From 8205073df05d11d951a0ca1d1efed82cdc2e3b2b Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 31 Mar 2025 13:19:07 +0300 Subject: [PATCH 01/62] * updated to v0.7.0 * lagre refactoring of memory management- moving from UtlAlloc to std::vector * big code cleanup ! not stable --- include/EQF.H | 11 - include/EQFDDE.H | 8 +- include/EQFFUZZ.H | 2 +- include/EQFMORPH.H | 14 +- include/EQFQDAMI.H | 6 +- include/EQFSETUP.H | 14 - include/EQFTA.H | 14 +- include/EQFTAG.H | 4 +- include/EQFTAG00.H | 1 - include/EQFTP.H | 3 +- include/OTMAPI.H | 11 - include/lowlevelotmdatastructs.h | 46 +- source/CMakeLists.txt | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp | 76 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp | 7 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 445 +--------- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 76 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 52 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 241 ++---- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 73 +- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 18 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 774 +++++------------- .../core/EqfMemoryPlugin/EqfMemory.cpp | 2 - .../opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp | 2 +- .../core/EqfMemoryPlugin/TMPluginWrapper.cpp | 318 +++---- source/opentm2/core/morph/MorphFactory.cpp | 1 + .../core/pluginmanager/PluginManagerImpl.h | 6 +- source/opentm2/core/tagtable/EQFTOKEN.cpp | 400 +++------ .../core/tagtable/MarkupPluginMapper.cpp | 44 - source/opentm2/core/tm.cpp | 4 +- source/opentm2/core/utilities/EQFDICT.cpp | 356 +++----- source/opentm2/core/utilities/EQFMORPH.cpp | 19 +- source/opentm2/core/utilities/EQFTADIT.cpp | 60 -- source/opentm2/core/utilities/EQFUTERR.cpp | 3 - .../core/utilities/FilesystemHelper.cpp | 21 - source/opentm2/core/utilities/LogWrapper.h | 2 +- .../opentm2/core/utilities/ThreadingWrapper.h | 2 +- .../OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c | 17 - .../OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.C | 271 ------ .../OtmMarkupTablePlugin/OtmMarkupTable.cpp | 72 -- .../OtmMarkupTablePlugin.cpp | 442 ---------- .../UserMarkupTablePlugin.cpp | 458 ----------- 42 files changed, 698 insertions(+), 3702 deletions(-) diff --git a/include/EQF.H b/include/EQF.H index aa8ccb02..92fa9aa6 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -3908,13 +3908,7 @@ USHORT SetSystemLanguage( PSZ pLanguage ); PSZ GetSystemLanguage( ); USHORT UtlSetFHandState( HFILE hf, USHORT fsState, BOOL fMsg ); USHORT UtlSetFHandStateHwnd( HFILE hf, USHORT fsState, BOOL fMsg, HWND hwnd ); -BOOL UtlAllocTrace ( PVOID *, LONG, LONG, USHORT, PSZ, USHORT ); -BOOL UtlAllocTraceHwnd ( PVOID *, LONG, LONG, USHORT, HWND, PSZ, USHORT ); #ifdef UTLFFST - #define UtlAlloc( par1, par2, par3, par4 ) \ - UtlAllocTrace(par1, par2, par3, par4, __FILE__, __LINE__ ) - #define UtlAllocHwnd( par1, par2, par3, par4, par5 ) \ - UtlAllocTraceHwnd(par1, par2, par3, par4, par5, __FILE__, __LINE__ ) #endif USHORT EqfDriveType(USHORT iDrive); @@ -3946,11 +3940,6 @@ BOOL UtlKeepInTWB( PSWP pswpWin ); USHORT UtlCheckDlgPos( HWND hwndDlg, BOOL fShow ); LONG UtlTime( PLONG plTime ); -#ifdef UTLFFST -#define UtlAlloc( par1, par2, par3, par4 ) \ - UtlAllocTrace(par1, par2, par3, par4, __FILE__, __LINE__ ) - -#endif VOID UtlWait( SHORT sWaitTime ); // wait the given time USHORT UtlLoadWidth( HAB, HMODULE, SHORT, PUSHORT ); diff --git a/include/EQFDDE.H b/include/EQFDDE.H index c05efd28..a2bf3ae2 100755 --- a/include/EQFDDE.H +++ b/include/EQFDDE.H @@ -587,10 +587,10 @@ typedef struct _BATCHIMPORTPARMS HGLOBAL MakeDDEDataSeg(USHORT, PSZ, PVOID, USHORT, USHORT ); -static BOOL EQFFunctionCmdLine (PDDECLIENT ); -static BATCHCMD ValidateToken ( PSZ *ppToken, PBATCHLIST pBatchList ); -static BOOL CheckBatchOccurrence (PDDECLIENT, PUSHORT); -static DDETASK ValidateTask ( PDDECLIENT pDDEClient ); +BOOL EQFFunctionCmdLine (PDDECLIENT ); +BATCHCMD ValidateToken ( PSZ *ppToken, PBATCHLIST pBatchList ); +BOOL CheckBatchOccurrence (PDDECLIENT, PUSHORT); +DDETASK ValidateTask ( PDDECLIENT pDDEClient ); BOOL DDEDocUnLoad( PDOCIMPEXP pDocExpIda ); BOOL DDEDocLoad( PDOCIMPEXP pDocImpIda ); diff --git a/include/EQFFUZZ.H b/include/EQFFUZZ.H index 75d5f9b2..40adb478 100755 --- a/include/EQFFUZZ.H +++ b/include/EQFFUZZ.H @@ -46,7 +46,7 @@ VOID MakeHashValue ( PULONG, USHORT, PSZ_W, PULONG ); //static -BOOL PrepareTokens ( PLOADEDTABLE, PBYTE, PBYTE, PSZ_W, SHORT, PFUZZYTOK *, ULONG ); +BOOL PrepareTokens ( PLOADEDTABLE, PBYTE, std::vector&, PSZ_W, SHORT, PFUZZYTOK *, ULONG ); //static BOOL FuzzyReplace ( PSZ_W, PSZ_W, PSZ_W, PREPLLIST, PREPLLIST ); //static diff --git a/include/EQFMORPH.H b/include/EQFMORPH.H index 97319b4c..66688efd 100755 --- a/include/EQFMORPH.H +++ b/include/EQFMORPH.H @@ -98,15 +98,7 @@ #include "LogWrapper.h" -/**********************************************************************/ -/* Structure to access MORPH_FLAG_OFFSLIST type term lists */ -/**********************************************************************/ -typedef struct _FLAGOFFSLIST -{ - LONG lFlags; // flags of term - USHORT usLen; // length of term - USHORT usOffs; // offset of term -} FLAGOFFSLIST, *PFLAGOFFSLIST; + //+----------------------------------------------------------------------------+ // Flags for flagged term lists @@ -328,9 +320,10 @@ USHORT MorphAddTermToList2 ); + USHORT MorphAddTermToList2W ( - PSZ_W *ppList, // ptr to term list pointer + PSZ_W* ppList, // ptr to term list pointer PULONG pulSize, // ptr to allocated size of term list PULONG pulUsed, // ptr to used bytes in term list PSZ_W pszTerm, // ptr to new term being added to list @@ -341,6 +334,7 @@ USHORT MorphAddTermToList2W ); + USHORT MorphCopyTermListASCII2Unicode ( PSZ pList, // ptr to ASCII termlist diff --git a/include/EQFQDAMI.H b/include/EQFQDAMI.H index b2add507..8b8570f9 100755 --- a/include/EQFQDAMI.H +++ b/include/EQFQDAMI.H @@ -894,7 +894,7 @@ typedef struct _DICTTIME SHORT QDAMDictNumEntriesLocal ( PBTREE, PULONG ); - SHORT QDAMDictFirstLocal ( PBTREE, PWCHAR, PULONG, PBYTE, PULONG ); + SHORT QDAMDictFirstLocal ( PBTREE, PWCHAR, PULONG, std::vector& ); SHORT QDAMInsertKey_V3( PBTREE, PBTREEBUFFER_V3, PWCHAR, RECPARAM, RECPARAM ); @@ -909,7 +909,7 @@ typedef struct _DICTTIME SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, PWCHAR, PULONG ); - SHORT QDAMGetszData_V3 ( PBTREE, RECPARAM, PBYTE, PULONG, CHAR ); + SHORT QDAMGetszData_V3 ( PBTREE, RECPARAM, std::vector&, CHAR ); BOOL QDAMDeleteKey_V3( PBTREE, PBTREEBUFFER_V3, PCHAR); SHORT QDAMDestroy ( PBTREE ); @@ -921,7 +921,7 @@ typedef struct _DICTTIME SHORT QDAMPrev_V3( PBTREE, PRECPARAM, PRECPARAM, PRECPARAM ); SHORT QDAMHeaderFirst ( PBTREE, PRECPARAM, PRECPARAM, PRECPARAM ); SHORT QDAMHeaderNext ( PBTREE, PRECPARAM, PRECPARAM,PRECPARAM ); - SHORT QDAMUnTerseData ( PBTREE, PUCHAR, ULONG, PULONG ); + SHORT QDAMUnTerseData ( PBTREE, std::vector& , ULONG ); SHORT QDAMValidateIndex_V3( PBTREE, PBTREEBUFFER_V3 * ); PSZ_W QDAMGetszKey_V3 ( PBTREEBUFFER_V3, USHORT, USHORT ); diff --git a/include/EQFSETUP.H b/include/EQFSETUP.H index 3fb8b6df..979bb9e5 100755 --- a/include/EQFSETUP.H +++ b/include/EQFSETUP.H @@ -79,34 +79,20 @@ typedef enum _UPDPROPTYPE VOID BuildPath( PSZ pszBuffer, CHAR chEqfDrive, PSZ pszFolder, USHORT usPathID ); -static VOID DeleteFiles( PSZ, USHORT ); -static USHORT ErrorMessage( USHORT, PSZ, USHORT ); -static BOOL FileExists ( PSZ ); -static BOOL InstCheckAndAdd( PSZ *, PSZ, USHORT ); PSZ InstLocateString ( PSZ, PSZ ); PSZ InstMatch ( PSZ, PSZ ); -static USHORT ReadAndPrepareConfigSys ( VOID ); USHORT UpdateConfigSys ( VOID ); int SetupMAT(); USHORT SetupMAT( HAB hab, CHAR chPrimaryDrive, PSZ pszSecondaryDrives, CHAR chLanDrive ); -static USHORT CreateDictProperties( CHAR chPrimaryDrive ); USHORT SetupTmCom (CHAR chPrimaryDrive, PSZ pszSecondaryDrives); USHORT SetupDemo (HAB hab, CHAR chPrimaryDrive, PSZ pszSecondaryDrives, CHAR chLanDrive); USHORT SetupDocu (HAB hab, CHAR chPrimaryDrive, PSZ pszSecondaryDrives); -static VOID DeleteDictCacheFiles( CHAR chPrimaryDrive, USHORT usPathId ); -static BOOL UtlPropFileExist( CHAR chDrive, PSZ pszFile ); - USHORT UpdatePropFile( USHORT usPropType ,CHAR chDrive, PSZ pszFileName, PSZ pszDocName ); -static USHORT UpdateFolderProp( PSZ pszFullFileName, CHAR chDrive ); -static USHORT UpdateDocumentProp( PSZ pszFullFileName, CHAR chDrive ); -static USHORT UpdateDictProp( PSZ pszFullFileName, CHAR chDrive ); -static USHORT UpdateTMProp( PSZ pszFullFileName, CHAR chDrive ); -static USHORT UpdateSystemProp( PSZ pszFullFileName, PSZ pszFolderName ); BOOL SetupUtils( HAB, PSZ ); BOOL TWBInit( HAB ); diff --git a/include/EQFTA.H b/include/EQFTA.H index 5ead7267..989f51f7 100755 --- a/include/EQFTA.H +++ b/include/EQFTA.H @@ -111,7 +111,7 @@ USHORT TACreateProtectTableW PSZ_W pszSegmentW, // ptr to text of segment being processed PVOID pVoidTable, // ptr to tag table USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens + std::vector& pTokBuffer, // buffer used temporarly for tokens USHORT usTokBufferSize, // size of token buffer in bytes PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr PFN pvUserExit, // ptr to user exit function @@ -128,7 +128,7 @@ USHORT TACreateProtectTableWEx PSZ_W pszSegment, // ptr to text of segment being processed PVOID pVoidTable, // ptr to tag table USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens + std::vector& pTokBuffer, // buffer used temporarly for tokens USHORT usTokBufferSize, // size of token buffer in bytes PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr PFN pvUserExit, // ptr to user exit function @@ -139,16 +139,6 @@ USHORT TACreateProtectTableWEx -USHORT TAPrepProtectTable -( - PVOID pVoidTable, // ptr to tag table (PLOADEDTABLE) - HMODULE *phModule, // address of user exit module handle - PFN *ppfnUserExit, // address of ptr to user exit function - PFN *ppfnCheckSegExit, // ptr to segment check function - PFN *ppfnShowTrans, // ptr to Show transl. function - PFN *ppfnTocGoto // ptr to TOC goto function -); - USHORT TAEndProtectTable ( HMODULE *phModule, // address of user exit module handle diff --git a/include/EQFTAG.H b/include/EQFTAG.H index 9980fd97..09313575 100755 --- a/include/EQFTAG.H +++ b/include/EQFTAG.H @@ -223,8 +223,8 @@ BOOL ContainsClassID( PLOADEDTABLE pLoadedTable, USHORT usClassID ); VOID EQFTagTokenize(PSZ, TAGTABLE *, BOOL, PSZ *, USHORT *, TOKENENTRY *, USHORT); -BOOL TATagTokenize (PSZ, PLOADEDTABLE, BOOL, PSZ *, USHORT *, TOKENENTRY *, USHORT, ULONG); -BOOL TATagTokenizeW(PSZ_W, PLOADEDTABLE, BOOL, PSZ_W *, USHORT *, TOKENENTRY *, USHORT); +//BOOL TATagTokenize (PSZ, PLOADEDTABLE, BOOL, PSZ *, USHORT *, TOKENENTRY *, USHORT, ULONG); +BOOL TATagTokenizeW(PSZ_W, PLOADEDTABLE, BOOL, PSZ_W *, USHORT *, std::vector&); /**********************************************************************/ diff --git a/include/EQFTAG00.H b/include/EQFTAG00.H index 5d3362dd..90f56d34 100755 --- a/include/EQFTAG00.H +++ b/include/EQFTAG00.H @@ -168,7 +168,6 @@ typedef enum { MUFILL_NAMES, MUFILL_CLBITEMS } MUFILLTYPES; BOOL MUGetMarkupTableFileName( char *pszMarkup, char *pszPlugin, char *pszBuffer, int iBufSize ); BOOL MUGetMarkupTableFilePath( char *pszMarkup, char *pszPlugin, char *pszBuffer, int iBufSize ); BOOL MUGetUserExitFileName( char *pszMarkup, char *pszPlugin, char *pszBuffer, int iBufSize ); - BOOL MUUpdateMarkupTableFiles( char *pszMarkup, char *pszPlugin, char *pszShortDescription, char *pszLongDescription, char *pszVersion, char *pszTableFile, char *pszUserExitFile, char *pszMarkupFiles ); BOOL MUDeleteMarkupTable( char *pszMarkup, char *pszPlugin ); #endif diff --git a/include/EQFTP.H b/include/EQFTP.H index 705461ee..ae69e144 100755 --- a/include/EQFTP.H +++ b/include/EQFTP.H @@ -982,7 +982,7 @@ VOID EQFBVioSetNewDocSize ( PTBDOCUMENT ); // refresh doc size VOID ImeSetFont( PTBDOCUMENT, HWND, PLOGFONT ); -BOOL EQFBFindDiffEx( PVOID, PBYTE, PBYTE, PSZ_W, PSZ_W, SHORT , PVOID * , PVOID *, ULONG); +BOOL EQFBFindDiffEx( PVOID, PBYTE, std::vector&, PSZ_W, PSZ_W, SHORT , PVOID * , PVOID *, ULONG); BOOL EQFBCountDiff( PVOID pvFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ); @@ -1169,7 +1169,6 @@ USHORT TAIteratedCreateProtectTableW( PTBDOCUMENT pDoc, PTBSEGMENT pSeg, int iM USHORT EQFBPrepareFuzzyPropEx ( PTBDOCUMENT, PSZ_W, PSZ_W, PSZ_W, SHORT, SHORT, PUSHORT, ULONG, ULONG); USHORT EQFBPrepareFuzzyPropEx2( PSZ_W, PSZ_W, PSZ_W, SHORT, SHORT, PUSHORT, PVOID, PVOID, PBYTE, PBYTE, ULONG, ULONG ); USHORT EQFBPrepareFuzzyPropEx3( PSZ_W, PSZ_W, PSZ_W, SHORT, SHORT, PUSHORT, PVOID, PVOID, PBYTE, PBYTE, ULONG, ULONG, PUSHORT ); - BOOL EQFBFindDiff ( PTBDOCUMENT, PSZ_W, PSZ_W, SHORT , PFUZZYTOK * , PUSHORT, PUSHORT, ULONG); BOOL EQFBFindWord ( PSZ_W ,SHORT, USHORT, PUSHORT, PUSHORT, ULONG, BOOL ); #endif //_EQFTP_H_INCLUDED_ diff --git a/include/OTMAPI.H b/include/OTMAPI.H index 5f2d4ae3..d39c5f65 100755 --- a/include/OTMAPI.H +++ b/include/OTMAPI.H @@ -1483,17 +1483,6 @@ EQF_BOOL EQFCHECKSEGW ( ); -EQF_BOOL EQFCHECKSEGEXW ( - PSZ_W pszPrevSrc, // previous source segment - PSZ_W pszSrc, // current source segment - PSZ_W pszTgt, // current translation - PEQF_BOOL pfChanged, // segment changed - LONG lInfo, // info handle to use with EQFGETPREVSEG(W),EQFGETNEXTSEG(W) - ULONG ulSegNum, // segment number to use with EQFGETPREVSEG(W),EQFGETNEXTSEG(W) - EQF_BOOL fMsg // message display requested -); - - /******************************************************************************* * * function: EQFCHECKSEGTYPE diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 41b671bc..84716802 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -30,7 +30,7 @@ class ChunkBuffer{ void setResponseHandler(proxygen::ResponseHandler* responseHandler){m_responseHandler = responseHandler;} bool isActive()const {return m_responseHandler != 0; } - ChunkBuffer(){//m_buff.reserve(chunkSize_+1); + ChunkBuffer(){//m_buff.resize(chunkSize_+1); } ~ChunkBuffer(){ //T5LOG(T5INFO)<< "called dctor of chunk buffer- sending last chunk"; @@ -465,13 +465,13 @@ typedef struct _TMX_TABLE_ENTRY // name table structure (TM version 5 and up) constexpr int NUM_OF_TMX_TABLE_ENTRIES = (BTREE_REC_SIZE_V3 - sizeof(ULONG)) / sizeof(TMX_TABLE_ENTRY); -typedef struct TMX_TABLE_OLD{ +struct TMX_TABLE_OLD{ ULONG ulMaxEntries = 0; TMX_TABLE_ENTRY stTmTableEntry[NUM_OF_TMX_TABLE_ENTRIES]; }; typedef TMX_TABLE_OLD * PTMX_TABLE_OLD; -typedef struct TMX_TABLE +struct TMX_TABLE { //ULONG ulAllocSize = 0; ULONG ulMaxEntries = 0; @@ -498,21 +498,29 @@ typedef char *PSZ; typedef struct _TMX_LONGNAME_TABLE_ENTRY { PSZ pszLongName; // ptr to long name in buffer area + int longNameStartOffset; + //std::string pszLongName; USHORT usId; // ID for long name } TMX_LONGNAME_TABLE_ENTRY, * PTMX_LONGNAME_TABLE_ENTRY; //table structure for long document table -typedef struct _TMX_LONGNAME_TABLE +typedef struct TMX_LONGNAME_TABLE { - PSZ pszBuffer; // buffer for names and IDs - ULONG ulBufUsed; // number of bytes used in buffer - ULONG ulBufSize; // size of buffer in bytes - ULONG ulTableSize; // table size (# entries) - ULONG ulEntries; // number of entries in table - TMX_LONGNAME_TABLE_ENTRY stTableEntry[1]; // dyn. array of table entries -} TMX_LONGNAMETABLE, * PTMX_LONGNAME_TABLE; + std::vector pszBuffer; // buffer for names and IDs + //ULONG ulBufUsed; // number of bytes used in buffer + //ULONG ulBufSize; // size of buffer in bytes + //ULONG ulTableSize; // table size (# entries) + //ULONG ulEntries; // number of entries in table + std::vector stTableEntry; // dyn. array of table entries + + void restoreStrPointersAfterRealloc(){ + for(auto& entry: stTableEntry){//during insertion to the pszBuffer, it could be reallocated, so we need to update poiners + entry.pszLongName = pszBuffer.data() + entry.longNameStartOffset; + }; + } +}; BOOL UtlAlloc ( void **, long, long, unsigned short ); BOOL UtlAllocHwnd ( void**, long, long, unsigned short, void * ); @@ -788,8 +796,7 @@ struct BTREE EQFNTMGet ( ULONG ulKey, // key to be searched for - PCHAR pchBuffer, // space for user data - PULONG pulLength // in/out length of returned user data + std::vector& uchBuff // space for user data ); @@ -890,13 +897,6 @@ struct BTREE //+----------------------------------------------------------------------------+ // Returncode type: SHORT //+----------------------------------------------------------------------------+ - SHORT EQFNTMGetUpdCounter - ( - PLONG plUpdCount, // ptr to buffer for update counter - SHORT sIndex, // index of requested update counter - SHORT sNumCounters // number of counters requested - ); - VOID QDAMUpdateList_V3( PBTREEBUFFER_V3 ); @@ -907,7 +907,7 @@ struct BTREE SHORT QDAMDeleteDataFromBuffer_V3( RECPARAM recParam); SHORT QDAMDictUpdateLocal ( PWCHAR, PBYTE, ULONG ); - SHORT QDAMDictExactLocal ( PWCHAR, PBYTE, PULONG, USHORT ); + SHORT QDAMDictExactLocal ( PWCHAR, std::vector&, USHORT ); SHORT QDAMDictCreateLocal ( TMX_SIGN*, ULONG, bool keepInRamOnly = false); @@ -1426,6 +1426,8 @@ typedef struct _TOKENENTRY // entry in tokenlist : USHORT ClassId; // class id of token CHAR_W * pDataStringW; // pointer to data string - Unicode // !!!! Attention: above has to match TOKENENTRYSEG definition .... !!!! + + void reset(){ memset(this, 0, sizeof(*this));} } TOKENENTRY, *PTOKENENTRY; class ImportStatusDetails; @@ -2322,7 +2324,7 @@ class ChunkBufferOld{ void setResponseHandler(proxygen::ResponseHandler* responseHandler){m_responseHandler = responseHandler;} bool isActive()const {return m_responseHandler != 0; } - ChunkBufferOld(){m_buff.reserve(chunkSize_+1);} + ChunkBufferOld(){m_buff.resize(chunkSize_+1);} ~ChunkBufferOld(){ T5LOG(T5INFO)<< "called dctor of chunk buffer- sending last chunk"; triggerChunkSend(); diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index da4aa20a..de603574 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) -SET(APP_VERSION_MAJOR 6) -SET(APP_VERSION_MINOR 87) +SET(APP_VERSION_MAJOR 7) +SET(APP_VERSION_MINOR 0) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp index 6e744e33..9eeeea81 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp @@ -160,7 +160,7 @@ PrepareTokens ( PLOADEDTABLE pTagTable, PBYTE pInBuf, - PBYTE pTokBuf, + std::vector& pTokBuf, PSZ_W pString, // pointer to string to be tokenized SHORT sLanguageId, // language ID PFUZZYTOK * ppTransTokList, // resulting list of tokens @@ -194,8 +194,7 @@ PrepareTokens TRUE, &pRest, &usColPos, - (PTOKENENTRY) pTokBuf, - TOK_BUFFER_SIZE / sizeof(TOKENENTRY) ); + pTokBuf ); /********************************************************************/ /* build tokenlist, i.e. */ @@ -205,7 +204,7 @@ PrepareTokens /* this is large enough, we can avoid any checking... */ /********************************************************************/ pstCurrent = (PFUZZYTOK) pInBuf; // use input buffer for table - pTok = (PTOKENENTRY) pTokBuf; + pTok = (PTOKENENTRY) pTokBuf.data(); pStart = pTok->pDataStringW; while ( (pTok->sTokenid != ENDOFLIST) ) { @@ -215,13 +214,13 @@ PrepareTokens pTermList = NULL; chTemp = *(pTok->pDataStringW+pTok->iLength); - *(pTok->pDataStringW+pTok->iLength) = EOS; + *(pTok->pDataStringW + pTok->iLength) = EOS; usRC = MorphTokenizeW( sLanguageId, pTok->pDataStringW, &usListSize, &pTermList, MORPH_OFFSLIST, ulOemCP ); - *(pTok->pDataStringW+pTok->iLength) = chTemp; + *(pTok->pDataStringW + pTok->iLength) = chTemp; if ( pTermList ) { @@ -272,6 +271,7 @@ PrepareTokens /* adjust current offset to point to new offset in string... */ /****************************************************************/ pTok++; + //pTokenBuf.emplace_back(Tok) iCurOffs = (USHORT)(pTok->pDataStringW - pStart); } /* endwhile */ @@ -1116,45 +1116,7 @@ TokStrCompare //| free forward and backward token lists | //| return success indicator | //+----------------------------------------------------------------------------+ -BOOL -EQFBFindDiff -( - PTBDOCUMENT pDoc, // pointer to document - PSZ_W pString1, // first string passed - PSZ_W pString2, // second string - SHORT sLanguageId, // language id to be used - PFUZZYTOK * ppFuzzyTok, // returned token list - PUSHORT pusModWords, // num of differences - PUSHORT pusTokens, - ULONG ulOemCP -) -{ - BOOL fOK; - PFUZZYTOK pFuzzyTgt = NULL; - - fOK = EQFBFindDiffEx( pDoc->pDocTagTable, - pDoc->pInBuf, - pDoc->pTokBuf, - pString1, - pString2, - sLanguageId, - (PVOID *)ppFuzzyTok, - (PVOID *)&pFuzzyTgt, ulOemCP ); - /**********************************************************************/ - /* pFuzzyTgt is not needed, so free space */ - /**********************************************************************/ - if (pFuzzyTgt ) - { - UtlAlloc ( (PVOID *)&pFuzzyTgt, 0L, 0L, NOMSG ); - } /* endif */ - if ( fOK ) - { - EQFBCountDiff( *ppFuzzyTok, pusTokens, pusModWords ); - } /* endif */ - - return fOK; -} // counts the number of tokens and the number of differences // in a fuzzy token list created using EQFBFindDiffEx @@ -1240,7 +1202,7 @@ EQFBFindDiffEx ( PVOID pTable, // pointer to loaded tagtable PBYTE pInBuf, // pointer to input buffer - PBYTE pTokBuf, // pointer to temp token buffer + std::vector& pTokBuf, // pointer to temp token buffer PSZ_W pString1, // first string passed PSZ_W pString2, // second string SHORT sLanguageId, // language id to be used for pString1 @@ -1262,11 +1224,9 @@ EQFBFindDiffEx /******************************************************************/ T5LOG( T5DEVELOP) <<"EQFBFindDiffEx::Preparing tokens for first string: "; //std::wstring ws1 = removeTagsFromString(pString1); - fOK = PrepareTokens( //pDoc, - pTagTable, + fOK = PrepareTokens( pTagTable, pInBuf, pTokBuf, - //(wchar_t*)ws1.c_str(), pString1, sLanguageId, &pTokenList1, ulOemCP ); if ( fOK ) @@ -1274,11 +1234,9 @@ EQFBFindDiffEx T5LOG( T5DEVELOP) <<"EQFBFindDiffEx::Preparing tokens for second string: "; //std::wstring ws2 = removeTagsFromString(pString2); - fOK = PrepareTokens( // pDoc, - pTagTable, + fOK = PrepareTokens( pTagTable, pInBuf, pTokBuf, - //(wchar_t*)ws2.c_str(), pString2, sLanguageId, &pTokenList2, ulOemCP ); } /* endif */ @@ -2511,8 +2469,8 @@ NTMCompareBetweenTokens SHORT sNumTags = 0; BOOL fStringEqual = TRUE; CHAR_W chW; - PBYTE pInBuf = NULL; - PBYTE pTokBuf = NULL; + std::vector pInBufL; + std::vector pTokBuf; USHORT usI = 0; BOOL fTokIsTag = FALSE; BOOL fTok2IsTag = FALSE; @@ -2522,8 +2480,10 @@ NTMCompareBetweenTokens PSZ_W pTemp = NULL; // allocate required buffers - fOK = UtlAlloc((PVOID *)&pInBuf, 0L, 50000L, NOMSG ); - if ( fOK ) fOK = UtlAlloc((PVOID *)&pTokBuf, 0L, (LONG)TOK_BUFFER_SIZE, NOMSG ); + pInBufL.resize(50000); + pTokBuf.reserve(TOK_BUFFER_SIZE/sizeof(TOKENENTRY)); + + fOK = true; // load tag table if ( fOK ) @@ -2545,14 +2505,14 @@ NTMCompareBetweenTokens /* prepare tokens for String1 and string 2 */ /******************************************************************/ fOK = PrepareTokens( pTable, - pInBuf, + pInBufL.data(), pTokBuf, pD1, sLangID, &pTokenList1, ulSrcCP ); } if ( fOK) { fOK = PrepareTokens( pTable, - pInBuf, + pInBufL.data(), pTokBuf, pD2, sLangID, &pTokenList2, ulSrcCP ); } @@ -2689,8 +2649,6 @@ NTMCompareBetweenTokens if (pTokenList1) UtlAlloc( (PVOID *)&pTokenList1, 0L, 0L, NOMSG ); if (pTokenList2) UtlAlloc( (PVOID *)&pTokenList2, 0L, 0L, NOMSG ); - if ( pInBuf ) UtlAlloc( (PVOID *) &pInBuf, 0L, 0L, NOMSG ); - if ( pTokBuf ) UtlAlloc( (PVOID *) &pTokBuf, 0L, 0L, NOMSG ); if ( pTable ) TAFreeTagTable( pTable); *pfStringEqual = fStringEqual; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp index 093273f8..0dcf7750 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp @@ -372,12 +372,7 @@ USHORT EQFMemExportEnd ( PPROCESSCOMMAREA pCommArea, if ( (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF) ) { - #ifdef TO_BE_REMOVED - if ( pExportIDA->pfnMemExpEnd ) - { - pExportIDA->pfnMemExpEnd( pExportIDA->lExternalExportHandle ); - } /* endif */ - #endif + if(pExportIDA->invalidXmlSegments){ T5LOG(T5ERROR) << pExportIDA->invalidXmlSegments <<" segments was not exported because of invalid xml"; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 521108b4..5508ff05 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -98,7 +98,7 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT ); -static void GetElapsedTime( long long *plTime ) +void GetElapsedTime( long long *plTime ) { if(plTime) *plTime = 0; //struct timespec now; @@ -106,12 +106,6 @@ static void GetElapsedTime( long long *plTime ) //*plTime = now.tv_sec + now.tv_nsec / 1000000000.0; } -USHORT MemHandleCodePageValue -( - MEM_LOAD_DLG_IDA* pLIDA, // ptr to memory load IDA - PSZ_W pszCodePage, // ptr to specified code page value - PBOOL pfRestartImport // caller's restart the import flag -); // import logging @@ -122,7 +116,6 @@ USHORT MemHandleCodePageValue //#endif -USHORT MemLoadAndConvert( MEM_LOAD_DLG_IDA* pLIDA, PSZ_W pszBuffer, ULONG usSize, PULONG pusBytesRead ); INT_PTR /*CALLBACK*/ NTMTagLangDlg( HWND, // Dialog window handle WINMSG, // Message ID @@ -286,71 +279,9 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // use the default token values if ( fOK ) { - PWCHAR pRest = NULL; - USHORT usLastColPos = 0; - /* - pRest = NULL; - TATagTokenizeW( MEM_CONTEXT_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CONTROL_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( NTM_DESCRIPTION_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_MEMORYDB_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( NTM_MEMORYDB_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_SEGMENT_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_SOURCE_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_TARGET_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CONTEXT_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CONTROL_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( NTM_DESCRIPTION_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_MEMORYDB_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( NTM_MEMORYDB_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_SEGMENT_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_SOURCE_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_TARGET_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CODEPAGE_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_CODEPAGE_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_ADDDATA_TOKEN, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - - pRest = NULL; - TATagTokenizeW( MEM_ADDDATA_TOKEN_END, pLIDA->pFormatTable, TRUE, &pRest, &usLastColPos, pLIDA->pTokenList, NUMB_OF_TOKENS); - //*/ pLIDA->pTokenList->sTokenid = ENDOFLIST; // enable correct start of import process } /* endif */ + // special handling for import in TMX format if ( fOK && ((pLIDA->usImpMode == MEM_FORMAT_TMX) || (pLIDA->usImpMode == MEM_FORMAT_TMX_TRADOS)) ) @@ -994,378 +925,6 @@ USHORT MemFuncImportProcess } /* end of function MemFuncImportProcess */ -// load some data from file and convert it if necessary -// usSize is size of pszBuffer(filled as output!) in number of CHAR_W's - -USHORT MemLoadAndConvert( MEM_LOAD_DLG_IDA* pLIDA, PSZ_W pszBuffer, ULONG ulSize, PULONG pulBytesRead ) -{ - USHORT usDosRc = NO_ERROR; - ULONG ulBytesRead = 0; - -#ifdef IMPORT_LOGGING - GetElapsedTime( &lOtherTime ); -#endif - - switch ( pLIDA->usImpMode ) - { - case SGMLFORMAT_ANSI : - { - PSZ pConvBuffer = NULL; - LONG lBytesLeft = 0L; - LONG lRc = 0; - ULONG ulUnicodeChars = 0; - - if ( UtlAlloc( (PVOID *)&pConvBuffer, 0L, ulSize+1, ERROR_STORAGE ) ) - { - usDosRc = UtlReadHwnd( pLIDA->hFile, pConvBuffer, ulSize, &ulBytesRead, - TRUE, pLIDA->hwndErrMsg ); - if (ulBytesRead != ulSize ) - { - pLIDA->fEOF = TRUE; - } /* endif */ - - // ensure that there are no DBCS characters splitted at the end of - // the data just read into memory - if ( ulBytesRead && !pLIDA->fEOF ) - { - // non DBCS case --- no problem at all - ulUnicodeChars = UtlDirectAnsi2UnicodeBuf( pConvBuffer, pszBuffer, - ulBytesRead, pLIDA->ulAnsiCP, FALSE, &lRc, - &lBytesLeft ); - usDosRc = (USHORT)lRc; - if ( lBytesLeft ) - { // undo the last character read... - // reposition file pointer - ULONG ulCurrentPos = 0; - UtlChgFilePtr( pLIDA->hFile, 0L, FILE_CURRENT, &ulCurrentPos, FALSE); - ulCurrentPos -= lBytesLeft; - UtlChgFilePtr( pLIDA->hFile, ulCurrentPos, FILE_BEGIN, &ulCurrentPos, FALSE); - } /* endif */ - // GQ: use number of Unicode characters (which may differ from number of chars read in) for processing - ulBytesRead = ulUnicodeChars; - } - else - { - // either at end or nothing read in... - ulUnicodeChars = UtlDirectAnsi2UnicodeBuf( pConvBuffer, pszBuffer, - ulBytesRead, pLIDA->ulAnsiCP, FALSE, &lRc, - &lBytesLeft ); - ulBytesRead = ulUnicodeChars; - usDosRc = (USHORT)lRc; - } /* endif */ - ulBytesRead *= sizeof(CHAR_W); - - if ( lRc ) - { - // find character causing the conversion error and show error message - UtlFindAndShowConversionError( pConvBuffer, ulBytesRead, pLIDA->ulAnsiCP ); - } /* endif */ - - UtlAlloc( (PVOID *)&pConvBuffer, 0L, 0L, NOMSG ); - } - else - { - LOG_AND_SET_RC(usDosRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } - break; - - case SGMLFORMAT_UNICODE : - { - // read data into conversion buffer - usDosRc = UtlReadHwnd( pLIDA->hFile, pszBuffer, (ulSize * sizeof(CHAR_W)), - &ulBytesRead, TRUE, pLIDA->hwndErrMsg ); - - // set EOF condition - if ( (usDosRc == NO_ERROR ) && (ulBytesRead != ulSize * sizeof(CHAR_W) ) ) - { - pLIDA->fEOF = TRUE; - } /* endif */ - - // if first read skip any unicode text prefix - if ( (usDosRc == NO_ERROR) // && (pLIDA->fFirstRead == FALSE)//seems like not used field - ) - { - PSZ pszPrefix = UNICODEFILEPREFIX; - int iLen = strlen(pszPrefix); - if ( memcmp( pszBuffer, pszPrefix, iLen ) == 0 ) - { - // skip prefix ... - ulBytesRead -= iLen; - memmove( (PBYTE)pszBuffer, (PBYTE)pszBuffer+iLen, ulBytesRead ); - } - else - { - PSZ pszImportFile = pLIDA->szFilePath; - usDosRc = UtlError( NO_VALID_UNICODEFORMAT, MB_YESNO, 1, &pszImportFile, EQF_WARNING ); - - //usDosRc = (usDosRc == MBID_YES ) ? NO_ERROR : ERROR_INVALID_DATA; - } /* endif */ - } /* endif */ - - } - break; - - case SGMLFORMAT_ASCII : - default: - { - PSZ pConvBuffer = NULL; - if ( UtlAlloc( (PVOID *)&pConvBuffer, 0L, ulSize+3, ERROR_STORAGE ) ) - { - usDosRc = UtlReadHwnd( pLIDA->hFile, pConvBuffer, (ulSize), &ulBytesRead, - TRUE, pLIDA->hwndErrMsg ); - if (ulBytesRead != ulSize ) - { - pLIDA->fEOF = TRUE; - } /* endif */ - - // ensure that there are no DBCS characters splitted at the end of - // the data just read into memory - if ( ulBytesRead && !pLIDA->fEOF ) - { - BYTE bTest = (BYTE)pConvBuffer[ulBytesRead-1]; -T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = x if ( IsDBCSLeadByteEx( pLIDA->ulOemCP, bTest ) )"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - if ( IsDBCSLeadByteEx( pLIDA->ulOemCP, bTest ) ) - { - int iTry = 5; - ULONG ulUnicodeChars = 0; - - while ( iTry > 0 && ulBytesRead) - { - // undo the last character read... - // reposition file pointer - ULONG ulCurrentPos = 0; - UtlChgFilePtr( pLIDA->hFile, 0L, FILE_CURRENT, &ulCurrentPos, FALSE); - ulCurrentPos--; - UtlChgFilePtr( pLIDA->hFile, ulCurrentPos, FILE_BEGIN, &ulCurrentPos, FALSE); - - // adjust counters - ulBytesRead--; - // return is number of wide chars written to pszBuffer - ulUnicodeChars = ASCII2UnicodeBuf( pConvBuffer, pszBuffer, ulBytesRead, pLIDA->ulOemCP ); - if ( ulUnicodeChars && pszBuffer[ulUnicodeChars-1] == 0) - { - // try again, we probably found a 2nd DBCSbyte which might be in the range of a 1st byte, too - iTry--; - } - else - { - // leave loop and set ulBytesRead - ulBytesRead = ulUnicodeChars; - iTry = 0; - } /* endif */ - } /* endwhile */ - - if ( iTry == 0 && ulUnicodeChars && pszBuffer[ulUnicodeChars-1] == 0) - { - // something went totally wrong - ulBytesRead = ulUnicodeChars-1; - } - } - else - { - LONG lBytesLeft = 0; - ULONG ulOutPut = 0; - LONG lRC = 0; - - // non DBCS case --- no problem at all - ulOutPut = ASCII2UnicodeBufEx( pConvBuffer, pszBuffer, ulBytesRead, pLIDA->ulOemCP, - FALSE, &lRC, &lBytesLeft ); - // For consistent behaviour with old ASCII2UnicodeBuf function where dwFlags was 0: - // Within TM : FileRead expects to get EOS at position ulOutPut - if ( lRC ) - { - // find character causing the conversion error and show error message - UtlFindAndShowConversionError( pConvBuffer, ulBytesRead, pLIDA->ulOemCP ); - } /* endif */ - - if (lBytesLeft && (ulOutPut < ulBytesRead)) - { - *(pszBuffer+ulOutPut) = EOS; - ulOutPut++; - } - ulBytesRead = ulOutPut; - } /* endif */ -#endif //TO_BE_REPLACED_WITH_LINUX_CODE - } - else - { - // either at end or nothing read in... - ulBytesRead = ASCII2UnicodeBuf( pConvBuffer, pszBuffer, ulBytesRead, pLIDA->ulOemCP ); - } /* endif */ - // return is number of wide chars written to pszBuffer - ulBytesRead *= sizeof(CHAR_W); - - UtlAlloc( (PVOID *)&pConvBuffer, 0L, 0L, NOMSG ); - } - else - { - LOG_AND_SET_RC(usDosRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } - break; - } /* endswitch */ - - // adjust read bytes and add EOS - if ( usDosRc == NO_ERROR ) - { - PBYTE pByte = (PBYTE) pszBuffer; - *((PSZ_W)(pByte+ulBytesRead)) = EOS; - - - *pulBytesRead = ulBytesRead; - } /* endif */ - -#ifdef IMPORT_LOGGING - GetElapsedTime( &lFileReadTime ); -#endif - - - return( usDosRc ); -} /* end of function MemLoadAndConvert */ - -USHORT MemHandleCodePageValue -( - MEM_LOAD_DLG_IDA* pLIDA, // ptr to memory load IDA - PSZ_W pszCodePage, // ptr to specified code page value - PBOOL pfRestartImport // caller's restart the import flag -) -{ - ULONG ulCP = 0; - PSZ_W pMsgError[2]; // ptr to error array - USHORT usRc = 0; - USHORT usImpMode = 0; - - *pfRestartImport = FALSE; - - // skip leading whitespace - while ( iswspace( *pszCodePage ) ) pszCodePage++; - - if ( !usRc ) - { - if ( UTF16strnicmp( pszCodePage, L"ANSI=", 5 ) == 0) - { - usImpMode = SGMLFORMAT_ANSI; - ulCP = wcstol(pszCodePage + 5, 0, 10); - - if ( usImpMode != pLIDA->usImpMode ) - { - if ( pLIDA->usImpMode == SGMLFORMAT_UNICODE ) - { - LOG_AND_SET_RC(usRc, T5INFO, NO_VALID_ASCIIANSIFORMAT); - pMsgError[0] = L"SGML ANSI"; - T5LOG(T5ERROR) << "::NO_VALID_ASCIIANSIFORMAT::" << EncodingHelper::convertToUTF8(pMsgError[0]); - } - else // usImpMode == SGMLFORMAT_ASCII - { - if ( (pLIDA->hwndErrMsg == HWND_FUNCIF) || pLIDA->fBatch ) - { - LOG_AND_SET_RC(usRc, T5INFO, MBID_OK); - } - else - { - pMsgError[1] = L"SGML ASCII"; - pMsgError[0] = pszCodePage; - T5LOG(T5ERROR) << ":: WARNING_DIMP_FORMAT_SGML ASCII::"<< EncodingHelper::convertToUTF8(pMsgError[0]); - LOG_AND_SET_RC(usRc, T5INFO, WARNING_DIMP_FORMAT); - } /* endif */ - - if ( usRc == MBID_OK ) - { - *pfRestartImport = TRUE; - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - } /* endif */ - } /* endif */ - } /* endif */ - - if ( !usRc && (ulCP != pLIDA->ulAnsiCP) ) - { - if ( !UtlIsValidCP( ulCP ) ) - { - LOG_AND_SET_RC(usRc, T5INFO, NO_VALID_ASCIIANSICP); - pMsgError[0] = pszCodePage + 5; - T5LOG(T5ERROR) << "::rc = " << usRc << ":: " << EncodingHelper::convertToUTF8(pMsgError[0]); - } /* endif */ - } - - if ( !usRc ) - { - pLIDA->ulAnsiCP = ADJUSTCP(ulCP); - pLIDA->usImpMode = usImpMode; - } /* endif */ - } - else if ( UTF16strnicmp( pszCodePage, L"ASCII=", 6 ) == 0) - { - usImpMode = SGMLFORMAT_ASCII; - ulCP = wcstol(pszCodePage + 6, 0, 10); - - if( usImpMode != pLIDA->usImpMode ) - { - if ( pLIDA->usImpMode == SGMLFORMAT_UNICODE ) - { - LOG_AND_SET_RC(usRc, T5INFO, NO_VALID_ASCIIANSIFORMAT); - pMsgError[0] = L"SGML ASCII"; - T5LOG(T5ERROR) << "::NO_VALID_ASCIIANSIFORMAT::SGML ASCII"; - } - else - { - if ( ( pLIDA->hwndErrMsg == HWND_FUNCIF) || pLIDA->fBatch ) - { - LOG_AND_SET_RC(usRc, T5INFO, MBID_OK); - } - else - { - pMsgError[1] = L"SGML ANSI"; - pMsgError[0] = pszCodePage; - T5LOG(T5ERROR) << ":: WARNING_DIMP_FORMAT::SGML ANSI"; - //usRc = T5LOG(T5ERROR) << ":: WARNING_DIMP_FORMAT::SGML ANSI"; - } /* endif */ - - if (usRc == MBID_OK) - { - *pfRestartImport = TRUE; - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - } /* endif */ - } /* endif */ - } /* endif */ - - if ( !usRc && (ulCP != pLIDA->ulOemCP) ) - { - if ( !UtlIsValidCP( ulCP ) ) - { - LOG_AND_SET_RC(usRc, T5INFO, NO_VALID_ASCIIANSICP); - pMsgError[0] = pszCodePage + 6; - T5LOG(T5ERROR) << ":: " << usRc << "::" << EncodingHelper::convertToUTF8(pMsgError[0]); - } /* endif */ - } /* endif */ - - if ( !usRc ) - { - pLIDA->ulOemCP = ADJUSTCP(ulCP); - pLIDA->usImpMode = usImpMode; - } /* endif */ - } - else if ( UTF16strnicmp( pszCodePage, L"UTF16", 5 ) == 0) - { - usImpMode = SGMLFORMAT_UNICODE; - - if ( usImpMode != pLIDA->usImpMode ) - { - LOG_AND_SET_RC(usRc, T5INFO, ERROR_DIMP_UTF16WRONG); - T5LOG(T5ERROR) << "::rc = ERROR_DIMP_UTF16WRONG = " << usRc; - } /* endif */ - } - else - { // error - this should not occur... - LOG_AND_SET_RC(usRc, T5INFO, ERROR_MEM_INVALID_SGML); - T5LOG(T5ERROR) << "::rc = ERROR_MEM_INVALID_SGML" ; - } /* endif */ - } /* endif */ - - return(usRc); -} /* end of function MemHandleCodePageValue */ - #include DECLARE_int32(maxBadSegmentsIdsSaved); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index f0f58594..83b68ba2 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -65,14 +65,12 @@ USHORT EqfMemory::TmtXDelSegm USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound = 0; // compact area hits ULONG ulLen = 0; // length indication - PTMX_RECORD pTmRecord = NULL; // pointer to tm record CHAR szString[MAX_EQF_PATH]; // character string PULONG pulSids = NULL; // ptr to sentence ids PULONG pulSidStart = NULL; // ptr to sentence ids - ULONG ulRecBufSize = TMX_REC_SIZE; // current size of record buffer - //allocate pSentence - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); + std::vector pTmRecord; // space for user data + pTmRecord.resize(TMX_REC_SIZE); if ( fOK ) { @@ -104,7 +102,7 @@ USHORT EqfMemory::TmtXDelSegm TmDelIn.szSourceLanguage); // set the tag table ID in the tag record (this can't be done in TokenizeSource anymore) - TmDelIn.pInputSentence->pTagRecord->usTagTableId = 1; + TmDelIn.pInputSentence->pTagRecord.data()->usTagTableId = 1; } @@ -130,34 +128,14 @@ USHORT EqfMemory::TmtXDelSegm { ulKey = *pulSids; ulLen = TMX_REC_SIZE; - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - // re-alloc buffer and try again if buffer overflow occured - if ( usRc == BTREE_BUFFER_SMALL ) - { - fOK = UtlAlloc( (PVOID *)&(pTmRecord), ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - - usRc = TmBtree.EQFNTMGet( - ulKey, - (PCHAR)pTmRecord, - &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + usRc = TmBtree.EQFNTMGet(ulKey, //tm record key + pTmRecord ); //length if ( usRc == NO_ERROR ) { //find target record and delete, if the target record was the //only target in the tm record, delete the entire record - usRc = FindTargetAndDelete( pTmRecord, TmDelIn, pTmDelOut, &ulKey ); + usRc = FindTargetAndDelete( toTmxRecord(pTmRecord), TmDelIn, pTmDelOut, &ulKey ); if ( usRc == SEG_NOT_FOUND ) { //get next tm record @@ -180,7 +158,6 @@ USHORT EqfMemory::TmtXDelSegm } /* endif */ //release memory - UtlAlloc( (PVOID *) &pTmRecord, 0L, 0L, NOMSG ); UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); pTmDelOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); @@ -200,15 +177,11 @@ USHORT EqfMemory::TmtXDelSegmByKey BOOL fOK; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound = 0; // compact area hits - ULONG ulLen = 0; // length indication - PTMX_RECORD pTmRecord = NULL; // pointer to tm record - CHAR szString[MAX_EQF_PATH]; // character string - //PULONG pulSids = NULL; // ptr to sentence ids - //PULONG pulSidStart = NULL; // ptr to sentence ids + CHAR szString[MAX_EQF_PATH]; // character strin ULONG ulRecBufSize = TMX_REC_SIZE; // current size of record buffer + std::vector pTmRecord; // space for user data + pTmRecord.resize(TMX_REC_SIZE); - //allocate pSentence - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); if ( !fOK ) { @@ -218,44 +191,19 @@ USHORT EqfMemory::TmtXDelSegmByKey // update TM databse if ( !usRc ) { - ulLen = TMX_REC_SIZE; - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - // re-alloc buffer and try again if buffer overflow occured - if ( usRc == BTREE_BUFFER_SMALL ) - { - fOK = UtlAlloc( (PVOID *)&(pTmRecord), ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - - usRc = TmBtree.EQFNTMGet( - ulKey, - (PCHAR)pTmRecord, - &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + usRc = TmBtree.EQFNTMGet( ulKey, //tm record key + pTmRecord ); //pointer to tm record data if ( usRc == NO_ERROR ) { //find target record and delete, if the target record was the //only target in the tm record, delete the entire record - usRc = FindTargetByKeyAndDelete( pTmRecord, + usRc = FindTargetByKeyAndDelete( toTmxRecord(pTmRecord), TmDelIn, pSentence, pTmDelOut, &ulKey ); } /* endif */ } /* endif */ - //release memory - - UtlAlloc( (PVOID *) &pTmRecord, 0L, 0L, NOMSG ); - pTmDelOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); pTmDelOut->stPrefixOut.usTmtXRc = usRc; return( usRc ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 8fe20c00..2416fa37 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -66,9 +66,6 @@ USHORT EqfMemory::TmtXExtract { USHORT usRc = NO_ERROR; //return code BOOL fOK; //success indicator - PTMX_RECORD pTmRecord = NULL; //pointer to tm record - ULONG ulRecBufSize = 0; // current size of record buffer - ULONG ulLen = 0; //length indicator PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record ULONG ulStartKey; //start tm key @@ -76,9 +73,10 @@ USHORT EqfMemory::TmtXExtract BOOL fSpecialMode = FALSE; //special mode flag ULONG ulOemCP = 1L; + std::vector pTmRecord; // space for user data + pTmRecord.resize(2*TMX_REC_SIZE); + //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) (2 * TMX_REC_SIZE), NOMSG ); - ulRecBufSize = 2 * TMX_REC_SIZE; //allocate pString if ( !fOK ) @@ -143,7 +141,7 @@ USHORT EqfMemory::TmtXExtract break; case MEM_OUTPUT_LONGNAMES : // different handling for long names, no setting of pTable - ulMaxEntries = pLongNames->ulEntries; + ulMaxEntries = LongNames.stTableEntry.size(); break; case MEM_OUTPUT_ALLDOCS : // different handling for long names, no setting of pTable @@ -202,8 +200,7 @@ USHORT EqfMemory::TmtXExtract { if ( pTmExtIn->usConvert == MEM_OUTPUT_LONGNAMES ) { - pszName = pLongNames-> - stTableEntry[pTmExtIn->usNextTarget].pszLongName; + pszName = LongNames.stTableEntry[pTmExtIn->usNextTarget].pszLongName; } else { @@ -213,15 +210,14 @@ USHORT EqfMemory::TmtXExtract if ( pTmExtIn->usConvert == MEM_OUTPUT_ALLDOCS ) { ULONG ulEntry = 0; - while ( (ulEntry < pLongNames->ulEntries) && - (pEntry->usId != pLongNames-> - stTableEntry[ulEntry].usId ) ) + while ( (ulEntry < LongNames.stTableEntry.size()) && + (pEntry->usId != LongNames.stTableEntry[ulEntry].usId ) ) { ulEntry++; } /* endwhile */ - if ( ulEntry < pLongNames->ulEntries ) + if ( ulEntry < LongNames.stTableEntry.size() ) { - pszName = pLongNames->stTableEntry[ulEntry].pszLongName; + pszName = LongNames.stTableEntry[ulEntry].pszLongName; } /* endif */ } /* endif */ } /* endif */ @@ -276,34 +272,11 @@ USHORT EqfMemory::TmtXExtract LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); while ( (pTmExtIn->ulTmKey < ulNextKey) && (usRc == BTREE_NOT_FOUND) ) { - ulLen = ulRecBufSize; - usRc = TmBtree.EQFNTMGet( - pTmExtIn->ulTmKey, - (PCHAR)pTmRecord, - &ulLen ); - - // re-alloc buffer and try again if buffer overflow occured - if ( usRc == BTREE_BUFFER_SMALL ) - { - fOK = UtlAlloc( (PVOID *)&(pTmRecord), ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - - usRc = TmBtree.EQFNTMGet( - pTmExtIn->ulTmKey, - (PCHAR)pTmRecord, - &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + usRc = TmBtree.EQFNTMGet(pTmExtIn->ulTmKey, pTmRecord ); if ( usRc == NO_ERROR ) { - usRc = ExtractRecordV6( pTmRecord, pTmExtIn, pTmExtOut ); + usRc = ExtractRecordV6( toTmxRecord(pTmRecord), pTmExtIn, pTmExtOut ); } /****************************************************************/ /* setup new starting point (do this even in the case we are */ @@ -337,9 +310,6 @@ USHORT EqfMemory::TmtXExtract pTmExtOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); pTmExtOut->stPrefixOut.usTmtXRc = usRc; - //release memory - UtlAlloc( (PVOID *) &pTmRecord, 0L, 0L, NOMSG ); - // in organize mode only: change any error code (except BTREE_EOF_REACHED) to BTREE_CORRUPTED if ( (usRc != NO_ERROR ) && (usRc != BTREE_EOF_REACHED) && ((usAccessMode & ASD_ORGANIZE) != 0) ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 18f3bfb1..ade89b5c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -358,7 +358,7 @@ USHORT EqfMemory::TmtXGet if ( usRc == NO_ERROR ) { //usRc = NTMGetIDFromName( pTmGetIn->szTagTable, NULL, (USHORT)TAGTABLE_KEY, &Sentence.pTagRecord->usTagTableId ); - Sentence.pTagRecord->usTagTableId = 1; + Sentence.pTagRecord.data()->usTagTableId = 1; } } /* endif */ @@ -648,27 +648,12 @@ USHORT GetExactMatch USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator ULONG ulKey; //tm record key - PTMX_RECORD pTmRecord = NULL; //pointer to tm record USHORT usMatchEntries = 0; //nr of found matches - ULONG ulRecBufSize = 0; // current size of record buffer + std::vector& pTmRecord = pTmClb->pvTmRecord; ULONG ulStrippedParm = pGetIn->ulParm & ~(GET_RESPECTCRLF | GET_IGNORE_PATH | GET_NO_GENERICREPLACE | GET_ALWAYS_WITH_TAGS | GET_IGNORE_COMMENT); - //allocate 32K for tm record - if ( pTmClb->pvTmRecord ) - { - pTmRecord = (PTMX_RECORD)pTmClb->pvTmRecord; - ulRecBufSize = pTmClb->ulRecBufSize; -// memset( pTmRecord, 0, ulRecBufSize ); - } - else - { - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - pTmClb->pvTmRecord = pTmRecord; - if ( fOK ) pTmClb->ulRecBufSize = ulRecBufSize = TMX_REC_SIZE; - } /* endif */ - //allocate for sentence ids if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pulSids), 0L, @@ -689,32 +674,14 @@ USHORT GetExactMatch while ( (*pulSids) && (usRc == NO_ERROR) ) { ulKey = *pulSids; - ulLen = TMX_REC_SIZE; T5LOG( T5INFO) << "GetExactMatch: EQFNTMGET of record #" <TmBtree.EQFNTMGet( ulKey, (PCHAR)pTmRecord, &ulLen ); + usRc = pTmClb->TmBtree.EQFNTMGet( ulKey, pTmRecord ); - if ( usRc == BTREE_BUFFER_SMALL) - { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - pTmClb->ulRecBufSize = ulRecBufSize = ulLen; - pTmClb->pvTmRecord = pTmRecord; - memset( pTmRecord, 0, ulLen ); - - usRc = pTmClb->TmBtree.EQFNTMGet( ulKey, (PCHAR)pTmRecord, - &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ if ( usRc == NO_ERROR ) { //compare tm record data with data passed in the get in structure T5LOG( T5INFO) << "GetExactMatch: ExactTest of record " << ulKey; - usRc = ExactTest( pTmClb, pTmRecord, pGetIn, pSentence, + usRc = ExactTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pSentence, pstMatchTable, &usMatchEntries, ulKey ); T5LOG( T5INFO) << "GetExactMatch: ExactTest found " << usMatchEntries << " matching entries" ; @@ -966,7 +933,7 @@ USHORT ExactTest ULONG ulKeyNum // number of key record ) { - BOOL fOK; //success indicator + BOOL fOK = true; //success indicator PBYTE pByte; //position ptr PBYTE pStartTarget; //position ptr PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record @@ -974,32 +941,23 @@ USHORT ExactTest PTMX_TARGET_CLB pTMXTargetClb; //ptr to target control block PTMX_TAGTABLE_RECORD pTMXSourceTagTable; //ptr to source tag info ULONG ulLen; //length indicator - PSZ_W pString = NULL; //pointer to character string USHORT usRc = NO_ERROR; //returned value from function USHORT usMatchLevel; //pointer to ushort value BOOL fStringEqual = FALSE; // indicator for string equal BOOL fEqualFound = FALSE; // tagging equal ?? USHORT usEqual = 0; - PSZ_W pContextBuffer = NULL; //pointer to buffer for context processing + std::string pContextBuffer, pString; //pointer to buffer for context processing USHORT usTargetTranslationFlag = (USHORT)-1; // translation flag of target CLB (processed) PSZ pszDocName = NULL; // document name to check for equal document names in GET_IGNORE_PATH mode + pString.reserve(MAX_SEGMENT_SIZE); //allocate pString and buffer for context processing (when necessary) - fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - if ( fOK ) + if ( pGetIn->szContext[0] || (pGetIn->pvGMOptList != NULL) ) { - if ( pGetIn->szContext[0] || (pGetIn->pvGMOptList != NULL) ) - { - fOK = UtlAlloc( (PVOID *) &(pContextBuffer), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - } /* endif */ - + pContextBuffer.reserve(MAX_SEGMENT_SIZE); } /* endif */ - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + { //position at beginning of source structure in tm record pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); @@ -1124,41 +1082,28 @@ USHORT ExactTest //position at source tag table record pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + fStringEqual=true; - //position at source tag table - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXSourceTagTable = (PTMX_TAGTABLE_RECORD)pByte; - - //compare tag table records - fStringEqual = true; // tag table should be always equal in t5memory - //fStringEqual = memcmp( pTMXSourceTagTable, pSentence->pTagRecord, - // RECLEN(pTMXSourceTagTable) ) == 0; - //if ( !fStringEqual ) + + /**********************************************************/ + /* if tagging record is unequal than we have some */ + /* (slight) differences */ + /* i.e. we will create a fully qualified string and try */ + /* another compare... */ + /**********************************************************/ + LONG lLenTmp = ulLen; // len of pString in # of w's + if ( !fOK ) { - /**********************************************************/ - /* if tagging record is unequal than we have some */ - /* (slight) differences */ - /* i.e. we will create a fully qualified string and try */ - /* another compare... */ - /**********************************************************/ - - if ( fOK ) - { - LONG lLenTmp = ulLen; // len of pString in # of w's - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5INFO, BTREE_CORRUPTED); - } - else - { - fStringEqual = FALSE; - auto genericTagsTmSeg = std::make_unique(pString); - fStringEqual = (UtlCompIgnWhiteSpaceW( - pSentence->pStrings->getNpReplStrC(), - genericTagsTmSeg->getNpReplStrC(), - 0 ) == 0 ); - } /* endif */ - } /* endif */ + LOG_AND_SET_RC(usRc, T5INFO, BTREE_CORRUPTED); + } + else + { + fStringEqual = FALSE; + auto genericTagsTmSeg = std::make_unique(pString); + fStringEqual = (UtlCompIgnWhiteSpaceW( + pSentence->pStrings->getNpReplStrC(), + genericTagsTmSeg->getNpReplStrC(), + 0 ) == 0 ); } /* endif */ if ( usRc == NO_ERROR ) @@ -1168,6 +1113,7 @@ USHORT ExactTest fEqualFound = TRUE; usEqual = EQUAL_EQUAL; } + else { usEqual = EQUAL_EQUAL - 3; @@ -1499,7 +1445,7 @@ USHORT ExactTest pusMatchEntries, &pSentence->usActVote, &pSentence->usActVote, ulKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId ); + pGetIn, pSentence->pTagRecord.data(), usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId ); } else { @@ -1537,10 +1483,6 @@ USHORT ExactTest } /* endif */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); - if ( pContextBuffer ) UtlAlloc( (PVOID *)&pContextBuffer, 0L, 0L, NOMSG ); - return( usRc ); } @@ -1648,7 +1590,8 @@ BOOL TMFuzzynessEx PFUZZYTOK pFuzzyTgt = NULL; PFUZZYTOK pFuzzyTok = NULL; // returned token list PSZ pInBuf = NULL; // buffer for EQFBFindDiffEx function - PSZ pTokBuf = NULL; // buffer for EQFBFindDiffEx function + //PSZ pTokBuf = NULL; // buffer for EQFBFindDiffEx function + std::vector pTokBuf; PLOADEDTABLE pTable = NULL; // ptr to loaded tag table @@ -1672,7 +1615,7 @@ BOOL TMFuzzynessEx // allocate required buffers fOK = UtlAlloc((PVOID *)&pInBuf, 0L, 64000L, NOMSG ); - if ( fOK ) fOK = UtlAlloc((PVOID *)&pTokBuf, 0L, 64000 /*(LONG)TOK_BUFFER_SIZE*/, NOMSG ); + //if ( fOK ) fOK = UtlAlloc((PVOID *)&pTokBuf, 0L, 64000 /*(LONG)TOK_BUFFER_SIZE*/, NOMSG ); // load tag table if ( fOK ) @@ -1685,7 +1628,7 @@ BOOL TMFuzzynessEx // call function to evaluate the differences if ( fOK ) { - fOK = EQFBFindDiffEx( pTable, (PBYTE)pInBuf, (PBYTE)pTokBuf, pszSource, + fOK = EQFBFindDiffEx( pTable, (PBYTE)pInBuf, pTokBuf, pszSource, pszMatch, sLanguageId, (PVOID *)&pFuzzyTok, (PVOID *)&pFuzzyTgt, ulOemCP ); } /* endif */ @@ -1699,7 +1642,7 @@ BOOL TMFuzzynessEx if ( pFuzzyTgt ) UtlAlloc( (PVOID *)&pFuzzyTgt, 0L, 0L, NOMSG ); if ( pFuzzyTok ) UtlAlloc( (PVOID *)&pFuzzyTok, 0L, 0L, NOMSG ); if ( pInBuf ) UtlAlloc( (PVOID *)&pInBuf, 0L, 0L, NOMSG ); - if ( pTokBuf ) UtlAlloc( (PVOID *)&pTokBuf, 0L, 0L, NOMSG ); + //if ( pTokBuf ) UtlAlloc( (PVOID *)&pTokBuf, 0L, 0L, NOMSG ); if ( pTable ) TAFreeTagTable( pTable ); if(*pusWords > *pusDiffs){ @@ -1834,35 +1777,14 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct //calculate length of target string lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - if ( UtlAlloc( (PVOID *) &pTarget, 0L, (LONG)MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG)) { - lTargetLen = EQFCompress2Unicode(pTarget, pByte, lTargetLen); + lTargetLen = EQFCompress2Unicode(pSubstProp->szPropTarget, pByte, lTargetLen); //position at target tag record pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usTargetTagTable; - - //pSubstProp->pTagsPropTarget = (PTMX_TAGTABLE_RECORD)pByte; - - //usRc = (AddTagsToStringW( pTarget, - // &lTargetLen, // in # of w's - // (PTMX_TAGTABLE_RECORD)pByte, - // pSubstProp->szPropTarget )) ? usRc : BTREE_CORRUPTED; - //instead of lines above we now save not normalize string to btree - wcsncpy(pSubstProp->szPropTarget, pTarget, lTargetLen); - UtlAlloc( (PVOID *) &pTarget, 0L, 0L, NOMSG ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); } if ( usRc == NO_ERROR ) { //fill in the markup table - //PBYTE p = ((PBYTE)pTMXTargetRecord)+pTMXTargetRecord->usTargetTagTable; - //pTmClb->NTMGetNameFromID( - // &(((PTMX_TAGTABLE_RECORD)p)->usTagTableId), - // (USHORT)TAGTABLE_KEY, - // pSubstProp->szPropTagTable, NULL ); strcpy(pSubstProp->szPropTagTable, "OTMXUXLF"); ulTgtOemCP = 1; @@ -1871,9 +1793,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, (USHORT)LANG_KEY, pSubstProp->szTargetLanguage, NULL ); - //pTmClb->NTMGetNameFromID( &usSrcLangId, - // (USHORT)LANG_KEY, - // pSubstProp->szOriginalSrcLanguage, NULL ); } /* endif */ } /* endif */ @@ -2420,30 +2339,16 @@ USHORT GetFuzzyMatch BOOL fOK = TRUE; //success indicator USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator - PTMX_RECORD pTmRecord = NULL; //pointer to tm record PTMX_MATCHENTRY pMatchEntry = NULL; //pointer to match entry structure PTMX_MATCHENTRY pMatchStart = NULL; //pointer to match entry structure USHORT usMatchEntries = 0; //nr of matches found USHORT usOverlaps = 0; //nr of overlapping triples - ULONG ulRecBufSize = 0L; // current size of record buffer #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lOtherTime) ); #endif - - //allocate 32K for tm record if not done yet - if ( pTmClb->pvTmRecord ) - { - pTmRecord = (PTMX_RECORD)pTmClb->pvTmRecord; - ulRecBufSize = pTmClb->ulRecBufSize; -// memset( pTmRecord, 0, ulRecBufSize ); - } - else - { - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - pTmClb->pvTmRecord = pTmRecord; - pTmClb->ulRecBufSize = ulRecBufSize = TMX_REC_SIZE; - } /* endif */ + + std::vector& pTmRecord = pTmClb->pvTmRecord; //allocate for match entry if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pMatchEntry), 0L, (LONG)(ABS_VOTES * sizeof(TMX_MATCHENTRY)), NOMSG ); @@ -2473,27 +2378,12 @@ USHORT GetFuzzyMatch #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFuzzyOtherTime) ); #endif - ulLen = ulRecBufSize; T5LOG( T5INFO) << "GetFuzzyMatch: EQFNTMGET of record " << pMatchEntry->ulKey; - usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->ulKey, (PCHAR)pTmRecord, &ulLen ); + usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->ulKey, pTmRecord ); - if ( usRc == BTREE_BUFFER_SMALL) - { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - pTmClb->ulRecBufSize = ulRecBufSize = ulLen; - pTmClb->pvTmRecord = pTmRecord; - memset( pTmRecord, 0, ulLen ); - usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->ulKey, (PCHAR)pTmRecord, &ulLen ); - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFuzzyGetTime) ); #endif @@ -2503,7 +2393,7 @@ USHORT GetFuzzyMatch //compare tm record data with data passed in the get in structure usOverlaps = std::min( pMatchEntry->usMaxVotes, pSentence->usActVote ); - usRc = FuzzyTest( pTmClb, pTmRecord, pGetIn, pstMatchTable, + usRc = FuzzyTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pstMatchTable, &usMatchEntries, &usOverlaps, &pMatchEntry->usMaxVotes, &pSentence->usActVote, pSentence, pMatchEntry->ulKey ); @@ -3046,7 +2936,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block pstMatchTable, &usFuzzy, fTagTableEqual, pusMatchesFound, pusTmMaxVotes, pusOverlaps, ulKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId ); + pGetIn, pSentence->pTagRecord.data(), usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId ); } /* endif */ } /* endif */ //position at next target @@ -3105,14 +2995,14 @@ USHORT FillMatchEntry PUSHORT pusMatchThreshold ) { - PTMX_INDEX_RECORD pIndexRecord = NULL; //pointer to index record + std::vector pIndexRecord; //pointer to index record BOOL fOK = TRUE; //success indicator PTMX_MATCHENTRY pTempMatch = NULL; //ptr to working match entry structure PTMX_MATCHENTRY pTempStart = NULL; //ptr to working match entry structure USHORT usRc = NO_ERROR; //return code PTMX_INDEX_ENTRY pIndexEntry = NULL; //pointer to index entry PULONG pulVotes; // pointer to votes - ULONG ulLen; // length paramter + //ULONG ulLen; // length paramter USHORT i, j; // index in for loop USHORT usMaxEntries; // nr of index entries in index record ULONG lMatchEntries = 0; // nr of entries in pTempMatch @@ -3125,31 +3015,19 @@ USHORT FillMatchEntry GetElapsedTime( &(pTmClb->lOtherTime) ); #endif - //allocate index record buffer if not done yet - if ( pTmClb->pvIndexRecord ) + pTmClb->pvIndexRecord.reserve(TMX_REC_SIZE); + + + // allocate match list array if not done yet + if ( pTmClb->pvTempMatchList ) { - pIndexRecord = (PTMX_INDEX_RECORD)pTmClb->pvIndexRecord; - memset( pIndexRecord, 0, TMX_REC_SIZE ); + pTempMatch = (PTMX_MATCHENTRY)pTmClb->pvTempMatchList; + memset( pTempMatch , 0, lMatchListSize ); } else { - fOK = UtlAlloc( (PVOID *) &(pIndexRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - pTmClb->pvIndexRecord = pIndexRecord; - } /* endif */ - - // allocate match list array if not done yet - if ( fOK ) - { - if ( pTmClb->pvTempMatchList ) - { - pTempMatch = (PTMX_MATCHENTRY)pTmClb->pvTempMatchList; - memset( pTempMatch , 0, lMatchListSize ); - } - else - { - fOK = UtlAlloc( (PVOID *) &(pTempMatch), 0L, lMatchListSize, NOMSG ); - pTmClb->pvTempMatchList = pTempMatch; - } /* endif */ + fOK = UtlAlloc( (PVOID *) &(pTempMatch), 0L, lMatchListSize, NOMSG ); + pTmClb->pvTempMatchList = pTempMatch; } /* endif */ if ( fOK ) @@ -3166,17 +3044,16 @@ USHORT FillMatchEntry #endif if ( fOK ) { - pulVotes = pSentence->pulVotes; + pulVotes = pSentence->pulVotes.data(); lMatchEntries = 0; for ( i = 0; i < pSentence->usActVote; i++, pulVotes++ ) { ulKey = (*pulVotes) & START_KEY; - ulLen = TMX_REC_SIZE; #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFillMatchOtherTime) ); #endif - usRc = pTmClb->InBtree.EQFNTMGet( ulKey, (PCHAR)pIndexRecord, &ulLen ); + usRc = pTmClb->InBtree.EQFNTMGet( ulKey, pIndexRecord ); #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFillMatchReadTime) ); @@ -3186,10 +3063,10 @@ USHORT FillMatchEntry //calculate number of index entries in index record T5LOG(T5INFO) <<"Processing index record "<usRecordLen; + ULONG ulLen = toIndexRecord(pIndexRecord)->usRecordLen; usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - pIndexEntry = &pIndexRecord->stIndexEntry; + pIndexEntry = &toIndexEntry(pIndexRecord)->stIndexEntry; pTempMatch = pTempStart; //end criteria are all sentence ids in index key or only one diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index 1ddec6c8..054c34a4 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -1626,9 +1626,8 @@ BTREE::EQFNTMUpdate SHORT BTREE::QDAMDictExactLocal ( PWCHAR pKey, // key to be searched for - PBYTE pchBuffer, // space for user data - PULONG pulLength, // in/out length of returned user data - USHORT usSearchSubType // special hyphenation lookup flag + std::vector& pchBuffer, // space for user data + USHORT usSearchSubType // special hyphenation lookup flag ) { SHORT i; @@ -1667,24 +1666,10 @@ SHORT BTREE::QDAMDictExactLocal usCurrentRecord = RECORDNUM( pRecord ); //memcpy( chHeadTerm, pKey, sizeof(ULONG) ); // save data - + recData = QDAMGetrecData_V3( pRecord, i, usVersion ); - if ( *pulLength == 0 || ! pchBuffer ) - { - *pulLength = recData.ulLen; - } - else if ( *pulLength < recData.ulLen ) - { - if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - T5LOG(T5ERROR) << "::BTREE_BUFFER_SMALL, pulLength = " << *pulLength << "; recData.ulLen = " << recData.ulLen; - } - *pulLength = recData.ulLen; - sRc = BTREE_BUFFER_SMALL; - } - else - { - sRc = QDAMGetszData_V3( this, recData, pchBuffer, pulLength, DATA_NODE ); - } /* endif */ + sRc = QDAMGetszData_V3( this, recData, pchBuffer, DATA_NODE ); + } else { @@ -1739,41 +1724,33 @@ SHORT BTREE::EQFNTMGet ( ULONG ulKey, // key to be searched for - PCHAR pchBuffer, // space for user data - PULONG pulLength // in/out length of returned user data + std::vector& pchBuffer // space for user data ) { - SHORT sRc; // return code - DEBUGEVENT( EQFNTMGET_LOC, FUNCENTRY_EVENT, 0 ); + SHORT sRc = 0; // return code - CHECKPBTREE(this, sRc ); - if ( !sRc ) + SHORT RetryCount = MAX_RETRY_COUNT; // retry counter for in-use condition + do { - SHORT RetryCount = MAX_RETRY_COUNT; // retry counter for in-use condition - ULONG ulLength; - do - { - /******************************************************************/ - /* disable corruption flag to allow get of data in case memory */ - /* is corrupted */ - /******************************************************************/ - BOOL _fCorrupted = fCorrupted; - fCorrupted = FALSE; - ulLength = *pulLength; + /******************************************************************/ + /* disable corruption flag to allow get of data in case memory */ + /* is corrupted */ + /******************************************************************/ + BOOL _fCorrupted = fCorrupted; + fCorrupted = FALSE; - sRc = QDAMDictExactLocal( (PWCHAR) &ulKey, (PBYTE)pchBuffer, &ulLength, FEXACT ); + sRc = QDAMDictExactLocal( (PWCHAR) &ulKey, pchBuffer, FEXACT ); - fCorrupted = _fCorrupted; + fCorrupted = _fCorrupted; + + if ( sRc == BTREE_IN_USE ) + { + RetryCount--; + UtlWait( MAX_WAIT_TIME ); + } /* endif */ + } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && + (RetryCount > 0) ); /* enddo */ - if ( sRc == BTREE_IN_USE ) - { - RetryCount--; - UtlWait( MAX_WAIT_TIME ); - } /* endif */ - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - *pulLength = ulLength; - } /* endif */ if ( sRc && (sRc != BTREE_NOT_FOUND) ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index 643940d3..283af2fb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -396,7 +396,7 @@ USHORT EqfMemory::LoadMem() // just for debugging purposes: check if short and long name table have the same numbe rof entries if ( usTempRc == NO_ERROR ) { - if ( pLongNames->ulEntries != FileNames.ulMaxEntries ) + if ( LongNames.stTableEntry.size() != FileNames.ulMaxEntries ) { // this is strange... int iSetBreakPointHere = 1; @@ -451,21 +451,6 @@ USHORT EqfMemory::LoadMem() /* Ensure that all required data areas have been loaded and the */ /* database files have been opened */ /********************************************************************/ - if ( (usRc == NO_ERROR) || - (usRc == BTREE_CORRUPTED) || - (usRc == VERSION_MISMATCH) ) - { - if ( false //(pLanguages == NULL) || - //(pAuthors == NULL) || - //(pTagTables == NULL) || - //(pFileNames == NULL) //|| - //(this.TmBtree == NULL) || - //(this.InBtree == NULL) - ) - { - usRc = TM_FILE_SCREWED_UP; - } /* endif */ - } /* endif */ if ( (usRc != NO_ERROR) && (usRc != BTREE_CORRUPTED) && @@ -474,7 +459,6 @@ USHORT EqfMemory::LoadMem() TmBtree.QDAMDictClose(); InBtree.QDAMDictClose(); - NTMDestroyLongNameTable( ); T5LOG(T5ERROR) << "Failed to open tm \""<< this->szName << "\" with rc = " << usRc; setErrorCode(usRc); eStatus = FAILED_TO_OPEN_STATUS; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index c9f5efb4..f4ffaf0e 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -117,41 +117,7 @@ BOOL CheckForAlloc PTMX_SENTENCE pSentence, PTMX_TERM_TOKEN * ppTermTokens, USHORT usEntries -) -{ - LONG lFilled = 0L; - USHORT usNewAlloc = 0; - BOOL fOK = TRUE; - PTMX_TERM_TOKEN pTermTokens; - - pTermTokens = *ppTermTokens; - lFilled = ( (PBYTE)pTermTokens - (PBYTE)pSentence->pTermTokens); - if ( (pSentence->lTermAlloc - lFilled) <= (LONG)(usEntries * sizeof(TMX_TERM_TOKEN)) ) - { - //remember offset of pTagEntry - lFilled = pTermTokens - pSentence->pTermTokens; - if (usEntries == 1) - { - usNewAlloc = TOK_SIZE; - } - else - { - usNewAlloc = usEntries * sizeof(TMX_TERM_TOKEN); - } - //allocate another 4k for pTagRecord - fOK = UtlAlloc( (PVOID *) &pSentence->pTermTokens, pSentence->lTermAlloc, - pSentence->lTermAlloc + (LONG)usNewAlloc, NOMSG ); - if ( fOK ) - { - pSentence->lTermAlloc += (LONG)usNewAlloc; - - //set new position of pTagEntry - pTermTokens = pSentence->pTermTokens + lFilled; - } /* endif */ - } /* endif */ - *ppTermTokens= pTermTokens; - return(fOK); -} +); @@ -248,11 +214,8 @@ USHORT EqfMemory::TmtXReplace if ( !usRc ) { HashSentence( TmProposal.pInputSentence ); - TmProposal.pInputSentence->pTagRecord->usTagTableId = 1; - - //usRc = NTMGetIDFromName( TmProposal.szMarkup, NULL, (USHORT)TAGTABLE_KEY, - // &TmProposal.pInputSentence->pTagRecord->usTagTableId ); - TmProposal.pInputSentence->pTagRecord->usTagTableId = 1; + TmProposal.pInputSentence->pTagRecord[0].usTagTableId = 1; + if(usRc){ T5LOG( T5WARNING) << ":: NTMGetIDFromName( tagtable ) returned " << usRc; } @@ -309,12 +272,6 @@ USHORT EqfMemory::TmtXReplace usRc = TmBtree.QDAMDictUpdSignLocal(&stTmSign); } - // unlock TM database if database has been locked - if ( fLocked ) - { - NTMLockTM( this, FALSE, &fLocked ); - } /* endif */ - pTmPutOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); pTmPutOut->stPrefixOut.usTmtXRc = usRc; @@ -384,28 +341,25 @@ VOID HashSentence ) { PSZ_W pNormOffset; // pointer to start of normalized string - PTMX_TERM_TOKEN pTermTokens; // pointer to term token structure USHORT usCount = 0; // counter - pTermTokens = pSentence->pTermTokens; - while ( pTermTokens->usLength ) + //while ( pTermTokens->usLength ) + for(auto& token: pSentence->pTermTokens) { - pNormOffset = pSentence->pStrings->getNormStrC() + pTermTokens->usOffset; - pTermTokens->usHash = HashTupelW( pNormOffset, pTermTokens->usLength ); + pNormOffset = pSentence->pStrings->getNormStrC() + token.usOffset; + token.usHash = HashTupelW( pNormOffset, token.usLength ); + if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP) || FLAGS_log_hashes_in_hash_sentence || FLAGS_add_tokens_to_fuzzy){ - auto str = EncodingHelper::convertToUTF8(pNormOffset).substr(0, pTermTokens->usLength); + auto str = EncodingHelper::convertToUTF8(pNormOffset).substr(0, token.usLength); pSentence->tokens.push_back(str); - if(FLAGS_log_hashes_in_hash_sentence){ - T5LOG( T5TRANSACTION) <<"HashSentence:: pNormOffset = \"" << str << "\"; len = " << pTermTokens->usLength <<"; hash = " <usHash; - - }else{ - T5LOG( T5DEBUG) <<"HashSentence:: pNormOffset = \"" << str << "\"; len = " << pTermTokens->usLength <<"; hash = " <usHash; - } + //if(FLAGS_log_hashes_in_hash_sentence){ + T5LOG( T5DEBUG) <<"HashSentence:: pNormOffset = \"" << str << "\"; len = " << token.usLength <<"; hash = " << token.usHash; + //} } //max nr of hashes built usCount++; - pTermTokens++; + //pTermTokens++; } /* endwhile */ @@ -415,11 +369,14 @@ VOID HashSentence /********************************************************************/ if ( usCount < 5 ) { - pTermTokens = pSentence->pTermTokens; + PTMX_TERM_TOKEN pTermTokens = pSentence->pTermTokens.data(); for (USHORT i=0; i < usCount; i++ ) { - pSentence->pulVotes[pSentence->usActVote] = pTermTokens[i].usHash; + //pSentence->pulVotes[pSentence->usActVote] = pTermTokens[i].usHash; + pSentence->pulVotes.push_back(pTermTokens[i].usHash); pSentence->usActVote++; + T5LOG(T5TRANSACTION) << "pSentence->pulVotes.size = " << pSentence->pulVotes.size() <<"; actVote = " << pSentence->usActVote; + } /* endfor */ } /* endif */ @@ -508,7 +465,7 @@ USHORT HashTupelW // with 7 words A B C D E F G // PREREQ: it is ensured that sentence has at least 3 tokens //------------------------------------------------------------------------------ -static VOID +VOID BuildVotes ( PTMX_SENTENCE pSentence // pointer to sentence structure @@ -519,7 +476,7 @@ BuildVotes PTMX_TERM_TOKEN pLastTerm; //run through list of tokens and build tuples - pTermTokens = pSentence->pTermTokens; + pTermTokens = pSentence->pTermTokens.data(); while ( pTermTokens->usLength && (pSentence->usActVote < ABS_VOTES) ) { @@ -529,14 +486,14 @@ BuildVotes pLastTerm = pTermTokens; pLastTerm --; - pTermTokens = pSentence->pTermTokens; + pTermTokens = pSentence->pTermTokens.data(); while ( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { Vote( pTermTokens, pSentence, 1 ); // ABD, BCE, CDF, DEG pTermTokens++; } /* endwhile */ - pTermTokens = pSentence->pTermTokens; + pTermTokens = pSentence->pTermTokens.data(); while ( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { @@ -544,7 +501,7 @@ BuildVotes pTermTokens++; } /* endwhile */ - pTermTokens = pSentence->pTermTokens; + pTermTokens = pSentence->pTermTokens.data(); if( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { Vote (pTermTokens, pSentence, 4 ); // 1 * AAC @@ -559,7 +516,7 @@ BuildVotes /********************************************************************/ /* use 1st token as triple */ /********************************************************************/ - pTermTokens = pSentence->pTermTokens; + pTermTokens = pSentence->pTermTokens.data(); if( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { Vote (pTermTokens, pSentence, 7 ); // 1 * AAA @@ -575,7 +532,7 @@ BuildVotes } /* endif */ usIndex = 0; - pTermTokens = pSentence->pTermTokens; + pTermTokens = pSentence->pTermTokens.data(); while ((usIndex < 3) && pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { @@ -604,7 +561,7 @@ BuildVotes // Function flow: build all possible 3 tupels out of a sequence of 4 consec // utive tokens. (base is the FLASH algorithm). //------------------------------------------------------------------------------ -static VOID +VOID Vote ( PTMX_TERM_TOKEN pTermTokens, //ptr to term tokens @@ -638,9 +595,15 @@ Vote { ulVote = (ulVote*131) + pTermTokens[usTLookUp[i+3*usTuple]].usHash; } /* endfor */ + + pSentence->pulVotes.push_back(ulVote); - pSentence->pulVotes[pSentence->usActVote] = ulVote; + //pSentence->pulVotes[pSentence->usActVote] = ulVote; pSentence->usActVote++; + + //#ifdef DEBUG + T5LOG(T5TRANSACTION) << "pulVodet.size()=" << pSentence->pulVotes.size() << "; actVote = " << pSentence->usActVote; + //#endif } /* endif */ } /* end of function Vote */ @@ -665,7 +628,7 @@ USHORT CheckCompactArea EqfMemory* pTmClb // pointer to tm control block ) { - PULONG pulVotes = pSentence->pulVotes; // pointer to begin of votes + PULONG pulVotes = pSentence->pulVotes.data(); // pointer to begin of votes USHORT usMatch = 0; // number of matches for (USHORT i = 0; ( i < pSentence->usActVote) ; i++, pulVotes++ ) @@ -727,201 +690,6 @@ USHORT CheckCompactArea // build normalized string // //------------------------------------------------------------------------------ -USHORT TokenizeTarget -( - StringTagVariants* pStrings, // ptr to target string - //PSZ_W pNormString, // ptr to normalized string - PTMX_TAGTABLE_RECORD *ppTagRecord, // ptr to tag record structure - PLONG pulTagAlloc, // size of allocated area for tag record - PSZ pTagTableName, // name of tag table - //PUSHORT pusNormLen, // length of normalized string - EqfMemory* pClb // pointer to control block -) -{ - PVOID pTokenList = NULL; // ptr to token table - BOOL fOK; // success indicator - PBYTE pTagEntry; // pointer to tag entries - PLOADEDTABLE pTable = NULL; // pointer to tagtable -// USHORT usI; // offset - USHORT usFilled = 0; // counter - USHORT usRc = NO_ERROR; // returned value - int iTagEntryLen; // length indicator - PTMX_TAGTABLE_RECORD pTagRecord; // ptr to tag record structure - PSTARTSTOP pStartStop = NULL; // ptr to start/stop table - int iIterations = 0; - USHORT usAddEntries = 0; - PSZ_W pStringToTokenize = pStrings->getNormalizedTargetStrC();//pStrings->getGenericTargetStrC(); - - pTagRecord = (*ppTagRecord); //get contents of pointer - - //allocate 4K pTokenlist for TaTagTokenize - fOK = UtlAlloc( (PVOID *) &(pTokenList), 0L, (LONG) TOK_SIZE, NOMSG ); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - pTagEntry = (PBYTE)pTagRecord; - pTagEntry += sizeof(TMX_TAGTABLE_RECORD); - RECLEN(pTagRecord) = 0; - pTagRecord->usFirstTagEntry = (USHORT)(pTagEntry - (PBYTE)pTagRecord); - - pTagRecord->usTagTableId = 1; - if ( !usRc ) - { - //tokenize target segment with correct tag table - //load tag table for tokenize function - - //build tag table path - usRc = TALoadTagTableExHwnd( pTagTableName, &pTable, FALSE, - TALOADUSEREXIT | TALOADPROTTABLEFUNC | - TALOADCOMPCONTEXTFUNC, - FALSE, NULLHANDLE ); - if ( usRc ) - { - LOG_AND_SET_RC(usRc, T5INFO, ERROR_TA_ACC_TAGTABLE); - } /* endif */ - } /* endif */ - - if ( !usRc ) - { - // build protect start/stop table for tag recognition - usRc = TACreateProtectTableW( pStringToTokenize, pTable, 0, - (PTOKENENTRY)pTokenList, - TOK_SIZE, &pStartStop, - pTable->pfnProtTable, pTable->pfnProtTableW, 0L ); - - while ((iIterations < 10) && (usRc == EQFRS_AREA_TOO_SMALL)) - { - // (re)allocate token buffer - LONG lOldSize = (usAddEntries * sizeof(TOKENENTRY)) + (LONG)TOK_SIZE; - LONG lNewSize = ((usAddEntries+128) * sizeof(TOKENENTRY)) + (LONG)TOK_SIZE; - - if (UtlAlloc((PVOID *) &pTokenList, lOldSize, lNewSize, NOMSG) ) - { - usAddEntries += 128; - iIterations++; - } - else - { - iIterations = 10; // force end of loop - } /* endif */ - // retry tokenization - if (iIterations < 10 ) - { - usRc = TACreateProtectTableW( pStringToTokenize, pTable, 0, - (PTOKENENTRY)pTokenList, - (USHORT)lNewSize, &pStartStop, - pTable->pfnProtTable, pTable->pfnProtTableW, 0L ); - } /* endif */ - - } /* endwhile */ - } /* endif */ - - if ( !usRc ) - { - PSTARTSTOP pEntry = pStartStop; - while ( (pEntry->usStart != 0) || - (pEntry->usStop != 0) || - (pEntry->usType != 0) ) - { - switch ( pEntry->usType ) - { - case UNPROTECTED_CHAR : - // handle translatable text - { - USHORT usLength = pEntry->usStop - pEntry->usStart + 1; - } /* end case UNPROTECTED_CHAR */ - break; - default : - // handle not-translatable data - { - // if next start/stop-entry is a protected one ... - if ( ((pEntry+1)->usStart != 0) && - ((pEntry+1)->usType != UNPROTECTED_CHAR) ) - { - // enlarge next entry and ignore current one - (pEntry+1)->usStart = pEntry->usStart; - } - else - { - // add tag data - iTagEntryLen = sizeof(TMX_TAGENTRY) + - (pEntry->usStop - pEntry->usStart + 1)*sizeof(CHAR_W); - if ( ((LONG)*pulTagAlloc - (LONG)(pTagEntry - (PBYTE)pTagRecord)) - <= (LONG)iTagEntryLen ) - { - //remember offset of pTagEntry - usFilled = (USHORT)(pTagEntry - (PBYTE)pTagRecord); - - LONG addedSpace = TOK_SIZE;//how many 4k pages we need for new alloc - while(((LONG)*pulTagAlloc + addedSpace - (LONG)(pTagEntry - (PBYTE)pTagRecord)) - <= (LONG)iTagEntryLen ){ - addedSpace += TOK_SIZE; - } - //allocate another 4k for pTagRecord - int i=0; - do{ - fOK = UtlAlloc( (PVOID *) &pTagRecord, *pulTagAlloc, - *pulTagAlloc + addedSpace, NOMSG ); - i++; - }while(i<10 && fOK == false); - - if ( fOK ) - { - *pulTagAlloc += (LONG)TOK_SIZE; - - //set new position of pTagEntry - pTagEntry = ((PBYTE)pTagRecord) + usFilled; - }else{ - T5LOG(T5ERROR) << "::ERROR: Segment not saved. It was tried to allocate " << *pulTagAlloc + addedSpace << - " bytes to save the tags of the segment, but this was still not enough. Since this should never happen, we did not try to allocate even more bytes and did not save the segment. This was the segment, that was not saved: " - << EncodingHelper::convertToUTF8(pStringToTokenize); - } - - } /* endif */ - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - ((PTMX_TAGENTRY)pTagEntry)->usOffset = pEntry->usStart; - ((PTMX_TAGENTRY)pTagEntry)->usTagLen = - (pEntry->usStop - pEntry->usStart + 1);// * sizeof(CHAR_W); - memcpy( &(((PTMX_TAGENTRY)pTagEntry)->bData), - pStringToTokenize + pEntry->usStart, - ((PTMX_TAGENTRY)pTagEntry)->usTagLen * sizeof(CHAR_W)); - pTagEntry += iTagEntryLen; - } /* endif */ - } /* endif */ - } /* end default */ - break; - } /* endswitch */ - pEntry++; - } /* endwhile */ - RECLEN(pTagRecord) = pTagEntry - (PBYTE)pTagRecord; - } /* endif */ - } /* endif */ - - - //release memory - if ( pStartStop ) UtlAlloc( (PVOID *) &pStartStop, 0L, 0L, NOMSG ); - if ( pTokenList ) UtlAlloc( (PVOID *) &pTokenList, 0L, 0L, NOMSG ); - - //free tag table / decrement use count - if ( pTable != NULL ) - { - TAFreeTagTable( pTable ); - } /* endif */ - - *ppTagRecord = pTagRecord; - - return( usRc ); -} //------------------------------------------------------------------------------ // External function @@ -1303,14 +1071,15 @@ USHORT EqfMemory::UpdateTmIndex PULONG pulVotes = NULL; // pointer to votes USHORT i; // index in for loop ULONG ulKey; // index key - PTMX_INDEX_RECORD pIndexRecord = NULL; // pointer to index structure + std::vector pIndexRecord; // pointer to index structure BOOL fOK = FALSE; // success indicator PBYTE pIndex; // position pointer //for all votes add the index to the corresponding list - pulVotes = pSentence->pulVotes; + pulVotes = pSentence->pulVotes.data(); //allocate 32K for tm index record + pIndexRecord.resize(1 + TMX_REC_SIZE/sizeof(TMX_INDEX_RECORD)); fOK = UtlAlloc( (PVOID *) &(pIndexRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); if ( !fOK ) @@ -1325,21 +1094,21 @@ USHORT EqfMemory::UpdateTmIndex { ulKey = (*pulVotes) & START_KEY; ulLen = TMX_REC_SIZE; - memset( pIndexRecord, 0, TMX_REC_SIZE ); + memset( pIndexRecord.data(), 0, TMX_REC_SIZE ); usRc = InBtree.EQFNTMGet( ulKey, //index key - (PCHAR)pIndexRecord, //pointer to index record + (PCHAR)pIndexRecord.data(), //pointer to index record &ulLen ); //length if ( usRc == BTREE_NOT_FOUND ) { //key is not in index file; add a new index entry - pIndexRecord->usRecordLen = sizeof( TMX_INDEX_RECORD ); + pIndexRecord.data()->usRecordLen = sizeof( TMX_INDEX_RECORD ); - pIndexRecord->stIndexEntry = NTMINDEX(pSentence->usActVote,ulSidKey); + pIndexRecord.data()->stIndexEntry = NTMINDEX(pSentence->usActVote,ulSidKey); usRc = InBtree.EQFNTMInsert( &ulKey, - (PBYTE)pIndexRecord, //pointer to index - pIndexRecord->usRecordLen ); //length + (PBYTE)pIndexRecord.data(), //pointer to index + pIndexRecord.data()->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) @@ -1348,7 +1117,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMInsert(&ulKey, (PBYTE)pIndexRecord, pIndexRecord->usRecordLen ); + usRc = InBtree.EQFNTMInsert(&ulKey, (PBYTE)pIndexRecord.data(), pIndexRecord.data()->usRecordLen ); } /* endif */ } /* endif */ @@ -1369,7 +1138,7 @@ USHORT EqfMemory::UpdateTmIndex BOOL fFound = FALSE; //key is in index file; update index entry with new sid - ulLen = pIndexRecord->usRecordLen; + ulLen = pIndexRecord.data()->usRecordLen; //calculate number of entries in index record usIndexEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); @@ -1377,7 +1146,7 @@ USHORT EqfMemory::UpdateTmIndex //// check if SID is already contained in list.. //{ // int i = (int)usIndexEntries; - // PULONG pulIndex = (PULONG)&(pIndexRecord->stIndexEntry); + // PULONG pulIndex = (PULONG)&(pIndexRecord.data()->stIndexEntry); // while ( i ) // { // if ( *pulIndex == ulNewSID ) @@ -1395,7 +1164,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usIndexEntries >= (MAX_INDEX_LEN -1)) { //position pointer at beginning of index record - pIndex = (PBYTE)pIndexRecord; + pIndex = (PBYTE)pIndexRecord.data(); memmove( pIndex, pIndex + sizeof(ULONG), ulLen - sizeof(ULONG) ); ulLen -= sizeof(ULONG); usIndexEntries--; @@ -1405,7 +1174,7 @@ USHORT EqfMemory::UpdateTmIndex && ((ulLen + sizeof( TMX_INDEX_ENTRY )) <= TMX_REC_SIZE) ) { //position pointer at beginning of index record - pIndex = (PBYTE)pIndexRecord; + pIndex = (PBYTE)pIndexRecord.data(); //move pointer to end of index record pIndex += ulLen; @@ -1415,12 +1184,12 @@ USHORT EqfMemory::UpdateTmIndex NTMINDEX(pSentence->usActVote,ulSidKey); //update index record size - pIndexRecord->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); + pIndexRecord.data()->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); usRc = InBtree.EQFNTMUpdate( ulKey, - (PBYTE)pIndexRecord, //pointer to index - pIndexRecord->usRecordLen ); //length + (PBYTE)pIndexRecord.data(), //pointer to index + pIndexRecord.data()->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) { @@ -1428,7 +1197,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMUpdate(ulKey, (PBYTE)pIndexRecord, pIndexRecord->usRecordLen ); + usRc = InBtree.EQFNTMUpdate(ulKey, (PBYTE)pIndexRecord.data(), pIndexRecord.data()->usRecordLen ); } /* endif */ } /* endif */ if ( !usRc ) @@ -1458,8 +1227,6 @@ USHORT EqfMemory::UpdateTmIndex } /* endfor */ } /* endif */ - //release allocated memory - UtlAlloc( (PVOID *) &(pIndexRecord), 0L, 0L, NOMSG); return( usRc ); } @@ -1505,7 +1272,7 @@ USHORT DetermineTmRecord ) { USHORT usRc = NO_ERROR; // return code - ULONG ulLen; // length paramter + ULONG ulLen = 0; // length paramter USHORT usSid = 0; // number of sentence ids found USHORT usPos; // position in pulSids PULONG pulVotes; // pointer to votes @@ -1513,152 +1280,141 @@ USHORT DetermineTmRecord USHORT i, j; // index in for loop USHORT usMaxEntries = 0; // nr of index entries in index record ULONG ulKey; // index key - PTMX_INDEX_RECORD pIndexRecord = NULL; // pointer to index structure - BOOL fOK; // success indicator + std::vector pIndexRecord; // pointer to index structure + BOOL fOk = true; // success indicator PTMX_INDEX_ENTRY pIndexEntry; // pointer to index entry structure pulSidStart = pulSids; //for all votes add the index to the corresponding list - pulVotes = pSentence->pulVotes; + pulVotes = pSentence->pulVotes.data(); //allocate 32K for tm index record - fOK = UtlAlloc( (PVOID *) &(pIndexRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); + pIndexRecord.resize(TMX_REC_SIZE); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - ulKey = (*pulVotes) & START_KEY; - ulLen = TMX_REC_SIZE; - memset( pIndexRecord, 0, TMX_REC_SIZE ); - usRc = pTmClb->InBtree.EQFNTMGet( - ulKey, //index key - (PCHAR)pIndexRecord, //pointer to index record - &ulLen ); //length - if ( usRc == NO_ERROR ) - { - //calculate number of index entries in index record - ulLen = pIndexRecord->usRecordLen; - usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - std::string msg = __func__ + std::string(":: Number Entries: ") + toStr(usMaxEntries).c_str() +"; Entries:"; - pIndexEntry = &pIndexRecord->stIndexEntry; - for (j=0 ; jInBtree.EQFNTMGet( ulKey, //index key + pIndexRecord); //pointer to index record - pIndexEntry = &pIndexRecord->stIndexEntry; + if ( usRc == NO_ERROR ) + { + //calculate number of index entries in index record + ulLen = toIndexRecord(pIndexRecord)->usRecordLen; + usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - for ( j = 0; j < usMaxEntries; j++, pIndexEntry++ ) + if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ + std::string msg = __func__ + std::string(":: Number Entries: ") + toStr(usMaxEntries).c_str() +"; Entries:"; + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; + for (j=0 ; jusActVote ) + if (j % 10 == 0) { - *pulSids = NTMKEY(*pIndexEntry); - usSid++; - pulSids++; + msg+="\n"; } /* endif */ + msg += std::to_string(NTMKEY(*pIndexEntry)) + "; " ; } /* endfor */ + T5LOG( T5DEBUG) << msg; + } - if ( (usSid > 1) ) + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; + + for ( j = 0; j < usMaxEntries; j++, pIndexEntry++ ) + { + if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) + { + *pulSids = NTMKEY(*pIndexEntry); + usSid++; + pulSids++; + } /* endif */ + } /* endfor */ + + if ( (usSid > 1) ) + { + //there is more than one sentence to check so go through other + //indices until all have been tried or usSid has been reduced to 1 + pulVotes++; + for ( i = 0; (i < pSentence->usActVote-1) && (usSid > 1); + i++, pulVotes++ ) { - //there is more than one sentence to check so go through other - //indices until all have been tried or usSid has been reduced to 1 - pulVotes++; - for ( i = 0; (i < pSentence->usActVote-1) && (usSid > 1); - i++, pulVotes++ ) + if ( usRc == NO_ERROR ) { + ulKey = (*pulVotes) & START_KEY; + memset( pIndexRecord.data(), 0, pIndexRecord.size() ); + usRc = pTmClb->InBtree.EQFNTMGet( ulKey, //index key + pIndexRecord); //pointer to index record + if ( usRc == NO_ERROR ) { - ulKey = (*pulVotes) & START_KEY; - ulLen = TMX_REC_SIZE; - memset( pIndexRecord, 0, TMX_REC_SIZE ); - usRc = pTmClb->InBtree.EQFNTMGet( - ulKey, //index key - (PCHAR)pIndexRecord, //pointer to index record - &ulLen ); //length - - if ( usRc == NO_ERROR ) + //calculate number of index entries in index record + ulLen = toIndexRecord(pIndexRecord)->usRecordLen; + usMaxEntries = (USHORT)((pIndexRecord.size() - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); + + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; + pulSids = pulSidStart; + usPos = 0; + + //end criteria are all sentence ids in index key or only one + //sentence id left in pulSids + for ( j = 0; (j < usMaxEntries) && (usSid > 1); + j++, pIndexEntry++ ) { - //calculate number of index entries in index record - ulLen = pIndexRecord->usRecordLen; - usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - - pIndexEntry = &pIndexRecord->stIndexEntry; - pulSids = pulSidStart; - usPos = 0; - - //end criteria are all sentence ids in index key or only one - //sentence id left in pulSids - for ( j = 0; (j < usMaxEntries) && (usSid > 1); - j++, pIndexEntry++ ) + if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) { - if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) + //before adding sentence id check if already in pulsids as the + //respective tm record need only be checked once + while ( (NTMKEY(*pIndexEntry) > *pulSids) && (usSid > 1) ) { - //before adding sentence id check if already in pulsids as the - //respective tm record need only be checked once - while ( (NTMKEY(*pIndexEntry) > *pulSids) && (usSid > 1) ) + //remove sid from pulSids and decrease sid counter + if ( usSid > usPos ) { - //remove sid from pulSids and decrease sid counter - if ( usSid > usPos ) - { - ulLen = (usSid - usPos) * sizeof(ULONG); - } - else - { - ulLen = sizeof(ULONG); - } /* endif */ - memmove( (PBYTE) pulSids, (PBYTE)(pulSids+1), ulLen ); - usSid--; - } /* endwhile */ - - if ( NTMKEY(*pIndexEntry) == *pulSids ) + ulLen = (usSid - usPos) * sizeof(ULONG); + } + else { - //move on one position in pulSids - pulSids++; - usPos++; + ulLen = sizeof(ULONG); } /* endif */ + memmove( (PBYTE) pulSids, (PBYTE)(pulSids+1), ulLen ); + usSid--; + } /* endwhile */ + + if ( NTMKEY(*pIndexEntry) == *pulSids ) + { + //move on one position in pulSids + pulSids++; + usPos++; } /* endif */ - } /* endfor */ - } - else - { - //in case the index record doesn't exist exit function and - //set return code to 0 - if ( usRc == BTREE_NOT_FOUND ) - { - LOG_AND_SET_RC(usRc, T5DEVELOP, NO_ERROR); } /* endif */ + } /* endfor */ + } + else + { + //in case the index record doesn't exist exit function and + //set return code to 0 + if ( usRc == BTREE_NOT_FOUND ) + { + LOG_AND_SET_RC(usRc, T5DEVELOP, NO_ERROR); } /* endif */ } /* endif */ - } /* endfor */ - } /* endif */ - } - else + } /* endif */ + } /* endfor */ + } /* endif */ + } + else + { + //in case the index record doesn't exist exit function and set return + //code to 0 + if ( usRc == BTREE_NOT_FOUND ) { - //in case the index record doesn't exist exit function and set return - //code to 0 - if ( usRc == BTREE_NOT_FOUND ) - { - LOG_AND_SET_RC(usRc, T5DEVELOP, NO_ERROR); - } /* endif */ + LOG_AND_SET_RC(usRc, T5DEVELOP, NO_ERROR); } /* endif */ } /* endif */ + if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ std::string msg = __func__ + std::string(":: Matching Sids: "); - pIndexEntry = &pIndexRecord->stIndexEntry; + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; while ( *pulSidStart ) { msg += std::to_string(*pulSidStart) + "; "; @@ -1667,8 +1423,6 @@ USHORT DetermineTmRecord T5LOG( T5DEBUG) << msg; } - UtlAlloc( (PVOID *) &(pIndexRecord), 0L, 0L, NOMSG ); - if ( usRc ) { ERREVENT2( DETERMINETMRECORD_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); @@ -1716,22 +1470,17 @@ USHORT EqfMemory::UpdateTmRecord OtmProposal& TmProposal //pointer to get in data ) { - BOOL fOK; //success indicator + BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indication to leave while loop PULONG pulSids = NULL; //ptr to sentence ids PULONG pulSidStart = NULL; //ptr to sentence ids USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator ULONG ulKey; //tm record key - PTMX_RECORD pTmRecord = NULL; //pointer to tm record - ULONG ulRecBufSize = 0L; // current size of record buffer + std::vector TmRecord; //pointer to tm record + + TmRecord.resize(TMX_REC_SIZE); - //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - if ( fOK ) - { - ulRecBufSize = TMX_REC_SIZE; - } /* endif */ //allocate for sentence ids if ( fOK ) @@ -1755,36 +1504,13 @@ USHORT EqfMemory::UpdateTmRecord while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) { ulKey = *pulSids; - ulLen = TMX_REC_SIZE; - memset( pTmRecord, 0, ulLen ); - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - if ( usRc == BTREE_BUFFER_SMALL) - { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - memset( pTmRecord, 0, ulLen ); - - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + usRc = TmBtree.EQFNTMGet( ulKey, //tm record key + TmRecord); //pointer to tm record data if ( usRc == NO_ERROR ) { //compare tm record data with data passed in the get in structure - usRc = ComparePutData( &pTmRecord, &ulRecBufSize, - TmProposal, &ulKey ); + usRc = ComparePutData( TmRecord, TmProposal, &ulKey ); if ( usRc == SOURCE_STRING_ERROR ) { @@ -1813,7 +1539,6 @@ USHORT EqfMemory::UpdateTmRecord //release memory UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pTmRecord, 0L, 0L, NOMSG ); return( usRc ); } @@ -1855,7 +1580,7 @@ USHORT EqfMemory::UpdateTmRecordByInternalKey OtmProposal& TmProposal ) { - BOOL fOK; //success indicator + BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indication to leave while loop PULONG pulSids = NULL; //ptr to sentence ids PULONG pulSidStart = NULL; //ptr to sentence ids @@ -1863,14 +1588,9 @@ USHORT EqfMemory::UpdateTmRecordByInternalKey ULONG ulLen; //length indicator ULONG ulKey; //tm record key PTMX_RECORD pTmRecord = NULL; //pointer to tm record - ULONG ulRecBufSize = 0L; // current size of record buffer - //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - if ( fOK ) - { - ulRecBufSize = TMX_REC_SIZE; - } /* endif */ + std::vector TmRecord; // space for user data + TmRecord.resize(TMX_REC_SIZE); //allocate for sentence ids if ( fOK ) @@ -1894,36 +1614,15 @@ USHORT EqfMemory::UpdateTmRecordByInternalKey while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) { ulKey = *pulSids; - ulLen = TMX_REC_SIZE; - memset( pTmRecord, 0, ulLen ); - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - if ( usRc == BTREE_BUFFER_SMALL) - { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - memset( pTmRecord, 0, ulLen ); - - usRc = TmBtree.EQFNTMGet( - ulKey, //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + memset( &TmRecord, 0, TmRecord.size() ); + usRc = TmBtree.EQFNTMGet(ulKey, //tm record key + TmRecord); //pointer to tm record data + if ( usRc == NO_ERROR ) { //compare tm record data with data passed in the get in structure - usRc = ComparePutData( &pTmRecord, &ulRecBufSize, - TmProposal, &ulKey ); + usRc = ComparePutData( TmRecord, TmProposal, &ulKey ); if ( usRc == SOURCE_STRING_ERROR ) { @@ -1951,7 +1650,6 @@ USHORT EqfMemory::UpdateTmRecordByInternalKey //release memory UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pTmRecord, 0L, 0L, NOMSG ); return( usRc ); } @@ -2009,13 +1707,12 @@ DECLARE_bool(log_memmove_in_compareputdata); //------------------------------------------------------------------------------ USHORT EqfMemory::ComparePutData ( - PTMX_RECORD *ppTmRecord, // ptr to ptr of tm record data buffer - PULONG pulRecBufSize, // current size of record buffer + std::vector& ppTmRecord, // ptr to ptr of tm record data buffer OtmProposal& TmProposal, // pointer to get in data PULONG pulKey // tm key ) { - BOOL fOK; //success indicator + BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indicates whether to leave loop or not PBYTE pByte; //position ptr PBYTE pStartTarget; //position ptr @@ -2024,7 +1721,8 @@ USHORT EqfMemory::ComparePutData PTMX_TARGET_CLB pClb = NULL; //ptr to target control block LONG lLen = 0; //length indicator USHORT usNormLen = 0; //length of normalized string - PSZ_W pString = NULL; //pointer to character string + //PSZ_W pString = NULL; //pointer to character string + std::vector pString(MAX_SEGMENT_SIZE); USHORT usRc = NO_ERROR; //returned value from function BOOL fUpdate = FALSE; // TRUE = record has been updated PTMX_RECORD pTmRecord = *ppTmRecord; // pointer to tm record data @@ -2033,7 +1731,7 @@ USHORT EqfMemory::ComparePutData int delTargetKey = 0, targetKey = 0; //allocate pString - fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG) MAX_SEGMENT_SIZE*sizeof(CHAR_W), NOMSG ); + //fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG) MAX_SEGMENT_SIZE*sizeof(CHAR_W), NOMSG ); if ( !fOK ) { @@ -2066,9 +1764,9 @@ USHORT EqfMemory::ComparePutData //calculate length of source string lLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); //copy and compare source string - memset( pString, 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); + memset( pString.data(), 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); lLen = EQFCompress2Unicode( pString, pByte, lLen ); - fStringEqual = ! UTF16strcmp( pString, TmProposal.pInputSentence->pStrings->getGenericTagStrC() ); + fStringEqual = ! UTF16strcmp( pString.data(), TmProposal.pInputSentence->pStrings->getGenericTagStrC() ); if ( fStringEqual ) { @@ -2122,10 +1820,10 @@ USHORT EqfMemory::ComparePutData pByte = NTRecPos(pStartTarget, REC_TGTSTRING); lLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; lLen = EQFCompress2Unicode( pString, pByte, lLen ); - pString[lLen] = EOS; + pString.data()[lLen] = EOS; //compare target strings and target tag record - if ( !UTF16strcmp( pString, TmProposal.pInputSentence->pStrings->getGenericTargetStrC())) + if ( !UTF16strcmp( pString.data(), TmProposal.pInputSentence->pStrings->getGenericTargetStrC())) { //target strings and target tag record are equal //position at first control block pClb = (PTMX_TARGET_CLB)NTRecPos(pStartTarget, REC_CLB); @@ -2299,7 +1997,7 @@ USHORT EqfMemory::ComparePutData } /* endif */ //release memory - UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); + //UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); //UtlAlloc( (PVOID *) &pTagRecord, 0L, 0L, NOMSG ); if ( usRc ) @@ -2349,42 +2047,24 @@ USHORT EqfMemory::AddTmTarget( PULONG pulRecBufSize, //ptr to current size of TM record buffer PULONG pulKey ) //tm key { - PTMX_TARGET_CLB pTargetClb ; // ptr to target ctl block - PTMX_TARGET_RECORD pTargetRecord = NULL; // ptr to target record - //PTMX_TAGTABLE_RECORD pTagRecord = NULL; // ptr to tag table record + PTMX_TARGET_CLB pTargetClb = nullptr ; // ptr to target ctl block + PTMX_TARGET_RECORD pTargetRecord = nullptr; // ptr to target record + std::vector vTargetClb, vTargetRecord;//for managing memory USHORT usRc = NO_ERROR; // return code BOOL fOK; // success indicator - //LONG lTagAlloc; // alloc size PBYTE pByte; // position pointer PTMX_RECORD pTmRecord = *ppTmRecord; //pointer to tm record data ULONG ulAddDataLen = 0; //allocate target control block record ulAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - fOK = UtlAlloc( (PVOID *) &pTargetClb, 0L, (LONG)(sizeof(TMX_TARGET_CLB)+ulAddDataLen), NOMSG ); + vTargetClb.resize(sizeof(TMX_TARGET_CLB)+ulAddDataLen); + vTargetRecord.resize(TMX_REC_SIZE); + pTargetClb = (PTMX_TARGET_CLB)vTargetClb.data(); + pTargetRecord = (PTMX_TARGET_RECORD)vTargetRecord.data(); - //allocate target record - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *) &(pTargetRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - } - - //allocate 4k for pTagRecord - //if ( fOK ) - //{ - //fOK = UtlAlloc( (PVOID *) &(pTagRecord), 0L, (LONG) TOK_SIZE, NOMSG ); - //if ( fOK ) - // lTagAlloc = (LONG)TOK_SIZE; - //} - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else { - //usRc = TokenizeTarget( TmProposal.pInputSentence->pStrings.get(), &pTagRecord, - // &lTagAlloc, TmProposal.szMarkup, this ); if ( usRc == NO_ERROR ) { usRc = FillClb( pTargetClb, TmProposal ); @@ -2392,11 +2072,8 @@ USHORT EqfMemory::AddTmTarget( { //fill target record FillTargetRecord( TmProposal.pInputSentence, //ptr to sentence structure - // pTagRecord, //ptr to target string tag table TmProposal.pInputSentence->pStrings->getGenericTargetStrC(), wcslen(TmProposal.pInputSentence->pStrings->getGenericTargetStrC()), - //pNormString,//ptr to target normalized string - //usNormLen, //length of target normalized string &pTargetRecord, //filled tm record returned pTargetClb ); //tm target control block @@ -2442,17 +2119,11 @@ USHORT EqfMemory::AddTmTarget( } /* endif */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &(pTargetClb), 0L, 0L, NOMSG); - UtlAlloc( (PVOID *) &(pTargetRecord), 0L, 0L, NOMSG); - //UtlAlloc( (PVOID *) &(pTagRecord), 0L, 0L, NOMSG); - if ( usRc ) { ERREVENT2( ADDTMTARGET_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); } /* endif */ - return( usRc ); } @@ -2590,7 +2261,6 @@ USHORT EqfMemory::TmtXUpdSeg { BOOL fOK; // success indicator USHORT usRc = NO_ERROR; // return code - PTMX_RECORD pTmRecord = NULL; // pointer to tm record BOOL fLocked = FALSE; // TM-database-has-been-locked flag ULONG ulLen = 0; //length indicator PBYTE pByte; //position ptr @@ -2598,18 +2268,9 @@ USHORT EqfMemory::TmtXUpdSeg PTMX_TARGET_RECORD pTMXTargetRecord; // ptr to target record PTMX_TARGET_CLB pTargetClb; // ptr to target CLB ULONG ulLeftClbLen; // remaining length of CLB area - ULONG ulRecBufSize = 0L; // current size of record buffer - - //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - ulRecBufSize = TMX_REC_SIZE; - } /* endif */ + + std::vector TmRecord; // space for user data + TmRecord.resize(TMX_REC_SIZE); // get TM record being modified and update the record if ( !usRc ) @@ -2617,27 +2278,8 @@ USHORT EqfMemory::TmtXUpdSeg LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); ulLen = TMX_REC_SIZE; - usRc = TmBtree.EQFNTMGet( pTmPutIn->currentInternalKey.getRecordKey(), (PCHAR)pTmRecord, - &ulLen ); - - if ( usRc == BTREE_BUFFER_SMALL) - { - fOK = UtlAlloc( (PVOID *)&pTmRecord, ulRecBufSize, ulLen, NOMSG ); - if ( fOK ) - { - ulRecBufSize = ulLen; - memset( pTmRecord, 0, ulLen ); - - usRc = TmBtree.EQFNTMGet( - pTmPutIn->currentInternalKey.getRecordKey(), //tm record key - (PCHAR)pTmRecord, //pointer to tm record data - &ulLen ); //length - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + usRc = TmBtree.EQFNTMGet( pTmPutIn->currentInternalKey.getRecordKey(), TmRecord ); + PTMX_RECORD pTmRecord = (PTMX_RECORD) TmRecord.data(); if ( usRc == NO_ERROR ) { @@ -2773,7 +2415,7 @@ USHORT EqfMemory::TmtXUpdSeg if ( usRc == NO_ERROR ) { usRc = TmBtree.EQFNTMUpdate( pTmPutIn->currentInternalKey.getRecordKey(), - (PBYTE)pTmRecord, RECLEN(pTmRecord) ); + pTmRecord, RECLEN(pTmRecord) ); } /* endif */ if(usRc == NO_ERROR) @@ -2802,15 +2444,6 @@ USHORT EqfMemory::TmtXUpdSeg } /* endif */ } /* endif */ - // unlock TM database if database has been locked - if ( fLocked ) - { - NTMLockTM( this, FALSE, &fLocked ); - } /* endif */ - - //release allocated memory - UtlAlloc( (PVOID *)&pTmRecord, 0L, 0L, NOMSG ); - return( usRc ); } /* end of function TmtXUpdSeg */ @@ -2958,26 +2591,26 @@ ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG ulLenChar ) } -LONG EQFCompress2Unicode( PSZ_W pOutput, PBYTE pTarget, ULONG ulLenComp ) +LONG EQFCompress2Unicode( std::string& pOutput, PBYTE pTarget, ULONG ulLenComp ) { LONG lLen = ulLenComp-1; + pOutput.clear(); - //T5LOG(T5TRANSACTION) <<"; lLen = " << lLen; if(lLen <= 0 ) return 0; + + pOutput.reserve(ulLenComp * 1.5); BYTE b = *pTarget++; - //T5LOG(T5TRANSACTION) << "b = " << b <<"; lLen = " << lLen; if ( b == 0 ) // no compression { - memcpy( pOutput, pTarget, lLen ); lLen = ulLenComp / sizeof(CHAR_W); + pOutput.insert(pOutput.begin(), pTarget, pTarget + lLen); pOutput[ lLen ] = EOS; } - //* else if (b == BOCU_COMPRESS) { - PSZ_W pTemp = pOutput; + //PSZ_W pTemp = pOutput; long delta = 0; USHORT iLen = 0; int oldCodePoint = 0x80; @@ -3001,22 +2634,21 @@ LONG EQFCompress2Unicode( PSZ_W pOutput, PBYTE pTarget, ULONG ulLenComp ) delta = delta + oldCodePoint - lOffset[i]; - *pOutput++ = (unsigned short) delta; + pOutput.insert(pOutput.end(), (unsigned short) delta); oldCodePoint = delta; break; } } } - *pOutput = 0; - lLen = (pOutput - pTemp); + pOutput.push_back('\0'); + lLen = pOutput.size();//(pOutput - pTemp); } else { //FilesystemHelper::FlushAllBuffers(); assert( 0 == 1); } - //*/ return lLen; } #endif @@ -3209,12 +2841,6 @@ USHORT TMLoopAndDelTargetClb NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTgtRec), 4713); lLeftTgtLen -= RECLEN(pTMXTgtRec); - // pos at source tag info - //pTMXSrcTTable = (PTMX_TAGTABLE_RECORD)NTRecPos((PBYTE)pTMXTgtRec, - // REC_SRCTAGTABLE); - // compare source tag record - //if ( (memcmp( pTMXSrcTTable, TmProposal.pInputSentence->pTagRecord, - // RECLEN(pTMXSrcTTable)) == 0) ) { pClb = (PTMX_TARGET_CLB)NTRecPos((PBYTE)pTMXTgtRec, REC_CLB); lLeftClbLen = RECLEN(pTMXTgtRec) - pTMXTgtRec->usClb; @@ -3224,9 +2850,7 @@ USHORT TMLoopAndDelTargetClb (*pTargetKey)++; if ( (pClb->usLangId == usPutLang) && (pClb->usAuthorId == usPutAuthor) && - //(pClb->ulSegmId == TmProposal.lSegmentId) && (pClb->usFileId == usPutFile) && - //&& !pClb->bMultiple isAddDataIsTheSame(TmProposal, *pClb) ) { // remove target CLB and target record (if only 1 CLB) diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 53ac161d..5d872f32 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -38,8 +38,6 @@ EqfMemory::~EqfMemory() T5LOG(T5TRANSACTION) << "Closing memory " << szName; } UnloadFromRAM(); - //free allocated memory - NTMDestroyLongNameTable(); } /*! \brief Get number of markups used for the proposals in this mmoryProvides a part of the memory in binary format diff --git a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp index c5f25eec..e82b72c5 100755 --- a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp @@ -3799,7 +3799,7 @@ USHORT APIENTRY WRITEEXPSEGMENT( LONG lMemHandle, PMEMEXPIMPSEG pSegment ) } // helper function: is character a list delimiter or space -static BOOL isDelimiter( CHAR chTest ) +BOOL isDelimiter( CHAR chTest ) { return ( (chTest == '(') || (chTest == ')') || (chTest == ',') || (chTest == ' ') || (chTest == EOS) ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp index f95d298e..e82e88b2 100755 --- a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp @@ -222,19 +222,6 @@ BOOL NTMDocMatch } /* end of function NTMDocMatch */ -BOOL -PrepareTokens -( - PLOADEDTABLE pTagTable, - PBYTE pInBuf, - PBYTE pTokBuf, - PSZ_W pString, // pointer to string to be tokenized - SHORT sLanguageId, // language ID - PFUZZYTOK * ppTransTokList, // resulting list of tokens - ULONG ulOemCP -); - - BOOL EQFBCallLCS @@ -256,8 +243,6 @@ BOOL EQFBMarkModDelIns USHORT usLenStr2 ); -static -BOOL CheckForAlloc( PTMX_SENTENCE pSentence, PTMX_TERM_TOKEN * ppTermTokens, USHORT usEntries ); static BOOL CheckForAlloc @@ -267,37 +252,17 @@ BOOL CheckForAlloc USHORT usEntries ) { - LONG lFilled = 0L; - USHORT usNewAlloc = 0; BOOL fOK = TRUE; PTMX_TERM_TOKEN pTermTokens; - pTermTokens = *ppTermTokens; - lFilled = ( (PBYTE)pTermTokens - (PBYTE)pSentence->pTermTokens); - if ( (pSentence->lTermAlloc - lFilled) <= (LONG)(usEntries * sizeof(TMX_TERM_TOKEN)) ) + auto lFilled = pTermTokens - pSentence->pTermTokens.data(); + + if ( (usEntries + lFilled/*pSentence->pTermTokens.size()*/) > pSentence->pTermTokens.size()) { //remember offset of pTagEntry - lFilled = pTermTokens - pSentence->pTermTokens; - if (usEntries == 1) - { - usNewAlloc = TOK_SIZE; - } - else - { - usNewAlloc = usEntries * sizeof(TMX_TERM_TOKEN); - } - //allocate another 4k for pTagRecord - fOK = UtlAlloc( (PVOID *) &pSentence->pTermTokens, pSentence->lTermAlloc, - pSentence->lTermAlloc + (LONG)usNewAlloc, NOMSG ); - if ( fOK ) - { - pSentence->lTermAlloc += (LONG)usNewAlloc; - - //set new position of pTagEntry - pTermTokens = pSentence->pTermTokens + lFilled; - } /* endif */ + pSentence->pTermTokens.resize( pSentence->pTermTokens.size() + usEntries + 1); } /* endif */ - *ppTermTokens= pTermTokens; + *ppTermTokens= &pSentence->pTermTokens[lFilled]; return(fOK); } @@ -352,15 +317,15 @@ USHORT EqfMemory::TokenizeSource PSZ pSourceLang // source language ) { - PVOID pTokenList = NULL; // ptr to token table - BOOL fOK; // success indicator - PBYTE pTagEntry; // pointer to tag entries - PTMX_TERM_TOKEN pTermTokens; // pointer to term tokens + std::vector pTokenList; // ptr to token table + BOOL fOK = true; // success indicator + PTMX_TAGENTRY pTagEntry = nullptr; // pointer to tag entries + //PTMX_TERM_TOKEN pTermTokens = nullptr; // pointer to term tokens PLOADEDTABLE pTable = NULL; // pointer to tagtable PTMX_TAGTABLE_RECORD pTagRecord; // pointer to record USHORT usLangId; // language id USHORT usRc = NO_ERROR; // returned value - USHORT usFilled; // counter + size_t sFilled; // counter USHORT usTagEntryLen; // length indicator CHAR szString[MAX_FNAME]; // name without extension USHORT usStart; // position counter @@ -403,7 +368,7 @@ USHORT EqfMemory::TokenizeSource } //allocate 4K pTokenlist for TaTagTokenize - fOK = UtlAlloc( (PVOID *) &(pTokenList), 0L, (LONG)TOK_SIZE, NOMSG ); + pTokenList.reserve(TOK_SIZE/sizeof(pTokenList[0])); if ( !fOK ) { @@ -411,13 +376,12 @@ USHORT EqfMemory::TokenizeSource } else { - pTagRecord = pSentence->pTagRecord; - pTermTokens = pSentence->pTermTokens; - pTagEntry = (PBYTE)pTagRecord; - pTagEntry += sizeof(TMX_TAGTABLE_RECORD); + pTagRecord = pSentence->pTagRecord.data(); + //pTermTokens = pSentence->pTermTokens.data(); + pTagEntry = (PTMX_TAGENTRY) ((PBYTE)pTagRecord + sizeof(TMX_TAGTABLE_RECORD)); RECLEN(pTagRecord) = 0; - pTagRecord->usFirstTagEntry = (USHORT)(pTagEntry - (PBYTE)pTagRecord); + pTagRecord->usFirstTagEntry = (USHORT)((PBYTE)pTagEntry - (PBYTE)pTagRecord); //get id of tag table, call Utlstrccpy( szString, UtlGetFnameFromPath( pTagTableName ), DOT ); @@ -463,12 +427,12 @@ USHORT EqfMemory::TokenizeSource // build protect start/stop table for tag recognition usRc = TACreateProtectTableWEx( pStringToTokenize, pTable, 0, - (PTOKENENTRY)pTokenList, + pTokenList, TOK_SIZE, &pStartStop, pTable->pfnProtTable, pTable->pfnProtTableW, 1, 0); - + #ifdef TO_BE_REMOVED while ((iIterations < 10) && (usRc == EQFRS_AREA_TOO_SMALL)) { // (re)allocate token buffer @@ -489,13 +453,14 @@ USHORT EqfMemory::TokenizeSource { usRc = TACreateProtectTableWEx( pStringToTokenize, pTable, 0, - (PTOKENENTRY)pTokenList, + pTokenList, (USHORT)lNewSize, &pStartStop, pTable->pfnProtTable, pTable->pfnProtTableW, 1, 0 ); } /* endif */ } /* endwhile */ + #endif } /* endif */ if ( !usRc ) @@ -513,8 +478,8 @@ USHORT EqfMemory::TokenizeSource // handle translatable text { PFLAGOFFSLIST pTerm; // pointer to terms list - PFLAGOFFSLIST pTermList = NULL; // pointer to offset/length term list - ULONG ulListSize = 0; + std::vector pTermList; // pointer to offset/length term list + //ULONG ulListSize = 0; USHORT usTokLen = pEntry->usStop - pEntry->usStart + 1; CHAR_W chTemp = pStringToTokenize[pEntry->usStop+1]; // buffer for character values @@ -522,47 +487,35 @@ USHORT EqfMemory::TokenizeSource usRc = NTMMorphTokenizeW( usLangId, pStringToTokenize + pEntry->usStart, - &ulListSize, (PVOID *)&pTermList, - MORPH_FLAG_OFFSLIST ); + pTermList ); pStringToTokenize[pEntry->usStop+1] = chTemp; if ( usRc == MORPH_OK ) { - pTerm = pTermList; - fOK = CheckForAlloc( pSentence, &pTermTokens, 1 ); - if ( !fOK ) + pTerm = pTermList.data(); + fOK = true; + //fOK = CheckForAlloc( pSentence, &pTermTokens, 1 ); + + usStart = pEntry->usStart; + if ( pTerm ) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - usStart = pEntry->usStart; - if ( pTerm ) - { - while ( pTerm->usLen && !usRc ) - { - if ( !(pTerm->lFlags & TF_NEWSENTENCE) ) - { - fOK = CheckForAlloc( pSentence, &pTermTokens, 1 ); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - pTermTokens->usOffset = pTerm->usOffs + usStart; - pTermTokens->usLength = pTerm->usLen; - pTermTokens->usHash = 0; - pTermTokens++; - } - } /* endif */ - pTerm++; - } /* endwhile */ + while ( pTerm->usLen && !usRc ) + { + if ( !(pTerm->lFlags & TF_NEWSENTENCE) ) + { + TMX_TERM_TOKEN TermToken; + + TermToken.usOffset = pTerm->usOffs + usStart; + TermToken.usLength = pTerm->usLen; + TermToken.usHash = 0; + + pSentence->pTermTokens.emplace_back(TermToken); + } /* endif */ + pTerm++; + } /* endwhile */ } /* endif */ - } /* endif */ } /* endif */ - UtlAlloc( (PVOID *) &pTermList, 0L, 0L, NOMSG ); } /* end case UNPROTECTED_CHAR */ break; default : @@ -578,28 +531,19 @@ USHORT EqfMemory::TokenizeSource else { // add tag data - usTagEntryLen = sizeof(TMX_TAGENTRY) + - (pEntry->usStop - pEntry->usStart + 1) * sizeof(CHAR_W); - if ( (pSentence->lTagAlloc - (pTagEntry - (PBYTE)pSentence->pTagRecord)) - <= (SHORT)usTagEntryLen ) - { - LONG lBytesToAlloc = get_max( ((LONG)TOK_SIZE), ((LONG)usTagEntryLen) ); + usTagEntryLen = sizeof(TMX_TAGENTRY) + (pEntry->usStop - pEntry->usStart + 1) * sizeof(CHAR_W); + + sFilled = (PBYTE)pTagEntry - (PBYTE)pSentence->pTagRecord.data(); + + if ( std::ceil((sFilled + usTagEntryLen) / static_cast(sizeof(TMX_TAGTABLE_RECORD))) // space in bytes/size of entry + <= pSentence->pTagRecord.size()) + { //remember offset of pTagEntry - usFilled = (USHORT)(pTagEntry - (PBYTE)pSentence->pTagRecord); - - //allocate another 4k for pTagRecord - fOK = UtlAlloc( (PVOID *) &pSentence->pTagRecord, pSentence->lTagAlloc, - pSentence->lTagAlloc + lBytesToAlloc, NOMSG ); - if ( fOK ) - { - pSentence->lTagAlloc += lBytesToAlloc; - - //set new position of pTagEntry - pTagEntry = (PBYTE)(pSentence->pTagRecord) + usFilled; - pTagRecord = pSentence->pTagRecord; - } /* endif */ - } /* endif */ + pSentence->pTagRecord.resize(pSentence->pTagRecord.size() + usTagEntryLen); + pTagEntry = (PTMX_TAGENTRY)((PBYTE)pSentence->pTagRecord.data() + sFilled); + } + if ( !fOK ) { @@ -607,12 +551,12 @@ USHORT EqfMemory::TokenizeSource } else { - ((PTMX_TAGENTRY)pTagEntry)->usOffset = pEntry->usStart; - ((PTMX_TAGENTRY)pTagEntry)->usTagLen = + pTagEntry->usOffset = pEntry->usStart; + pTagEntry->usTagLen = (pEntry->usStop - pEntry->usStart + 1); - memcpy( &(((PTMX_TAGENTRY)pTagEntry)->bData), + memcpy( &(pTagEntry->bData), pStringToTokenize + pEntry->usStart, - ((PTMX_TAGENTRY)pTagEntry)->usTagLen * sizeof(CHAR_W)); + pTagEntry->usTagLen * sizeof(CHAR_W)); pTagEntry += usTagEntryLen; } /* endif */ } /* endif */ @@ -627,28 +571,29 @@ USHORT EqfMemory::TokenizeSource /* check if we filled at least one term token -- if not */ /* use a dummy token - just to get an exact match ... */ /********************************************************/ - if ( pTermTokens == pSentence->pTermTokens ) + if (pSentence->pTermTokens.empty()) //pTermTokens == pSentence->pTermTokens.data() ) { - pTermTokens->usOffset = 0; - pTermTokens->usLength = (USHORT)UTF16strlenCHAR( pStringToTokenize ); - pTermTokens->usHash = 0; - pTermTokens++; + TMX_TERM_TOKEN TermToken; + TermToken.usOffset = 0; + TermToken.usLength = (USHORT)UTF16strlenCHAR( pStringToTokenize ); + TermToken.usHash = 0; + pSentence->pTermTokens.emplace_back(TermToken); + } /* endif */ - fOK = CheckForAlloc( pSentence, &pTermTokens, 3 ); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } + //fOK = CheckForAlloc( pSentence, &pTermTokens, 3 ); + //if ( !fOK ) + //{ + // LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); + //} //set total tag record length - RECLEN(pTagRecord) = pTagEntry - (PBYTE)pTagRecord; + RECLEN(pTagRecord) = (PBYTE)pTagEntry - (PBYTE)pTagRecord; } /* endif */ } /* endif */ //release allocated memory if ( pStartStop ) UtlAlloc( (PVOID *) &pStartStop, 0L, 0L, NOMSG ); - if ( pTokenList ) UtlAlloc( (PVOID *) &pTokenList, 0L, 0L, NOMSG ); //free tag table / decrement use count if ( pTable != NULL ) @@ -663,47 +608,26 @@ USHORT EqfMemory::TokenizeSource USHORT NTMTokenizeW ( PSZ_W pszInData, // IN : ptr to data being tokenized - PULONG pulTermListSize, // IN/OUT: address of variable - // containing size of term list buffer - PVOID *ppTermList, // IN/OUT: address of term list pointer - USHORT usListType // IN: type of term list MORPH_ZTERMLIST, - // MORPH_OFFSLIST, MORPH_FLAG_OFFSLIST, - // or MORPH_FLAG_ZTERMLIST + std::vector& ppTermList // IN/OUT: address of term list pointer ); USHORT NTMMorphTokenizeW ( SHORT sLanguageID, // language ID PSZ_W pszInData, // pointer to input segment - PULONG pulBufferSize, // address of variable containing size of - // term list buffer - PVOID *ppTermList, // address of caller's term list pointer - USHORT usListType // type of term list MORPH_ZTERMLIST or - // MORPH_OFFSLIST + std::vector& pTermList // address of caller's term list pointer ) { USHORT usRC = MORPH_OK; // function return code - ULONG ulTermBufUsed = 0; // number of bytes used in term buffer /********************************************************************/ /* Check input data */ /********************************************************************/ - if ( (pszInData == NULL) || - (pulBufferSize == NULL) || - (ppTermList == NULL) || - ((*ppTermList == NULL) && (*pulBufferSize != 0) ) ) + if ( (pszInData == NULL) ) { LOG_AND_SET_RC(usRC, T5INFO, MORPH_INV_PARMS); } /* endif */ -///********************************************************************/ -///* Get language control block pointer -- not needed */ -///********************************************************************/ -//if ( usRC == MORPH_OK ) -//{ -// usRC = MorphGetLCB( sLanguageID, &pLCB ); -//} /* endif */ - /********************************************************************/ /* call language exit to tokenize the input data */ @@ -713,21 +637,17 @@ USHORT NTMMorphTokenizeW if ( *pszInData != EOS ) { usRC = NTMTokenizeW(pszInData, - pulBufferSize, - ppTermList, - usListType - ); + pTermList ); } else { - usRC = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulBufferSize, - &ulTermBufUsed, - (PSZ_W)EMPTY_STRING, - 0, - 0, - 0L, - usListType ); + + usRC = MorphAddTermToList2W( pTermList, + (PSZ_W)EMPTY_STRING, //term + 0, // Length + 0, //Offset + 0L //Flags + ); } /* endif */ } /* endif */ @@ -738,43 +658,28 @@ USHORT NTMMorphTokenizeW USHORT NTMTokenizeW ( PSZ_W pszInData, // IN : ptr to data being tokenized - PULONG pulTermListSize, // IN/OUT: address of variable - // containing size of term list buffer - PVOID *ppTermList, // IN/OUT: address of term list pointer - USHORT usListType // IN: type of term list MORPH_ZTERMLIST, - // MORPH_OFFSLIST, MORPH_FLAG_OFFSLIST, - // or MORPH_FLAG_ZTERMLIST + std::vector& pTermList // IN/OUT: address of term list pointer ) { USHORT usReturn = 0; // returncode ULONG ulTermBufUsed = 0; // amount of space used in term buffer PSZ_W pTerm; // pointer to current token string PSZ_W pszCurPos = pszInData; // current position in input data - BOOL fOffsList; // TRUE = return a offset/length list LONG lFlags; // flags for current term/token BOOL fAllCaps, fAlNum, fNumber;// character classification flags BOOL fSkip, fNewSentence, // token processing flags fSingleToken, fEndOfToken; CHAR_W c, d; - fOffsList = (usListType == MORPH_OFFSLIST) || - (usListType == MORPH_FLAG_OFFSLIST); /********************************************************************/ /* Always assume start of a new sentence */ /********************************************************************/ - if ( (usListType == MORPH_FLAG_ZTERMLIST) || - (usListType == MORPH_FLAG_OFFSLIST) ) - { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, - L" ", - 1, - 0, // no offset possible - TF_NEWSENTENCE, - usListType ); - } /* endif */ + usReturn = MorphAddTermToList2W( pTermList, + L" ", + 1, + 0, // no offset possible + TF_NEWSENTENCE); /********************************************************************/ /* Initialize processing flags */ @@ -946,14 +851,13 @@ USHORT NTMTokenizeW if ( fAllCaps ) lFlags |= TF_ALLCAPS; if ( !fAlNum ) lFlags |= TF_NOLOOKUP; if ( fNumber ) lFlags |= TF_NUMBER; - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, - pTerm, + + usReturn = MorphAddTermToList2W( pTermList, + pTerm, (USHORT)(pszCurPos - pTerm), - (USHORT)(fOffsList ? (pTerm-pszInData) : 0), - lFlags, - usListType ); + pTerm-pszInData, + lFlags ); + pTerm = pszCurPos; fAllCaps = TRUE; fAlNum = TRUE; @@ -965,14 +869,11 @@ USHORT NTMTokenizeW /******************************************************************/ if ( fSingleToken && !usReturn ) { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, + usReturn = MorphAddTermToList2W( pTermList, pszCurPos, 1, - (USHORT)(fOffsList ? (pszCurPos-pszInData) : 0), - TF_NOLOOKUP, - usListType ); + pszCurPos-pszInData, + TF_NOLOOKUP ); pTerm = pszCurPos + 1; } /* endif */ @@ -981,18 +882,11 @@ USHORT NTMTokenizeW /******************************************************************/ if ( fNewSentence && !usReturn ) { - if ( (usListType == MORPH_FLAG_ZTERMLIST) || - (usListType == MORPH_FLAG_OFFSLIST) ) - { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, + usReturn = MorphAddTermToList2W( pTermList, L" ", 1, 0, // no offset possible - TF_NEWSENTENCE, - usListType ); - } /* endif */ + TF_NEWSENTENCE); } /* endif */ @@ -1022,14 +916,11 @@ USHORT NTMTokenizeW if ( fAllCaps ) lFlags |= TF_ALLCAPS; if ( !fAlNum ) lFlags |= TF_NOLOOKUP; if ( fNumber ) lFlags |= TF_NUMBER; - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, + usReturn = MorphAddTermToList2W( pTermList, pTerm, (USHORT)(pszCurPos - pTerm), - (USHORT)(fOffsList ? (pTerm-pszInData) : 0), - lFlags, - usListType ); + pTerm - pszInData, + lFlags); } /* endif */ /*****************************************************************/ @@ -1037,14 +928,11 @@ USHORT NTMTokenizeW /*****************************************************************/ if ( !usReturn ) { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, - pulTermListSize, - &ulTermBufUsed, + usReturn = MorphAddTermToList2W( pTermList, (PSZ_W)EMPTY_STRING, 0, 0, - 0L, - usListType ); + 0L ); } /* endif */ return (usReturn); diff --git a/source/opentm2/core/morph/MorphFactory.cpp b/source/opentm2/core/morph/MorphFactory.cpp index 55ba0517..c36836cb 100755 --- a/source/opentm2/core/morph/MorphFactory.cpp +++ b/source/opentm2/core/morph/MorphFactory.cpp @@ -156,6 +156,7 @@ int MorphFactory::getLanguageList( return pOtmMorphPlugin->getLanguageList(vLanguageList); } } + return 0; } /* \brief check if the given language is supported by any of the available Morph plugins diff --git a/source/opentm2/core/pluginmanager/PluginManagerImpl.h b/source/opentm2/core/pluginmanager/PluginManagerImpl.h index e26e8eea..2b4e2625 100755 --- a/source/opentm2/core/pluginmanager/PluginManagerImpl.h +++ b/source/opentm2/core/pluginmanager/PluginManagerImpl.h @@ -19,10 +19,6 @@ class PluginListener; */ struct LoadedPluginDLL { - #ifdef TO_BE_REMOVED - HMODULE hMod; // DLL module handle - #endif - char strDll[MAX_PATH]; // Add for P402792, dll path std::vector vPluginList; // list of plugins contained in DLL }; @@ -134,7 +130,7 @@ int m_iNextCommandID; PluginManagerImpl(const PluginManager&) {}; /*! \brief Assignment-operator is private */ -PluginManagerImpl& operator=(const PluginManagerImpl&) {}; +PluginManagerImpl& operator=(const PluginManagerImpl&) { return *this; }; /*! \brief A set containing all registered plugins. */ PLUGINSET* pluginSet; diff --git a/source/opentm2/core/tagtable/EQFTOKEN.cpp b/source/opentm2/core/tagtable/EQFTOKEN.cpp index 8b432e8c..f262f4b0 100755 --- a/source/opentm2/core/tagtable/EQFTOKEN.cpp +++ b/source/opentm2/core/tagtable/EQFTOKEN.cpp @@ -123,7 +123,7 @@ USHORT TAProtectedPartsInQuotedText int iAttrStartOffset, PSTARTSTOP * pStartStop, PSTARTSTOP * pCurrent, - PTOKENENTRY pAttrTokBuffer, + std::vector& pAttrTokBuffer, PLOADEDTABLE pTagTable, PULONG pulTableAlloc, PULONG pulTableUsed, @@ -437,13 +437,13 @@ BOOL TATagTokenizeW BOOL fComplete, // TRUE = no more buffers to follow PWCHAR *ppRest, // pointer to not processed data in buffer PUSHORT pusColPos, // column position - PTOKENENTRY pTokBuf, // buffer for created tokens - USHORT usTokens // max entries in token buffer + std::vector& pTokBuf//, // buffer for created tokens + //USHORT usTokens // max entries in token buffer ) { PWCHAR pszData; // pointer for input data processing PWCHAR pszDataStart; // pointer for end of data block - PTOKENENTRY pToken; // pointer to current token + TOKENENTRY Token; // pointer to current token PWCHAR pszEnd; // pointer to end of data BOOL fFound = FALSE; // flag used in setting pRest PLOADEDTABLE pTable = (PLOADEDTABLE)pVoidTable; // pointer to loaded tag table @@ -451,7 +451,8 @@ BOOL TATagTokenizeW TOKENENTRY NewToken; // buffer for new tokens MSSTACK MSStack[10]; PTAG pTag; // pointer to first tag in tagtable - PTOKENENTRY pLastTag = NULL; // pointer to last tag token + //PTOKENENTRY pLastTag = NULL; // pointer to last tag token + int iLastTagPosInBuf = -1; USHORT usTagColPos = 0; // column position of last tag USHORT usCurColPos; // current column position BOOL fProcessingAttributes; // TRUE = attributes are processed @@ -473,7 +474,7 @@ BOOL TATagTokenizeW } else { - pTagAddInfo = NULL; + pTagAddInfo = NULL; } /* endif */ @@ -516,13 +517,14 @@ BOOL TATagTokenizeW pTag = (PTAG)((PBYTE)pTable->pTagTable + pTable->pTagTable->stFixTag.uOffset ); pszData = pszInput; // start at begin of input buffer - pToken = pTokBuf; // start at begin of token buffer - pToken->sTokenid = TEXT_TOKEN; // assume text data - pToken->pDataStringW = pszData; // remember start of data - usTokens--; // reserve space for end of list token + Token.reset();// = pTokBuf.data(); // start at begin of token buffer + + Token.sTokenid = TEXT_TOKEN; // assume text data + Token.pDataStringW = pszData; // remember start of data + //usTokens--; // reserve space for end of list token fProcessingAttributes = FALSE; // we are waiting for tags ... - memset(&NewToken, 0, sizeof(NewToken)); - while( fOK && *pszData && usTokens )// while not end of data and free tokens + NewToken.reset(); + while( fOK && *pszData)// && usTokens )// while not end of data and free tokens { USHORT usProcessFlag; // processing flag /* @KWT0010A */ @@ -546,20 +548,22 @@ BOOL TATagTokenizeW /*************************************************************/ if ( pszDataStart != NewToken.pDataStringW ) { - pToken->sTokenid = TEXT_TOKEN; // text data - pToken->pDataStringW = pszDataStart; // set start of data - pToken->iLength = (USHORT)(NewToken.pDataStringW - pszDataStart); - pToken->sAddInfo = 0; - pToken->ClassId = CLS_DEFAULT; - pToken->usOrgId = 0; - usTokens--; - pToken++; + Token.sTokenid = TEXT_TOKEN; // text data + Token.pDataStringW = pszDataStart; // set start of data + Token.iLength = (USHORT)(NewToken.pDataStringW - pszDataStart); + Token.sAddInfo = 0; + Token.ClassId = CLS_DEFAULT; + Token.usOrgId = 0; + //usTokens--; + pTokBuf.push_back(Token); + Token.reset(); + //Token++; } /* endif */ /*************************************************************/ /* Add tag token */ /*************************************************************/ - if ( usTokens ) + //if ( usTokens ) { // special handling for tags which may be followed by attributes: // check if tag end character is a blank, when it is one look-ahead @@ -599,17 +603,20 @@ BOOL TATagTokenizeW } /* endif */ // add tag token to token list - pLastTag = pToken; + iLastTagPosInBuf = pTokBuf.size();//pToken; usTagColPos = TokStatus.usStartColPos; - memcpy( pToken, &NewToken, sizeof(TOKENENTRY) ); - pToken->sAddInfo = pTag[NewToken.sTokenid].BitFlags.AddInfo; + memcpy( &Token, &NewToken, sizeof(TOKENENTRY) ); + Token.sAddInfo = pTag[NewToken.sTokenid].BitFlags.AddInfo; if ( pTagAddInfo ) { - pToken->ClassId = pTagAddInfo[NewToken.sTokenid].ClassId; - pToken->usOrgId = pTagAddInfo[NewToken.sTokenid].usFixedTagId; + Token.ClassId = pTagAddInfo[NewToken.sTokenid].ClassId; + Token.usOrgId = pTagAddInfo[NewToken.sTokenid].usFixedTagId; } /* endif */ - pToken++; - usTokens--; + + //pToken++; + pTokBuf.push_back(Token); + Token.reset(); + //usTokens--; } /* endif */ /**********************************************************/ @@ -664,20 +671,23 @@ BOOL TATagTokenizeW // } /* endif */ } /* endif */ - if ( fFound && usTokens ) + if ( fFound)// && usTokens ) { - memcpy( pToken, &NewToken, sizeof(TOKENENTRY) ); - pToken->sAddInfo = pAttr[NewToken.sTokenid-sTags].BitFlags.AddInfo; + memcpy( &Token, &NewToken, sizeof(TOKENENTRY) ); + Token.sAddInfo = pAttr[NewToken.sTokenid-sTags].BitFlags.AddInfo; if ( pTagAddInfo ) { - pToken->ClassId = pTagAddInfo[NewToken.sTokenid].ClassId; - pToken->usOrgId = pTagAddInfo[NewToken.sTokenid].usFixedTagId; + Token.ClassId = pTagAddInfo[NewToken.sTokenid].ClassId; + Token.usOrgId = pTagAddInfo[NewToken.sTokenid].usFixedTagId; } /* endif */ - pToken++; - usTokens--; + //pToken++; + + pTokBuf.push_back(Token); + Token.reset(); + //usTokens--; } /* endif */ - } while ( fFound && usTokens ); /* enddo */ + } while ( fFound ); //&& usTokens ); /* enddo */ } /* endif */ } else @@ -690,12 +700,15 @@ BOOL TATagTokenizeW /***********************************************************/ /* Add remaining data as TEXT token */ /***********************************************************/ - pToken->sTokenid = TEXT_TOKEN; // text data - pToken->pDataStringW = pszDataStart; // set start of data - pToken->iLength = (USHORT)UTF16strlenCHAR(pszDataStart); - pszData = pszDataStart + pToken->iLength; - usTokens--; - pToken++; + Token.sTokenid = TEXT_TOKEN; // text data + Token.pDataStringW = pszDataStart; // set start of data + Token.iLength = (USHORT)UTF16strlenCHAR(pszDataStart); + pszData = pszDataStart + Token.iLength; + + pTokBuf.push_back(Token); + Token.reset(); + //usTokens--; + //pToken++; } else { @@ -715,11 +728,13 @@ BOOL TATagTokenizeW // nothing was processed due to errors (memory allocation?) *ppRest = pszInput; } - else if ( !usTokens || // end of token table reached + else if ( //!usTokens || // end of token table reached !fComplete ) // or not last block of data ??? { + + TOKENENTRY* pLastToken = pTokBuf.size()>0? & pTokBuf[pTokBuf.size()-1] : nullptr; // at least two tokens in buffer .... - if ( (pToken != pTokBuf) && ( pToken != (pTokBuf+1)) ) // if tokens in buffer + if ( pTokBuf.size()>2)//(pToken != pTokBuf.data()) && ( pToken != (pTokBuf.data()+1)) ) // if tokens in buffer { /**************************************************************/ /* There are tokens in the buffer, let's see how we have to */ @@ -730,19 +745,20 @@ BOOL TATagTokenizeW // processing in the next tokenization block (unless this tag // is the first token in the token buffer or the remaining data // is longer than twice the max segment size ) - int iRemainLength = UTF16strlenCHAR(pToken[-1].pDataStringW + pToken[-1].iLength); - if ( ( (pLastTag != pTokBuf) && // more than one tag in buffer - (iRemainLength < (2*MAX_SEGMENT_SIZE)) ) || // rest is smaller than 2 segments - (usTokens == 0) ) // token buffer overflow + int iRemainLength = UTF16strlenCHAR(pLastToken->pDataStringW + pLastToken->iLength); + if ( ( (iLastTagPosInBuf != pTokBuf.size()-1) && // more than one tag in buffer + (iRemainLength < (2*MAX_SEGMENT_SIZE)) ) //|| // rest is smaller than 2 segments + //(usTokens == 0) + ) // token buffer overflow { - pToken = pLastTag; + Token = pTokBuf[iLastTagPosInBuf]; TokStatus.usColPos = usTagColPos; - *ppRest = pToken->pDataStringW; + *ppRest = Token.pDataStringW; } - else + else if(pLastToken) { // continue right behind the last recognized token - *ppRest = pToken[-1].pDataStringW + pToken[-1].iLength; + *ppRest = pLastToken->pDataStringW + pLastToken->iLength; if ( (*pszData == EOS) && (*ppRest >= pszData) ) { // no more data to follow, segment has been processed completely @@ -761,7 +777,7 @@ BOOL TATagTokenizeW PSZ_W pszAreaStart = NULL; pszEnd = pszData; pszData--; - if ( pToken == pTokBuf ) + if ( pTokBuf.empty() ) { // no token in buffer pszAreaStart = pszInput; // set start of data @@ -769,7 +785,7 @@ BOOL TATagTokenizeW else { // one token in buffer - pszAreaStart = pToken[-1].pDataStringW + pToken[-1].iLength; + pszAreaStart = pLastToken->pDataStringW + pLastToken->iLength; } /* endif */ while ( pszData > pszAreaStart ) @@ -792,12 +808,14 @@ BOOL TATagTokenizeW { *ppRest = pszData; // set pRest to last linefeed } /* endif */ - pToken->sTokenid = TEXT_TOKEN; // text data - pToken->pDataStringW = pszAreaStart; // set start of data - pToken->iLength = (USHORT)(*ppRest - pszAreaStart); + Token.sTokenid = TEXT_TOKEN; // text data + Token.pDataStringW = pszAreaStart; // set start of data + Token.iLength = (USHORT)(*ppRest - pszAreaStart); - usTokens--; - pToken++; + //usTokens--; + //pToken++; + pTokBuf.push_back(Token); + Token.reset(); } /* endif */ } else @@ -808,13 +826,24 @@ BOOL TATagTokenizeW // add end-of-list token if ( fOK ) { - memset( pToken, 0, sizeof(TOKENENTRY) ); - pToken->iLength = 0; - pToken->sTokenid = ENDOFLIST; + Token.reset(); + Token.iLength = 0; + Token.sTokenid = ENDOFLIST; + pTokBuf.push_back(Token); + Token.reset(); *pusColPos = TokStatus.usColPos; } /* endif */ + //#define DEBUG_PRINT_TOKENS + //#ifdef DEBUG_PRINT_TOKENS + for(auto& tok: pTokBuf){ + T5LOG(T5DEVELOP) << "pTokBuf[" << &tok - pTokBuf.data() <<"]: ClassId=" << tok.ClassId << "; iLength=" << tok.iLength << "; pDataStringW=" << (tok.pDataStringW? EncodingHelper::convertToUTF8(tok.pDataStringW): "" ) + << "; sAddInfo=" << tok.sAddInfo << "; sTokenid=" << tok.sTokenid << "; usOrgId=" << tok.usOrgId; + } + + //#endif + /*******************************************************************/ /* cleanup */ /*******************************************************************/ @@ -2197,7 +2226,7 @@ USHORT TACreateProtectTableW PSZ_W pszSegment, // ptr to text of segment being processed PVOID pVoidTable, // ptr to tag table USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens + std::vector& pTokBuffer, // buffer used temporarly for tokens USHORT usTokBufferSize, // size of token buffer in bytes PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr PFN pvUserExit, // ptr to user exit function @@ -2218,7 +2247,7 @@ USHORT TACreateProtectTableWEx PSZ_W pszSegment, // ptr to text of segment being processed PVOID pVoidTable, // ptr to tag table USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens + std::vector& pTokBuffer, // buffer used temporarly for tokens USHORT usTokBufferSize, // size of token buffer in bytes PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr PFN pvUserExit, // ptr to user exit function @@ -2249,7 +2278,7 @@ USHORT TACreateProtectTableWEx BOOL fTRNoteFound = 0; // true if TRNOTE found in curr segment BOOL fTRTag= FALSE; // true if inside of a trnote CHAR chAsciiSeg[MAX_SEGMENT_SIZE]; - PTOKENENTRY pAttrTokBuffer = NULL; // used temp.for translatable attributetokens + std::vector pAttrTokBuffer; // used temp.for translatable attributetokens LONG lAttrSize = 0L; PTOKENENTRY pNextTok; // ptr for next token in token list @@ -2260,7 +2289,7 @@ USHORT TACreateProtectTableWEx if ( (pszSegment == NULL) || (pTagTable == NULL) || - (pTokBuffer == NULL) || + //(pTokBuffer == NULL) || (usTokBufferSize == 0) || (ppStartStop == NULL) ) { @@ -2323,8 +2352,7 @@ USHORT TACreateProtectTableWEx TRUE, &pRest, &usColPos, - pTokBuffer, - (USHORT)(usTokBufferSize / sizeof(TOKENENTRY)) ); + pTokBuffer); usRC = ( pRest == NULL ) ? NO_ERROR : EQFRS_AREA_TOO_SMALL; } /* endif */ @@ -2333,7 +2361,7 @@ USHORT TACreateProtectTableWEx /********************************************************************/ if ( usRC == NO_ERROR ) { - pTok = pTokBuffer; + pTok = pTokBuffer.data(); while ( pTok->sTokenid != ENDOFLIST ) { @@ -2388,7 +2416,7 @@ USHORT TACreateProtectTableWEx ulTableUsed = 0; } /* endif */ - TATagAddWSpace(pszSegment, pTokBuffer, sNumTags); + TATagAddWSpace(pszSegment, pTokBuffer.data(), sNumTags); /********************************************************************/ /* Fill start/stop table */ @@ -2396,7 +2424,7 @@ USHORT TACreateProtectTableWEx if ( usRC == NO_ERROR ) { pCurrent = pStartStop; - pTok = pTokBuffer; + pTok = pTokBuffer.data(); while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) { /****************************************************************/ @@ -2426,14 +2454,8 @@ USHORT TACreateProtectTableWEx /**************************************************************/ while ( pTok->iLength && (usRC == NO_ERROR) ) { // P016238: check whether quoted text contains protected parts! - if (!pAttrTokBuffer) - { - lAttrSize = MAX_ATTR_TOKENS * sizeof(TOKENENTRY); - if (!UtlAlloc((PVOID *) &pAttrTokBuffer, 0L, lAttrSize, NOMSG) ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + pAttrTokBuffer.reserve(MAX_ATTR_TOKENS); + if ( usRC == NO_ERROR) { usRC = TAProtectedPartsInQuotedText( pTok, @@ -2549,7 +2571,7 @@ USHORT TACreateProtectTableWEx if ( usRC == NO_ERROR ) { pAct = pCurrent = pStartStop; - pTok = pTokBuffer; + pTok = pTokBuffer.data(); pNextTok = pTok + 1; if ( (pAct->usType == TAGPROT_CHAR) && TA_IS_TAG(pTok->sTokenid, sNumTags)) { @@ -2708,11 +2730,6 @@ USHORT TACreateProtectTableWEx } /* endif */ } /* endif */ - if ( pAttrTokBuffer ) - { - UtlAlloc( (PVOID *)&pAttrTokBuffer, 0L, 0L, NOMSG ); - } /* endif */ - /********************************************************************/ /* Return start/stop list to caller or cleanup */ /********************************************************************/ @@ -2836,7 +2853,7 @@ USHORT TAProtectedPartsInQuotedText int iAttrStartOffset, PSTARTSTOP * ppStartStop, PSTARTSTOP * ppCurrent, - PTOKENENTRY pAttrTokBuffer, + std::vector& pAttrTokBuffer, PLOADEDTABLE pTagTable, PULONG pulTableAlloc, PULONG pulTableUsed, @@ -2880,8 +2897,7 @@ USHORT TAProtectedPartsInQuotedText TRUE, &pRest, pusColPos, - pAttrTokBuffer, - (USHORT)lAttrSize ); + pAttrTokBuffer ); if(pRest == nullptr){ LOG_AND_SET_RC(usRC, T5INFO, NO_ERROR); }else{ @@ -2891,7 +2907,7 @@ USHORT TAProtectedPartsInQuotedText if ( usRC == NO_ERROR ) { - pAttrTok = pAttrTokBuffer; + pAttrTok = pAttrTokBuffer.data(); while ( (pAttrTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) { pCurrent->usStart = (USHORT)(pAttrTok->pDataStringW - pAttrStart @@ -2934,212 +2950,6 @@ USHORT TAProtectedPartsInQuotedText -USHORT TACreateProtectTable -( - PSZ pszSegment, // ptr to text of segment being processed - PVOID pVoidTable, // ptr to tag table - USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens - USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function - ULONG ulCP -) -{ - USHORT usRC = 0; - PSZ_W pszInputW = &chInputW[0]; - - ASCII2Unicode( pszSegment, pszInputW, ulCP ); - - usRC = TACreateProtectTableW( pszInputW, pVoidTable, usColPos, pTokBuffer, usTokBufferSize, - ppStartStop, pvUserExit, NULL, ulCP ); - - // attention: - // for DBCS code pages the returned offsets and lengths do not match the actual - // position in the source data and have to be re-adjusted - - return usRC; -} - -USHORT TAPrepProtectTable -( - PVOID pVoidTable, // ptr to tag table (PLOADEDTABLE) - HMODULE *phModule, // address of user exit module handle - PFN *ppfnUserExit, // address of ptr to user exit function - PFN *ppfnCheckSegExit, // address of ptr to segment check func - PFN *ppfnWYSIWYGExit, // address of ptr to show transl. func - PFN *ppfnTocGotoExit // address of ptr to TOC func -) -{ - return( TALoadEditUserExit( pVoidTable, "", phModule, ppfnUserExit, ppfnCheckSegExit, - ppfnWYSIWYGExit, ppfnTocGotoExit, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL ) ); - -} /* end of function TAPrepProtectTable */ - -USHORT TALoadEditUserExit -( - PVOID pVoidTable, // ptr to tag table (PLOADEDTABLE) - PSZ pszTableName, // name of tag table (w/o path and ext.) - HMODULE *phModule, // address of user exit module handle - PFN *ppfnUserExit, // address of ptr to user exit function - PFN *ppfnCheckSegExit, // address of ptr to segment check func - PFN *ppfnWYSIWYGExit, // address of ptr to show transl. func - PFN *ppfnTocGotoExit, // address of ptr to TOC func - PFN *ppfnGetSegContext, // address of ptr to EQFGETSEGCONTEXT function - PFN *ppfnUpdateContext, // address of ptr to EQFUPDATECONTEXT function - PFN *ppfnFormatContext, // address of ptr to EQFFORMATCONTEXT function - PFN *ppfnCompareContext, // address of ptr to EQFCOMPARECONTEXT function - PFN *ppfnUserExitW, // unicode user exit to create start-stop-tbl - PFN *ppfnCheckSegExitW, // unicode user exit to check segment - PFN *ppfnCheckSegExExitW,// unicode user exit to check segment Ex - PFN *ppfnCheckSegType // user exit to check segment type -) -{ - PTAGTABLE pTagTable; // pointer to tag table - USHORT usRC = NO_ERROR; // function return code - - /********************************************************************/ - /* initialize user's data fields */ - /********************************************************************/ - *phModule = NULLHANDLE; - if ( ppfnUserExit ) *ppfnUserExit = NULL; - if ( ppfnCheckSegExit ) *ppfnCheckSegExit = NULL; - if ( ppfnWYSIWYGExit ) *ppfnWYSIWYGExit = NULL; - if ( ppfnTocGotoExit ) *ppfnTocGotoExit = NULL; - if ( ppfnGetSegContext ) *ppfnGetSegContext = NULL; - if ( ppfnUpdateContext ) *ppfnUpdateContext = NULL; - if ( ppfnFormatContext ) *ppfnFormatContext = NULL; - if ( ppfnCompareContext ) *ppfnCompareContext = NULL; - if ( ppfnUserExitW ) *ppfnUserExitW = NULL; - if ( ppfnCheckSegExitW ) *ppfnCheckSegExitW = NULL; - if ( ppfnCheckSegExExitW ) *ppfnCheckSegExExitW = NULL; - if ( ppfnCheckSegType ) *ppfnCheckSegType = NULL; - - - /********************************************************************/ - /* Address tag table */ - /********************************************************************/ - pTagTable = ((PLOADEDTABLE)pVoidTable)->pTagTable; - - /********************************************************************/ - /* Load user exit if tag table has one */ - /********************************************************************/ - if ( pTagTable->szSegmentExit[0] != EOS ) - { - CHAR szExit[MAX_LONGFILESPEC]; // buffer for library name - CHAR *ptr ; - -// strcpy( szExit, pTagTable->szSegmentExit ); -// strcat( szExit, EXT_OF_DLL ); -// if ( MUGetUserExitFileName( pTagTable->szSegmentExit, NULL, szExit, sizeof(szExit) ) ) - if ( MUGetUserExitFileName( pszTableName, NULL, szExit, sizeof(szExit) ) ) - { - ptr = strrchr( szExit, '\\' ) ; - if ( ptr ) - strcpy( ++ptr, pTagTable->szSegmentExit ) ; - - usRC = DosLoadModule( NULL, 0, szExit, phModule ); - - if ( usRC == NO_ERROR ) - { - if ( ppfnUserExit ) - { - - usRC = DosGetProcAddr( *phModule, EQFPROTTABLE_EXIT, (PFN*)ppfnUserExit); - if ( usRC != NO_ERROR ) - { - *ppfnUserExit = NULL; - } /* endif */ - } - /****************************************************************/ - /* try to load SegmentCheckExit function */ - /****************************************************************/ - if ( ppfnCheckSegExit ) - { - usRC = DosGetProcAddr( *phModule, EQFCHECKSEG_EXIT, - (PFN*)ppfnCheckSegExit); - if ( usRC != NO_ERROR ) - { - *ppfnCheckSegExit = NULL; - } /* endif */ - } - /****************************************************************/ - /* try to load ShowTransl Exit function */ - /****************************************************************/ - if ( ppfnWYSIWYGExit ) - { - usRC = DosGetProcAddr( *phModule, EQFSHOWTRANS_EXIT, - (PFN*)ppfnWYSIWYGExit); - if ( usRC != NO_ERROR ) - { - *ppfnWYSIWYGExit = NULL; - } /* endif */ - } - /****************************************************************/ - /* try to load CheckSegType Exit function */ - /****************************************************************/ - if ( ppfnCheckSegType ) - { - *ppfnCheckSegType = NULL; - DosGetProcAddr( *phModule, EQFCHECKSEGTYPE_EXIT, (PFN*)ppfnCheckSegType); - } - /****************************************************************/ - /* try to load TOC user Exit function */ - /****************************************************************/ - if ( ppfnTocGotoExit ) - { - usRC = DosGetProcAddr( *phModule, EQFTOCGOTO_EXIT, - (PFN*)ppfnTocGotoExit); - if ( usRC != NO_ERROR ) - { - *ppfnTocGotoExit = NULL; - } /* endif */ - } - // try to load context related user exit functions - if ( ppfnGetSegContext ) - { - usRC = DosGetProcAddr( *phModule, EQFGETSEGCONTEXT_EXIT, (PFN*)ppfnGetSegContext ); - if ( usRC != NO_ERROR ) *ppfnGetSegContext = NULL; - } - if ( ppfnUpdateContext ) - { - usRC = DosGetProcAddr( *phModule, EQFUPDATECONTEXT_EXIT, (PFN*)ppfnUpdateContext ); - if ( usRC != NO_ERROR ) *ppfnUpdateContext = NULL; - } - if ( ppfnCompareContext ) - { - usRC = DosGetProcAddr( *phModule, EQFCOMPARECONTEXT_EXIT, (PFN*)ppfnCompareContext ); - if ( usRC != NO_ERROR ) *ppfnCompareContext = NULL; - } - if ( ppfnFormatContext ) - { - usRC = DosGetProcAddr( *phModule, EQFFORMATCONTEXT_EXIT, (PFN*)ppfnFormatContext ); - if ( usRC != NO_ERROR ) *ppfnFormatContext = NULL; - } - if ( ppfnUserExitW ) - { - usRC = DosGetProcAddr( *phModule, EQFPROTTABLEW_EXIT, (PFN*)ppfnUserExitW); - if ( usRC != NO_ERROR ) *ppfnUserExitW = NULL; - } - if ( ppfnCheckSegExitW ) - { - usRC = DosGetProcAddr( *phModule, EQFCHECKSEGW_EXIT, (PFN*)ppfnCheckSegExitW); - if ( usRC != NO_ERROR ) *ppfnCheckSegExitW = NULL; - } - if ( ppfnCheckSegExExitW ) - { - usRC = DosGetProcAddr( *phModule, EQFCHECKSEGEXW_EXIT, (PFN*)ppfnCheckSegExExitW); - if ( usRC != NO_ERROR ) *ppfnCheckSegExExitW = NULL; - } - - } /* endif */ - } - } /* endif */ - - return( NO_ERROR ); -} /* end of function TAPrepProtectTable */ - USHORT TAEndProtectTable ( diff --git a/source/opentm2/core/tagtable/MarkupPluginMapper.cpp b/source/opentm2/core/tagtable/MarkupPluginMapper.cpp index 418c7258..7d796e4b 100755 --- a/source/opentm2/core/tagtable/MarkupPluginMapper.cpp +++ b/source/opentm2/core/tagtable/MarkupPluginMapper.cpp @@ -339,50 +339,6 @@ BOOL MUGetMarkupTablePlugin( char *pszMarkup, char *pszBuffer, int iBufSize ) } -// update the files of a markup table - -BOOL MUUpdateMarkupTableFiles( - char *pszMarkupTableName, - char *pszPluginName, - char *pszShortDescription, - char *pszLongDescription, - char *pszMarkupTableVersion, - char *pszTableFileName, - char *pszUserExitFilename, - char *pszFileList -) -{ - static char szBuffer[MAX_LONGFILESPEC]; // buffer for markup names - BOOL bReturn = false ; - - if ( ( pszPluginName ) && - ( pszMarkupTableName ) ) { - - // loop through all markup table plugins - for( int i = 0 ; i < (int)pluginList->size() ; i++ ) { - OtmMarkupPlugin* curPlugin = (*pluginList)[i]; - if ( ( ! strcmp( curPlugin->getName(), pszPluginName ) ) || - ( ! strcmp( curPlugin->getShortName(), pszPluginName ) ) ) { - int iReturn = curPlugin->updateFiles( pszMarkupTableName, - pszShortDescription, pszLongDescription, - pszMarkupTableVersion, pszTableFileName, - pszUserExitFilename, pszFileList ) ; - if ( iReturn != UPDATE_MARKUP_ERROR ) { - bReturn = true; - // OtmMarkup *markup = curPlugin->getMarkup( pszMarkupTableName ); - // if ( markup ) { - // markup->updateInfo( pszMarkupTableName, - // pszShortDescription, pszLongDescription, - // NULL, pszUserExitFilename ) ; - // } - } - } - } - } - - return( bReturn ); -} - // delete a markup table diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index 11db3fb1..1347ce04 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -395,8 +395,8 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem ulKey = LONGNAME_KEY; // write long document name buffer area to the database _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)NewMem->pLongNames->pszBuffer, - NewMem->pLongNames->ulBufUsed ); + (PBYTE)NewMem->LongNames.pszBuffer.data(), + NewMem->LongNames.pszBuffer.size() ); } /* endif */ diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index f50c65b3..6101f272 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -1257,7 +1257,7 @@ SHORT BTREE::QDAMReadRecord_V3 USHORT i; SHORT sRc = 0; // return code BOOL fMemOK = FALSE; - PLOOKUPENTRY_V3 pLEntry; + PLOOKUPENTRY_V3 pLEntry = nullptr; //PACCESSCTRTABLEENTRY pACTEntry; DEBUGEVENT2( QDAMREADRECORD_LOC, FUNCENTRY_EVENT, usNumber, DB_GROUP, "" ); @@ -2359,9 +2359,9 @@ RECPARAM QDAMGetrecData_V3 pData += sizeof(USHORT ); // get pointer to datarec memcpy( &recData, (PRECPARAM) pData, sizeof( RECPARAM )); - memcpy( &recData.usNum, (PUSHORT) pData, sizeof( USHORT ) ); - memcpy( &recData.usOffset,(PUSHORT) (pData+2), sizeof( USHORT ) ); - memcpy( &recData.ulLen, (PULONG) (pData+8), sizeof( PULONG ) ); + //memcpy( &recData.usNum, (PUSHORT) pData, sizeof( USHORT ) ); + //memcpy( &recData.usOffset,(PUSHORT) (pData+2), sizeof( USHORT ) ); + //memcpy( &recData.ulLen, (PULONG) (pData+8), sizeof( PULONG ) ); return ( recData ); } @@ -4061,10 +4061,9 @@ SHORT QDAMSplitNode_V3 //+----------------------------------------------------------------------------+ SHORT QDAMUnTerseData ( - PBTREE pBTIda, // - PUCHAR pData, // pointer to data - ULONG ulDataLen, // data length (uncompressed) - PULONG pulLen // length of the string + PBTREE pBTIda, // + std::vector& pData, // pointer to data + ULONG ulDataLen // data length (uncompressed) ) { ULONG ulLen ; // length of string @@ -4073,14 +4072,17 @@ SHORT QDAMUnTerseData USHORT usStartBit = 0; // start bit looking at from high to low USHORT usDecode; // value to be decoded SHORT sRc = 0; // okay; - PSZ pInData = (PSZ)pData; // pointer to input data + PSZ pInData = (PSZ)pData.data(); // pointer to input data + PSZ ppData = (PSZ)pData.data(); PBTREE pBT = pBTIda; + long lLen = 0; #if defined(MEASURE) ulBeg = pGlobInfoSeg->msecs; #endif // get length of passed string ulLen = ulDataLen; + T5LOG(T5TRANSACTION) << "pData.size = " << pData.size() << "; ulDataLen = " << ulDataLen; // try to uncompress the passed string ( use pTempData as temporary space) @@ -4095,7 +4097,7 @@ SHORT QDAMUnTerseData /***************************************************************/ if ( usStartBit < 8 ) { - usCurByte = (usCurByte << 8) + *pData++; + usCurByte = (usCurByte << 8) + *ppData++; usStartBit += 8; } /* endif */ @@ -4113,7 +4115,7 @@ SHORT QDAMUnTerseData case 0: // get next character if ( usStartBit < 8 ) { - usCurByte = (usCurByte << 8) + *pData++; + usCurByte = (usCurByte << 8) + *ppData++; usStartBit += 8; } /* endif */ *pTempData++ = @@ -4122,7 +4124,7 @@ SHORT QDAMUnTerseData usStartBit -= 8; if ( usStartBit < 8 ) { - usCurByte = (usCurByte << 8) + *pData++; + usCurByte = (usCurByte << 8) + *ppData++; usStartBit += 8; } /* endif */ break; @@ -4162,16 +4164,9 @@ SHORT QDAMUnTerseData /*******************************************************************/ if ( !sRc ) { - if ( *pulLen >= ulDataLen ) - { - memcpy( pInData, &pBT->TempRecord[0], *pulLen ); - *pulLen = ulDataLen; - } - else - { - T5LOG(T5ERROR) << ":: BufferTooSmall 1"; - sRc = BTREE_BUFFER_SMALL; - } /* endif */ + // Replace old data with new data + pData.clear(); // Remove old data + pData.insert(pData.begin(), &pBT->TempRecord[0], &pBT->TempRecord[0] + lLen); // Insert new data at the same position } /* endif */ #if defined(MEASURE) ulUnTerseEnd += (pGlobInfoSeg->msecs - ulBeg); @@ -4220,8 +4215,7 @@ SHORT QDAMGetszData_V3 ( PBTREE pBTIda, RECPARAM recDataParam, - PBYTE pData, - PULONG pulDataLen, + std::vector& pData, CHAR chType // type of record key/data ) { @@ -4229,7 +4223,6 @@ SHORT QDAMGetszData_V3 PBTREEBUFFER_V3 pRecord; // pointer to record PBTREEHEADER pHeader; // pointer to header PCHAR pTempData = NULL; // pointer to data pointer - PCHAR pStartData = (PCHAR)pData; // pointer to data pointer LONG lLen = 0; // length of string LONG lTerseLen = 0; // length of tersed string BOOL fTerse = FALSE; // entry tersed?? @@ -4282,14 +4275,6 @@ SHORT QDAMGetszData_V3 sRc = BTREE_CORRUPTED; } pTempData += sizeof(ULONG ); // get pointer to data -#ifdef _DEBUG - { - CHAR szTemp[40]; - sprintf( szTemp, "%ld", lLen ); - INFOEVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, recDataParam.usNum, - DB_GROUP, szTemp ); - } -#endif pHeader = &(pRecord->contents.header); } /* endif */ @@ -4304,109 +4289,83 @@ SHORT QDAMGetszData_V3 } /* endif */ } /* endif */ - if ( !sRc ) - { - if ( *pulDataLen == 0 || !pData ) - { - // give back only length - if ( fTerse ) - { - // first field contains real length - *pulDataLen = LENGTHOFDATA(pBT,pTempData); - } - else - { - *pulDataLen = lLen; // give back only length - } /* endif */ - } - else if ( *pulDataLen >= lLen ) - { - lFitLen = BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER) - *pusOffset; - lFitLen = lLen < (lFitLen - usLenFieldSize) ? lLen : (lFitLen - usLenFieldSize) ; - - if ( fTerse ) - { - memcpy(pData,pTempData+usLenFieldSize,lFitLen-usLenFieldSize); - lTerseLen = LENGTHOFDATA(pBT,pTempData); - /**********************************************************/ - /* adjust pointers */ - /**********************************************************/ - lLen -= lFitLen; - pData += (lFitLen - usLenFieldSize); - } - else - { - memcpy( pData, pTempData, lFitLen ); - *pulDataLen = lLen; - /**********************************************************/ - /* adjust pointers */ - /**********************************************************/ - lLen -= lFitLen; - pData += lFitLen; - } /* endif */ + if ( !sRc ) + { + lFitLen = BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER) - *pusOffset; + lFitLen = lLen < (lFitLen - usLenFieldSize) ? lLen : (lFitLen - usLenFieldSize) ; + if (pTempData && lFitLen > 0) { + if ( fTerse ) + { + long bytesToCopy = lFitLen - usLenFieldSize; + pData.insert(pData.end(), pTempData+usLenFieldSize, pTempData + bytesToCopy); + lTerseLen = LENGTHOFDATA(pBT,pTempData); /**********************************************************/ - /* copy as long as still data are available */ + /* adjust pointers */ /**********************************************************/ - while ( !sRc && lLen>0 ) - { - usNum = NEXT( pRecord ); - BTREEUNLOCKRECORD( pRecord ); - fRecLocked = FALSE; - if ( usNum ) - { - sRc = pBTIda->QDAMReadRecord_V3( usNum, &pRecord, FALSE ); - if ( !sRc && TYPE( pRecord ) != DATA_NEXTNODE ) - { - sRc = BTREE_CORRUPTED; - ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 3, DB_GROUP, "" ); - } /* endif */ - if ( !sRc ) - { - BTREELOCKRECORD( pRecord ); - fRecLocked = TRUE; - pusOffset = (PUSHORT) pRecord->contents.uchData; - pTempData = (PCHAR)(pRecord->contents.uchData + *pusOffset); - - lFitLen = LENGTHOFDATA( pBT, pTempData ); - lFitLen = lLen < lFitLen? lLen: lFitLen ; - pTempData += usLenFieldSize; // get pointer to data - - memcpy( pData, pTempData, lFitLen ); - lLen -= lFitLen; - pData += lFitLen; - } /* endif */ - } - else - { - sRc = BTREE_CORRUPTED; - ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 4, DB_GROUP, "" ); - } /* endif */ - } /* endwhile */ - if ( !sRc && fTerse ) + lLen -= lFitLen; + } + else + { + pData.insert(pData.end(), pTempData, pTempData + lFitLen); + lLen -= lFitLen; + } /* endif */ + } + + + /**********************************************************/ + /* copy as long as still data are available */ + /**********************************************************/ + while ( !sRc && lLen>0 ) + { + usNum = NEXT( pRecord ); + BTREEUNLOCKRECORD( pRecord ); + fRecLocked = FALSE; + if ( usNum ) { - switch ( pBT->fTerse ) + sRc = pBTIda->QDAMReadRecord_V3( usNum, &pRecord, FALSE ); + if ( !sRc && TYPE( pRecord ) != DATA_NEXTNODE ) { - case BTREE_TERSE_HUFFMAN : - { - sRc = QDAMUnTerseData( pBTIda, (PUCHAR)pStartData, lTerseLen, - pulDataLen ); - } - break; - default : - sRc = BTREE_CORRUPTED; - ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 5, DB_GROUP, "" ); - break; - } /* endswitch */ + sRc = BTREE_CORRUPTED; + ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 3, DB_GROUP, "" ); + } /* endif */ + if ( !sRc ) + { + BTREELOCKRECORD( pRecord ); + fRecLocked = TRUE; + pusOffset = (PUSHORT) pRecord->contents.uchData; + pTempData = (PCHAR)(pRecord->contents.uchData + *pusOffset); + + lFitLen = LENGTHOFDATA( pBT, pTempData ); + lFitLen = lLen < lFitLen? lLen: lFitLen ; + pTempData += usLenFieldSize; // get pointer to data + + pData.insert(pData.end(), pTempData, pTempData + lFitLen); + lLen -= lFitLen; + } /* endif */ + } + else + { + sRc = BTREE_CORRUPTED; + ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 4, DB_GROUP, "" ); } /* endif */ - } - else - { - T5LOG(T5ERROR) << ":: BufferTooSmall 3"; - sRc = BTREE_BUFFER_SMALL; - } /* endif */ - + } /* endwhile */ + if ( !sRc && fTerse ) + { + switch ( pBT->fTerse ) + { + case BTREE_TERSE_HUFFMAN : + { + sRc = QDAMUnTerseData( pBTIda, pData, lTerseLen ); + } + break; + default : + sRc = BTREE_CORRUPTED; + ERREVENT2( QDAMGETSZDATA_LOC, STATE_EVENT, 5, DB_GROUP, "" ); + break; + } /* endswitch */ + } /* endif */ } /* endif */ if ( fRecLocked ) { @@ -5800,119 +5759,6 @@ BTREE::QDAMCheckDict // endif // return Rc //------------------------------------------------------------------------------ -SHORT QDAMLocSubstr_V3 -( - PBTREE pBTIda, - PBTREEBUFFER_V3 pRecord, - PCHAR_W pKey, - PBYTE pchBuffer, // space for key data - PULONG pulLength, // in/out length of returned key data - PBYTE pchUserData, // space for user data - PULONG pulUserLen // in/out length of returned user data -) -{ - SHORT i; // index - SHORT sNearKey; // nearest key - USHORT usLen; // length of key - SHORT sRc = 0; // return code - PCHAR_W pKey2; // key to be compared with - RECPARAM recData; - PBTREE pBT = pBTIda; - - sRc = pBTIda->QDAMLocateKey_V3( pRecord, pKey, &i, FSUBSTR, &sNearKey); - if ( !sRc ) - { - if ( i != -1 ) - { - sNearKey = i; - } /* endif */ - - // set new current position - pBTIda->sCurrentIndex = sNearKey; - pBTIda->usCurrentRecord = RECORDNUM( pRecord ); - - BTREELOCKRECORD( pRecord ); - // check if the key fulfills the substring option - if ( sNearKey != -1) - { - pKey2 = QDAMGetszKey_V3( pRecord, sNearKey, pBT->usVersion ); - if ( pKey2 ) - { - if ( UTF16strnicmp(pKey, pKey2, (USHORT)UTF16strlenCHAR(pKey)) ) - { - if ( sNearKey < (SHORT)OCCUPIED(pRecord)-1 ) - { - sNearKey++; - } /* endif */ - - pKey2 = QDAMGetszKey_V3( pRecord, sNearKey, pBT->usVersion ); - if ( pKey2 ) - { - if ( UTF16strnicmp(pKey, pKey2, (USHORT)UTF16strlenCHAR(pKey)) ) - { - SET_AND_LOG(sRc, BTREE_NOT_FOUND); - } - else - { - pBTIda->sCurrentIndex = sNearKey; - } /* endif */ - } - else - { - SET_AND_LOG(sRc, BTREE_NOT_FOUND); - } /* endif */ - } /* endif */ - } - else - { - sRc = BTREE_CORRUPTED; - ERREVENT2( QDAMLOCSUBSTR_LOC, STATE_EVENT, 1, DB_GROUP, "" ); - } /* endif */ - - if ( !sRc ) - { - usLen = (USHORT)(UTF16strlenBYTE( pKey2 ) + sizeof(CHAR_W)); - if ( !pchBuffer || *pulLength == 0) - { - *pulLength = usLen ; // give back length only - } - else - { - if ( *pulLength >= usLen ) - { - *pulLength = usLen; - memcpy( pchBuffer, pKey2, usLen ); - } - else - { - //ERREVENT2( QDAMLOCSUBSTR_LOC, STATE_EVENT, 2, DB_GROUP, "" ); - T5LOG(T5ERROR) <<":: BufferTooSmall 4"; - sRc = BTREE_BUFFER_SMALL; - } /* endif */ - } /* endif */ - if ( !sRc ) - { - recData = QDAMGetrecData_V3( pRecord, i, pBT->usVersion ); - if ( *pulUserLen == 0 || ! pchUserData ) - { - *pulUserLen = recData.ulLen; - } - else - { - sRc = QDAMGetszData_V3( pBTIda, recData, pchUserData, pulUserLen, DATA_NODE ); - } /* endif */ - } /* endif */ - } /* endif */ - } - else - { - SET_AND_LOG(sRc, BTREE_NOT_FOUND); - } /* endif */ - BTREEUNLOCKRECORD( pRecord ); - } /* endif */ - - return sRc; -} //------------------------------------------------------------------------------ @@ -6365,8 +6211,7 @@ SHORT QDAMDictFirstLocal PBTREE pBTIda, PCHAR_W pKeyData, // pointer to space for key data PULONG pulKeyLen, // length of space for key data - PBYTE pUserData, // pointer to space for user data - PULONG pulUserLen // length of space for user data + std::vector& pUserData // pointer to space for user data ) { SHORT sRc = 0; // return code @@ -6408,7 +6253,7 @@ SHORT QDAMDictFirstLocal if ( !sRc ) { - sRc = QDAMGetszData_V3( pBTIda, recData, pUserData, pulUserLen, DATA_NODE ); + sRc = QDAMGetszData_V3( pBTIda, recData, pUserData, DATA_NODE ); } /* endif */ if ( (sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED) ) @@ -6447,8 +6292,7 @@ USHORT EQFNTMOrganizeIndex //PBTREE pbTree = (PBTREE)(*ppBTIda); PCHAR_W pchKeyBuffer = NULL; // buffer for record keys ULONG ulKeyBufSize = 0; // current size of key buffer (number of characters) - PBYTE pbData = NULL; // buffer for record data - ULONG ulDataBufSize = 0; // current size of record data buffer (number of bytes) + std::vector pbData; // buffer for record data BOOL fNewIndexCreated = FALSE; // new-index-has-been-created flag CHAR szNewIndex[MAX_LONGPATH]; // buffer for new index name PBTREE pBtreeOut = NULL; // structure for output BTREE @@ -6456,21 +6300,13 @@ USHORT EQFNTMOrganizeIndex ULONG ulKey; // allocate buffer areas + pbData.resize(MAX_INDEX_LEN * sizeof(LONG) * 4); ulKeyBufSize = 256; if ( !UtlAlloc( (PVOID *)&pchKeyBuffer, 0, ulKeyBufSize*sizeof(CHAR_W) , ERROR_STORAGE ) ) { LOG_AND_SET_RC(sRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); } /* endif */ - if ( !sRc ) - { - ulDataBufSize = MAX_INDEX_LEN * sizeof(LONG) * 4; - if ( !UtlAlloc( (PVOID *)&pbData, 0, ulDataBufSize, ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(sRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - if ( !sRc ) { if ( !UtlAlloc( (PVOID *)&pBtreeOut, 0, sizeof(BTREE), ERROR_STORAGE ) ) @@ -6483,22 +6319,20 @@ USHORT EQFNTMOrganizeIndex if ( !sRc ) { //T5LOG( T5WARNING) << "TEMPORARY HARDCODED EQFNTMOrganizeIndex:: usSigLen = (USHORT)ulDataBufSize => usSigLen = (SHORT)ulDataBufSize"); - usSigLen = (USHORT)ulDataBufSize; - sRc = pbTree->QDAMDictSignLocal( (PCHAR)pbData, &usSigLen ); + usSigLen = (USHORT)pbData.size(); + sRc = pbTree->QDAMDictSignLocal( (PCHAR)pbData.data(), &usSigLen ); } /* endif */ // check if index is empty if ( !sRc ) { - ULONG ulDataLen = ulDataBufSize; ULONG ulKeyLen = sizeof(ULONG) + 1; // ulKeyBufSize; - sRc = QDAMDictFirstLocal( pbTree, (PCHAR_W)&ulKey, &ulKeyLen, pbData, &ulDataLen ); + sRc = QDAMDictFirstLocal( pbTree, (PCHAR_W)&ulKey, &ulKeyLen, pbData ); } /* endif */ // cleanup if ( pchKeyBuffer ) UtlAlloc( (PVOID *)&pchKeyBuffer, 0, 0, NOMSG ); - if ( pbData ) UtlAlloc( (PVOID *)&pbData, 0, 0, NOMSG ); if ( pBtreeOut ) UtlAlloc( (PVOID *)pBtreeOut, 0, 0, NOMSG ); // re-map some return codes.. diff --git a/source/opentm2/core/utilities/EQFMORPH.cpp b/source/opentm2/core/utilities/EQFMORPH.cpp index 8c5d0fae..3bd02ccb 100755 --- a/source/opentm2/core/utilities/EQFMORPH.cpp +++ b/source/opentm2/core/utilities/EQFMORPH.cpp @@ -1112,10 +1112,27 @@ USHORT MorphGetLCB } /* end of function MorphGetLCB */ +USHORT MorphAddTermToList2W +( + std::vector& pList, // ptr to term list pointer + PSZ_W pszTerm, // ptr to new term being added to list + USHORT usLength, // length of term + USHORT usOffs, // offset of term + ULONG ulFlags // flags for the ter +) +{ + FLAGOFFSLIST data; + data.lFlags = ulFlags; + data.usLen = usLength; + data.usOffs = usOffs; + pList.emplace_back(data); + return 0; +} + USHORT MorphAddTermToList2W ( - PSZ_W *ppList, // ptr to term list pointer + PSZ_W* ppList, // ptr to term list pointer PULONG pulSize, // alloc. size of term list in # of w's PULONG pulUsed, // used # of w's in term list PSZ_W pszTerm, // ptr to new term being added to list diff --git a/source/opentm2/core/utilities/EQFTADIT.cpp b/source/opentm2/core/utilities/EQFTADIT.cpp index 14ec7f8d..e69de29b 100755 --- a/source/opentm2/core/utilities/EQFTADIT.cpp +++ b/source/opentm2/core/utilities/EQFTADIT.cpp @@ -1,60 +0,0 @@ -#ifdef TO_BE_REMOVED -//+----------------------------------------------------------------------------+ -//| EQFTADIT.CPP | -//+----------------------------------------------------------------------------+ -//|Copyright Notice: | -//| | -//| Copyright (C) 1990-2012, International Business Machines | -//| Corporation and others. All rights reserved | -//+----------------------------------------------------------------------------+ -//| Author: Gerhard Queck | -//| | -//+----------------------------------------------------------------------------+ -//| Description: SGML-DITA special processing functions | -//| | -//| | -//+----------------------------------------------------------------------------+ - -//#include -#include -#include -#include -#include - -// the Win32 Xerces build requires the default structure packing... -//#pragma pack( push ) -//#pragma pack(8) - -#include -#include -#include -#include -#include -#include -#include - -//#pragma pack( pop ) - - -XERCES_CPP_NAMESPACE_USE - - -//#pragma pack( push, TM2StructPacking, 1 ) - -#define INCL_EQF_TP // public translation processor functions -#define INCL_EQF_EDITORAPI // editor API -#define INCL_EQF_TAGTABLE // tagtable defines and functions -#define INCL_EQF_ANALYSIS // analysis functions -#define INCL_EQF_TM // general Transl. Memory functions -#define INCL_EQF_TMREMOTE // remote Transl. Memory functions (U-Code) -#define INCL_EQF_DLGUTILS // dialog utilities -#define INCL_EQF_MORPH -#define INCL_EQF_DAM -#include "EQF.H" // General .H for EQF -#include "EQFMEMIE.H" - -//#pragma pack( pop, TM2StructPacking ) - - -#endif - diff --git a/source/opentm2/core/utilities/EQFUTERR.cpp b/source/opentm2/core/utilities/EQFUTERR.cpp index 4395040e..d695868f 100755 --- a/source/opentm2/core/utilities/EQFUTERR.cpp +++ b/source/opentm2/core/utilities/EQFUTERR.cpp @@ -73,9 +73,6 @@ CHAR INTERNAL_ERROR_TXT [30]; // "INTERNAL ERROR" /* Protoypes for internal functions */ /**********************************************************************/ SHORT UtlQdamMsgTxt ( SHORT ); -static SHORT UtlDosMsgTxt ( SHORT, PSZ, PUSHORT ); -static SHORT UtlQldbMsgTxt ( SHORT ); -static SHORT UtlPropMsgTxt ( SHORT ); USHORT UtlErrorHwnd diff --git a/source/opentm2/core/utilities/FilesystemHelper.cpp b/source/opentm2/core/utilities/FilesystemHelper.cpp index 8262b9c5..02b840d8 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.cpp +++ b/source/opentm2/core/utilities/FilesystemHelper.cpp @@ -611,28 +611,7 @@ int FileBuffer::Flush(bool forced){ oldSize = FilesystemHelper::GetFileSize(fileName); } - #ifdef TO_BE_REMOVED - PUCHAR bufStart = &data[0]; - int size = data.size(); - //if(fileWasOpened && file) fclose(file); - if(!fileWasOpened) file = fopen(fileName.c_str(),"w+b"); - - if(file==nullptr){ - T5LOG(T5ERROR) <<"Cant open the file " << fileName; - return FilesystemHelper::FILEHELPER_FILE_PTR_IS_NULL; - } - if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - oldSize = FilesystemHelper::GetFileSize(file); - } - - fseek(file, 0, SEEK_SET); - //if(!fileWasOpened) - { - fclose(file); - file = nullptr; - } - #endif //writenBytes = fwrite(bufStart, size, 1, file) * size; rc = atomicWriteWithBackup(); if ( rc ){ diff --git a/source/opentm2/core/utilities/LogWrapper.h b/source/opentm2/core/utilities/LogWrapper.h index 0f42a46d..03c28132 100755 --- a/source/opentm2/core/utilities/LogWrapper.h +++ b/source/opentm2/core/utilities/LogWrapper.h @@ -140,7 +140,7 @@ class T5Logger{ std::string FlushBuffers(int severity); }; -static void GetElapsedTime( long long *plTime ); +void GetElapsedTime( long long *plTime ); std::string getTimeStr(); int getBuffIdForLog(int severity); diff --git a/source/opentm2/core/utilities/ThreadingWrapper.h b/source/opentm2/core/utilities/ThreadingWrapper.h index 62b032a8..05d99bcb 100755 --- a/source/opentm2/core/utilities/ThreadingWrapper.h +++ b/source/opentm2/core/utilities/ThreadingWrapper.h @@ -45,7 +45,7 @@ class MutexTimeout{ return m_timeout_failed; } - bool reset(){ + void reset(){ m_timeout_failed = false; m_errMsg = ""; m_timeout_ms = 0; diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c index 211f420c..aa5e1b9e 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c @@ -439,23 +439,6 @@ USHORT EQFQUERYEXITINFO( *******************************************************************************/ -EQF_BOOL EQFCHECKSEGEXW( - PSZ_W pszPrevSrc, - PSZ_W pszSrc, - PSZ_W pszTgt, - PEQF_BOOL pfChanged, - LONG lInfo, - ULONG ulSegNum, - EQF_BOOL fMsg) -{ - BOOL bReturn = TRUE ; - - PrepDocLanguageInfo( NULL ) ; /* Set language unqiue processing, like DBCS */ - - bReturn = CheckXmlText( pszPrevSrc, pszSrc, pszTgt, pfChanged, lInfo, ulSegNum, fMsg ) ; - - return( bReturn ) ; -} diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.C b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.C index 251a29ef..01ad501a 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.C +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.C @@ -72,7 +72,6 @@ typedef struct _MRIPARSFLAGS BOOL fEscapeChars; // TRUE = process escape chars (prefixed with a backslash) } MRIPARSEFLAGS, *PMRIPARSEFLAGS; -static USHORT MriGetSettings( PSZ pszTagTable, PMRIPARSEFLAGS pFlags ); // static array with markup table names and associated flags typedef struct _MRIPARSMARKUPINFO @@ -274,57 +273,6 @@ int MriCountQuotes( PSZ_W pszString, CHAR_W chQuote, BOOL fEscapeChars ) return( iQuotes ); } -// check segment content before it is saved - -EQF_BOOL /*APIENTRY*/ EQFCHECKSEGEXW -( - PSZ_W pszPrevSrc, // previous source segment - PSZ_W pszSrc, // current source segment - PSZ_W pszTgt, // current translation - PEQF_BOOL pfChanged, // segment changed - LONG lInfo, // info handle to use with EQFGETPREVSEG(W),EQFGETNEXTSEG(W) - ULONG ulSegNum, // segment number to use with EQFGETPREVSEG(W),EQFGETNEXTSEG(W) - EQF_BOOL fMsg // message display requested -) -{ - CHAR_W chQuote = L' '; - int iSourceQuotes, iTargetQuotes; - EQF_BOOL fContinue = TRUE; - MRIPARSEFLAGS Flags; - - // adress TBDOCUMENT for access to markup table name - PTBDOCUMENT pDoc = (PTBDOCUMENT)lInfo; - PLOADEDTABLE pTable = (PLOADEDTABLE)pDoc->pDocTagTable; - - pszPrevSrc; pfChanged; ulSegNum; - - - MriGetSettings( pTable->szName, &Flags ); - - chQuote = Flags.fSingleQuotes ? QUOTE_DELIMITER_W : STRING_DELIMITER_W; - - // count not escaped delimiters in source and target - iSourceQuotes = MriCountQuotes( pszSrc, chQuote, Flags.fEscapeChars ); - iTargetQuotes = MriCountQuotes( pszTgt, chQuote, Flags.fEscapeChars ); - if ( iSourceQuotes != iTargetQuotes ) - { - if ( fMsg ) - { - USHORT usMBCode = UtlError( EQFQUOT_QUOTE_MISMATCH, MB_YESNO | MB_DEFBUTTON2, 0, NULL, EQF_QUERY ); - if ( usMBCode == MBID_NO ) - { - fContinue = FALSE; - } /* endif */ - } - else - { - fContinue = FALSE; - } /* endif */ - } /* endif */ - - return( fContinue ); -} - /**********************************************************************/ @@ -358,225 +306,6 @@ SETTINGKEYWORD SettingsKeywords[] = }; -/**********************************************************************/ -/* MriGetSettings */ -/* */ -/* Get any settings file (name of settings file is name of tag table */ -/* with an extension of .CHR) and set the flags in the passed */ -/* flags structure */ -/**********************************************************************/ -static USHORT MriGetSettings -( - PSZ pszTagTable, // name of tag table - PMRIPARSEFLAGS pFlags // reference to MRI parser flag structure -) -{ - // private data area - typedef struct _GETSETTINGSDATA - { - CHAR chInBuf[8096]; // input buffer - CHAR szLine[1024]; // buffer for current line - CHAR szSettingsFile[MAX_EQF_PATH]; // path name of settings file - } GETSETTINGSDATA, *PGETSETTINGSDATA; - - PGETSETTINGSDATA pData = NULL; // pointer to private data - HFILE hfSettings = NULLHANDLE; // file handle of settings file - BOOL fOK = TRUE; // internal O.K. flag - LONG lBytesToRead = 0L; // number of bytes to read from file - ULONG ulBytesInBuffer = 0; // number of bytes in buffer - PMRIPARSMARKUPINFO pMarkupInfo; - - // initialize caller's flags - memset( pFlags, 0, sizeof(MRIPARSEFLAGS) ); - - // check if info is already contained in our static buffer - pMarkupInfo = MarkupInfo; - while ( (pMarkupInfo->szMarkup[0] != EOS) && (_stricmp( pMarkupInfo->szMarkup, pszTagTable ) != 0) ) pMarkupInfo++; - if ( pMarkupInfo->szMarkup[0] != EOS ) - { - memcpy( pFlags, &(pMarkupInfo->Flags), sizeof(MRIPARSEFLAGS) ); - return( NO_ERROR ); - } /* endif */ - - - // allocate our private data structure - fOK = UtlAlloc( (PVOID *)&pData, 0L, sizeof(GETSETTINGSDATA), NOMSG ); - - // setup name of settings file and open settings file - if ( fOK ) - { - USHORT usActionTaken; // action performed by file open - - UtlMakeEQFPath( pData->szSettingsFile, NULC, TABLE_PATH, NULL ); - GetOTMTablePath( pData->szSettingsFile, pData->szLine ) ; - strcpy( pData->szSettingsFile, pData->szLine ) ; - Utlstrccpy( pData->szSettingsFile + strlen(pData->szSettingsFile), - pszTagTable, DOT ); - strcat( pData->szSettingsFile, SETTINGS_EXTENSION ); - fOK = UtlOpen( pData->szSettingsFile, &hfSettings, &usActionTaken, 0L, - FILE_NORMAL, FILE_OPEN, - OPEN_ACCESS_READONLY | OPEN_SHARE_DENYWRITE, - 0L, FALSE ) == NO_ERROR; - } /* endif */ - - // get size of input file - if ( fOK ) - { - FILESTATUS stStatus; // File status information - - fOK = UtlQFileInfo( hfSettings, 1, (PBYTE)&stStatus, - sizeof(FILESTATUS), FALSE ) == NO_ERROR; - lBytesToRead = stStatus.cbFile; - } /* endif */ - - // read and process until done - while( fOK && ((lBytesToRead != 0L)|| (ulBytesInBuffer != 0)) ) - { - PSETTINGKEYWORD pKeyWord = NULL; // ptr for keyword processing - - // fill input buffer - if ( lBytesToRead && (ulBytesInBuffer < sizeof(pData->chInBuf)) ) - { - USHORT usBytesRead; // number of bytes read from file - ULONG ulBytesToRead; - - ulBytesToRead = min( ((LONG)sizeof(pData->chInBuf)- ulBytesInBuffer), - (ULONG)lBytesToRead ); - // do not upgrade to UtlReadL, since downward compatibility would be lost! - // RJ - 030325 - GQ agreed - fOK = UtlRead( hfSettings, pData->chInBuf + ulBytesInBuffer, - (USHORT)ulBytesToRead, - &usBytesRead, FALSE ) == NO_ERROR; - ulBytesInBuffer += usBytesRead; - lBytesToRead -= usBytesRead; - } /* endif */ - - // extract next line from input buffer - if ( fOK ) - { - USHORT usPos = 0; - - // copy bytes to line buffer until EOL detected - while ( (usPos < ulBytesInBuffer) && - (pData->chInBuf[usPos] != CR) && - (pData->chInBuf[usPos] != LF ) ) - { - pData->szLine[usPos] = pData->chInBuf[usPos]; - usPos++; - } /* endwhile */ - - // terminate line buffer and skip EOL characters - pData->szLine[usPos] = EOS; - while ( (usPos < ulBytesInBuffer) && - ( (pData->chInBuf[usPos] == CR) || - (pData->chInBuf[usPos] == LF ) ) ) - { - usPos++; - } /* endwhile */ - - // adjust data in input buffer - memmove( pData->chInBuf, pData->chInBuf + usPos, ulBytesInBuffer - usPos ); - ulBytesInBuffer -= usPos; - } /* endif */ - - // check for settings keywords - if ( fOK ) - { - // lookup settings keyword in keyword table - pKeyWord = SettingsKeywords; - while ( (pKeyWord->szKey[0] != EOS) && - (strnicmp( pKeyWord->szKey, pData->szLine, pKeyWord->sLen ) != 0) ) - { - pKeyWord++; - } - } /* endif */ - - // check supplied value if keyword found and do any settings - if ( pKeyWord->szKey[0] != EOS ) - { - PSZ pszTemp; - PSZ pszSpecifiedValue = pData->szLine + pKeyWord->sLen; - PSZ pszValue; - SHORT sValueIndex = 0; - BOOL fDone = FALSE; - - // truncate value at first non-alphanumeric character - pszTemp = pszSpecifiedValue; - while ( isalnum(*pszTemp) ) pszTemp++; - *pszTemp = EOS; - - // compare specified value with possible values - pszValue = pKeyWord->szValues; - - while ( (*pszValue != EOS) && !fDone ) - { - CHAR chTemp; // buffer for value end character - - // isolate current value of possible value list - pszTemp = pszValue; - while ( (*pszTemp != ',') && (*pszTemp != EOS) ) pszTemp++; - chTemp = *pszTemp; - *pszTemp = EOS; - - // compare against specified value - if ( stricmp( pszValue, pszSpecifiedValue ) == 0 ) - { - fDone = TRUE; - } /* endif */ - - // continue with next value (if any) - *pszTemp = chTemp; - if ( !fDone && (chTemp != EOS) ) - { - pszValue = pszTemp + 1; - sValueIndex += 1; - } - else - { - pszValue = pszTemp; - } /* endif */ - } /* endwhile */ - - // set flags if valid - if ( fDone ) - { - switch ( pKeyWord->sID ) - { - case DOUBLEQUOTES_ID : - pFlags->fDoubleQuotes = sValueIndex; - break; - case SINGLEQUOTES_ID : - pFlags->fSingleQuotes = sValueIndex; - break; - case ESCAPECHARS_ID : - pFlags->fEscapeChars = sValueIndex; - break; - default: - break; - } /* endswitch */ - } /* endif */ - } /* endif */ - } /* endwhile */ - - // update static markup table info array - if ( !pFlags->fDoubleQuotes && !pFlags->fSingleQuotes ) - { - pFlags->fDoubleQuotes = TRUE; - } /* end */ - strcpy( pMarkupInfo->szMarkup, pszTagTable ); - memcpy( &(pMarkupInfo->Flags), pFlags, sizeof(MRIPARSEFLAGS) ); - pMarkupInfo[1].szMarkup[0] = EOS; - - - // cleanup - if ( hfSettings ) UtlClose( hfSettings, FALSE ); - if ( pData ) UtlAlloc( (PVOID *)&pData, 0L, 0L, NOMSG ); - - // return to caller - return( NO_ERROR ); -} /* end of function MriGetSettings */ - - //+----------------------------------------------------------------------------+ //| ParseQuotedFile - parse a MRI file | diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTable.cpp b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTable.cpp index cb852186..a01e5fa7 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTable.cpp +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTable.cpp @@ -270,78 +270,6 @@ bool OtmMarkupTable::isProtected() return( true ); } - /*! \brief Update interface for the markup table - - This method can update its internal table files with the files provided in the - comma separated update file list - - \param pszTableName Pointer to name of markup table - \param pszShortDescription Pointer to markup table short description or NULL - \param pszLongDescription Pointer to markup table long description or NULL - \param pszVersion Pointer to version of markup table or NULL - \param pTableFileName Pointer to name of TBL file - \param pUserExitFileName Pointer to name of user exit DLL file or NULL - \param pFileList Pointer to other markup table files (with TABLE\ or BIN\ prefix) or NULL - - \returns 0 when the update failed - 1 when the markup table files have been updated - 2 when the update has been delayed and will occur at restart - - */ -int OtmMarkupTable::updateFiles( - char *pszTableName, - char *pszShortDescription, - char *pszLongDescription, - char *pszVersion, - char *pszTableFileName, - char *pszUserExitFileName, - char *pszFileList ) -{ - bool bNewMarkup = false ; - int iReturn = UPDATE_MARKUP_OK ; - - - if ( ( pszTableName ) && - ( ! pInfo->pszName ) ) { - bNewMarkup = true ; - SaveValue2( &(pInfo->pszName), pszTableName ) ; - } - if ( pszShortDescription ) { - if ( ( ! pInfo->pszShortDescription ) || - ( strcmp( pszShortDescription, pInfo->pszShortDescription ) >= 0 ) ) - SaveValue2( &(pInfo->pszShortDescription), pszShortDescription ) ; - } - if ( pszLongDescription ) { - if ( ( ! pInfo->pszLongDescription ) || - ( strcmp( pszLongDescription, pInfo->pszLongDescription ) >= 0 ) ) - SaveValue2( &(pInfo->pszLongDescription), pszLongDescription ) ; - } - if ( ( pszVersion ) && - ( ( ! pInfo->pszVersion ) || - ( stricmp( pszVersion, pInfo->pszVersion ) >= 0 ) ) ) { - SaveValue2( &(pInfo->pszVersion), pszVersion ) ; - } - if ( ( pszTableFileName ) && - ( ! pInfo->pszTable ) ) { - SaveValue2( &(pInfo->pszTable), pszTableFileName ) ; - } - if ( ( ( pszUserExitFileName ) && - ( ( ! pInfo->pszUserExit ) || - ( stricmp( pszUserExitFileName, pInfo->pszUserExit ) ) ) ) || - ( ( ! pszUserExitFileName ) && - ( pInfo->pszUserExit ) ) ) { - SaveValue2( &(pInfo->pszUserExit), pszUserExitFileName ) ; - } - if ( ( ( pszFileList ) && - ( ( ! pInfo->pszFileList ) || - ( stricmp( pszFileList, pInfo->pszFileList ) ) ) ) || - ( ( ! pszFileList ) && - ( pInfo->pszFileList ) ) ) { - SaveValue2( &(pInfo->pszFileList), pszFileList ) ; - } - - return( iReturn ) ; -} /*! \brief Update XML information for the markup table diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp index f6550003..a84195f2 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp @@ -532,448 +532,6 @@ const bool OtmMarkupTablePlugin::isProtected() } -/* -------------------------------------------------------------- */ -/* updateFiles. */ -/* -------------------------------------------------------------- */ -/*! \brief Update the files contained in this markup table - - \param pszMarkupName Pointer to name of markup table - \param pszShortDescription Pointer to markup table short description or NULL - \param pszLongDescription Pointer to markup table long description or NULL - \param pszVersion Pointer to version of markup table or NULL - \param pTableFileName Pointer to name of TBL file - \param pUserExitFileName Pointer to name of user exit DLL file or NULL - \param pFileList Pointer to other markup table files (with TABLE\ or BIN\ prefix) or NULL - - \returns 0 when the update failed - 1 when the markup table files have been updated - 2 when the update has been delayed and will occur at restart -*/ -const int OtmMarkupTablePlugin::updateFiles( - char *pszMarkupName, - char *pszShortDescription, - char *pszLongDescription, - char *pszVersion, - char *pszTableFileName, - char *pszUserExitFileName, - char *pszFileList ) - { - - PTAGTABLE m_pTagTable; - FILE *fInControl, *fOutControl ; - - char szNewFile[512] ; - char szOldFile[512] ; - char szInLine[512] ; - char szOutLine[512] ; - char szNode[512] ; - char szContent[512] ; - - char szMarkupName[512] ; - char szShortDescription[512] ; - char szLongDescription[512] ; - char szVersion[512] ; - char szTableDirFileName[512] ; - char szTableFileName[512] ; /* xxx.TBL */ - char szUserExitDirFileName[512] ; /* TABLE\xxx.TBL */ - char szUserExitFileName[512] ; /* xxx.DLL */ - char szFileList[MAX_FILELIST] ; /* BIN\xxx.DLL */ - - char szTemp[MAX_FILELIST] ; - char *ptrChar, *ptrChar2, *ptrChar3 ; - - int iRC ; - bool bFileChanged = FALSE ; - bool bNewMarkup = FALSE ; - bool bUpdateMarkup = FALSE ; - bool bLogOpen = FALSE ; - int iReturn = UPDATE_MARKUP_ERROR ; - - /* ------------------------------------------------------------------- */ - /* Initialization. */ - /* ------------------------------------------------------------------- */ - strcpy( szMarkupName, pszMarkupName ) ; - strupr( szMarkupName ) ; -// szDescription[0] = '\0' ; - strcpy( szShortDescription, "-" ) ; - strcpy( szLongDescription, "-" ) ; - strcpy( szVersion, "0.0" ) ; - sprintf( szTableFileName, "%s.TBL", szMarkupName ) ; - sprintf( szTableDirFileName, "%s\\%s", pszTableDirectory, szTableFileName ) ; - sprintf( szTableFileName, "%s.TBL", szMarkupName ) ; - sprintf( szTableDirFileName, "%s\\%s", pszTableDirectory, szTableFileName ) ; - szUserExitFileName[0] = '\0' ; - szUserExitDirFileName[0] = '\0' ; - szFileList[0] = '\0' ; - bLogOpen = FALSE ; - - if ( ( pszShortDescription ) && - ( *pszShortDescription ) ) { - strcpy( szShortDescription, pszShortDescription ) ; - } - if ( ( pszLongDescription ) && - ( *pszLongDescription ) ) { - strcpy( szLongDescription, pszLongDescription ) ; - } - if ( ( pszVersion ) && - ( *pszVersion ) ) { - strcpy( szVersion, pszVersion ) ; - } - if ( pszTableFileName ) { - ptrChar = strrchr( pszTableFileName, '\\' ) ; - if ( ptrChar ) { - strcpy( szTableFileName, ptrChar+1 ) ; - if ( ! strstr( szTableFileName, ".TBL" ) ) - strcat( szTableFileName, ".TBL" ) ; - sprintf( szTableDirFileName, "%s\\%s", pszTableDirectory, szTableFileName ) ; - } - } - if ( ( pszUserExitFileName ) && - ( *pszUserExitFileName ) ) { - ptrChar = strrchr( pszUserExitFileName, '\\' ) ; - if ( ptrChar ) { - strcpy( szUserExitFileName, ptrChar+1 ) ; - } else { - strcpy( szUserExitFileName, pszUserExitFileName ) ; - } - if ( ! strstr( szUserExitFileName, ".DLL" ) ) - strcat( szUserExitFileName, ".DLL" ) ; - sprintf( szUserExitDirFileName, "%s\\%s", pszUserExitDirectory, szUserExitFileName ) ; - } - if ( pszFileList ) { - if ( *pszFileList ) { - szFileList[0] = '\0' ; - sprintf( szNode, "\\%s\\", pszTableDirectory ) ; - sprintf( szContent, "\\%s\\", pszUserExitDirectory ) ; - strcpy( szTemp, pszFileList ) ; - strupr( szTemp ) ; - ptrChar = strtok( szTemp, " ," ) ; - while( ptrChar ) { - ptrChar2 = strstr( ptrChar, szNode ) ; - if ( ! ptrChar2 ) - ptrChar2 = strstr( ptrChar, szContent ) ; - if ( ! ptrChar2 ) { - ptrChar2 = strrchr( ptrChar, '\\' ) ; - if ( ! ptrChar2 ) - ptrChar2 = ptrChar - 1 ; - ptrChar3 = strrchr( ptrChar, '.' ) ; - if ( ( ptrChar3 ) && - ( ( ! strcmp( ptrChar3+1, "DLL" ) ) || - ( ! strcmp( ptrChar3+1, "EXE" ) ) ) ) - sprintf( szInLine, " %s\\%s", pszUserExitDirectory, ptrChar2+1 ) ; - else - sprintf( szInLine, " %s\\%s", pszTableDirectory, ptrChar2+1 ) ; - ptrChar2 = szInLine ; - } - if ( szFileList[0] ) - strcat( szFileList, "," ) ; - strcat( szFileList, ptrChar2+1 ) ; - ptrChar = strtok( NULL, " ," ) ; - } - } - } - strupr( szTableFileName ) ; - strupr( szTableDirFileName ) ; - strupr( szUserExitFileName ) ; - strupr( szUserExitDirFileName ) ; -#ifdef MARKUPTABLE_LOGGING - T5LOG(T5DEBUG) << "Updating markup table... " <szDescription ) ; - } - UtlAlloc( (PVOID *)&m_pTagTable, 0L, 0L, NOMSG ); /* Free */ - } - - /* ------------------------------------------------------------------- */ - /* Log that this markup table is being updated. */ - /* ------------------------------------------------------------------- */ -#ifdef MARKUPTABLE_LOGGING - if ( pszShortDescription ) T5LOG(T5DEBUG) << " SDs: " << pszShortDescription; - if ( pszLongDescription ) T5LOG(T5DEBUG) << " LDs: " <updateFiles( szMarkupName, szShortDescription, szLongDescription, szVersion, - szTableDirFileName, szUserExitDirFileName, szFileList ) ; - } - - bUpdateMarkup = FALSE ; - if ( ( pszTableFileName ) && - ( *pszTableFileName != NULL ) && - ( *(pszTableFileName+1) == ':' ) ) { - markup->getTableFileName( szTemp, sizeof(szTemp) ) ; - if ( ! stricmp( pszTableFileName, szTemp ) ) { - bUpdateMarkup = TRUE ; - } else { - iRC = CopyMarkupFile( pszTableFileName, szTemp, FALSE ) ; - if ( iRC != COPY_MARKUP_NOCHANGE ) { - if ( iRC == COPY_MARKUP_COPIED ) { - bUpdateMarkup = TRUE ; - } else { - iReturn = UPDATE_MARKUP_ERROR ; - #ifdef MARKUPTABLE_LOGGING - T5LOG(T5DEBUG) << " Copy file failed: "< " <getUserExitFileName( szTemp, sizeof(szTemp) ) ; - if ( ! stricmp( pszUserExitFileName, szTemp ) ) { - bUpdateMarkup = TRUE ; - } else { - iRC = CopyMarkupFile( pszUserExitFileName, szTemp, TRUE ) ; - if ( iRC != COPY_MARKUP_NOCHANGE ) { - if ( iRC == COPY_MARKUP_COPIED ) { - bUpdateMarkup = TRUE ; - } else - if ( iRC == COPY_MARKUP_INUSE ) { - bUpdateMarkup = TRUE ; - iReturn = iRC ; - } else { - if ( ! bNewMarkup ) { - iReturn = UPDATE_MARKUP_ERROR ; - #ifdef MARKUPTABLE_LOGGING - T5LOG(T5DEBUG) << " Copy file failed: "< " << szTemp; - #endif - } - } - } - } - } - if ( ( iReturn == UPDATE_MARKUP_OK ) && - ( pszFileList ) && - ( *pszFileList != NULL ) ) { - markup->getAllFiles( szTemp, sizeof(szTemp) ) ; - iRC = CopyMarkupFiles( pszFileList, szTemp, szBasePath, pszTableDirectory, pszUserExitDirectory, TRUE ) ; - if ( iRC != COPY_MARKUP_NOCHANGE ) { - if ( iRC == COPY_MARKUP_COPIED ) { - bUpdateMarkup = TRUE ; - } else - if ( iRC == COPY_MARKUP_INUSE ) { - bUpdateMarkup = TRUE ; - iReturn = iRC ; - } else { - if ( ! bNewMarkup ) { - iReturn = UPDATE_MARKUP_ERROR ; - #ifdef MARKUPTABLE_LOGGING - T5LOG(T5DEBUG) << " Copy file failed: "< " << szTemp; - #endif - } - } - } - } - - - /* ------------------------------------------------------------------- */ - /* Update the markup table object with this new information. */ - /* ------------------------------------------------------------------- */ - if ( ( iReturn != UPDATE_MARKUP_ERROR ) && - ( ( bNewMarkup ) || - ( ( bUpdateMarkup ) && - ( markup->updateFiles( szMarkupName, szShortDescription, szLongDescription, szVersion, - szTableDirFileName, szUserExitDirFileName, - szFileList ) ) ) ) ) { - - /* ------------------------------------------------------------------- */ - /* Update control XML file with new information. */ - /* ------------------------------------------------------------------- */ - - strcpy( szOldFile, szBasePath ) ; - strcat( szOldFile, "\\" ) ; - strcat( szOldFile, PluginControlFile ) ; - strcpy( szNewFile, szOldFile ) ; - strcat( szNewFile, ".NEW" ) ; - - fInControl = fopen( szOldFile, "r" ) ; - fOutControl = fopen( szNewFile, "w" ) ; - if ( fInControl && fOutControl ) { - bool bInMarkups = FALSE ; - bool bInMarkup = FALSE ; - bUpdateMarkup = FALSE ; - - while( fgets( szInLine, sizeof(szInLine), fInControl ) != NULL ) { - - strcpy( szOutLine, szInLine ) ; - GetControlNode( szInLine, szNode, szContent ) ; - - if ( szNode[0] ) { - if ( ! strcmp( szNode, szXml_Markups ) ) { - bInMarkups = TRUE ; - } else - if ( bInMarkups ) { - if ( ( szNode[0] == '/' ) && - ( ! strcmp( &szNode[1], szXml_Markups ) ) ) { - bInMarkups = FALSE ; - if ( bNewMarkup ) { - bFileChanged = TRUE ; - fprintf( fOutControl, "<%s>\n", szXml_Markup ) ; - fprintf( fOutControl, szXml_NewRecord, szXml_Name, szMarkupName, szXml_Name ) ; - fprintf( fOutControl, szXml_NewRecord, szXml_ShortDescription, szShortDescription, szXml_ShortDescription ) ; - fprintf( fOutControl, szXml_NewRecord, szXml_LongDescription, szLongDescription, szXml_LongDescription ) ; - fprintf( fOutControl, szXml_NewRecord, szXml_Version, szVersion, szXml_Version ) ; - fprintf( fOutControl, szXml_NewRecord, szXml_Table, szTableFileName, szXml_Table ) ; - if ( szUserExitFileName[0] ) - fprintf( fOutControl, szXml_NewRecord, szXml_UserExit, szUserExitFileName, szXml_UserExit ) ; - else - fprintf( fOutControl, szXml_NewRecord, szXml_UserExit, szXml_none, szXml_UserExit ) ; - if ( szFileList[0] ) - fprintf( fOutControl, szXml_NewRecord, szXml_Files, szFileList, szXml_Files ) ; - else - fprintf( fOutControl, szXml_NewRecord, szXml_Files, szXml_none, szXml_Files ) ; - fprintf( fOutControl, "\n", szXml_Markup ) ; - } - bUpdateMarkup = FALSE ; - } else - if ( ! strcmp( szNode, szXml_Markup ) ) { - bInMarkup = TRUE ; - } else - if ( bInMarkup ) { - if ( ( szNode[0] == '/' ) && - ( ! strcmp( &szNode[1], szXml_Markup ) ) ) { - bInMarkup = FALSE ; - bUpdateMarkup = FALSE ; - } else - if ( ! strcmp( szNode, szXml_Name ) ) { - if ( ! stricmp( szContent, szMarkupName ) ) { - bUpdateMarkup = TRUE ; - #ifdef MARKUPTABLE_LOGGING - T5LOG(T5DEBUG) << " Updating: " <szDescription ) ; - } - UtlAlloc( (PVOID *)&m_pTagTable, 0L, 0L, NOMSG ); /* Free */ - } - - /* ------------------------------------------------------------------- */ - /* Log that this markup table is being updated. */ - /* ------------------------------------------------------------------- */ - #ifdef MARKUPTABLE_LOGGING - if ( pszShortDescription T5LOG(T5DEBUG) << " SDs: " <updateFiles( szMarkupName, szShortDescription, szLongDescription, szVersion, - szTableDirFileName, szUserExitDirFileName, szFileList ) ; - } - - bUpdateMarkup = FALSE ; - if ( ( pszTableFileName ) && - ( *pszTableFileName != NULL ) && - ( *(pszTableFileName+1) == ':' ) ) { - markup->getTableFileName( szTemp, sizeof(szTemp) ) ; - if ( ! stricmp( pszTableFileName, szTemp ) ) { - bUpdateMarkup = TRUE ; - } else { - iRC = CopyMarkupFile( pszTableFileName, szTemp, FALSE ) ; - if ( iRC != COPY_MARKUP_NOCHANGE ) { - if ( iRC == COPY_MARKUP_COPIED ) { - bUpdateMarkup = TRUE ; - } else { - iReturn = UPDATE_MARKUP_ERROR ; - #ifdef MARKUPTABLE_LOGGING - T5LOG(T5DEBUG) << " Copy file failed: "< "<getUserExitFileName( szTemp, sizeof(szTemp) ) ; - if ( ! stricmp( pszUserExitFileName, szTemp ) ) { - bUpdateMarkup = TRUE ; - } else { - iRC = CopyMarkupFile( pszUserExitFileName, szTemp, TRUE ) ; - if ( iRC != COPY_MARKUP_NOCHANGE ) { - if ( iRC == COPY_MARKUP_COPIED ) { - bUpdateMarkup = TRUE ; - } else - if ( iRC == COPY_MARKUP_INUSE ) { - bUpdateMarkup = TRUE ; - iReturn = iRC ; - } else { - if ( ! bNewMarkup ) { - iReturn = UPDATE_MARKUP_ERROR ; - #ifdef MARKUPTABLE_LOGGING - T5LOG(T5DEBUG) << " Copy file failed: "< " << szTemp; - #endif - } - } - } - } - } - if ( ( iReturn == UPDATE_MARKUP_OK ) && - ( pszFileList ) && - ( *pszFileList != NULL ) ) { - markup->getAllFiles( szTemp, sizeof(szTemp) ) ; - iRC = CopyMarkupFiles( pszFileList, szTemp, szBasePath, pszTableDirectory, pszUserExitDirectory, TRUE ) ; - if ( iRC != COPY_MARKUP_NOCHANGE ) { - if ( iRC == COPY_MARKUP_COPIED ) { - bUpdateMarkup = TRUE ; - } else - if ( iRC == COPY_MARKUP_INUSE ) { - bUpdateMarkup = TRUE ; - iReturn = iRC ; - } else { - if ( ! bNewMarkup ) { - iReturn = UPDATE_MARKUP_ERROR ; - #ifdef MARKUPTABLE_LOGGING - T5LOG(T5DEBUG) << " Copy file failed: "<< pszFileList << " -> "<updateFiles( szMarkupName, szShortDescription, szLongDescription, - szVersion, szTableDirFileName, szUserExitDirFileName, - szFileList ) ) ) ) ) { - - iReturn = UpdateXMLControlFile( szMarkupName, szShortDescription, - szLongDescription, szVersion, - szTableFileName, szUserExitFileName, - szFileList, szBasePath, bNewMarkup ) ; - - #ifdef MARKUPTABLE_LOGGING - if ( iReturn == UPATE_MARKUP_OK ) { - T5LOG(T5DEBUG) << " Update control file failed: " << szMarkupName; - } else { - T5LOG(T5DEBUG) << " Update control file OK: " << szMarkupName ; - } - #endif - -// strcpy( szOldFile, szBasePath ) ; -// strcat( szOldFile, "\\" ) ; -// strcat( szOldFile, PluginControlFile ) ; -// strcpy( szNewFile, szOldFile ) ; -// strcat( szNewFile, ".NEW" ) ; -// -// fInControl = fopen( szOldFile, "r" ) ; -// fOutControl = fopen( szNewFile, "w" ) ; -// if ( fInControl && fOutControl ) { -// bool bInMarkups = FALSE ; -// bool bInMarkup = FALSE ; -// bUpdateMarkup = FALSE ; -// -// while( fgets( szInLine, sizeof(szInLine), fInControl ) != NULL ) { -// -// strcpy( szOutLine, szInLine ) ; -// GetControlNode( szInLine, szNode, szContent ) ; -// -// if ( szNode[0] ) { -// if ( ! strcmp( szNode, szXml_Markups ) ) { -// bInMarkups = TRUE ; -// } else -// if ( bInMarkups ) { -// if ( ( szNode[0] == '/' ) && -// ( ! strcmp( &szNode[1], szXml_Markups ) ) ) { -// bInMarkups = FALSE ; -// if ( bNewMarkup ) { -// bFileChanged = TRUE ; -// fprintf( fOutControl, "<%s>\n", szXml_Markup ) ; -// fprintf( fOutControl, szXml_NewRecord, szXml_Name, szMarkupName, szXml_Name ) ; -// fprintf( fOutControl, szXml_NewRecord, szXml_ShortDescription, szDescription, szXml_ShortDescription ) ; -// fprintf( fOutControl, szXml_NewRecord, szXml_LongDescription, szDescription, szXml_LongDescription ) ; -// fprintf( fOutControl, szXml_NewRecord, szXml_Version, szVersion, szXml_Version ) ; -// fprintf( fOutControl, szXml_NewRecord, szXml_Table, szTableFileName, szXml_Table ) ; -// if ( szUserExitFileName[0] ) -// fprintf( fOutControl, szXml_NewRecord, szXml_UserExit, szUserExitFileName, szXml_UserExit ) ; -// else -// fprintf( fOutControl, szXml_NewRecord, szXml_UserExit, szXml_none, szXml_UserExit ) ; -// if ( szFileList[0] ) -// fprintf( fOutControl, szXml_NewRecord, szXml_Files, szFileList, szXml_Files ) ; -// else -// fprintf( fOutControl, szXml_NewRecord, szXml_Files, szXml_none, szXml_Files ) ; -// fprintf( fOutControl, "\n", szXml_Markup ) ; -// } -// bUpdateMarkup = FALSE ; -// } else -// if ( ! strcmp( szNode, szXml_Markup ) ) { -// bInMarkup = TRUE ; -// } else -// if ( bInMarkup ) { -// if ( ( szNode[0] == '/' ) && -// ( ! strcmp( &szNode[1], szXml_Markup ) ) ) { -// bInMarkup = FALSE ; -// bUpdateMarkup = FALSE ; -// } else -// if ( ! strcmp( szNode, szXml_Name ) ) { -// if ( ! stricmp( szContent, szMarkupName ) ) { -// bUpdateMarkup = TRUE ; -//#ifdef MARKUPTABLE_LOGGING -// T5LOG(T5DEVELOP) << " Updating: %s" < Date: Thu, 3 Apr 2025 19:40:05 +0300 Subject: [PATCH 02/62] * fixing raw pointers and refactoring --- docker/workdir/TABLE/otmxml.c | 17 - include/lowlevelotmdatastructs.h | 4 +- include/tm.h | 6165 ++++++++--------- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 43 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 40 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 123 +- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 8 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 396 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 1510 ++-- .../core/EqfMemoryPlugin/EqfMemory.cpp | 13 +- source/opentm2/core/tm.cpp | 5 +- 12 files changed, 3652 insertions(+), 4674 deletions(-) diff --git a/docker/workdir/TABLE/otmxml.c b/docker/workdir/TABLE/otmxml.c index b607be07..9302cb94 100755 --- a/docker/workdir/TABLE/otmxml.c +++ b/docker/workdir/TABLE/otmxml.c @@ -236,23 +236,6 @@ EQF_BOOL /*APIENTRY*/ EQFPOSTUNSEGW( -USHORT /*APIENTRY*/ EQFQUERYEXITINFO(PSZ pszTagTable, // name of the markup table, e.g. "OTMHTM32" - USHORT usMode, // type of information being queried - PSZ pszBuffer, // buffer area receiving the information returned by the exit - USHORT usBufLen // length of buffer area -) -{ - - if( usMode == QUERYEXIT_ADDFILES) { - QueryExportFiles(pszTagTable, pszBuffer, usBufLen, FALSE); - } - - return 0; -} - - - - /******************************************************************************* * * function: IsFileUTF16 diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 84716802..a0c1f3b8 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -507,7 +507,7 @@ typedef struct _TMX_LONGNAME_TABLE_ENTRY //table structure for long document table typedef struct TMX_LONGNAME_TABLE { - std::vector pszBuffer; // buffer for names and IDs + std::vector pszBuffer; // buffer for names and IDs //ULONG ulBufUsed; // number of bytes used in buffer //ULONG ulBufSize; // size of buffer in bytes //ULONG ulTableSize; // table size (# entries) @@ -516,7 +516,7 @@ typedef struct TMX_LONGNAME_TABLE void restoreStrPointersAfterRealloc(){ for(auto& entry: stTableEntry){//during insertion to the pszBuffer, it could be reallocated, so we need to update poiners - entry.pszLongName = pszBuffer.data() + entry.longNameStartOffset; + entry.pszLongName = (PSZ) (pszBuffer[entry.longNameStartOffset]); }; } diff --git a/include/tm.h b/include/tm.h index 25319dca..8907e06d 100644 --- a/include/tm.h +++ b/include/tm.h @@ -17,7 +17,7 @@ #include "ThreadingWrapper.h" -#define INCL_EQF_TAGTABLE // tag table and format functions +#define INCL_EQF_TAGTABLE // tag table and format functions #define INCL_EQF_TP #define INCL_EQF_TM #define INCL_EQF_DAM @@ -29,134 +29,128 @@ #include "../source/opentm2/core/pluginmanager/OtmPlugin.h" #include "../source/opentm2/core/utilities/Stopwatch.hpp" -#define MEM_START_ORGANIZE USER_TASK + 1 -#define MEM_ORGANIZE_TASK USER_TASK + 2 -#define MEM_END_ORGANIZE USER_TASK + 3 - - - -#define KEY_DIR_SIZE 4096 // key directory size -#define TM_PREFIX_SIZE 8 // length of prefix bytes in TMT db -#define MAX_SEC_LENGTH 30 // max length of secondary key -#define MAX_LINE_LENGTH 80 // max length of each line in lang file -#define FN_LENGTH 13 // length of input filename -#define PRIM_KEY_LENGTH 4 // number of characters in primary key -#define DATA_IN_SIZE 3900 // buffer size for input -#define DATA_OUT_SIZE 3900 // buffer size for output -#define CODEPAGE_SIZE 256 // size of codepage of the language -#define SEG_MARKER_LENGTH 3 // length of segment marker -#define MAX_TGT_LENGTH 2047 // max length of each target. -#define MAX_MATCH_TAB_ENTRIES 5 // number of entries in match table -#define CREATE_BUFFER_SIZE 40000 // buffer size for create_in -#define MAX_TM_LIST_NUMBER 500 // max. number of TMs that can be listed - // by the TMC_GET_SERVER_TM_LIST command -#define GETPART_BUFFER_SIZE 16384 // read a 16 KB block at a time -#define MEM_PROP_SIZE 2048 // Global size of all memory database properties -#define _MAX_DIR 256 - -typedef CHAR SHORT_FN [FN_LENGTH]; -typedef CHAR BUFFERIN [DATA_IN_SIZE]; -typedef UCHAR BUFFEROUT [DATA_OUT_SIZE], - ACHPRIMKEY [PRIM_KEY_LENGTH], - SZSECKEY [MAX_SEC_LENGTH + 1]; - -typedef CHAR LANG_LINE [MAX_LINE_LENGTH + 1]; -typedef CHAR LONG_FN [MAX_LONGFILESPEC]; - - +#define MEM_START_ORGANIZE USER_TASK + 1 +#define MEM_ORGANIZE_TASK USER_TASK + 2 +#define MEM_END_ORGANIZE USER_TASK + 3 + +#define KEY_DIR_SIZE 4096 // key directory size +#define TM_PREFIX_SIZE 8 // length of prefix bytes in TMT db +#define MAX_SEC_LENGTH 30 // max length of secondary key +#define MAX_LINE_LENGTH 80 // max length of each line in lang file +#define FN_LENGTH 13 // length of input filename +#define PRIM_KEY_LENGTH 4 // number of characters in primary key +#define DATA_IN_SIZE 3900 // buffer size for input +#define DATA_OUT_SIZE 3900 // buffer size for output +#define CODEPAGE_SIZE 256 // size of codepage of the language +#define SEG_MARKER_LENGTH 3 // length of segment marker +#define MAX_TGT_LENGTH 2047 // max length of each target. +#define MAX_MATCH_TAB_ENTRIES 5 // number of entries in match table +#define CREATE_BUFFER_SIZE 40000 // buffer size for create_in +#define MAX_TM_LIST_NUMBER 500 // max. number of TMs that can be listed + // by the TMC_GET_SERVER_TM_LIST command +#define GETPART_BUFFER_SIZE 16384 // read a 16 KB block at a time +#define MEM_PROP_SIZE 2048 // Global size of all memory database properties +#define _MAX_DIR 256 + +typedef CHAR SHORT_FN[FN_LENGTH]; +typedef CHAR BUFFERIN[DATA_IN_SIZE]; +typedef UCHAR BUFFEROUT[DATA_OUT_SIZE], + ACHPRIMKEY[PRIM_KEY_LENGTH], + SZSECKEY[MAX_SEC_LENGTH + 1]; + +typedef CHAR LANG_LINE[MAX_LINE_LENGTH + 1]; +typedef CHAR LONG_FN[MAX_LONGFILESPEC]; struct TMX_EXT_OUT_W { - TMX_PREFIX_OUT stPrefixOut; //prefix of output buffer - - CHAR_W szSource[MAX_SEGMENT_SIZE]; //source sentence - CHAR_W szTarget[MAX_SEGMENT_SIZE]; //target sentence - CHAR szOriginalSourceLanguage[MAX_LANG_LENGTH]; //language name of the source - CHAR szTagTable[MAX_FNAME]; //tag table name - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language name of target - CHAR szAuthorName[MAX_FILESPEC]; //author name of target - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - CHAR szFileName[MAX_FILESPEC]; //where source comes from name+ext - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSourceSegmentId; //seg. num. of source sentence from analysis - TIME_L lTargetTime; //time stamp of target - CHAR_W szContext[MAX_SEGMENT_SIZE]; //segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information + TMX_PREFIX_OUT stPrefixOut; // prefix of output buffer + + CHAR_W szSource[MAX_SEGMENT_SIZE]; // source sentence + CHAR_W szTarget[MAX_SEGMENT_SIZE]; // target sentence + CHAR szOriginalSourceLanguage[MAX_LANG_LENGTH]; // language name of the source + CHAR szTagTable[MAX_FNAME]; // tag table name + CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language name of target + CHAR szAuthorName[MAX_FILESPEC]; // author name of target + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + CHAR szFileName[MAX_FILESPEC]; // where source comes from name+ext + LONG_FN szLongName; // name of source file (long name or EOS) + ULONG ulSourceSegmentId; // seg. num. of source sentence from analysis + TIME_L lTargetTime; // time stamp of target + std::wstring szContext, szAddInfo; + //CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context + //CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information ULONG ulRecKey; - USHORT usTargetKey; + USHORT usTargetKey; - ULONG ulTmKey; //tm record key - USHORT usNextTarget; //which target record to address next - ULONG ulMaxEntries; //number of entries in tm data file + ULONG ulTmKey; // tm record key + USHORT usNextTarget; // which target record to address next + ULONG ulMaxEntries; // number of entries in tm data file }; using PTMX_EXT_OUT_W = TMX_EXT_OUT_W *; - - -//#include "../pluginmanager/PluginManager.h" +// #include "../pluginmanager/PluginManager.h" #include "EQF.H" // ************ Translation memory flags ******************************** #define far -#define MEM_OUTPUT_ASIS 0 // Do not convert translation memory output -#define MEM_OUTPUT_CRLF 1 // Convert translation memory output to CRLF -#define MEM_OUTPUT_LF 2 // Convert translation memory output to LF +#define MEM_OUTPUT_ASIS 0 // Do not convert translation memory output +#define MEM_OUTPUT_CRLF 1 // Convert translation memory output to CRLF +#define MEM_OUTPUT_LF 2 // Convert translation memory output to LF // defines for special names mode of TMExtract (only valid with TMExtract!) -#define MEM_OUTPUT_TAGTABLES 3 // return list of tag tables of TM -#define MEM_OUTPUT_AUTHORS 4 // return list of authors of TM -#define MEM_OUTPUT_DOCUMENTS 5 // return list of documents of TM -#define MEM_OUTPUT_LANGUAGES 6 // return list of languages of TM -#define MEM_OUTPUT_LONGNAMES 7 // return list of document long names of TM -#define MEM_OUTPUT_ALLDOCS 8 // return list of all documensts (long - // names and short names for docs w/o long name) - -#define EXCLUSIVE 0 -#define NONEXCLUSIVE 1 -#define EXCLUSIVE_FOR_GET_PART 2 -#define FOR_ORGANIZE 3 -#define READONLYACCESS 4 +#define MEM_OUTPUT_TAGTABLES 3 // return list of tag tables of TM +#define MEM_OUTPUT_AUTHORS 4 // return list of authors of TM +#define MEM_OUTPUT_DOCUMENTS 5 // return list of documents of TM +#define MEM_OUTPUT_LANGUAGES 6 // return list of languages of TM +#define MEM_OUTPUT_LONGNAMES 7 // return list of document long names of TM +#define MEM_OUTPUT_ALLDOCS 8 // return list of all documensts (long + // names and short names for docs w/o long name) + +#define EXCLUSIVE 0 +#define NONEXCLUSIVE 1 +#define EXCLUSIVE_FOR_GET_PART 2 +#define FOR_ORGANIZE 3 +#define READONLYACCESS 4 // defines used in TmOpen and MemCreateDlg -#define TM_LOCAL 0 // TM resides local -#define TM_REMOTE 1 // TM resides remote on a server -#define TM_LOCALREMOTE 2 // TM may be local or remote -#define TM_SHARED 3 // TM resides on a shared drive +#define TM_LOCAL 0 // TM resides local +#define TM_REMOTE 1 // TM resides remote on a server +#define TM_LOCALREMOTE 2 // TM may be local or remote +#define TM_SHARED 3 // TM resides on a shared drive // Match value constants: the similarity level classes (lSimLevel) -#define BASE_SIMILAR 0L -#define BASE_EXACT_DATE 200L -#define BASE_EXACT_IND 300L -#define BASE_EXACT_SEG_NUM 400L - -#define MAX_SIMILAR_VAL 100L -#define MAX_EXACT_DATE_VAL (BASE_EXACT_IND - 1L) -#define MAX_EXACT_IND_VAL (BASE_EXACT_SEG_NUM - 1L) +#define BASE_SIMILAR 0L +#define BASE_EXACT_DATE 200L +#define BASE_EXACT_IND 300L +#define BASE_EXACT_SEG_NUM 400L + +#define MAX_SIMILAR_VAL 100L +#define MAX_EXACT_DATE_VAL (BASE_EXACT_IND - 1L) +#define MAX_EXACT_IND_VAL (BASE_EXACT_SEG_NUM - 1L) #define MAX_EXACT_SEG_NUM_VAL (500L - 1L) -#define EXTENT_SIMILAR (MAX_SIMILAR_VAL - BASE_SIMILAR) -#define EXTENT_EXACT_DATE (MAX_EXACT_DATE_VAL - BASE_EXACT_DATE) -#define EXTENT_EXACT_IND (MAX_EXACT_IND_VAL - BASE_EXACT_IND) -#define EXTENT_EXACT_SEG_NUM (MAX_EXACT_SEG_NUM_VAL - BASE_EXACT_SEG_NUM) +#define EXTENT_SIMILAR (MAX_SIMILAR_VAL - BASE_SIMILAR) +#define EXTENT_EXACT_DATE (MAX_EXACT_DATE_VAL - BASE_EXACT_DATE) +#define EXTENT_EXACT_IND (MAX_EXACT_IND_VAL - BASE_EXACT_IND) +#define EXTENT_EXACT_SEG_NUM (MAX_EXACT_SEG_NUM_VAL - BASE_EXACT_SEG_NUM) // translation flag values -#define TRANSLFLAG_NORMAL 0 -#define TRANSLFLAG_MACHINE 1 -#define TRANSLFLAG_GLOBMEM 2 +#define TRANSLFLAG_NORMAL 0 +#define TRANSLFLAG_MACHINE 1 +#define TRANSLFLAG_GLOBMEM 2 #define TRANSLFLAG_GLOBMEMSTAR 3 - /*----------------------------------------------------------------------------*\ System wide threshold values \*----------------------------------------------------------------------------*/ -#define LENGTH_THR 50L // Length threshold value -#define MAX_LENGTH_THR 100L // maximal Length threshold value -#define INIT_MATCH_THR 59L // Initial match threshold value -#define SHORTER_MATCH_THR 50L // match threshold value for shorter - // segments, defined thru LENGTH_SHORTER_VALUE -#define WORDS_MATCH_THR 0L // Not used yet +#define LENGTH_THR 50L // Length threshold value +#define MAX_LENGTH_THR 100L // maximal Length threshold value +#define INIT_MATCH_THR 59L // Initial match threshold value +#define SHORTER_MATCH_THR 50L // match threshold value for shorter + // segments, defined thru LENGTH_SHORTER_VALUE +#define WORDS_MATCH_THR 0L // Not used yet //--- Used in GET_IN structure to specify kind of matches. //--- This defines should be bitwise 'ored' with the number of required matches. @@ -167,61 +161,56 @@ using PTMX_EXT_OUT_W = TMX_EXT_OUT_W *; //--- If you want to retrieve translation memory hits with exact context only, //--- use GET_EXACT_CONTEXT. //--- If Generic replace should be disabled, you have to or with GET_NO_GENERICREPLACE -#define GET_MOREPROP_INDIC 0x00000200 -#define GET_ALL_EXACT_MATCHES 0x00000400 -#define GET_RESPECTCRLF 0x00000800 -#define GET_IGNORE_PATH 0x00001000 -#define GET_NO_GENERICREPLACE 0x00002000 -#define GET_EXACT_AND_CONTEXT 0x00004000 -#define GET_EXACT_AND_FUZZY 0x00008000 -#define GET_ALWAYS_WITH_TAGS 0x00010000 -#define GET_IGNORE_COMMENT 0x00020000 -#define GET_EXACT 0x00040000 - -#define EQUAL_EQUAL 100 -#define TAGS_EQUAL 98 -#define TAGS_UNEQUAL 95 - -#define NON_EXCLUSIVE 1 -#define EXCLUSIVE 0 - -#define LANG_KEY 1L -#define FILE_KEY 2L -#define AUTHOR_KEY 3L -#define TAGTABLE_KEY 4L -//#define RESERVED_KEY 5L -#define LONGNAME_KEY 5L -#define COMPACT_KEY 6L -#define FIRST_KEY 7L +#define GET_MOREPROP_INDIC 0x00000200 +#define GET_ALL_EXACT_MATCHES 0x00000400 +#define GET_RESPECTCRLF 0x00000800 +#define GET_IGNORE_PATH 0x00001000 +#define GET_NO_GENERICREPLACE 0x00002000 +#define GET_EXACT_AND_CONTEXT 0x00004000 +#define GET_EXACT_AND_FUZZY 0x00008000 +#define GET_ALWAYS_WITH_TAGS 0x00010000 +#define GET_IGNORE_COMMENT 0x00020000 +#define GET_EXACT 0x00040000 + +#define EQUAL_EQUAL 100 +#define TAGS_EQUAL 98 +#define TAGS_UNEQUAL 95 + +#define NON_EXCLUSIVE 1 +#define EXCLUSIVE 0 + +#define LANG_KEY 1L +#define FILE_KEY 2L +#define AUTHOR_KEY 3L +#define TAGTABLE_KEY 4L +// #define RESERVED_KEY 5L +#define LONGNAME_KEY 5L +#define COMPACT_KEY 6L +#define FIRST_KEY 7L // Note: the following key is NOT used as the key of a QDAM record // it is only a symbolic value used in the Name-to-ID functions -#define LANGGROUP_KEY 8L - +#define LANGGROUP_KEY 8L // used in GET_OUT structure -#define GET_MORE_EXACTS_AVAIL 0x8000 +#define GET_MORE_EXACTS_AVAIL 0x8000 #define GET_ADDITIONAL_FUZZY_AVAIL 0x4000 - - /* The segment marker */ -#define SEGMARKER "###" - +#define SEGMARKER "###" /**********************************************************************/ /* indicatos for old or new TM */ /**********************************************************************/ -#define OLD_TM 0 -#define NEW_TM 1 +#define OLD_TM 0 +#define NEW_TM 1 /**********************************************************************/ /* indicator for organize */ /* TM_CONVERT - convert an ols TM to an new one */ /* TM_ORGANIZE - organize a new TM */ /**********************************************************************/ -#define TM_CONVERT 0 -#define TM_ORGANIZE 1 - +#define TM_CONVERT 0 +#define TM_ORGANIZE 1 /**********************************************************************/ /* default threshold value for get */ @@ -230,91 +219,92 @@ using PTMX_EXT_OUT_W = TMX_EXT_OUT_W *; // against the new computed fuzziness, the old value is used // for the triple threshold checking /**********************************************************************/ -#define TM_DEFAULT_THRESHOLD 33 // prior: 40 -#define TM_FUZZINESS_THRESHOLD 10 - +#define TM_DEFAULT_THRESHOLD 33 // prior: 40 +#define TM_FUZZINESS_THRESHOLD 10 #include "win_types.h" - - -typedef BYTE ABGROUP [CODEPAGE_SIZE]; -typedef ABGROUP * PABGROUP; - -typedef struct _TM_ADDRSS { /* addr */ - USHORT usEntryInDir, /* >= 0, entry# of the cluster in KeyDir */ - /* <= 4095 */ - usBlockNumber, /* > 0, indicates block number in TM */ - /* 0 used for chaining purposes as null */ - usDispBlockPtr; /* size(block header) <= BLOCK_SIZE */ - /* indicates location within a block */ -} TM_ADDRESS, * PTM_ADDRESS ; - -typedef struct _SEGMENT { /* seg */ - UCHAR achSegMarker[SEG_MARKER_LENGTH] ; - /* marker to beginning of segment */ - USHORT usLenSegment, /* total length of segment */ - usDispIndustry, /* industry codes location in bufData */ - usLenIndustry, /* length of indus. codes list(BYTES) */ - usDispSource, /* source location relative to bufData */ - usLenSource, /* source length in bytes */ - usDispTarget, /* target location relative to bufData */ - usLenTarget, /* target length in bytes */ - usDispReserved, /* Reserved area location in bufData */ - usLenReserved, /* length of reserved area */ - usDispSecKey, /* secondary key location in bufData */ - usLenSecKey, /* length of secondary key */ - usDispContext, /* Context area location in bufData */ - usLenContext; /* length of context area */ - SHORT_FN szFileName; /* name of source file (short name) */ - LONG_FN szLongName; /* name of source file (long name or EOS)*/ - USHORT usSegNumber; /* segment number in file */ - EQF_BOOL fLogicalDel, /* set when logically deleted segment */ - fAscii; /* */ - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - TIME_L tStamp; /* time taken from c function */ - BUFFERIN bufData; /* fields with variable lengths */ -} SEGMENT, * PSEGMENT; - -typedef struct _OLDSEGMENT { /* seg */ - UCHAR achSegMarker[SEG_MARKER_LENGTH] ; - /* marker to beginning of segment */ - USHORT usLenSegment, /* total length of segment */ - usDispIndustry, /* industry codes location in bufData */ - usLenIndustry, /* length of indus. codes list(BYTES) */ - usDispSource, /* source location relative to bufData */ - usLenSource, /* source length in bytes */ - usDispTarget, /* target location relative to bufData */ - usLenTarget, /* target length in bytes */ - usDispReserved, /* Reserved area location in bufData */ - usLenReserved, /* length of reserved area */ - usDispSecKey, /* secondary key location in bufData */ - usLenSecKey; /* length of secondary key */ - SHORT_FN szFileName; /* name of source file (short name) */ - USHORT usSegNumber; /* segment number in file */ - EQF_BOOL fLogicalDel, /* set when logically deleted segment */ - fAscii; /* */ - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - TIME_L tStamp; /* time taken from c function */ - BUFFERIN bufData; /* fields with variable lengths */ -} OLDSEGMENT, * POLDSEGMENT; - -typedef struct _MATCH { /* mtch */ - TM_ADDRESS addr; /* address of the matching segment */ - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - USHORT usNumExactBytes;/* no. of exact bytes in the */ - /* input and matched sentence */ - TIME_L tStamp; /* time stamp of segment */ - LONG lSimLevel, /* <= 100 indicates similar */ - /* >= 200 indicate exact */ - /* >= 300 indicate exact+same ind code */ - /* >= 400 indicate exact+same filename */ - lLengthTest, /* value of lengths test */ - lInitMatchTest, /* value of initials test */ - lWordsTest; /* value of words test */ - /* last 3 values are meaningless for exact matches*/ - UCHAR szTarget[MAX_TGT_LENGTH + 1]; /* The target */ -} MATCH , * PMATCH; +typedef BYTE ABGROUP[CODEPAGE_SIZE]; +typedef ABGROUP *PABGROUP; + +typedef struct _TM_ADDRSS +{ /* addr */ + USHORT usEntryInDir, /* >= 0, entry# of the cluster in KeyDir */ + /* <= 4095 */ + usBlockNumber, /* > 0, indicates block number in TM */ + /* 0 used for chaining purposes as null */ + usDispBlockPtr; /* size(block header) <= BLOCK_SIZE */ + /* indicates location within a block */ +} TM_ADDRESS, *PTM_ADDRESS; + +typedef struct _SEGMENT +{ /* seg */ + UCHAR achSegMarker[SEG_MARKER_LENGTH]; + /* marker to beginning of segment */ + USHORT usLenSegment, /* total length of segment */ + usDispIndustry, /* industry codes location in bufData */ + usLenIndustry, /* length of indus. codes list(BYTES) */ + usDispSource, /* source location relative to bufData */ + usLenSource, /* source length in bytes */ + usDispTarget, /* target location relative to bufData */ + usLenTarget, /* target length in bytes */ + usDispReserved, /* Reserved area location in bufData */ + usLenReserved, /* length of reserved area */ + usDispSecKey, /* secondary key location in bufData */ + usLenSecKey, /* length of secondary key */ + usDispContext, /* Context area location in bufData */ + usLenContext; /* length of context area */ + SHORT_FN szFileName; /* name of source file (short name) */ + LONG_FN szLongName; /* name of source file (long name or EOS)*/ + USHORT usSegNumber; /* segment number in file */ + EQF_BOOL fLogicalDel, /* set when logically deleted segment */ + fAscii; /* */ + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + TIME_L tStamp; /* time taken from c function */ + BUFFERIN bufData; /* fields with variable lengths */ +} SEGMENT, *PSEGMENT; + +typedef struct _OLDSEGMENT +{ /* seg */ + UCHAR achSegMarker[SEG_MARKER_LENGTH]; + /* marker to beginning of segment */ + USHORT usLenSegment, /* total length of segment */ + usDispIndustry, /* industry codes location in bufData */ + usLenIndustry, /* length of indus. codes list(BYTES) */ + usDispSource, /* source location relative to bufData */ + usLenSource, /* source length in bytes */ + usDispTarget, /* target location relative to bufData */ + usLenTarget, /* target length in bytes */ + usDispReserved, /* Reserved area location in bufData */ + usLenReserved, /* length of reserved area */ + usDispSecKey, /* secondary key location in bufData */ + usLenSecKey; /* length of secondary key */ + SHORT_FN szFileName; /* name of source file (short name) */ + USHORT usSegNumber; /* segment number in file */ + EQF_BOOL fLogicalDel, /* set when logically deleted segment */ + fAscii; /* */ + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + TIME_L tStamp; /* time taken from c function */ + BUFFERIN bufData; /* fields with variable lengths */ +} OLDSEGMENT, *POLDSEGMENT; + +typedef struct _MATCH +{ /* mtch */ + TM_ADDRESS addr; /* address of the matching segment */ + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + USHORT usNumExactBytes; /* no. of exact bytes in the */ + /* input and matched sentence */ + TIME_L tStamp; /* time stamp of segment */ + LONG lSimLevel, /* <= 100 indicates similar */ + /* >= 200 indicate exact */ + /* >= 300 indicate exact+same ind code */ + /* >= 400 indicate exact+same filename */ + lLengthTest, /* value of lengths test */ + lInitMatchTest, /* value of initials test */ + lWordsTest; /* value of words test */ + /* last 3 values are meaningless for exact matches*/ + UCHAR szTarget[MAX_TGT_LENGTH + 1]; /* The target */ +} MATCH, *PMATCH; /* More details about the match ranking method. @@ -351,106 +341,117 @@ typedef struct _MATCH { /* mtch */ described both in the LLD document and in code comments. */ -typedef struct _PREFIX_IN { - USHORT usLenIn, /* total length of input buffer */ - idCommand ; /* command id, previously defined. */ -// } PREFIX_IN, * PPREFIX_IN, IN, * PIN; -} PREFIX_IN, * PPREFIX_IN, * PIN; +typedef struct _PREFIX_IN +{ + USHORT usLenIn, /* total length of input buffer */ + idCommand; /* command id, previously defined. */ + // } PREFIX_IN, * PPREFIX_IN, IN, * PIN; +} PREFIX_IN, *PPREFIX_IN, *PIN; -typedef struct _PREFIX_OUT { - USHORT usLenOut; /* total length of output buffer */ - BOOL fDiskFull, /* for Add /Replace / Create when number*/ - fDBfull; /* of blocks is a cluster exceeds */ - /* 2**16 - 1. Filled by FormatMore */ - USHORT rcTmt; /* returned by TMT function */ +typedef struct _PREFIX_OUT +{ + USHORT usLenOut; /* total length of output buffer */ + BOOL fDiskFull, /* for Add /Replace / Create when number*/ + fDBfull; /* of blocks is a cluster exceeds */ + /* 2**16 - 1. Filled by FormatMore */ + USHORT rcTmt; /* returned by TMT function */ -//} PREFIX_OUT, * PPREFIX_OUT, OUT, * POUT; -} PREFIX_OUT, * PPREFIX_OUT, * POUT; + //} PREFIX_OUT, * PPREFIX_OUT, OUT, * POUT; +} PREFIX_OUT, *PPREFIX_OUT, *POUT; -typedef struct _UPDATE_IN { - PREFIX_IN prefin; /* prefix of each command */ - SEGMENT segIn; /* the rest of this type is a SEGMENT */ -} UPDATE_IN, * PUPDATE_IN; +typedef struct _UPDATE_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + SEGMENT segIn; /* the rest of this type is a SEGMENT */ +} UPDATE_IN, *PUPDATE_IN; -typedef UPDATE_IN ADD_IN, DEL_IN, REP_IN; +typedef UPDATE_IN ADD_IN, DEL_IN, REP_IN; typedef PUPDATE_IN PADD_IN, PDEL_IN, PREP_IN; -typedef struct _ADD_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - TM_ADDRESS addr; /* address of added segment */ -} ADD_OUT, * PADD_OUT; - -typedef struct _DEL_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - TM_ADDRESS addr; /* address of deleted segment */ -} DEL_OUT , * PDEL_OUT ; - -typedef struct _REP_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - SERVERNAME szServer; /* server name */ - TM_ADDRESS addrDel, /* address to delete */ - addrAdd; /* address to add */ -} REP_OUT , * PREP_OUT; - -typedef struct _CLOSE_IN { - PREFIX_IN prefin ; /* prefix of each command */ -} CLOSE_IN, * PCLOSE_IN; - -typedef struct _CLOSE_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ -} CLOSE_OUT, * PCLOSE_OUT; - -typedef struct _EXT_IN { - PREFIX_IN prefin; /* prefix of each command */ - TM_ADDRESS addr; /* address of extracted segment */ - USHORT usConvert; // Indicates how the output should be converted -} EXT_IN , * PEXT_IN ; - -typedef struct _EXT_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - SERVERNAME szServer; /* server name */ - TM_ADDRESS addr, /* address of current segment */ - addrNext; /* address of next segment */ - SEGMENT segOut; /* fields with variable lengths */ -} EXT_OUT , * PEXT_OUT; - -typedef struct _GET_IN { - PREFIX_IN prefin; /* prefix of each command */ - SHORT_FN szFileName; /* name of source file (short name) */ - LONG_FN szLongName; /* name of source file (long name or EOS) */ - USHORT usConvert; // Indicates how the output should be converted - USHORT usSegNumber, /* */ - usNumMatchesReq,/* number of matches required */ - usDispIndustry,/* industry codes location in bufData */ - usLenIndustry, /* length of indus. codes list(BYTES) */ - usDispSource, /* source location relative to bufData */ - usLenSource; /* source length in bytes */ - LONG lLengthThr, /* needed to filter out sentences */ - /* with different lengths */ - lInitMatchThr, /* needed to filter out sentences */ - /* with non matching initials */ - lWordsMatchThr;/* needed to filter out sentences */ - /* with non matching words */ - BOOL fAscii; /* */ - BUFFERIN bufData; /* fields with variable lengths */ -} GET_IN, * PGET_IN; +typedef struct _ADD_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + TM_ADDRESS addr; /* address of added segment */ +} ADD_OUT, *PADD_OUT; + +typedef struct _DEL_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + TM_ADDRESS addr; /* address of deleted segment */ +} DEL_OUT, *PDEL_OUT; + +typedef struct _REP_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + SERVERNAME szServer; /* server name */ + TM_ADDRESS addrDel, /* address to delete */ + addrAdd; /* address to add */ +} REP_OUT, *PREP_OUT; + +typedef struct _CLOSE_IN +{ + PREFIX_IN prefin; /* prefix of each command */ +} CLOSE_IN, *PCLOSE_IN; + +typedef struct _CLOSE_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ +} CLOSE_OUT, *PCLOSE_OUT; + +typedef struct _EXT_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + TM_ADDRESS addr; /* address of extracted segment */ + USHORT usConvert; // Indicates how the output should be converted +} EXT_IN, *PEXT_IN; + +typedef struct _EXT_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + SERVERNAME szServer; /* server name */ + TM_ADDRESS addr, /* address of current segment */ + addrNext; /* address of next segment */ + SEGMENT segOut; /* fields with variable lengths */ +} EXT_OUT, *PEXT_OUT; + +typedef struct _GET_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + SHORT_FN szFileName; /* name of source file (short name) */ + LONG_FN szLongName; /* name of source file (long name or EOS) */ + USHORT usConvert; // Indicates how the output should be converted + USHORT usSegNumber, /* */ + usNumMatchesReq, /* number of matches required */ + usDispIndustry, /* industry codes location in bufData */ + usLenIndustry, /* length of indus. codes list(BYTES) */ + usDispSource, /* source location relative to bufData */ + usLenSource; /* source length in bytes */ + LONG lLengthThr, /* needed to filter out sentences */ + /* with different lengths */ + lInitMatchThr, /* needed to filter out sentences */ + /* with non matching initials */ + lWordsMatchThr; /* needed to filter out sentences */ + /* with non matching words */ + BOOL fAscii; /* */ + BUFFERIN bufData; /* fields with variable lengths */ +} GET_IN, *PGET_IN; /**********************************************************************/ /* structure passed in case of rename a file ... */ /**********************************************************************/ typedef struct _RENFILE_IN { - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; // server name - CHAR szOldFile[ MAX_EQF_PATH ]; // old file name - CHAR szNewFile[ MAX_EQF_PATH ]; // new file name - CHAR szUserId [MAX_USERID]; // userId logged on to requester -} RENFILE_IN, * PRENFILE_IN; + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; // server name + CHAR szOldFile[MAX_EQF_PATH]; // old file name + CHAR szNewFile[MAX_EQF_PATH]; // new file name + CHAR szUserId[MAX_USERID]; // userId logged on to requester +} RENFILE_IN, *PRENFILE_IN; typedef struct _RENFILE_OUT { - PREFIX_OUT prefout; /* prefix of each command */ -} RENFILE_OUT, * PRENFILE_OUT; + PREFIX_OUT prefout; /* prefix of each command */ +} RENFILE_OUT, *PRENFILE_OUT; /* More details about lLengthThr and lInitMatchThr: @@ -505,78 +506,84 @@ typedef struct _RENFILE_OUT */ -typedef struct _GET_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - - USHORT usNumMatchesFound,/* number of matches found */ - usNumMatchesValid,/* number of matches valid */ - ausSortedMatches[MAX_MATCH_TAB_ENTRIES]; - MATCH amtchBest[MAX_MATCH_TAB_ENTRIES]; /* Matches array */ -} GET_OUT, * PGET_OUT; - -typedef struct _OPEN_IN { - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* servername */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ - FILENAME szTmFileName; /* name of Tm database to open */ - BOOL fExclusive; /* indicate the mode for opening the file */ - BOOL fOpenGetPart; /* indicate if Open was due to a get part */ -} OPEN_IN , * POPEN_IN; - - -typedef struct _OPEN_OUT { - PREFIX_OUT prefout; /* prefix of Output buffer */ - HTM htm; /* handle to the TM */ -} OPEN_OUT , * POPEN_OUT; - -typedef struct _BLOCK_HEADER { /* bh */ - USHORT usBlockNum, /* >= 1, points to previous block */ - usPrevBlock, /* >= 1, points to previous block */ - usNextBlock, /* > 1, points to next block. */ - /* 0 indicates no chaining */ - usFirstAvailByte; /* >= size of block header, */ - /* <= block size */ - /* points to first available byte in block */ - /* If block is full it sets to BlockSize */ -} BLOCK_HEADER , * PBLOCK_HEADER; - -typedef struct _CREATE_TMH { /* ctmh */ - FILENAME szTmFileName; /* Tm database name */ - USHORT usBlockSize; - LANGUAGE szSourceLang, /* source language */ - szTargetLang; /* target language */ - EQF_BOOL fDbcs; /* double byte char support 0/1 */ +typedef struct _GET_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + + USHORT usNumMatchesFound, /* number of matches found */ + usNumMatchesValid, /* number of matches valid */ + ausSortedMatches[MAX_MATCH_TAB_ENTRIES]; + MATCH amtchBest[MAX_MATCH_TAB_ENTRIES]; /* Matches array */ +} GET_OUT, *PGET_OUT; + +typedef struct _OPEN_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* servername */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ + FILENAME szTmFileName; /* name of Tm database to open */ + BOOL fExclusive; /* indicate the mode for opening the file */ + BOOL fOpenGetPart; /* indicate if Open was due to a get part */ +} OPEN_IN, *POPEN_IN; + +typedef struct _OPEN_OUT +{ + PREFIX_OUT prefout; /* prefix of Output buffer */ + HTM htm; /* handle to the TM */ +} OPEN_OUT, *POPEN_OUT; + +typedef struct _BLOCK_HEADER +{ /* bh */ + USHORT usBlockNum, /* >= 1, points to previous block */ + usPrevBlock, /* >= 1, points to previous block */ + usNextBlock, /* > 1, points to next block. */ + /* 0 indicates no chaining */ + usFirstAvailByte; /* >= size of block header, */ + /* <= block size */ + /* points to first available byte in block */ + /* If block is full it sets to BlockSize */ +} BLOCK_HEADER, *PBLOCK_HEADER; + +typedef struct _CREATE_TMH +{ /* ctmh */ + FILENAME szTmFileName; /* Tm database name */ + USHORT usBlockSize; + LANGUAGE szSourceLang, /* source language */ + szTargetLang; /* target language */ + EQF_BOOL fDbcs; /* double byte char support 0/1 */ } CREATE_TMH; -typedef struct _CREATE_IN { /* crei */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; - CREATE_TMH ctmh; /* parameters that copied into Tm header */ - TIME_L tCreate; /* creation time stamp */ - ABGROUP abABGrouping; - USHORT usDispExclTagList, /* offset of tag list */ - usLenExclTagList, /* length of tag list */ - usDispExclWordList,/* offset of word list */ - usLenExclWordList; /* length of word list */ - CHAR bufData[CREATE_BUFFER_SIZE]; - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} CREATE_IN , * PCREATE_IN; +typedef struct _CREATE_IN +{ /* crei */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; + CREATE_TMH ctmh; /* parameters that copied into Tm header */ + TIME_L tCreate; /* creation time stamp */ + ABGROUP abABGrouping; + USHORT usDispExclTagList, /* offset of tag list */ + usLenExclTagList, /* length of tag list */ + usDispExclWordList, /* offset of word list */ + usLenExclWordList; /* length of word list */ + CHAR bufData[CREATE_BUFFER_SIZE]; + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} CREATE_IN, *PCREATE_IN; typedef OPEN_OUT CREATE_OUT, *PCREATE_OUT; -typedef CLOSE_IN INFO_IN, * PINFO_IN; - -typedef struct _INFO_OUT { /* infi */ - PREFIX_OUT prefout; /* prefix of each command */ - CREATE_TMH ctmh; /* parameters that copied into Tm header */ - TIME_L tCreate; /* creation time stamp */ - ABGROUP abABGrouping; - USHORT usDispExclTagList, /* offset of tag list */ - usLenExclTagList, /* length of tag list */ - usDispExclWordList,/* offset of word list */ - usLenExclWordList; /* length of word list */ - CHAR bufData[CREATE_BUFFER_SIZE]; -} INFO_OUT , * PINFO_OUT; +typedef CLOSE_IN INFO_IN, *PINFO_IN; + +typedef struct _INFO_OUT +{ /* infi */ + PREFIX_OUT prefout; /* prefix of each command */ + CREATE_TMH ctmh; /* parameters that copied into Tm header */ + TIME_L tCreate; /* creation time stamp */ + ABGROUP abABGrouping; + USHORT usDispExclTagList, /* offset of tag list */ + usLenExclTagList, /* length of tag list */ + usDispExclWordList, /* offset of word list */ + usLenExclWordList; /* length of word list */ + CHAR bufData[CREATE_BUFFER_SIZE]; +} INFO_OUT, *PINFO_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_GET_SERVER_DRIVES command which lists all @@ -586,49 +593,52 @@ typedef struct _INFO_OUT { /* infi */ // any of the secondary drives) // ---------------------------------------------------------------------------- -typedef struct _DRIVES_IN { /* indr */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} DRIVES_IN, * PDRIVES_IN; - -typedef struct _DRIVE_INFO { /* drin */ - CHAR cDriveLetter; /* drive letter of disk drive */ - ULONG ulFreeSpace; /* number of bytes left on this disk drive */ -} DRIVE_INFO, * PDRIVE_INFO; - -typedef struct _DRIVES_OUT { /* outdr */ - PREFIX_OUT prefout; /* prefix of each command */ - USHORT usValidDrives; /* how much drives have been */ - /* found on the server ? */ - DRIVE_INFO adrinDrives [MAX_DRIVELIST]; /* info for each valid */ - /* server drive */ -} DRIVES_OUT, * PDRIVES_OUT; - +typedef struct _DRIVES_IN +{ /* indr */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} DRIVES_IN, *PDRIVES_IN; + +typedef struct _DRIVE_INFO +{ /* drin */ + CHAR cDriveLetter; /* drive letter of disk drive */ + ULONG ulFreeSpace; /* number of bytes left on this disk drive */ +} DRIVE_INFO, *PDRIVE_INFO; + +typedef struct _DRIVES_OUT +{ /* outdr */ + PREFIX_OUT prefout; /* prefix of each command */ + USHORT usValidDrives; /* how much drives have been */ + /* found on the server ? */ + DRIVE_INFO adrinDrives[MAX_DRIVELIST]; /* info for each valid */ + /* server drive */ +} DRIVES_OUT, *PDRIVES_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_GET_SERVER_TM_LIST command which lists all // translation memories that have been found on the selected server; for each // TM its full path name and its current file size is returned // ---------------------------------------------------------------------------- -typedef struct _FILE_LIST_IN { /* intl !!! CHM */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} FILE_LIST_IN, * PFILE_LIST_IN; // !!! CHM - -typedef struct _FILE_INFO { /* tmin !!! CHM */ - CHAR szPathFileName [MAX_EQF_PATH]; /* full filename of file */ - ULONG ulFileSize; /* filesize in bytes */ -} FILE_INFO, * PFILE_INFO; // !!! CHM - - -typedef struct _FILE_LIST_OUT { /* outtl !!! CHM */ - PREFIX_OUT prefout; /* prefix of each command */ - USHORT usValidFiles; /* valid entries in array */ - FILE_INFO aflinFileList [MAX_TM_LIST_NUMBER]; /* info for avail. files */ -} FILE_LIST_OUT, * PFILE_LIST_OUT; // !!! CHM - +typedef struct _FILE_LIST_IN +{ /* intl !!! CHM */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} FILE_LIST_IN, *PFILE_LIST_IN; // !!! CHM + +typedef struct _FILE_INFO +{ /* tmin !!! CHM */ + CHAR szPathFileName[MAX_EQF_PATH]; /* full filename of file */ + ULONG ulFileSize; /* filesize in bytes */ +} FILE_INFO, *PFILE_INFO; // !!! CHM + +typedef struct _FILE_LIST_OUT +{ /* outtl !!! CHM */ + PREFIX_OUT prefout; /* prefix of each command */ + USHORT usValidFiles; /* valid entries in array */ + FILE_INFO aflinFileList[MAX_TM_LIST_NUMBER]; /* info for avail. files */ +} FILE_LIST_OUT, *PFILE_LIST_OUT; // !!! CHM // ---------------------------------------------------------------------------- // structures needed for the TMC_GET_PART_OF_TM_FILE command that transfers @@ -639,64 +649,64 @@ typedef struct _FILE_LIST_OUT { /* outtl !!! CHM */ // maximum size to transfer at a time is GETPART_BUFFER_SIZE // ---------------------------------------------------------------------------- - -typedef struct _GETPART_IN { /* ingp */ - PREFIX_IN prefin; /* prefix of each command */ - ULONG ulFilePos; /* file position of first byte to transfer */ - ULONG ulBytesToRead; /* number of bytes to transfer */ -} GETPART_IN, * PGETPART_IN; - -typedef struct _GETPART_OUT { /* outgp */ - PREFIX_OUT prefout; /* prefix of each command */ - ULONG ulBytesRead; /* number of bytes actually read */ - ULONG ulNextFilePos; /* file position of next byte to read */ - UCHAR aucOutBuffer [GETPART_BUFFER_SIZE]; /* output buffer */ -} GETPART_OUT, * PGETPART_OUT; - -typedef struct _GETDICTPART_IN { - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - PVOID pBTree ; // ptr to dict file handles - ULONG ulFilePos; /* file position of first byte to transfer */ - ULONG ulBytesToRead; /* number of bytes to transfer */ -} GETDICTPART_IN, * PGETDICTPART_IN; - - -#define MAX_DICT_DESCR 40 // max length of a dictionary description +typedef struct _GETPART_IN +{ /* ingp */ + PREFIX_IN prefin; /* prefix of each command */ + ULONG ulFilePos; /* file position of first byte to transfer */ + ULONG ulBytesToRead; /* number of bytes to transfer */ +} GETPART_IN, *PGETPART_IN; + +typedef struct _GETPART_OUT +{ /* outgp */ + PREFIX_OUT prefout; /* prefix of each command */ + ULONG ulBytesRead; /* number of bytes actually read */ + ULONG ulNextFilePos; /* file position of next byte to read */ + UCHAR aucOutBuffer[GETPART_BUFFER_SIZE]; /* output buffer */ +} GETPART_OUT, *PGETPART_OUT; + +typedef struct _GETDICTPART_IN +{ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + PVOID pBTree; // ptr to dict file handles + ULONG ulFilePos; /* file position of first byte to transfer */ + ULONG ulBytesToRead; /* number of bytes to transfer */ +} GETDICTPART_IN, *PGETDICTPART_IN; + +#define MAX_DICT_DESCR 40 // max length of a dictionary description // Structure of EQF dictionary properties typedef struct _PROPDICTIONARY { - //--- common property part --- - PROPHEAD PropHead; // header of properties - //--- general dictionary information --- - CHAR szDescription[MAX_DICT_DESCR]; // dictionary description - CHAR szSourceLang[MAX_LANG_LENGTH]; // dictionary source language - // dictionary target language(s) - CHAR szTargetLang[MAX_TGT_LANG][MAX_LANG_LENGTH]; - EQF_BOOL fCopyRight; // dictionary-is-copyrighted flag - CHAR szDictPath[MAX_EQF_PATH]; // fully qualified dict file name - CHAR szIndexPath[MAX_EQF_PATH]; // fully qualified index file name - //--- dictionary profile --- - USHORT usLength; // number of user profile entries - USHORT usUserNameCount; // number of user created dict names - COLFONT ColFontDictEntry; // colour/font for dict entry - COLFONT ColFontEntryVal; // colour/font for dict entry value - PROFENTRY ProfEntry[MAX_PROF_ENTRIES]; // user profile entries - EQF_BOOL fProtected; // dictionary-is-protected flag - ULONG ulPassWord; - CHAR szServer[MAX_SERVER_NAME]; // Server Name of TM or \0 if TM is local - CHAR szUserid[MAX_USERID]; // LAN Userid of TM: if local '\0' - CHAR chRemPrimDrive; // LAN primary drive - USHORT usLocation; // location of dictionary - CHAR szLongName[MAX_LONGFILESPEC]; // dictionary long (descriptive) name - CHAR szLongDesc[MAX_LONG_DESCRIPTION]; // dictionary long (descriptive) name - USHORT usVersion; // Version of dictionary - used for - // comparison during organize if copyrighted - //--- reserved space --- - CHAR chReserve[3402]; // reserve space / filler + //--- common property part --- + PROPHEAD PropHead; // header of properties + //--- general dictionary information --- + CHAR szDescription[MAX_DICT_DESCR]; // dictionary description + CHAR szSourceLang[MAX_LANG_LENGTH]; // dictionary source language + // dictionary target language(s) + CHAR szTargetLang[MAX_TGT_LANG][MAX_LANG_LENGTH]; + EQF_BOOL fCopyRight; // dictionary-is-copyrighted flag + CHAR szDictPath[MAX_EQF_PATH]; // fully qualified dict file name + CHAR szIndexPath[MAX_EQF_PATH]; // fully qualified index file name + //--- dictionary profile --- + USHORT usLength; // number of user profile entries + USHORT usUserNameCount; // number of user created dict names + COLFONT ColFontDictEntry; // colour/font for dict entry + COLFONT ColFontEntryVal; // colour/font for dict entry value + PROFENTRY ProfEntry[MAX_PROF_ENTRIES]; // user profile entries + EQF_BOOL fProtected; // dictionary-is-protected flag + ULONG ulPassWord; + CHAR szServer[MAX_SERVER_NAME]; // Server Name of TM or \0 if TM is local + CHAR szUserid[MAX_USERID]; // LAN Userid of TM: if local '\0' + CHAR chRemPrimDrive; // LAN primary drive + USHORT usLocation; // location of dictionary + CHAR szLongName[MAX_LONGFILESPEC]; // dictionary long (descriptive) name + CHAR szLongDesc[MAX_LONG_DESCRIPTION]; // dictionary long (descriptive) name + USHORT usVersion; // Version of dictionary - used for + // comparison during organize if copyrighted + //--- reserved space --- + CHAR chReserve[3402]; // reserve space / filler } PROPDICTIONARY, *PPROPDICTIONARY; - // ---------------------------------------------------------------------------- // structures needed for the TMC_PUT_TM_PROPERTIES command which is used to // copy a property file for a remote TM to the server (during TM creation @@ -707,37 +717,39 @@ typedef struct _PROPDICTIONARY // the drive and path) which is then used to create a filename for the property // file (same drive and path but new extension (will be .PRP) // ---------------------------------------------------------------------------- -typedef struct _PUTPROP_IN { /* inpp */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szPathFileName [MAX_EQF_PATH]; /* full filename */ - ULONG ulPropLength; /* length of property file */ - UCHAR aucInBuffer [MEM_PROP_SIZE]; /* buffer for file data */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} PUTPROP_IN, * PPUTPROP_IN; +typedef struct _PUTPROP_IN +{ /* inpp */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szPathFileName[MAX_EQF_PATH]; /* full filename */ + ULONG ulPropLength; /* length of property file */ + UCHAR aucInBuffer[MEM_PROP_SIZE]; /* buffer for file data */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} PUTPROP_IN, *PPUTPROP_IN; // !!! CHM Start -typedef struct _PUTDICTPROP_IN { /* inpp */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szPathFileName [MAX_EQF_PATH]; /* full filename */ - ULONG ulPropLength; /* length of property file */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ - PROPDICTIONARY DictProp; /* dictionary property buffer */ -} PUTDICTPROP_IN, * PPUTDICTPROP_IN; +typedef struct _PUTDICTPROP_IN +{ /* inpp */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szPathFileName[MAX_EQF_PATH]; /* full filename */ + ULONG ulPropLength; /* length of property file */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ + PROPDICTIONARY DictProp; /* dictionary property buffer */ +} PUTDICTPROP_IN, *PPUTDICTPROP_IN; /* define union to allow combined processing in server code */ typedef union _PUTPROPCOMBINEDIN { - PUTPROP_IN TmPropIn; - PUTDICTPROP_IN DictPropIn; -} PUTPROPCOMBINED_IN, * PPUTPROPCOMBINED_IN; + PUTPROP_IN TmPropIn; + PUTDICTPROP_IN DictPropIn; +} PUTPROPCOMBINED_IN, *PPUTPROPCOMBINED_IN; // !!! CHM End -typedef struct _PUTPROP_OUT { /* outpp */ - PREFIX_OUT prefout; /* prefix of each command */ -} PUTPROP_OUT, * PPUTPROP_OUT; - +typedef struct _PUTPROP_OUT +{ /* outpp */ + PREFIX_OUT prefout; /* prefix of each command */ +} PUTPROP_OUT, *PPUTPROP_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_GET_TM_PROPERTIES command which is used to @@ -750,29 +762,31 @@ typedef struct _PUTPROP_OUT { /* outpp // property file which is stored on the same disk in the same path with the // extension .PRP // ---------------------------------------------------------------------------- -typedef struct _GETPROP_IN { /* ingp */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szPathFileName [MAX_EQF_PATH]; /* full filename */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} GETPROP_IN, * PGETPROP_IN; - -typedef struct _GETPROP_OUT { /* outgp */ - PREFIX_OUT prefout; /* prefix of each command */ - ULONG ulPropLength; /* length of property file */ - UCHAR aucOutBuffer [MEM_PROP_SIZE]; /* buffer for file data */ -} GETPROP_OUT, * PGETPROP_OUT; +typedef struct _GETPROP_IN +{ /* ingp */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szPathFileName[MAX_EQF_PATH]; /* full filename */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} GETPROP_IN, *PGETPROP_IN; + +typedef struct _GETPROP_OUT +{ /* outgp */ + PREFIX_OUT prefout; /* prefix of each command */ + ULONG ulPropLength; /* length of property file */ + UCHAR aucOutBuffer[MEM_PROP_SIZE]; /* buffer for file data */ +} GETPROP_OUT, *PGETPROP_OUT; // !!! CHM Start -typedef struct _GETDICTPROP_OUT { /* outgp */ - PREFIX_OUT prefout; /* prefix of each command */ - ULONG ulPropLength; /* length of property file */ - PROPDICTIONARY DictProp; /* dictionary property buffer*/ -} GETDICTPROP_OUT, * PGETDICTPROP_OUT; +typedef struct _GETDICTPROP_OUT +{ /* outgp */ + PREFIX_OUT prefout; /* prefix of each command */ + ULONG ulPropLength; /* length of property file */ + PROPDICTIONARY DictProp; /* dictionary property buffer*/ +} GETDICTPROP_OUT, *PGETDICTPROP_OUT; /* define union to allow combined processing in server code */ - // ---------------------------------------------------------------------------- // structures needed for the TMC_END_ORGANIZE command which is used to end a // running 'Organize' process; this command is sent to the original TM instead @@ -782,16 +796,17 @@ typedef struct _GETDICTPROP_OUT { /* outgp * // original TM file to be deleted and the temporary file to be renamed to the // original TM filename; // ---------------------------------------------------------------------------- -typedef struct _ENDORG_IN { /* ineo */ - PREFIX_IN prefin; /* prefix of each command */ - CHAR szOrgTM [MAX_EQF_PATH];/* full filename of original TM file */ - CHAR szTmpTM [MAX_EQF_PATH];/* full filename of temporary TM file */ -} ENDORG_IN, * PENDORG_IN; - -typedef struct _ENDORG_OUT { /* outeo */ - PREFIX_OUT prefout; /* prefix of each command */ -} ENDORG_OUT, * PENDORG_OUT; - +typedef struct _ENDORG_IN +{ /* ineo */ + PREFIX_IN prefin; /* prefix of each command */ + CHAR szOrgTM[MAX_EQF_PATH]; /* full filename of original TM file */ + CHAR szTmpTM[MAX_EQF_PATH]; /* full filename of temporary TM file */ +} ENDORG_IN, *PENDORG_IN; + +typedef struct _ENDORG_OUT +{ /* outeo */ + PREFIX_OUT prefout; /* prefix of each command */ +} ENDORG_OUT, *PENDORG_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_DELETE_TM command which is used to physical @@ -799,16 +814,17 @@ typedef struct _ENDORG_OUT { /* outeo */ // before this command can be submitted, the file has to opened exclusively; // the supplied filename must contain the full drive and path information; // ---------------------------------------------------------------------------- -typedef struct _DELTM_IN { /* indt */ - PREFIX_IN prefin; /* prefix of each command */ - CHAR szTMPathFileName [MAX_EQF_PATH]; /* full filename of TM file */ - CHAR szPropPathFileName[MAX_EQF_PATH]; /* full filename of Prop file */ -} DELTM_IN, * PDELTM_IN; - -typedef struct _DELTM_OUT { /* outdt */ - PREFIX_OUT prefout; /* prefix of each command */ -} DELTM_OUT, * PDELTM_OUT; - +typedef struct _DELTM_IN +{ /* indt */ + PREFIX_IN prefin; /* prefix of each command */ + CHAR szTMPathFileName[MAX_EQF_PATH]; /* full filename of TM file */ + CHAR szPropPathFileName[MAX_EQF_PATH]; /* full filename of Prop file */ +} DELTM_IN, *PDELTM_IN; + +typedef struct _DELTM_OUT +{ /* outdt */ + PREFIX_OUT prefout; /* prefix of each command */ +} DELTM_OUT, *PDELTM_OUT; // ---------------------------------------------------------------------------- // structures needed for the TMC_DELETE_FILE command which is used to physical @@ -817,32 +833,34 @@ typedef struct _DELTM_OUT { /* outdt */ // a previous organize process that died during execution); // the supplied filename must contain the full drive and path information; // ---------------------------------------------------------------------------- -typedef struct _DELFILE_IN { /* indf */ - PREFIX_IN prefin; /* prefix of each command */ - SERVERNAME szServer; /* which server ? */ - CHAR szFileName [MAX_EQF_PATH]; /* full filename of file to delete*/ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester */ -} DELFILE_IN, * PDELFILE_IN; - -typedef struct _DELFILE_OUT { /* outdf */ - PREFIX_OUT prefout; /* prefix of each command */ -} DELFILE_OUT, * PDELFILE_OUT; - -typedef struct _CLOSEHANDLER_IN +typedef struct _DELFILE_IN +{ /* indf */ + PREFIX_IN prefin; /* prefix of each command */ + SERVERNAME szServer; /* which server ? */ + CHAR szFileName[MAX_EQF_PATH]; /* full filename of file to delete*/ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester */ +} DELFILE_IN, *PDELFILE_IN; + +typedef struct _DELFILE_OUT +{ /* outdf */ + PREFIX_OUT prefout; /* prefix of each command */ +} DELFILE_OUT, *PDELFILE_OUT; + +typedef struct _CLOSEHANDLER_IN { - PREFIX_IN prefin; - USHORT hth; -} CLOSEHANDLER_IN, * PCLOSEHANDLER_IN; + PREFIX_IN prefin; + USHORT hth; +} CLOSEHANDLER_IN, *PCLOSEHANDLER_IN; -typedef struct _EXIT_IN +typedef struct _EXIT_IN { - PREFIX_IN prefin; -} EXIT_IN, * PEXIT_IN; + PREFIX_IN prefin; +} EXIT_IN, *PEXIT_IN; -typedef struct _EXIT_OUT +typedef struct _EXIT_OUT { - PREFIX_OUT prefout; -} EXIT_OUT, * PEXIT_OUT; + PREFIX_OUT prefout; +} EXIT_OUT, *PEXIT_OUT; // ---------------------------------------------------------------------------- // data structure used for the TMC_QUERY_FILE_INFO which returns the file info @@ -850,61 +868,60 @@ typedef struct _EXIT_OUT // server // ---------------------------------------------------------------------------- -typedef struct _FILEINFO_IN { /* infi */ - PREFIX_IN prefin; /* prefix of each command */ - CHAR szFileName [MAX_EQF_PATH]; /* full filename of file to query */ - SERVERNAME szServer; /* which server ? */ - CHAR szUserId [MAX_USERID]; /* userId logged on to requester !!!! CHM */ -} FILEINFO_IN, * PFILEINFO_IN; - -typedef struct _FILEINFO_OUT { /* outfi */ - PREFIX_OUT prefout; /* prefix of each command */ - FILEFINDBUF stFile; /* info about file */ -} FILEINFO_OUT, * PFILEINFO_OUT; - - -typedef struct _TM_HEADER { /* tmh */ -/* fixed part */ - UCHAR achTmPrefix[TM_PREFIX_SIZE]; - /* should be initiated to EQFTMT$$ 8 */ - USHORT usTmHeaderSize, /* actual length of tm header */ - usDbVersion; - LONG ldispFirstBlock; /* disp of first block from */ - /* beginning of TM */ - CREATE_TMH ctmh; /* parameters provided by the create */ - TIME_L tCreate; /* creation time stamp */ - ABGROUP abABGrouping; - -/* updated part */ - EQF_BOOL fCorruption; - USHORT usNumTMBlocks, /* number of blocks in the TM */ - usNumFreeBlocks, /* # of free, pre-formatted blocks */ - usFirstAvailBlock,/* points to first available block */ - ausKeyDirectory[KEY_DIR_SIZE], /* key directory */ - - usDispExclTagList, /* offset of tag list */ - usLenExclTagList, /* length of tag list */ - usDispExclWordList,/* offset of word list */ - usLenExclWordList; /* length of word list */ - -} TM_HEADER, * PTM_HEADER, ** PPTM_HEADER; - -#define TAG_LEN 35 -#define MAX_NAME 8 -#define MAX_RANDOM 20 -//#define MAX_VOTES 20 // 30 -#define MAX_VOTES 30 // change 27.2.2000 -#define ABS_VOTES 400 -#define MAX_MATCHES 15 -#define TOK_SIZE 4000 -#define TMX_REC_SIZE 32760 -#define TMX_TABLE_SIZE 512 -#define MAX_INDEX_LEN 8150 // 2048 - - - - - // name table structure (TM Version 1 - 4) +typedef struct _FILEINFO_IN +{ /* infi */ + PREFIX_IN prefin; /* prefix of each command */ + CHAR szFileName[MAX_EQF_PATH]; /* full filename of file to query */ + SERVERNAME szServer; /* which server ? */ + CHAR szUserId[MAX_USERID]; /* userId logged on to requester !!!! CHM */ +} FILEINFO_IN, *PFILEINFO_IN; + +typedef struct _FILEINFO_OUT +{ /* outfi */ + PREFIX_OUT prefout; /* prefix of each command */ + FILEFINDBUF stFile; /* info about file */ +} FILEINFO_OUT, *PFILEINFO_OUT; + +typedef struct _TM_HEADER +{ /* tmh */ + /* fixed part */ + UCHAR achTmPrefix[TM_PREFIX_SIZE]; + /* should be initiated to EQFTMT$$ 8 */ + USHORT usTmHeaderSize, /* actual length of tm header */ + usDbVersion; + LONG ldispFirstBlock; /* disp of first block from */ + /* beginning of TM */ + CREATE_TMH ctmh; /* parameters provided by the create */ + TIME_L tCreate; /* creation time stamp */ + ABGROUP abABGrouping; + + /* updated part */ + EQF_BOOL fCorruption; + USHORT usNumTMBlocks, /* number of blocks in the TM */ + usNumFreeBlocks, /* # of free, pre-formatted blocks */ + usFirstAvailBlock, /* points to first available block */ + ausKeyDirectory[KEY_DIR_SIZE], /* key directory */ + + usDispExclTagList, /* offset of tag list */ + usLenExclTagList, /* length of tag list */ + usDispExclWordList, /* offset of word list */ + usLenExclWordList; /* length of word list */ + +} TM_HEADER, *PTM_HEADER, **PPTM_HEADER; + +#define TAG_LEN 35 +#define MAX_NAME 8 +#define MAX_RANDOM 20 +// #define MAX_VOTES 20 // 30 +#define MAX_VOTES 30 // change 27.2.2000 +#define ABS_VOTES 400 +#define MAX_MATCHES 15 +#define TOK_SIZE 4000 +#define TMX_REC_SIZE 32760 +#define TMX_TABLE_SIZE 512 +#define MAX_INDEX_LEN 8150 // 2048 + +// name table structure (TM Version 1 - 4) // typedef struct _TMX_VER1_TABLE // { // USHORT usAllocSize; @@ -912,8 +929,6 @@ typedef struct _TM_HEADER { /* tmh */ // TMX_TABLE_ENTRY stTmTableEntry; // } TMX_VER1_TABLE, * PTMX_VER1_TABLE; - - /**********************************************************************/ /* Defines and structures for the long document name support */ /**********************************************************************/ @@ -925,67 +940,67 @@ typedef struct _TM_HEADER { /* tmh */ // and increment for table enlargements #define LONGNAMETABLE_ENTRIES 32 - - - -//complete entry id tm data file +// complete entry id tm data file typedef struct _TMX_RECORD { - LONG lRecordLen; - USHORT usSourceRecord; - USHORT usFirstTargetRecord; -} TMX_RECORD, * PTMX_RECORD; + LONG lRecordLen; + USHORT usSourceRecord; + USHORT usFirstTargetRecord; +} TMX_RECORD, *PTMX_RECORD; + + -//structure of the source segment +// structure of the source segment typedef struct _TMX_SOURCE_RECORD { - LONG lRecordLen; - USHORT usSource; + LONG lRecordLen; + USHORT usSource; USHORT usLangId; - void reset(){ + void reset() + { memset(this, 0, sizeof(*this)); } -} TMX_SOURCE_RECORD, * PTMX_SOURCE_RECORD; +} TMX_SOURCE_RECORD, *PTMX_SOURCE_RECORD; -//structure of the target segment +// structure of the target segment typedef struct _TMX_TARGET_RECORD { - LONG lRecordLen = 0; - //USHORT usSourceTagTable; - //USHORT usTargetTagTable; - USHORT usTarget = 0; - USHORT usClb = 0; - void reset(){ + LONG lRecordLen = 0; + // USHORT usSourceTagTable; + // USHORT usTargetTagTable; + USHORT usTarget = 0; + USHORT usClb = 0; + void reset() + { memset(this, 0, sizeof(*this)); } -} TMX_TARGET_RECORD, * PTMX_TARGET_RECORD; +} TMX_TARGET_RECORD, *PTMX_TARGET_RECORD; - -//control block structure in target record +// control block structure in target record typedef struct _TMX_TARGET_CLB -{ - TIME_L lTime; - TIME_L lUpdateTime; - ULONG ulSegmId; - USHORT usLangId; - USHORT usFileId; - USHORT usAuthorId; - USHORT usAddDataLen; // new for Major_version6: Length of following context and additional info data - BYTE bMultiple; - BYTE bTranslationFlag; -} TMX_TARGET_CLB, * PTMX_TARGET_CLB; +{ + TIME_L lTime; + TIME_L lUpdateTime; + ULONG ulSegmId; + USHORT usLangId; + USHORT usFileId; + USHORT usAuthorId; + USHORT usAddDataLen; // new for Major_version6: Length of following context and additional info data + BYTE bMultiple; + BYTE bTranslationFlag; +} TMX_TARGET_CLB, *PTMX_TARGET_CLB; // helper macros for working with TMX_TARGET_CLBs -#define TARGETCLBLEN( pClb ) (sizeof(TMX_TARGET_CLB) + pClb->usAddDataLen) +#define TARGETCLBLEN(pClb) (sizeof(TMX_TARGET_CLB) + pClb->usAddDataLen) -#define NEXTTARGETCLB( pClb ) ((PTMX_TARGET_CLB)(((PBYTE)pClb) + TARGETCLBLEN(pClb))) +#define NEXTTARGETCLB(pClb) ((PTMX_TARGET_CLB)(((PBYTE)pClb) + TARGETCLBLEN(pClb))) -#define PCONTEXTFROMCLB( pClb ) ((PSZ_W)(((PBYTE)pClb)+sizeof(TMX_TARGET_CLB))) +#define PCONTEXTFROMCLB(pClb) ((PSZ_W)(((PBYTE)pClb) + sizeof(TMX_TARGET_CLB))) // max size of additional data area // (currently 2 * MAX_SEGMENT_SIZE for context and additional info // plus size of three identifierrs (USHORT) and two size fields (USHORT) ) -#define MAX_ADD_DATA_LEN ((2 * MAX_SEGMENT_SIZE)*sizeof(WCHAR) + (5 * sizeof(USHORT))) +#define MAX_ADD_DATA_LEN ((2 * MAX_SEGMENT_SIZE) * sizeof(WCHAR) + (5 * sizeof(USHORT))) // identifier for additional info data in additional data area #define ADDDATA_ADDINFO_ID 1 @@ -997,58 +1012,67 @@ typedef struct _TMX_TARGET_CLB #define ADDDATA_ENDOFDATA_ID 9 // compute the size of the additional data for the given input -USHORT NTMComputeAddDataSize( PSZ_W pszContext, PSZ_W pszAddInfo ); +USHORT NTMComputeAddDataSize(PSZ_W pszContext, PSZ_W pszAddInfo); // get length of specific data in the combined data area, returns length of data area -USHORT NtmGetAddDataLen( PTMX_TARGET_CLB pCLB, USHORT usDataID ); +USHORT NtmGetAddDataLen(PTMX_TARGET_CLB pCLB, USHORT usDataID); // store/combine additional data in the combined area, returns new size of combined data area or 0 in case of errors -USHORT NtmStoreAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszAddData ); +USHORT NtmStoreAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszAddData); // retrieve specific data from the combined data area, returns length of retrieved data (incl. string end delimiter) -USHORT NtmGetAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszBuffer, USHORT usBufSize ); +USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, std::wstring& pszBuffer); // find a string in a specific data area -BOOL NtmFindInAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch ); +BOOL NtmFindInAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch); -//tag table structure for both source and target +// tag table structure for both source and target typedef struct _TMX_TAGTABLE_RECORD { - LONG lRecordLen; - USHORT usTagTableId; - USHORT usFirstTagEntry; -} TMX_TAGTABLE_RECORD, * PTMX_TAGTABLE_RECORD; - + LONG lRecordLen; + USHORT usTagTableId; + USHORT usFirstTagEntry; +} TMX_TAGTABLE_RECORD, *PTMX_TAGTABLE_RECORD; -//individual tag entry in tag table record +// individual tag entry in tag table record typedef struct _TMX_TAGENTRY { - USHORT usOffset; - USHORT usTagLen; - BYTE bData; -} TMX_TAGENTRY, * PTMX_TAGENTRY; + USHORT usOffset; + USHORT usTagLen; + BYTE bData; +} TMX_TAGENTRY, *PTMX_TAGENTRY; // macros to access certain TM records fields - #define RECLEN(pRec) pRec->lRecordLen - +#define RECLEN(pRec) pRec->lRecordLen +#define RECLENB(pRec) toTmxRecord(pRec)->lRecordLen -#define NTMVOTES(l) ((BYTE)(((ULONG)(l) >> 24) & 0xFF)) -#define NTMKEY(l) ((ULONG)(l) & 0xFFFFFF) -#define NTMINDEX(b,l) ((ULONG)(((l) & 0xFFFFFF) | (((ULONG)(b)) << 24 ))) +#define NTMVOTES(l) ((BYTE)(((ULONG)(l) >> 24) & 0xFF)) +#define NTMKEY(l) ((ULONG)(l) & 0xFFFFFF) +#define NTMINDEX(b, l) ((ULONG)(((l) & 0xFFFFFF) | (((ULONG)(b)) << 24))) -typedef ULONG TMX_INDEX_ENTRY, *PTMX_INDEX_ENTRY; +typedef ULONG TMX_INDEX_ENTRY, *PTMX_INDEX_ENTRY; typedef struct _TMX_INDEX_RECORD { - USHORT usRecordLen; - TMX_INDEX_ENTRY stIndexEntry; -} TMX_INDEX_RECORD, * PTMX_INDEX_RECORD; + USHORT usRecordLen; + TMX_INDEX_ENTRY stIndexEntry; +} TMX_INDEX_RECORD, *PTMX_INDEX_RECORD; +inline TMX_RECORD *toTmxRecord(std::vector &buffer) +{ + return reinterpret_cast(buffer.data()); +} + +inline TMX_INDEX_RECORD *toIndexRecord(std::vector &buffer) +{ + return reinterpret_cast(buffer.data()); +} typedef struct _TMX_TERM_TOKEN { - USHORT usOffset; - USHORT usLength; - USHORT usHash; + USHORT usOffset; + USHORT usLength; + USHORT usHash; + void reset() { memset(this, 0, sizeof(*this)); } } TMX_TERM_TOKEN, *PTMX_TERM_TOKEN; typedef struct _TMX_MATCHENTRY @@ -1057,13 +1081,13 @@ typedef struct _TMX_MATCHENTRY USHORT usMaxVotes; USHORT usMatchVotes; BYTE cCount; -} TMX_MATCHENTRY, * PTMX_MATCHENTRY; +} TMX_MATCHENTRY, *PTMX_MATCHENTRY; -//#pragma pack(show) -//#pragma pack(push, 8) -//#pragma pack(8) +// #pragma pack(show) +// #pragma pack(push, 8) +// #pragma pack(8) -#include +#include #include #include @@ -1073,140 +1097,159 @@ typedef struct _TMX_MATCHENTRY #include #include - class ImportStatusDetails; - // IDs of TMX elelements typedef enum { - UNKNOWN_ELEMENT =-1, - TMX_ELEMENT = 1, - PROP_ELEMENT = 2, - HEADER_ELEMENT = 3, - TU_ELEMENT = 4, - TUV_ELEMENT = 5, - BODY_ELEMENT = 6, - SEG_ELEMENT = 7, - SEGMENT_TAGS = 8, + UNKNOWN_ELEMENT = -1, + TMX_ELEMENT = 1, + PROP_ELEMENT = 2, + HEADER_ELEMENT = 3, + TU_ELEMENT = 4, + TUV_ELEMENT = 5, + BODY_ELEMENT = 6, + SEG_ELEMENT = 7, + SEGMENT_TAGS = 8, BEGIN_INLINE_TAGS = 10, - //pair tags + // pair tags BEGIN_PAIR_TAGS = 10, - BPT_ELEMENT = 10, - EPT_ELEMENT = 11, - G_ELEMENT = 12, - HI_ELEMENT = 13, - SUB_ELEMENT = 14, - BX_ELEMENT = 15, - EX_ELEMENT = 16, - END_PAIR_TAGS = 16, - - //standalone tags + BPT_ELEMENT = 10, + EPT_ELEMENT = 11, + G_ELEMENT = 12, + HI_ELEMENT = 13, + SUB_ELEMENT = 14, + BX_ELEMENT = 15, + EX_ELEMENT = 16, + END_PAIR_TAGS = 16, + + // standalone tags BEGIN_STANDALONE_TAGS = 20, - PH_ELEMENT = 20, - X_ELEMENT = 21, - IT_ELEMENT = 22, - UT_ELEMENT = 23, - T5_N_ELEMENT = 24, - END_STANDALONE_TAGS = 25, + PH_ELEMENT = 20, + X_ELEMENT = 21, + IT_ELEMENT = 22, + UT_ELEMENT = 23, + T5_N_ELEMENT = 24, + END_STANDALONE_TAGS = 25, - END_INLINE_TAGS = 29, + END_INLINE_TAGS = 29, - TMX_SENTENCE_ELEMENT = 30, - INVCHAR_ELEMENT = 31 + TMX_SENTENCE_ELEMENT = 30, + INVCHAR_ELEMENT = 31 } ELEMENTID; - -enum ACTIVE_SEGMENT{ - SOURCE_SEGMENT = 0, - TARGET_SEGMENT = 1, - REQUEST_SEGMENT = 2 //from fuzzy search request +enum ACTIVE_SEGMENT +{ + SOURCE_SEGMENT = 0, + TARGET_SEGMENT = 1, + REQUEST_SEGMENT = 2 // from fuzzy search request }; - typedef ACTIVE_SEGMENT TagLocation; struct TagInfo { TagLocation tagLocation = SOURCE_SEGMENT; - //bool fPairedTag; // is tag is paired tag (bpt, ept, ) - bool fPairedTagClosed = false; // false for bpt/ept tag - waiting for matching ept/bpt tag + // bool fPairedTag; // is tag is paired tag (bpt, ept, ) + bool fPairedTagClosed = false; // false for bpt/ept tag - waiting for matching ept/bpt tag bool fTagAlreadyUsedInTarget = false; // we save tags only from source segment and then try to match\bind them in target - int generated_i = 0; // for pair tags - generated identifier to find matching tag. the same as in original_i if it's not binded to other tag in segment - int generated_x = 0; // id of tag. should match original_x, if it's not occupied by other tags - ELEMENTID generated_tagType = UNKNOWN_ELEMENT; // replaced tagType, could be PH_ELEMENT, BPT_ELEMENT, EPT_ELEMENT + int generated_i = 0; // for pair tags - generated identifier to find matching tag. the same as in original_i if it's not binded to other tag in segment + int generated_x = 0; // id of tag. should match original_x, if it's not occupied by other tags + ELEMENTID generated_tagType = UNKNOWN_ELEMENT; // replaced tagType, could be PH_ELEMENT, BPT_ELEMENT, EPT_ELEMENT - int original_i = 0; // original paired tags i - int original_x = 0; // original id of tag - ELEMENTID original_tagType; // original tagType - //if targetTag -> matching tag from source tags - //if sourceTag -> matching tag from target tags - //TagInfo* matchingTag; - //int matchingTagIndex = -1; + int original_i = 0; // original paired tags i + int original_x = 0; // original id of tag + ELEMENTID original_tagType; // original tagType + // if targetTag -> matching tag from source tags + // if sourceTag -> matching tag from target tags + // TagInfo* matchingTag; + // int matchingTagIndex = -1; bool fDeleted = false; - - //t5n + + // t5n std::string t5n_key; std::string t5n_value; }; - -class TagReplacer{ - public: - +class TagReplacer +{ +public: std::vector sourceTagList; std::vector targetTagList; std::vector requestTagList; ACTIVE_SEGMENT activeSegment = SOURCE_SEGMENT; - int iHighestPTI = 0; // increments with each opening pair tags - int iHighestPTId = 500; // increments with pair tag - int iHighestPHId = 100; // increments with ph tag - bool fFuzzyRequest = false; // if re are dealing with import or fuzzy request - bool fReplaceNumberProtectionTagsWithHashes = false;// + int iHighestPTI = 0; // increments with each opening pair tags + int iHighestPTId = 500; // increments with pair tag + int iHighestPHId = 100; // increments with ph tag + bool fFuzzyRequest = false; // if re are dealing with import or fuzzy request + bool fReplaceNumberProtectionTagsWithHashes = false; // bool fSkipTags = false; - //to track id and i attributes in request and then generate new values for tags in srt and trg that is not matching - int iHighestRequestsOriginalI = 0; + // to track id and i attributes in request and then generate new values for tags in srt and trg that is not matching + int iHighestRequestsOriginalI = 0; int iHighestRequestsOriginalId = 0; - - //idAttr==0 -> look for id in attributes - //iAttr==0 ->i attribute not used - //tag should be only "ph", "bpt", or "ept" - TagInfo GenerateReplacingTag(ELEMENTID tagType, xercesc::AttributeList* attributes, bool saveTagToTagList = true); - std::wstring PrintTag(TagInfo& tag); + // idAttr==0 -> look for id in attributes + // iAttr==0 ->i attribute not used + // tag should be only "ph", "bpt", or "ept" + TagInfo GenerateReplacingTag(ELEMENTID tagType, xercesc::AttributeList *attributes, bool saveTagToTagList = true); + std::wstring PrintTag(TagInfo &tag); void reset(); - std::string static LogTag(TagInfo& tag); - std::wstring PrintReplaceTagWithKey(TagInfo& tag); - - TagReplacer(){ - sourceTagList.reserve(50); - targetTagList.reserve(50); - requestTagList.reserve(50); - reset(); - } + std::string static LogTag(TagInfo &tag); + std::wstring PrintReplaceTagWithKey(TagInfo &tag); + + TagReplacer() + { + sourceTagList.resize(50); + targetTagList.resize(50); + requestTagList.resize(50); + reset(); + } }; // PROP types -typedef enum { TMLANGUAGE_PROP, TMMARKUP_PROP, TMDOCNAME_PROP, TMCONTEXT_PROP, TMADDINFO_PROP, TMAUTHOR_PROP, MACHINEFLAG_PROP, SEG_PROP, TMDESCRIPTION_PROP, TMNOTE_PROP, TMNOTESTYLE_PROP, - TRANSLATIONFLAG_PROP, TMTMMATCHTYPE_PROP, TMMTSERVICE_PROP, TMMTMETRICNAME_PROP, TMMTMETRICVALUE_PROP, TMPEEDITDISTANCECHARS_PROP, TMPEEDITDISTANCEWORDS_PROP, - TMMTFIELDS_PROP, TMWORDS_PROP, TMMATCHSEGID_PROP, UNKNOWN_PROP } TMXPROPID, PROPID; +typedef enum +{ + TMLANGUAGE_PROP, + TMMARKUP_PROP, + TMDOCNAME_PROP, + TMCONTEXT_PROP, + TMADDINFO_PROP, + TMAUTHOR_PROP, + MACHINEFLAG_PROP, + SEG_PROP, + TMDESCRIPTION_PROP, + TMNOTE_PROP, + TMNOTESTYLE_PROP, + TRANSLATIONFLAG_PROP, + TMTMMATCHTYPE_PROP, + TMMTSERVICE_PROP, + TMMTMETRICNAME_PROP, + TMMTMETRICVALUE_PROP, + TMPEEDITDISTANCECHARS_PROP, + TMPEEDITDISTANCEWORDS_PROP, + TMMTFIELDS_PROP, + TMWORDS_PROP, + TMMATCHSEGID_PROP, + UNKNOWN_PROP +} TMXPROPID, + PROPID; // stack elements typedef struct _TMXELEMENT { - ELEMENTID ID; // ID of element - BOOL fInlineTagging; // TRUE = we are processing inline tagging - BOOL fInsideTagging; // TRUE = we are currently inside inline tagging - TMXPROPID PropID; // ID of prop element (only used for props) - CHAR szDataType[50]; // data type of current element - CHAR szTMXLanguage[50]; // TMX language of element - CHAR szTMLanguage[50]; // TM language of element - CHAR szTMMarkup[50]; // TM markup of element - LONG lSegNum; // TM segment number + ELEMENTID ID; // ID of element + BOOL fInlineTagging; // TRUE = we are processing inline tagging + BOOL fInsideTagging; // TRUE = we are currently inside inline tagging + TMXPROPID PropID; // ID of prop element (only used for props) + CHAR szDataType[50]; // data type of current element + CHAR szTMXLanguage[50]; // TMX language of element + CHAR szTMLanguage[50]; // TM language of element + CHAR szTMMarkup[50]; // TM markup of element + LONG lSegNum; // TM segment number } TMXELEMENT, *PTMXELEMENT; // @@ -1218,25 +1261,25 @@ class TMXParseHandler : public xercesc::HandlerBase // ----------------------------------------------------------------------- // Constructors and Destructor // ----------------------------------------------------------------------- - TMXParseHandler(InclosingTagsBehaviour InclosingTagsBehaviour = InclosingTagsBehaviour::saveAll); + TMXParseHandler(InclosingTagsBehaviour InclosingTagsBehaviour = InclosingTagsBehaviour::saveAll); virtual ~TMXParseHandler(); // setter functions for import info - void SetMemInfo( PMEMEXPIMPINFO m_pMemInfo ); - void SetImportData(ImportStatusDetails * _pImportDetails) { pImportDetails = _pImportDetails;} - void SetMemInterface( PFN_MEMINSERTSEGMENT pfnInsertSegment, LONG lMemHandle, //LOADEDTABLE* pTable, - PTOKENENTRY pTokBuf, int iTokBufSize ); - void SetSourceLanguage( char *pszSourceLang ); - - // getter functions - void GetDescription( char *pszDescription, int iBufSize ); - void GetSourceLanguage( char *pszSourceLang, int iBufSize ); - BOOL IsHeaderDone( void ); - BOOL ErrorOccured( void ); - void GetErrorText( char *pszTextBuffer, int iBufSize ); + void SetMemInfo(PMEMEXPIMPINFO m_pMemInfo); + void SetImportData(ImportStatusDetails *_pImportDetails) { pImportDetails = _pImportDetails; } + void SetMemInterface(PFN_MEMINSERTSEGMENT pfnInsertSegment, LONG lMemHandle, // LOADEDTABLE* pTable, + PTOKENENTRY pTokBuf, int iTokBufSize); + void SetSourceLanguage(char *pszSourceLang); + + // getter functions + void GetDescription(char *pszDescription, int iBufSize); + void GetSourceLanguage(char *pszSourceLang, int iBufSize); + BOOL IsHeaderDone(void); + BOOL ErrorOccured(void); + void GetErrorText(char *pszTextBuffer, int iBufSize); std::wstring GetParsedData() const; - std::wstring GetParsedDataWithReplacedNpTags()const; - std::wstring GetParsedNormalizedData()const; + std::wstring GetParsedDataWithReplacedNpTags() const; + std::wstring GetParsedNormalizedData() const; int iStopImportWRc = 0; bool fReplaceWithTagsWithoutAttributes; @@ -1245,125 +1288,123 @@ class TMXParseHandler : public xercesc::HandlerBase // ----------------------------------------------------------------------- // Handlers for the SAX DocumentHandler interface // ----------------------------------------------------------------------- - void startElement(const XMLCh* const name, xercesc::AttributeList& attributes); - void endElement(const XMLCh* const name ); - void characters(const XMLCh* const chars, const XMLSize_t length); - //void ignorableWhitespace(const XMLCh* const chars, const unsigned int length); - //void resetDocument(); - + void startElement(const XMLCh *const name, xercesc::AttributeList &attributes); + void endElement(const XMLCh *const name); + void characters(const XMLCh *const chars, const XMLSize_t length); + // void ignorableWhitespace(const XMLCh* const chars, const unsigned int length); + // void resetDocument(); // ----------------------------------------------------------------------- // Handlers for the SAX ErrorHandler interface // ----------------------------------------------------------------------- - void warning(const xercesc::SAXParseException& exc); - void error(const xercesc::SAXParseException& exc ); - void fatalError(const xercesc::SAXParseException& exc); - void fatalInternalError(const xercesc::SAXException& exc); + void warning(const xercesc::SAXParseException &exc); + void error(const xercesc::SAXParseException &exc); + void fatalError(const xercesc::SAXParseException &exc); + void fatalInternalError(const xercesc::SAXException &exc); - bool tmxEndReached()const {return fBodyDone;} - //void resetErrors(); + bool tmxEndReached() const { return fBodyDone; } + // void resetErrors(); - TagReplacer tagReplacer; BOOL fInitialized = false; BOOL fCreateNormalizedStr = false; USHORT insertSegUsRC{0}; - void setDocumentLocator(const xercesc::Locator* const locator) override; + void setDocumentLocator(const xercesc::Locator *const locator) override; private: - const xercesc::Locator* m_locator = nullptr; - ImportStatusDetails* pImportDetails = nullptr; - ELEMENTID GetElementID( PSZ pszName ); - void Push( PTMXELEMENT pElement ); - void Pop( PTMXELEMENT pElement ); - BOOL GetValue( PSZ pszString, int iLen, int *piResult ); - BOOL TMXLanguage2TMLanguage( PSZ pszTMLanguage, PSZ pszTMXLanguage, PSZ pszResultingLanguage ); - USHORT RemoveRTFTags( PSZ_W pszString, BOOL fTagsInCurlyBracesOnly ); + const xercesc::Locator *m_locator = nullptr; + ImportStatusDetails *pImportDetails = nullptr; + ELEMENTID GetElementID(PSZ pszName); + void Push(PTMXELEMENT pElement); + void Pop(PTMXELEMENT pElement); + BOOL GetValue(PSZ pszString, int iLen, int *piResult); + BOOL TMXLanguage2TMLanguage(PSZ pszTMLanguage, PSZ pszTMXLanguage, PSZ pszResultingLanguage); + USHORT RemoveRTFTags(PSZ_W pszString, BOOL fTagsInCurlyBracesOnly); // date conversion help functions - BOOL IsLeapYear( const int iYear ); - int GetDaysOfMonth( const int iMonth, const int iYear ); - int GetDaysOfYear( const int iYear ); - int GetYearDay( const int iDay, const int iMonth, const int iYear ); + BOOL IsLeapYear(const int iYear); + int GetDaysOfMonth(const int iMonth, const int iYear); + int GetDaysOfYear(const int iYear); + int GetYearDay(const int iDay, const int iMonth, const int iYear); // mem import interface data PMEMEXPIMPINFO m_pMemInfo; PFN_MEMINSERTSEGMENT pfnInsertSegment; - LONG lMemHandle; + LONG lMemHandle; // processing flags - BOOL fSource; // TRUE = source data collected - BOOL fTarget; // TRUE = target data collected - BOOL fCatchData; // TRUE = catch data - BOOL fWithTagging; // TRUE = add tagging to data - BOOL fWithTMXTags; // TRUE = segment contains TMX tags - BOOL fTMXTagStarted; // TRUE = TMX inline tag started - BOOL fHeaderDone; // TRUE = header has been processed - BOOL fError; // TRUE = parsing ended with error + BOOL fSource; // TRUE = source data collected + BOOL fTarget; // TRUE = target data collected + BOOL fCatchData; // TRUE = catch data + BOOL fWithTagging; // TRUE = add tagging to data + BOOL fWithTMXTags; // TRUE = segment contains TMX tags + BOOL fTMXTagStarted; // TRUE = TMX inline tag started + BOOL fHeaderDone; // TRUE = header has been processed + BOOL fError; // TRUE = parsing ended with error bool fBodyDone = false; - + // segment data - ULONG ulSegNo; // segmet number - LONG lTime; // segment date/time - USHORT usTranslationFlag; // type of translation flag - int iNumOfTu; + ULONG ulSegNo; // segmet number + LONG lTime; // segment date/time + USHORT usTranslationFlag; // type of translation flag + int iNumOfTu; size_t _invalidCharacterErrorCount{0}; - // buffers - #define DATABUFFERSIZE 4098 +// buffers +#define DATABUFFERSIZE 4098 typedef struct _BUFFERAREAS { - CHAR_W szData[DATABUFFERSIZE]; // buffer for collected data - CHAR_W szReplacedNpData[DATABUFFERSIZE]; - CHAR_W szNormalizedData[DATABUFFERSIZE]; - CHAR_W szPropW[DATABUFFERSIZE]; // buffer for collected prop values - CHAR_W szContext[DATABUFFERSIZE]; // buffer for collected prop values - CHAR_W szAddInfo[DATABUFFERSIZE]; // buffer for collected prop values - CHAR szLang[50]; // buffer for language - CHAR szDocument[EQF_DOCNAMELEN];// buffer for document name - - CHAR szAuthor[EQF_DOCNAMELEN];// buffer for author name - CHAR szChangeId[EQF_DOCNAMELEN];// buffer for author/changeId name - CHAR szCreationId[EQF_DOCNAMELEN];// buffer for author/creationId name - - MEMEXPIMPSEG SegmentData; // buffer for segment data - CHAR szDescription[1024]; // buffer for memory descripion - CHAR szMemSourceLang[50]; // buffer for memory source language - CHAR szMemSourceIsoLang[50]; - CHAR szErrorMessage[1024]; // buffer for error message text - CHAR_W szNote[MAX_SEGMENT_SIZE];// buffer for note text - CHAR_W szNoteStyle[100]; // buffer for note style - CHAR_W szMTMetrics[MAX_SEGMENT_SIZE]; // buffer for MT metrics data - ULONG ulWords; // number of words in the segment text - CHAR_W szMatchSegID[MAX_SEGMENT_SIZE];// buffer for match segment ID - - //for inclosing tags skipping. If pFirstBptTag is nullptr->segments don't start from bpt tag-> inclosing tags skipping should be ignored - int inclosingTagsAtTheStartOfSegment = 0; - int inclosingTagsAtTheEndOfSegment = 0; - + CHAR_W szData[DATABUFFERSIZE]; // buffer for collected data + CHAR_W szReplacedNpData[DATABUFFERSIZE]; + CHAR_W szNormalizedData[DATABUFFERSIZE]; + CHAR_W szPropW[DATABUFFERSIZE]; // buffer for collected prop values + CHAR_W szContext[DATABUFFERSIZE]; // buffer for collected prop values + CHAR_W szAddInfo[DATABUFFERSIZE]; // buffer for collected prop values + CHAR szLang[50]; // buffer for language + CHAR szDocument[EQF_DOCNAMELEN]; // buffer for document name + + CHAR szAuthor[EQF_DOCNAMELEN]; // buffer for author name + CHAR szChangeId[EQF_DOCNAMELEN]; // buffer for author/changeId name + CHAR szCreationId[EQF_DOCNAMELEN]; // buffer for author/creationId name + + MEMEXPIMPSEG SegmentData; // buffer for segment data + CHAR szDescription[1024]; // buffer for memory descripion + CHAR szMemSourceLang[50]; // buffer for memory source language + CHAR szMemSourceIsoLang[50]; + CHAR szErrorMessage[1024]; // buffer for error message text + CHAR_W szNote[MAX_SEGMENT_SIZE]; // buffer for note text + CHAR_W szNoteStyle[100]; // buffer for note style + CHAR_W szMTMetrics[MAX_SEGMENT_SIZE]; // buffer for MT metrics data + ULONG ulWords; // number of words in the segment text + CHAR_W szMatchSegID[MAX_SEGMENT_SIZE]; // buffer for match segment ID + + // for inclosing tags skipping. If pFirstBptTag is nullptr->segments don't start from bpt tag-> inclosing tags skipping should be ignored + int inclosingTagsAtTheStartOfSegment = 0; + int inclosingTagsAtTheEndOfSegment = 0; + InclosingTagsBehaviour inclosingTagsBehaviour = saveAll; - - bool fUseMajorLanguage; + + bool fUseMajorLanguage; } BUFFERAREAS, *PBUFFERAREAS; - PBUFFERAREAS pBuf; + PBUFFERAREAS pBuf; // TUV data area typedef struct _TMXTUV { - CHAR_W szText[DATABUFFERSIZE]; // buffer for TUV text - CHAR szLang[50]; // buffer for TUV language (converted to Tmgr language name) or original language in case of fInvalidLang - CHAR szIsoLang[10]; - BOOL fInvalidChars; // TRUE = contains invalid characters - BOOL fInlineTags; // TRUE = contains inline tagging - BOOL fInvalidLang; // TRUE = the language of the TUV is invalid + CHAR_W szText[DATABUFFERSIZE]; // buffer for TUV text + CHAR szLang[50]; // buffer for TUV language (converted to Tmgr language name) or original language in case of fInvalidLang + CHAR szIsoLang[10]; + BOOL fInvalidChars; // TRUE = contains invalid characters + BOOL fInlineTags; // TRUE = contains inline tagging + BOOL fInvalidLang; // TRUE = the language of the TUV is invalid } TMXTUV, *PTMXTUV; PTMXTUV pTuvArray; - int iCurTuv; // current TUV index - int iTuvArraySize; // current size of TUV array + int iCurTuv; // current TUV index + int iTuvArraySize; // current size of TUV array // element stack int iStackSize; @@ -1371,49 +1412,50 @@ class TMXParseHandler : public xercesc::HandlerBase TMXELEMENT CurElement; PTMXELEMENT pStack; - bool fSawErrors; - BOOL fInvalidChars; // TRUE = current TUV contains invalid characters - BOOL fInlineTags; // TRUE = current TUV contains inline tagging + bool fSawErrors; + BOOL fInvalidChars; // TRUE = current TUV contains invalid characters + BOOL fInlineTags; // TRUE = current TUV contains inline tagging // data for remove tag function PTOKENENTRY pTokBuf; int iTokBufSize; - //ULONG ulCP; - - void fillSegmentInfo( PTMXTUV pSource, PTMXTUV pTarget, PMEMEXPIMPSEG pSegment ); + // ULONG ulCP; + void fillSegmentInfo(PTMXTUV pSource, PTMXTUV pTarget, PMEMEXPIMPSEG pSegment); }; - -class StringVariants{ +class StringVariants +{ protected: std::wstring original; // full original str std::wstring originalTarget; - bool fSuccess = false; + bool fSuccess = false; - xercesc::SAXParser parser; TMXParseHandler handler; xercesc::XMLPScanToken saxToken; + public: - StringVariants(std::wstring&& w_src, std::wstring&& w_trg){ + StringVariants(std::wstring &&w_src, std::wstring &&w_trg) + { original = std::move(w_src); originalTarget = std::move(w_trg); } - bool isParsed() const{ return fSuccess; } - std::wstring& getOriginalSrcStr(){ return original; } - std::wstring& getOriginalTrgStr(){ return original; } - wchar_t* getOriginalStrC() { return &original[0]; } - wchar_t* getOriginalTrgStrC() {return &originalTarget[0];} + bool isParsed() const { return fSuccess; } + std::wstring &getOriginalSrcStr() { return original; } + std::wstring &getOriginalTrgStr() { return original; } + wchar_t *getOriginalStrC() { return &original[0]; } + wchar_t *getOriginalTrgStrC() { return &originalTarget[0]; } }; -class StringTagVariants: public StringVariants{ +class StringTagVariants : public StringVariants +{ protected: - std::wstring norm; // without any tags - std::wstring npReplaced; // with generic(generated) tags but np tags is replaced with their keys(r attr) + std::wstring norm; // without any tags + std::wstring npReplaced; // with generic(generated) tags but np tags is replaced with their keys(r attr) std::wstring genericTags; // generated tags so there could be only ph or bpt/ept with id and r attributes that would map input str to common tag format - std::wstring genericTarget; + std::wstring genericTarget; std::wstring normTarget; void initParser(); @@ -1424,272 +1466,251 @@ class StringTagVariants: public StringVariants{ std::string trg; public: - StringTagVariants(std::wstring&& w_src): StringTagVariants(std::move(w_src), L""){} - StringTagVariants(std::wstring&& w_src, std::wstring&& w_trg): StringVariants(std::move(w_src), std::move(w_trg)){ + StringTagVariants(std::wstring &&w_src) : StringTagVariants(std::move(w_src), L"") {} + StringTagVariants(std::wstring &&w_src, std::wstring &&w_trg) : StringVariants(std::move(w_src), std::move(w_trg)) + { initParser(); parseSrc(); - if(fSuccess){ + if (fSuccess) + { parseTrg(); logResults(); } } bool hasTarget() const { return !genericTarget.empty(); } - bool replaceTags(std::wstring&& w_request_input); - - std::wstring& getNormStr(){ return norm; } - std::wstring& getNpReplacedStr(){ return npReplaced; } - std::wstring& getGenericTagsString() { return genericTags; } - std::wstring& getNormTargetStr(){return normTarget; } - - wchar_t* getNormStrC(){return &norm[0];} - wchar_t* getNpReplStrC(){return &npReplaced[0];} - wchar_t* getGenericTagStrC() { return &genericTags[0]; } - wchar_t* getGenericTargetStrC() {return &genericTarget[0];} - wchar_t* getNormalizedTargetStrC() { return &normTarget[0]; } + bool replaceTags(std::wstring &&w_request_input); + + std::wstring &getNormStr() { return norm; } + std::wstring &getNpReplacedStr() { return npReplaced; } + std::wstring &getGenericTagsString() { return genericTags; } + std::wstring &getGenericTargetStr() { return genericTarget; } + std::wstring &getNormTargetStr() { return normTarget; } + + wchar_t *getNormStrC() { return &norm[0]; } + wchar_t *getNpReplStrC() { return &npReplaced[0]; } + wchar_t *getGenericTagStrC() { return &genericTags[0]; } + wchar_t *getGenericTargetStrC() { return &genericTarget[0]; } + wchar_t *getNormalizedTargetStrC() { return &normTarget[0]; } }; -class RequestTagReplacer: public StringVariants{ +class RequestTagReplacer : public StringVariants +{ std::wstring replacedTagsSrc, replacedTagsTrg, genericTagsReq, originalReq; void parseAndReplaceTags(); - public: - RequestTagReplacer(std::wstring&& w_src, std::wstring&& w_trg, std::wstring&& w_req): StringVariants(std::move(w_src), std::move(w_trg)){ - originalReq = std::move(w_req); - parseAndReplaceTags(); +public: + RequestTagReplacer(std::wstring &&w_src, std::wstring &&w_trg, std::wstring &&w_req) : StringVariants(std::move(w_src), std::move(w_trg)) + { + originalReq = std::move(w_req); + parseAndReplaceTags(); }; - wchar_t* getSrcWithTagsFromRequestC(){ return &replacedTagsSrc[0];} - wchar_t* getTrgWithTagsFromRequestC(){ return &replacedTagsTrg[0];} - wchar_t* getReqGenericTagStrC(){return &genericTagsReq[0]; } - wchar_t* getOrigReqC(){return &originalReq[0];} + wchar_t *getSrcWithTagsFromRequestC() { return &replacedTagsSrc[0]; } + wchar_t *getTrgWithTagsFromRequestC() { return &replacedTagsTrg[0]; } + wchar_t *getReqGenericTagStrC() { return &genericTagsReq[0]; } + wchar_t *getOrigReqC() { return &originalReq[0]; } - std::wstring& getSrcWithTagsFromRequest(){ return replacedTagsSrc;} - std::wstring& getTrgWithTagsFromRequest(){ return replacedTagsTrg;} - std::wstring& getReqGenericTagStr(){return genericTagsReq; } - std::wstring& getOrigReq(){return originalReq;} + std::wstring &getSrcWithTagsFromRequest() { return replacedTagsSrc; } + std::wstring &getTrgWithTagsFromRequest() { return replacedTagsTrg; } + std::wstring &getReqGenericTagStr() { return genericTagsReq; } + std::wstring &getOrigReq() { return originalReq; } }; struct TMX_SENTENCE { - //std::unique_ptr pStrings = nullptr; - StringTagVariants* pStrings = nullptr; + // std::unique_ptr pStrings = nullptr; + StringTagVariants *pStrings = nullptr; PSZ_W pAddString = nullptr; - //USHORT usNormLen = 0; - PTMX_TERM_TOKEN pTermTokens = nullptr; - LONG lTermAlloc = 0; - PTMX_TAGTABLE_RECORD pTagRecord = nullptr; - LONG lTagAlloc = 0; - PTMX_TAGENTRY pTagEntryList = nullptr; - USHORT usActVote = 0; - PULONG pulVotes = nullptr; - //std::unique_ptr pPropString = nullptr; - StringTagVariants * pPropString = nullptr; - - PTMX_TERM_TOKEN pPropTermTokens = nullptr; // buffer for Termtokens - - //ADDINFO - CHAR szSourceLanguage[MAX_LANG_LENGTH]; //language name of source - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language name of target - CHAR szAuthorName[MAX_FILESPEC]; //author name of target - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - CHAR szFileName[MAX_FILESPEC]; //where source comes from name+ext - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSourceSegmentId; //seg. num. of source sentence from analysis - CHAR szTagTable[MAX_FNAME]; //tag table name - TIME_L lTime; //time stamp - CHAR_W szContext[MAX_SEGMENT_SIZE]; //segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import - //end ADDINFO + std::vector pTermTokens; + std::vector pTagRecord; + std::vector pTagEntryList; + USHORT usActVote = 0; + std::vector pulVotes; + // std::unique_ptr pPropString = nullptr; + StringTagVariants *pPropString = nullptr; + + std::vector pPropTermTokens; // buffer for Termtokens + + // ADDINFO + CHAR szSourceLanguage[MAX_LANG_LENGTH]; // language name of source + CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language name of target + CHAR szAuthorName[MAX_FILESPEC]; // author name of target + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ + CHAR szFileName[MAX_FILESPEC]; // where source comes from name+ext + LONG_FN szLongName; // name of source file (long name or EOS) + ULONG ulSourceSegmentId; // seg. num. of source sentence from analysis + CHAR szTagTable[MAX_FNAME]; // tag table name + TIME_L lTime; // time stamp + CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context + CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information + BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import + // end ADDINFO std::vector tokens; - - TMX_SENTENCE(std::wstring&& w_src, std::wstring &&w_trg){ - if(w_trg.empty()){ + TMX_SENTENCE(std::wstring &&w_src, std::wstring &&w_trg) + { + if (w_trg.empty()) + { pStrings = new StringTagVariants(std::move(w_src)); //;std::make_unique (std::move(w_src)); - }else{ - pStrings = new StringTagVariants(std::move(w_src), std::move(w_trg)); // std::make_unique (std::move(w_src), std::move(w_trg)); } - bool fOK = true; - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pulVotes), 0L, (LONG)(ABS_VOTES * sizeof(ULONG)), NOMSG ); - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pTagRecord), 0L, (LONG)(2*TOK_SIZE), NOMSG); - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pTermTokens), 0L, (LONG)TOK_SIZE, NOMSG ); - - if ( fOK ) + else { - lTermAlloc = (LONG)TOK_SIZE; - lTagAlloc = (LONG)(2*TOK_SIZE); - } /* endif */ - if(!fOK){ - throw ;//new Exception(); + pStrings = new StringTagVariants(std::move(w_src), std::move(w_trg)); // std::make_unique (std::move(w_src), std::move(w_trg)); } - //pStrings = input; - //usNormLen = pStrings->getNormStr().length(); + bool fOK = true; + pulVotes.reserve(ABS_VOTES / sizeof(pulVotes[0])); + pTagRecord.reserve(2 * TOK_SIZE / sizeof(pTagRecord[0])); + pTermTokens.reserve(TOK_SIZE / sizeof(pTermTokens[0])); } - TMX_SENTENCE(wchar_t* w_src): TMX_SENTENCE(std::move(w_src), L""){} + TMX_SENTENCE(wchar_t *w_src) : TMX_SENTENCE(std::move(w_src), L"") {} - ~TMX_SENTENCE(){ - if(pStrings){ + ~TMX_SENTENCE() + { + if (pStrings) + { delete pStrings; pStrings = nullptr; } - if(pPropString){ + if (pPropString) + { delete pPropString; pPropString = nullptr; } - //if(pPropString) pPropString.reset(); - //if(pStrings) pStrings.reset(); - UtlAlloc( (PVOID *) &pTermTokens, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pTagRecord, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pulVotes, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pAddString, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *) &pTermTokens, 0L, 0L, NOMSG ); } - bool wasParsedSuccessfully()const {return pStrings && pStrings->isParsed(); } -}; + bool wasParsedSuccessfully() const { return pStrings && pStrings->isParsed(); } +}; using PTMX_SENTENCE = TMX_SENTENCE *; +// #pragma pack(pop) +// #pragma pack() -//#pragma pack(pop) -//#pragma pack() - - -//#pragma pack(pop) -//#pragma pack() +// #pragma pack(pop) +// #pragma pack() typedef struct _TMX_REPLTAGPAIR { - PBYTE pSrcTok; // token for paired toks - PBYTE pPropTok; - BOOL fUsed; //TRUE = Tag pair has been used in replacement already -} TMX_REPLTAGPAIR, * PTMX_REPLTAGPAIR; + PBYTE pSrcTok; // token for paired toks + PBYTE pPropTok; + BOOL fUsed; // TRUE = Tag pair has been used in replacement already +} TMX_REPLTAGPAIR, *PTMX_REPLTAGPAIR; typedef struct _TMX_SUBSTPROP { - CHAR_W szSource [ MAX_SEGMENT_SIZE ]; // source segment - CHAR_W szPropSource[ MAX_SEGMENT_SIZE ]; // source proposal - CHAR_W szPropTarget[ MAX_SEGMENT_SIZE ]; // translation of proposal - char szSourceLanguage[MAX_LANG_LENGTH];// source language - char szSourceTagTable[MAX_FNAME]; // source tagtable - char szTargetLanguage[MAX_LANG_LENGTH];// target language - char szPropTagTable[MAX_FNAME]; // tagtable for proposal - PBYTE pTokSource; // token buffer for source tokens - PBYTE pTokPropSource; // token buffer for proposal tokens - PBYTE pTokPropTarget; // token buffer for proposal target .. - //PTMX_TAGTABLE_RECORD pTagsSource; // tag table record for source .. - //PTMX_TAGTABLE_RECORD pTagsPropSource; // tag table record for prop source - //PTMX_TAGTABLE_RECORD pTagsPropTarget; // tag table record for prop target - CHAR_W chBuffer[ MAX_SEGMENT_SIZE * 2 ]; // generic token buffer + CHAR_W szSource[MAX_SEGMENT_SIZE]; // source segment + CHAR_W szPropSource[MAX_SEGMENT_SIZE]; // source proposal + CHAR_W szPropTarget[MAX_SEGMENT_SIZE]; // translation of proposal + char szSourceLanguage[MAX_LANG_LENGTH]; // source language + char szSourceTagTable[MAX_FNAME]; // source tagtable + char szTargetLanguage[MAX_LANG_LENGTH]; // target language + char szPropTagTable[MAX_FNAME]; // tagtable for proposal + PBYTE pTokSource; // token buffer for source tokens + PBYTE pTokPropSource; // token buffer for proposal tokens + PBYTE pTokPropTarget; // token buffer for proposal target .. + // PTMX_TAGTABLE_RECORD pTagsSource; // tag table record for source .. + // PTMX_TAGTABLE_RECORD pTagsPropSource; // tag table record for prop source + // PTMX_TAGTABLE_RECORD pTagsPropTarget; // tag table record for prop target + CHAR_W chBuffer[MAX_SEGMENT_SIZE * 2]; // generic token buffer USHORT usTokenSource; // number of source tokens USHORT usTokenPropSource; // number of prop source tokens USHORT usTokenPropTarget; // number of prop target tokens - ULONG ulRandom[MAX_RANDOM]; // random sequence number - PTMX_REPLTAGPAIR pTagPairs; // tags to be replaced by each other - PTMX_REPLTAGPAIR pDelTagPairs; // tags to be deleted in prop -} TMX_SUBSTPROP, * PTMX_SUBSTPROP; - - + ULONG ulRandom[MAX_RANDOM]; // random sequence number + PTMX_REPLTAGPAIR pTagPairs; // tags to be replaced by each other + PTMX_REPLTAGPAIR pDelTagPairs; // tags to be deleted in prop +} TMX_SUBSTPROP, *PTMX_SUBSTPROP; //======================================================================= typedef struct _TMX_PREFIX_IN { - USHORT usLengthInput; //length of complete input structure - USHORT usTmCommand; //TM command id -} TMX_PREFIX_IN, * PTMX_PREFIX_IN, XIN, * PXIN; + USHORT usLengthInput; // length of complete input structure + USHORT usTmCommand; // TM command id +} TMX_PREFIX_IN, *PTMX_PREFIX_IN, XIN, *PXIN; typedef struct _TMX_PUT_W { - CHAR_W szSource[MAX_SEGMENT_SIZE]; //source sentence - CHAR_W szTarget[MAX_SEGMENT_SIZE]; //target sentence - CHAR szSourceLanguage[MAX_LANG_LENGTH]; //language name of source - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language name of target - CHAR szAuthorName[MAX_LANG_LENGTH]; //author name of target - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - CHAR szFileName[MAX_FILESPEC]; //where source comes from name+ext - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSourceSegmentId; //seg. num. of source sentence from analysis - CHAR szTagTable[MAX_FNAME]; //tag table name - TIME_L lTime; //time stamp - CHAR_W szContext[MAX_SEGMENT_SIZE]; //segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import -} TMX_PUT_W, * PTMX_PUT_W; - - + CHAR_W szSource[MAX_SEGMENT_SIZE]; // source sentence + CHAR_W szTarget[MAX_SEGMENT_SIZE]; // target sentence + CHAR szSourceLanguage[MAX_LANG_LENGTH]; // language name of source + CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language name of target + CHAR szAuthorName[MAX_LANG_LENGTH]; // author name of target + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ + CHAR szFileName[MAX_FILESPEC]; // where source comes from name+ext + LONG_FN szLongName; // name of source file (long name or EOS) + ULONG ulSourceSegmentId; // seg. num. of source sentence from analysis + CHAR szTagTable[MAX_FNAME]; // tag table name + TIME_L lTime; // time stamp + CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context + CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information + BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import +} TMX_PUT_W, *PTMX_PUT_W; typedef struct _TMX_GET_W { - CHAR_W szSource[MAX_SEGMENT_SIZE]; //source sentence - CHAR szTagTable[MAX_FNAME]; //tag table name of source - CHAR szSourceLanguage[MAX_LANG_LENGTH]; //language of source - CHAR szFileName[MAX_FILESPEC]; //file name the source comes from - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSegmentId; //segment number from analysis - CHAR szAuthorName[MAX_LANG_LENGTH]; //author name - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language of translation - USHORT usRequestedMatches; //number of requested matches - USHORT usMatchThreshold; //threshold for match level - USHORT usConvert; //how the output should be converted - ULONG ulParm; //for future use, xmp. GET_ONLY_MT_MATCHES, GET_ONLY_EXACT_MATCHES...) - CHAR_W szContext[MAX_SEGMENT_SIZE]; //segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - ULONG ulSrcOemCP; - ULONG ulTgtOemCP; - PVOID pvReplacementList; // ptr to a SGML-DITA replacement list or NULL - PVOID pvGMOptList; // ptr to a global memory option list or NULL + CHAR_W szSource[MAX_SEGMENT_SIZE]; // source sentence + CHAR szTagTable[MAX_FNAME]; // tag table name of source + CHAR szSourceLanguage[MAX_LANG_LENGTH]; // language of source + CHAR szFileName[MAX_FILESPEC]; // file name the source comes from + LONG_FN szLongName; // name of source file (long name or EOS) + ULONG ulSegmentId; // segment number from analysis + CHAR szAuthorName[MAX_LANG_LENGTH]; // author name + CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language of translation + USHORT usRequestedMatches; // number of requested matches + USHORT usMatchThreshold; // threshold for match level + USHORT usConvert; // how the output should be converted + ULONG ulParm; // for future use, xmp. GET_ONLY_MT_MATCHES, GET_ONLY_EXACT_MATCHES...) + CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context + CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information + ULONG ulSrcOemCP; + ULONG ulTgtOemCP; + PVOID pvReplacementList; // ptr to a SGML-DITA replacement list or NULL + PVOID pvGMOptList; // ptr to a global memory option list or NULL bool fSourceLangIsPrefered; bool fTargetLangIsPrefered; -} TMX_GET_W, * PTMX_GET_W; - +} TMX_GET_W, *PTMX_GET_W; //======================================================================= // structure TMX_GET_OUT typedef struct _TMX_MATCH_TABLE_W { - CHAR_W szSource[MAX_SEGMENT_SIZE]; //source sentence with tags - CHAR szFileName[MAX_FILESPEC]; //file name where the source comes from. - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSegmentId; //segment number from analysis - CHAR_W szTarget[MAX_SEGMENT_SIZE]; //target sentence with tags - CHAR szTargetLanguage[MAX_LANG_LENGTH]; //language of translation - CHAR szOriginalSrcLanguage[MAX_LANG_LENGTH]; //language of src of translation - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - CHAR szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]; //author name of target - TIME_L lTargetTime; //time stamp of target - USHORT usMatchLevel; //similarity of the source + CHAR_W szSource[MAX_SEGMENT_SIZE]; // source sentence with tags + CHAR szFileName[MAX_FILESPEC]; // file name where the source comes from. + LONG_FN szLongName; // name of source file (long name or EOS) + ULONG ulSegmentId; // segment number from analysis + CHAR_W szTarget[MAX_SEGMENT_SIZE]; // target sentence with tags + CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language of translation + CHAR szOriginalSrcLanguage[MAX_LANG_LENGTH]; // language of src of translation + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ + CHAR szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]; // author name of target + TIME_L lTargetTime; // time stamp of target + USHORT usMatchLevel; // similarity of the source int iWords; int iDiffs; - USHORT usOverlaps; //temp field - nr of overlapping triples - CHAR szTagTable[MAX_FNAME]; //tag table name of source - ULONG ulKey; // key of match - USHORT usTargetNum; // number of target - USHORT usDBIndex; // number of memory in current hierarchy - CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context info - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - USHORT usContextRanking; // context ranking from user exit context processing - USHORT usMatchInfo; // for future use: special info for match -} TMX_MATCH_TABLE_W, * PTMX_MATCH_TABLE_W; + USHORT usOverlaps; // temp field - nr of overlapping triples + CHAR szTagTable[MAX_FNAME]; // tag table name of source + ULONG ulKey; // key of match + USHORT usTargetNum; // number of target + USHORT usDBIndex; // number of memory in current hierarchy + //CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context info + //CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information + std::wstring szContext, szAddInfo; + USHORT usContextRanking; // context ranking from user exit context processing + USHORT usMatchInfo; // for future use: special info for match +} TMX_MATCH_TABLE_W, *PTMX_MATCH_TABLE_W; using TMX_GET_OUT_W = struct _TMX_GET_OUT_W : public TMX_EXT_OUT_W { - //TMX_PREFIX_OUT stPrefixOut; //prefix of Output buffer - USHORT usNumMatchesFound; //number of matches found - TMX_MATCH_TABLE_W stMatchTable[MAX_MATCHES]; //match structure + // TMX_PREFIX_OUT stPrefixOut; //prefix of Output buffer + USHORT usNumMatchesFound; // number of matches found + TMX_MATCH_TABLE_W stMatchTable[MAX_MATCHES]; // match structure USHORT fsAvailFlags; // additional flags (more exact/fuzzy matches avail) std::vector tokens; }; using PTMX_GET_OUT_W = TMX_GET_OUT_W *; - - - - - //======================================================================= /**********************************************************************/ /* Attention: for special names mode of TMExtract, the TMX_EXT_IN */ @@ -1705,13 +1726,13 @@ using PTMX_GET_OUT_W = TMX_GET_OUT_W *; /* names are available than fit into the output buffers. */ /* The remining fields are unused. */ /**********************************************************************/ -//structure TMX_EXT_IN +// structure TMX_EXT_IN typedef struct _TMX_EXT_IN { - ULONG ulTmKey; //tm get to get - USHORT usConvert; //how the output is to appear - USHORT usNextTarget; //which target record to address next -} TMX_EXT_IN, * PTMX_EXT_IN; + ULONG ulTmKey; // tm get to get + USHORT usConvert; // how the output is to appear + USHORT usNextTarget; // which target record to address next +} TMX_EXT_IN, *PTMX_EXT_IN; typedef TMX_EXT_IN TMX_EXT_IN_W, *PTMX_EXT_IN_W; //======================================================================= @@ -1729,26 +1750,27 @@ typedef TMX_EXT_IN TMX_EXT_IN_W, *PTMX_EXT_IN_W; // structure TMX_EXT_OUT class EqfMemoryPlugin; -class Dummy{}; +class Dummy +{ +}; class RequestData; - -#define ASD_ORGANIZE 0x100 // work in organize mode +#define ASD_ORGANIZE 0x100 // work in organize mode std::string StatusToString(int eStatus); #include class EqfMemory //: public TMX_CLB /*! \brief This class implements the standard translation memory (EQF) for OpenTM2. -*/ + */ { - + USHORT LoadMem(); public: std::recursive_timed_mutex tmMutex; - //bool fOpen = false; + // bool fOpen = false; BTREE TmBtree; BTREE InBtree; TMX_TABLE Languages; @@ -1758,637 +1780,573 @@ class EqfMemory //: public TMX_CLB USHORT usAccessMode = 0; USHORT usThreshold = 0; TMX_SIGN stTmSign; - BYTE bCompact[MAX_COMPACT_SIZE-1]; - BYTE bCompactChanged = 0; - LONG alUpdCounter[MAX_UPD_COUNTERS]; - PTMX_LONGNAME_TABLE pLongNames = nullptr; - TMX_TABLE LangGroups; // table containing language group names - PSHORT psLangIdToGroupTable = nullptr; // language ID to group ID table - LONG lLangIdToGroupTableSize = 0; // size of table (alloc size) - LONG lLangIdToGroupTableUsed = 0; // size of table (bytes in use) - PVOID pTagTable = nullptr; // tag table loaded for segment markup (TBLOADEDTABLE) + std::vector bCompact; + BYTE bCompactChanged = 0; + LONG alUpdCounter[MAX_UPD_COUNTERS]; + // std::vector pLongNames ; + TMX_TABLE LangGroups; // table containing language group names + std::vector psLangIdToGroupTable; // language ID to group ID table + LONG lLangIdToGroupTableUsed = 0; // size of table (bytes in use) + PVOID pTagTable = nullptr; // tag table loaded for segment markup (TBLOADEDTABLE) // copy of long name table sorted ignoring the case of the file names // Note: only the stTableEntry array is filled in this area, for all other // information use the entries in the pLongNames structure - PTMX_LONGNAME_TABLE pLongNamesCaseIgnore = nullptr; + // std::vector pLongNamesCaseIgnore; + TMX_LONGNAME_TABLE LongNames, LongNamesCaseIgnore; // fields for work area pointers of various subfunctions which are allocated // only once for performance reasons - PVOID pvTempMatchList = nullptr; // matchlist of FillMatchEntry function - PVOID pvIndexRecord = nullptr; // index record area of FillMatchEntry function - PVOID pvTmRecord = nullptr; // buffer for memory record used by GetFuzzy and GetExact - ULONG ulRecBufSize = 0; // current size of pvTMRecord; + PVOID pvTempMatchList = nullptr; // matchlist of FillMatchEntry function + std::vector pvIndexRecord; // index record area of FillMatchEntry function + std::vector pvTmRecord; // buffer for memory record used by GetFuzzy and GetExact + ULONG ulRecBufSize = 0; // current size of pvTMRecord; // fields for time measurements and logging - BOOL fTimeLogging = 0; // TRUE = Time logging is active - LONG64 lAllocTime = 0; // time for memory allocation - LONG64 lTokenizeTime = 0; // time for tokenization - LONG64 lGetExactTime = 0; // time for GetExact - LONG64 lOtherTime = 0; // time for other activities - LONG64 lGetFuzzyTime = 0; // time for GetFuzzy - LONG64 lFuzzyOtherTime = 0; // other time spent in GetFuzzy - LONG64 lFuzzyTestTime = 0; // FuzzyTest time spent in GetFuzzy - LONG64 lFuzzyGetTime = 0; // NTMGet time spent in GetFuzzy - LONG64 lFuzzyFillMatchEntry = 0; // FillMatchEntry time spent in GetFuzzy - LONG64 lFillMatchAllocTime = 0; // FillMatchEntry: allocation time - LONG64 lFillMatchOtherTime = 0; // FillMatchEntry: other times - LONG64 lFillMatchReadTime = 0; // FillMatchEntry: read index DB time - LONG64 lFillMatchFillTime = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchCleanupTime = 0; // FillMatchEntry: cleanup match list time - LONG64 lFillMatchFill1Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill2Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill3Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill4Time = 0; // FillMatchEntry: fill match list time - - //std::shared_ptr readOnlyPtr; - //std::shared_ptr writePtr; + BOOL fTimeLogging = 0; // TRUE = Time logging is active + LONG64 lAllocTime = 0; // time for memory allocation + LONG64 lTokenizeTime = 0; // time for tokenization + LONG64 lGetExactTime = 0; // time for GetExact + LONG64 lOtherTime = 0; // time for other activities + LONG64 lGetFuzzyTime = 0; // time for GetFuzzy + LONG64 lFuzzyOtherTime = 0; // other time spent in GetFuzzy + LONG64 lFuzzyTestTime = 0; // FuzzyTest time spent in GetFuzzy + LONG64 lFuzzyGetTime = 0; // NTMGet time spent in GetFuzzy + LONG64 lFuzzyFillMatchEntry = 0; // FillMatchEntry time spent in GetFuzzy + LONG64 lFillMatchAllocTime = 0; // FillMatchEntry: allocation time + LONG64 lFillMatchOtherTime = 0; // FillMatchEntry: other times + LONG64 lFillMatchReadTime = 0; // FillMatchEntry: read index DB time + LONG64 lFillMatchFillTime = 0; // FillMatchEntry: fill match list time + LONG64 lFillMatchCleanupTime = 0; // FillMatchEntry: cleanup match list time + LONG64 lFillMatchFill1Time = 0; // FillMatchEntry: fill match list time + LONG64 lFillMatchFill2Time = 0; // FillMatchEntry: fill match list time + LONG64 lFillMatchFill3Time = 0; // FillMatchEntry: fill match list time + LONG64 lFillMatchFill4Time = 0; // FillMatchEntry: fill match list time + + // std::shared_ptr readOnlyPtr; + // std::shared_ptr writePtr; std::shared_ptr readOnlyCnt; std::shared_ptr writeCnt; - //std::weak_ptr pActiveRequest; - RequestData* pActiveRequest; + // std::weak_ptr pActiveRequest; + RequestData *pActiveRequest; int lastErrorCode = 0; size_t expectedSizeInRam = 0; - int getErrorCode()const; + int getErrorCode() const; int setErrorCode(int rc); void reserErrorCode(); COMMAND getActiveRequestCommand(); void setActiveRequest(std::weak_ptr request); - void setActiveRequest(RequestData* request); + void setActiveRequest(RequestData *request); void resetActiveRequest(); - + COMMAND activeCommand = UNKNOWN_COMMAND; void setActiveRequestCommand(COMMAND command); void resetActiveRequestCommand(); - void setExpecedSizeInRAM(size_t newSize){ + void setExpecedSizeInRAM(size_t newSize) + { expectedSizeInRam = newSize; } - size_t getExpectedSizeInRAM()const{ + size_t getExpectedSizeInRAM() const + { return expectedSizeInRam; } - - std::string getStatusString()const; - std::string getActiveRequestString()const; - bool isWaitingToBeLoaded()const{ + std::string getStatusString() const; + std::string getActiveRequestString() const; + + bool isWaitingToBeLoaded() const + { return WAITING_FOR_LOADING_STATUS == eStatus; } - bool isLoading()const{ + bool isLoading() const + { return LOADING_STATUS == eStatus; } - bool isLoaded()const{ + bool isLoaded() const + { return OPEN_STATUS == eStatus; } - bool isFailedToLoad()const{ + bool isFailedToLoad() const + { return FAILED_TO_OPEN_STATUS == eStatus; } - bool isReorganizeRunning()const{ + bool isReorganizeRunning() const + { return REORGANIZE_RUNNING_STATUS == eStatus; } - bool isImportRunning()const{ + bool isImportRunning() const + { return IMPORT_RUNNING_STATUS == eStatus; } - //bool fReorganizeOnly = false; - bool isOpenedOnlyForReorganize()const{ + // bool fReorganizeOnly = false; + bool isOpenedOnlyForReorganize() const + { return usAccessMode & ASD_ORGANIZE; } - void resetOpenedOnlyForReorganize(){ + void resetOpenedOnlyForReorganize() + { unsigned int mask = ~ASD_ORGANIZE; usAccessMode &= mask; } - void updateLastUseTime(){ + void updateLastUseTime() + { tLastAccessTime = time(0); } -/*! \brief OtmMemory related return codes + /*! \brief OtmMemory related return codes -*/ + */ static const int ERROR_INTERNALKEY_MISSING = 1001; -/*! \brief Constructors -*/ - + /*! \brief Constructors + */ -/*! \brief Destructor -*/ - ~EqfMemory(); + /*! \brief Destructor + */ + ~EqfMemory(); USHORT Load(); - -/*! \brief Info structure for an open translation memory -*/ - //typedef struct _OPENEDMEMORY + + /*! \brief Info structure for an open translation memory + */ + // typedef struct _OPENEDMEMORY //{ - //char szName[260]; // name of the memory - time_t tLastAccessTime = 0; // last time memory has been used - long lHandle = 0; // memory handle - MEMORY_STATUS eStatus; // status of the memory - MEMORY_STATUS eImportStatus; // status of the current/last memory import - //std::atomic dImportProcess; - //ushort * pusImportPersent = nullptr; - ImportStatusDetails* importDetails = nullptr; - std::string strError; // pointer to an error message (only when eStatus is IMPORT_FAILED_STATUS) - + // char szName[260]; // name of the memory + time_t tLastAccessTime = 0; // last time memory has been used + long lHandle = 0; // memory handle + MEMORY_STATUS eStatus; // status of the memory + MEMORY_STATUS eImportStatus; // status of the current/last memory import + // std::atomic dImportProcess; + // ushort * pusImportPersent = nullptr; + ImportStatusDetails *importDetails = nullptr; + std::string strError; // pointer to an error message (only when eStatus is IMPORT_FAILED_STATUS) + PPROP_NTM memoryProperties; - /*! \brief structure for memory information */ + /*! \brief structure for memory information */ // typedef struct _MEMORYINFO // { - - // } MEMORYINFO, *PMEMORYINFO; -//} OPENEDMEMORY ; -/*! \brief Error code definition -*/ - static const int ERROR_MEMORYOBJECTISNULL = 8003; - static const int ERROR_BUFFERTOOSMALL = 8004; - static const int ERROR_INVALIDOBJNAME = 8005; - static const int ERROR_MEMORYEXISTS = 8006; - static const int INFO_ENDREACHED = 8008; - static const int ERROR_ENTRYISCORRUPTED = 8009; + // } MEMORYINFO, *PMEMORYINFO; + //} OPENEDMEMORY ; + /*! \brief Error code definition + */ + static const int ERROR_MEMORYOBJECTISNULL = 8003; + static const int ERROR_BUFFERTOOSMALL = 8004; + static const int ERROR_INVALIDOBJNAME = 8005; + static const int ERROR_MEMORYEXISTS = 8006; + static const int INFO_ENDREACHED = 8008; + static const int ERROR_ENTRYISCORRUPTED = 8009; /*! \brief Flags for the update of proposals */ - static const int UPDATE_MARKUP = 0x01; // update markup/tag table - static const int UPDATE_MTFLAG = 0x02; // update machine translation flag - static const int UPDATE_TARGLANG = 0x04; // update target language - static const int UPDATE_DATE = 0x08; // update proposal update time + static const int UPDATE_MARKUP = 0x01; // update markup/tag table + static const int UPDATE_MTFLAG = 0x02; // update machine translation flag + static const int UPDATE_TARGLANG = 0x04; // update target language + static const int UPDATE_DATE = 0x08; // update proposal update time - //void setInternalCursor(ULONG recordKey, USHORT targetKey); - //void resetInternalCursor(); + // void setInternalCursor(ULONG recordKey, USHORT targetKey); + // void resetInternalCursor(); - ushort TmtXDelSegm -( - OtmProposal& pTmDelIn, //ptr to input struct - TMX_EXT_OUT_W* pTmDelOut //ptr to output struct -); + ushort TmtXDelSegm( + OtmProposal &pTmDelIn, // ptr to input struct + TMX_EXT_OUT_W *pTmDelOut // ptr to output struct + ); -USHORT TmtXDelSegmByKey -( - OtmProposal& TmDelIn, //ptr to input struct - TMX_EXT_OUT_W* pTmDelOut //ptr to output struct -); + USHORT TmtXDelSegmByKey( + OtmProposal &TmDelIn, // ptr to input struct + TMX_EXT_OUT_W *pTmDelOut // ptr to output struct + ); -USHORT FindTargetAndDelete( PTMX_RECORD pTmRecord, - OtmProposal& TmDel, - TMX_EXT_OUT_W * pTmExtOut, - PULONG pulKey ); - -USHORT FindTargetByKeyAndDelete( - PTMX_RECORD pTmRecord, - OtmProposal& pTmDel, - PTMX_SENTENCE pSentence, - TMX_EXT_OUT_W * pTmExtOut, - PULONG pulKey ); - - -USHORT TokenizeSource( PTMX_SENTENCE, PSZ, PSZ ); - -USHORT NTMGetIDFromName( PSZ, PSZ, USHORT, PUSHORT ); -USHORT NTMGetIDFromNameEx -( - PSZ pszName, // input, name being looked up - PSZ pszLongName, // input, long name (only for FILE_KEY) - USHORT usTableType, // input, type of table to use - PUSHORT pusID, // output, ID for name being looked up - LONG lOptions, - PUSHORT pusAlternativeID // output, alternative ID -); -USHORT NTMGetNameFromID( PUSHORT, USHORT, PSZ, PSZ ); -USHORT NTMGetPointersToTable( USHORT, PTMX_TABLE *, - PTMX_TABLE_ENTRY * ); -USHORT NTMAddNameToTable( PSZ, USHORT, PUSHORT ); -PSZ NTMFindNameForID( - PUSHORT pusID, //intput - USHORT usTableType ); - -USHORT FillClb( PTMX_TARGET_CLB, OtmProposal& ); -USHORT ComparePutData -( - PTMX_RECORD *ppTmRecord, // ptr to ptr of tm record data buffer - PULONG pulRecBufSize, // current size of record buffer - OtmProposal& TmProposal, // pointer to get in data - PULONG pulKey // tm key -); -USHORT AddTmTarget( - OtmProposal& TmProposal, //pointer to get in data - PTMX_RECORD *ppTmRecord, //pointer to tm record data pointer - PULONG pulRecBufSize, //ptr to current size of TM record buffer - PULONG pulKey ); //tm key + USHORT FindTargetAndDelete(PTMX_RECORD pTmRecord, + OtmProposal &TmDel, + TMX_EXT_OUT_W *pTmExtOut, + PULONG pulKey); + + USHORT FindTargetByKeyAndDelete( + PTMX_RECORD pTmRecord, + OtmProposal &pTmDel, + PTMX_SENTENCE pSentence, + TMX_EXT_OUT_W *pTmExtOut, + PULONG pulKey); + + USHORT TokenizeSource(PTMX_SENTENCE, PSZ, PSZ); + + USHORT NTMGetIDFromName(PSZ, PSZ, USHORT, PUSHORT); + USHORT NTMGetIDFromNameEx( + PSZ pszName, // input, name being looked up + PSZ pszLongName, // input, long name (only for FILE_KEY) + USHORT usTableType, // input, type of table to use + PUSHORT pusID, // output, ID for name being looked up + LONG lOptions, + PUSHORT pusAlternativeID // output, alternative ID + ); + USHORT NTMGetNameFromID(PUSHORT, USHORT, PSZ, PSZ); + USHORT NTMGetPointersToTable(USHORT, PTMX_TABLE *, + PTMX_TABLE_ENTRY *); + USHORT NTMAddNameToTable(PSZ, USHORT, PUSHORT); + PSZ NTMFindNameForID( + PUSHORT pusID, // intput + USHORT usTableType); + + USHORT FillClb(PTMX_TARGET_CLB, OtmProposal &); + USHORT ComparePutData( + std::vector &ppTmRecord, // ptr to ptr of tm record data buffer + OtmProposal &TmProposal, // pointer to get in data + PULONG pulKey // tm key + ); + USHORT AddTmTarget( + OtmProposal &TmProposal, // pointer to get in data + std::vector& pTmRecord, // pointer to tm record data pointer + PULONG pulKey); // tm key -/*! \brief Store the supplied proposal in the memory - When the proposal aready exists it will be overwritten with the supplied data + /*! \brief Store the supplied proposal in the memory + When the proposal aready exists it will be overwritten with the supplied data - \param pProposal pointer to a OtmProposal object + \param pProposal pointer to a OtmProposal object - \returns 0 or error code in case of errors -*/ - int putProposal - ( - OtmProposal &Proposal - ); - - //fuzzy search - USHORT TmtXGet - ( - PTMX_GET_W pTmGetIn, //ptr to input struct - PTMX_GET_OUT_W pTmGetOut //ptr to output struct - ); + \returns 0 or error code in case of errors + */ + int putProposal( + OtmProposal &Proposal); - // TM segment update prototypes - USHORT TmtXUpdSeg - ( - OtmProposal* pTmPutIn, // ptr to put input data - PTMX_EXT_OUT_W pTmPutOut, //ptr to output struct - USHORT usFlags // flags controlling the updated fields + // fuzzy search + USHORT TmtXGet( + PTMX_GET_W pTmGetIn, // ptr to input struct + PTMX_GET_OUT_W pTmGetOut // ptr to output struct ); - USHORT TmtXExtract - ( - PTMX_EXT_IN_W pTmExtIn, //ptr to input struct - TMX_EXT_OUT_W * pTmExtOut //ptr to output struct - ); - //tm extract prototypes - USHORT FillExtStructure( PTMX_TARGET_RECORD, - PTMX_TARGET_CLB, - PSZ_W, PLONG, PTMX_EXT_OUT_W ); + USHORT TmtXExtract( + PTMX_EXT_IN_W pTmExtIn, // ptr to input struct + TMX_EXT_OUT_W *pTmExtOut // ptr to output struct + ); + + // tm extract prototypes + USHORT FillExtStructure(PTMX_TARGET_RECORD, + PTMX_TARGET_CLB, + std::wstring&, PLONG, PTMX_EXT_OUT_W); int RewriteCompactTable(); - -/*! \brief Get the next proposal from the memory (with progress info) - \param lHandle the hande returned by GetFirstProposal - \param Proposal reference to a OtmProposal object which will be filled with the proposal data - \param piProgress pointer to buffer for progress indicator, this indicator goes from 0 up to 100 - \returns 0 or error code in case of errors -*/ - int getNextProposal - ( - OtmProposal &Proposal, - int *piProgress = nullptr - ); - - /*! \brief Get the current sequential access key (the key for the next proposal in the memory) - \param pszKeyBuffer pointer to the buffer to store the sequential access key - \param iKeyBufferSize size of the key buffer in number of characters - \returns 0 or error code in case of errors + + /*! \brief Get the next proposal from the memory (with progress info) + \param lHandle the hande returned by GetFirstProposal + \param Proposal reference to a OtmProposal object which will be filled with the proposal data + \param piProgress pointer to buffer for progress indicator, this indicator goes from 0 up to 100 + \returns 0 or error code in case of errors */ - int getSequentialAccessKey - ( - char *pszKeyBuffer, - int iKeyBufferSize - ); + int getNextProposal( + OtmProposal &Proposal, + int *piProgress = nullptr); + + /*! \brief Get the current sequential access key (the key for the next proposal in the memory) + \param pszKeyBuffer pointer to the buffer to store the sequential access key + \param iKeyBufferSize size of the key buffer in number of characters + \returns 0 or error code in case of errors +*/ + int getSequentialAccessKey( + char *pszKeyBuffer, + int iKeyBufferSize); + USHORT AddToTm(OtmProposal &, PULONG); + USHORT UpdateTmIndex(PTMX_SENTENCE, ULONG); - USHORT AddToTm( OtmProposal&, PULONG ); - USHORT UpdateTmIndex( PTMX_SENTENCE, ULONG ); + void importDone(int iRC, char *pszError); + void reorganizeDone(int iRC, char *pszError); - void importDone(int iRC, char *pszError ); - void reorganizeDone(int iRC, char *pszError ); - /*! \brief Set the current sequential access key to resume the sequential access at the given position \param pszKey a sequential access key previously returned by getSequentialAccessKey - \returns 0 or error code in case of errors + \returns 0 or error code in case of errors */ - int setSequentialAccessKey - ( - char *pszKey - ); - - size_t GetRAMSize()const; - size_t GetExpectedRAMSize()const; - - USHORT ExtractRecordV6 - ( - PTMX_RECORD pTmRecord, - PTMX_EXT_IN_W pTmExtIn, - TMX_EXT_OUT_W * pTmExtOut - ); + int setSequentialAccessKey( + char *pszKey); + size_t GetRAMSize() const; + size_t GetExpectedRAMSize() const; - USHORT TmtXReplace - ( - OtmProposal& TmProposal, //ptr to input struct - PTMX_EXT_OUT_W pTmPutOut //ptr to output struct - ); + USHORT ExtractRecordV6( + PTMX_RECORD pTmRecord, + PTMX_EXT_IN_W pTmExtIn, + TMX_EXT_OUT_W *pTmExtOut); -USHORT NTMLoadNameTable -( - ULONG ulTableKey, // key of table record - PTMX_TABLE pTMTable, // ptr to table data pointer - PULONG pulSize // ptr to buffer for size of table data -); + USHORT TmtXReplace( + OtmProposal &TmProposal, // ptr to input struct + PTMX_EXT_OUT_W pTmPutOut // ptr to output struct + ); -/*! \brief Get a list of memory proposals matching the given search key + USHORT NTMLoadNameTable( + ULONG ulTableKey, // key of table record + PTMX_TABLE pTMTable, // ptr to table data pointer + PULONG pulSize // ptr to buffer for size of table data + ); - This method uses the search data contained in the search key to find one or more - matching proposals in the memory. At least the szSource and the szTargetLang members of the - search key have to be filled by the caller. - The caller provides a list of OtmProposals which will be filled with the data of the matching - proposals. The number of requested proposals is determined by the number - of proposals in the list. + /*! \brief Get a list of memory proposals matching the given search key - \param SearchKey proposal containing search string and meta data - \param FoundProposals refernce to vector with OtmProposal objects - \param ulOptions options for the lookup + This method uses the search data contained in the search key to find one or more + matching proposals in the memory. At least the szSource and the szTargetLang members of the + search key have to be filled by the caller. + The caller provides a list of OtmProposals which will be filled with the data of the matching + proposals. The number of requested proposals is determined by the number + of proposals in the list. - \returns 0 or error code in case of errors -*/ - int SearchProposal - ( - OtmProposal &SearchKey, - std::vector &FoundProposals, - unsigned long ulOptions - ); - - USHORT UpdateTmRecord( OtmProposal& ); - - USHORT UpdateTmRecordByInternalKey - ( - OtmProposal& - ); - - int getNumOfMarkupNames(); + \param SearchKey proposal containing search string and meta data + \param FoundProposals refernce to vector with OtmProposal objects + \param ulOptions options for the lookup -/*! \brief Get markup name at position n [n = 0.. GetNumOfMarkupNames()-1] - \param iPos position of markup table name - \param pszBuffer pointer to a buffer for the markup table name - \param iSize size of buffer in number of characters - \returns number of characters copied to buffer -*/ - int getMarkupName - ( - int iPos, - char *pszBuffer, - int iSize - ); + \returns 0 or error code in case of errors + */ + int SearchProposal( + OtmProposal &SearchKey, + std::vector &FoundProposals, + unsigned long ulOptions); + USHORT UpdateTmRecord(OtmProposal &); -/*! \brief Get language at position n [n = 0.. GetNumOfLanguages()-1] - \param iPos position of language - \param pszBuffer pointer to a buffer for the document name - \param iSize size of buffer in number of characters - \returns number of characters copied to buffer -*/ - int getLanguage - ( - int iPos, - char *pszBuffer, - int iSize - ); - + USHORT UpdateTmRecordByInternalKey( + OtmProposal &); + int getNumOfMarkupNames(); -/*! \brief Get source language of the memory - \param pszBuffer pointer to a buffer for the source language name - \param iSize size of buffer in number of characters - \returns number of characters copied to the buffer -*/ - int getSourceLanguage - ( - char *pszBuffer, - int iSize - ); + /*! \brief Get markup name at position n [n = 0.. GetNumOfMarkupNames()-1] + \param iPos position of markup table name + \param pszBuffer pointer to a buffer for the markup table name + \param iSize size of buffer in number of characters + \returns number of characters copied to buffer + */ + int getMarkupName( + int iPos, + char *pszBuffer, + int iSize); + + /*! \brief Get language at position n [n = 0.. GetNumOfLanguages()-1] + \param iPos position of language + \param pszBuffer pointer to a buffer for the document name + \param iSize size of buffer in number of characters + \returns number of characters copied to buffer + */ + int getLanguage( + int iPos, + char *pszBuffer, + int iSize); + + /*! \brief Get source language of the memory + \param pszBuffer pointer to a buffer for the source language name + \param iSize size of buffer in number of characters + \returns number of characters copied to the buffer + */ + int getSourceLanguage( + char *pszBuffer, + int iSize); -/*! \brief Get the name of the memory - \param pszBuffer pointer to a buffer for the name - \param iSize size of buffer in number of characters - \returns number of characters copied to the buffer -*/ - int getName - ( - char *pszBuffer, - int iSize - ); + /*! \brief Get the name of the memory + \param pszBuffer pointer to a buffer for the name + \param iSize size of buffer in number of characters + \returns number of characters copied to the buffer + */ + int getName( + char *pszBuffer, + int iSize); /*! \brief Get the name of the memory \param strName reference of a string receiving the memory name */ - int getName - ( - std::string &strName - ); - + int getName( + std::string &strName); -/*! \brief Get description of the memory - \param pszBuffer pointer to a buffer for the description - \param iSize size of buffer in number of characters - \returns number of characters copied to the buffer -*/ - int getDescription - ( - char *pszBuffer, - int iSize - ); + /*! \brief Get description of the memory + \param pszBuffer pointer to a buffer for the description + \param iSize size of buffer in number of characters + \returns number of characters copied to the buffer + */ + int getDescription( + char *pszBuffer, + int iSize); /*! \brief Set the description of the memory \param pszBuffer pointer to the description text */ - void setDescription - ( - char *pszBuffer - ); + void setDescription( + char *pszBuffer); -/*! \brief Get plugin responsible for this memory - \returns pointer to memory plugin object -*/ + /*! \brief Get plugin responsible for this memory + \returns pointer to memory plugin object + */ void *getPlugin(); - -/*! \brief Get overall file size of this memory - \returns size of memory in number of bytes -*/ + /*! \brief Get overall file size of this memory + \returns size of memory in number of bytes + */ unsigned long getFileSize(); -/*! \brief Get the error message for the last error occured + /*! \brief Get the error message for the last error occured - \param strError reference to a string receiving the error mesage text - \returns last error code -*/ - int getLastError - ( - std::string &strError - ); -/*! \brief Get the error message for the last error occured - - \param pszError pointer to a buffer for the error text - \param iBufSize size of error text buffer in number of characters - \returns last error code -*/ - int getLastError - ( - char *pszError, - int iBufSize - ); - -/*! \brief Set or clear the pointer to a loaded global memory option file + \param strError reference to a string receiving the error mesage text + \returns last error code + */ + int getLastError( + std::string &strError); + /*! \brief Get the error message for the last error occured - This method sets a pointer to a loaded global memory option file. - When set the option file will be used to decide how global memory proposals will be processed. + \param pszError pointer to a buffer for the error text + \param iBufSize size of error text buffer in number of characters + \returns last error code + */ + int getLastError( + char *pszError, + int iBufSize); - \param pvGlobalMemoryOptions pointer to a loaded global memory option file or NULL to clear the current option file pointer + /*! \brief Set or clear the pointer to a loaded global memory option file - \returns 0 or error code in case of errors -*/ - int setGlobalMemoryOptions - ( - void *pvGlobalMemoryOptions - ); + This method sets a pointer to a loaded global memory option file. + When set the option file will be used to decide how global memory proposals will be processed. - std::string strLastError; - int iLastError = 0; - void *pvGlobalMemoryOptions = nullptr; // pointert to global memory options to be used for global memory proposals + \param pvGlobalMemoryOptions pointer to a loaded global memory option file or NULL to clear the current option file pointer + + \returns 0 or error code in case of errors + */ + int setGlobalMemoryOptions( + void *pvGlobalMemoryOptions); + + std::string strLastError; + int iLastError = 0; + void *pvGlobalMemoryOptions = nullptr; // pointert to global memory options to be used for global memory proposals public: - char szPlugin[256]; // name of the plugin controlling this memory - char szName[256]; // name of the memory - char szDescription[256]; // description of the memory - char szFullPath[256]; // full path to memory file(s) (if applicable only) - char szFullDataFilePath[256]; // full path to memory file(s) (if applicable only) - char szFullIndexFilePath[256]; // full path to memory file(s) (if applicable only) - char szSourceLanguage[MAX_LANG_LENGTH+1]; // memory source language - char szOwner[256]; // ID of the memory owner - char szDescrMemoryType[256]; // descriptive name of the memory type - unsigned long ulSize = 0; // size of the memory - BOOL fEnabled = 0; // memory-is-enabled flag - - std::string getName()const { return szName; } + char szPlugin[256]; // name of the plugin controlling this memory + char szName[256]; // name of the memory + char szDescription[256]; // description of the memory + char szFullPath[256]; // full path to memory file(s) (if applicable only) + char szFullDataFilePath[256]; // full path to memory file(s) (if applicable only) + char szFullIndexFilePath[256]; // full path to memory file(s) (if applicable only) + char szSourceLanguage[MAX_LANG_LENGTH + 1]; // memory source language + char szOwner[256]; // ID of the memory owner + char szDescrMemoryType[256]; // descriptive name of the memory type + unsigned long ulSize = 0; // size of the memory + BOOL fEnabled = 0; // memory-is-enabled flag + + std::string getName() const { return szName; } // functions dealing with long document tables USHORT NTMCreateLongNameTable(); USHORT NTMReadLongNameTable(); USHORT NTMWriteLongNameTable(); USHORT NTMDestroyLongNameTable(); USHORT NTMCreateLangGroupTable(); - USHORT NTMAddLangGroup( PSZ pszLang, USHORT sLangID ); + USHORT NTMAddLangGroup(PSZ pszLang, USHORT sLangID); USHORT NTMOrganizeIndexFile(); -//private: -/*! \brief Fill OtmProposal from TMX_GET_OUT_W structure - \param ulKey key of record containing the proposal - \param usTargetNum number of target within record - \param Proposal reference to the OtmProposal being filled - \returns 0 or error code in case of errors -*/ -int SetProposalKey -( - ULONG ulKey, - USHORT usTargetNum, - OtmProposal *pProposal -); - - -/*! \brief Fill OtmProposal from TMX_GET_OUT_W structure - \param pExtOut pointer to the TMX_GET_OUT_W structure - \param Proposal reference to the OtmProposal being filled - \returns 0 or error code in case of errors -*/ -int ExtOutToOtmProposal -( - TMX_EXT_OUT_W* pExtOut, - OtmProposal &Proposal -); - -/*! \brief Fill OtmProposal from TMX_MATCH_TABLE_W structure - \param pMatch pointer to the TMX_MATCH_TABLE_W structure - \param Proposal reference to the OtmProposal being filled - \returns 0 or error code in case of errors -*/ -int MatchToOtmProposal -( - PTMX_MATCH_TABLE_W pMatch, - OtmProposal *pProposal -); - - -/*! \brief Fill TMX_GET_W structure with OtmProposal data - \param Proposal reference to the OtmProposal containing the data - \param pGetIn pointer to the TMX_GET_W structure - \returns 0 or error code in case of errors -*/ -int OtmProposalToGetIn -( - OtmProposal &Proposal, - PTMX_GET_W pGetIn -); - -int OtmProposalToPutIn -( - OtmProposal &Proposal, - PTMX_PUT_W pPutIn -); - -/*! \brief Handle a return code from the memory functions and create - the approbriate error message text for it - \param iRC return code from memory function - \param pszMemName long memory name - \param pszMarkup markup table name - \returns original or modified error return code -*/ -int handleError( int iRC, char *pszMemName, char *pszMarkup = "OTMXUXLF" ); - + // private: + /*! \brief Fill OtmProposal from TMX_GET_OUT_W structure + \param ulKey key of record containing the proposal + \param usTargetNum number of target within record + \param Proposal reference to the OtmProposal being filled + \returns 0 or error code in case of errors + */ + int SetProposalKey( + ULONG ulKey, + USHORT usTargetNum, + OtmProposal *pProposal); + + /*! \brief Fill OtmProposal from TMX_GET_OUT_W structure + \param pExtOut pointer to the TMX_GET_OUT_W structure + \param Proposal reference to the OtmProposal being filled + \returns 0 or error code in case of errors + */ + int ExtOutToOtmProposal( + TMX_EXT_OUT_W *pExtOut, + OtmProposal &Proposal); + + /*! \brief Fill OtmProposal from TMX_MATCH_TABLE_W structure + \param pMatch pointer to the TMX_MATCH_TABLE_W structure + \param Proposal reference to the OtmProposal being filled + \returns 0 or error code in case of errors + */ + int MatchToOtmProposal( + PTMX_MATCH_TABLE_W pMatch, + OtmProposal *pProposal); + + /*! \brief Fill TMX_GET_W structure with OtmProposal data + \param Proposal reference to the OtmProposal containing the data + \param pGetIn pointer to the TMX_GET_W structure + \returns 0 or error code in case of errors + */ + int OtmProposalToGetIn( + OtmProposal &Proposal, + PTMX_GET_W pGetIn); + + int OtmProposalToPutIn( + OtmProposal &Proposal, + PTMX_PUT_W pPutIn); + + /*! \brief Handle a return code from the memory functions and create + the approbriate error message text for it + \param iRC return code from memory function + \param pszMemName long memory name + \param pszMarkup markup table name + \returns original or modified error return code + */ + int handleError(int iRC, char *pszMemName, char *pszMarkup = "OTMXUXLF"); -/// newTM code -public: - EqfMemory(const std::string& tmName); + /// newTM code +public: + EqfMemory(const std::string &tmName); bool IsLoadedInRAM(); int ReloadFromDisk(); int UnloadFromRAM(); - int FlushFilebuffers(MutexTimeout& tmLockTimeout); + int FlushFilebuffers(MutexTimeout &tmLockTimeout); -private: +private: }; - -OtmProposal::eProposalType FlagToProposalType( USHORT usTranslationFlag ); +OtmProposal::eProposalType FlagToProposalType(USHORT usTranslationFlag); BYTE ProposalTypeToFlag(OtmProposal::eProposalType t); -USHORT TmGetServerDrives( PDRIVES_IN pDrivesIn, // Pointer to DRIVES_IN struct - PDRIVES_OUT pDrivesOut, // Pointer to DRIVES_OUT struct - USHORT usMsgHandling );// Message handling parameter -USHORT TmGetServerDrivesHwnd( PDRIVES_IN, PDRIVES_OUT, USHORT, HWND ); -USHORT TmGetServerTMs( PFILE_LIST_IN pTmListIn, // Pointer to FILE_LIST_IN struct !!! CHM - PFILE_LIST_OUT pTmListOut, // Pointer to FILE_LIST_OUT struct !!! CHM - USHORT usMsgHandling ); // Message handling parameter -USHORT TmGetTMPart( HTM htm, // Memory database handle - PSZ pszMemPat, // Full translation memory path - PGETPART_IN pGetPartIn, // Pointer to GETPART_IN struct - PGETPART_OUT pGetPartOut, // Pointer to GETPART_OUT struct - USHORT usMsgHandling ); // Message handling parameter -USHORT TmSendTMProp( SERVERNAME szServer, // Servername - PSZ pszMemName, // Translation memory name - USHORT usMsgHandling ); // Message handling parameter -USHORT TmReceiveTMProp( SERVERNAME szServer, // Servername - PSZ pszMemName, // Translation memory name - USHORT usMsgHandling ); // Message handling parameter - -//use this define as usMsgHandling for special handling of greyedout TMs +USHORT TmGetServerDrives(PDRIVES_IN pDrivesIn, // Pointer to DRIVES_IN struct + PDRIVES_OUT pDrivesOut, // Pointer to DRIVES_OUT struct + USHORT usMsgHandling); // Message handling parameter +USHORT TmGetServerDrivesHwnd(PDRIVES_IN, PDRIVES_OUT, USHORT, HWND); +USHORT TmGetServerTMs(PFILE_LIST_IN pTmListIn, // Pointer to FILE_LIST_IN struct !!! CHM + PFILE_LIST_OUT pTmListOut, // Pointer to FILE_LIST_OUT struct !!! CHM + USHORT usMsgHandling); // Message handling parameter +USHORT TmGetTMPart(HTM htm, // Memory database handle + PSZ pszMemPat, // Full translation memory path + PGETPART_IN pGetPartIn, // Pointer to GETPART_IN struct + PGETPART_OUT pGetPartOut, // Pointer to GETPART_OUT struct + USHORT usMsgHandling); // Message handling parameter +USHORT TmSendTMProp(SERVERNAME szServer, // Servername + PSZ pszMemName, // Translation memory name + USHORT usMsgHandling); // Message handling parameter +USHORT TmReceiveTMProp(SERVERNAME szServer, // Servername + PSZ pszMemName, // Translation memory name + USHORT usMsgHandling); // Message handling parameter + +// use this define as usMsgHandling for special handling of greyedout TMs #define DELETE_GREYEDOUT 2 -USHORT TmDeleteFile( SERVERNAME szServer, // Servername - PSZ pszFilePath, // Full file path - USHORT usMsgHandling ); // Message handling parameter -USHORT TmGetServerFileInfo( SERVERNAME szServer, //Servername - PSZ pszFilePath, //Full file path - PFILEFINDBUF pstFile, // File info - USHORT usMsgHandling ); // Message handling parameter -USHORT TmGetServerFileInfoHwnd( SERVERNAME, PSZ, PFILEFINDBUF, USHORT, HWND ); -USHORT TmCompareLocalRemoteProperties( SERVERNAME, PSZ, USHORT ); -USHORT MemRcHandling( USHORT, PSZ, HTM *, PSZ ); -USHORT MemRcHandlingHwnd( USHORT, PSZ, HTM *, PSZ, HWND ); +USHORT TmDeleteFile(SERVERNAME szServer, // Servername + PSZ pszFilePath, // Full file path + USHORT usMsgHandling); // Message handling parameter +USHORT TmGetServerFileInfo(SERVERNAME szServer, // Servername + PSZ pszFilePath, // Full file path + PFILEFINDBUF pstFile, // File info + USHORT usMsgHandling); // Message handling parameter +USHORT TmGetServerFileInfoHwnd(SERVERNAME, PSZ, PFILEFINDBUF, USHORT, HWND); +USHORT TmCompareLocalRemoteProperties(SERVERNAME, PSZ, USHORT); +USHORT MemRcHandling(USHORT, PSZ, HTM *, PSZ); +USHORT MemRcHandlingHwnd(USHORT, PSZ, HTM *, PSZ, HWND); /**********************************************************************/ /* MemReadWriteSegment */ @@ -2401,89 +2359,87 @@ USHORT MemRcHandlingHwnd( USHORT, PSZ, HTM *, PSZ, HWND ); // TMRWS_NOMSG: show no error message at all // TMRWS_ALLMSG: show all error messages // TMRWS_WRITEMSG: show only error messages for write segment (for TM organize!) -#define TMRWS_NOMSG 0 -#define TMRWS_ALLMSG 1 +#define TMRWS_NOMSG 0 +#define TMRWS_ALLMSG 1 #define TMRWS_WRITEMSG 2 -USHORT NTMReadWriteSegmentW( HTM, //Handle of Input TM - PSZ, //Full path to Input TM - HTM, //Handle of Output TM - PSZ, //Full path to Output TM - PTMX_EXT_IN_W,//Pointer to the EXTRACT_IN structure - TMX_EXT_OUT_W *, //Pointer to the EXTRACT_OUT structure - PTMX_PUT_W, //Pointer to the REPLACE_OUT structure - PTMX_EXT_OUT_W, //Pointer to the REPLACE_OUT structure - ULONG *, //Pointer to a segment counter - ULONG *, //Pointer to the invalid segment counter - PSZ, //pointer to TM source language - USHORT ); //message handling flags -USHORT NTMReadWriteSegmentHwndW( HTM, //Handle of Input TM - PSZ, //Full path to Input TM - HTM, //Handle of Output TM - PSZ, //Full path to Output TM - PTMX_EXT_IN_W, //Pointer to the EXTRACT_IN structure - TMX_EXT_OUT_W *, //Pointer to the EXTRACT_OUT structure - PTMX_PUT_W, //Pointer to the REPLACE_OUT structure - PTMX_EXT_OUT_W, //Pointer to the REPLACE_OUT structure - ULONG *, //Pointer to a segment counter - ULONG *, //Pointer to the invalid segment counter - PSZ, //pointer to TM source language - USHORT, //message handling flags - HWND ); //handle for error messages parent - - - -BOOL fMemIsAvail( PSZ, PSZ, PSZ ); // memory is available and accessible -USHORT MemFillCBNames( HWND, PSZ, PSZ ); -USHORT MemFillTableLB( HWND hListBox, USHORT usBoxType, PSZ pszLastUsed ); +USHORT NTMReadWriteSegmentW(HTM, // Handle of Input TM + PSZ, // Full path to Input TM + HTM, // Handle of Output TM + PSZ, // Full path to Output TM + PTMX_EXT_IN_W, // Pointer to the EXTRACT_IN structure + TMX_EXT_OUT_W *, // Pointer to the EXTRACT_OUT structure + PTMX_PUT_W, // Pointer to the REPLACE_OUT structure + PTMX_EXT_OUT_W, // Pointer to the REPLACE_OUT structure + ULONG *, // Pointer to a segment counter + ULONG *, // Pointer to the invalid segment counter + PSZ, // pointer to TM source language + USHORT); // message handling flags +USHORT NTMReadWriteSegmentHwndW(HTM, // Handle of Input TM + PSZ, // Full path to Input TM + HTM, // Handle of Output TM + PSZ, // Full path to Output TM + PTMX_EXT_IN_W, // Pointer to the EXTRACT_IN structure + TMX_EXT_OUT_W *, // Pointer to the EXTRACT_OUT structure + PTMX_PUT_W, // Pointer to the REPLACE_OUT structure + PTMX_EXT_OUT_W, // Pointer to the REPLACE_OUT structure + ULONG *, // Pointer to a segment counter + ULONG *, // Pointer to the invalid segment counter + PSZ, // pointer to TM source language + USHORT, // message handling flags + HWND); // handle for error messages parent + +BOOL fMemIsAvail(PSZ, PSZ, PSZ); // memory is available and accessible +USHORT MemFillCBNames(HWND, PSZ, PSZ); +USHORT MemFillTableLB(HWND hListBox, USHORT usBoxType, PSZ pszLastUsed); /**********************************************************************/ /* TmOpen */ /**********************************************************************/ -USHORT TmOpen( PSZ, HTM *, USHORT, USHORT, //(in) location: TM_LOCAL - // TM_REMOTE - // TM_LOCAL_REMOTE - USHORT, //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message - HWND ); //(in) window handle for error messages +USHORT TmOpen(PSZ, HTM *, USHORT, USHORT, //(in) location: TM_LOCAL + // TM_REMOTE + // TM_LOCAL_REMOTE + USHORT, //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message + HWND); //(in) window handle for error messages /**********************************************************************/ /* TmClose */ /**********************************************************************/ USHORT -TmClose( HTM, //(in) TM handle returned from open - PSZ, //(in) full TM name x:\eqf\mem\mem.mem - USHORT, //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message - HWND ); //(in) window handle for error messages +TmClose(HTM, //(in) TM handle returned from open + PSZ, //(in) full TM name x:\eqf\mem\mem.mem + USHORT, //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message + HWND); //(in) window handle for error messages /**********************************************************************/ /* TmReplace */ /**********************************************************************/ USHORT -TmReplace( HTM, PSZ, PTMX_PUT_W, PTMX_EXT_OUT_W, USHORT, HWND ); +TmReplace(HTM, PSZ, PTMX_PUT_W, PTMX_EXT_OUT_W, USHORT, HWND); /**********************************************************************/ /* TmGet */ /**********************************************************************/ USHORT -TmGetW(EqfMemory*, //(in) TM handle - PSZ, //(in) full TM name x:\eqf\mem\mem.tmd +TmGetW(EqfMemory *, //(in) TM handle + PSZ, //(in) full TM name x:\eqf\mem\mem.tmd PTMX_GET_W, //(in) pointer to get input structure - PTMX_GET_OUT_W, //(out) pointer to get output structure - USHORT ); //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message + PTMX_GET_OUT_W, //(out) pointer to get output structure + USHORT); //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message /**********************************************************************/ /* TmExtract */ /**********************************************************************/ USHORT -TmExtractW( HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT ); +TmExtractW(HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT); USHORT -TmExtractHwndW( HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT, HWND ); +TmExtractHwndW(HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT, HWND); /**********************************************************************/ /* information levels for TmInfo function call */ @@ -2494,319 +2450,306 @@ TmExtractHwndW( HTM, PSZ, PTMX_EXT_IN_W, TMX_EXT_OUT_W *, USHORT, HWND ); /**********************************************************************/ /* TmDeleteTM */ /**********************************************************************/ -USHORT TmDeleteTM( PSZ, USHORT, HWND, PUSHORT ); +USHORT TmDeleteTM(PSZ, USHORT, HWND, PUSHORT); /**********************************************************************/ /* TmDeleteSegment */ /**********************************************************************/ USHORT -TmDeleteSegmentW( HTM, PSZ, PTMX_PUT_W, PTMX_EXT_OUT_W, USHORT ); - - +TmDeleteSegmentW(HTM, PSZ, PTMX_PUT_W, PTMX_EXT_OUT_W, USHORT); /**********************************************************************/ /* NTMConvertCRLF */ /**********************************************************************/ VOID -NTMConvertCRLF( PSZ, - PSZ, - USHORT ); -VOID NTMConvertCRLFW( PSZ_W, PSZ_W, USHORT ); - + NTMConvertCRLF(PSZ, + PSZ, + USHORT); +VOID NTMConvertCRLFW(PSZ_W, PSZ_W, USHORT); /**********************************************************************/ /* NTMOpenProperties */ /**********************************************************************/ USHORT -NTMOpenProperties( HPROP *, - PVOID *, - PSZ, - PSZ, - USHORT, - BOOL ); - - - +NTMOpenProperties(HPROP *, + PVOID *, + PSZ, + PSZ, + USHORT, + BOOL); // Update specific parts of segment // flags controlling the updated segment parts -#define TMUPDSEG_MARKUP 0x01 // update markup/tag table -#define TMUPDSEG_MTFLAG 0x02 // update MT flag -#define TMUPDSEG_TARGLANG 0x04 // update target language -#define TMUPDSEG_DATE 0x08 // update segment time - - -USHORT TmUpdSegW -( - HTM htm, //(in) TM handle - PSZ szMemPath, //(in) full TM name x:\eqf\mem\mem - PTMX_PUT_W pstPutIn, //(in) pointer to put input structure - ULONG ulUpdKey, //(in) key of record being updated - USHORT usUpdTarget, //(in) number of target being updated - USHORT usFlags, //(in) flags controlling the updated fields - USHORT usMsgHandling //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message +#define TMUPDSEG_MARKUP 0x01 // update markup/tag table +#define TMUPDSEG_MTFLAG 0x02 // update MT flag +#define TMUPDSEG_TARGLANG 0x04 // update target language +#define TMUPDSEG_DATE 0x08 // update segment time + +USHORT TmUpdSegW( + HTM htm, //(in) TM handle + PSZ szMemPath, //(in) full TM name x:\eqf\mem\mem + PTMX_PUT_W pstPutIn, //(in) pointer to put input structure + ULONG ulUpdKey, //(in) key of record being updated + USHORT usUpdTarget, //(in) number of target being updated + USHORT usFlags, //(in) flags controlling the updated fields + USHORT usMsgHandling //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message ); -USHORT TmUpdSegHwndW -( - HTM htm, //(in) TM handle - PSZ szMemPath, //(in) full TM name x:\eqf\mem\mem - PTMX_PUT_W pstPutIn, //(in) pointer to put input structure - ULONG ulUpdKey, //(in) key of record being updated - USHORT usUpdTarget, //(in) number of target being updated - USHORT usFlags, //(in) flags controlling the updated fields - USHORT usMsgHandling, //(in) message handling parameter - // TRUE: display error message - // FALSE: display no error message - HWND hwnd //(in) handle for error messages +USHORT TmUpdSegHwndW( + HTM htm, //(in) TM handle + PSZ szMemPath, //(in) full TM name x:\eqf\mem\mem + PTMX_PUT_W pstPutIn, //(in) pointer to put input structure + ULONG ulUpdKey, //(in) key of record being updated + USHORT usUpdTarget, //(in) number of target being updated + USHORT usFlags, //(in) flags controlling the updated fields + USHORT usMsgHandling, //(in) message handling parameter + // TRUE: display error message + // FALSE: display no error message + HWND hwnd //(in) handle for error messages ); - - // structure for MT_TMMERGE pointer typedef struct _MT_TMMERGE { - CHAR chMemory[MAX_EQF_PATH]; // TM name to be merged into - CHAR chSGMLFile[MAX_EQF_PATH]; // external memory - OBJNAME szObjName; // document object name - HWND hwndNotify; // parent to be notified - CHAR szTargetLang[MAX_LANG_LENGTH]; - OBJNAME szFolObjName; // object name of folder + CHAR chMemory[MAX_EQF_PATH]; // TM name to be merged into + CHAR chSGMLFile[MAX_EQF_PATH]; // external memory + OBJNAME szObjName; // document object name + HWND hwndNotify; // parent to be notified + CHAR szTargetLang[MAX_LANG_LENGTH]; + OBJNAME szFolObjName; // object name of folder } MT_TMMERGE, *PMT_TMMERGE; -BOOL MTTMMergeStart -( - PLISTCOMMAREA pCommArea, - PMT_TMMERGE pTMMerge -); - +BOOL MTTMMergeStart( + PLISTCOMMAREA pCommArea, + PMT_TMMERGE pTMMerge); // return codes of function MemConvertMem // memory converted successfully -#define MEM_CONVERTMEM_SUCCESS 0 +#define MEM_CONVERTMEM_SUCCESS 0 // memory conversion not required, memory is in new format already #define MEM_CONVERTMEM_ALREADYNEWFORMAT 7001 - // do a cleanup of temporary memories -void TMCleanupTempMem -( - PSZ pszPrefix // ptr to memory prefix +void TMCleanupTempMem( + PSZ pszPrefix // ptr to memory prefix ); // create a temporary memory -USHORT TMCreateTempMem -( - PSZ pszPrefix, // short prefix to be used for memory name (should start with a dollar sign) - PSZ pszMemPath, // ptr to buffer for memory path - HTM *pHtm, // ptr to buffer for memory handle - HTM htm, // htm of similar memory - PSZ pszSourceLang, // memory source language - HWND hwnd // window handle for error messages +USHORT TMCreateTempMem( + PSZ pszPrefix, // short prefix to be used for memory name (should start with a dollar sign) + PSZ pszMemPath, // ptr to buffer for memory path + HTM *pHtm, // ptr to buffer for memory handle + HTM htm, // htm of similar memory + PSZ pszSourceLang, // memory source language + HWND hwnd // window handle for error messages ); // delete a temporary memory -void TMDeleteTempMem -( - PSZ pszMemPath // ptr to memory path +void TMDeleteTempMem( + PSZ pszMemPath // ptr to memory path ); -BOOL TMFuzzynessEx -( - PSZ pszMarkup, // markup table name - PSZ_W pszSource, // original string - PSZ_W pszMatch, // found match - SHORT sLanguageId, // language id to be used - PUSHORT pusFuzzy, // fuzzyness - ULONG ulOemCP, - PUSHORT pusWords, // number of words in segment - PUSHORT pusDiffs // number of diffs in segment +BOOL TMFuzzynessEx( + PSZ pszMarkup, // markup table name + PSZ_W pszSource, // original string + PSZ_W pszMatch, // found match + SHORT sLanguageId, // language id to be used + PUSHORT pusFuzzy, // fuzzyness + ULONG ulOemCP, + PUSHORT pusWords, // number of words in segment + PUSHORT pusDiffs // number of diffs in segment ); - -USHORT NTMMorphTokenizeW( SHORT, PSZ_W, PULONG, PVOID*, USHORT); - -BOOL NTMTagSubst // generic inline tagging for TM -( - PTMX_SUBSTPROP pSubstProp, - ULONG ulSrcOemCP, - ULONG ulTgtOemCP +/**********************************************************************/ +/* Structure to access MORPH_FLAG_OFFSLIST type term lists */ +/**********************************************************************/ +typedef struct _FLAGOFFSLIST +{ + LONG lFlags; // flags of term + USHORT usLen; // length of term + USHORT usOffs; // offset of term +} FLAGOFFSLIST, *PFLAGOFFSLIST; + +USHORT NTMMorphTokenizeW(SHORT, PSZ_W, std::vector &); + +USHORT MorphAddTermToList2W( + std::vector &pList, // ptr to term list pointer + PSZ_W pszTerm, // ptr to new term being added to list + USHORT usLength, // length of term + USHORT usOffs, // offset of term + ULONG ulFlags // flags for the ter ); -BOOL NTMDocMatch( PSZ pszShort1, PSZ pszLong1, PSZ pszShort2, PSZ pszLong2 ); +BOOL NTMTagSubst // generic inline tagging for TM + ( + PTMX_SUBSTPROP pSubstProp, + ULONG ulSrcOemCP, + ULONG ulTgtOemCP); +BOOL NTMDocMatch(PSZ pszShort1, PSZ pszLong1, PSZ pszShort2, PSZ pszLong2); -void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); +void NTMFreeSubstProp(PTMX_SUBSTPROP pSubstProp); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /*! tm.h Internal header file for Translation Memory functions - Copyright (c) 1990-2016, International Business Machines Corporation and others. All rights reserved. + Copyright (c) 1990-2016, International Business Machines Corporation and others. All rights reserved. */ -//#ifndef _EQFTMI_H_ -//#define _EQFTMI_H_ +// #ifndef _EQFTMI_H_ +// #define _EQFTMI_H_ /**********************************************************************/ /* This include file requires EQFDDE.H!!! */ /**********************************************************************/ #ifndef EQFDDE_INCLUDED - #include "EQFDDE.H" +#include "EQFDDE.H" #endif // include external memory export/import interface #include "EQFMEMIE.H" - #ifndef FIELDOFFSET - #define FIELDOFFSET(type, field) (LOWORD(&(((type *)0)->field))) +#define FIELDOFFSET(type, field) (LOWORD(&(((type *)0)->field))) #endif // file extension used for shared memory (LAN based) property files in property directory #define LANSHARED_MEM_PROP ".SLM" - /*---------------------------------------------------------------------*\ #Define \*---------------------------------------------------------------------*/ // memory export and import formats -#define MEM_SGMLFORMAT_ANSI 1 -#define MEM_SGMLFORMAT_ASCII 2 -#define MEM_SGMLFORMAT_UNICODE 3 -#define MEM_FORMAT_TMX 4 +#define MEM_SGMLFORMAT_ANSI 1 +#define MEM_SGMLFORMAT_ASCII 2 +#define MEM_SGMLFORMAT_UNICODE 3 +#define MEM_FORMAT_TMX 4 // the same define value is used for the import as TMX (Trados) and for export as TMX (UTF-8) -#define MEM_FORMAT_TMX_UTF8 5 -#define MEM_FORMAT_TMX_TRADOS 5 +#define MEM_FORMAT_TMX_UTF8 5 +#define MEM_FORMAT_TMX_TRADOS 5 -#define MEM_FORMAT_TMX_NOCRLF 6 +#define MEM_FORMAT_TMX_NOCRLF 6 #define MEM_FORMAT_TMX_UTF8_NOCRLF 7 -#define MEM_FORMAT_XLIFF_MT 8 - +#define MEM_FORMAT_XLIFF_MT 8 // filter for mem SGML formats (same sequence as memory export and import formats required) -//#define MEM_FORMAT_FILTERS "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML Unicode\0*.*\0\0\0" +// #define MEM_FORMAT_FILTERS "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML Unicode\0*.*\0\0\0" #define MEM_FORMAT_FILTERS "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML UTF-16\0*.*\0TMX\0*.*\0TMX (Trados)\0*.*\0XLIFF (MT)\0*.*\0\0\0" // filter for mem SGML formats (same sequence as memory export and import formats required) -//#define MEM_FORMAT_FILTERS_EXP "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML Unicode\0*.*\0\0\0" +// #define MEM_FORMAT_FILTERS_EXP "SGML ANSI\0*.*\0SGML ASCII\0*.*\0SGML Unicode\0*.*\0\0\0" #define MEM_FORMAT_FILTERS_EXP "SGML ANSI (*.EXP)\0*.EXP\0SGML ASCII (*.EXP)\0*.EXP\0SGML UTF-16 (*.EXP)\0*.EXP\0TMX (UTF-16) (*.TMX)\0*.TMX\0TMX (UTF-8) (*.TMX)\0*.TMX\0TMX (UTF-16) (remove CRLF) (*.TMX)\0*.TMX\0TMX (UTF-8) (remove CRLF) (*.TMX)\0*.TMX\0\0\0" // defines for retries in case of BTREE_IN_USE conditions #define MAX_RETRY_COUNT 30 -#define MAX_WAIT_TIME 100 +#define MAX_WAIT_TIME 100 -#define TMT_CODE_VERSION 5 // Code version -#define TM_PREFIX "EQFTMT$$" // TM DB identifier -#define HUNDRED 100 // +#define TMT_CODE_VERSION 5 // Code version +#define TM_PREFIX "EQFTMT$$" // TM DB identifier +#define HUNDRED 100 // // ID to be used if a table overflow occurs -#define OVERFLOW_ID 32000 +#define OVERFLOW_ID 32000 // name to be used if a table overflow occurs -#define OVERFLOW_NAME "$$$$$$$$" +#define OVERFLOW_NAME "$$$$$$$$" -#define LAST_CLUSTER_NUMBER 4095 +#define LAST_CLUSTER_NUMBER 4095 //--- size of the message buffer for the Error message /*@1108A*/ //--- ERROR_MEM_SEGMENT_TOO_LARGE_CON /*@1108A*/ -#define MSG_BUFFER 35 /*@1108A*/ +#define MSG_BUFFER 35 /*@1108A*/ - -#define MIN_SPOOL_SIZE 20480 // 20k bytes for preformatted TM blocks -#define SECTOR_SIZE 4096 // Disk sector. Fixed now, dynamic later -#define BLOCK_HEADER_SIZE 8 // size of block header -#define MAX_TM_HEADER_SIZE 47600 // max size of TM header -#define MAX_BLOCK_SIZE 4096 // Maximum block size +#define MIN_SPOOL_SIZE 20480 // 20k bytes for preformatted TM blocks +#define SECTOR_SIZE 4096 // Disk sector. Fixed now, dynamic later +#define BLOCK_HEADER_SIZE 8 // size of block header +#define MAX_TM_HEADER_SIZE 47600 // max size of TM header +#define MAX_BLOCK_SIZE 4096 // Maximum block size #define MAX_TEXT_TAB_SIZE (MAX_TAGS_TAB_ENTRIES * sizeof(TOKENENTRY)) #define MAX_WORDS_TAB_SIZE (MAX_WORDS_TAB_ENTRIES * sizeof(TOKENENTRY)) -#define MAX_TM_WORK_AREA ( MAX_TGT_LENGTH + 300 ) - /* Work areas to convert LF into CRFL */ - /* and the other way round */ +#define MAX_TM_WORK_AREA (MAX_TGT_LENGTH + 300) +/* Work areas to convert LF into CRFL */ +/* and the other way round */ // The following define should not exceed 65535 Bytes -#define MAX_TMT_GLOBALS_SIZE (sizeof(TMT_GLOBALS) +\ - MAX_TM_HEADER_SIZE +\ - MAX_BLOCK_SIZE+\ - MAX_TEXT_TAB_SIZE+\ - MAX_WORDS_TAB_SIZE) - +#define MAX_TMT_GLOBALS_SIZE (sizeof(TMT_GLOBALS) + \ + MAX_TM_HEADER_SIZE + \ + MAX_BLOCK_SIZE + \ + MAX_TEXT_TAB_SIZE + \ + MAX_WORDS_TAB_SIZE) -#define MAX_WORDS_TAB_ENTRIES 200 /* Number of entries in WordsTable */ -#define MAX_TAGS_TAB_ENTRIES 100 /* number of entries in TextTable */ -#define KEY_DIR_ENTRIES_NUM 4096 /* number of entries key directory */ +#define MAX_WORDS_TAB_ENTRIES 200 /* Number of entries in WordsTable */ +#define MAX_TAGS_TAB_ENTRIES 100 /* number of entries in TextTable */ +#define KEY_DIR_ENTRIES_NUM 4096 /* number of entries key directory */ #define TOP_BLOCK_NUMBER (USHORT)65535 - /* 2**16 - 1, the max number of a block */ -#define MAX_BLOCKS_NUM (USHORT)65535 - /* 2**16 - 1, maximal number of blocks */ -#define GROUP_NUM 8 /* number of distribution groups */ - -#define TAG_STARTING_VALUE 0 /* the lowest attribute to tags in text */ - /* table */ -#ifndef TM_ADM -#define MAX_SERVER_NAME 15 /* length of server name */ +/* 2**16 - 1, the max number of a block */ +#define MAX_BLOCKS_NUM (USHORT)65535 +/* 2**16 - 1, maximal number of blocks */ +#define GROUP_NUM 8 /* number of distribution groups */ + +#define TAG_STARTING_VALUE 0 /* the lowest attribute to tags in text */ + /* table */ +#ifndef TM_ADM +#define MAX_SERVER_NAME 15 /* length of server name */ #endif /* The two defines below should not be used. Displacement can be */ /* computed in runtime from the structure. */ -#define SEG_LENGTH_DISP 5 /* disp of length field in a SEGMENT */ -#define DEL_FLAG_DISP 6 /* disp of del flag from segment start */ +#define SEG_LENGTH_DISP 5 /* disp of length field in a SEGMENT */ +#define DEL_FLAG_DISP 6 /* disp of del flag from segment start */ // data compress flag -#define BOCU_COMPRESS 0x01 - +#define BOCU_COMPRESS 0x01 /* define for write Tm Header function */ -#define WF_ALL 0x0000 /* Write the whole header............*/ -#define WF_CORRUPT 0x0001 -#define WF_TMBLOCKS 0x0002 -#define WF_FREEBLOCKS 0x0004 -#define WF_FIRSTAVAIL 0x0008 -#define WF_UPDATE 0x4000 /* Write all but the key directory...*/ -#define WF_KEYDIR 0x8000 /* Write the key directory...........*/ - +#define WF_ALL 0x0000 /* Write the whole header............*/ +#define WF_CORRUPT 0x0001 +#define WF_TMBLOCKS 0x0002 +#define WF_FREEBLOCKS 0x0004 +#define WF_FIRSTAVAIL 0x0008 +#define WF_UPDATE 0x4000 /* Write all but the key directory...*/ +#define WF_KEYDIR 0x8000 /* Write the key directory...........*/ /*---------------------------------------------------------------------*\ match type definitions \*---------------------------------------------------------------------*/ -#define USE_CRLF 1 -#define ALIGN_CRLF 2 -#define IGNORE_CRLF 3 +#define USE_CRLF 1 +#define ALIGN_CRLF 2 +#define IGNORE_CRLF 3 /* return codes for the Tmt Commands */ -#define OK NO_ERROR /* for Tmt commands */ - - +#define OK NO_ERROR /* for Tmt commands */ /*----------------------------------------------------------------------------*\ System wide lengthes for correction of the thresholds in dependency of the length of the input segment \*----------------------------------------------------------------------------*/ //@@@ // segments, defined thru LENGTH_SHORTER_VALUE -#define LENGTH_SHORTER_VALUE 2 // Length for shorter segment +#define LENGTH_SHORTER_VALUE 2 // Length for shorter segment //@@@ // segments, defined thru LENGTH_SHORTER_VALUE -#define LENGTH_SHORT_VALUE 6 // Length for short segment -#define LENGTH_MEDIUM_VALUE 10 // Length for medium segment -#define LENGTH_LONG_VALUE 20 // Length for long segment +#define LENGTH_SHORT_VALUE 6 // Length for short segment +#define LENGTH_MEDIUM_VALUE 10 // Length for medium segment +#define LENGTH_LONG_VALUE 20 // Length for long segment /*----------------------------------------------------------------------------*\ System wide percentages for correction of the MATCH threshold \*----------------------------------------------------------------------------*/ -#define MATCH_PERCENTAGE_SHORT_VALUE 1L // percentage for short segment -#define MATCH_PERCENTAGE_MEDIUM_VALUE 10L // percentage for medium segment -#define MATCH_PERCENTAGE_LONG_VALUE 20L // percentage for long segment -#define MATCH_PERCENTAGE_LONGER_VALUE 40L // percentage for longer segment +#define MATCH_PERCENTAGE_SHORT_VALUE 1L // percentage for short segment +#define MATCH_PERCENTAGE_MEDIUM_VALUE 10L // percentage for medium segment +#define MATCH_PERCENTAGE_LONG_VALUE 20L // percentage for long segment +#define MATCH_PERCENTAGE_LONGER_VALUE 40L // percentage for longer segment /*----------------------------------------------------------------------------*\ System wide percentages for correction of the LENGTH threshold \*----------------------------------------------------------------------------*/ -#define LENGTH_PERCENTAGE_SHORT_VALUE 1L // percentage for short segment -#define LENGTH_PERCENTAGE_MEDIUM_VALUE 10L // percentage for medium segment -#define LENGTH_PERCENTAGE_LONG_VALUE 20L // percentage for long segment -#define LENGTH_PERCENTAGE_LONGER_VALUE 30L // percentage for longer segment +#define LENGTH_PERCENTAGE_SHORT_VALUE 1L // percentage for short segment +#define LENGTH_PERCENTAGE_MEDIUM_VALUE 10L // percentage for medium segment +#define LENGTH_PERCENTAGE_LONG_VALUE 20L // percentage for long segment +#define LENGTH_PERCENTAGE_LONGER_VALUE 30L // percentage for longer segment /*---------------------------------------------------------------------*\ Macro Definition @@ -2816,16 +2759,15 @@ void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); /**********************************************************************/ /* macro to build index name from fully qualified data name */ /**********************************************************************/ -#define INDEXNAMEFROMMEMPATH( mem, ind ) \ - { \ - Utlstrccpy( ind, mem, DOT ); \ - if ( strcmp( strrchr( mem, DOT ), EXT_OF_SHARED_MEM ) == 0 ) \ - strcat( ind, EXT_OF_SHARED_MEMINDEX ); \ - else \ - strcat( ind, EXT_OF_TMINDEX ); \ +#define INDEXNAMEFROMMEMPATH(mem, ind) \ + { \ + Utlstrccpy(ind, mem, DOT); \ + if (strcmp(strrchr(mem, DOT), EXT_OF_SHARED_MEM) == 0) \ + strcat(ind, EXT_OF_SHARED_MEMINDEX); \ + else \ + strcat(ind, EXT_OF_TMINDEX); \ } - #include "EQFTAG.H" /*---------------------------------------------------------------------*\ Typedefs Definition @@ -2836,66 +2778,66 @@ void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); CHAR buf[] \*---------------------------------------------------------------------*/ - -typedef struct _TM_HEADER_UPDATE { /* tmhu */ - BOOL fCorruption ; - USHORT usNumTMBlocks, /* number of blocks in the TM */ - usNumFreeBlocks, /* # of free, pre-formatted blocks */ - usFirstAvailBlock;/* points to first available block */ +typedef struct _TM_HEADER_UPDATE +{ /* tmhu */ + BOOL fCorruption; + USHORT usNumTMBlocks, /* number of blocks in the TM */ + usNumFreeBlocks, /* # of free, pre-formatted blocks */ + usFirstAvailBlock; /* points to first available block */ } TM_HEADER_UPDATE; -typedef union _UNIONIN +typedef union _UNIONIN { - PREFIX_IN in; - ADD_IN ain; - DEL_IN din; - REP_IN rin; - EXT_IN ein; - GET_IN gin; - CLOSE_IN clin; - OPEN_IN oin; - CREATE_IN crin; - INFO_IN infoin; - CLOSEHANDLER_IN chin; - EXIT_IN xin; - DRIVES_IN drivesin; - FILE_LIST_IN filelistin; // !!! CHM - GETPART_IN getpartin; - PUTPROP_IN putpropin; - PUTDICTPROP_IN PutDictPropIn; // !!! CHM added - PUTPROPCOMBINED_IN PutPropCombinedIn; // !!! CHM added - GETPROP_IN getpropin; - ENDORG_IN endorgin; - DELTM_IN deltmin; - DELFILE_IN delfilein; - FILEINFO_IN fileinfoin; - RENFILE_IN renfilein; -} UNIONIN, * PUNIONIN; /* uin */ - -typedef union _UNIONOUT + PREFIX_IN in; + ADD_IN ain; + DEL_IN din; + REP_IN rin; + EXT_IN ein; + GET_IN gin; + CLOSE_IN clin; + OPEN_IN oin; + CREATE_IN crin; + INFO_IN infoin; + CLOSEHANDLER_IN chin; + EXIT_IN xin; + DRIVES_IN drivesin; + FILE_LIST_IN filelistin; // !!! CHM + GETPART_IN getpartin; + PUTPROP_IN putpropin; + PUTDICTPROP_IN PutDictPropIn; // !!! CHM added + PUTPROPCOMBINED_IN PutPropCombinedIn; // !!! CHM added + GETPROP_IN getpropin; + ENDORG_IN endorgin; + DELTM_IN deltmin; + DELFILE_IN delfilein; + FILEINFO_IN fileinfoin; + RENFILE_IN renfilein; +} UNIONIN, *PUNIONIN; /* uin */ + +typedef union _UNIONOUT { - PREFIX_OUT out; - ADD_OUT aout; - DEL_OUT dout; - REP_OUT rout; - EXT_OUT eout; - GET_OUT gout; - CLOSE_OUT clout; - OPEN_OUT oout; - CREATE_OUT crout; - INFO_OUT infoout; - EXIT_OUT xout; - DRIVES_OUT drivesout; - FILE_LIST_OUT filelistout; // !!! CHM - GETPART_OUT getpartout; - PUTPROP_OUT putpropout; - GETPROP_OUT getpropout; - ENDORG_OUT endorgout; - DELTM_OUT deltmout; - DELFILE_OUT delfileout; - FILEINFO_OUT fileinfoout; - RENFILE_OUT renfileout; -} UNIONOUT, * PUNIONOUT; /* uout */ + PREFIX_OUT out; + ADD_OUT aout; + DEL_OUT dout; + REP_OUT rout; + EXT_OUT eout; + GET_OUT gout; + CLOSE_OUT clout; + OPEN_OUT oout; + CREATE_OUT crout; + INFO_OUT infoout; + EXIT_OUT xout; + DRIVES_OUT drivesout; + FILE_LIST_OUT filelistout; // !!! CHM + GETPART_OUT getpartout; + PUTPROP_OUT putpropout; + GETPROP_OUT getpropout; + ENDORG_OUT endorgout; + DELTM_OUT deltmout; + DELFILE_OUT delfileout; + FILEINFO_OUT fileinfoout; + RENFILE_OUT renfileout; +} UNIONOUT, *PUNIONOUT; /* uout */ /* +---------------------------------------------------------------------------+ @@ -2910,117 +2852,111 @@ typedef union _UNIONOUT Samples: fRc = fTmComInit (); +---------------------------------------------------------------------------+ */ -BOOL fTmComInit (VOID); - - - -typedef struct _TMT_GLOBALS { /* tmtg */ - /* this structure keeps data and pointers to variables that exist from*/ - /* the point that the TM was created or opened, until close command */ - /* BlockImage, TextTable, WordsTable and TmHeader are allocated */ - /* dynamically during Create/Open and are freed at Close time. */ - HFILE hfTM; /* handle to TM db (from Dos call) */ - PUCHAR pchBlockImage; /* pointer to block image */ - PTOKENENTRY pteTextTable, /* pointer to TextTable */ - pteWordsTable, /* pointer to WordsTable */ - pteFirstSigWord, /* pointer to first significant word */ - pteSecondSigWord; /* pointer to second significant word*/ - PPREFIX_OUT pPrefixOut; /* pointer to the current PrefixOut */ - /* to be used by FormatMore */ - SEGMENT seg; /* placed here instead of allocating */ - /* this space on stack (point 276) */ - MATCH mtch; /* placed here instead of allocating */ - /* this space on stack (point 276) */ - PCHAR pWorkArea1; /* Pointer to work area 1 */ - PCHAR pWorkArea2; /* Pointer to work area 2 */ - PTM_HEADER pTmHeader; /* pointer to Tm header structure */ - LONG lActLengthThr; /* length corrected length threshold values*/ - LONG lActMatchThr; /* length corrected match threshold values */ - PLOADEDTABLE pLoadedTagTable; /* pointer to loaded tagtable */ - PVOID pstLoadedTagTable; /* pointer to loaded tagtable, used for */ - /* function TATagTokenize */ -} TMT_GLOBALS, * PTMT_GLOBALS, ** PPTMT_GLOBALS; +BOOL fTmComInit(VOID); + +typedef struct _TMT_GLOBALS +{ /* tmtg */ + /* this structure keeps data and pointers to variables that exist from*/ + /* the point that the TM was created or opened, until close command */ + /* BlockImage, TextTable, WordsTable and TmHeader are allocated */ + /* dynamically during Create/Open and are freed at Close time. */ + HFILE hfTM; /* handle to TM db (from Dos call) */ + PUCHAR pchBlockImage; /* pointer to block image */ + PTOKENENTRY pteTextTable, /* pointer to TextTable */ + pteWordsTable, /* pointer to WordsTable */ + pteFirstSigWord, /* pointer to first significant word */ + pteSecondSigWord; /* pointer to second significant word*/ + PPREFIX_OUT pPrefixOut; /* pointer to the current PrefixOut */ + /* to be used by FormatMore */ + SEGMENT seg; /* placed here instead of allocating */ + /* this space on stack (point 276) */ + MATCH mtch; /* placed here instead of allocating */ + /* this space on stack (point 276) */ + PCHAR pWorkArea1; /* Pointer to work area 1 */ + PCHAR pWorkArea2; /* Pointer to work area 2 */ + PTM_HEADER pTmHeader; /* pointer to Tm header structure */ + LONG lActLengthThr; /* length corrected length threshold values*/ + LONG lActMatchThr; /* length corrected match threshold values */ + PLOADEDTABLE pLoadedTagTable; /* pointer to loaded tagtable */ + PVOID pstLoadedTagTable; /* pointer to loaded tagtable, used for */ + /* function TATagTokenize */ +} TMT_GLOBALS, *PTMT_GLOBALS, **PPTMT_GLOBALS; // ***************** Work Constants global ************************** -#define INIT_NUMB_OF_ENTRIES 5 // Initial number of entries in a table or List -#define SEG_NUMB 6 // Segment number in char. -#define ASCII_FLAG 1 // Ascii flag converted to char. -#define MACH_TRANS 1 // Machine trans.flag converted to char. -#define TIME_STAMP 16 // Time integer converted to character -#define IND_CODES 20 // External industry codes space to hold a maximum of 6 codes -#define ASCII_IND_CODES_LENGTH 3 // Length of Industry codes in external format -#define NUMB_OF_IND_CODES 6 // Number of Industry codes in external format -#define MEM_NAME 8 // Length of memory database name without extension -#define DRIVE_NAME 12 // Special drive name -#define PTR_MEMHANDLER_IDA 0 // Relative position in the extra bytes for handler -#define PTR_MEM_IDA 0 // Relative position in the extra bytes for instance -#define PTR_CRT_IDA 0 // Relative position in the extra bytes for create dialog -#define PTR_DLG_IDA 0 // Relative position in the extra bytes for dialogs -#define TEMP 40 // Size of temporaty work area - // of WinRegisterClass call. -#define TEXT_100_CHAR 100 // string length for temporary strings -//#define NUMB_OF_TOKENS 1000 // Number of tokens in the token list -#define NUMB_OF_TOKENS 8000 // Number of tokens in the toklist:RJ increase nec. - // if 65520 bytes are read-in at once during MemImport -#define MEM_PROP_SIZE 2048 // Global size of all memory database properties -#define MEM_BLOCK_SIZE 1024 // Translation Memory block size -#define PROCESS_NUMB_OF_MESSAGES 10 // Number of messages to be process till next message is issued -#define MEM_DBCS 0 // 0 = No support, 1 = DBCS support -#define MEM_LOAD_PATTERN_NAME "*" // Default names to be shown for load dialog -#define MEM_LOAD_PATTERN_EXT ".*" // Default extension to be shown for load dialog -#define EQF_IDENTIFICATION "EQF" // EQF identification -#define CLBCOL_TITLE_STRING " " // Placeholder for of CLBCOL titles - +#define INIT_NUMB_OF_ENTRIES 5 // Initial number of entries in a table or List +#define SEG_NUMB 6 // Segment number in char. +#define ASCII_FLAG 1 // Ascii flag converted to char. +#define MACH_TRANS 1 // Machine trans.flag converted to char. +#define TIME_STAMP 16 // Time integer converted to character +#define IND_CODES 20 // External industry codes space to hold a maximum of 6 codes +#define ASCII_IND_CODES_LENGTH 3 // Length of Industry codes in external format +#define NUMB_OF_IND_CODES 6 // Number of Industry codes in external format +#define MEM_NAME 8 // Length of memory database name without extension +#define DRIVE_NAME 12 // Special drive name +#define PTR_MEMHANDLER_IDA 0 // Relative position in the extra bytes for handler +#define PTR_MEM_IDA 0 // Relative position in the extra bytes for instance +#define PTR_CRT_IDA 0 // Relative position in the extra bytes for create dialog +#define PTR_DLG_IDA 0 // Relative position in the extra bytes for dialogs +#define TEMP 40 // Size of temporaty work area + // of WinRegisterClass call. +#define TEXT_100_CHAR 100 // string length for temporary strings +// #define NUMB_OF_TOKENS 1000 // Number of tokens in the token list +#define NUMB_OF_TOKENS 8000 // Number of tokens in the toklist:RJ increase nec. + // if 65520 bytes are read-in at once during MemImport +#define MEM_PROP_SIZE 2048 // Global size of all memory database properties +#define MEM_BLOCK_SIZE 1024 // Translation Memory block size +#define PROCESS_NUMB_OF_MESSAGES 10 // Number of messages to be process till next message is issued +#define MEM_DBCS 0 // 0 = No support, 1 = DBCS support +#define MEM_LOAD_PATTERN_NAME "*" // Default names to be shown for load dialog +#define MEM_LOAD_PATTERN_EXT ".*" // Default extension to be shown for load dialog +#define EQF_IDENTIFICATION "EQF" // EQF identification +#define CLBCOL_TITLE_STRING " " // Placeholder for of CLBCOL titles // ***************** Memory database load ************************** -//#define MEM_CONTEXT_TOKEN_END 0 // -//#define MEM_CONTROL_TOKEN_END 1 // UNUSED! -//#define NTM_DESCRIPTION_TOKEN_END 2 // -//#define MEM_MEMORYDB_TOKEN_END 3 // -//#define NTM_MEMORYDB_TOKEN_END 4 // -//#define MEM_SEGMENT_TOKEN_END 5 // -//#define MEM_SOURCE_TOKEN_END 6 // -//#define MEM_TARGET_TOKEN_END 7 // -//#define MEM_CONTEXT_TOKEN 8 // -//#define MEM_CONTROL_TOKEN 9 // -//#define NTM_DESCRIPTION_TOKEN 10 // -//#define MEM_MEMORYDB_TOKEN 11 // -//#define NTM_MEMORYDB_TOKEN 12 // -//#define MEM_SEGMENT_TOKEN 13 // -//#define MEM_SOURCE_TOKEN 14 // -//#define MEM_TARGET_TOKEN 15 // +// #define MEM_CONTEXT_TOKEN_END 0 // +// #define MEM_CONTROL_TOKEN_END 1 // UNUSED! +// #define NTM_DESCRIPTION_TOKEN_END 2 // +// #define MEM_MEMORYDB_TOKEN_END 3 // +// #define NTM_MEMORYDB_TOKEN_END 4 // +// #define MEM_SEGMENT_TOKEN_END 5 // +// #define MEM_SOURCE_TOKEN_END 6 // +// #define MEM_TARGET_TOKEN_END 7 // +// #define MEM_CONTEXT_TOKEN 8 // +// #define MEM_CONTROL_TOKEN 9 // +// #define NTM_DESCRIPTION_TOKEN 10 // +// #define MEM_MEMORYDB_TOKEN 11 // +// #define NTM_MEMORYDB_TOKEN 12 // +// #define MEM_SEGMENT_TOKEN 13 // +// #define MEM_SOURCE_TOKEN 14 // +// #define MEM_TARGET_TOKEN 15 // // strings to create the token IDs for the tags dynamically using TATagTokenize -#define MEM_CONTEXT_TOKEN_END L"" -#define MEM_CONTROL_TOKEN_END L"" -#define NTM_DESCRIPTION_TOKEN_END L"" -#define MEM_MEMORYDB_TOKEN_END L"" -#define NTM_MEMORYDB_TOKEN_END L"" -#define MEM_SEGMENT_TOKEN_END L"" -#define MEM_SOURCE_TOKEN_END L"" -#define MEM_TARGET_TOKEN_END L"" -#define MEM_CONTEXT_TOKEN L"" -#define MEM_CONTROL_TOKEN L"" -#define NTM_DESCRIPTION_TOKEN L"" -#define MEM_MEMORYDB_TOKEN L"" -#define NTM_MEMORYDB_TOKEN L"" -#define MEM_SEGMENT_TOKEN L"" -#define MEM_SOURCE_TOKEN L"" -#define MEM_TARGET_TOKEN L"" -#define MEM_CODEPAGE_TOKEN L"" -#define MEM_CODEPAGE_TOKEN_END L"" -#define MEM_ADDDATA_TOKEN_END L"" -#define MEM_ADDDATA_TOKEN L"" - - - - -#define TM_IMPORT_OK 0 // Translation memory import OK -#define TM_IMPORT_FAILED 1 // Translation memory import failed -#define TM_IMPORT_FORCED 2 // Translation memory import was forced - - -//used for input from create folder dialog +#define MEM_CONTEXT_TOKEN_END L"" +#define MEM_CONTROL_TOKEN_END L"" +#define NTM_DESCRIPTION_TOKEN_END L"" +#define MEM_MEMORYDB_TOKEN_END L"" +#define NTM_MEMORYDB_TOKEN_END L"" +#define MEM_SEGMENT_TOKEN_END L"" +#define MEM_SOURCE_TOKEN_END L"" +#define MEM_TARGET_TOKEN_END L"" +#define MEM_CONTEXT_TOKEN L"" +#define MEM_CONTROL_TOKEN L"" +#define NTM_DESCRIPTION_TOKEN L"" +#define MEM_MEMORYDB_TOKEN L"" +#define NTM_MEMORYDB_TOKEN L"" +#define MEM_SEGMENT_TOKEN L"" +#define MEM_SOURCE_TOKEN L"" +#define MEM_TARGET_TOKEN L"" +#define MEM_CODEPAGE_TOKEN L"" +#define MEM_CODEPAGE_TOKEN_END L"" +#define MEM_ADDDATA_TOKEN_END L"" +#define MEM_ADDDATA_TOKEN L"" + +#define TM_IMPORT_OK 0 // Translation memory import OK +#define TM_IMPORT_FAILED 1 // Translation memory import failed +#define TM_IMPORT_FORCED 2 // Translation memory import was forced + +// used for input from create folder dialog typedef enum { MEM_CREATE_NAME_IND, @@ -3029,8 +2965,7 @@ typedef enum MEM_CREATE_TARGETLANG_IND } MEM_CREATEINDEX; -#define MEM_TEXT_BUFFER 65520 // Length of the text buffer - +#define MEM_TEXT_BUFFER 65520 // Length of the text buffer // ************ Definitions for TM list box display ********************* typedef enum @@ -3051,14 +2986,14 @@ typedef enum /* string IDs to parse the CONTROL string of the TM exported format */ /* using UtlParseX15 */ /**********************************************************************/ -//#define NTM_SEGNR_ID 0 //seg nr from analysis -//#define NTM_MTFLAG_ID 1 //machine translation flag -//#define NTM_TIME_ID 2 //time stamp -//#define NTM_SOURCELNG_ID 3 //source language -//#define NTM_TARGETLNG_ID 4 //target language -//#define NTM_AUTHORNAME_ID 5 //author name -//#define NTM_TAGTABLENAME_ID 6 //tag table name -//#define NTM_FILENAME_ID 7 //file name +// #define NTM_SEGNR_ID 0 //seg nr from analysis +// #define NTM_MTFLAG_ID 1 //machine translation flag +// #define NTM_TIME_ID 2 //time stamp +// #define NTM_SOURCELNG_ID 3 //source language +// #define NTM_TARGETLNG_ID 4 //target language +// #define NTM_AUTHORNAME_ID 5 //author name +// #define NTM_TAGTABLENAME_ID 6 //tag table name +// #define NTM_FILENAME_ID 7 //file name typedef enum { @@ -3073,15 +3008,15 @@ typedef enum NTM_LONGNAME_ID } NTM_IMPORTINDEX; -#define START_KEY 0xFFFFFF +#define START_KEY 0xFFFFFF -#define EXT_OF_RENAMED_TMDATA ".TRD" +#define EXT_OF_RENAMED_TMDATA ".TRD" #define EXT_OF_RENAMED_TMINDEX ".TRI" -#define EXT_OF_TEMP_TMPROP ".TMR" -#define EXT_OF_RENAMED_TMPROP ".RMR" -#define EXT_OF_TEMP_TMDATA ".TTD" -#define EXT_OF_TEMP_TMINDEX ".TTI" -#define EXT_OF_TMPROP EXT_OF_MEM +#define EXT_OF_TEMP_TMPROP ".TMR" +#define EXT_OF_RENAMED_TMPROP ".RMR" +#define EXT_OF_TEMP_TMDATA ".TTD" +#define EXT_OF_TEMP_TMINDEX ".TTI" +#define EXT_OF_TMPROP EXT_OF_MEM #include class ProposalFilter; @@ -3090,164 +3025,147 @@ class ProposalFilter; typedef struct _SEG_CTRL_DATA { - CHAR chSegmentNumber[SEG_NUMB]; /* Segment number converted to character*/ - CHAR chAsciiFlag[ASCII_FLAG]; /* Ascii flag converted to char. */ - CHAR chMachineTrans[MACH_TRANS]; /* Machine trans.flag converted to char.*/ - CHAR chTimeStamp[TIME_STAMP]; /* Time integer converted to character */ - CHAR chIndustryCodes[IND_CODES]; /* Industry codes */ - CHAR chFileName[MAX_FILESPEC-1]; /* Segment origin. from that file name */ -}SEG_CTRL_DATA, * PSEG_CTRL_DATA; - - + CHAR chSegmentNumber[SEG_NUMB]; /* Segment number converted to character*/ + CHAR chAsciiFlag[ASCII_FLAG]; /* Ascii flag converted to char. */ + CHAR chMachineTrans[MACH_TRANS]; /* Machine trans.flag converted to char.*/ + CHAR chTimeStamp[TIME_STAMP]; /* Time integer converted to character */ + CHAR chIndustryCodes[IND_CODES]; /* Industry codes */ + CHAR chFileName[MAX_FILESPEC - 1]; /* Segment origin. from that file name */ +} SEG_CTRL_DATA, *PSEG_CTRL_DATA; /*! \brief search a string in a proposal - \param pProposal pointer to the proposal + \param pProposal pointer to the proposal \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ -BOOL searchInProposal -( - OtmProposal *pProposal, - PSZ_W pszSearch, - LONG lSearchOptions -); - +BOOL searchInProposal( + OtmProposal *pProposal, + PSZ_W pszSearch, + LONG lSearchOptions); /*! \brief search a string in a proposal - \param pProposal pointer to the proposal + \param pProposal pointer to the proposal \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ -BOOL searchExtendedInProposal -( - OtmProposal *pProposal, - std::vector& filters, - LONG lSearchOptions -); - - - +BOOL searchExtendedInProposal( + OtmProposal *pProposal, + std::vector &filters, + LONG lSearchOptions); /*! \brief check if search string matches current data \param pData pointer to current position in data area \param pSearch pointer to search string \returns 0 if search string matches data */ -SHORT compareString -( - PSZ_W pData, - PSZ_W pSearch -); +SHORT compareString( + PSZ_W pData, + PSZ_W pSearch); /*! \brief find the given string in the provided data \param pszData pointer to the data being searched \param pszSearch pointer to the search string \returns TRUE if the data contains the searched string otherwise FALSE is returned */ -BOOL findString -( - PSZ_W pszData, - PSZ_W pszSearch -); +BOOL findString( + PSZ_W pszData, + PSZ_W pszSearch); typedef struct _MEM_MERGE_IDA { - CHAR szPathMergeMem[MAX_LONGPATH]; // Full path and name of TM to be merged - CHAR szPathMergeProp[MAX_LONGPATH]; // Full path and name of Properties to be imported - CHAR szDriveMergeMem[MAX_DRIVE]; // Drive letter of TM to be merged w o : ( d ) - CHAR szNameMergeMem[MAX_LONGFILESPEC];// Name of TM to be merged w o ext ( heller ) - CHAR szShortNameMergeMem[MAX_FILESPEC];// Short name of TM being merged - CHAR szDirMergeMem[MAX_LONGPATH]; // Path of directories fo TM to be merged ( \eqf\import\ ) - CHAR szExtMergeMem[MAX_FEXT]; // Ext of TM to be merged ( MIP or MEM or ? ) - EqfMemory *pMergeMem; // TM handle of TM to be merged - CHAR szInvokingHandler[40]; // ??? 40 ???? to be in Gerds stuff. Name of handler which invoked the process - CHAR szSystemPath[MAX_EQF_PATH]; // Path to the EQF system - CHAR szTemp[MAX_EQF_PATH]; // Temporary path area - CHAR szPropName[MAX_FILESPEC]; // TM property name ( xxxx.MEM ) - CHAR szPathMem[MAX_LONGPATH]; // Full name and Path to the TM ( c:\EQF\MEM\xxxx.MEM ) - CHAR szPathProp[MAX_LONGPATH]; // Full name and Path to the Properties - CHAR szDriveMem[MAX_DRIVE]; // Drive letter of TM w o : ( h ) - CHAR szNameMem[MAX_LONGFILESPEC]; // Long name of TM w o ext ( gallus ) - CHAR szShortNameMem[MAX_FILESPEC]; // Short name of TM w o ext ( gallus ) - EqfMemory *pOutputMem; // TM handle - BOOL fPropExist; // Existence of properties 0=No 1=Yes - BOOL fPropCreated; // Tm properties have been created - BOOL fMsg; // A message has been issued already - BOOL fImport; // If set the merge is started via an import - USHORT usImportRc; // Import function return code - ULONG ulSegmentCounter; // Number of segments merged - ULONG ulInvSegmentCounter; // Invalid Segment counter - EXT_IN stOldExtIn; // input for TmOldExtract - EXT_OUT stOldExtOut; // output for TmOldExtract - TMX_EXT_IN_W stExtIn; // TMX_EXTRACT_IN structure - TMX_EXT_OUT_W stExtOut; // TMX_EXTRACT_OUT structure - TMX_PUT_W stPutIn; // TMX_PUT_IN structure - TMX_EXT_OUT_W stPutOut; // TMX_PUT_OUT structure - TIME_L tStamp; // Time stamp of merge start time - HWND hwndMemLb; // Handle to the TM listbox - HWND hProgressWindow; // Handle of progress indicator window - ULONG ulProgressPos; // position of progress indicator - CHAR szServer[MAX_LONGPATH]; // Server Name of TM or \0 if TM is local - BOOL fPropReceived; // Property file got via Receive ? - BOOL fOrganizeInvoked; // TRUE if Organize is invoked - BOOL fImportAnyway; // FALSE if import of TM with - // different IDs is canceled - CHAR szSourceMemSourceLang[MAX_LANG_LENGTH]; - CHAR szSourceMemTargetLang[MAX_LANG_LENGTH]; - CHAR szSourceMemMarkup[MAX_FNAME]; /*@1276A*/ - CHAR szTargetMemSourceLang[MAX_LANG_LENGTH]; - CHAR szTargetMemTargetLang[MAX_LANG_LENGTH]; - CHAR szTargetMemMarkup[MAX_FNAME]; /*@1276A*/ - BOOL fDisplayMsg; // message flag if segment not /*@47A*/ - // not replaced message should /*@47A*/ - // be displayed or not /*@47A*/ - BOOL fOldPropFile; // used for folder import - CHAR szTagTable[MAX_FILESPEC]; // tag table of org TM - CHAR szPathMergeIndex[MAX_LONGPATH];// Full path and name of index to be merged - CHAR szPathIndex[MAX_LONGPATH]; // name and path to index C:\EQF\MEM\xxxx.TMI ) - BOOL fBatch; // TRUE = we are in batch mode - HWND hwndErrMsg; // parent handle for error messages - PDDEMEMEXP pDDEMemExp; // ptr to batch memory export data - CHAR szLongName[MAX_LONGFILESPEC]; // buffer for long TM names - USHORT usTask; // current taskl to do (batch mode) - OtmProposal *pProposal; // buffer for proposal data - BOOL fFirstExtract; // TRUE = this ist the first extract call - int iComplete; // process completion rate -}MEM_MERGE_IDA, * PMEM_MERGE_IDA; + CHAR szPathMergeMem[MAX_LONGPATH]; // Full path and name of TM to be merged + CHAR szPathMergeProp[MAX_LONGPATH]; // Full path and name of Properties to be imported + CHAR szDriveMergeMem[MAX_DRIVE]; // Drive letter of TM to be merged w o : ( d ) + CHAR szNameMergeMem[MAX_LONGFILESPEC]; // Name of TM to be merged w o ext ( heller ) + CHAR szShortNameMergeMem[MAX_FILESPEC]; // Short name of TM being merged + CHAR szDirMergeMem[MAX_LONGPATH]; // Path of directories fo TM to be merged ( \eqf\import\ ) + CHAR szExtMergeMem[MAX_FEXT]; // Ext of TM to be merged ( MIP or MEM or ? ) + EqfMemory *pMergeMem; // TM handle of TM to be merged + CHAR szInvokingHandler[40]; // ??? 40 ???? to be in Gerds stuff. Name of handler which invoked the process + CHAR szSystemPath[MAX_EQF_PATH]; // Path to the EQF system + CHAR szTemp[MAX_EQF_PATH]; // Temporary path area + CHAR szPropName[MAX_FILESPEC]; // TM property name ( xxxx.MEM ) + CHAR szPathMem[MAX_LONGPATH]; // Full name and Path to the TM ( c:\EQF\MEM\xxxx.MEM ) + CHAR szPathProp[MAX_LONGPATH]; // Full name and Path to the Properties + CHAR szDriveMem[MAX_DRIVE]; // Drive letter of TM w o : ( h ) + CHAR szNameMem[MAX_LONGFILESPEC]; // Long name of TM w o ext ( gallus ) + CHAR szShortNameMem[MAX_FILESPEC]; // Short name of TM w o ext ( gallus ) + EqfMemory *pOutputMem; // TM handle + BOOL fPropExist; // Existence of properties 0=No 1=Yes + BOOL fPropCreated; // Tm properties have been created + BOOL fMsg; // A message has been issued already + BOOL fImport; // If set the merge is started via an import + USHORT usImportRc; // Import function return code + ULONG ulSegmentCounter; // Number of segments merged + ULONG ulInvSegmentCounter; // Invalid Segment counter + EXT_IN stOldExtIn; // input for TmOldExtract + EXT_OUT stOldExtOut; // output for TmOldExtract + TMX_EXT_IN_W stExtIn; // TMX_EXTRACT_IN structure + TMX_EXT_OUT_W stExtOut; // TMX_EXTRACT_OUT structure + TMX_PUT_W stPutIn; // TMX_PUT_IN structure + TMX_EXT_OUT_W stPutOut; // TMX_PUT_OUT structure + TIME_L tStamp; // Time stamp of merge start time + HWND hwndMemLb; // Handle to the TM listbox + HWND hProgressWindow; // Handle of progress indicator window + ULONG ulProgressPos; // position of progress indicator + CHAR szServer[MAX_LONGPATH]; // Server Name of TM or \0 if TM is local + BOOL fPropReceived; // Property file got via Receive ? + BOOL fOrganizeInvoked; // TRUE if Organize is invoked + BOOL fImportAnyway; // FALSE if import of TM with + // different IDs is canceled + CHAR szSourceMemSourceLang[MAX_LANG_LENGTH]; + CHAR szSourceMemTargetLang[MAX_LANG_LENGTH]; + CHAR szSourceMemMarkup[MAX_FNAME]; /*@1276A*/ + CHAR szTargetMemSourceLang[MAX_LANG_LENGTH]; + CHAR szTargetMemTargetLang[MAX_LANG_LENGTH]; + CHAR szTargetMemMarkup[MAX_FNAME]; /*@1276A*/ + BOOL fDisplayMsg; // message flag if segment not /*@47A*/ + // not replaced message should /*@47A*/ + // be displayed or not /*@47A*/ + BOOL fOldPropFile; // used for folder import + CHAR szTagTable[MAX_FILESPEC]; // tag table of org TM + CHAR szPathMergeIndex[MAX_LONGPATH]; // Full path and name of index to be merged + CHAR szPathIndex[MAX_LONGPATH]; // name and path to index C:\EQF\MEM\xxxx.TMI ) + BOOL fBatch; // TRUE = we are in batch mode + HWND hwndErrMsg; // parent handle for error messages + PDDEMEMEXP pDDEMemExp; // ptr to batch memory export data + CHAR szLongName[MAX_LONGFILESPEC]; // buffer for long TM names + USHORT usTask; // current taskl to do (batch mode) + OtmProposal *pProposal; // buffer for proposal data + BOOL fFirstExtract; // TRUE = this ist the first extract call + int iComplete; // process completion rate +} MEM_MERGE_IDA, *PMEM_MERGE_IDA; /**********************************************************************/ /* Dialog IDA for TM property dialog */ /**********************************************************************/ typedef struct _MEM_PROP_IDA { - CHAR szMemName[MAX_LONGFILESPEC];// Memory database name without extension - EqfMemory *pMem; // Handle of memory database - //HPROP hPropMem; // Memory database property handle - //PPROP_NTM pPropMem; // pointer to TM properties - CHAR szPropName[MAX_FILESPEC]; // buffer for property name - CHAR szPropPath[MAX_EQF_PATH]; // buffer for property path - CHAR szTempPath[MAX_EQF_PATH]; // buffer for path names -}MEM_PROP_IDA, * PMEM_PROP_IDA; - - -BOOL EqfMemPropsToHtml( HWND hwndParent, PMEM_IDA pIDA, EqfMemory *pMem ); - -VOID EQFMemImportTrojaEnd( PMEM_MERGE_IDA ); -USHORT CloseMergeTmAndTm( PMEM_MERGE_IDA, BOOL ); /*@1139A*/ -USHORT MemFuncMergeTM -( - PMEM_MERGE_IDA pMIDA // Pointer to the merge IDA + CHAR szMemName[MAX_LONGFILESPEC]; // Memory database name without extension + EqfMemory *pMem; // Handle of memory database + // HPROP hPropMem; // Memory database property handle + // PPROP_NTM pPropMem; // pointer to TM properties + CHAR szPropName[MAX_FILESPEC]; // buffer for property name + CHAR szPropPath[MAX_EQF_PATH]; // buffer for property path + CHAR szTempPath[MAX_EQF_PATH]; // buffer for path names +} MEM_PROP_IDA, *PMEM_PROP_IDA; + +BOOL EqfMemPropsToHtml(HWND hwndParent, PMEM_IDA pIDA, EqfMemory *pMem); + +VOID EQFMemImportTrojaEnd(PMEM_MERGE_IDA); +USHORT CloseMergeTmAndTm(PMEM_MERGE_IDA, BOOL); /*@1139A*/ +USHORT MemFuncMergeTM( + PMEM_MERGE_IDA pMIDA // Pointer to the merge IDA ); -//#endif +// #endif typedef struct _MEMORY_HANDLER_DATA { - IDA_HEAD stIdaHead; // Standard Ida head -}MEMORY_HANDLER_DATA, * PMEMORY_HANDLER_DATA; - + IDA_HEAD stIdaHead; // Standard Ida head +} MEMORY_HANDLER_DATA, *PMEMORY_HANDLER_DATA; /**********************************************************************/ /* TMX_ENDORG_IN, TMX_ENDORG_OUT used by NTMCloseOrganize */ @@ -3259,8 +3177,6 @@ typedef struct _MEMORY_HANDLER_DATA /* The temporary file has to be closed previously. */ /**********************************************************************/ - - /**********************************************************************/ /* The following structures for dialog IDAs are defined only */ /* if INCL_EQFMEM_DLGIDAS has been defined */ @@ -3269,374 +3185,328 @@ typedef struct _MEMORY_HANDLER_DATA typedef struct _MEM_INCL_DLG_IDA { - BOOL fInitErrorOccurred; // Important init error occurred -}MEM_INCL_DLG_IDA, * PMEM_INCL_DLG_IDA; + BOOL fInitErrorOccurred; // Important init error occurred +} MEM_INCL_DLG_IDA, *PMEM_INCL_DLG_IDA; #endif - typedef struct _SLIDER_DATA { - CHAR szLine [5] [TEXT_100_CHAR+MAX_PATH144]; // 5 text lines of 100 characters - // + max path length -} SLIDER_DATA, * PSLIDER_DATA; - - + CHAR szLine[5][TEXT_100_CHAR + MAX_PATH144]; // 5 text lines of 100 characters + // + max path length +} SLIDER_DATA, *PSLIDER_DATA; /*--------------------------------------------------------------------------*\ Function prototypes. \*--------------------------------------------------------------------------*/ USHORT - OpenTmFile (PTMT_GLOBALS ptmtg, /*.The TM Globals Area.............*/ - PSZ pszFileName,/*.The TM Full File Name...........*/ - USHORT idCommand), /* TMC_CREATE / TMC_OPEN...........*/ +OpenTmFile(PTMT_GLOBALS ptmtg, /*.The TM Globals Area.............*/ + PSZ pszFileName, /*.The TM Full File Name...........*/ + USHORT idCommand), /* TMC_CREATE / TMC_OPEN...........*/ - ReadTmHeader (PTMT_GLOBALS ptmtg), /* Tmt Globals area ................*/ + ReadTmHeader(PTMT_GLOBALS ptmtg), /* Tmt Globals area ................*/ - AllocTmtGlobals (USHORT usTmtGlobalsSize, /* The allocateion size.......*/ - PPTMT_GLOBALS pptmtg), /* The TM Globals Area........*/ + AllocTmtGlobals(USHORT usTmtGlobalsSize, /* The allocateion size.......*/ + PPTMT_GLOBALS pptmtg), /* The TM Globals Area........*/ - SetTmtWorkPointers (PTMT_GLOBALS ptmtg); /* The TM Globals Area........*/ + SetTmtWorkPointers(PTMT_GLOBALS ptmtg); /* The TM Globals Area........*/ /*--------------------------------------------------------------------------*\ Function prototypes. \*--------------------------------------------------------------------------*/ -VOID -GetFirstSegAddress -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PSZ pszSource, // pointer to source segment - PTM_ADDRESS pAddr // pointer to address +VOID GetFirstSegAddress( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PSZ pszSource, // pointer to source segment + PTM_ADDRESS pAddr // pointer to address ); -VOID -CleanSource -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PSZ pszSource // Pointer to string to be tokenized +VOID CleanSource( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PSZ pszSource // Pointer to string to be tokenized ); -VOID -CalcPrimaryKey -( - PTMT_GLOBALS ptmtg, // Pointer to Globals structure - PUCHAR pchPrimaryKey // Pointer to primary key +VOID CalcPrimaryKey( + PTMT_GLOBALS ptmtg, // Pointer to Globals structure + PUCHAR pchPrimaryKey // Pointer to primary key ); -VOID -CalcSecondaryKey -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PSZ pszSortedSecKey // Ptr to sorted secondary key +VOID CalcSecondaryKey( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PSZ pszSortedSecKey // Ptr to sorted secondary key ); -VOID -Get4Chars -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - SHORT sTokenId, // WORD/USELESS/NOISE/TEXT/TAG - PTOKENENTRY pteFirst, // Ptr to Text or Word Table - PUCHAR pchPrimaryKey // 4 chars for primary key +VOID Get4Chars( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + SHORT sTokenId, // WORD/USELESS/NOISE/TEXT/TAG + PTOKENENTRY pteFirst, // Ptr to Text or Word Table + PUCHAR pchPrimaryKey // 4 chars for primary key ); -VOID -WordsTokenize -( - PTMT_GLOBALS ptmtg // Pointer to Tmt globals +VOID WordsTokenize( + PTMT_GLOBALS ptmtg // Pointer to Tmt globals ); -VOID -UselessFiltering -( - PTOKENENTRY pteWord // Pointer to word in WordsTable +VOID UselessFiltering( + PTOKENENTRY pteWord // Pointer to word in WordsTable ); USHORT -Tmt -( - HTM htm, // Pointer to TmtGlobals - PIN pIn, // Pointer to input buffer - POUT pOut // Pointer to output buffer +Tmt( + HTM htm, // Pointer to TmtGlobals + PIN pIn, // Pointer to input buffer + POUT pOut // Pointer to output buffer ); - /**********************************************************************/ /* TmtX */ /**********************************************************************/ USHORT -TmtX ( HTM htm, // Pointer to TmtGlobals - PXIN pIn, // Pointer to input buffer - PXOUT pOut ); // Pointer to output buffer - +TmtX(HTM htm, // Pointer to TmtGlobals + PXIN pIn, // Pointer to input buffer + PXOUT pOut); // Pointer to output buffer USHORT -CalcEntryInKeyDir -( - PTM_HEADER ptmh, // Pointer to Tm Header - PUCHAR pchPrimaryKey // Primary Key +CalcEntryInKeyDir( + PTM_HEADER ptmh, // Pointer to Tm Header + PUCHAR pchPrimaryKey // Primary Key ); USHORT -WriteTmhToDisk -( - PTMT_GLOBALS ptmtg, // Pointer to globals - USHORT fsWrite // Write control flags word +WriteTmhToDisk( + PTMT_GLOBALS ptmtg, // Pointer to globals + USHORT fsWrite // Write control flags word ); USHORT -WriteToDisk -( - PTMT_GLOBALS ptmtg, // TMT globals - ULONG ldispPtr, // New file pointer location - PVOID pvWrite, // Pointer to write buffer - USHORT numWrite // Number of bytes to write +WriteToDisk( + PTMT_GLOBALS ptmtg, // TMT globals + ULONG ldispPtr, // New file pointer location + PVOID pvWrite, // Pointer to write buffer + USHORT numWrite // Number of bytes to write ); USHORT -FormatMore -( - PTMT_GLOBALS ptmtg // Pointer to Tmt globals +FormatMore( + PTMT_GLOBALS ptmtg // Pointer to Tmt globals ); USHORT -ReadStringFromDisk -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PCHAR bufRead, // Read buffer - USHORT lenString, // String length - PTM_ADDRESS paddr // Pointer to string buffer +ReadStringFromDisk( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PCHAR bufRead, // Read buffer + USHORT lenString, // String length + PTM_ADDRESS paddr // Pointer to string buffer ); USHORT -ReadSegmentFromDisk -( - PTMT_GLOBALS ptmtg, // pointer to Tmt globals - PTM_ADDRESS pAddr, // Pointer to an address - PSEGMENT pSegment, // Pointer to seg. buffer - PBOOL pfFirstSeg, // Pointer to 1st seg. flag - PBOOL pfLastSeg // Pointer to last seg. flag +ReadSegmentFromDisk( + PTMT_GLOBALS ptmtg, // pointer to Tmt globals + PTM_ADDRESS pAddr, // Pointer to an address + PSEGMENT pSegment, // Pointer to seg. buffer + PBOOL pfFirstSeg, // Pointer to 1st seg. flag + PBOOL pfLastSeg // Pointer to last seg. flag ); USHORT -ReadBlock -( - PTMT_GLOBALS ptmtg, // Pointer to Tmt globals - PTM_ADDRESS paddr, // Pointer to an address - BOOL fAddressInParameter // Address in pAddr +ReadBlock( + PTMT_GLOBALS ptmtg, // Pointer to Tmt globals + PTM_ADDRESS paddr, // Pointer to an address + BOOL fAddressInParameter // Address in pAddr ); -INT -CharCompare -( - const void * arg1, // First comparand - const void * arg2 // Second comparand +INT CharCompare( + const void *arg1, // First comparand + const void *arg2 // Second comparand ); -BOOL -fStrcmpCRLF -( - PSZ pszStr1, // pointer to first string to be compared - PSZ pszStr2, // pointer to second string to be compared - PUSHORT pusEqualChars, // (return value) pointer to USHORT where the - // number of equal characters (first string) - USHORT fCompareType); // flag indicating whether CRLF and LF are ignored - // ALIGN_CRLF -> CRLF and LF are equal equal - // USE_CRLF -> exact matches required s - // IGNORE_CRL -> characters are equal - +BOOL fStrcmpCRLF( + PSZ pszStr1, // pointer to first string to be compared + PSZ pszStr2, // pointer to second string to be compared + PUSHORT pusEqualChars, // (return value) pointer to USHORT where the + // number of equal characters (first string) + USHORT fCompareType); // flag indicating whether CRLF and LF are ignored + // ALIGN_CRLF -> CRLF and LF are equal equal + // USE_CRLF -> exact matches required s + // IGNORE_CRL -> characters are equal /*--------------------------------------------------------------------------*\ Function prototypes. \*--------------------------------------------------------------------------*/ USHORT - TmtExtract (PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ - PEXT_IN pExtIn, /* pointer to input buffer...*/ - PEXT_OUT pExtOut); /* pointer to output buffer..*/ +TmtExtract(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ + PEXT_IN pExtIn, /* pointer to input buffer...*/ + PEXT_OUT pExtOut); /* pointer to output buffer..*/ USHORT - TmtGet (PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ - PGET_IN pGetIn, /* pointer to input buffer...*/ - PGET_OUT pGetOut), /* pointer to output buffer..*/ - - TmtGetTMPart( PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ - PGETPART_IN pGetPartIn, /* pointer to input buffer...*/ - PGETPART_OUT pGetPartOut), /* pointer to output buffer..*/ - - GetSegByAddr (PTMT_GLOBALS ptmtg, /* pointer to TmtGlobals.....*/ - PTM_ADDRESS pAddr, /* pointer to input address..*/ - PTM_ADDRESS pAddrNext, /* pointer to next address...*/ - PSEGMENT pseg), /* pointer to segment........*/ - - - TmtInfo (PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ - PINFO_OUT pInfoOut); /* pointer to output buffer..*/ - - -LONG - CheckSimilar (PTMT_GLOBALS ptmtg, /* pointer to Tmt globals.....*/ - PSEGMENT pCurrentSegment,/* pointer to current segmnt .*/ - PTOKENENTRY pTokenEntry, /* points to 1'st & 2'nd word */ - PGET_IN pGetIn, /* pointer to input buffer....*/ - PSZ pszInputKey, /* pointer to sortedkey of....*/ - /* input segment..............*/ - PMATCH pmtch), /* pointer to stMatch.........*/ - - CheckExact (PGET_IN pGetIn, /* pointer to input segment....*/ - PSEGMENT pseg, /* pointer to current seg......*/ - BOOL fSimilarMod, /* Similar= TRUE, Exact=FALSE..*/ - PMATCH pmtch), /* pointer to stMatch..........*/ - - CalcExactIndustry (PGET_IN pGetIn, /* pointer to input segment....*/ - PSEGMENT pseg); /* pointer to current seg......*/ - -VOID - RankNewMatch (USHORT numMatchesReq, // number of matches required - USHORT usConvert, // Conversion indicator - PMATCH pmtchCurrent, // pointer to stMatch - PSEGMENT pseg, // pointer to current seg - PGET_OUT pGetOut); // pointer to GET_OUT struct - -BOOL - CheckWord (PTOKENENTRY pteFirst, /*Point to first word ..........*/ - PTOKENENTRY pteSecond) ; /*Point to second word .........*/ - -VOID -LengthCorrectThresholds( PTMT_GLOBALS ptmtg, //pointer to Tmt globals - USHORT usLenInput, //number of words in inp. seg. - PGET_IN pGetIn ); //pointer to input buffer +TmtGet(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ + PGET_IN pGetIn, /* pointer to input buffer...*/ + PGET_OUT pGetOut), /* pointer to output buffer..*/ + + TmtGetTMPart(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ + PGETPART_IN pGetPartIn, /* pointer to input buffer...*/ + PGETPART_OUT pGetPartOut), /* pointer to output buffer..*/ + + GetSegByAddr(PTMT_GLOBALS ptmtg, /* pointer to TmtGlobals.....*/ + PTM_ADDRESS pAddr, /* pointer to input address..*/ + PTM_ADDRESS pAddrNext, /* pointer to next address...*/ + PSEGMENT pseg), /* pointer to segment........*/ + + TmtInfo(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals....*/ + PINFO_OUT pInfoOut); /* pointer to output buffer..*/ + +LONG CheckSimilar(PTMT_GLOBALS ptmtg, /* pointer to Tmt globals.....*/ + PSEGMENT pCurrentSegment, /* pointer to current segmnt .*/ + PTOKENENTRY pTokenEntry, /* points to 1'st & 2'nd word */ + PGET_IN pGetIn, /* pointer to input buffer....*/ + PSZ pszInputKey, /* pointer to sortedkey of....*/ + /* input segment..............*/ + PMATCH pmtch), /* pointer to stMatch.........*/ + + CheckExact(PGET_IN pGetIn, /* pointer to input segment....*/ + PSEGMENT pseg, /* pointer to current seg......*/ + BOOL fSimilarMod, /* Similar= TRUE, Exact=FALSE..*/ + PMATCH pmtch), /* pointer to stMatch..........*/ + + CalcExactIndustry(PGET_IN pGetIn, /* pointer to input segment....*/ + PSEGMENT pseg); /* pointer to current seg......*/ + +VOID RankNewMatch(USHORT numMatchesReq, // number of matches required + USHORT usConvert, // Conversion indicator + PMATCH pmtchCurrent, // pointer to stMatch + PSEGMENT pseg, // pointer to current seg + PGET_OUT pGetOut); // pointer to GET_OUT struct + +BOOL CheckWord(PTOKENENTRY pteFirst, /*Point to first word ..........*/ + PTOKENENTRY pteSecond); /*Point to second word .........*/ + +VOID LengthCorrectThresholds(PTMT_GLOBALS ptmtg, // pointer to Tmt globals + USHORT usLenInput, // number of words in inp. seg. + PGET_IN pGetIn); // pointer to input buffer USHORT - TmtAdd (PTMT_GLOBALS pTmtGlobals, /* Pointer to Tmt Globals........*/ - PADD_IN pAddIn, /* Pointer to input buffer.......*/ - PADD_OUT pAddOut), /* Pointer to output buffer......*/ +TmtAdd(PTMT_GLOBALS pTmtGlobals, /* Pointer to Tmt Globals........*/ + PADD_IN pAddIn, /* Pointer to input buffer.......*/ + PADD_OUT pAddOut), /* Pointer to output buffer......*/ - TmtDelete (PTMT_GLOBALS ptmtg, /* Pointer to Tmt Globals....*/ - PDEL_IN pDeleteIn, /* Pointer to input buffer...*/ - PDEL_OUT pDeleteOut), /* Pointer to output buffer..*/ + TmtDelete(PTMT_GLOBALS ptmtg, /* Pointer to Tmt Globals....*/ + PDEL_IN pDeleteIn, /* Pointer to input buffer...*/ + PDEL_OUT pDeleteOut), /* Pointer to output buffer..*/ - TmtReplace (PTMT_GLOBALS ptmtg, /* Pointer to Tmt Globals....*/ - PREP_IN pReplaceIn, /* Pointer to input buffer...*/ - PREP_OUT pReplaceOut), /* Pointer to output buffer..*/ + TmtReplace(PTMT_GLOBALS ptmtg, /* Pointer to Tmt Globals....*/ + PREP_IN pReplaceIn, /* Pointer to input buffer...*/ + PREP_OUT pReplaceOut), /* Pointer to output buffer..*/ - AddSegToCluster (PTMT_GLOBALS ptmtg, /* Pointer to globals area..*/ - PTM_ADDRESS pAddr, /* Ptr to initial address...*/ - PSEGMENT pseg), /* Pointer to a segment.....*/ + AddSegToCluster(PTMT_GLOBALS ptmtg, /* Pointer to globals area..*/ + PTM_ADDRESS pAddr, /* Ptr to initial address...*/ + PSEGMENT pseg), /* Pointer to a segment.....*/ - FindFreshAddrInCluster (PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.*/ - PTM_ADDRESS pAddr, /* Pointer to address.....*/ - BOOL fReadFirstBlock), /* Read block flag........*/ + FindFreshAddrInCluster(PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.*/ + PTM_ADDRESS pAddr, /* Pointer to address.....*/ + BOOL fReadFirstBlock), /* Read block flag........*/ - TakeBlockFromSpool (PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.....*/ - USHORT rcPrevious, /* RC of the previous call....*/ - PTM_ADDRESS pAddr), /* Pointer to an address......*/ + TakeBlockFromSpool(PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.....*/ + USHORT rcPrevious, /* RC of the previous call....*/ + PTM_ADDRESS pAddr), /* Pointer to an address......*/ - WriteStringToDisk (PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.....*/ - PCHAR bufWrite, /* Pointer to the string......*/ - USHORT lenString, /* String length..............*/ - PTM_ADDRESS pAddr), /* Pointer to string address..*/ + WriteStringToDisk(PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals.....*/ + PCHAR bufWrite, /* Pointer to the string......*/ + USHORT lenString, /* String length..............*/ + PTM_ADDRESS pAddr), /* Pointer to string address..*/ - WriteBlock (PTMT_GLOBALS ptmtg), /* Pointer to Tmt globals.....*/ + WriteBlock(PTMT_GLOBALS ptmtg), /* Pointer to Tmt globals.....*/ - FindMatchSegInCluster (PTMT_GLOBALS ptmtg, /* Ptr to Tmt globals...*/ - PTM_ADDRESS pAddr, /* Pointer to address...*/ - PBOOL pfLastSeg, /* Last segment flag....*/ - PSEGMENT psegIn, /* Ptr to input buffer..*/ - PSEGMENT pseg), /* Pointer to segment...*/ + FindMatchSegInCluster(PTMT_GLOBALS ptmtg, /* Ptr to Tmt globals...*/ + PTM_ADDRESS pAddr, /* Pointer to address...*/ + PBOOL pfLastSeg, /* Last segment flag....*/ + PSEGMENT psegIn, /* Ptr to input buffer..*/ + PSEGMENT pseg), /* Pointer to segment...*/ - DeleteSegment (PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals........*/ - PTM_ADDRESS pAddr, /* Address of segment to delete..*/ - BOOL fLastSeg, /* Last segment in cluster flag..*/ - PSEGMENT pseg); /* Pointer to the segment........*/ + DeleteSegment(PTMT_GLOBALS ptmtg, /* Pointer to Tmt globals........*/ + PTM_ADDRESS pAddr, /* Address of segment to delete..*/ + BOOL fLastSeg, /* Last segment in cluster flag..*/ + PSEGMENT pseg); /* Pointer to the segment........*/ -BOOL - CheckMatchForDelete (PSEGMENT psegIn, /* pointer to input buffer.......*/ - PSEGMENT pseg); /* currently, output buffer......*/ +BOOL CheckMatchForDelete(PSEGMENT psegIn, /* pointer to input buffer.......*/ + PSEGMENT pseg); /* currently, output buffer......*/ -USHORT TmtDeleteTM( HTM hMem, // TM handle - PDELTM_IN pDelTmIn, // Input structure - PDELTM_OUT pDelTmOut ); // Output structure +USHORT TmtDeleteTM(HTM hMem, // TM handle + PDELTM_IN pDelTmIn, // Input structure + PDELTM_OUT pDelTmOut); // Output structure +USHORT TmtCloseOrganize(HTM hMem, // TM handle + PENDORG_IN pEndOrgIn, // Input structure + PENDORG_OUT pEndOrgOut); // Output structure -USHORT TmtCloseOrganize( HTM hMem, // TM handle - PENDORG_IN pEndOrgIn, // Input structure - PENDORG_OUT pEndOrgOut ); // Output structure +VOID TmtDeleteFile(PDELFILE_IN pDelFileIn, // Input structure + PDELFILE_OUT pDelFileOut); // Output structure +VOID GetFileInfo(PFILEINFO_IN pFileInfoIn, // Input structure + PFILEINFO_OUT pFileInfoOut); // Output structure +VOID RenameFile(PRENFILE_IN pRenFileIn, // Input structure + PRENFILE_OUT pRenFileOut); // Output structure -VOID TmtDeleteFile( PDELFILE_IN pDelFileIn, // Input structure - PDELFILE_OUT pDelFileOut ); // Output structure - -VOID GetFileInfo( PFILEINFO_IN pFileInfoIn, // Input structure - PFILEINFO_OUT pFileInfoOut ); // Output structure - -VOID RenameFile( PRENFILE_IN pRenFileIn, // Input structure - PRENFILE_OUT pRenFileOut ); // Output structure - -BOOL fCheckFileClosed ( PSZ ); //--- pointer to file name - +BOOL fCheckFileClosed(PSZ); //--- pointer to file name /*--------------------------------------------------------------------------*\ Function prototypes. \*--------------------------------------------------------------------------*/ USHORT - TmtOpen (POPEN_IN pOpenIn, /* Pointer to input structure....*/ - POPEN_OUT pOpenOut), /* Pointer to output structure...*/ - - TmtCreate (PCREATE_IN pCreateIn, /* Pointer to input structure....*/ - PCREATE_OUT pCreateOut), /* Pointer to output structure...*/ - - TmtClose (PTMT_GLOBALS ptmtg, /* Pointer to Globals structure.....*/ - PCLOSE_OUT pCloseOut); /* Pointer to output buffer.........*/ - -INT_PTR /*CALLBACK*/ MEMCREATEDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMLOADDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMEXPORTDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMMERGEDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMPROPDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MemCreateName( HWND, USHORT, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ UTLSERVERLISTDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ UTLSERVERLISTNAMEDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ MEMINCLUDEDLG( HWND, WINMSG, WPARAM, LPARAM ); -INT_PTR /*CALLBACK*/ SERVWAITDLG( HWND, WINMSG, WPARAM, LPARAM ); -USHORT MemCreateProcess( PMEM_IDA, PSZ, USHORT ); -VOID MemDestroyProcess( PMEM_IDA, USHORT * ); -USHORT MemGetAddressOfProcessIDA( PMEM_IDA, WPARAM, USHORT *, PVOID * ); -VOID MemRcHandlingErrorUndefined( USHORT, PSZ ); -VOID MemRcHandlingErrorUndefinedHwnd( USHORT, PSZ, HWND ); -USHORT EQFMemOrganizeStart( PPROCESSCOMMAREA ); -VOID EQFMemOrganizeProcess( PPROCESSCOMMAREA ); -VOID EQFMemOrganizeEnd( PPROCESSCOMMAREA ); -VOID EQFMemLoadStart( PPROCESSCOMMAREA, HWND ); -VOID EQFMemLoadProcess( PPROCESSCOMMAREA, HWND ); -VOID EQFMemLoadEnd( PPROCESSCOMMAREA, HWND, LPARAM ); -USHORT EQFMemExportStart( PPROCESSCOMMAREA, HWND ); -USHORT EQFMemExportProcess( PPROCESSCOMMAREA, HWND ); -USHORT EQFMemExportEnd( PPROCESSCOMMAREA, HWND, LPARAM ); -VOID EQFMemMergeStart( PPROCESSCOMMAREA, HWND ); -VOID EQFMemMergeProcess( PPROCESSCOMMAREA, HWND ); -VOID EQFMemMergeEnd( PPROCESSCOMMAREA, HWND, LPARAM ); -USHORT EQFMemImportTrojaStart( HWND, LPARAM, HWND ); -USHORT MemInitSlider( HWND, USHORT, PSZ, USHORT, PSZ, PHWND, USHORT, USHORT); -USHORT ReadABGrouping( PSZ, PSZ, ABGROUP ); -MRESULT MemOrganizeCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); -MRESULT MemExportCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); -MRESULT MemMergeCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); -MRESULT MemImportCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); - - +TmtOpen(POPEN_IN pOpenIn, /* Pointer to input structure....*/ + POPEN_OUT pOpenOut), /* Pointer to output structure...*/ + + TmtCreate(PCREATE_IN pCreateIn, /* Pointer to input structure....*/ + PCREATE_OUT pCreateOut), /* Pointer to output structure...*/ + + TmtClose(PTMT_GLOBALS ptmtg, /* Pointer to Globals structure.....*/ + PCLOSE_OUT pCloseOut); /* Pointer to output buffer.........*/ + +INT_PTR /*CALLBACK*/ MEMCREATEDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMLOADDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMEXPORTDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMMERGEDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMPROPDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MemCreateName(HWND, USHORT, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ UTLSERVERLISTDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ UTLSERVERLISTNAMEDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ MEMINCLUDEDLG(HWND, WINMSG, WPARAM, LPARAM); +INT_PTR /*CALLBACK*/ SERVWAITDLG(HWND, WINMSG, WPARAM, LPARAM); +USHORT MemCreateProcess(PMEM_IDA, PSZ, USHORT); +VOID MemDestroyProcess(PMEM_IDA, USHORT *); +USHORT MemGetAddressOfProcessIDA(PMEM_IDA, WPARAM, USHORT *, PVOID *); +VOID MemRcHandlingErrorUndefined(USHORT, PSZ); +VOID MemRcHandlingErrorUndefinedHwnd(USHORT, PSZ, HWND); +USHORT EQFMemOrganizeStart(PPROCESSCOMMAREA); +VOID EQFMemOrganizeProcess(PPROCESSCOMMAREA); +VOID EQFMemOrganizeEnd(PPROCESSCOMMAREA); +VOID EQFMemLoadStart(PPROCESSCOMMAREA, HWND); +VOID EQFMemLoadProcess(PPROCESSCOMMAREA, HWND); +VOID EQFMemLoadEnd(PPROCESSCOMMAREA, HWND, LPARAM); +USHORT EQFMemExportStart(PPROCESSCOMMAREA, HWND); +USHORT EQFMemExportProcess(PPROCESSCOMMAREA, HWND); +USHORT EQFMemExportEnd(PPROCESSCOMMAREA, HWND, LPARAM); +VOID EQFMemMergeStart(PPROCESSCOMMAREA, HWND); +VOID EQFMemMergeProcess(PPROCESSCOMMAREA, HWND); +VOID EQFMemMergeEnd(PPROCESSCOMMAREA, HWND, LPARAM); +USHORT EQFMemImportTrojaStart(HWND, LPARAM, HWND); +USHORT MemInitSlider(HWND, USHORT, PSZ, USHORT, PSZ, PHWND, USHORT, USHORT); +USHORT ReadABGrouping(PSZ, PSZ, ABGROUP); +MRESULT MemOrganizeCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); +MRESULT MemExportCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); +MRESULT MemMergeCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); +MRESULT MemImportCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); // ************* Memory macro definitions ******************************* //======================================================================= -//prototypes and definitions for TM utilities +// prototypes and definitions for TM utilities #define SIZE_32K 32768 #define ERROR_TABLE_FULL 1111 -#define ID_NOT_FOUND 2222 - +#define ID_NOT_FOUND 2222 // adjust pointger for a new location -#define ADJUSTPTR( new, old, offsptr ) \ - ((PBYTE)new + ((PBYTE)offsptr - (PBYTE)old)) +#define ADJUSTPTR(new, old, offsptr) \ + ((PBYTE) new + ((PBYTE)offsptr - (PBYTE)old)) // do not update name tables when name is not contained in the table #define NTMGETID_NOUPDATE_OPT 0x00000001 @@ -3644,54 +3514,49 @@ MRESULT MemImportCallBack( PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM ); // returned ID if a name is not found in the table #define NTMGETID_NOTFOUND_ID 0xFFFF +USHORT TmtXGet(EqfMemory *, PTMX_GET_W, PTMX_GET_OUT_W); -USHORT TmtXGet( EqfMemory*, PTMX_GET_W, PTMX_GET_OUT_W ); +// tm put prototypes +VOID HashSentence(PTMX_SENTENCE); +USHORT HashTupel(PBYTE, USHORT, USHORT); +USHORT HashTupelW(PSZ_W, USHORT); +VOID BuildVotes(PTMX_SENTENCE); +VOID Vote(PTMX_TERM_TOKEN, PTMX_SENTENCE, USHORT); +USHORT CheckCompactArea(PTMX_SENTENCE, EqfMemory *); -//tm put prototypes -VOID HashSentence( PTMX_SENTENCE ); -USHORT HashTupel( PBYTE, USHORT, USHORT ); -USHORT HashTupelW( PSZ_W, USHORT ); -static VOID BuildVotes( PTMX_SENTENCE ); -static VOID Vote( PTMX_TERM_TOKEN, PTMX_SENTENCE, USHORT ); -USHORT CheckCompactArea( PTMX_SENTENCE, EqfMemory* ); +VOID FillTmRecord(PTMX_SENTENCE, + PTMX_RECORD, PTMX_TARGET_CLB, USHORT); -USHORT TokenizeTarget( StringTagVariants*, PTMX_TAGTABLE_RECORD*, PLONG, PSZ, EqfMemory* ); +USHORT DetermineTmRecord(EqfMemory *, PTMX_SENTENCE, PULONG); -VOID FillTmRecord( PTMX_SENTENCE, - PTMX_RECORD, PTMX_TARGET_CLB, USHORT ); +VOID FillTargetRecord(PTMX_SENTENCE, // PTMX_TAGTABLE_RECORD, + PSZ_W, USHORT, PTMX_TARGET_RECORD *, PTMX_TARGET_CLB); -USHORT DetermineTmRecord( EqfMemory*, PTMX_SENTENCE, PULONG ); +VOID DeleteOldestRecord(PTMX_RECORD, PULONG); -VOID FillTargetRecord( PTMX_SENTENCE, //PTMX_TAGTABLE_RECORD, - PSZ_W, USHORT, PTMX_TARGET_RECORD *, PTMX_TARGET_CLB ); +// tm get prototypes +USHORT GetExactMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, + PUSHORT, PTMX_GET_OUT_W); +USHORT ExactTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_SENTENCE, + PTMX_MATCH_TABLE_W, PUSHORT, ULONG); -VOID DeleteOldestRecord( PTMX_RECORD, PULONG ); +BOOL AddTagsToString(PSZ, PULONG, PTMX_TAGTABLE_RECORD, PSZ); +BOOL AddTagsToStringW(PSZ_W, PLONG, PTMX_TAGTABLE_RECORD, PSZ_W); -//tm get prototypes -USHORT GetExactMatch( EqfMemory*, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, - PUSHORT, PTMX_GET_OUT_W ); -USHORT ExactTest( EqfMemory*, PTMX_RECORD, PTMX_GET_W, PTMX_SENTENCE, - PTMX_MATCH_TABLE_W, PUSHORT, ULONG ); +INT CompCount(const void *, const void *); +INT CompCountVotes(const void *, const void *); +VOID CleanupTempMatch(PTMX_MATCHENTRY, PTMX_MATCHENTRY *, PUSHORT, PUSHORT); +USHORT FillMatchEntry(EqfMemory *, PTMX_SENTENCE, PTMX_MATCHENTRY, PUSHORT); +USHORT FuzzyTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, + PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, ULONG); +USHORT GetFuzzyMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT); -BOOL AddTagsToString( PSZ, PULONG, PTMX_TAGTABLE_RECORD, PSZ ); -BOOL AddTagsToStringW( PSZ_W, PLONG, PTMX_TAGTABLE_RECORD, PSZ_W ); - -INT CompCount( const void *, const void * ); -INT CompCountVotes( const void *, const void * ); -VOID CleanupTempMatch( PTMX_MATCHENTRY, PTMX_MATCHENTRY *, PUSHORT, PUSHORT ); -USHORT FillMatchEntry( EqfMemory*, PTMX_SENTENCE, PTMX_MATCHENTRY, PUSHORT ); -USHORT FuzzyTest( EqfMemory*, PTMX_RECORD, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, - PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, ULONG ); -USHORT GetFuzzyMatch( EqfMemory*, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT ); - - -//tm delete segment prototypes -USHORT NTMCheckForUpdates( EqfMemory* ); -USHORT NTMLockTM( EqfMemory*, BOOL, PBOOL ); -USHORT MemBatchTMCreate( HWND hwnd, PDDEMEMCRT pMemCrt ); -USHORT MemBatchTMExport( HWND hwnd, PDDEMEMEXP pMemExp ); -USHORT MemBatchTMImport( HWND hwnd, PDDEMEMIMP pMemImp ); -USHORT MemBatchTMDelete( HWND hwnd, PDDEMEMDEL pMemDel ); +// tm delete segment prototypes +USHORT NTMCheckForUpdates(EqfMemory *); +USHORT MemBatchTMCreate(HWND hwnd, PDDEMEMCRT pMemCrt); +USHORT MemBatchTMExport(HWND hwnd, PDDEMEMEXP pMemExp); +USHORT MemBatchTMImport(HWND hwnd, PDDEMEMIMP pMemImp); +USHORT MemBatchTMDelete(HWND hwnd, PDDEMEMDEL pMemDel); // definitions for terse of TM name tables // max size of in-memory name table @@ -3705,35 +3570,33 @@ USHORT MemBatchTMDelete( HWND hwnd, PDDEMEMDEL pMemDel ); // structure of header for tersed name tables typedef struct _TERSEHEADER { - USHORT usAllocSize; // alloc size in untersed tables - USHORT usMaxEntries; // max entries in untersed tables - ULONG ulMagicWord; // magic word for tersed name - USHORT usDataSize; // size of data area when expanded - USHORT usCompression; // type of compression used for data - // (values as defined in EQFCMPR.H) + USHORT usAllocSize; // alloc size in untersed tables + USHORT usMaxEntries; // max entries in untersed tables + ULONG ulMagicWord; // magic word for tersed name + USHORT usDataSize; // size of data area when expanded + USHORT usCompression; // type of compression used for data + // (values as defined in EQFCMPR.H) } TERSEHEADER, *PTERSEHEADER; -BOOL TMDelTargetClb -( - PTMX_RECORD pTmRecord, // ptr to TM record - PTMX_TARGET_RECORD pTargetRecord, // ptr to target record within TM record - PTMX_TARGET_CLB pTargetClb // ptr to target control record +BOOL TMDelTargetClb( + PTMX_RECORD pTmRecord, // ptr to TM record + PTMX_TARGET_RECORD pTargetRecord, // ptr to target record within TM record + PTMX_TARGET_CLB pTargetClb // ptr to target control record ); -ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG usLenChar ); -LONG EQFCompress2Unicode( PSZ_W pOutput, PBYTE pTarget, ULONG usLenComp ); +ULONG EQFUnicode2Compress(PBYTE pTarget, PSZ_W pInput, ULONG usLenChar); +LONG EQFCompress2Unicode(std::wstring &pOutput, PBYTE pTarget, ULONG usLenComp); #include "EncodingHelper.h" - /* class FilterParam{ public: - FilterParam(std::string& search, FilterField field, FilterType type): m_field(field), m_type(type), + FilterParam(std::string& search, FilterField field, FilterType type): m_field(field), m_type(type), m_searchString(search), m_searchStringW(convertStrToWstr(search)){}; - + FilterParam(long t1, long t2, FilterField field = TIMESTAMP): m_timestamp1(t1), m_timestamp2(t2), m_field{field}{} private: @@ -3743,8 +3606,6 @@ class FilterParam{ long m_timestamp1=0, m_timestamp2=0; };//*/ - - /* class StringFilterParam{ std::string m_searchString; @@ -3754,487 +3615,467 @@ class StringFilterParam{ StringFilterParam(std::string& search, ProposalFilter::FilterType type): FilterParam(type, field), {} }//*/ -typedef struct _MEM_ORGANIZE_IDA +struct MEM_ORGANIZE_IDA { - std::shared_ptr memRef; - TMX_PUT_W stPutIn; // input for TmReplace - TMX_EXT_OUT_W stPutOut; // The REPLACE_OUT structure - TMX_EXT_IN_W stExtIn; // input for TmExtract - TMX_EXT_OUT_W stExtOut; // output for TmExtract - CHAR szPathOrganizeMem[2048]; // Full path and name of TM to be organized - CHAR szDrive[MAX_DRIVE]; // Drive letter without colum - CHAR szMemName[MAX_LONGFILESPEC]; // Translation memory name - CHAR szTempMemName[MAX_LONGFILESPEC];// name of temporary translation memory - CHAR szPluginName[MAX_LONGFILESPEC]; // name of plugin used for the memory - CHAR szPathTempMem[2048]; // Full path of temporary transl. mem. - OtmProposal *pProposal; // buffer for memory proposal - std::shared_ptr pMem; // Handle of transl. memory - std::shared_ptr pMemTemp; // Handle of tmporary transl. memory - BOOL fMsg; // A message has been issued already - //ULONG ulSegmentCounter; // Number of segments organized - //ULONG ulInvSegmentCounter; // Invalid Segment counter - TIME_L tStamp; // Time stamp of organize start time - HWND hProgressWindow; // Handle of progress indicator window - ULONG ulProgressPos; // position of progress indicator - CHAR szPathTempIndex[MAX_EQF_PATH]; //full temporary index name - - LANGUAGE szSourceLanguage; // source language of org TM - LANGUAGE szTargetLanguage; // target language of org TM - CHAR szTagTable[MAX_FILESPEC]; // tag table of org TM - - CHAR szPropertyName[MAX_FILESPEC]; // property name with ext. - CHAR szTempPropertyName[MAX_FILESPEC]; // property name temp TM with ext - - CHAR szOrgProp[MAX_EQF_PATH]; // full name of original property file - CHAR szTmpProp[MAX_EQF_PATH]; // full name of temporary property file - - CHAR szEqfPath[MAX_EQF_PATH]; // system path D:\EQF - USHORT usOrgType; // type ORGANIZE, CONVERT - BOOL fBatch; // TRUE if organizing in batch mode - HWND hwndErrMsg; // handle of window to be used for error msgs - PDDEMEMORG pDDEMemOrg; // ptr to batch memory organize data - USHORT usRC; // return code / error code - LONG NextTask; // next task in non-DDE batch mode - PSZ pszNameList; // pointer to list of TMs being organized - PSZ pszActiveName; // points to current name in pszNameList - CHAR szBuffer[2048]; // general purpose buffer - BOOL fFirstGet; // TRUE = this is the first get access - std::vector m_reorganizeFilters; - MutexTimeout tmListTimeout ; - MutexTimeout tmLockTimeout ; -}MEM_ORGANIZE_IDA, * PMEM_ORGANIZE_IDA; - -USHORT TmCloseOrganize( PMEM_ORGANIZE_IDA, USHORT ); - -USHORT NTMCloseOrganize ( PMEM_ORGANIZE_IDA, USHORT ); - -USHORT NTMConvertProperties( PPROPTRANSLMEM, PMEM_ORGANIZE_IDA ); + std::shared_ptr memRef; + TMX_PUT_W stPutIn; // input for TmReplace + TMX_EXT_OUT_W stPutOut; // The REPLACE_OUT structure + TMX_EXT_IN_W stExtIn; // input for TmExtract + TMX_EXT_OUT_W stExtOut; // output for TmExtract + CHAR szPathOrganizeMem[2048]; // Full path and name of TM to be organized + CHAR szDrive[MAX_DRIVE]; // Drive letter without colum + CHAR szMemName[MAX_LONGFILESPEC]; // Translation memory name + CHAR szTempMemName[MAX_LONGFILESPEC]; // name of temporary translation memory + CHAR szPluginName[MAX_LONGFILESPEC]; // name of plugin used for the memory + CHAR szPathTempMem[2048]; // Full path of temporary transl. mem. + OtmProposal *pProposal; // buffer for memory proposal + std::shared_ptr pMem; // Handle of transl. memory + std::shared_ptr pMemTemp; // Handle of tmporary transl. memory + BOOL fMsg; // A message has been issued already + // ULONG ulSegmentCounter; // Number of segments organized + // ULONG ulInvSegmentCounter; // Invalid Segment counter + TIME_L tStamp; // Time stamp of organize start time + HWND hProgressWindow; // Handle of progress indicator window + ULONG ulProgressPos; // position of progress indicator + CHAR szPathTempIndex[MAX_EQF_PATH]; // full temporary index name + + LANGUAGE szSourceLanguage; // source language of org TM + LANGUAGE szTargetLanguage; // target language of org TM + CHAR szTagTable[MAX_FILESPEC]; // tag table of org TM + + CHAR szPropertyName[MAX_FILESPEC]; // property name with ext. + CHAR szTempPropertyName[MAX_FILESPEC]; // property name temp TM with ext + + CHAR szOrgProp[MAX_EQF_PATH]; // full name of original property file + CHAR szTmpProp[MAX_EQF_PATH]; // full name of temporary property file + + CHAR szEqfPath[MAX_EQF_PATH]; // system path D:\EQF + USHORT usOrgType; // type ORGANIZE, CONVERT + BOOL fBatch; // TRUE if organizing in batch mode + HWND hwndErrMsg; // handle of window to be used for error msgs + PDDEMEMORG pDDEMemOrg; // ptr to batch memory organize data + USHORT usRC; // return code / error code + LONG NextTask; // next task in non-DDE batch mode + PSZ pszNameList; // pointer to list of TMs being organized + PSZ pszActiveName; // points to current name in pszNameList + CHAR szBuffer[2048]; // general purpose buffer + BOOL fFirstGet; // TRUE = this is the first get access + std::vector m_reorganizeFilters; + MutexTimeout tmListTimeout; + MutexTimeout tmLockTimeout; +}; +using PMEM_ORGANIZE_IDA = MEM_ORGANIZE_IDA*; +USHORT TmCloseOrganize(PMEM_ORGANIZE_IDA, USHORT); + +USHORT NTMCloseOrganize(PMEM_ORGANIZE_IDA, USHORT); + +USHORT NTMConvertProperties(PPROPTRANSLMEM, PMEM_ORGANIZE_IDA); /*! \brief constant defining the maximum number of opened memories -*/ + */ #define OTMMEMSERVICE_MAX_NUMBER_OF_OPEN_MEMORIES 40 -//5MB reserved for service +// 5MB reserved for service #define MEMORY_RESERVED_FOR_SERVICE 5000000 - class CreateMemRequestData; class EqfMemoryPlugin : public OtmPlugin /*! \brief This class implements the standard translation memory plugin (EQF) for OpenTM2. -*/ + */ { - //static EqfMemoryPlugin* _instance; + // static EqfMemoryPlugin* _instance; public: -/*! \brief Constructor -*/ - EqfMemoryPlugin(); -/*! \brief Destructor -*/ - ~EqfMemoryPlugin(); + /*! \brief Constructor + */ + EqfMemoryPlugin(); + /*! \brief Destructor + */ + ~EqfMemoryPlugin(); - static EqfMemoryPlugin* GetInstance(); + static EqfMemoryPlugin *GetInstance(); - virtual bool isUsable() - {return OtmPlugin::isUsable();}; + virtual bool isUsable() + { + return OtmPlugin::isUsable(); + }; - -/*! \enum eRegRc - Possible return values of EqfMemory and EqfMemoryPlugin methods -*/ - enum eRc - { - eSuccess = 0, /*!< method completed successfully */ - eUnknownPlugin, /*!< the specified memory plugin is not available */ -// eInvalidName, /*!< plugin-name is invalid */ -// eAlreadyRegistered, /*!< plugin with same name was already registered before */ -// eInvalidRequest, /*!< method may only be called from within registerPlugins call */ - eMemoryNotFound, /*!< the specified memory does not exist or is not controlled by this memory plugin*/ - eUnknown, /*!< plugin with that name was not registered before */ - eNotSupported, /*!< method is not supported by this plugin */ - eBufferTooSmall, /*!< the provided buffer is too small */ + /*! \enum eRegRc + Possible return values of EqfMemory and EqfMemoryPlugin methods + */ + enum eRc + { + eSuccess = 0, /*!< method completed successfully */ + eUnknownPlugin, /*!< the specified memory plugin is not available */ + // eInvalidName, /*!< plugin-name is invalid */ + // eAlreadyRegistered, /*!< plugin with same name was already registered before */ + // eInvalidRequest, /*!< method may only be called from within registerPlugins call */ + eMemoryNotFound, /*!< the specified memory does not exist or is not controlled by this memory plugin*/ + eUnknown, /*!< plugin with that name was not registered before */ + eNotSupported, /*!< method is not supported by this plugin */ + eBufferTooSmall, /*!< the provided buffer is too small */ eNotSupportedMemoryType, - eNotEnoughMemory, /*!< not enough system memory to process the request */ - eRepeat /*!< repeat calling this method until processing has been completed*/ - }; - - -// options for the importFromMemoryFiles method -static const int IMPORTFROMMEMFILES_COMPLETEINONECALL_OPT = 1; // complete the import in one call, do not divide the processing into smaller steps - - -/*! \brief Returns the name of the plugin -*/ - const char* getName(); -/*! \brief Returns a short plugin-Description -*/ - const char* getShortDescription(); -/*! \brief Returns a verbose plugin-Description -*/ - const char* getLongDescription(); -/*! \brief Returns the version of the plugin -*/ - const char* getVersion(); -/*! \brief Returns the name of the plugin-supplier -*/ - const char* getSupplier(); - - EqfMemory* initTM(const std::string& memName, - size_t requiredMemory, - unsigned short usAccessMode, - bool fReorganizeOnly); + eNotEnoughMemory, /*!< not enough system memory to process the request */ + eRepeat /*!< repeat calling this method until processing has been completed*/ + }; -/*! \brief Close a memory - \param pMemory pointer to memory object -*/ + // options for the importFromMemoryFiles method + static const int IMPORTFROMMEMFILES_COMPLETEINONECALL_OPT = 1; // complete the import in one call, do not divide the processing into smaller steps + + /*! \brief Returns the name of the plugin + */ + const char *getName(); + /*! \brief Returns a short plugin-Description + */ + const char *getShortDescription(); + /*! \brief Returns a verbose plugin-Description + */ + const char *getLongDescription(); + /*! \brief Returns the version of the plugin + */ + const char *getVersion(); + /*! \brief Returns the name of the plugin-supplier + */ + const char *getSupplier(); + + EqfMemory *initTM(const std::string &memName, + size_t requiredMemory, + unsigned short usAccessMode, + bool fReorganizeOnly); + + /*! \brief Close a memory + \param pMemory pointer to memory object + */ int closeMemory( - EqfMemory *pMemory - ); + EqfMemory *pMemory); -/*! \brief set description of a memory - \param pszName name of the memory - \param pszDesc description information - \returns 0 if successful or error return code -*/ + /*! \brief set description of a memory + \param pszName name of the memory + \param pszDesc description information + \returns 0 if successful or error return code + */ int setDescription( - const char* pszName, - const char* pszDesc); + const char *pszName, + const char *pszDesc); -/*! \brief Get the error message for the last error occured + /*! \brief Get the error message for the last error occured - \param strError reference to a string receiving the error mesage text - \returns last error code -*/ - int getLastError - ( - std::string &strError - ); -/*! \brief Get the error message for the last error occured - - \param pszError pointer to a buffer for the error text - \param iBufSize size of error text buffer in number of characters - \returns last error code -*/ - int getLastError - ( - char *pszError, - int iBufSize - ); - - -/*! \brief Stops the plugin. - Terminating-function for the plugin, will be called directly before - the DLL containing the plugin will be unloaded.\n - The method should call PluginManager::deregisterPlugin() to tell the PluginManager - that the plugin is not active anymore. - Warning: THIS METHOD SHOULD BE CALLED BY THE PLUGINMANAGER ONLY! - \param fForce, TRUE = force stop of the plugin even if functions are active, FALSE = only stop plugin when it is inactive - \returns TRUE when successful */ - bool stopPlugin( bool fForce = false ); - -/*! \brief Handle a return code from the memory functions and create the approbriate error message text for it - \param iRC return code from memory function - \param pszMemName long memory name - \param pszMarkup markup table name or NULL if not available - \param pszMemPath fully qualified memory path name or NULL if not available - \param strLastError reference to string object receiving the message text - \param iLastError reference to a integer variable receiving the error code - \returns original or modified error return code -*/ -static int handleError( int iRC, char *pszMemName, char *pszMarkup, char *pszMemPath, std::string &strLastError, int &iLastError ); - - int iLastError; - std::string strLastError; - std::vector< std::shared_ptr > m_MemInfoVector; + \param strError reference to a string receiving the error mesage text + \returns last error code + */ + int getLastError( + std::string &strError); + /*! \brief Get the error message for the last error occured -private: + \param pszError pointer to a buffer for the error text + \param iBufSize size of error text buffer in number of characters + \returns last error code + */ + int getLastError( + char *pszError, + int iBufSize); + + /*! \brief Stops the plugin. + Terminating-function for the plugin, will be called directly before + the DLL containing the plugin will be unloaded.\n + The method should call PluginManager::deregisterPlugin() to tell the PluginManager + that the plugin is not active anymore. + Warning: THIS METHOD SHOULD BE CALLED BY THE PLUGINMANAGER ONLY! + \param fForce, TRUE = force stop of the plugin even if functions are active, FALSE = only stop plugin when it is inactive + \returns TRUE when successful */ + bool stopPlugin(bool fForce = false); + + /*! \brief Handle a return code from the memory functions and create the approbriate error message text for it + \param iRC return code from memory function + \param pszMemName long memory name + \param pszMarkup markup table name or NULL if not available + \param pszMemPath fully qualified memory path name or NULL if not available + \param strLastError reference to string object receiving the message text + \param iLastError reference to a integer variable receiving the error code + \returns original or modified error return code + */ + static int handleError(int iRC, char *pszMemName, char *pszMarkup, char *pszMemPath, std::string &strLastError, int &iLastError); + + int iLastError; + std::string strLastError; + std::vector> m_MemInfoVector; - std::shared_ptr findMemory( const char *pszName ); +private: + std::shared_ptr findMemory(const char *pszName); int findMemoryIndex(const char *pszName); - std::string name; - std::string shortDesc; - std::string longDesc; - std::string version; - std::string supplier; - std::string descrType; - char szBuffer[4000]; // general purpose buffer area + std::string name; + std::string shortDesc; + std::string longDesc; + std::string version; + std::string supplier; + std::string descrType; + char szBuffer[4000]; // general purpose buffer area char szSupportedDrives[27]; // list of supported drives - }; -//enum TIMED_BOOL{ -// FALSE = 0, -// TRUE = 1, -// TIMEOUT = -2 -//}; +// enum TIMED_BOOL{ +// FALSE = 0, +// TRUE = 1, +// TIMEOUT = -2 +// }; -class TMManager{ +class TMManager +{ - public: - - +public: std::atomic_bool fWriteRequestsAllowed{0}; std::atomic_bool fServiceIsRunning{1}; - /*! \brief Pointer to the list of opened memories - */ - //std::vector vMemoryList; - typedef std::map > TMMap; + /*! \brief Pointer to the list of opened memories + */ + // std::vector vMemoryList; + typedef std::map> TMMap; TMMap tms; std::recursive_timed_mutex mutex_requestTM; std::recursive_timed_mutex mutex_access_tms; - enum TMManagerCodes{ + enum TMManagerCodes + { TMM_NO_ERROR = 0, TMM_TMD_NOT_FOUND = 16, TMM_TMI_NOT_FOUND = 32, TMM_TM_NOT_FOUND = TMM_TMD_NOT_FOUND + TMM_TMI_NOT_FOUND, - }; - bool IsMemoryLoaded(const std::string& strMemName, MutexTimeout& tmListTimeout); - bool IsMemoryInList(const std::string& strMemName, MutexTimeout& tmListTimeout); - bool IsMemoryLoading(const std::string& strMemName, MutexTimeout& tmListTimeout); - bool IsMemoryFailedToLoad(const std::string& strMemName, MutexTimeout& tmListTimeout); - int TMExistsOnDisk(const std::string& tmName, bool logErrorIfNotExists = true); + bool IsMemoryLoaded(const std::string &strMemName, MutexTimeout &tmListTimeout); + bool IsMemoryInList(const std::string &strMemName, MutexTimeout &tmListTimeout); + bool IsMemoryLoading(const std::string &strMemName, MutexTimeout &tmListTimeout); + bool IsMemoryFailedToLoad(const std::string &strMemName, MutexTimeout &tmListTimeout); + int TMExistsOnDisk(const std::string &tmName, bool logErrorIfNotExists = true); - int AddMem(const std::shared_ptr NewMem, MutexTimeout& tmListTimeout); - int OpenTM(const std::string& strMemName, MutexTimeout& tmListTimeout, COMMAND command); - int CloseTM(const std::string& strMemName, MutexTimeout& tmListTimeout); - int DeleteTM(const std::string& strMemName, std::string& outputMsg); + int AddMem(const std::shared_ptr NewMem, MutexTimeout &tmListTimeout); + int OpenTM(const std::string &strMemName, MutexTimeout &tmListTimeout, COMMAND command); + int CloseTM(const std::string &strMemName, MutexTimeout &tmListTimeout); + int DeleteTM(const std::string &strMemName, std::string &outputMsg); - int RenameTM(const std::string& oldMemName, const std::string& newMemName, std::string& outputMsg,MutexTimeout& tmListTimeout); + int RenameTM(const std::string &oldMemName, const std::string &newMemName, std::string &outputMsg, MutexTimeout &tmListTimeout); int MoveTM( - const std::string& oldMemoryName, - const std::string& newMemoryName, - std::string &strError, - MutexTimeout& tmListTimeout - ); - - std::shared_ptr requestServicePointer(const std::string& strMemName, MutexTimeout& requestTMTimeout, MutexTimeout& tmListTimeout, COMMAND command); - std::shared_ptr requestReadOnlyTMPointer(const std::string& strMemName, std::shared_ptr& refBack, MutexTimeout& requestTMTimeout, MutexTimeout& tmListTimeout, COMMAND command); - std::shared_ptr requestWriteTMPointer(const std::string& strMemName, std::shared_ptr& refBack, MutexTimeout& requestTMTimeout, MutexTimeout& tmListTimeout, COMMAND command); - - std::shared_ptr CreateNewEmptyTM(const std::string& strMemName, const std::string& strSrcLang, - const std::string& strMemDescription, int& _rc_, bool keepInRamOnly = false); - - - /*! \brief OpenTM2 API session handle - */ - LONG hSession = 0; - /*! \brief close a memory and remove it from the open list - \param iIndex index of memory in the open list - \returns 0 - */ - - /*! \brief find a memory in our list of active memories - \param pszMemory name of the memory - \returns index in the memory table if successful, -1 if memory is not contained in the list - */ - int findMemoryInList( const std::string& memName ); - - /*! \brief close any memories which haven't been used for a long time - \returns 0 - */ - size_t CleanupMemoryList(size_t memoryRequested, MutexTimeout& tmListTimeout); - - - /*! \brief close any memories which haven't been used for a long time - \returns size of closed tms - */ - size_t CloseTmsThatWasNotUsedForTooLong(MutexTimeout& tmListTimeout, const std::string& tmToOpen); - - /*! \brief calcuate total amount of RAM occupied by opened memory files - \returns 0 - */ - size_t CalculateOccupiedRAM(MutexTimeout& tmListTimeout); - - std::shared_ptr findOpenedMemory( const std::string& memName); - - int GetMemImportInProcessCount(MutexTimeout& tmListTimeout); - - /*! \brief Close all open memories - \returns http return code0 if successful or an error code in case of failures - */ - int closeAll(); - -///MEMORY FACTORY REGION - /*! \brief This class provides factory methods for EqfMemory objects + const std::string &oldMemoryName, + const std::string &newMemoryName, + std::string &strError, + MutexTimeout &tmListTimeout); + + std::shared_ptr requestServicePointer(const std::string &strMemName, MutexTimeout &requestTMTimeout, MutexTimeout &tmListTimeout, COMMAND command); + std::shared_ptr requestReadOnlyTMPointer(const std::string &strMemName, std::shared_ptr &refBack, MutexTimeout &requestTMTimeout, MutexTimeout &tmListTimeout, COMMAND command); + std::shared_ptr requestWriteTMPointer(const std::string &strMemName, std::shared_ptr &refBack, MutexTimeout &requestTMTimeout, MutexTimeout &tmListTimeout, COMMAND command); + + std::shared_ptr CreateNewEmptyTM(const std::string &strMemName, const std::string &strSrcLang, + const std::string &strMemDescription, int &_rc_, bool keepInRamOnly = false); + + /*! \brief OpenTM2 API session handle + */ + LONG hSession = 0; + /*! \brief close a memory and remove it from the open list + \param iIndex index of memory in the open list + \returns 0 + */ -*/ + /*! \brief find a memory in our list of active memories + \param pszMemory name of the memory + \returns index in the memory table if successful, -1 if memory is not contained in the list + */ + int findMemoryInList(const std::string &memName); -//public: + /*! \brief close any memories which haven't been used for a long time + \returns 0 + */ + size_t CleanupMemoryList(size_t memoryRequested, MutexTimeout &tmListTimeout); + /*! \brief close any memories which haven't been used for a long time + \returns size of closed tms + */ + size_t CloseTmsThatWasNotUsedForTooLong(MutexTimeout &tmListTimeout, const std::string &tmToOpen); -/*! \brief Error code definition -*/ - static const int ERROR_PLUGINNOTAVAILABLE = 1002; - static const int ERROR_MEMORYOBJECTISNULL = 1003; - static const int ERROR_BUFFERTOOSMALL = 1004; - static const int ERROR_INVALIDOBJNAME = 1005; - static const int ERROR_MISSINGPARAMETER = 1006; + /*! \brief calcuate total amount of RAM occupied by opened memory files + \returns 0 + */ + size_t CalculateOccupiedRAM(MutexTimeout &tmListTimeout); -/*! \brief Options for proposal sorting -*/ - static const LONG LATESTPROPOSAL_FIRST = 0x00000001; + std::shared_ptr findOpenedMemory(const std::string &memName); - /*! \brief This static method returns a pointer to the TMManager object. - The first call of the method creates the TMManager instance. -*/ - static TMManager* GetInstance(); + int GetMemImportInProcessCount(MutexTimeout &tmListTimeout); - TMManager(){ - T5LOG( T5DEBUG) << "::Ctor of TMManager"; - pluginList = NULL; - pHandleToMemoryList = new std::vector; - refreshPluginList(); - } + /*! \brief Close all open memories + \returns http return code0 if successful or an error code in case of failures + */ + int closeAll(); -/* \brief Close a memory - Close the memory object and free all memory related resources. - The memory object is not valid anymore. - \param pMemory pointer to the memory object being closes - \returns 0 when successful or a error return code -*/ -int closeMemory -( - EqfMemory *pMemory -); + /// MEMORY FACTORY REGION + /*! \brief This class provides factory methods for EqfMemory objects -/*! \brief Show error message for the last error - \param pszPlugin plugin-name or NULL if not available or memory object name is used - \param pszMemoryName name of the memory causing the problem - memory object name (pluginname + colon + memoryname) - \param pMemory pointer to existing memory object or NULL if not available - \param hwndErrMsg handle of parent window message box */ -void showLastError( - char *pszPluginName, - char *pszMemoryName, - EqfMemory *pMemory, - HWND hwndErrMsg -); -/*! \brief get error message for the last error - \param pMemory pointer to existing memory object or NULL if not available - \param iLastError the last error number - \param strError the error string returned with - \returns the last error string -*/ -std::string& getLastError( - EqfMemory *pMemory, - int& iLastError, - std::string& strError); - -/*! \brief Copy best matches from one proposal vector into another - and sort the proposals - \param SourceProposals refernce to a vector containing the source proposals - \param TargetProposals reference to a vector receiving the copied proposals - the vector may already contain proposals. The proposals are - inserted on their relevance - \param iMaxProposals maximum number of proposals to be filled in TargetProposals - When there are more proposals available proposals with lesser relevance will be replaced -*/ -void copyBestMatches( - std::vector &SourceProposals, - std::vector &TargetProposals, - int iMaxProposals -); + // public: -/*! \brief Copy best matches from one proposal vector into another - and sort the proposals - \param SourceProposals refernce to a vector containing the source proposals - \param TargetProposals reference to a vector receiving the copied proposals - the vector may already contain proposals. The proposals are - inserted on their relevance - \param iMaxProposals maximum number of proposals to be filled in TargetProposals - When there are more proposals available proposals with lesser relevance will be replaced - \param iMTDisplayFactor factor for the placement of machine matches within the table - \param fExactAndFuzzies switch to control the handling of fuzzy matches when exact matches exist, TRUE = keep fuzzy matches even when exact matches exist - \param lOptions options for the sorting of the proposals -*/ -void copyBestMatches( - std::vector &SourceProposals, - std::vector &TargetProposals, - int iMaxProposals, - int iMTDisplayFactor, - BOOL fExactAndFuzzies, - LONG lOptions = 0 -); + /*! \brief Error code definition + */ + static const int ERROR_PLUGINNOTAVAILABLE = 1002; + static const int ERROR_MEMORYOBJECTISNULL = 1003; + static const int ERROR_BUFFERTOOSMALL = 1004; + static const int ERROR_INVALIDOBJNAME = 1005; + static const int ERROR_MISSINGPARAMETER = 1006; -/*! \brief Insert proposal into proposal vector at the correct position and - remove a proposal with lesser relevance when iMaxPropoals have already been filled - \param NewProposal pointer to proposal being inserted - \param SourceProposals refernce to a vector containing the source proposals - \param TargetProposals reference to a vector receiving the copied proposals - the vector may already contain proposals. The proposals are - inserted on their relevance - \param iMaxProposals maximum number of proposals to be filled in TargetProposals - When there are more proposals available proposals with lesser relevance will be replaced - \param fLastEntry true = this is the last entry in the table - \param iMTDisplayFactor factor for the placement of machine matches within the table - \param lOptions options for the sorting of the proposals -*/ -void insertProposalData( - OtmProposal *newProposal, - std::vector &Proposals, - int iMaxProposals, - BOOL fLastEntry, - int iMTDisplayFactor = -1, - LONG lOptions = 0 -); + /*! \brief Options for proposal sorting + */ + static const LONG LATESTPROPOSAL_FIRST = 0x00000001; -/*! \brief Get the object name for the memory - \param pMemory pointer to the memory object - \param pszObjName pointer to a buffer for the object name - \param iBufSize size of the object name buffer - \returns 0 when successful or the error code -*/ -int getObjectName( EqfMemory *pMemory, char *pszObjName, int iBufSize ); - -/*! \brief Get the plugin name and the memory name from a memory object name - \param pszObjName pointer to the memory object name - \param pszPluginName pointer to the buffer for the plugin name or - NULL if no plugin name is requested - \param iPluginBufSize size of the buffer for the plugin name - \param pszluginName pointer to the buffer for the plugin name or - NULL if no memory name is requested - \param iNameBufSize size of the buffer for the memory name - \returns 0 when successful or the error code -*/ -int splitObjName( char *pszObjName, char *pszPluginName, int iPluginBufSize, char *pszMemoryName, int iNameBufSize ); - - /*! \brief Get the sort order key for a memory match - \param Proposal reference to a proposal for which the sort key is evaluated - \param iMTDisplayFactor the machine translation display factor, -1 to ignore the factor - \param usContextRanking the context ranking for the proposal - \param fEndIfTable TRUE when this proposal is the last in a proposal table - When there are more proposals available proposals with lesser relevance will be replaced - \returns the proposal sort key + /*! \brief This static method returns a pointer to the TMManager object. + The first call of the method creates the TMManager instance. */ - int getProposalSortKey( OtmProposal &Proposal, int iMTDisplayFactor, USHORT usContextRanking, BOOL fEndOfTable, LONG lOptions = 0 ); + static TMManager *GetInstance(); - /*! \brief Get the sort order key for a memory match - \param Proposal reference to a proposal for which the sort key is evaluated - \returns the proposal sort key -*/ -int getProposalSortKey( OtmProposal &Proposal ); - - /*! \brief Get the sort order key for a memory match - \param MatchType match type of the match - \param ProposalType type of the proposal - \param iFuzziness fuzziness of the proposal - \param iMTDisplayFactor the machine translation display factor, -1 to use the system MT display factor - \param usContextRanking the context ranking for the proposal - \param fEndIfTable TRUE when this proposal is the last in a proposal table - When there are more proposals available proposals with lesser relevance will be replaced - \returns the proposal sort key -*/ -int getProposalSortKey( OtmProposal::eMatchType MatchType, OtmProposal::eProposalType ProposalType, int iFuzzyness, int iMTDisplayFactor, USHORT usContextRanking, BOOL fEndOfTable, LONG lOptions = 0 ); + TMManager() + { + T5LOG(T5DEBUG) << "::Ctor of TMManager"; + pluginList = NULL; + pHandleToMemoryList = new std::vector; + refreshPluginList(); + } + + /* \brief Close a memory + Close the memory object and free all memory related resources. + The memory object is not valid anymore. + \param pMemory pointer to the memory object being closes + \returns 0 when successful or a error return code + */ + int closeMemory( + EqfMemory *pMemory); + + /*! \brief Show error message for the last error + \param pszPlugin plugin-name or NULL if not available or memory object name is used + \param pszMemoryName name of the memory causing the problem + memory object name (pluginname + colon + memoryname) + \param pMemory pointer to existing memory object or NULL if not available + \param hwndErrMsg handle of parent window message box + */ + void showLastError( + char *pszPluginName, + char *pszMemoryName, + EqfMemory *pMemory, + HWND hwndErrMsg); + + /*! \brief get error message for the last error + \param pMemory pointer to existing memory object or NULL if not available + \param iLastError the last error number + \param strError the error string returned with + \returns the last error string + */ + std::string &getLastError( + EqfMemory *pMemory, + int &iLastError, + std::string &strError); + + /*! \brief Copy best matches from one proposal vector into another + and sort the proposals + \param SourceProposals refernce to a vector containing the source proposals + \param TargetProposals reference to a vector receiving the copied proposals + the vector may already contain proposals. The proposals are + inserted on their relevance + \param iMaxProposals maximum number of proposals to be filled in TargetProposals + When there are more proposals available proposals with lesser relevance will be replaced + */ + void copyBestMatches( + std::vector &SourceProposals, + std::vector &TargetProposals, + int iMaxProposals); + + /*! \brief Copy best matches from one proposal vector into another + and sort the proposals + \param SourceProposals refernce to a vector containing the source proposals + \param TargetProposals reference to a vector receiving the copied proposals + the vector may already contain proposals. The proposals are + inserted on their relevance + \param iMaxProposals maximum number of proposals to be filled in TargetProposals + When there are more proposals available proposals with lesser relevance will be replaced + \param iMTDisplayFactor factor for the placement of machine matches within the table + \param fExactAndFuzzies switch to control the handling of fuzzy matches when exact matches exist, TRUE = keep fuzzy matches even when exact matches exist + \param lOptions options for the sorting of the proposals + */ + void copyBestMatches( + std::vector &SourceProposals, + std::vector &TargetProposals, + int iMaxProposals, + int iMTDisplayFactor, + BOOL fExactAndFuzzies, + LONG lOptions = 0); + + /*! \brief Insert proposal into proposal vector at the correct position and + remove a proposal with lesser relevance when iMaxPropoals have already been filled + \param NewProposal pointer to proposal being inserted + \param SourceProposals refernce to a vector containing the source proposals + \param TargetProposals reference to a vector receiving the copied proposals + the vector may already contain proposals. The proposals are + inserted on their relevance + \param iMaxProposals maximum number of proposals to be filled in TargetProposals + When there are more proposals available proposals with lesser relevance will be replaced + \param fLastEntry true = this is the last entry in the table + \param iMTDisplayFactor factor for the placement of machine matches within the table + \param lOptions options for the sorting of the proposals + */ + void insertProposalData( + OtmProposal *newProposal, + std::vector &Proposals, + int iMaxProposals, + BOOL fLastEntry, + int iMTDisplayFactor = -1, + LONG lOptions = 0); + + /*! \brief Get the object name for the memory + \param pMemory pointer to the memory object + \param pszObjName pointer to a buffer for the object name + \param iBufSize size of the object name buffer + \returns 0 when successful or the error code + */ + int getObjectName(EqfMemory *pMemory, char *pszObjName, int iBufSize); + + /*! \brief Get the plugin name and the memory name from a memory object name + \param pszObjName pointer to the memory object name + \param pszPluginName pointer to the buffer for the plugin name or + NULL if no plugin name is requested + \param iPluginBufSize size of the buffer for the plugin name + \param pszluginName pointer to the buffer for the plugin name or + NULL if no memory name is requested + \param iNameBufSize size of the buffer for the memory name + \returns 0 when successful or the error code + */ + int splitObjName(char *pszObjName, char *pszPluginName, int iPluginBufSize, char *pszMemoryName, int iNameBufSize); + + /*! \brief Get the sort order key for a memory match + \param Proposal reference to a proposal for which the sort key is evaluated + \param iMTDisplayFactor the machine translation display factor, -1 to ignore the factor + \param usContextRanking the context ranking for the proposal + \param fEndIfTable TRUE when this proposal is the last in a proposal table + When there are more proposals available proposals with lesser relevance will be replaced + \returns the proposal sort key + */ + int getProposalSortKey(OtmProposal &Proposal, int iMTDisplayFactor, USHORT usContextRanking, BOOL fEndOfTable, LONG lOptions = 0); + + /*! \brief Get the sort order key for a memory match + \param Proposal reference to a proposal for which the sort key is evaluated + \returns the proposal sort key + */ + int getProposalSortKey(OtmProposal &Proposal); + + /*! \brief Get the sort order key for a memory match + \param MatchType match type of the match + \param ProposalType type of the proposal + \param iFuzziness fuzziness of the proposal + \param iMTDisplayFactor the machine translation display factor, -1 to use the system MT display factor + \param usContextRanking the context ranking for the proposal + \param fEndIfTable TRUE when this proposal is the last in a proposal table + When there are more proposals available proposals with lesser relevance will be replaced + \returns the proposal sort key + */ + int getProposalSortKey(OtmProposal::eMatchType MatchType, OtmProposal::eProposalType ProposalType, int iFuzzyness, int iMTDisplayFactor, USHORT usContextRanking, BOOL fEndOfTable, LONG lOptions = 0); /*! \brief Replace a memory with the data from another memory This method bevaves like deleting the replace memory and renaming the @@ -4243,99 +4084,90 @@ int getProposalSortKey( OtmProposal::eMatchType MatchType, OtmProposal::ePropos \param pszPluginName name of plugin of the memory \param pszReplace name of the memory being replaced \param pszReplaceWith name of the memory replacing the pszReplace memory - returns 0 if successful or error return code + returns 0 if successful or error return code */ - int ReplaceMemory - ( - const std::string& strReplace, - const std::string& strReplaceWith, - std::string& outputMsg, - MutexTimeout& tmListTimeout - ); - + int ReplaceMemory( + const std::string &strReplace, + const std::string &strReplaceWith, + std::string &outputMsg, + MutexTimeout &tmListTimeout); private: - /* \brief Get memory plugin with the given plugin name \param pszPlugin plugin-name \returns pointer to plugin or NULL if no memory pluging with the given name is specified */ -OtmPlugin *getPlugin -( - const char *pszPluginName -); + OtmPlugin *getPlugin( + const char *pszPluginName); -/* \brief Refresh internal list of memory plugins -*/ -void refreshPluginList(); + /* \brief Refresh internal list of memory plugins + */ + void refreshPluginList(); /*! \brief search option flags -*/ -#define SEARCH_SOURCE 1 -#define SEARCH_TARGET 2 + */ +#define SEARCH_SOURCE 1 +#define SEARCH_TARGET 2 #define SEARCH_CASEINSENSITIVE 4 #define SEARCH_WHITESPACETOLERANT 8 + /*! \brief Pointer to the list of installed memory plugins + */ + std::vector *pluginList; -/*! \brief Pointer to the list of installed memory plugins -*/ - std::vector *pluginList; + /*! \brief error code of last error occured + */ + int iLastError; - /*! \brief error code of last error occured -*/ - int iLastError; + /*! \brief error text for last error occured + */ + std::string strLastError; - /*! \brief error text for last error occured -*/ - std::string strLastError; + /*! \brief buffer for memory object names + */ + char szMemObjName[512]; -/*! \brief buffer for memory object names -*/ - char szMemObjName[512]; + /*! \brief buffer for name of default memory plugin + */ + char szDefaultMemoryPlugin[512]; -/*! \brief buffer for name of default memory plugin -*/ - char szDefaultMemoryPlugin[512]; + /*! \brief buffer for name of default shared memory plugin + */ + char szDefaultSharedMemoryPlugin[512]; -/*! \brief buffer for name of default shared memory plugin -*/ - char szDefaultSharedMemoryPlugin[512]; + /*! \brief array containing the memory objects referred to by a handle + */ + std::vector *pHandleToMemoryList; -/*! \brief array containing the memory objects referred to by a handle -*/ -std::vector *pHandleToMemoryList; - -/*! \brief Buffer for segment text -*/ - CHAR_W m_szSegmentText[MAX_SEGMENT_SIZE+1]; + /*! \brief Buffer for segment text + */ + CHAR_W m_szSegmentText[MAX_SEGMENT_SIZE + 1]; /*! \brief Buffer for search string - */ + */ CHAR_W m_szSearchString[MAX_SEGMENT_SIZE + 1]; - -//end of MEMORY FACTORY region + // end of MEMORY FACTORY region }; - /*! \brief copy the data of a MEMPROPOSAL structure to a OtmProposal object - \param pMemProposal pointer to MEMPROPOSAL structure + \param pMemProposal pointer to MEMPROPOSAL structure \param pOtmProposal pointer to OtmProposal object \returns 0 in any case */ -void copyMemProposalToOtmProposal( PMEMPROPOSAL pProposal, OtmProposal *pOtmProposal ); +void copyMemProposalToOtmProposal(PMEMPROPOSAL pProposal, OtmProposal *pOtmProposal); ////OTMFUNC region /** -* \file tm.h -* -* \brief External header file for OpenTM2 API functions -* -* Copyright Notice: -* -* Copyright (C) 1990-2017, International Business Machines -* Corporation and others. All rights reserved -**/ + * \file tm.h + * + * \brief External header file for OpenTM2 API functions + * + * Copyright Notice: + * + * Copyright (C) 1990-2017, International Business Machines + * Corporation and others. All rights reserved + **/ #ifndef _OTMFUNC_H_INCLUDED #define _OTMFUNC_H_INCLUDED @@ -4345,9 +4177,8 @@ void copyMemProposalToOtmProposal( PMEMPROPOSAL pProposal, OtmProposal *pOtmProp // include EQFPAPI.H for segment definitions #ifndef _EQFPAPI_H_INCLUDED - #include "EQFPAPI.H" -#endif - +#include "EQFPAPI.H" +#endif typedef LONG HSESSION; typedef HSESSION *PHSESSION; @@ -4356,46 +4187,42 @@ typedef int BOOL; typedef struct _REPORTTYPE { - PSZ pszReport; + PSZ pszReport; LONG lRepType; - PSZ pszDescription; + PSZ pszDescription; } REPORTTYPE, *PREPORTTYPE; - typedef struct _REPORTSETTINGS { - PSZ pszCountType; + PSZ pszCountType; BOOL bShow; BOOL bSummary; - PSZ pszRepLayout; + PSZ pszRepLayout; BOOL bShrink; - PSZ pszStatisticType; + PSZ pszStatisticType; BOOL bExProposal; } REPORTSETTINGS, *PREPORTSETTINGS; - typedef struct _FACTSHEET { float lComplexity; float lPayFactor; -} FACTSHEET,*PFACTSHEET; - +} FACTSHEET, *PFACTSHEET; typedef struct _FINALFACTORS { - LONG lUnit; + LONG lUnit; float lCurrFactor; - PSZ pszLocalCurrency; -} FINALFACTORS,*PFINALFACTORS; - + PSZ pszLocalCurrency; +} FINALFACTORS, *PFINALFACTORS; typedef struct _EXTFOLPROP { - CHAR chDrive; - CHAR szTargetLang[MAX_LANG_LENGTH]; - CHAR szRWMemory[MAX_LONGFILESPEC]; - CHAR szROMemTbl[MAX_NUM_OF_READONLY_MDB][MAX_LONGFILESPEC]; - CHAR szDicTbl[ NUM_OF_FOLDER_DICS][MAX_FILESPEC]; + CHAR chDrive; + CHAR szTargetLang[MAX_LANG_LENGTH]; + CHAR szRWMemory[MAX_LONGFILESPEC]; + CHAR szROMemTbl[MAX_NUM_OF_READONLY_MDB][MAX_LONGFILESPEC]; + CHAR szDicTbl[NUM_OF_FOLDER_DICS][MAX_FILESPEC]; } EXTFOLPROP, *PEXTFOLPROP; // // option flags used by the function call interface @@ -4404,54 +4231,53 @@ typedef struct _EXTFOLPROP #define NO_ERROR 0 // location flags for TMs and dictionaries -#define LOCAL_OPT 0x00000001L -#define SHARED_OPT 0x00000002L - -// analysis flags -#define TMMATCH_OPT 0x00000010L -#define ADDTOMEM_OPT 0x00000020L -#define AUTOSUBST_OPT 0x00000040L -#define UNTRANSLATED_OPT 0x00000080L -#define AUTOLAST_OPT 0x00000100L -#define AUTOJOIN_OPT 0x00000200L -#define AUTOCONTEXT_OPT 0x00000400L -#define REDUNDCOUNT_OPT 0x00000800L -#define IGNOREPATH_OPT 0x00001000L -#define ADJUSTLEADWS_OPT 0x00002000L -#define ADJUSTTRAILWS_OPT 0x00004000L -#define RESPECTCRLF_OPT 0x00008000L -#define NOBLANKATSEGEND_OPT 0x00010000L +#define LOCAL_OPT 0x00000001L +#define SHARED_OPT 0x00000002L + +// analysis flags +#define TMMATCH_OPT 0x00000010L +#define ADDTOMEM_OPT 0x00000020L +#define AUTOSUBST_OPT 0x00000040L +#define UNTRANSLATED_OPT 0x00000080L +#define AUTOLAST_OPT 0x00000100L +#define AUTOJOIN_OPT 0x00000200L +#define AUTOCONTEXT_OPT 0x00000400L +#define REDUNDCOUNT_OPT 0x00000800L +#define IGNOREPATH_OPT 0x00001000L +#define ADJUSTLEADWS_OPT 0x00002000L +#define ADJUSTTRAILWS_OPT 0x00004000L +#define RESPECTCRLF_OPT 0x00008000L +#define NOBLANKATSEGEND_OPT 0x00010000L #define NOSUBSTIFIDENTICAL_OPT 0x00020000L -#define SENDTOMT_OPT 0x00080000L -#define STOPATFIRSTEXACT_OPT 0x00100000L +#define SENDTOMT_OPT 0x00080000L +#define STOPATFIRSTEXACT_OPT 0x00100000L // ignore memory proposals which are flagged with a comment -#define IGNORECOMMENTED_OPT 0x00200000L +#define IGNORECOMMENTED_OPT 0x00200000L /** @brief protect text inside XMP sections, MsgNum sections, Meta sections, screen sections, and code block sections */ -#define PROTECTXMPSCREEN_OPT 0x00040000L +#define PROTECTXMPSCREEN_OPT 0x00040000L /** @brief protect text inside XMP sections */ -#define PROTECTXMP_OPT 0x00400000L +#define PROTECTXMP_OPT 0x00400000L /** @brief protect text inside MsgNum sections */ -#define PROTECTMSGNUM_OPT 0x00800000L +#define PROTECTMSGNUM_OPT 0x00800000L /** @brief protect text inside Meta sections */ -#define PROTECTMETA_OPT 0x01000000L +#define PROTECTMETA_OPT 0x01000000L /** @brief protect text inside screen sections */ -#define PROTECTSCREEN_OPT 0x02000000L +#define PROTECTSCREEN_OPT 0x02000000L /** @brief protect text inside CodeBlock sections */ -#define PROTECTCODEBLOCK_OPT 0x04000000L - +#define PROTECTCODEBLOCK_OPT 0x04000000L // folder export and folder import flags -#define WITHDICT_OPT 0x00001000L -#define WITHMEM_OPT 0x00002000L -#define DELETE_OPT 0x00004000L -#define WITHREADONLYMEM_OPT 0x00008000L -#define WITHDOCMEM_OPT 0x00010000L -#define MASTERFOLDER_OPT 0x00020000L -#define XLIFF_OPT 0x00040000L -#define NOMARKUP_UPDATE_OPT 0x00080000L +#define WITHDICT_OPT 0x00001000L +#define WITHMEM_OPT 0x00002000L +#define DELETE_OPT 0x00004000L +#define WITHREADONLYMEM_OPT 0x00008000L +#define WITHDOCMEM_OPT 0x00010000L +#define MASTERFOLDER_OPT 0x00020000L +#define XLIFF_OPT 0x00040000L +#define NOMARKUP_UPDATE_OPT 0x00080000L #define WO_REDUNDANCY_DATA_OPT 0x00100000L // @@ -4459,85 +4285,85 @@ typedef struct _EXTFOLPROP // // do not search in sub-directories for specified document names -#define NOSUBDIRSEARCH_OPT 0x00001000L +#define NOSUBDIRSEARCH_OPT 0x00001000L // // flags for document export and wordcount // // Count replace matches separately option (can only be used in conjunction with TMMATCH_OPT) -#define SEPERATEREPLMATCH_OPT 0x02000000L +#define SEPERATEREPLMATCH_OPT 0x02000000L // create word count without duplicates (R009560) -#define DUPMEMMATCH_OPT 0x01000000L +#define DUPMEMMATCH_OPT 0x01000000L // create word count without duplicates (R009560) -#define DUPLICATE_OPT 0x00010000L +#define DUPLICATE_OPT 0x00010000L // create source word count / export source document -#define SOURCE_OPT 0x00020000L +#define SOURCE_OPT 0x00020000L // create translated/untranslated word count / export target document -#define TARGET_OPT 0x00040000L +#define TARGET_OPT 0x00040000L // export SNOMATCH document -#define SNOMATCH_OPT 0x00080000L +#define SNOMATCH_OPT 0x00080000L // count fuzzy matches (only EqfCountWords) -#define FUZZYMATCH_OPT 0x00080000L +#define FUZZYMATCH_OPT 0x00080000L // // flags for dictionary import merge options // (IGNORE_OPT also used in EqfImportMem to skip invalid segments) // -#define IGNORE_OPT 0x00000020L -#define REPLACE_OPT 0x00000040L -#define COMBINE_OPT 0x00000080L +#define IGNORE_OPT 0x00000020L +#define REPLACE_OPT 0x00000040L +#define COMBINE_OPT 0x00000080L // flags for dictionary import/export in DXT UTF8 format -#define DXT_UTF8_OPT 0x00200000L +#define DXT_UTF8_OPT 0x00200000L // external/internal flags -#define EXTERNAL_OPT 0x01000000L -#define INTERNAL_OPT 0x02000000L +#define EXTERNAL_OPT 0x01000000L +#define INTERNAL_OPT 0x02000000L // flags for build archive TM function EqfArchiveTM -#define USEASFOLDERTM_OPT 0x20000000L +#define USEASFOLDERTM_OPT 0x20000000L #define SOURCESOURCEMEM_OPT 0x40000000L -#define SETMFLAG_OPT 0x80000000L +#define SETMFLAG_OPT 0x80000000L // general flags -#define OVERWRITE_OPT 0x10000000L +#define OVERWRITE_OPT 0x10000000L // characterset flags for dictionary/memory import and export -#define ASCII_OPT 0x00020000L -#define ANSI_OPT 0x00040000L -#define UTF16_OPT 0x00080000L +#define ASCII_OPT 0x00020000L +#define ANSI_OPT 0x00040000L +#define UTF16_OPT 0x00080000L // import in TMX format (UTF-16 and UTF-8) -#define TMX_OPT 0x00100000L +#define TMX_OPT 0x00100000L // export in TMX format -#define TMX_UTF16_OPT 0x00100000L -#define TMX_UTF8_OPT 0x00200000L +#define TMX_UTF16_OPT 0x00100000L +#define TMX_UTF8_OPT 0x00200000L // clean/remove RTF tags -#define CLEANRTF_OPT 0x00400000L +#define CLEANRTF_OPT 0x00400000L // restrict RTF tag removal to tags enclosed in curly braces only -#define INCURLYBRACE_OPT 0x00800000L +#define INCURLYBRACE_OPT 0x00800000L // remove linebreaks in text of translation units -#define TMX_NOCRLF_OPT 0x01000000L +#define TMX_NOCRLF_OPT 0x01000000L // import in XLIFF machine translation (MT) format -#define XLIFF_MT_OPT 0x02000000L +#define XLIFF_MT_OPT 0x02000000L // how to handle unknown markup table when importing memory -#define CANCEL_UNKNOWN_MARKUP_OPT 0x10000000L -#define SKIP_UNKNOWN_MARKUP_OPT 0x20000000L -#define GENERIC_UNKNOWN_MARKUP_OPT 0x40000000L +#define CANCEL_UNKNOWN_MARKUP_OPT 0x10000000L +#define SKIP_UNKNOWN_MARKUP_OPT 0x20000000L +#define GENERIC_UNKNOWN_MARKUP_OPT 0x40000000L // option for EqfImportMemEx only: create a new segment match ID even if the match has one already #define FORCENEWMATCHID_OPT 0x00000004L @@ -4546,29 +4372,29 @@ typedef struct _EXTFOLPROP // export documents in validation format (cannot be used together with the other export options) // export validation document in XML format -#define VALFORMAT_XML_OPT 0x00200000L +#define VALFORMAT_XML_OPT 0x00200000L // export validation document in HML format -#define VALFORMAT_HTML_OPT 0x00400000L -//export validation document in DOC format -#define VALFORMAT_DOC_OPT 0x00800000L -//export validation document in ODT format -#define VALFORMAT_ODT_OPT 0x00100000L -//export validation document in DOCX format -#define VALFORMAT_DOCX_OPT 0x08000000L +#define VALFORMAT_HTML_OPT 0x00400000L +// export validation document in DOC format +#define VALFORMAT_DOC_OPT 0x00800000L +// export validation document in ODT format +#define VALFORMAT_ODT_OPT 0x00100000L +// export validation document in DOCX format +#define VALFORMAT_DOCX_OPT 0x08000000L // flags which can be used together with the export options above -// combine documents into a single file -#define VALFORMAT_COMBINE_OPT 0x01000000L +// combine documents into a single file +#define VALFORMAT_COMBINE_OPT 0x01000000L // include protected segments in the validation document -#define VALFORMAT_PROTSEGS_OPT 0x02000000L +#define VALFORMAT_PROTSEGS_OPT 0x02000000L // export documents as plain XML -#define PLAINXML_OPT 0x04000000L +#define PLAINXML_OPT 0x04000000L // relative path document export options -#define WITHRELATIVEPATH_OPT 0x20000000L -#define WITHOUTRELATIVEPATH_OPT 0x40000000L +#define WITHRELATIVEPATH_OPT 0x20000000L +#define WITHOUTRELATIVEPATH_OPT 0x40000000L // document export/import in internal format #define OPENTM2FORMAT_OPT 0x80000000L @@ -4579,59 +4405,57 @@ typedef struct _EXTFOLPROP // // output type options for EqfCountWords and EqfCreateCountReport // -#define TEXT_OUTPUT_OPT 0x00100000L -#define XML_OUTPUT_OPT 0x00200000L -#define HTML_OUTPUT_OPT 0x00400000L +#define TEXT_OUTPUT_OPT 0x00100000L +#define XML_OUTPUT_OPT 0x00200000L +#define HTML_OUTPUT_OPT 0x00400000L // report IDs for function EqfCreateCountReport -#define HISTORY_REP 1 -#define COUNTING_REP 2 -#define CALCULATING_REP 3 -#define PREANALYSIS_REP 4 -#define REDUNDANCY_REP 5 -#define REDUNDANCYSEGMENT_REP 6 +#define HISTORY_REP 1 +#define COUNTING_REP 2 +#define CALCULATING_REP 3 +#define PREANALYSIS_REP 4 +#define REDUNDANCY_REP 5 +#define REDUNDANCYSEGMENT_REP 6 // report types for function EqfCreateCountReport // types for HISTORY_REPORT -#define BRIEF_SORTBYDATE_REPTYPE 1 -#define BRIEF_SORTBYDOC_REPTYPE 2 -#define DETAIL_REPTYPE 3 +#define BRIEF_SORTBYDATE_REPTYPE 1 +#define BRIEF_SORTBYDOC_REPTYPE 2 +#define DETAIL_REPTYPE 3 // types for COUNTING_REPORT -#define WITHTOTALS_REPTYPE 4 -#define WITHOUTTOTALS_REPTYPE 5 +#define WITHTOTALS_REPTYPE 4 +#define WITHOUTTOTALS_REPTYPE 5 // types for CALCULATING_REPORT, PREANALYSIS_REPORT, REDUNDANCY_REPORT, // and REDUNDANCYSEGMENT_REPORT -#define BASE_REPTYPE 6 -#define BASE_SUMMARY_REPTYPE 7 -#define BASE_SUMMARY_FACTSHEET_REPTYPE 8 -#define SUMMARY_FACTSHEET_REPTYPE 9 -#define FACTSHEET_REPTYPE 10 - +#define BASE_REPTYPE 6 +#define BASE_SUMMARY_REPTYPE 7 +#define BASE_SUMMARY_FACTSHEET_REPTYPE 8 +#define SUMMARY_FACTSHEET_REPTYPE 9 +#define FACTSHEET_REPTYPE 10 -//options for EqfCreateITM lType -#define NOANA_TYP 0x1 -#define NOTM_TYP 0x2 -#define PREPARE_TYP 0x4 +// options for EqfCreateITM lType +#define NOANA_TYP 0x1 +#define NOTM_TYP 0x2 +#define PREPARE_TYP 0x4 -//options for EqfChangeMFlag lAction -#define CLEAR_MMOPT 0 -#define SET_MMOPT 1 +// options for EqfChangeMFlag lAction +#define CLEAR_MMOPT 0 +#define SET_MMOPT 1 // options for non-dde interface (calculating report) -#define BASE_TYP 1 -#define FACT_TYP 2 -#define SUM_TYP 4 -#define BRIEF_SORTBYDATE_TYP 8 -#define BRIEF_SORTBYDOC_TYP 16 -#define DETAIL_TYP 32 +#define BASE_TYP 1 +#define FACT_TYP 2 +#define SUM_TYP 4 +#define BRIEF_SORTBYDATE_TYP 8 +#define BRIEF_SORTBYDOC_TYP 16 +#define DETAIL_TYP 32 - -#define PLAUS_OPT 1 // (Plausibilty check) -#define LOST_OPT 2 // (Lost Data: Force new shipment) -#define LIST_OPT 4 // (List of Documents) +#define PLAUS_OPT 1 // (Plausibilty check) +#define LOST_OPT 2 // (Lost Data: Force new shipment) +#define LIST_OPT 4 // (List of Documents) // option for fuzzy segment search to mark the differences in the output file #define MARKDIFFERENCES_OPT 0x08000000L @@ -4641,75 +4465,73 @@ typedef struct _EXTFOLPROP #define SELECTEDCLASSANDHIGHER_MODE 1 #define ONLYSELECTEDCLASS_MODE 2 - // return when all activitiy has been completed (instead of iterative processing) #define COMPLETE_IN_ONE_CALL_OPT 0x01000000 // non-DDE function interface specific return codes // Info: return code for functions not implemented yet -#define NOTIMPLEMENTED_RC 10001 +#define NOTIMPLEMENTED_RC 10001 // Error: passed session handle is invalid -#define ERROR_INVALID_SESSION_HANDLE 10002 +#define ERROR_INVALID_SESSION_HANDLE 10002 // Notification: current function has not been completed yet // re-call function until a non-CONTINUE_RC // return code is retuned -#define CONTINUE_RC 10003 +#define CONTINUE_RC 10003 // Error: Wrong or conflicting options specified -#define WRONG_OPTIONS_RC 10004 +#define WRONG_OPTIONS_RC 10004 // Error: different function has been called although // current task has not been completed yet -#define LASTTASK_INCOMPLETE_RC 10005 +#define LASTTASK_INCOMPLETE_RC 10005 // Error: Segment has been joined to a previous one and is not // part of the visible document. It cannot be used // in function EqfGetSourceLine -#define SEGMENTISJOINED_RC 10006 +#define SEGMENTISJOINED_RC 10006 // Error: Given segment number is invalid // (probably not contained in document) -#define INVALIDSEGMENT_RC 10007 +#define INVALIDSEGMENT_RC 10007 // Error: Given handle of loaded file is invalid -#define INVALIDFILEHANDLE_RC 10008 +#define INVALIDFILEHANDLE_RC 10008 // Error: There is no segment matching the given position -#define NOMATCHINGSEGMENT_RC 10009 +#define NOMATCHINGSEGMENT_RC 10009 // Info: End if data reached -#define ENDREACHED_RC 10010 +#define ENDREACHED_RC 10010 // Info: Time out occurend (eceeded given search time) -#define TIMEOUT_RC 10011 - -#define VAL_COMBINE_OPT 0x00000001L -#define VAL_PRESERVE_LINKS_OPT 0x00000002L -#define VAL_MAN_EXACTMATCH_OPT 0x00000004L -#define VAL_REMOVE_INLINE_OPT 0x00000008L -#define VAL_TRANSTEXT_ONLY_OPT 0x00000010L -#define VAL_INCLUDE_COUNT_OPT 0x00000020L -#define VAL_INCLUDE_MATCH_OPT 0x00000040L -#define VAL_MISMATCHES_ONLY_OPT 0x00000080L - -#define VAL_AUTOSUBST_OPT 0x00000100L -#define VAL_MOD_AUTOSUBST_OPT 0x00000200L -#define VAL_EXACT_OPT 0x00000400L -#define VAL_MOD_EXACT_OPT 0x00000800L -#define VAL_GLOBAL_MEM_OPT 0x00001000L -#define VAL_MACHINE_OPT 0x00002000L -#define VAL_FUZZY_OPT 0x00004000L -#define VAL_NEW_OPT 0x00008000L -#define VAL_NOT_TRANS_OPT 0x00010000L -#define VAL_PROTECTED_OPT 0x00020000L -#define VAL_REPLACE_OPT 0x00040000L - -#define VAL_VALIDATION_OPT 0x01000000L -#define VAL_PROOFREAD_OPT 0x02000000L - +#define TIMEOUT_RC 10011 + +#define VAL_COMBINE_OPT 0x00000001L +#define VAL_PRESERVE_LINKS_OPT 0x00000002L +#define VAL_MAN_EXACTMATCH_OPT 0x00000004L +#define VAL_REMOVE_INLINE_OPT 0x00000008L +#define VAL_TRANSTEXT_ONLY_OPT 0x00000010L +#define VAL_INCLUDE_COUNT_OPT 0x00000020L +#define VAL_INCLUDE_MATCH_OPT 0x00000040L +#define VAL_MISMATCHES_ONLY_OPT 0x00000080L + +#define VAL_AUTOSUBST_OPT 0x00000100L +#define VAL_MOD_AUTOSUBST_OPT 0x00000200L +#define VAL_EXACT_OPT 0x00000400L +#define VAL_MOD_EXACT_OPT 0x00000800L +#define VAL_GLOBAL_MEM_OPT 0x00001000L +#define VAL_MACHINE_OPT 0x00002000L +#define VAL_FUZZY_OPT 0x00004000L +#define VAL_NEW_OPT 0x00008000L +#define VAL_NOT_TRANS_OPT 0x00010000L +#define VAL_PROTECTED_OPT 0x00020000L +#define VAL_REPLACE_OPT 0x00040000L + +#define VAL_VALIDATION_OPT 0x01000000L +#define VAL_PROOFREAD_OPT 0x02000000L class IMPORTMEMORYDATA; /*! \brief Import a Translation Memory @@ -4717,7 +4539,7 @@ class IMPORTMEMORYDATA; \param pszMemName name of the Translation Memory \param pszInFile fully qualified name of the input file \param lOptions options for Translation Memory import - one of the format options + one of the format options - TMX_OPT import in TMX format - XLIFF_MT_OPT import in XLIFF format - UTF16_OPT import in the EXP format (Unicode UTF-16 encoded) @@ -4731,22 +4553,20 @@ class IMPORTMEMORYDATA; . additional options which can be used - CLEANRTF_OPT to remove RTF inline tags (only in combination with TMX_OPT) - - IGNORE_OPT ignore invalid segments and continue the import + - IGNORE_OPT ignore invalid segments and continue the import \returns 0 if successful or an error code in case of failures */ /*@ADDTOSCRIPTER*/ -USHORT EqfImportMem -( - IMPORTMEMORYDATA* pData, - //std::shared_ptr hSession, // mand: Eqf session handle - //PSZ pszInFile, // mand: fully qualified name of input file - LONG lOptions // opt: options for Translation Memory import - // @Import Mode: TMX_OPT{CLEANRTF_OPT}, XLIFF_MT_OPT,UTF16_OPT,ANSI_OPT,ASCII_OPT(default) - // @Markup Table Handling: CANCEL_UNKNOWN_MARKUP_OPT(default), SKIP_UNKNOWN_MARKUP_OPT, GENERIC_UNKNOWN_MARKUP_OPT - // @Other: {CLEANRTF_OPT,IGNORE_OPT} - -); +USHORT EqfImportMem( + IMPORTMEMORYDATA *pData, + // std::shared_ptr hSession, // mand: Eqf session handle + // PSZ pszInFile, // mand: fully qualified name of input file + LONG lOptions // opt: options for Translation Memory import + // @Import Mode: TMX_OPT{CLEANRTF_OPT}, XLIFF_MT_OPT,UTF16_OPT,ANSI_OPT,ASCII_OPT(default) + // @Markup Table Handling: CANCEL_UNKNOWN_MARKUP_OPT(default), SKIP_UNKNOWN_MARKUP_OPT, GENERIC_UNKNOWN_MARKUP_OPT + // @Other: {CLEANRTF_OPT,IGNORE_OPT} +); /*! \brief Organize a Translation Memory \param hSession the session handle returned by the EqfStartSession call @@ -4754,38 +4574,33 @@ USHORT EqfImportMem \returns 0 if successful or an error code in case of failures */ /*@ADDTOSCRIPTER*/ -USHORT EqfOrganizeMem -( - HSESSION hSession, // mand: Eqf session handle - std::shared_ptr pMem, // mand: name of Translation Memory - long& reorgSegCount, - long& invSegCount -); +USHORT EqfOrganizeMem( + HSESSION hSession, // mand: Eqf session handle + std::shared_ptr pMem, // mand: name of Translation Memory + long &reorgSegCount, + long &invSegCount); /*! \brief Start a OpenTM2 API call session \param phSession pointer to a buffer receiving the session handle \returns 0 if successful or an error code in case of failures */ /*@ADDTOSCRIPTER*/ -USHORT EqfStartSession -( - PHSESSION phSession // mand: ptr to callers Eqf session handle variable +USHORT EqfStartSession( + PHSESSION phSession // mand: ptr to callers Eqf session handle variable ); - /*! \brief Get information about the last occured error \param hSession the session handle returned by the EqfStartSession call \param pusRC pointer to the buffer for last return code - \param pszMsgBuffer pointer to a buffer receiving the error message text + \param pszMsgBuffer pointer to a buffer receiving the error message text \param usBufSize size of message buffer in number of bytes \returns 0 if successful or an error code in case of failures */ -USHORT EqfGetLastError -( - HSESSION hSession, // Eqf session handle - PUSHORT pusRC, // ptr to buffer for last return code - PSZ pszMsgBuffer, // ptr to buffer receiving the message - USHORT usBufSize // size of message buffer in bytes +USHORT EqfGetLastError( + HSESSION hSession, // Eqf session handle + PUSHORT pusRC, // ptr to buffer for last return code + PSZ pszMsgBuffer, // ptr to buffer receiving the message + USHORT usBufSize // size of message buffer in bytes ); /*! \brief Get information about the last occured error (in UTF16 encoding) \param hSession the session handle returned by the EqfStartSession call @@ -4794,94 +4609,92 @@ USHORT EqfGetLastError \param usBufSize size of message buffer in number of characters \returns 0 if successful or an error code in case of failures */ -USHORT EqfGetLastErrorW -( - HSESSION hSession, // Eqf session handle - PUSHORT pusRC, // ptr to buffer for last return code - wchar_t *pszMsgBuffer, // ptr to buffer receiving the message - USHORT usBufSize // size of message buffer in bytes +USHORT EqfGetLastErrorW( + HSESSION hSession, // Eqf session handle + PUSHORT pusRC, // ptr to buffer for last return code + wchar_t *pszMsgBuffer, // ptr to buffer receiving the message + USHORT usBufSize // size of message buffer in bytes ); /*! \brief Structure containing the segment or memory match information This structure is used by the EqfGetMatchLevel API for the segment from the document and for the memory proposal */ -//#pragma pack( push, TM2StructPacking, 1 ) +// #pragma pack( push, TM2StructPacking, 1 ) typedef struct _EQFSEGINFO { - WCHAR szSource[EQF_SEGLEN]; // segment source text (UTF-16 encoded) - WCHAR szTarget[EQF_SEGLEN]; // segment target text (UTF-16 encoded) (empty for document segment) - LONG lSegNumber; // segment number - CHAR szDocument[MAX_LONGFILESPEC]; // name of document - CHAR szSourceLanguage[MAX_LANG_LENGTH]; // source language of segment - CHAR szTargetLanguage[MAX_LANG_LENGTH]; // target language of segment - CHAR szMarkup[MAX_FILESPEC]; // markup of segment + WCHAR szSource[EQF_SEGLEN]; // segment source text (UTF-16 encoded) + WCHAR szTarget[EQF_SEGLEN]; // segment target text (UTF-16 encoded) (empty for document segment) + LONG lSegNumber; // segment number + CHAR szDocument[MAX_LONGFILESPEC]; // name of document + CHAR szSourceLanguage[MAX_LANG_LENGTH]; // source language of segment + CHAR szTargetLanguage[MAX_LANG_LENGTH]; // target language of segment + CHAR szMarkup[MAX_FILESPEC]; // markup of segment } EQFSEGINFO, *PEQFSEGINFO; -//#pragma pack( pop, TM2StructPacking ) +// #pragma pack( pop, TM2StructPacking ) // // options for EqfGetMatchLevel API // // suppress generic inline tag replacement -#define NO_GENERIC_INLINETAG_REPL_OPT 0x0001 +#define NO_GENERIC_INLINETAG_REPL_OPT 0x0001 // use generic inline tag replacement function -#define USE_GENERIC_INLINETAG_REPL_OPT 0x0002 +#define USE_GENERIC_INLINETAG_REPL_OPT 0x0002 /*! \brief State/type of matches used by EqfGetMatchLevel API -*/ + */ typedef enum _EQFMATCHSTATE { - REPLACE_MATCHSTATE, // replace match - FUZZYREPLACE_MATCHSTATE, // fuzzy replace match - FUZZY_MATCHSTATE, // fuzzy match - NONE_MATCHSTATE, // no match - EXACT_MATCHSTATE, // exact match - EXACTEXACT_MATCHSTATE // exact-exact match (match from same document and same segment) -} EQFMATCHSTATE, *PEQFMATCHSTATE; + REPLACE_MATCHSTATE, // replace match + FUZZYREPLACE_MATCHSTATE, // fuzzy replace match + FUZZY_MATCHSTATE, // fuzzy match + NONE_MATCHSTATE, // no match + EXACT_MATCHSTATE, // exact match + EXACTEXACT_MATCHSTATE // exact-exact match (match from same document and same segment) +} EQFMATCHSTATE, + *PEQFMATCHSTATE; // options fpr EqfCleanMatch API call // create internal memory -#define CLEANMEM_INTERNAL_MEMORY_OPT 0x00000001 +#define CLEANMEM_INTERNAL_MEMORY_OPT 0x00000001 // create external memory -#define CLEANMEM_EXTERNAL_MEMORY_OPT 0x00000002 +#define CLEANMEM_EXTERNAL_MEMORY_OPT 0x00000002 // return when all activitiy has been completed (instead of iterative processing) #define CLEANMEM_COMPLETE_IN_ONE_CALL_OPT 0x01000000 #define COMPLETE_IN_ONE_CALL_OPT 0x01000000 - // activate logging (will append log info to the log file \EQF\LOGS\CLEANMEMORY.LOG) -#define CLEANMEM_LOGGING_OPT 0x00010000 +#define CLEANMEM_LOGGING_OPT 0x00010000 // if specified only the best match will be written to the output memory // if not specified the best 3 fuzzy matches will be written to the output memory -#define CLEANMEM_BESTMATCH_OPT 0x00020000 +#define CLEANMEM_BESTMATCH_OPT 0x00020000 // merge into existing internal memory -#define CLEANMEM_MERGE_OPT 0x20000000 +#define CLEANMEM_MERGE_OPT 0x20000000 // keep duplicate exact matches and fuzzy matches in the memory -#define CLEANMEM_KEEP_DUPS_OPT 0x40000000 +#define CLEANMEM_KEEP_DUPS_OPT 0x40000000 // rename modes -#define RENAME_FOLDER 1 -#define RENAME_MEMORY 2 -#define RENAME_DICTIONARY 4 +#define RENAME_FOLDER 1 +#define RENAME_MEMORY 2 +#define RENAME_DICTIONARY 4 // options -#define ADJUSTREFERENCES_OPT 0x04000000 +#define ADJUSTREFERENCES_OPT 0x04000000 // constants for API call EqfGetShortName -#define FOLDER_OBJ 1 -#define MEMORY_OBJ 2 -#define DICT_OBJ 3 -#define DOCUMENT_OBJ 4 - +#define FOLDER_OBJ 1 +#define MEMORY_OBJ 2 +#define DICT_OBJ 3 +#define DOCUMENT_OBJ 4 /*! \brief Search segments having fuzzy memory proposals \param hSession the session handle returned by the EqfStartSession call @@ -4903,35 +4716,29 @@ typedef enum _EQFMATCHSTATE \param pSearchKey pointer to a MemProposal structure containing the searched criteria \param *piNumOfProposals pointer to the number of requested memory proposals, will be changed on return to the number of proposals found \param pProposals pointer to a array of MemProposal structures receiving the search results - \param lOptions processing options + \param lOptions processing options \returns 0 if successful or an error code in case of failures */ -USHORT EqfQueryMem -( - HSESSION hSession, - EqfMemory* lHandle, - PMEMPROPOSAL pSearchKey, - int *piNumOfProposals, - PMEMPROPOSAL pProposals, - LONG lOptions -); - +USHORT EqfQueryMem( + HSESSION hSession, + EqfMemory *lHandle, + PMEMPROPOSAL pSearchKey, + int *piNumOfProposals, + PMEMPROPOSAL pProposals, + LONG lOptions); /*! \brief Update a segment in the memory \param hSession the session handle returned by the EqfStartSession call \param lHandle handle of a previously opened memory \param pNewProposal pointer to an MemProposal structure containing the segment data - \param lOptions processing options + \param lOptions processing options \returns 0 if successful or an error code in case of failures */ -USHORT EqfUpdateMem -( - HSESSION hSession, - EqfMemory* lHandle, - PMEMPROPOSAL pNewProposal, - LONG lOptions -); - +USHORT EqfUpdateMem( + HSESSION hSession, + EqfMemory *lHandle, + PMEMPROPOSAL pNewProposal, + LONG lOptions); /*! \brief Get the OpenTM2 language name for a ISO language identifier \param hSession the session handle returned by the EqfStartSession call @@ -4939,14 +4746,11 @@ USHORT EqfUpdateMem \param pszOpenTM2Lang buffer for the OpenTM2 language name \returns 0 if successful or an error code in case of failures */ -USHORT EqfGetOpenTM2Lang -( - HSESSION hSession, - PSZ pszISOLang, - PSZ pszOpenTM2Lang, - bool* pfPrefered = nullptr -); - +USHORT EqfGetOpenTM2Lang( + HSESSION hSession, + PSZ pszISOLang, + PSZ pszOpenTM2Lang, + bool *pfPrefered = nullptr); /*! \brief Get the ISO language identifier for a OpenTM2 language name \param hSession the session handle returned by the EqfStartSession call @@ -4954,16 +4758,13 @@ USHORT EqfGetOpenTM2Lang \param pszISOLang pointer to a buffer for the ISO language identifier \returns 0 if successful or an error code in case of failures */ -USHORT EqfGetIsoLang -( - HSESSION hSession, - PSZ pszOpenTM2Lang, - PSZ pszISOLang -); +USHORT EqfGetIsoLang( + HSESSION hSession, + PSZ pszOpenTM2Lang, + PSZ pszISOLang); #endif - -///end of OTMFUNC region +/// end of OTMFUNC region #endif //_tm_h_included_ \ No newline at end of file diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 83b68ba2..e76ac1fa 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -280,18 +280,14 @@ USHORT EqfMemory::FindTargetAndDelete( LONG lTagAlloc; //allocate length ULONG ulLen = 0; //length indicator LONG lSrcLen = 0; - PSZ_W pString = NULL; //pointer to character string + std::wstring pString; //pointer to character string USHORT usId = 0; //returned id from function USHORT usRc = NO_ERROR; //returned value from function USHORT usTarget = 0; //initialize counter - //allocate pString - fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - //allocate 4k for pTagRecord if ( fOK ) { - //fOK = UtlAlloc( (PVOID *) &(pTagRecord), 0L, (LONG) TOK_SIZE, NOMSG ); if ( fOK ) lTagAlloc = (LONG)TOK_SIZE; } @@ -312,9 +308,6 @@ USHORT EqfMemory::FindTargetAndDelete( //calculate length of source string ulLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); - //clear pString for later use - memset( pString, 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W) ); - //copy source string for later compare function // memcpy( pString, pByte, ulLen ); ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); @@ -322,7 +315,7 @@ USHORT EqfMemory::FindTargetAndDelete( //compare source strings //if ( !UTF16strcmp( pString, pSentence->pNormString ) ) - if ( !UTF16strcmp( pString, TmDel.pInputSentence->pStrings->getGenericTagStrC() ) ) + if ( !UTF16strcmp( pString.data(), TmDel.pInputSentence->pStrings->getGenericTagStrC() ) ) { ULONG ulLeftTgtLen; // remaining target length /****************************************************************/ @@ -365,13 +358,6 @@ USHORT EqfMemory::FindTargetAndDelete( pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //position at source tag table - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXSourceTagTable = (PTMX_TAGTABLE_RECORD)pByte; - - //compare tag table records - if(//UtlCompIgnWhiteSpaceW(pSentence->pInputString, (wchar_t*)pStringWNormalizedTags.c_str(), pStringWNormalizedTags.size() ) == 0 - true) // we use the same tag tables in t5? { //source tag tables are identical pByte = pStartTarget; @@ -381,11 +367,8 @@ USHORT EqfMemory::FindTargetAndDelete( //calculate length of target string ulLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - //clear pString for later use - memset( pString, 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); - //copy target string for later compare function -// memcpy( pString, pByte, ulLen ); + pString.clear(); ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); //tokenize target string in del structure @@ -395,7 +378,7 @@ USHORT EqfMemory::FindTargetAndDelete( { //compare target strings //if ( !UTF16strcmp( pString, pNormString ) ) - if ( !UTF16strcmp( pString, TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) + if ( !UTF16strcmp( pString.data(), TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) { //target strings are equal so compare target tag records //position at target tag table record @@ -447,7 +430,7 @@ USHORT EqfMemory::FindTargetAndDelete( //fill out the put structure as output of the extract function usRc = FillExtStructure( pTMXTargetRecord, pClb, - TmDel.pInputSentence->pStrings->getGenericTargetStrC(), &lSrcLen, + TmDel.pInputSentence->pStrings->getGenericTargetStr(), &lSrcLen, pTmExtOut ); NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); @@ -506,10 +489,6 @@ USHORT EqfMemory::FindTargetAndDelete( } /* endif */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); - //UtlAlloc( (PVOID *) &(pTagRecord), 0L, 0L, NOMSG ); - return( usRc ); } @@ -528,18 +507,10 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( PBYTE pByte; //position ptr LONG lSourceLen = 0; //length of source string USHORT usTarget = 0; //nr of target records in tm record - PSZ_W pSourceString = NULL; //pointer to source string + std::wstring pSourceString; //pointer to source string PBYTE pSource; //position ptr - UtlAlloc( (PVOID *) &(pSourceString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - - if ( !pSourceString ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - if ( !usRc ) { //check if record is empty @@ -739,8 +710,6 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( } /* endif */ - //release memory - return( usRc ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 2416fa37..e653e2e1 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -334,18 +334,10 @@ USHORT EqfMemory::ExtractRecordV6 PBYTE pByte; //position ptr LONG lSourceLen = 0; //length of source string USHORT usTarget; //nr of target records in tm record - PSZ_W pSourceString = NULL; //pointer to source string + std::wstring pSourceString; //pointer to source string PBYTE pSource; //position ptr - UtlAlloc( (PVOID *) &(pSourceString), 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - - if ( !pSourceString ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - if ( !usRc ) { //check if record is empty @@ -541,10 +533,6 @@ USHORT EqfMemory::ExtractRecordV6 } - // release memory - UtlAlloc( (PVOID *) &pSourceString, 0L, 0L, NOMSG ); - - return usRc; } @@ -583,7 +571,7 @@ USHORT EqfMemory::FillExtStructure ( PTMX_TARGET_RECORD pTMXTargetRecord, //ptr to tm target PTMX_TARGET_CLB pTargetClb, // ptr to current target CLB - PSZ_W pSourceString, //ptr to source string + std::wstring& pSourceString, //ptr to source string PLONG plSourceLen, //length of source string PTMX_EXT_OUT_W pstExt //extout ext struct ) @@ -591,19 +579,12 @@ USHORT EqfMemory::FillExtStructure PTMX_TARGET_CLB pTMXTargetClb = NULL; //ptr to target control block PTMX_TAGTABLE_RECORD pTMXTagTableRecord = NULL; //ptr to tag table record PBYTE pByte; //position pointer - BOOL fOK; //success indicator + BOOL fOK = true; //success indicator USHORT usRc = NO_ERROR; //return code LONG lTargetLen = 0; //length indicator - PSZ_W pTargetString = NULL; //pointer to target string + std::wstring pTargetString; //pointer to target string - //allocate pString - fOK = UtlAlloc( (PVOID *) &(pTargetString), 0L, (LONG)MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else { pByte = (PBYTE)pTMXTargetRecord; //pByte += pTMXTargetRecord->usSourceTagTable; @@ -612,7 +593,7 @@ USHORT EqfMemory::FillExtStructure if(*plSourceLen > 0) { //else copy string - memcpy( pstExt->szSource, pSourceString, *plSourceLen * sizeof(CHAR_W) ); + memcpy( pstExt->szSource, pSourceString.c_str(), *plSourceLen * sizeof(CHAR_W) ); pstExt->szSource[*plSourceLen] = EOS; } /* endif */ @@ -651,15 +632,13 @@ USHORT EqfMemory::FillExtStructure //position at target tag record pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usTargetTagTable; - //pTMXTagTableRecord = (PTMX_TAGTABLE_RECORD)pByte; //fill in the tag table name strcpy(pstExt->szTagTable, "OTMXUXLF"); if(lTargetLen >= 0){ - memcpy( pstExt->szTarget, pTargetString, lTargetLen * sizeof(CHAR_W)); + memcpy( pstExt->szTarget, pTargetString.c_str(), lTargetLen * sizeof(CHAR_W)); pstExt->szTarget[lTargetLen] = EOS; } /* endif */ @@ -711,8 +690,8 @@ USHORT EqfMemory::FillExtStructure } else if ( pTMXTargetClb->usAddDataLen ) { - NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstExt->szContext, sizeof(pstExt->szContext) / sizeof(CHAR_W) ); - NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstExt->szAddInfo, sizeof(pstExt->szAddInfo) / sizeof(CHAR_W) ); + NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstExt->szContext); + NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstExt->szAddInfo ); } /* endif */ } /* endif */ if(fOK){ @@ -722,8 +701,5 @@ USHORT EqfMemory::FillExtStructure } } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pTargetString, 0L, 0L, NOMSG ); - return( usRc ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index ade89b5c..b50c69dd 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -75,7 +75,7 @@ void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ); -USHORT FillMatchTable( EqfMemory*, PSZ_W, PLONG, PTMX_TARGET_RECORD, +USHORT FillMatchTable( EqfMemory*, std::wstring&, PLONG, PTMX_TARGET_RECORD, PTMX_TARGET_CLB, PTMX_MATCH_TABLE_W, PUSHORT, BOOL, PUSHORT, PUSHORT, PUSHORT, ULONG, USHORT, PTMX_GET_W, PTMX_TAGTABLE_RECORD, USHORT, USHORT, int, int, USHORT); @@ -946,7 +946,7 @@ USHORT ExactTest BOOL fStringEqual = FALSE; // indicator for string equal BOOL fEqualFound = FALSE; // tagging equal ?? USHORT usEqual = 0; - std::string pContextBuffer, pString; //pointer to buffer for context processing + std::wstring pContextBuffer, pString; //pointer to buffer for context processing USHORT usTargetTranslationFlag = (USHORT)-1; // translation flag of target CLB (processed) PSZ pszDocName = NULL; // document name to check for equal document names in GET_IGNORE_PATH mode @@ -971,7 +971,7 @@ USHORT ExactTest //copy source string for later compare function ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - auto normalizedTmStr = std::make_unique(pString); + auto normalizedTmStr = std::make_unique(pString.c_str()); //compare source strings fStringEqual = UtlCompIgnWhiteSpaceW(normalizedTmStr->getNormStrC(), pSentence->pStrings->getNormStrC(), 0) == 0L; @@ -1099,7 +1099,7 @@ USHORT ExactTest else { fStringEqual = FALSE; - auto genericTagsTmSeg = std::make_unique(pString); + auto genericTagsTmSeg = std::make_unique(pString.c_str()); fStringEqual = (UtlCompIgnWhiteSpaceW( pSentence->pStrings->getNpReplStrC(), genericTagsTmSeg->getNpReplStrC(), @@ -1159,10 +1159,10 @@ USHORT ExactTest { if ( (pGetIn->pvGMOptList != NULL) && pClb->usAddDataLen ) { - USHORT usAddDataLen = NtmGetAddData( pClb, ADDDATA_ADDINFO_ID, pContextBuffer, MAX_SEGMENT_SIZE ); + USHORT usAddDataLen = NtmGetAddData( pClb, ADDDATA_ADDINFO_ID, pContextBuffer ); if ( usAddDataLen ) { - GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pContextBuffer ); + GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pContextBuffer.data() ); switch ( GobMemOpt ) { case GM_SUBSTITUTE_OPT: usTranslationFlag = TRANSLFLAG_NORMAL; break; @@ -1184,10 +1184,10 @@ USHORT ExactTest && pGetIn->szContext[0] && pClb->usAddDataLen ) { - USHORT usContextLen = NtmGetAddData( pClb, ADDDATA_CONTEXT_ID, pContextBuffer, MAX_SEGMENT_SIZE ); + USHORT usContextLen = NtmGetAddData( pClb, ADDDATA_CONTEXT_ID, pContextBuffer ); if ( usContextLen != 0 ) { - usCurContextRanking = NTMCompareContext( pTmClb, pGetIn->szTagTable, pGetIn->szContext, pContextBuffer ); + usCurContextRanking = NTMCompareContext( pTmClb, pGetIn->szTagTable, pGetIn->szContext, pContextBuffer.data() ); } /* endif */ } /* endif */ @@ -1690,7 +1690,7 @@ BOOL TMFuzzynessEx // ----------------------------------------------------------------------------+ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct - PSZ_W pSourceString, //pointer to normalized source string + std::wstring& pSourceString, //pointer to normalized source string PLONG plSourceLen, //length of source string PTMX_TARGET_RECORD pTMXTargetRecord, //pointer to tm target PTMX_TARGET_CLB pTMXTargetClb, //ptr to target control block @@ -1715,7 +1715,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct USHORT usRc = NO_ERROR; //return code LONG lTargetLen; //length indicator LONG ulLen; //length indicator - PSZ_W pString = NULL; //pointer to temp string + std::wstring pString; //pointer to temp string USHORT usCurrentEntry; //counter BOOL fFound = FALSE; //indicates if correct pos found PTMX_SUBSTPROP pSubstProp = NULL; // tag substitution array @@ -1753,19 +1753,12 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct strcpy( pSubstProp->szSourceLanguage, pGetIn->szSourceLanguage); strcpy( pSubstProp->szTargetLanguage, pGetIn->szTargetLanguage ); UTF16strcpy( pSubstProp->szSource, pGetIn->szSource ); - //pSubstProp->pTagsSource = pTagRecord; pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usSourceTagTable; MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); - //pSubstProp->pTagsPropSource = (PTMX_TAGTABLE_RECORD)pByte; - //usRc = (AddTagsToStringW( pSourceString, - // plSourceLen, // in # of w's - // (PTMX_TAGTABLE_RECORD)pByte, - // pSubstProp->szPropSource )) ? usRc : BTREE_CORRUPTED; //instead of lines above we now save not normalize string to btree - wcsncpy(pSubstProp->szPropSource, pSourceString, *plSourceLen); + wcsncpy(pSubstProp->szPropSource, pSourceString.c_str(), *plSourceLen); if ( usRc == NO_ERROR ) { PSZ_W pTarget; @@ -1778,7 +1771,9 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; { - lTargetLen = EQFCompress2Unicode(pSubstProp->szPropTarget, pByte, lTargetLen); + std::wstring output; + lTargetLen = EQFCompress2Unicode(output, pByte, lTargetLen); + wcsncpy(pSubstProp->szPropTarget, output.c_str(), sizeof(pSubstProp->szPropTarget)/sizeof(pSubstProp->szPropTarget[0])); //position at target tag record pByte = (PBYTE)pTMXTargetRecord; } @@ -1907,9 +1902,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct if ( fFound ) { - //allocate pString - fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG)MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); - if ( !fOK ) { LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); @@ -1933,7 +1925,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct { // copy normalized propsource string if(*plSourceLen > 0 ){ - memcpy( pstMatchTable->szSource, pSourceString, *plSourceLen * sizeof(CHAR_W)); + memcpy( pstMatchTable->szSource, pSourceString.c_str(), *plSourceLen * sizeof(CHAR_W)); } } /* endif */ @@ -2007,7 +1999,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct } else { - memcpy( pstMatchTable->szTarget, pString, lTargetLen * sizeof(CHAR_W)); + memcpy( pstMatchTable->szTarget, pString.data(), lTargetLen * sizeof(CHAR_W)); } /* endif */ if ( usRc == NO_ERROR ) @@ -2064,8 +2056,8 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct pstMatchTable->szAddInfo[0] = 0; if ( pTMXTargetClb->usAddDataLen ) { - NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstMatchTable->szContext, sizeof(pstMatchTable->szContext) / sizeof(CHAR_W) ); - NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstMatchTable->szAddInfo, sizeof(pstMatchTable->szAddInfo) / sizeof(CHAR_W) ); + NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstMatchTable->szContext ); + NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstMatchTable->szAddInfo ); } /* endif */ /****************************************************************/ @@ -2534,7 +2526,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block PTMX_TAGTABLE_RECORD pTMXTargetTagTable = NULL; //ptr to target tag record PTMX_SOURCE_RECORD pTMXSourceRecord = NULL; //ptr to source record PTMX_TARGET_CLB pTMXTargetClb = NULL; //ptr to target control block - PSZ_W pString = NULL; //ptr to source string + std::wstring pString; //ptr to source string BOOL fOK = TRUE; //success indicator ULONG ulSourceLen = 0; //length of normalized source string USHORT usRc = NO_ERROR; //return code @@ -2551,9 +2543,6 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block } T5LOG( T5INFO) << "FuzzyTest for record " <usSourceTagTable; - //pTMXSourceTagTable = (PTMX_TAGTABLE_RECORD)pByte; if ( !fOK ) { LOG_AND_SET_RC(usRc, T5INFO, BTREE_CORRUPTED); @@ -2830,23 +2777,6 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //position at source tag table - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXSourceTagTable = (PTMX_TAGTABLE_RECORD)pByte; - - //compare tag table records - //fStringEqual = memcmp( pTMXSourceTagTable, pSentence->pTagRecord, - // RECLEN(pTMXSourceTagTable) ) == 0; - //if ( !fStringEqual ) - //{ - /**********************************************************/ - /* if tagging record is unequal than we have some */ - /* (slight) differences */ - /* i.e. we will create a fully qualified string and try */ - /* another compare... */ - /**********************************************************/ - //allocate pString - if ( fOK ) { //calculate length of source string @@ -2951,9 +2881,6 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block } /* endif */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); - if ( usRc ) { ERREVENT2( FUZZYTEST_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); @@ -3066,7 +2993,7 @@ USHORT FillMatchEntry ULONG ulLen = toIndexRecord(pIndexRecord)->usRecordLen; usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - pIndexEntry = &toIndexEntry(pIndexRecord)->stIndexEntry; + pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; pTempMatch = pTempStart; //end criteria are all sentence ids in index key or only one diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index 054c34a4..db74227f 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -1677,7 +1677,7 @@ SHORT BTREE::QDAMDictExactLocal // set new current position sCurrentIndex = sNearKey; usCurrentRecord = RECORDNUM( pRecord ); - *pulLength = 0; // init returned length + //*pulLength = 0; // init returned length } /* endif */ } /* endif */ } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index 283af2fb..dd9839ef 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -193,7 +193,7 @@ USHORT EqfMemory::LoadMem() BOOL fOK; //success indicator USHORT usRc = NO_ERROR; //return value USHORT usRc1 = NO_ERROR; //return value - ULONG ulLen; //length indicator + ULONG ulLen = 0; //length indicator //if(!usRc) { @@ -245,9 +245,8 @@ USHORT EqfMemory::LoadMem() (usRc == BTREE_CORRUPTED) || (usRc == VERSION_MISMATCH) ) { - ulLen = MAX_COMPACT_SIZE-1; //get compact area and add to control block - USHORT usTempRc = TmBtree.EQFNTMGet( COMPACT_KEY, (PCHAR)bCompact, &ulLen ); + USHORT usTempRc = TmBtree.EQFNTMGet( COMPACT_KEY, bCompact ); // in organize mode allow continue if compact area is corrupted if ( (usTempRc != NO_ERROR) && (usTempRc != VERSION_MISMATCH) ) @@ -257,7 +256,8 @@ USHORT EqfMemory::LoadMem() if ( usTempRc == BTREE_CORRUPTED ) { - memset( bCompact, 0, sizeof(bCompact) ); + bCompact.resize(MAX_COMPACT_SIZE); + memset( bCompact.data(), 0, bCompact.size() ); } /* endif */ if ( usTempRc != NO_ERROR ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index f4ffaf0e..ecdb0845 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -45,18 +45,16 @@ PBYTE NTRecPos(PBYTE pStart, int iType); USHORT NTMAdjustAddDataInTgtCLB ( - PTMX_RECORD *ppTmRecordStart, - PULONG pulRecBufSize, + std::vector& pTmRecordStart, OtmProposal& TmProposal, PTMX_TARGET_CLB *ppClb, - PTMX_RECORD *ppCurTmRecord, PTMX_TARGET_RECORD *ppTMXTargetRecord, PLONG pulLeftClbLen, PBOOL pfUpdate); USHORT TMLoopAndDelTargetClb ( - PTMX_RECORD pTmRecord, + std::vector& pTmRecord, OtmProposal& TmProposal, USHORT usPutLang, USHORT usPutFile, @@ -92,7 +90,7 @@ USHORT TMLoopAndDelTargetClb bool isAddDataIsTheSame(OtmProposal& TmProposal,TMX_TARGET_CLB& Clb) { - wchar_t szAddInfo[OTMPROPOSAL_MAXSEGLEN+1]; + std::wstring szBuff; USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); if(Clb.usAddDataLen != usAddDataLen) @@ -100,12 +98,12 @@ bool isAddDataIsTheSame(OtmProposal& TmProposal,TMX_TARGET_CLB& Clb) if (Clb.usAddDataLen) { - NtmGetAddData( &Clb, ADDDATA_CONTEXT_ID, szAddInfo, sizeof(szAddInfo) / sizeof(CHAR_W) ); - if(wcscmp(szAddInfo,TmProposal.szContext)) + NtmGetAddData( &Clb, ADDDATA_CONTEXT_ID, szBuff ); + if(wcscmp(szBuff.c_str(),TmProposal.szContext)) return false; - - NtmGetAddData( &Clb, ADDDATA_ADDINFO_ID, szAddInfo, sizeof(szAddInfo) / sizeof(CHAR_W) ); - if(wcscmp(szAddInfo,TmProposal.szAddInfo)) + szBuff.clear(); + NtmGetAddData( &Clb, ADDDATA_ADDINFO_ID, szBuff ); + if(wcscmp(szBuff.c_str(),TmProposal.szAddInfo)) return false; } return true; @@ -639,7 +637,7 @@ USHORT CheckCompactArea BYTE bTuple = (BYTE) ulVote; // active byte BYTE bRest = bTuple & 0x7; // relevant bit of byte - PBYTE pByte = ((PBYTE)pTmClb->bCompact) + (ulVote >> 3); // byte pointer + PBYTE pByte = pTmClb->bCompact.data() + (ulVote >> 3); // byte pointer if ( *pByte & (1 << bRest) ) { usMatch++; @@ -1071,16 +1069,15 @@ USHORT EqfMemory::UpdateTmIndex PULONG pulVotes = NULL; // pointer to votes USHORT i; // index in for loop ULONG ulKey; // index key - std::vector pIndexRecord; // pointer to index structure - BOOL fOK = FALSE; // success indicator + std::vector pIndexRecord; // pointer to index structure + BOOL fOK = TRUE; // success indicator PBYTE pIndex; // position pointer //for all votes add the index to the corresponding list pulVotes = pSentence->pulVotes.data(); //allocate 32K for tm index record - pIndexRecord.resize(1 + TMX_REC_SIZE/sizeof(TMX_INDEX_RECORD)); - fOK = UtlAlloc( (PVOID *) &(pIndexRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); + pIndexRecord.resize(TMX_REC_SIZE); if ( !fOK ) { @@ -1093,22 +1090,20 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { ulKey = (*pulVotes) & START_KEY; - ulLen = TMX_REC_SIZE; memset( pIndexRecord.data(), 0, TMX_REC_SIZE ); usRc = InBtree.EQFNTMGet( ulKey, //index key - (PCHAR)pIndexRecord.data(), //pointer to index record - &ulLen ); //length + pIndexRecord); //pointer to index record if ( usRc == BTREE_NOT_FOUND ) { //key is not in index file; add a new index entry - pIndexRecord.data()->usRecordLen = sizeof( TMX_INDEX_RECORD ); + toIndexRecord(pIndexRecord)->usRecordLen = sizeof( TMX_INDEX_RECORD ); - pIndexRecord.data()->stIndexEntry = NTMINDEX(pSentence->usActVote,ulSidKey); + toIndexRecord(pIndexRecord)->stIndexEntry = NTMINDEX(pSentence->usActVote,ulSidKey); usRc = InBtree.EQFNTMInsert( &ulKey, - (PBYTE)pIndexRecord.data(), //pointer to index - pIndexRecord.data()->usRecordLen ); //length + pIndexRecord.data(), //pointer to index + toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) @@ -1117,7 +1112,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMInsert(&ulKey, (PBYTE)pIndexRecord.data(), pIndexRecord.data()->usRecordLen ); + usRc = InBtree.EQFNTMInsert(&ulKey, pIndexRecord.data(), toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ @@ -1125,7 +1120,7 @@ USHORT EqfMemory::UpdateTmIndex { //add the match(tuple) to compact area ulKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); - *((PBYTE)bCompact + (ulKey >> 3)) |= + *(bCompact.data() + (ulKey >> 3)) |= 1 << ((BYTE)ulKey & 0x07); // 1 << (USHORT)(ulKey & 0x07); @01M bCompactChanged = TRUE; @@ -1138,7 +1133,7 @@ USHORT EqfMemory::UpdateTmIndex BOOL fFound = FALSE; //key is in index file; update index entry with new sid - ulLen = pIndexRecord.data()->usRecordLen; + ULONG ulLen = toIndexRecord(pIndexRecord)->usRecordLen; //calculate number of entries in index record usIndexEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); @@ -1146,7 +1141,7 @@ USHORT EqfMemory::UpdateTmIndex //// check if SID is already contained in list.. //{ // int i = (int)usIndexEntries; - // PULONG pulIndex = (PULONG)&(pIndexRecord.data()->stIndexEntry); + // PULONG pulIndex = (PULONG)&(toIndexRecord(pIndexRecord)->stIndexEntry); // while ( i ) // { // if ( *pulIndex == ulNewSID ) @@ -1164,7 +1159,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usIndexEntries >= (MAX_INDEX_LEN -1)) { //position pointer at beginning of index record - pIndex = (PBYTE)pIndexRecord.data(); + pIndex = pIndexRecord.data(); memmove( pIndex, pIndex + sizeof(ULONG), ulLen - sizeof(ULONG) ); ulLen -= sizeof(ULONG); usIndexEntries--; @@ -1174,7 +1169,7 @@ USHORT EqfMemory::UpdateTmIndex && ((ulLen + sizeof( TMX_INDEX_ENTRY )) <= TMX_REC_SIZE) ) { //position pointer at beginning of index record - pIndex = (PBYTE)pIndexRecord.data(); + pIndex = pIndexRecord.data(); //move pointer to end of index record pIndex += ulLen; @@ -1184,12 +1179,12 @@ USHORT EqfMemory::UpdateTmIndex NTMINDEX(pSentence->usActVote,ulSidKey); //update index record size - pIndexRecord.data()->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); + toIndexRecord(pIndexRecord)->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); usRc = InBtree.EQFNTMUpdate( ulKey, - (PBYTE)pIndexRecord.data(), //pointer to index - pIndexRecord.data()->usRecordLen ); //length + pIndexRecord.data(), //pointer to index + toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) { @@ -1197,14 +1192,14 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMUpdate(ulKey, (PBYTE)pIndexRecord.data(), pIndexRecord.data()->usRecordLen ); + usRc = InBtree.EQFNTMUpdate(ulKey, pIndexRecord.data(), toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ if ( !usRc ) { //add the match(tuple) to compact area ulKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); - *((PBYTE)bCompact + (ulKey >> 3)) |= + *((PBYTE)bCompact.data() + (ulKey >> 3)) |= 1 << ((BYTE)ulKey & 0x07); bCompactChanged = TRUE; } /* endif */ @@ -1707,7 +1702,7 @@ DECLARE_bool(log_memmove_in_compareputdata); //------------------------------------------------------------------------------ USHORT EqfMemory::ComparePutData ( - std::vector& ppTmRecord, // ptr to ptr of tm record data buffer + std::vector& pTmRecord, // ptr to ptr of tm record data buffer OtmProposal& TmProposal, // pointer to get in data PULONG pulKey // tm key ) @@ -1722,17 +1717,13 @@ USHORT EqfMemory::ComparePutData LONG lLen = 0; //length indicator USHORT usNormLen = 0; //length of normalized string //PSZ_W pString = NULL; //pointer to character string - std::vector pString(MAX_SEGMENT_SIZE); + std::wstring pString; USHORT usRc = NO_ERROR; //returned value from function BOOL fUpdate = FALSE; // TRUE = record has been updated - PTMX_RECORD pTmRecord = *ppTmRecord; // pointer to tm record data USHORT usAuthorId; // ID for author string LONG lLeftClbLen; int delTargetKey = 0, targetKey = 0; - //allocate pString - //fOK = UtlAlloc( (PVOID *) &(pString), 0L, (LONG) MAX_SEGMENT_SIZE*sizeof(CHAR_W), NOMSG ); - if ( !fOK ) { LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); @@ -1758,13 +1749,13 @@ USHORT EqfMemory::ComparePutData if ( !usRc ) { //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - pByte = NTRecPos((PBYTE)(pTmRecord+1), REC_SRCSTRING); + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(toTmxRecord(pTmRecord)+1); + pByte = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_SRCSTRING); //calculate length of source string lLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); //copy and compare source string - memset( pString.data(), 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); + //memset( pString.data(), 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); lLen = EQFCompress2Unicode( pString, pByte, lLen ); fStringEqual = ! UTF16strcmp( pString.data(), TmProposal.pInputSentence->pStrings->getGenericTagStrC() ); @@ -1773,12 +1764,12 @@ USHORT EqfMemory::ComparePutData BOOL fNewerTargetExists = FALSE; LONG lLeftTgtLen; // remaining target length - NTASSERTLEN(RECLEN(pTmRecord), pTmRecord->usFirstTargetRecord, 4712) + NTASSERTLEN(RECLENB(pTmRecord), toTmxRecord(pTmRecord)->usFirstTargetRecord, 4712) // get length of target block to work with //source strings equal - position at first target record - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - pStartTarget = NTRecPos((PBYTE)(pTmRecord+1), REC_FIRSTTARGET); + lLeftTgtLen = RECLENB(pTmRecord) - toTmxRecord(pTmRecord)->usFirstTargetRecord; + pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); pTMXTargetRecord = (PTMX_TARGET_RECORD)pStartTarget; fStop = (usRc != 0); @@ -1788,8 +1779,8 @@ USHORT EqfMemory::ComparePutData TMLoopAndDelTargetClb(pTmRecord, TmProposal, usPutLang, usPutFile, usAuthorId, &fNewerTargetExists, &delTargetKey ); // recalc since record may have changed during delete above! - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - pStartTarget = NTRecPos((PBYTE)(pTmRecord+1), REC_FIRSTTARGET); + lLeftTgtLen = RECLENB(pTmRecord) - toTmxRecord(pTmRecord)->usFirstTargetRecord; + pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); pTMXTargetRecord = (PTMX_TARGET_RECORD)pStartTarget; if ( fNewerTargetExists && !FLAGS_ignore_newer_target_exists_check) @@ -1851,7 +1842,7 @@ USHORT EqfMemory::ComparePutData fStop = TRUE; // adjust context part of target control block if necessary - NTMAdjustAddDataInTgtCLB( ppTmRecord, pulRecBufSize, TmProposal, &pClb, &pTmRecord, &pTMXTargetRecord, &lLeftClbLen, &fUpdate ); + NTMAdjustAddDataInTgtCLB( pTmRecord, TmProposal, &pClb, &pTMXTargetRecord, &lLeftClbLen, &fUpdate ); } /* endif */ // continue with next target CLB @@ -1868,21 +1859,14 @@ USHORT EqfMemory::ComparePutData targetKey++; // re-alloc record buffer if too small USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - ULONG ulNewSize = RECLEN(pTmRecord) + sizeof(TMX_TARGET_CLB) + usAddDataLen; - if ( ulNewSize > *pulRecBufSize) + ULONG ulNewSize = RECLENB((pTmRecord)) + sizeof(TMX_TARGET_CLB) + usAddDataLen; + if ( ulNewSize > pTmRecord.size()) { - fOK = UtlAlloc( (PVOID *)ppTmRecord, *pulRecBufSize, ulNewSize, NOMSG ); - if ( fOK ) - { - *pulRecBufSize = ulNewSize; - pClb = (PTMX_TARGET_CLB) ADJUSTPTR( *ppTmRecord, pTmRecord, pClb ); - pTMXTargetRecord = (PTMX_TARGET_RECORD) ADJUSTPTR( *ppTmRecord, pTmRecord, pTMXTargetRecord ); - pTmRecord = *ppTmRecord; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + size_t pClbOffs = (PBYTE)pClb - pTmRecord.data(); + size_t pTMXTargetRecordOffs = (PBYTE)pTMXTargetRecord - pTmRecord.data(); + pTmRecord.resize(ulNewSize); + pClb = (PTMX_TARGET_CLB) pTmRecord.data() + pClbOffs; + pTMXTargetRecord = (PTMX_TARGET_RECORD) pTmRecord.data() + pTMXTargetRecordOffs; } /* endif */ // make room at pCLB for a new CLB and adjust TM record @@ -1896,7 +1880,7 @@ USHORT EqfMemory::ComparePutData //try{ if(size>=0) { - size_t occupiedSize = RECLEN(pTmRecord) + sizeof( TMX_SOURCE_RECORD ) + sizeof(TMX_TARGET_RECORD); + size_t occupiedSize = RECLEN(toTmxRecord(pTmRecord)) + sizeof( TMX_SOURCE_RECORD ) + sizeof(TMX_TARGET_RECORD); if(lNewClbLen + occupiedSize >= TMX_REC_SIZE){ usRc = BTREE_NODE_IS_FULL; fStop = true; @@ -1905,11 +1889,11 @@ USHORT EqfMemory::ComparePutData if(FLAGS_log_memmove_in_compareputdata) { T5LOG(T5ERROR) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " - << RECLEN(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize + << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize << "; ulKey = " << *pulKey ; } memmove( (((PBYTE)pClb) + lNewClbLen), pClb, size); - RECLEN(pTmRecord) += lNewClbLen; + RECLENB(pTmRecord) += lNewClbLen; RECLEN(pTMXTargetRecord) += lNewClbLen; } } @@ -1965,7 +1949,7 @@ USHORT EqfMemory::ComparePutData // update TM record if required if ( fUpdate ) { - usRc = TmBtree.EQFNTMUpdate( *pulKey, (PBYTE)pTmRecord, RECLEN(pTmRecord) ); + usRc = TmBtree.EQFNTMUpdate( *pulKey, pTmRecord.data(), RECLENB(pTmRecord) ); } /* endif */ if ( fStop ) @@ -1976,14 +1960,11 @@ USHORT EqfMemory::ComparePutData }else{ //all target records have been checked but nothing overlapped //so add new target record to end of tm record - usRc = AddTmTarget( TmProposal, ppTmRecord, pulRecBufSize, pulKey ); - pTmRecord = *ppTmRecord; + usRc = AddTmTarget( TmProposal, pTmRecord, pulKey ); if(!usRc) { TmProposal.currentInternalKey.setInternalKey(*pulKey, 1); - //TmProposal.recordKey = *pulKey; - //TmProposal.targetKey = 1; } } /* endif */ } @@ -1996,10 +1977,6 @@ USHORT EqfMemory::ComparePutData } /* endif */ } /* endif */ - //release memory - //UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); - //UtlAlloc( (PVOID *) &pTagRecord, 0L, 0L, NOMSG ); - if ( usRc ) { ERREVENT2( COMPAREPUTDATA_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); @@ -2043,8 +2020,7 @@ USHORT EqfMemory::ComparePutData //------------------------------------------------------------------------------ USHORT EqfMemory::AddTmTarget( OtmProposal& TmProposal, //pointer to get in data - PTMX_RECORD *ppTmRecord, //pointer to tm record data pointer - PULONG pulRecBufSize, //ptr to current size of TM record buffer + std::vector& pTmRecord, //pointer to tm record data pointer PULONG pulKey ) //tm key { PTMX_TARGET_CLB pTargetClb = nullptr ; // ptr to target ctl block @@ -2053,7 +2029,6 @@ USHORT EqfMemory::AddTmTarget( USHORT usRc = NO_ERROR; // return code BOOL fOK; // success indicator PBYTE pByte; // position pointer - PTMX_RECORD pTmRecord = *ppTmRecord; //pointer to tm record data ULONG ulAddDataLen = 0; //allocate target control block record @@ -2080,50 +2055,36 @@ USHORT EqfMemory::AddTmTarget( //check space requirements { // re-alloc record buffer if too small - ULONG ulNewSize = RECLEN(pTmRecord) + sizeof(TMX_TARGET_RECORD) + + ULONG ulNewSize = RECLENB(pTmRecord) + sizeof(TMX_TARGET_RECORD) + RECLEN(pTargetRecord); - if ( ulNewSize > *pulRecBufSize) + if ( ulNewSize > pTmRecord.size()) { - fOK = UtlAlloc( (PVOID *)ppTmRecord, *pulRecBufSize, ulNewSize, NOMSG ); - if ( fOK ) - { - *pulRecBufSize = ulNewSize; - pTmRecord = *ppTmRecord; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + pTmRecord.resize(ulNewSize); } /* endif */ } if ( fOK ) { //postion at end on tm record - pByte = (PBYTE)pTmRecord; - pByte += RECLEN(pTmRecord); + pByte = pTmRecord.data(); + pByte += RECLENB(pTmRecord); //add new target record to end memcpy( pByte, pTargetRecord, RECLEN(pTargetRecord) ); //update overall length of tm record - RECLEN(pTmRecord) += RECLEN(pTargetRecord); + RECLENB(pTmRecord) += RECLEN(pTargetRecord); //add updated tm record to database usRc = TmBtree.EQFNTMUpdate( *pulKey, - (PBYTE)pTmRecord, - RECLEN(pTmRecord) ); + pTmRecord.data(), + RECLENB(pTmRecord) ); } /* endif */ } /* endif */ } /* endif */ } /* endif */ - if ( usRc ) - { - ERREVENT2( ADDTMTARGET_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ - return( usRc ); } @@ -2252,200 +2213,6 @@ VOID FillTargetRecord // USHORT usUpdTarget number of target being updated // USHORT usFlags flags controlling the update //------------------------------------------------------------------------------ -USHORT EqfMemory::TmtXUpdSeg -( - OtmProposal* pTmPutIn, // ptr to put input data - PTMX_EXT_OUT_W pTmPutOut, //ptr to output struct - USHORT usFlags // flags controlling the updated fields -) -{ - BOOL fOK; // success indicator - USHORT usRc = NO_ERROR; // return code - BOOL fLocked = FALSE; // TM-database-has-been-locked flag - ULONG ulLen = 0; //length indicator - PBYTE pByte; //position ptr - USHORT usTarget; //nr of target records in tm record - PTMX_TARGET_RECORD pTMXTargetRecord; // ptr to target record - PTMX_TARGET_CLB pTargetClb; // ptr to target CLB - ULONG ulLeftClbLen; // remaining length of CLB area - - std::vector TmRecord; // space for user data - TmRecord.resize(TMX_REC_SIZE); - - // get TM record being modified and update the record - if ( !usRc ) - { - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - - ulLen = TMX_REC_SIZE; - usRc = TmBtree.EQFNTMGet( pTmPutIn->currentInternalKey.getRecordKey(), TmRecord ); - PTMX_RECORD pTmRecord = (PTMX_RECORD) TmRecord.data(); - - if ( usRc == NO_ERROR ) - { - //check if record is empty - if ( ( RECLEN(pTmRecord) != 0) && - (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) - { - ULONG ulLeftTgtLen; // remaining target length - ulLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - - if ( !usRc ) - { - //find target record specified in usUpdTarget - //move pointer to first target - pByte = (PBYTE)(pTmRecord); - pByte += pTmRecord->usFirstTargetRecord; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - ulLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - ulLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB - usTarget = 1; //initialize counter - - //loop until correct target is found - while ( (usTarget < pTmPutIn->currentInternalKey.getTargetKey()) //pTmPutIn->targetKey) - && ulLeftTgtLen ) - { - // position to first target CLB - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - ulLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - ulLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB - - // loop over all target CLBs - while ( (usTarget < pTmPutIn->currentInternalKey.getTargetKey()) && ulLeftClbLen ) - { - usTarget++; - pTargetClb = NEXTTARGETCLB(pTargetClb); - ulLeftClbLen -= TARGETCLBLEN(pTargetClb); - } /* endwhile */ - - // continue with next target if no match yet - if ( usTarget < pTmPutIn->currentInternalKey.getTargetKey() ) - { - // position at next target - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target - if (ulLeftTgtLen >= RECLEN(pTMXTargetRecord)) - { - ulLeftTgtLen -= RECLEN(pTMXTargetRecord); - } - else - { - ulLeftTgtLen = 0; - } /* endif */ - pByte += RECLEN(pTMXTargetRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - usTarget++; - } /* endif */ - } /* endwhile */ - - if ( usTarget == pTmPutIn->currentInternalKey.getTargetKey())//pTmPutIn->targetKey ) - { - //position at start of target record - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //if target record exists - if ( ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0) ) - { - // update requested fields - - // change markup/tag table if requested - if ( //(usFlags & TMUPDSEG_MARKUP) && - (pTmPutIn->szMarkup[0] != EOS) ) - { - PTMX_TAGTABLE_RECORD pTMXTagTableRecord; - USHORT usNewId; - PBYTE pByte; - - // get ID for new tag table - //usRc = NTMGetIDFromName( pTmPutIn->szMarkup, - // NULL, - // (USHORT)TAGTABLE_KEY, - // &usNewId ); - usNewId = 1; - - // update source tag table record - //pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXTagTableRecord = (PTMX_TAGTABLE_RECORD)pByte; - //pTMXTagTableRecord->usTagTableId = 1;//usNewId; - - // update target tag table record - //pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usTargetTagTable; - //pTMXTagTableRecord = (PTMX_TAGTABLE_RECORD)pByte; - //pTMXTagTableRecord->usTagTableId = 1;//usNewId; - } /* endif */ - - // update MT flag if requested - if ( (usRc == NO_ERROR) - // && (usFlags & TMUPDSEG_MTFLAG) - ) - { - // set type of translation flag - pTargetClb->bTranslationFlag = (BYTE)pTmPutIn->eType; - } /* endif */ - - // update target language if requested - if ( (usRc == NO_ERROR) - // && (usFlags & TMUPDSEG_TARGLANG) - ) - { - // set target language - usRc = NTMGetIDFromName( pTmPutIn->szTargetLanguage, - NULL, - (USHORT)LANG_KEY, - &pTargetClb->usLangId ); - } /* endif */ - - // update segment date if requested - if ( (usRc == NO_ERROR) - // && (usFlags & TMUPDSEG_DATE) - ) - { - pTargetClb->lTime = pTmPutIn->lTargetTime; - } /* endif */ - - - // rewrite TM record - if ( usRc == NO_ERROR ) - { - usRc = TmBtree.EQFNTMUpdate( pTmPutIn->currentInternalKey.getRecordKey(), - pTmRecord, RECLEN(pTmRecord) ); - } /* endif */ - - if(usRc == NO_ERROR) - { - usRc = RewriteCompactTable(); - } - } - else - { - // target not found - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - } - else - { - // record not found - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - } /* endif */ - } - else - { - // record is empty and should not be updated - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - } /* endif */ - } /* endif */ - - return( usRc ); -} /* end of function TmtXUpdSeg */ /**********************************************************************/ @@ -2591,7 +2358,7 @@ ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG ulLenChar ) } -LONG EQFCompress2Unicode( std::string& pOutput, PBYTE pTarget, ULONG ulLenComp ) +LONG EQFCompress2Unicode( std::wstring& pOutput, PBYTE pTarget, ULONG ulLenComp ) { LONG lLen = ulLenComp-1; pOutput.clear(); @@ -2659,11 +2426,9 @@ LONG EQFCompress2Unicode( std::string& pOutput, PBYTE pTarget, ULONG ulLenComp ) USHORT NTMAdjustAddDataInTgtCLB ( - PTMX_RECORD *ppTmRecordStart, - PULONG pulRecBufSize, + std::vector& pTmRecord, OtmProposal& TmProposal, PTMX_TARGET_CLB *ppClb, - PTMX_RECORD *ppCurTmRecord, PTMX_TARGET_RECORD *ppTMXTargetRecord, PLONG pulLeftClbLen, PBOOL pfUpdate) @@ -2678,7 +2443,7 @@ USHORT NTMAdjustAddDataInTgtCLB PTMX_TARGET_CLB pNextClb = NULL; pTMXTargetRecord = *ppTMXTargetRecord; - pCurTmRecord = *ppCurTmRecord; + pCurTmRecord = toTmxRecord(pTmRecord); pCurClb = *ppClb; usOldLen = pCurClb->usAddDataLen; @@ -2709,23 +2474,16 @@ USHORT NTMAdjustAddDataInTgtCLB // re-alloc record buffer if too small ULONG ulNewSize = RECLEN(pCurTmRecord) + (ULONG)ulDiff; - if ( ulNewSize > *pulRecBufSize) + if ( ulNewSize > pTmRecord.size()) { - fOK = UtlAlloc( (PVOID *)ppTmRecordStart, *pulRecBufSize, - ulNewSize, NOMSG ); - if ( fOK ) - { - *pulRecBufSize = ulNewSize; - pCurClb = (PTMX_TARGET_CLB) ADJUSTPTR( *ppTmRecordStart, pCurTmRecord, pCurClb ); - // *ppTMRecordStart + (pClb-ppCurTmRecord) - pTMXTargetRecord = (PTMX_TARGET_RECORD) ADJUSTPTR( *ppTmRecordStart, pCurTmRecord, pTMXTargetRecord ); - pCurTmRecord = *ppTmRecordStart; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + size_t pCurClbOffs = (PBYTE)pCurClb - pTmRecord.data(); + size_t pTMXTargetRecordOffs = (PBYTE) pTMXTargetRecord - pTmRecord.data(); + pTmRecord.resize(ulNewSize); + + pCurTmRecord = toTmxRecord(pTmRecord); + pCurClb = (PTMX_TARGET_CLB) pTmRecord.data() + pCurClbOffs; + pTMXTargetRecord = (PTMX_TARGET_RECORD) pTmRecord.data() + pTMXTargetRecordOffs; + } // enlarge current control block and adjust TM record // length and target record length @@ -2758,7 +2516,7 @@ USHORT NTMAdjustAddDataInTgtCLB } /* endif */ *ppTMXTargetRecord = pTMXTargetRecord; - *ppCurTmRecord = pCurTmRecord; + //*ppCurTmRecord = pCurTmRecord; *ppClb = pCurClb; return (usRc); } // end of context handling @@ -2808,7 +2566,7 @@ USHORT NTMAdjustAddDataInTgtCLB USHORT TMLoopAndDelTargetClb ( - PTMX_RECORD pTmRecord, + std::vector& pTmRecord, OtmProposal& TmProposal, USHORT usPutLang, USHORT usPutFile, @@ -2830,10 +2588,10 @@ USHORT TMLoopAndDelTargetClb *pfNewerTargetExists = FALSE; // loop until either end of record or one occ. found&deleted - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + lLeftTgtLen = RECLENB(pTmRecord) - toTmxRecord(pTmRecord)->usFirstTargetRecord; //source strings equal, position at first target record - pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)(pTmRecord+1), REC_FIRSTTARGET); + pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); if(pTargetKey) *pTargetKey = 0; while ( ( lLeftTgtLen <= RECLEN(pTMXTgtRec) ) && (RECLEN(pTMXTgtRec) != 0) && !fDel ) @@ -2860,7 +2618,7 @@ USHORT TMLoopAndDelTargetClb lLeftClbLen -= TARGETCLBLEN(pClb); // loop over all CLBs of this target record and remove // any CLB for the current segment - fTgtRemoved = TMDelTargetClb( pTmRecord, pTMXTgtRec, pClb); + fTgtRemoved = TMDelTargetClb( toTmxRecord(pTmRecord), pTMXTgtRec, pClb); fDel = TRUE; } else diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index f61e8952..90e1e44a 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -53,7 +53,16 @@ #include "LogWrapper.h" #include "OSWrapper.h" -//#include "EQFCMPR.H" +#include // for std::transform +//#include // for std::tolower + +void toLower(std::vector& vec) { + std::transform(vec.begin(), vec.end(), vec.begin(), [](unsigned char c) { + return std::tolower(c); + }); +} + +// #include "EQFCMPR.H" /**********************************************************************/ /* prototypes for internal functions */ /**********************************************************************/ @@ -169,14 +178,13 @@ int NTMLongNameTableCompCaseIgnore( const void *, const void * ); // ----------------------------------------------------------------------------+ USHORT EqfMemory::NTMGetIDFromName( - PSZ pszName, // input - PSZ pszLongName, // input, long name (only for FILE_KEY) - USHORT usTableType, //input - PUSHORT pusID ) //output + PSZ pszName, // input + PSZ pszLongName, // input, long name (only for FILE_KEY) + USHORT usTableType, // input + PUSHORT pusID) // output { - return( NTMGetIDFromNameEx( pszName, pszLongName, usTableType, - pusID, 0, NULL ) ); - + return (NTMGetIDFromNameEx(pszName, pszLongName, usTableType, + pusID, 0, NULL)); } /* end of function NTMGetIDFromName */ // NTMGetIDFromNameEx @@ -185,350 +193,273 @@ EqfMemory::NTMGetIDFromName( // suppressed for new names (important for get operations which should // not change the database) // the pusAlternativeID is filled with the ID of the name in the long name -// table if no pszLongName has been specified and the pszName is found in +// table if no pszLongName has been specified and the pszName is found in // the short name table as well -USHORT EqfMemory::NTMGetIDFromNameEx -( - PSZ pszName, // input, name being looked up - PSZ pszLongName, // input, long name (only for FILE_KEY) - USHORT usTableType, // input, type of table to use - PUSHORT pusID, // output, ID for name being looked up - LONG lOptions, - PUSHORT pusAlternativeID // output, alternative ID +USHORT EqfMemory::NTMGetIDFromNameEx( + PSZ pszName, // input, name being looked up + PSZ pszLongName, // input, long name (only for FILE_KEY) + USHORT usTableType, // input, type of table to use + PUSHORT pusID, // output, ID for name being looked up + LONG lOptions, + PUSHORT pusAlternativeID // output, alternative ID ) { - USHORT usRc = NO_ERROR; //function return coed - PTMX_TABLE_ENTRY pstTMTableEntry = NULL; //ptr to a table entry for bsearch - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; //ptr to first table entry - PTMX_TABLE pstTMTable = NULL; //ptr to table structure - BOOL fLongName = FALSE; + USHORT usRc = NO_ERROR; // function return coed + PTMX_TABLE_ENTRY pstTMTableEntry = NULL; // ptr to a table entry for bsearch + PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry + PTMX_TABLE pstTMTable = NULL; // ptr to table structure + BOOL fLongName = FALSE; // initialize ID - + // initialize ID - if ( lOptions & NTMGETID_NOUPDATE_OPT ) + if (lOptions & NTMGETID_NOUPDATE_OPT) { - *pusID = NTMGETID_NOTFOUND_ID; // use as not-found indicator + *pusID = NTMGETID_NOTFOUND_ID; // use as not-found indicator } else { *pusID = 0; } /* endif */ - if ( pusAlternativeID ) *pusAlternativeID = NTMGETID_NOTFOUND_ID; + if (pusAlternativeID) + *pusAlternativeID = NTMGETID_NOTFOUND_ID; *pusID = 0; - if ( pusAlternativeID ) *pusAlternativeID = NTMGETID_NOTFOUND_ID; + if (pusAlternativeID) + *pusAlternativeID = NTMGETID_NOTFOUND_ID; - if ( pszName[0] != EOS) + if (pszName[0] != EOS) { //--- if input parameters OK - if ( pszName[0] != EOS ) + if (pszName[0] != EOS) { //--- capitalize input string - if(usTableType != FILE_KEY) + if (usTableType != FILE_KEY) { - strupr( pszName ); + strupr(pszName); } /******************************************************************/ /* get pointer to table and table entries in dependency of the */ /* table type */ /******************************************************************/ - usRc = NTMGetPointersToTable( usTableType, - &pstTMTable, - &pstTMTableEntries ); + usRc = NTMGetPointersToTable(usTableType, + &pstTMTable, + &pstTMTableEntries); /****************************************************************/ /* If we are looking for documents and a long document name is */ /* available ... */ /****************************************************************/ // skip any leading backslash (added by bug in ITM code ...) - if ( pszLongName != NULL ) + if (pszLongName != NULL) { - if ( *pszLongName == BACKSLASH ) pszLongName++; + if (*pszLongName == BACKSLASH) + pszLongName++; } /* endif */ fLongName = (usTableType == FILE_KEY) && - (pszLongName != NULL) && - (pszLongName[0] != EOS) - //&& (strcmp( pszLongName, pszName ) != 0) - ; + (pszLongName != NULL) && + (pszLongName[0] != EOS) + //&& (strcmp( pszLongName, pszName ) != 0) + ; - if ( (usRc == NO_ERROR) && fLongName ) + if ((usRc == NO_ERROR) && fLongName) { /**************************************************************/ /* ... check first against our long name table */ /**************************************************************/ - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to entry found - TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched + PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to entry found + TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched // prepare search entry SearchEntry.pszLongName = pszLongName; // do the actual search - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch( &SearchEntry, - pLongNames->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp ); - + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + LongNames.stTableEntry.data(), + LongNames.stTableEntry.size(), + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableComp); // if search fails try again using case insenstive search - if ( pEntry == NULL ) + if (pEntry == NULL) { - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch( &SearchEntry, - pLongNamesCaseIgnore->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore ); + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + LongNamesCaseIgnore.stTableEntry.data(), + LongNames.stTableEntry.size(), + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); } /* endif */ - if ( pEntry != NULL ) + if (pEntry != NULL) { // return ID of found entry - *pusID = pEntry->usId; + *pusID = pEntry->usId; } - else if ( !(lOptions & NTMGETID_NOUPDATE_OPT) ) + else if (!(lOptions & NTMGETID_NOUPDATE_OPT)) { - ULONG ulNameLen = strlen(pszLongName) + 1; - ULONG ulAddLen = ulNameLen + sizeof(USHORT); - + ULONG ulNameLen = strlen(pszLongName) + 1; + ULONG ulAddLen = ulNameLen + sizeof(USHORT); // add a new entry for the given short name - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { // for some (yet unknown) reasons, there are memories which have fewer entries in // the short name table when in the long name table, using the ID generated when // adding the short name will lead to incorrect IDs for the long name table. // In order to circumvent this we call NTMAddNameToTable for the short name table - // until the returned ID is larger than or equal to pTmClb->pLongNames->ulEntries + // until the returned ID is larger than or equal to pTmClb->LongNames->ulEntries do { - DEBUGEVENT2( NTMGETIDFROMNAME_LOC, INFO_EVENT, 1, TM_GROUP, pszName ); - usRc = NTMAddNameToTable( pszName, usTableType, pusID ); - } while ( (usRc == NO_ERROR) && (*pusID < pLongNames->ulEntries) ); /* enddo */ + DEBUGEVENT2(NTMGETIDFROMNAME_LOC, INFO_EVENT, 1, TM_GROUP, pszName); + usRc = NTMAddNameToTable(pszName, usTableType, pusID); + } while ((usRc == NO_ERROR) && (*pusID < LongNames.stTableEntry.size())); /* enddo */ } /* endif */ // add a new entry to the long name table - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - // enlarge array if necessary - if ( pLongNames->ulEntries >= pLongNames->ulTableSize) - { - ULONG ulOldSize = (ULONG)sizeof(TMX_LONGNAMETABLE) + - (ULONG)(sizeof(TMX_LONGNAME_TABLE_ENTRY) * - pLongNames->ulTableSize); - ULONG ulNewSize = ulOldSize + (ULONG) - (sizeof(TMX_LONGNAME_TABLE_ENTRY) * LONGNAMETABLE_ENTRIES); - - if ( UtlAlloc( (PVOID *)&pLongNames, - ulOldSize, ulNewSize, NOMSG ) ) - { - pLongNames->ulTableSize += LONGNAMETABLE_ENTRIES; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - // enlarge case ignore table as well - if ( usRc == NO_ERROR ) - { - if ( !UtlAlloc( (PVOID *)&pLongNamesCaseIgnore, - ulOldSize, ulNewSize, NOMSG ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - } /* endif */ - - // enlarge buffer if necessary - if ( usRc == NO_ERROR ) - { - if ( (pLongNames->ulBufUsed + ulAddLen ) >= - pLongNames->ulBufSize) - { - PSZ pszOldBuffer; // ptr to old buffer area - - ULONG ulNewLen = pLongNames->ulBufUsed + - ulAddLen + 256; - pszOldBuffer = pLongNames->pszBuffer; - if ( UtlAlloc( (PVOID *)&(pLongNames->pszBuffer), - pLongNames->ulBufSize, - ulNewLen, NOMSG ) ) - { - // remember new buffer size - pLongNames->ulBufSize = ulNewLen; - - // adjust pointers in our table - { - ULONG ulI; - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to table entry - - pEntry = pLongNames->stTableEntry; - - for ( ulI = 0; ulI < pLongNames->ulEntries; ulI++ ) - { - pEntry->pszLongName = pLongNames->pszBuffer + - (pEntry->pszLongName - pszOldBuffer); - pEntry++; - } /* endfor */ - } - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - } /* endif */ - // add new entry to buffer - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to table entry - PSZ pszTarget; // ptr into buffer - - // position to table entry - pEntry = pLongNames->stTableEntry + - pLongNames->ulEntries; - - // position to free area in buffer (overwrite end delimiter!) - pszTarget = pLongNames->pszBuffer + - (pLongNames->ulBufUsed - sizeof(USHORT)); + TMX_LONGNAME_TABLE_ENTRY Entry; // ptr to table entry - // add ID to buffer and table - *((PUSHORT)pszTarget) = *pusID; - pszTarget += sizeof(USHORT); - pEntry->usId = *pusID; + LongNames.pszBuffer.push_back(*pusID); + Entry.longNameStartOffset = LongNames.stTableEntry.size(); + LongNames.pszBuffer.insert(LongNames.pszBuffer.end(), pszLongName, pszLongName + strlen(pszLongName)); + LongNames.pszBuffer.push_back(0);//as USHORT separator + LongNames.pszBuffer.push_back(0); + Entry.usId = *pusID; // add long name to buffer and table - strcpy( pszTarget, pszLongName ); - pEntry->pszLongName = pszTarget; - pszTarget += ulNameLen; - - // add new end delimiter - *((PUSHORT)pszTarget) = 0; // end delimiter + Entry.pszLongName = pszLongName; // adjust entry count and buffer used size - pLongNames->ulEntries++; - pLongNames->ulBufUsed += ulAddLen; + LongNames.stTableEntry.push_back(Entry); + LongNames.restoreStrPointersAfterRealloc(); } /* endif */ // sort long name array - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - qsort( pLongNames->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp ); + qsort(LongNames.stTableEntry.data(), + LongNames.stTableEntry.size(), + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableComp); } /* endif */ // copy to case ignore table and sort it - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - memcpy( pLongNamesCaseIgnore->stTableEntry, - pLongNames->stTableEntry, - pLongNames->ulEntries * - sizeof(TMX_LONGNAME_TABLE_ENTRY) ); - - qsort( pLongNamesCaseIgnore->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore ); + LongNamesCaseIgnore.stTableEntry = LongNames.stTableEntry; + LongNamesCaseIgnore.pszBuffer = LongNames.pszBuffer; + toLower(LongNamesCaseIgnore.pszBuffer); + LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); + + //memcpy(LongNamesCaseIgnore.stTableEntry.data(), + // LongNames.stTableEntry.data(), + // LongNames.stTableEntry.size() * sizeof(TMX_LONGNAME_TABLE_ENTRY)); + + qsort(LongNamesCaseIgnore.stTableEntry.data(), + LongNamesCaseIgnore.stTableEntry.size(), + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); } /* endif */ // update TM record for long names - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - DEBUGEVENT2( NTMGETIDFROMNAME_LOC, INFO_EVENT, 2, TM_GROUP, pszLongName ); + DEBUGEVENT2(NTMGETIDFROMNAME_LOC, INFO_EVENT, 2, TM_GROUP, pszLongName); - usRc = TmBtree.EQFNTMUpdate( LONGNAME_KEY, - (PBYTE)pLongNames->pszBuffer, - pLongNames->ulBufUsed ); + usRc = TmBtree.EQFNTMUpdate(LONGNAME_KEY, + (PBYTE)LongNames.pszBuffer.data(), + //LongNames.ulBufUsed); + LongNames.pszBuffer.size()); } /* endif */ } /* endif */ } /* endif */ } else { - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { /********************************************************************/ /* search name passed in pszName in passed table usTableType */ /********************************************************************/ - pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch( pszName, - pstTMTableEntries, - pstTMTable->ulMaxEntries, - sizeof(TMX_TABLE_ENTRY), - NTMCompNames ); - //--- if passed name found in table - if ( pstTMTableEntry ) - { - //-- get ID of entry and set output parameter - *pusID = pstTMTableEntry->usId; - } - else - { - // name is not contained in name table - if ( !(lOptions & NTMGETID_NOUPDATE_OPT) ) - { - usRc = NTMAddNameToTable( pszName, usTableType, - pusID ); - - // for new languages update our language group table - if ( (usRc == NO_ERROR) && - (usTableType == LANG_KEY) ) - { - usRc = NTMAddLangGroup( pszName, *pusID ); - } /* endif */ - } /* endif */ - } /* endif */ + pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch(pszName, + pstTMTableEntries, + pstTMTable->ulMaxEntries, + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); + //--- if passed name found in table + if (pstTMTableEntry) + { + //-- get ID of entry and set output parameter + *pusID = pstTMTableEntry->usId; + } + else + { + // name is not contained in name table + if (!(lOptions & NTMGETID_NOUPDATE_OPT)) + { + usRc = NTMAddNameToTable(pszName, usTableType, + pusID); + + // for new languages update our language group table + if ((usRc == NO_ERROR) && + (usTableType == LANG_KEY)) + { + usRc = NTMAddLangGroup(pszName, *pusID); + } /* endif */ + } /* endif */ + } /* endif */ } /* endif */ } /* endif */ // file name table only: // look for given short name in long name table if not found yet or // an alternative file ID is requested - if ( (usRc == NO_ERROR) && (usTableType == FILE_KEY) && !fLongName && - ((*pusID == NTMGETID_NOTFOUND_ID) || (pusAlternativeID != NULL)) ) + if ((usRc == NO_ERROR) && (usTableType == FILE_KEY) && !fLongName && + ((*pusID == NTMGETID_NOTFOUND_ID) || (pusAlternativeID != NULL))) { - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to entry found - TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched + PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to entry found + TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched // prepare search entry SearchEntry.pszLongName = pszName; // do the actual search - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch( &SearchEntry, - pLongNames->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp ); + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + LongNames.stTableEntry.data(), + LongNames.stTableEntry.size(), + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableComp); - if ( pEntry != NULL ) + if (pEntry != NULL) { - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch( &SearchEntry, - pLongNamesCaseIgnore->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore ); + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + LongNamesCaseIgnore.stTableEntry.data(), + LongNamesCaseIgnore.stTableEntry.size(), + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); } /* endif */ - if ( pEntry != NULL ) + if (pEntry != NULL) { // return ID of found entry - if ( *pusID == NTMGETID_NOTFOUND_ID ) + if (*pusID == NTMGETID_NOTFOUND_ID) { *pusID = pEntry->usId; } - else if ( pusAlternativeID != NULL ) + else if (pusAlternativeID != NULL) { *pusAlternativeID = pEntry->usId; } /* endif */ } } /* endif */ } - else //--- pTmClb is NULL pointer or pszName is empty + else //--- pTmClb is NULL pointer or pszName is empty { //--- wrong function paramters LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); @@ -538,7 +469,6 @@ USHORT EqfMemory::NTMGetIDFromNameEx return usRc; } /* end of function NTMGetIDFromNameEx */ - //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ @@ -608,80 +538,77 @@ USHORT EqfMemory::NTMGetIDFromNameEx //| | //| return usRc; | // ----------------------------------------------------------------------------+ -USHORT -EqfMemory::NTMGetNameFromID( PUSHORT pusID, //intput - USHORT usTableType, //input - PSZ pszName, //output - PSZ pszLongName ) //output, long name (only for FILE_KEY) - +USHORT EqfMemory::NTMGetNameFromID(PUSHORT pusID, // intput + USHORT usTableType, // input + PSZ pszName, // output + PSZ pszLongName) // output, long name (only for FILE_KEY) { - USHORT usRc = NO_ERROR; - BOOL fFound = FALSE; - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; //ptr to first table entry - PTMX_TABLE pstTMTable = NULL; //ptr to table structure - + USHORT usRc = NO_ERROR; + BOOL fFound = FALSE; + PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry + PTMX_TABLE pstTMTable = NULL; // ptr to table structure - if ( pszLongName != NULL ) + if (pszLongName != NULL) { pszLongName[0] = EOS; } /* endif */ //--- if input parameters OK - //if ( pTmClb ) + // if ( pTmClb ) { /******************************************************************/ /* get pointer to table and table entries in dependency of the */ /* table type */ /******************************************************************/ - usRc = NTMGetPointersToTable( usTableType, - &pstTMTable, - &pstTMTableEntries ); + usRc = NTMGetPointersToTable(usTableType, + &pstTMTable, + &pstTMTableEntries); /****************************************************************/ /* search ID in passed table usTableTye and get name of ID */ /****************************************************************/ - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - if ( *pusID == OVERFLOW_ID ) + if (*pusID == OVERFLOW_ID) { - strcpy( pszName, OVERFLOW_NAME ); + strcpy(pszName, OVERFLOW_NAME); fFound = TRUE; } else { ULONG ulI; - for ( ulI = 0; ulI < pstTMTable->ulMaxEntries && !fFound; ulI++ ) + for (ulI = 0; ulI < pstTMTable->ulMaxEntries && !fFound; ulI++) { - if ( *pusID == pstTMTableEntries[ulI].usId ) + if (*pusID == pstTMTableEntries[ulI].usId) { fFound = TRUE; - strcpy( pszName, pstTMTableEntries[ulI].szName ); + strcpy(pszName, pstTMTableEntries[ulI].szName); } /* endif */ } /* endfor */ /**************************************************************/ /* Get any long document name for this ID */ /**************************************************************/ - if ( fFound && (usTableType == FILE_KEY) && (pszLongName != NULL) ) + if (fFound && (usTableType == FILE_KEY) && (pszLongName != NULL)) { BOOL fLongFound = FALSE; - for ( ulI = 0; - ulI < pLongNames->ulEntries && !fLongFound; - ulI++ ) + for (ulI = 0; + ulI < LongNames.stTableEntry.size() && !fLongFound; + ulI++) { - if ( *pusID == pLongNames->stTableEntry[ulI].usId ) + if (*pusID == LongNames.stTableEntry[ulI].usId) { fLongFound = TRUE; - strcpy( pszLongName, - pLongNames->stTableEntry[ulI].pszLongName ); + strcpy(pszLongName, + LongNames.stTableEntry[ulI].pszLongName); } /* endif */ } /* endfor */ - if(!fLongFound){ + if (!fLongFound) + { strcpy(pszLongName, pszName); } } /* endif */ - - if ( !fFound ) + if (!fFound) { /************************************************************/ /* the ID was not found in the table */ @@ -696,34 +623,32 @@ EqfMemory::NTMGetNameFromID( PUSHORT pusID, //intput return usRc; } /* end of function NTMGetNameFromID */ - PSZ EqfMemory::NTMFindNameForID( - PUSHORT pusID, //intput - USHORT usTableType ) // input - + PUSHORT pusID, // intput + USHORT usTableType) // input { - USHORT usRc = NO_ERROR; - BOOL fFound = FALSE; - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; //ptr to first table entry - PTMX_TABLE pstTMTable = NULL; //ptr to table structure + USHORT usRc = NO_ERROR; + BOOL fFound = FALSE; + PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry + PTMX_TABLE pstTMTable = NULL; // ptr to table structure PSZ pszFoundName = NULL; //--- if input parameters OK - //if ( pTmClb ) + // if ( pTmClb ) { /******************************************************************/ /* get pointer to table and table entries in dependency of the */ /* table type */ /******************************************************************/ - usRc = NTMGetPointersToTable( usTableType, - &pstTMTable, - &pstTMTableEntries ); + usRc = NTMGetPointersToTable(usTableType, + &pstTMTable, + &pstTMTableEntries); /****************************************************************/ /* search ID in passed table usTableTye and get name of ID */ /****************************************************************/ - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - if ( *pusID == OVERFLOW_ID ) + if (*pusID == OVERFLOW_ID) { pszFoundName = OVERFLOW_NAME; fFound = TRUE; @@ -731,9 +656,9 @@ PSZ EqfMemory::NTMFindNameForID( else { ULONG ulI; - for ( ulI = 0; ulI < pstTMTable->ulMaxEntries && !fFound; ulI++ ) + for (ulI = 0; ulI < pstTMTable->ulMaxEntries && !fFound; ulI++) { - if ( *pusID == pstTMTableEntries[ulI].usId ) + if (*pusID == pstTMTableEntries[ulI].usId) { fFound = TRUE; pszFoundName = pstTMTableEntries[ulI].szName; @@ -742,23 +667,22 @@ PSZ EqfMemory::NTMFindNameForID( /**************************************************************/ /* Get any long document name for this ID */ /**************************************************************/ - if ( fFound && (usTableType == FILE_KEY) ) + if (fFound && (usTableType == FILE_KEY)) { BOOL fLongFound = FALSE; - for ( ulI = 0; - ulI < pLongNames->ulEntries && !fLongFound; - ulI++ ) + for (ulI = 0; + ulI < LongNames.stTableEntry.size() && !fLongFound; + ulI++) { - if ( *pusID == pLongNames->stTableEntry[ulI].usId ) + if (*pusID == LongNames.stTableEntry[ulI].usId) { fLongFound = TRUE; - pszFoundName = pLongNames->stTableEntry[ulI].pszLongName; + pszFoundName = LongNames.stTableEntry[ulI].pszLongName; } /* endif */ } /* endfor */ } /* endif */ - - if ( !fFound ) + if (!fFound) { /************************************************************/ /* the ID was not found in the table */ @@ -771,10 +695,9 @@ PSZ EqfMemory::NTMFindNameForID( } /* endif */ } - return( pszFoundName ); + return (pszFoundName); } /* end of function NTMFindNameForID */ - //+----------------------------------------------------------------------------+ //|Internal function | //+----------------------------------------------------------------------------+ @@ -814,15 +737,14 @@ PSZ EqfMemory::NTMFindNameForID( //| compare the names of the passed tabele entries using strcmp | //| return compare result | // ----------------------------------------------------------------------------+ -int -NTMCompNames( const void * pstTMTableEntry1, //input - const void * pstTMTableEntry2 ) //input +int NTMCompNames(const void *pstTMTableEntry1, // input + const void *pstTMTableEntry2) // input { /********************************************************************/ /* compare the names of the passed tabele entries using strcmp */ /********************************************************************/ - return( strcmp( ((PTMX_TABLE_ENTRY) pstTMTableEntry1)->szName, - ((PTMX_TABLE_ENTRY) pstTMTableEntry2)->szName )); + return (strcmp(((PTMX_TABLE_ENTRY)pstTMTableEntry1)->szName, + ((PTMX_TABLE_ENTRY)pstTMTableEntry2)->szName)); } /* end of function NTMCompNames */ //+----------------------------------------------------------------------------+ @@ -870,47 +792,47 @@ NTMCompNames( const void * pstTMTableEntry1, //input //| return usRc | // ----------------------------------------------------------------------------+ USHORT -EqfMemory::NTMGetPointersToTable( - USHORT usTableType, //input - PTMX_TABLE *ppstTMTable, //output - PTMX_TABLE_ENTRY *ppstTMTableEntries ) //output +EqfMemory::NTMGetPointersToTable( + USHORT usTableType, // input + PTMX_TABLE *ppstTMTable, // output + PTMX_TABLE_ENTRY *ppstTMTableEntries) // output { - USHORT usRc = NO_ERROR; //function return code + USHORT usRc = NO_ERROR; // function return code /********************************************************************/ /* get pointer to table in dependency of passed usTableType */ /********************************************************************/ - switch ( usTableType ) + switch (usTableType) { - //----------------------------------------------------------------------- - case LANG_KEY : - *ppstTMTable = (PTMX_TABLE)&Languages; - break; - //----------------------------------------------------------------------- - case FILE_KEY : - *ppstTMTable = (PTMX_TABLE)&FileNames; - break; - //----------------------------------------------------------------------- - case AUTHOR_KEY : - *ppstTMTable = (PTMX_TABLE)&Authors; - break; - //----------------------------------------------------------------------- - case TAGTABLE_KEY : - *ppstTMTable = (PTMX_TABLE)&TagTables; - break; - //----------------------------------------------------------------------- - case LANGGROUP_KEY : - *ppstTMTable = (PTMX_TABLE)&LangGroups; - break; - //----------------------------------------------------------------------- - default : - LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); - *ppstTMTable = NULL; - *ppstTMTableEntries = NULL; - break; + //----------------------------------------------------------------------- + case LANG_KEY: + *ppstTMTable = (PTMX_TABLE)&Languages; + break; + //----------------------------------------------------------------------- + case FILE_KEY: + *ppstTMTable = (PTMX_TABLE)&FileNames; + break; + //----------------------------------------------------------------------- + case AUTHOR_KEY: + *ppstTMTable = (PTMX_TABLE)&Authors; + break; + //----------------------------------------------------------------------- + case TAGTABLE_KEY: + *ppstTMTable = (PTMX_TABLE)&TagTables; + break; + //----------------------------------------------------------------------- + case LANGGROUP_KEY: + *ppstTMTable = (PTMX_TABLE)&LangGroups; + break; + //----------------------------------------------------------------------- + default: + LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); + *ppstTMTable = NULL; + *ppstTMTableEntries = NULL; + break; } /* end switch */ - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { /******************************************************************/ /* get pointer to table entries */ @@ -922,79 +844,6 @@ EqfMemory::NTMGetPointersToTable( } /* end of function NTMGetPointersToTable */ - -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMLockTM | -//+----------------------------------------------------------------------------+ -//|Function call: NTMLockTM( clb, fLock, pfLocked ); | -//+----------------------------------------------------------------------------+ -//|Description: Physically lock or unlock the data and the index file | -//| of the given TM | -//+----------------------------------------------------------------------------+ -//|Parameters: EqfMemory* pTmClb, pointer to control block | -//| BOOL fLock TRUE = Lock, FALSE = Unlock | -//| PBOOL pfLocked set to TRUE if TM has been | -//| locked | -//+----------------------------------------------------------------------------+ -//|Returncode type: USHORT | -// ----------------------------------------------------------------------------+ -USHORT NTMLockTM -( - EqfMemory* pTmClb, // pointer to control block - BOOL fLock, // TRUE = Lock, FALSE = Unlock - PBOOL pfLocked // set to TRUE if TM has been locked -) -{ - USHORT usRc = 0; // function return code - BOOL fLockedData = FALSE; // data-file-has-been-locked flag - BOOL fLockedIndex = FALSE; // index-file-has-been-locked flagf - - DEBUGEVENT( NTMLOCKTM_LOC, FUNCENTRY_EVENT, usRc ); - - if ( fLock ) - { - // Lock data file of TM - usRc = pTmClb->TmBtree.EQFNTMPhysLock( TRUE, &fLockedData ); - - - // Lock Index file of TM - if ( !usRc ) - { - usRc = pTmClb->InBtree.EQFNTMPhysLock( TRUE, &fLockedIndex ); - } /* endif */ - - // Set caller's lock flag - *pfLocked = fLockedData && fLockedIndex; - - // Unlock data file if lock of index failed - if ( !fLockedIndex && fLockedData ) - { - usRc = pTmClb->TmBtree.EQFNTMPhysLock( FALSE, &fLockedData ); - } /* endif */ - } - else - { - // Rewrite compact area if compact area has been changed - pTmClb->RewriteCompactTable(); - - // Unlock index file of TM - usRc = pTmClb->InBtree.EQFNTMPhysLock( FALSE, &fLockedIndex ); - - // Unlock data file of TM - usRc = pTmClb->TmBtree.EQFNTMPhysLock( FALSE, &fLockedData ); - } /* endif */ - - if ( usRc != NO_ERROR ) - { - ERREVENT( NTMLOCKTM_LOC, ERROR_EVENT, usRc ); - } /* endif */ - - return( usRc ); -} /* end of function NTMLockTM */ - - //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ @@ -1009,106 +858,17 @@ USHORT NTMLockTM //|Returncode type: USHORT error return code or NO_ERROR if O.K. | // ----------------------------------------------------------------------------+ USHORT EqfMemory::NTMCreateLongNameTable() -{ - USHORT usRC = NO_ERROR; // function return code - - // allocate initial long name pointer array - if ( UtlAlloc( (PVOID *)&(pLongNames), 0L, (ULONG) - (sizeof(TMX_LONGNAMETABLE) + - sizeof(TMX_LONGNAME_TABLE_ENTRY) * LONGNAMETABLE_ENTRIES), - NOMSG ) ) - { - pLongNames->ulTableSize = LONGNAMETABLE_ENTRIES; - pLongNames->ulEntries = 0; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ +{ + LongNames.pszBuffer.resize(LONGNAMETABLE_ENTRIES); + LongNames.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); - // allocate initial long name pointer array for case ignore search - if ( UtlAlloc( (PVOID *)&(pLongNamesCaseIgnore), 0L, (ULONG) - (sizeof(TMX_LONGNAMETABLE) + - sizeof(TMX_LONGNAME_TABLE_ENTRY) * LONGNAMETABLE_ENTRIES), - NOMSG ) ) - { - pLongNamesCaseIgnore->ulTableSize = LONGNAMETABLE_ENTRIES; - pLongNamesCaseIgnore->ulEntries = 0; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + LongNamesCaseIgnore.pszBuffer.resize(LONGNAMETABLE_ENTRIES); + LongNamesCaseIgnore.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); - // allocate initial long name buffer area - if ( usRC == NO_ERROR ) - { - if ( UtlAlloc( (PVOID *)&(pLongNames->pszBuffer), 0L, - (ULONG) LONGNAMEBUFFER_SIZE, NOMSG ) ) - { - pLongNames->ulBufSize = (ULONG)LONGNAMEBUFFER_SIZE; - pLongNames->ulBufUsed = (ULONG)sizeof(USHORT); // end-of-table delimiter - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - - // cleanup in case of errors - if ( usRC != NO_ERROR ) - { - NTMDestroyLongNameTable( ); - } /* endif */ - - // return to caller - return( usRC ); + return 0; } /* end of function NTMCreateLongNameTable */ -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMDestroyLongNameTable | -//+----------------------------------------------------------------------------+ -//|Function call: usRC = NTMDestroyLongnameTable(); | -//+----------------------------------------------------------------------------+ -//|Description: Destroys a long name table and frees the memory occupied | -//| by it. | -//+----------------------------------------------------------------------------+ -//|Parameters: EqfMemory* pTmClb pointer to control block | -//+----------------------------------------------------------------------------+ -//|Returncode type: USHORT error return code or NO_ERROR if O.K. | -// ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMDestroyLongNameTable() -{ - USHORT usRC = NO_ERROR; // function return code - - // if there is a long name table ... - if ( pLongNames != NULL ) - { - // free any buffer area - if ( pLongNames->pszBuffer != NULL ) - { - UtlAlloc( (PVOID *)&(pLongNames->pszBuffer), 0L, 0L, NOMSG ); - } /* endif */ - - // free table - UtlAlloc( (PVOID *)&(pLongNames), 0L, 0L, NOMSG ); - pLongNames = NULL; - } /* endif */ - - if ( pLongNamesCaseIgnore != NULL ) - { - // free table - UtlAlloc( (PVOID *)&(pLongNamesCaseIgnore), 0L, 0L, NOMSG ); - pLongNamesCaseIgnore = NULL; - } /* endif */ - - // return to caller - return( usRC ); -} /* end of function NTMDestroyLongNameTable */ - //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ @@ -1122,198 +882,117 @@ USHORT EqfMemory::NTMDestroyLongNameTable() //+----------------------------------------------------------------------------+ //|Returncode type: USHORT error return code or NO_ERROR if O.K. | // ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMReadLongNameTable(){ - USHORT usRC = NO_ERROR; // function return code - ULONG ulLen = 0; // record length +USHORT EqfMemory::NTMReadLongNameTable() +{ + USHORT usRC = NO_ERROR; // function return code // call to obtain exact length of record - ulLen = 0; - usRC = TmBtree.EQFNTMGet( LONGNAME_KEY, 0, &ulLen ); - - if ( usRC == NO_ERROR ) - { - // allocate buffer area if it is too small - if ( pLongNames->ulBufSize < ulLen ) - { - if ( UtlAlloc( (PVOID *)&pLongNames->pszBuffer, 0L, - ulLen, NOMSG )) - { - pLongNames->ulBufSize = ulLen; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - - // read long name table from database - if ( usRC == NO_ERROR ) - { - usRC = TmBtree.EQFNTMGet( LONGNAME_KEY, - (PCHAR)pLongNames->pszBuffer, &ulLen ); - } /* endif */ + usRC = TmBtree.EQFNTMGet(LONGNAME_KEY, LongNames.pszBuffer); + if (usRC == NO_ERROR) + { // unterse data if table data is tersed - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { - PTERSEHEADER pTerseHeader = (PTERSEHEADER)pLongNames->pszBuffer; - PBYTE pNewArea = NULL; // ptr to unterse data area + PTERSEHEADER pTerseHeader = (PTERSEHEADER)LongNames.pszBuffer.data(); + std::vector pNewArea; // ptr to unterse data area - if ( pTerseHeader->ulMagicWord == TERSEMAGICWORD ) + if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) { // table is tersed... - + pNewArea.resize(pTerseHeader->usDataSize); // allocate buffer for untersed data - if ( !UtlAlloc( (PVOID *)&pNewArea, 0L, - (LONG)pTerseHeader->usDataSize, NOMSG ) ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ // unterse data - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { - ULONG ulNewLen = 0; + ULONG ulNewLen = 0; - memcpy( pNewArea, - (PBYTE)pLongNames->pszBuffer + sizeof(TERSEHEADER), - ulLen - sizeof(TERSEHEADER) ); + memcpy(pNewArea.data(), + LongNames.pszBuffer.data() + sizeof(TERSEHEADER), + LongNames.pszBuffer.size() - sizeof(TERSEHEADER)); T5LOG(T5ERROR) << "::TEMPORARY_COMMENTED in NTMReadLongNameTable, fUtlHuffmanExpand"; #ifdef TEMPORARY_COMMENTED - if ( !fUtlHuffmanExpand( (PUCHAR)pNewArea, pTerseHeader->usDataSize, - &ulNewLen ) ) + if (!fUtlHuffmanExpand((PUCHAR)pNewArea, pTerseHeader->usDataSize, + &ulNewLen)) { LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); // expand failed most likely to - // memory shortage + // memory shortage } /* endif */ - #endif +#endif } /* endif */ // set table data and cleanup - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { // set size of buffer area - pLongNames->ulBufSize = (ULONG)pTerseHeader->usDataSize; + //LongNames.data()->ulBufSize = (ULONG)pTerseHeader->usDataSize; + //LongNames.pszBuffer.resize(ulLen); + LongNames.pszBuffer = pNewArea; + //memcpy(LongNames.pszBuffer.data(), pNewArea.data(), pNewArea.size()); // free tersed data area - UtlAlloc( (PVOID *)&pLongNames->pszBuffer, 0L, 0L, NOMSG ); + // UtlAlloc( (PVOID *)&LongNames.data()->pszBuffer, 0L, 0L, NOMSG ); // anchor untersed data area - pLongNames->pszBuffer = (PSZ)pNewArea; + //LongNames.data()->pszBuffer = (PSZ)pNewArea; // avoid free of untersed data area - pNewArea = NULL; + // pNewArea = NULL; } /* endif */ - if ( pNewArea != NULL ) UtlAlloc( (PVOID *)&pNewArea, 0L, 0L, NOMSG ); + // if ( pNewArea != NULL ) UtlAlloc( (PVOID *)&pNewArea, 0L, 0L, NOMSG ); } /* endif */ } /* endif */ // setup pointer array for long names - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { - ULONG ulEntries = 0; // number of entries in buffer - PSZ pszTemp; // ptr for buffer processing - - // remember used space in buffer area - pLongNames->ulBufUsed = ulLen; - - // count number of entries in long name buffer - pszTemp = pLongNames->pszBuffer; - while ( *((PUSHORT)pszTemp) != 0 ) + // fill pointer array + PSZ pszTemp = (PSZ)LongNames.pszBuffer.data(); + TMX_LONGNAME_TABLE_ENTRY Entry; + + while (*((PUSHORT)pszTemp) != 0) { - ulEntries++; - pszTemp += sizeof(USHORT); // skip ID - pszTemp += strlen(pszTemp)+1;// skip long name + memset(&Entry, 0, sizeof(Entry)); + // get ID of string + Entry.usId = *((PUSHORT)pszTemp); + pszTemp += sizeof(USHORT); // skip ID + Entry.longNameStartOffset = pszTemp - (PSZ)LongNames.pszBuffer.data(); + + // set pointer to long document name + Entry.pszLongName = pszTemp; + + // continue with next entry + LongNames.stTableEntry.push_back(Entry); + pszTemp += strlen(pszTemp) + 1; // skip long name } /* endwhile */ - - // enlarge pointer array if necessary - if ( pLongNames->ulTableSize < ulEntries ) - { - ULONG ulOldSize = sizeof(TMX_LONGNAMETABLE) + - (sizeof(TMX_LONGNAME_TABLE_ENTRY) * - pLongNames->ulTableSize); - ULONG ulNewSize = sizeof(TMX_LONGNAMETABLE) + - (sizeof(TMX_LONGNAME_TABLE_ENTRY) * ulEntries); - if ( UtlAlloc( (PVOID *)&pLongNames, - ulOldSize, ulNewSize, NOMSG ) ) - { - pLongNames->ulTableSize = ulEntries; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - // enlarge pointer array for case ignore search as well - if ( usRC == NO_ERROR ) - { - if ( UtlAlloc( (PVOID *)&pLongNamesCaseIgnore, - ulOldSize, ulNewSize, NOMSG ) ) - { - pLongNamesCaseIgnore->ulTableSize = ulEntries; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - } /* endif */ - - // fill pointer array - if ( usRC == NO_ERROR ) - { - PTMX_LONGNAME_TABLE_ENTRY pEntry = pLongNames->stTableEntry; - pLongNames->ulEntries = 0; - pszTemp = pLongNames->pszBuffer; - while ( *((PUSHORT)pszTemp) != 0 ) - { - - // get ID of string - pEntry->usId = *((PUSHORT)pszTemp); - pszTemp += sizeof(USHORT); // skip ID - - // set pointer to long document name - pEntry->pszLongName = pszTemp; - - // continue with next entry - pEntry++; - pLongNames->ulEntries++; - pszTemp += strlen(pszTemp)+1;// skip long name - } /* endwhile */ - } /* endif */ - + // sort long name array - if ( usRC == NO_ERROR ) - { - qsort( pLongNames->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp ); - } /* endif */ + qsort(LongNames.stTableEntry.data(), + LongNames.stTableEntry.size(), + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableComp); // make copy of long name array for case ignore search - if ( usRC == NO_ERROR ) - { - memcpy( pLongNamesCaseIgnore->stTableEntry, - pLongNames->stTableEntry, - pLongNames->ulEntries * - sizeof(TMX_LONGNAME_TABLE_ENTRY) ); - - qsort( pLongNamesCaseIgnore->stTableEntry, - pLongNames->ulEntries, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore ); - } /* endif */ + LongNamesCaseIgnore.stTableEntry = LongNames.stTableEntry; + LongNamesCaseIgnore.pszBuffer = LongNames.pszBuffer; + toLower(LongNamesCaseIgnore.pszBuffer); + + LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); + + qsort(LongNamesCaseIgnore.stTableEntry.data(), + LongNamesCaseIgnore.stTableEntry.size(), + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); } /* endif */ } /* endif */ // return to caller - return( usRC ); + return (usRC); } /* end of function NTMReadLongNameTable */ - //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ @@ -1349,18 +1028,17 @@ USHORT EqfMemory::NTMReadLongNameTable(){ //| ERROR_NOT_ENOUGH_MEMORY - reallocation of table failed | //| others - return codes from QDAM | // ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMAddNameToTable -( - PSZ pszName, // input - USHORT usTableType, //input - PUSHORT pusID //output +USHORT EqfMemory::NTMAddNameToTable( + PSZ pszName, // input + USHORT usTableType, // input + PUSHORT pusID // output ) { - USHORT usRc = NO_ERROR; //function return coed - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; //ptr to first table entry - PTMX_TABLE pstTMTable = NULL; //ptr to table structure - ULONG ulNewSize; //for size calculation - ULONG ulReallocSize; //size for realloc of the table + USHORT usRc = NO_ERROR; // function return coed + PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry + PTMX_TABLE pstTMTable = NULL; // ptr to table structure + ULONG ulNewSize; // for size calculation + ULONG ulReallocSize; // size for realloc of the table /********************************************************************/ /* initialize ID, that in error case a 0 - ID is returned */ @@ -1368,144 +1046,86 @@ USHORT EqfMemory::NTMAddNameToTable *pusID = 0; //--- if input parameters OK - if ( pszName[0] != EOS ) + if (pszName[0] != EOS) { /******************************************************************/ /* get pointer to table and table entries in dependency of the */ /* table type */ /******************************************************************/ - usRc = NTMGetPointersToTable( usTableType, - &pstTMTable, - &pstTMTableEntries ); + usRc = NTMGetPointersToTable(usTableType, + &pstTMTable, + &pstTMTableEntries); - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - /*************************************************************/ - /* Check if space is available in table so that new entry */ - /* will fit into the table. */ - /* Check if the allocated size can hold the new entry. */ - /* If not reallocate the table and check that the table */ - /* will not exceed the maximum size of a QDAM record (32K) */ - /*************************************************************/ - #ifdef TEMPORARY_COMMENTED - ulNewSize = (ULONG)sizeof(TMX_TABLE) + - (pstTMTable->ulMaxEntries * (ULONG)sizeof(TMX_TABLE_ENTRY) ); - if ( ulNewSize >= pstTMTable->ulAllocSize ) - { - ulReallocSize = (ULONG)pstTMTable->ulAllocSize + (ULONG)TMX_TABLE_SIZE; - { - if ( UtlAlloc( (PVOID *) &pstTMTable, - (LONG)pstTMTable->ulAllocSize, - (LONG)ulReallocSize, - NOMSG ) ) - { - /*****************************************************/ - /* no error from UtlAlloc */ - /* reset the pointer in TM_CLB because UtlAlloc */ - /* reallocates the storage on another place */ - /*****************************************************/ - switch ( usTableType ) - { - //--------------------------------------------------------- - case LANG_KEY : - pTmClb->Languages = *pstTMTable; - pstTMTableEntries = pTmClb->Languages.stTmTableEntry; - break; - //--------------------------------------------------------- - case FILE_KEY : - pTmClb->FileNames = *pstTMTable; - pstTMTableEntries = pTmClb->FileNames.stTmTableEntry; - break; - //--------------------------------------------------------- - case AUTHOR_KEY : - pTmClb->Authors = *pstTMTable; - pstTMTableEntries = pTmClb->Authors.stTmTableEntry; - break; - //--------------------------------------------------------- - case TAGTABLE_KEY : - pTmClb->TagTables = *pstTMTable; - pstTMTableEntries = pTmClb->TagTables.stTmTableEntry; - break; - //--------------------------------------------------------- - case LANGGROUP_KEY : - pTmClb->LangGroups = *pstTMTable; - pstTMTableEntries = pTmClb->LangGroups.stTmTableEntry; - break; - } /* end switch */ - //pstTMTable->ulAllocSize = ulReallocSize; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ - - } /* endif */ - #endif - //if( (pstTMTable->ulMaxEntries+1) * sizeof(pstTMTableEntries[0]) + sizeof(ULONG) > TMX_REC_SIZE){ - // T5LOG(T5ERROR) << ":: cannot increase table size, it reached it's limit ; maxEntries="<< pstTMTable->ulMaxEntries << "; tagTableType=" << usTableType; - //usRc = ERROR_NOT_ENOUGH_MEMORY; - //} +/*************************************************************/ +/* Check if space is available in table so that new entry */ +/* will fit into the table. */ +/* Check if the allocated size can hold the new entry. */ +/* If not reallocate the table and check that the table */ +/* will not exceed the maximum size of a QDAM record (32K) */ +/*************************************************************/ + const size_t currentCap = pstTMTable->stTmTableEntry.size(); - if(pstTMTable->ulMaxEntries >= currentCap){ + if (pstTMTable->ulMaxEntries >= currentCap) + { size_t newCap = currentCap + 10; - if(newCap >= pstTMTable->stTmTableEntry.max_size()){ - T5LOG(T5FATAL) << "tried to set too big capacity for table " << usTableType <<"; newCap = " << newCap; + if (newCap >= pstTMTable->stTmTableEntry.max_size()) + { + T5LOG(T5FATAL) << "tried to set too big capacity for table " << usTableType << "; newCap = " << newCap; } - //T5LOG(T5ERROR)<<"remove this log later: tableType = " << usTableType <<" changing capacity to "<< newCap; + // T5LOG(T5ERROR)<<"remove this log later: tableType = " << usTableType <<" changing capacity to "<< newCap; pstTMTable->stTmTableEntry.resize(newCap); pstTMTableEntries = &pstTMTable->stTmTableEntry[0]; } - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { /*************************************************************/ /* insert name and id to table and sort table */ /*************************************************************/ int pos = pstTMTable->ulMaxEntries; *pusID = ++pstTMTable->ulMaxEntries; - strncpy( pstTMTableEntries[pos].szName, - pszName, MAX_LANG_LENGTH-1 ); + strncpy(pstTMTableEntries[pos].szName, + pszName, MAX_LANG_LENGTH - 1); pstTMTableEntries[pos].usId = *pusID; - qsort( pstTMTableEntries, - pstTMTable->ulMaxEntries, - sizeof(TMX_TABLE_ENTRY), - NTMCompNames ); + qsort(pstTMTableEntries, + pstTMTable->ulMaxEntries, + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); // update table record in TM QDAM file (if not in read-only mode) - if ( !(usAccessMode & ASD_READONLY) ) + if (!(usAccessMode & ASD_READONLY)) { - if ( usTableType != LANGGROUP_KEY ) + if (usTableType != LANGGROUP_KEY) { - int occupSize = (pstTMTable->ulMaxEntries+1) * sizeof(TMX_TABLE_ENTRY); - PBYTE pOldTable = new BYTE[occupSize]; - TMX_TABLE_OLD* table = (TMX_TABLE_OLD*) pOldTable; + int occupSize = (pstTMTable->ulMaxEntries + 1) * sizeof(TMX_TABLE_ENTRY); + std::vector pOldTable; + pOldTable.resize(occupSize);; + TMX_TABLE_OLD *table = (TMX_TABLE_OLD *)pOldTable.data(); + //pOldTable = pstTMTable->stTmTableEntry; table->ulMaxEntries = pstTMTable->ulMaxEntries; - if(table->ulMaxEntries){ - memcpy( &table->stTmTableEntry, - &pstTMTable->stTmTableEntry[0], sizeof(TMX_TABLE_ENTRY) * table->ulMaxEntries); + if (table->ulMaxEntries) + { + memcpy(&table->stTmTableEntry, + &pstTMTable->stTmTableEntry[0], sizeof(TMX_TABLE_ENTRY) * table->ulMaxEntries); } usRc = TmBtree.EQFNTMUpdate( - (ULONG)usTableType, - //(PBYTE)pstTMTable, - (PBYTE)table, - //pstTMTable->ulAllocSize - //BTREE_REC_SIZE_V3 - occupSize - ); - delete[] pOldTable; + (ULONG)usTableType, + (PBYTE)table, + occupSize); } /* endif */ } /* endif */ } /* endif */ } /* endif */ } - else //--- pTmClb is NULL pointer or pszName is empty + else //--- pTmClb is NULL pointer or pszName is empty { //--- wrong function paramters LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); } /* endif */ - if ( usRc == ERROR_TABLE_FULL ) + if (usRc == ERROR_TABLE_FULL) { usRc = 0; *pusID = OVERFLOW_ID; @@ -1514,7 +1134,6 @@ USHORT EqfMemory::NTMAddNameToTable return usRc; } /* end of function NTMAddNameToTable */ - //+----------------------------------------------------------------------------+ //|Internal function | //+----------------------------------------------------------------------------+ @@ -1524,10 +1143,9 @@ USHORT EqfMemory::NTMAddNameToTable //| of the passed table entries are compared and the | //| compare result is returned. | // ----------------------------------------------------------------------------+ -int NTMLongNameTableComp -( - const void * pEntry1, //input - const void * pEntry2 //input +int NTMLongNameTableComp( + const void *pEntry1, // input + const void *pEntry2 // input ) { /********************************************************************/ @@ -1535,28 +1153,25 @@ int NTMLongNameTableComp /********************************************************************/ PSZ lname1 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName; PSZ lname2 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName; - return( strcmp( lname1, lname2 ) ); + return (strcmp(lname1, lname2)); } /* end of function NTMCompNames */ -int NTMLongNameTableCompCaseIgnore -( - const void * pEntry1, //input - const void * pEntry2 //input +int NTMLongNameTableCompCaseIgnore( + const void *pEntry1, // input + const void *pEntry2 // input ) { /********************************************************************/ /* compare the long names of the passed table entries using strcmp */ /********************************************************************/ - //return( _stricmp( ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName, - // ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName )); - return( strcmp( ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName, - ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName )); + // return( _stricmp( ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName, + // ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName )); + return (strcasecmp(((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName, + ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName)); } /* end of function NTMCompNames */ //+----------------------------------------------------------------------------+ - - //+----------------------------------------------------------------------------+ //|Internal function | //+----------------------------------------------------------------------------+ @@ -1565,93 +1180,78 @@ int NTMLongNameTableCompCaseIgnore //|Description: This function loads a TM name table from the database. | //| If the table is tersed it is uncompressed. | // ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMLoadNameTable -( - ULONG ulTableKey, // key of table record - PTMX_TABLE pTMTable, // ptr to table data pointer - PULONG pulSize // ptr to buffer for size of table data +USHORT EqfMemory::NTMLoadNameTable( + ULONG ulTableKey, // key of table record + PTMX_TABLE pTMTable, // ptr to table data pointer + PULONG pulSize // ptr to buffer for size of table data ) { - USHORT usRc = NO_ERROR; // function return code + USHORT usRc = NO_ERROR; // function return code + std::vector pOldTable; // call to obtain exact length of record *pulSize = 0; - usRc = TmBtree.EQFNTMGet( ulTableKey, 0, pulSize ); - - PBYTE pOldTable = nullptr; - // read table data - if ( usRc == NO_ERROR ) - { - ULONG ulLen = *pulSize; - pOldTable = new BYTE[ulLen + 1]; - usRc = TmBtree.EQFNTMGet( ulTableKey, (PCHAR)(pOldTable), &ulLen ); - } /* endif */ + usRc = TmBtree.EQFNTMGet(ulTableKey, pOldTable); // handle tersed name tables - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { - PTERSEHEADER pTerseHeader = (PTERSEHEADER)pOldTable; - PBYTE pNewArea = NULL; // ptr to unterse data area + PTERSEHEADER pTerseHeader = (PTERSEHEADER)pOldTable.data(); + std::vector pNewArea; // ptr to unterse data area - if ( pTerseHeader->ulMagicWord == TERSEMAGICWORD ) + if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) { // table is tersed... - + pNewArea.resize(pTerseHeader->usDataSize); // allocate buffer for untersed data - if ( !UtlAlloc( (PVOID *)&pNewArea, 0L, - (LONG)pTerseHeader->usDataSize, NOMSG ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ // unterse data - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { ULONG ulNewLen = 0; - memcpy( pNewArea, (PBYTE)pOldTable + sizeof(TERSEHEADER), - *pulSize - sizeof(TERSEHEADER) ); - + memcpy(pNewArea.data(), (PBYTE)pOldTable.data() + sizeof(TERSEHEADER), + *pulSize - sizeof(TERSEHEADER)); + T5LOG(T5ERROR) << ":: TEMPORARY_COMMENTED temcom_id = 51 in NTMLoadNameTable"; #ifdef TEMPORARY_COMMENTED - if ( !fUtlHuffmanExpand( (PUCHAR)pNewArea, pTerseHeader->usDataSize, - &ulNewLen ) ) + if (!fUtlHuffmanExpand((PUCHAR)pNewArea, pTerseHeader->usDataSize, + &ulNewLen)) { LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); // expand failed most likely to - // memory shortage + // memory shortage } /* endif */ - #endif +#endif } /* endif */ // set table data and cleanup - if ( usRc == NO_ERROR ) + if (usRc == NO_ERROR) { // set size of name table *pulSize = pTerseHeader->usDataSize; // free tersed data area - //UtlAlloc( (PVOID *)pTMTable, 0L, 0L, NOMSG ); + // UtlAlloc( (PVOID *)pTMTable, 0L, 0L, NOMSG ); // anchor untersed data area //*pTMTable = pNewArea; // avoid free of untersed data area - pNewArea = NULL; + // pNewArea = NULL; } /* endif */ - if ( pNewArea != NULL ) UtlAlloc( (PVOID *)&pNewArea, 0L, 0L, NOMSG ); } /* endif */ } /* endif */ // special handling for language name table: // correct word delimiter in "Other Languages" which is // 0xA0 under Windows (due to OemToAnsi) but should be 0xFF - if ( (usRc == NO_ERROR) && (ulTableKey == LANG_KEY) ) + if ((usRc == NO_ERROR) && (ulTableKey == LANG_KEY)) { LONG lRest = *pulSize; - PBYTE pbTemp = (PBYTE)pOldTable; - while ( lRest ) + PBYTE pbTemp = (PBYTE)pOldTable.data(); + while (lRest) { - if ( *pbTemp == 0xA0 ) + if (*pbTemp == 0xA0) { *pbTemp = 0xFF; } /* endif */ @@ -1660,38 +1260,34 @@ USHORT EqfMemory::NTMLoadNameTable } /* endwhile */ } /* endif */ - if(usRc == NO_ERROR){ - pTMTable->ulMaxEntries = ((PTMX_TABLE_OLD)pOldTable)->ulMaxEntries; + if (usRc == NO_ERROR) + { + pTMTable->ulMaxEntries = ((PTMX_TABLE_OLD)pOldTable.data())->ulMaxEntries; pTMTable->stTmTableEntry.clear(); - pTMTable->stTmTableEntry.resize(pTMTable->ulMaxEntries+1); - if(pTMTable->ulMaxEntries){ - memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)pOldTable)->stTmTableEntry, pTMTable->ulMaxEntries * sizeof(TMX_TABLE_ENTRY)); + pTMTable->stTmTableEntry.resize(pTMTable->ulMaxEntries + 1); + if (pTMTable->ulMaxEntries) + { + memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)pOldTable.data())->stTmTableEntry, pTMTable->ulMaxEntries * sizeof(TMX_TABLE_ENTRY)); } } - if(pOldTable){ - delete[] pOldTable; - pOldTable = nullptr; - } // return to caller - return( usRc ); + return (usRc); } /* end of function NTMLoadNameTable */ - -USHORT EqfMemory::NTMAddLangGroup -( - PSZ pszLang, // ptr to language name - USHORT sLangID // ID of language in our tables +USHORT EqfMemory::NTMAddLangGroup( + PSZ pszLang, // ptr to language name + USHORT sLangID // ID of language in our tables ) { - SHORT sGroupID = 0; - USHORT usRC = NO_ERROR; // function return code - CHAR szLangGroup[MAX_LANGUAGE_PROPERTIES]; // buffer for language group name + SHORT sGroupID = 0; + USHORT usRC = NO_ERROR; // function return code + CHAR szLangGroup[MAX_LANGUAGE_PROPERTIES]; // buffer for language group name // get language properties (incl. group name) - if ( GetLanguageGroup( pszLang, szLangGroup ) ) + if (GetLanguageGroup(pszLang, szLangGroup)) { - if ( szLangGroup[0] == EOS ) + if (szLangGroup[0] == EOS) { // no language group for language, so use language ID as // language group ID but multiply it with -1 to distinguish @@ -1704,84 +1300,66 @@ USHORT EqfMemory::NTMAddLangGroup // no language properties for language, so use language ID as // language group ID but multiply it with -1 to distinguish // language group IDs from language IDs - sGroupID = sLangID * -1; + sGroupID = sLangID * -1; } /* endif */ // get ID of group name - if ( (usRC == NO_ERROR) && (sGroupID == 0) ) + if ((usRC == NO_ERROR) && (sGroupID == 0)) { - usRC = NTMGetIDFromName( szLangGroup, NULL, LANGGROUP_KEY, (PUSHORT)&sGroupID ); + usRC = NTMGetIDFromName(szLangGroup, NULL, LANGGROUP_KEY, (PUSHORT)&sGroupID); } /* endif */ // enlarge language-ID-to-group-ID table if necessary - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { LONG lRequiredSize = (sLangID + 1) * sizeof(SHORT); - if ( lLangIdToGroupTableSize < lRequiredSize ) + if (psLangIdToGroupTable.size() < lRequiredSize) { - lRequiredSize += (10 * sizeof(SHORT)); - if ( UtlAlloc( (PVOID *)&(psLangIdToGroupTable), - lLangIdToGroupTableSize, - lRequiredSize, NOMSG ) ) - { - lLangIdToGroupTableSize = lRequiredSize; - lLangIdToGroupTableUsed = lRequiredSize; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + psLangIdToGroupTable.resize(lRequiredSize + 10); + lLangIdToGroupTableUsed = lRequiredSize; } /* endif */ } /* endif */ // update language ID to group ID table - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { psLangIdToGroupTable[sLangID] = sGroupID; } /* endif */ - return( usRC ); + return (usRC); } /* end of function NTMAddLangGroup */ USHORT EqfMemory::NTMCreateLangGroupTable() { USHORT usRC = NO_ERROR; - //pTmClb->LangGroups.ulAllocSize = TMX_TABLE_SIZE; + // pTmClb->LangGroups.ulAllocSize = TMX_TABLE_SIZE; // allocate language-ID-to-group-ID-table - - LONG lSize = Languages.ulMaxEntries > 100L ? Languages.ulMaxEntries : 100L; - if(lSize <=0) lSize = 1; - lSize *= sizeof(SHORT); - if( UtlAlloc( (PVOID *)&(psLangIdToGroupTable), - 0L, lSize, NOMSG ) ) - { - lLangIdToGroupTableSize = lSize; - lLangIdToGroupTableUsed = 0L; - } - else - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + + LONG lSize = Languages.ulMaxEntries > 100L ? Languages.ulMaxEntries : 100L; + if (lSize <= 0) + lSize = 1; + psLangIdToGroupTable.resize(lSize); + + lLangIdToGroupTableUsed = 0L; // get group IDs for all languages and fill map table - if ( usRC == NO_ERROR ) + if (usRC == NO_ERROR) { int i = 0; - while ( (usRC == NO_ERROR) && - (i < (int)Languages.ulMaxEntries) ) + while ((usRC == NO_ERROR) && + (i < (int)Languages.ulMaxEntries)) { - usRC = NTMAddLangGroup( Languages.stTmTableEntry[i].szName, - Languages.stTmTableEntry[i].usId ); + usRC = NTMAddLangGroup(Languages.stTmTableEntry[i].szName, + Languages.stTmTableEntry[i].usId); i++; } /* endwhile */ } /* endif */ - return( usRC ); + return (usRC); } /* end of function NTMCreateLangGroupTable */ - -// +// // function NTMOrganizeIndexFile // // compact/organize the index part of a memory @@ -1789,236 +1367,224 @@ USHORT EqfMemory::NTMCreateLangGroupTable() USHORT EqfMemory::NTMOrganizeIndexFile() { USHORT usRC = NO_ERROR; - usRC = EQFNTMOrganizeIndex( &(InBtree), usAccessMode, START_KEY ); + usRC = EQFNTMOrganizeIndex(&(InBtree), usAccessMode, START_KEY); - return( usRC ); + return (usRC); } /* end of function NTMOrganizeIndexFile */ /////////////////////////////////////////////////////////////////////////////////////////////// /// functions for working with the additional data area following the target CLB /// /////////////////////////////////////////////////////////////////////////////////////////////// -// compute the size of the additional data for the given input -USHORT NTMComputeAddDataSize( PSZ_W pszContext, PSZ_W pszAddInfo ) +// compute the size of the additional data for the given input +USHORT NTMComputeAddDataSize(PSZ_W pszContext, PSZ_W pszAddInfo) { USHORT usLength = 0; - if ( (pszContext != NULL) && (*pszContext != 0) ) + if ((pszContext != NULL) && (*pszContext != 0)) { // length = characters in context + end delimiter + size field + IF field - usLength = usLength + (USHORT)((wcslen( pszContext) + 2) * sizeof(CHAR_W)); - } /* endif */ + usLength = usLength + (USHORT)((wcslen(pszContext) + 2) * sizeof(CHAR_W)); + } /* endif */ - if ( (pszAddInfo != NULL) && (*pszAddInfo != 0) ) + if ((pszAddInfo != NULL) && (*pszAddInfo != 0)) { // length = characters in add.info + end delimiter + size field + IF field - usLength = usLength + (USHORT)((wcslen( pszAddInfo) + 2) * sizeof(CHAR_W)); - } /* endif */ + usLength = usLength + (USHORT)((wcslen(pszAddInfo) + 2) * sizeof(CHAR_W)); + } /* endif */ - if ( usLength != 0 ) + if (usLength != 0) { // if data is available we need an area end delimiter usLength += sizeof(CHAR_W); - } /* endif */ + } /* endif */ - return( usLength ); + return (usLength); } - // check if the data is old context data or new additional data -BOOL NTMIsAddData( PSZ_W pData ) +BOOL NTMIsAddData(PSZ_W pData) { BOOL fAddData = FALSE; - PUSHORT pDataShort = (PUSHORT) pData; - if ( (*pDataShort == ADDDATA_ADDINFO_ID) || (*pDataShort == ADDDATA_CONTEXT_ID) || (*pDataShort == ADDDATA_ENDOFDATA_ID) ) + PUSHORT pDataShort = (PUSHORT)pData; + if ((*pDataShort == ADDDATA_ADDINFO_ID) || (*pDataShort == ADDDATA_CONTEXT_ID) || (*pDataShort == ADDDATA_ENDOFDATA_ID)) { fAddData = TRUE; - } /* endif */ - return( fAddData ); -} /* end of function NTMIsAddData */ + } /* endif */ + return (fAddData); +} /* end of function NTMIsAddData */ // find specified data area in additional data, returns ptr to start of area or NULL -PSZ_W NTMFindData( PSZ_W pData, USHORT usDataID ) +PSZ_W NTMFindData(PSZ_W pData, USHORT usDataID) { - PUSHORT pDataShort = (PUSHORT) pData; - while ( (*pDataShort != 0) && (*pDataShort != ADDDATA_ENDOFDATA_ID) && (*pDataShort != usDataID) ) + PUSHORT pDataShort = (PUSHORT)pData; + while ((*pDataShort != 0) && (*pDataShort != ADDDATA_ENDOFDATA_ID) && (*pDataShort != usDataID)) { USHORT usLen = pDataShort[1]; pData += (usLen + 1); - pDataShort = (PUSHORT) pData; - } /* endwhile */ + pDataShort = (PUSHORT)pData; + } /* endwhile */ - return( (*pDataShort == usDataID) ? pData : NULL ); -} /* end of function NTMFindData */ + return ((*pDataShort == usDataID) ? pData : NULL); +} /* end of function NTMFindData */ // get length of specific data in the combined data area, returns length of data area in number of CHAR_Ws -USHORT NtmGetAddDataLen( PTMX_TARGET_CLB pCLB, USHORT usDataID ) +USHORT NtmGetAddDataLen(PTMX_TARGET_CLB pCLB, USHORT usDataID) { USHORT usLength = 0; - if ( pCLB->usAddDataLen != 0 ) // do we have additional data ? + if (pCLB->usAddDataLen != 0) // do we have additional data ? { PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); - if ( NTMIsAddData( pData ) ) + if (NTMIsAddData(pData)) { - pData = NTMFindData( pData, usDataID ); + pData = NTMFindData(pData, usDataID); - if ( pData != NULL ) + if (pData != NULL) { - PUSHORT pDataPShort = (PUSHORT) pData; + PUSHORT pDataPShort = (PUSHORT)pData; usLength = pDataPShort[1]; - } /* endif */ - } - } /* endif */ + } /* endif */ + } + } /* endif */ - return( usLength ); + return (usLength); } /* end of function NtmGetAddDataLen */ // store/combine additional data in the combined area, returns new size of combined data area or 0 in case of errors -USHORT NtmStoreAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszNewData ) +USHORT NtmStoreAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszNewData) { PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); PSZ_W pDataStart = pData; - //PSZ_W pNewDataStr = nullptr; + // PSZ_W pNewDataStr = nullptr; PUSHORT pDataShort = (PUSHORT)pData; - USHORT usNewLength = (USHORT)(wcslen( pszNewData )); + USHORT usNewLength = (USHORT)(wcslen(pszNewData)); - if ( usNewLength != 0 ) usNewLength += 1; + if (usNewLength != 0) + usNewLength += 1; - if ( pCLB->usAddDataLen != 0 ) + if (pCLB->usAddDataLen != 0) { // remove any old data for this ID - PSZ_W pOldData = NTMFindData( pData, usDataID ); - if ( pOldData != NULL ) + PSZ_W pOldData = NTMFindData(pData, usDataID); + if (pOldData != NULL) { - PUSHORT pOldDataPShort = (PUSHORT) pOldData; - PSZ_W pEndOfData = (PSZ_W)(((PBYTE)pOldData ) + pCLB->usAddDataLen - 2); - USHORT usOldLen = pOldDataPShort [1]; + PUSHORT pOldDataPShort = (PUSHORT)pOldData; + PSZ_W pEndOfData = (PSZ_W)(((PBYTE)pOldData) + pCLB->usAddDataLen - 2); + USHORT usOldLen = pOldDataPShort[1]; PSZ_W pSource = pOldData + usOldLen + 2; - PSZ_W pTarget = pOldData ; + PSZ_W pTarget = pOldData; - while ( pSource <= pEndOfData ) + while (pSource <= pEndOfData) { *pTarget++ = *pSource++; - } /* endwhile */ + } /* endwhile */ - pCLB->usAddDataLen = pCLB->usAddDataLen - ((usOldLen + 2)*sizeof(PSZ_W)); - } /* endif */ + pCLB->usAddDataLen = pCLB->usAddDataLen - ((usOldLen + 2) * sizeof(PSZ_W)); + } /* endif */ // add new data to end of data area - if ( usNewLength != 0 ) + if (usNewLength != 0) { - PSZ_W pTarget = (PSZ_W)(((PBYTE)pData) + pCLB->usAddDataLen - 1 ); + PSZ_W pTarget = (PSZ_W)(((PBYTE)pData) + pCLB->usAddDataLen - 1); PUSHORT pTargetPShort = (PUSHORT)pTarget; *pTargetPShort++ = usDataID; *pTargetPShort++ = usNewLength; pTarget++; - //pNewDataStr = pTarget; + // pNewDataStr = pTarget; pszNewData[usNewLength] = 0; - //wcsncpy( pTarget, pszNewData, usNewLength); - memcpy( pTarget, pszNewData, sizeof(pData[0])*(usNewLength) ); + // wcsncpy( pTarget, pszNewData, usNewLength); + memcpy(pTarget, pszNewData, sizeof(pData[0]) * (usNewLength)); pTarget += usNewLength; - pTargetPShort = (PUSHORT) pTarget; + pTargetPShort = (PUSHORT)pTarget; *pTargetPShort++ = ADDDATA_ENDOFDATA_ID; *pTargetPShort = 0; - pCLB->usAddDataLen += ((usNewLength + 1 ) * sizeof(pData[0])); - } /* endif */ + pCLB->usAddDataLen += ((usNewLength + 1) * sizeof(pData[0])); + } /* endif */ } - else if ( usNewLength != 0 ) + else if (usNewLength != 0) { // no additional data yet, copy new data *pDataShort++ = usDataID; *pDataShort = usNewLength; pData++; pszNewData[usNewLength] = 0; - //wcsncpy( pData, pszNewData, usNewLength); - memcpy( pData, pszNewData, sizeof(pData[0])*(usNewLength) ); - //pNewDataStr = pData; + // wcsncpy( pData, pszNewData, usNewLength); + memcpy(pData, pszNewData, sizeof(pData[0]) * (usNewLength)); + // pNewDataStr = pData; pData += usNewLength; - pDataShort = (PUSHORT) pData; + pDataShort = (PUSHORT)pData; *pDataShort++ = ADDDATA_ENDOFDATA_ID; *pDataShort = 0; - pCLB->usAddDataLen = (usNewLength + 2) * sizeof(pData[0]);// addinfo_metadana(len+type - 1 wchar) + (usNewLength = strlen + endDelim(1 wchar)) + endAddinfoDelim(1wchar) - } /* endif */ + pCLB->usAddDataLen = (usNewLength + 2) * sizeof(pData[0]); // addinfo_metadana(len+type - 1 wchar) + (usNewLength = strlen + endDelim(1 wchar)) + endAddinfoDelim(1wchar) + } /* endif */ - return( pCLB->usAddDataLen ); + return (pCLB->usAddDataLen); } /* end of function NtmStoreAddData */ // retrieve specific data from the combined data area, returns length of retrieved data (incl. string end delimiter) -USHORT NtmGetAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszBuffer, USHORT usBufSize ) +USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, std::wstring& pszBuffer) { USHORT usLength = 0; - if ( pCLB->usAddDataLen != 0 ) // do we have additional data ? + if (pCLB->usAddDataLen != 0) // do we have additional data ? { PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); - if ( NTMIsAddData( pData ) ) // is it additional data? + if (NTMIsAddData(pData)) // is it additional data? { - pData = NTMFindData( pData, usDataID ); + pData = NTMFindData(pData, usDataID); - if ( pData != NULL ) + if (pData != NULL) { PUSHORT pDataPUShort = (PUSHORT)pData; usLength = pDataPUShort[1]; pData++; - if ( usLength < usBufSize ) - { - pszBuffer[0] = 0; - //wcsncpy(pszBuffer, pData, usLength+1 ); - memcpy(pszBuffer, pData, usLength*sizeof(pData[0])); - pszBuffer[usLength+1] = 0; - //wcscpy( pszBuffer, (wchar_t *) pusData ); - } - else - { - usLength = 0; - } /* endif */ - } /* endif */ + pszBuffer.assign(pData, usLength); + } /* endif */ } else { - T5LOG(T5WARNING) <<"Add info was not saved properly"; - } /* endif */ - } /* endif */ + T5LOG(T5WARNING) << "Add info was not saved properly"; + } /* endif */ + } /* endif */ - return( usLength ); + return (usLength); } /* end of function NtmGetAddData */ - // find a string in a specific data area -BOOL NtmFindInAddData( PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch ) +BOOL NtmFindInAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch) { USHORT usLength = 0; BOOL fFound = FALSE; - if ( pCLB->usAddDataLen != 0 ) // do we have additional data ? + if (pCLB->usAddDataLen != 0) // do we have additional data ? { PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); - if ( NTMIsAddData( pData ) ) // is it additional data? + if (NTMIsAddData(pData)) // is it additional data? { - pData = NTMFindData( pData, usDataID ); + pData = NTMFindData(pData, usDataID); - if ( pData != NULL ) + if (pData != NULL) { - PUSHORT pDataUShort = (PUSHORT) pData; + PUSHORT pDataUShort = (PUSHORT)pData; usLength = pDataUShort[1]; - if ( usLength > 0 ) + if (usLength > 0) { - if ( wcsstr( (pData + 1), pszSearch ) != NULL ) + if (wcsstr((pData + 1), pszSearch) != NULL) { fFound = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ + } /* endif */ + } /* endif */ + } /* endif */ } else { - T5LOG(T5WARNING) <<"Add info was not saved properly"; - } /* endif */ - } /* endif */ + T5LOG(T5WARNING) << "Add info was not saved properly"; + } /* endif */ + } /* endif */ - return( fFound ); + return (fFound); } /* end of function NtmFindInAddData */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 5d872f32..a4eb038d 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -231,13 +231,10 @@ int EqfMemory::RewriteCompactTable(){ if ( bCompactChanged ) { usRc = TmBtree.EQFNTMUpdate( COMPACT_KEY, - bCompact, MAX_COMPACT_SIZE-1 ); + bCompact.data(), MAX_COMPACT_SIZE-1 ); if ( !usRc ) { bCompactChanged = FALSE; - - //usRc = TmBtree.EQFNTMIncrUpdCounter( COMPACTAREA_UPD_COUNTER, - // &(alUpdCounter[COMPACTAREA_UPD_COUNTER]) ); } /* endif */ } /* endif */ @@ -496,8 +493,8 @@ int EqfMemory::ExtOutToOtmProposal Proposal.setMarkup( pExtOut->szTagTable ); Proposal.setTargetLanguage( pExtOut->szTargetLanguage ); Proposal.setSourceLanguage( stTmSign.szSourceLanguage); - Proposal.setAddInfo( pExtOut->szAddInfo ); - Proposal.setContext( pExtOut->szContext ); + Proposal.setAddInfo( pExtOut->szAddInfo.data() ); + Proposal.setContext( pExtOut->szContext.data() ); Proposal.setDocName( pExtOut->szLongName ); Proposal.setSegmentId( pExtOut->ulSourceSegmentId ); Proposal.setType( FlagToProposalType( pExtOut->usTranslationFlag ) ); @@ -531,8 +528,8 @@ int EqfMemory::MatchToOtmProposal pProposal->setMarkup( pMatch->szTagTable ); pProposal->setTargetLanguage( pMatch->szTargetLanguage ); pProposal->setSourceLanguage( stTmSign.szSourceLanguage); - pProposal->setAddInfo( pMatch->szAddInfo ); - pProposal->setContext( pMatch->szContext ); + pProposal->setAddInfo( pMatch->szAddInfo.data() ); + pProposal->setContext( pMatch->szContext.data() ); pProposal->setDocName( pMatch->szLongName ); pProposal->setSegmentId( pMatch->ulSegmentId ); pProposal->setType( FlagToProposalType( pMatch->usTranslationFlag ) ); diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index 1347ce04..acb656a9 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -381,11 +381,12 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem { int size = MAX_COMPACT_SIZE-1 ; //initialize and insert compact area record - memset( NewMem->bCompact, 0, size ); + //memset( NewMem->bCompact, 0, size ); + NewMem->bCompact.resize(size); ulKey = COMPACT_KEY; _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - NewMem->bCompact, size); + NewMem->bCompact.data(), size); } /* endif */ From 28cd862614c1bcd3507c46e783b0e5165ef98ca6 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 7 Apr 2025 13:58:23 +0300 Subject: [PATCH 03/62] * Refactoring lookup tables, switching to smart pointers * refactoring creating tms and tm tables --- include/EQFQDAMI.H | 42 +- include/lowlevelotmdatastructs.h | 96 ++- include/tm.h | 56 +- source/RestAPI/OtmMemoryServiceWorker.cpp | 58 +- source/RestAPI/OtmMemoryServiceWorker.h | 14 +- source/RestAPI/ProxygenServer.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp | 8 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp | 280 ++------- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 98 +-- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 58 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 7 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 146 ++--- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 211 ++----- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 6 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 286 ++++----- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 140 +++-- .../core/EqfMemoryPlugin/EqfMemory.cpp | 38 +- .../core/EqfMemoryPlugin/EqfMemoryPlugin.cpp | 11 - source/opentm2/core/requestdata.cpp | 66 +- source/opentm2/core/tm.cpp | 56 +- source/opentm2/core/utilities/EQFDICT.cpp | 565 ++++++------------ source/opentm2/core/utilities/UtlAlloc.cpp | 36 -- 22 files changed, 737 insertions(+), 1543 deletions(-) diff --git a/include/EQFQDAMI.H b/include/EQFQDAMI.H index 8b8570f9..a6472ddb 100755 --- a/include/EQFQDAMI.H +++ b/include/EQFQDAMI.H @@ -897,21 +897,21 @@ typedef struct _DICTTIME SHORT QDAMDictFirstLocal ( PBTREE, PWCHAR, PULONG, std::vector& ); - SHORT QDAMInsertKey_V3( PBTREE, PBTREEBUFFER_V3, PWCHAR, RECPARAM, RECPARAM ); - SHORT QDAMFindParent_V3(PBTREE, PBTREEBUFFER_V3, PUSHORT ); + SHORT QDAMInsertKey_V3( PBTREE, std::shared_ptr&, PWCHAR, RECPARAM, RECPARAM ); + SHORT QDAMFindParent_V3(PBTREE, std::shared_ptr&, PUSHORT ); - SHORT QDAMFindChild_V3 ( PBTREE, PWCHAR, USHORT, PBTREEBUFFER_V3 * ); + SHORT QDAMFindChild_V3 ( PBTREE, PWCHAR, USHORT, std::shared_ptr& ); SHORT QDAMChangeKey_V3(PBTREE, USHORT, PWCHAR, PWCHAR ); - SHORT QDAMNewRecord_V3( PBTREE, PBTREEBUFFER_V3 *, RECTYPE ); + SHORT QDAMNewRecord_V3( PBTREE, std::shared_ptr&, RECTYPE ); SHORT QDAMDeleteDataFromBuffer ( PBTREE, RECPARAM ); - SHORT QDAMSplitNode_V3( PBTREE, PBTREEBUFFER_V3 *, PWCHAR ); + SHORT QDAMSplitNode_V3( PBTREE, std::shared_ptr&, PWCHAR ); SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, PWCHAR, PULONG ); SHORT QDAMGetszData_V3 ( PBTREE, RECPARAM, std::vector&, CHAR ); - BOOL QDAMDeleteKey_V3( PBTREE, PBTREEBUFFER_V3, PCHAR); + BOOL QDAMDeleteKey_V3( PBTREE, std::shared_ptr&, PCHAR); SHORT QDAMDestroy ( PBTREE ); SHORT QDAMKeyCompare ( PVOID, PVOID, PVOID ); @@ -922,23 +922,23 @@ typedef struct _DICTTIME SHORT QDAMHeaderFirst ( PBTREE, PRECPARAM, PRECPARAM, PRECPARAM ); SHORT QDAMHeaderNext ( PBTREE, PRECPARAM, PRECPARAM,PRECPARAM ); SHORT QDAMUnTerseData ( PBTREE, std::vector& , ULONG ); - SHORT QDAMValidateIndex_V3( PBTREE, PBTREEBUFFER_V3 * ); + SHORT QDAMValidateIndex_V3( PBTREE, std::shared_ptr& ); - PSZ_W QDAMGetszKey_V3 ( PBTREEBUFFER_V3, USHORT, USHORT ); - RECPARAM QDAMGetrecData_V3 ( PBTREEBUFFER_V3, SHORT, USHORT ); - VOID QDAMSetrecData_V3 ( PBTREEBUFFER_V3, SHORT, RECPARAM, USHORT ); - RECPARAM QDAMGetrecKey_V3 ( PBTREEBUFFER_V3, SHORT ); - VOID QDAMReArrangeKRec_V3 ( PBTREE, PBTREEBUFFER_V3 ); - VOID QDAMCopyKeyTo_V3( PBTREEBUFFER_V3, SHORT, PBTREEBUFFER_V3, SHORT, USHORT ); - VOID QDAMCopyDataTo_V3( PBTREEBUFFER_V3, SHORT, PBTREEBUFFER_V3, SHORT, USHORT ); - SHORT QDAMLastEntry_V3( PBTREE, PBTREEBUFFER_V3 * ); - SHORT QDAMFirstEntry_V3( PBTREE, PBTREEBUFFER_V3 * ); - BOOL QDAMFetchFromIndexList_V3( PBTREE, PBTREEBUFFER_V3, USHORT ); - VOID QDAMAddToIndexList_V3( PBTREE, PBTREEBUFFER_V3 ); + PSZ_W QDAMGetszKey_V3 ( std::shared_ptr&, USHORT, USHORT ); + RECPARAM QDAMGetrecData_V3 ( std::shared_ptr&, SHORT, USHORT ); + VOID QDAMSetrecData_V3 ( std::shared_ptr&, SHORT, RECPARAM, USHORT ); + RECPARAM QDAMGetrecKey_V3 ( std::shared_ptr&, SHORT ); + VOID QDAMReArrangeKRec_V3 ( PBTREE, std::shared_ptr& ); + VOID QDAMCopyKeyTo_V3( std::shared_ptr&, SHORT, std::shared_ptr&, SHORT, USHORT ); + VOID QDAMCopyDataTo_V3( std::shared_ptr&, SHORT, std::shared_ptr&, SHORT, USHORT ); + SHORT QDAMLastEntry_V3( PBTREE, std::shared_ptr& ); + SHORT QDAMFirstEntry_V3( PBTREE, std::shared_ptr& ); + BOOL QDAMFetchFromIndexList_V3( PBTREE, std::shared_ptr&, USHORT ); + VOID QDAMAddToIndexList_V3( PBTREE, std::shared_ptr& ); SHORT QDAMAllocKeyRecords ( PBTREE, USHORT ); - SHORT QDAMLocSubstr_V3( PBTREE, PBTREEBUFFER_V3, PWCHAR, PBYTE, PULONG, PBYTE, PULONG ); + SHORT QDAMLocSubstr_V3( PBTREE, std::shared_ptr&, PWCHAR, PBYTE, PULONG, PBYTE, PULONG ); @@ -1331,8 +1331,8 @@ BOOL QDAMMatchWild PUCHAR pPattern ); -ULONG QDAMComputeCheckSum_V3 (PBTREEBUFFER_V3 pRecord ); -BOOL QDAMCheckCheckSum_V3( PBTREEBUFFER_V3 pRecord, SHORT sLocation ); +ULONG QDAMComputeCheckSum_V3 (std::shared_ptr& pRecord ); +BOOL QDAMCheckCheckSum_V3( std::shared_ptr& pRecord, SHORT sLocation ); // get nect dictionary of an association list diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index a0c1f3b8..d7b1fbbf 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -396,7 +396,7 @@ typedef struct _BTREERECORD_V3 /* BTREEBUFFER is the format of the buffers when read in to the buffer */ /* cache. It maps a record number and its properties to its contents */ /*****************************************************************************/ -typedef struct _BTREEBUFFER_V3 +struct BTREEBUFFER { USHORT usRecordNumber=0; // index of rec in buffer BOOL fLocked=0; // Is the record locked ? @@ -404,12 +404,14 @@ typedef struct _BTREEBUFFER_V3 SHORT sUsed=0; // buffer used count ULONG ulCheckSum=0; // CheckSum of contents data BTREERECORD_V3 contents; // data from disk -} BTREEBUFFER_V3, *PBTREEBUFFER_V3; + + BTREEBUFFER(){memset(this, 0, sizeof(*this));} +}; typedef struct _BTREEINDEX_V3 { struct _BTREEINDEX_V3 * pNext; // point to next index buffer - BTREEBUFFER_V3 btreeBuffer; // data from disk + BTREEBUFFER btreeBuffer; // data from disk } BTREEINDEX_V3, * PBTREEINDEX_V3; @@ -425,12 +427,6 @@ typedef struct _BTREEINDEX_V3 typedef SHORT _PFN_QDAMCOMPARE( PVOID, PVOID, PVOID ); typedef _PFN_QDAMCOMPARE *PFN_QDAMCOMPARE; -typedef struct _LOOKUPENTRY_V3 -{ - PBTREEBUFFER_V3 pBuffer; // Pointer to BTREEBUFFER - ULONG ulAccessCounter; -} LOOKUPENTRY_V3, *PLOOKUPENTRY_V3; - typedef LHANDLE HTM; @@ -471,6 +467,7 @@ struct TMX_TABLE_OLD{ }; typedef TMX_TABLE_OLD * PTMX_TABLE_OLD; + struct TMX_TABLE { //ULONG ulAllocSize = 0; @@ -487,8 +484,8 @@ struct TMX_TABLE }; typedef TMX_TABLE * PTMX_TABLE; -//signature structure +int tmxTableToBuffer(PTMX_TABLE pTable, std::vector& buff); @@ -505,18 +502,14 @@ typedef struct _TMX_LONGNAME_TABLE_ENTRY //table structure for long document table -typedef struct TMX_LONGNAME_TABLE +struct TMX_LONGNAME_TABLE { std::vector pszBuffer; // buffer for names and IDs - //ULONG ulBufUsed; // number of bytes used in buffer - //ULONG ulBufSize; // size of buffer in bytes - //ULONG ulTableSize; // table size (# entries) - //ULONG ulEntries; // number of entries in table std::vector stTableEntry; // dyn. array of table entries void restoreStrPointersAfterRealloc(){ for(auto& entry: stTableEntry){//during insertion to the pszBuffer, it could be reallocated, so we need to update poiners - entry.pszLongName = (PSZ) (pszBuffer[entry.longNameStartOffset]); + entry.pszLongName = (PSZ) &(pszBuffer[entry.longNameStartOffset]); }; } @@ -577,7 +570,7 @@ struct BTREE ushort usFreeKeyBuffer=0; // index of buffer to use ushort usFreeDataBuffer=0; // first data buffer chain ushort usFirstDataBuffer=0; // first data buffer - BTREEBUFFER_V3 BTreeTempBuffer_V3; // temporary V3 buffer + std::shared_ptr BTreeTempBuffer_V3; // temporary V3 buffer long lTime=0; // time of last update/open ushort usVersion=0; // version identification... char chEQF[7]; // The type of file @@ -591,7 +584,6 @@ struct BTREE char sCurrentIndex=0; // current sequence array ushort usCurrentRecord=0; // current sequence record - ushort usDictNum=0; // index in global structure //wchar_t chHeadTerm[HEADTERM_SIZE]; // last active head term bool fLock =false; // head term is locked unsigned long ulNum=0; @@ -600,7 +592,7 @@ struct BTREE //new fields FileBuffer fb; std::vector TempRecord; - std::vector LookupTable_V3; // Pointer to lookup-table + std::map> LookupTable; // Pointer to lookup-table BTREE(){ TempRecord = std::vector(MAXDATASIZE, 0); @@ -612,7 +604,7 @@ struct BTREE std::fill(std::begin(chCaseMap), std::end(chCaseMap), 0); //std::fill(std::begin(chCollate), std::end(chCollate), 0); std::fill(std::begin(chEQF), std::end(chEQF), '\0'); - memset(&BTreeTempBuffer_V3, 0, sizeof(BTreeTempBuffer_V3)); + BTreeTempBuffer_V3 = std::make_shared(); //std::fill(std::begin(chHeadTerm), std::end(chHeadTerm), L'\0'); } @@ -626,9 +618,9 @@ struct BTREE int initLookupTable(); int resetLookupTable(); - void freeLookupTable(); - int checkLookupTableAndRealocate(int number); - int allocateNewLookupTableBuffer(int number, PLOOKUPENTRY_V3& pLEntry, PBTREEBUFFER_V3& pBuffer); + void freeLookupTable(); + int allocateNewLookupTableBuffer(int number,std::shared_ptr& pBuffer); + int deallocateLookupTableBuffer(int number); //+----------------------------------------------------------------------------+ // Internal function @@ -695,11 +687,7 @@ struct BTREE // for consistency reasons, the little overhead was used... //+----------------------------------------------------------------------------+ - SHORT EQFNTMSign - ( - PCHAR pUserData, // pointer to user data - PUSHORT pusLen // length of user data - ); + SHORT EQFNTMSign(TMX_SIGN& ); VOID QDAMTerseInit (unsigned char* ); @@ -739,8 +727,7 @@ struct BTREE EQFNTMInsert ( PULONG pulKey, // pointer to key - PBYTE pData, // pointer to user data - ULONG ulLen // length of user data + std::vector& pData // pointer to user data ); //+----------------------------------------------------------------------------+ @@ -769,8 +756,7 @@ struct BTREE EQFNTMUpdate ( ULONG ulKey, // key value - PBYTE pData, // pointer to user data - ULONG ulLen // length of user data + std::vector& pData // pointer to user data ); @@ -898,24 +884,24 @@ struct BTREE // Returncode type: SHORT //+----------------------------------------------------------------------------+ - VOID QDAMUpdateList_V3( PBTREEBUFFER_V3 ); + VOID QDAMUpdateList_V3( std::shared_ptr& ); - VOID QDAMFreeFromList_V3(PRECPARAM ,PBTREEBUFFER_V3 ); - SHORT QDAMFreeRecord_V3( PBTREEBUFFER_V3 pRecord, RECTYPE recType/* data or key record*/); + VOID QDAMFreeFromList_V3(PRECPARAM ,std::shared_ptr& ); + SHORT QDAMFreeRecord_V3( std::shared_ptr& pRecord, RECTYPE recType/* data or key record*/); - ULONG QDAMGetrecDataLen_V3 ( PBTREEBUFFER_V3, SHORT ); + ULONG QDAMGetrecDataLen_V3 ( std::shared_ptr&, SHORT ); SHORT QDAMDeleteDataFromBuffer_V3( RECPARAM recParam); - SHORT QDAMDictUpdateLocal ( PWCHAR, PBYTE, ULONG ); + SHORT QDAMDictUpdateLocal ( PWCHAR, std::vector& ); SHORT QDAMDictExactLocal ( PWCHAR, std::vector&, USHORT ); SHORT QDAMDictCreateLocal ( TMX_SIGN*, ULONG, bool keepInRamOnly = false); - SHORT QDAMDictInsertLocal ( PWCHAR, PBYTE, ULONG ); + SHORT QDAMDictInsertLocal ( PWCHAR, std::vector& pData ); BOOL QDAMDictLockStatus ( PWCHAR ); VOID QDAMDictUpdStatus (); - SHORT QDAMFindRecord_V3( PWCHAR, PBTREEBUFFER_V3 * ); + SHORT QDAMFindRecord_V3( PWCHAR, std::shared_ptr&); //------------------------------------------------------------------------------ @@ -1024,12 +1010,12 @@ struct BTREE SHORT QDAMReadRecord_V3 ( USHORT usNumber, - PBTREEBUFFER_V3 * ppReadBuffer, + std::shared_ptr& pReadBuffer, BOOL fNewRec ); - SHORT QDAMAddToBuffer_V3( PBYTE, ULONG, PRECPARAM); + SHORT QDAMAddToBuffer_V3( std::vector&, PRECPARAM); SHORT QDAMWriteHeader (); SHORT QDAMDictUpdSignLocal @@ -1037,10 +1023,10 @@ struct BTREE PTMX_SIGN pSign // pointer to user data ); - SHORT QDAMReadRecordFromDisk_V3( USHORT, PBTREEBUFFER_V3 *, BOOL ); + SHORT QDAMReadRecordFromDisk_V3( USHORT, std::shared_ptr&, BOOL ); - SHORT QDAMWRecordToDisk_V3( PBTREEBUFFER_V3 ); - SHORT QDAMWriteRecord_V3( PBTREEBUFFER_V3 pBuffer); + SHORT QDAMWRecordToDisk_V3( std::shared_ptr& ); + SHORT QDAMWriteRecord_V3( std::shared_ptr& pBuffer); SHORT QDAMAllocKeyRecords( USHORT usNum); SHORT QDAMDictOpenLocal( SHORT sNumberOfBuffers, // number of buffers @@ -1108,13 +1094,13 @@ SHORT EQFNTMClose(); SHORT QDAMNewRecord_V3 ( - PBTREEBUFFER_V3 * ppRecord, + std::shared_ptr& pRecord, RECTYPE recType // data or key record ); SHORT QDAMInsertKey_V3 ( - PBTREEBUFFER_V3 pRecord, // record where key is to be inserted + std::shared_ptr& pRecord, // record where key is to be inserted PWCHAR pKey, RECPARAM recKey, // position/offset for key RECPARAM recData // position/offset for data @@ -1124,14 +1110,14 @@ size_t GetFileSize()const; BOOL QDAMTerseData -( - PUCHAR pData, // pointer to data - PULONG pulLen // length of the string +( + std::vector& , + PULONG pulLen // length of the string ); SHORT QDAMLocateKey_V3( - PBTREEBUFFER_V3, PWCHAR, PSHORT, SEARCHTYPE, PSHORT); + std::shared_ptr&, PWCHAR, PSHORT, SEARCHTYPE, PSHORT); };//BTREE @@ -1519,7 +1505,6 @@ typedef struct _MEM_EXPORT_IDA USHORT usRC; // error code in non-DDE batch mode USHORT usExpMode; // mode for export CHAR szConvArea[MEM_EXPORT_WORK_AREA]; // Work area for extract - PSZ pszNameList; // pointer to list of TMs being organized PSZ pszActiveName; // points to current name in pszNameList USHORT usYesToAllMode; // yes-to-all mode for replace existing files ULONG ulOemCP; // ASCII cp of system preferences language @@ -1527,8 +1512,8 @@ typedef struct _MEM_EXPORT_IDA // fields for external memory export methods CTMXExportImport* lExternalExportHandle; // handle of external memory export functions //HMODULE hmodMemExtExport; // handle of external export module/DLL - MEMEXPIMPINFO* pstMemInfo; // buffer for memory information - MEMEXPIMPSEG* pstSegment; // buffer for segment data + MEMEXPIMPINFO pstMemInfo; // buffer for memory information + MEMEXPIMPSEG pstSegment; // buffer for segment data BOOL fDataCorrupted; // TRUE = data has been corrupted during export CHAR_W szBackConvArea[MEM_EXPORT_WORK_AREA]; // Work area for conversion check @@ -1537,7 +1522,6 @@ typedef struct _MEM_EXPORT_IDA CHAR_W szSegmentBuffer[MAX_SEGMENT_SIZE]; // Buffer for segment data CHAR_W szSegmentBuffer2[MAX_SEGMENT_SIZE]; // Buffer for preprocessed segment data int iComplete; // completion rate - CHAR szPlugin[MAX_LONGFILESPEC]; // name of memory plugin handling the current memory database int invalidXmlSegments=0; long numOfRequestedSegmentsForExport = 0; @@ -2445,13 +2429,13 @@ struct MEM_LOAD_DLG_IDA PTOKENENTRY pTokenEntry; // A pointer to token entries PTOKENENTRY pTokenEntryWork; // A work pointer to token entries - MEMEXPIMPINFO* pstMemInfo; // buffer for memory information + MEMEXPIMPINFO pstMemInfo; // buffer for memory information CTMXExportImport TMXImport; // handle of external memory import functions PLOADEDTABLE pFormatTable; // Pointer to Format Table - MEMEXPIMPSEG* pstSegment; // buffer for segment data + MEMEXPIMPSEG pstSegment; // buffer for segment data BOOL fEOF; // Indicates end of file }; diff --git a/include/tm.h b/include/tm.h index 8907e06d..34d0156f 100644 --- a/include/tm.h +++ b/include/tm.h @@ -966,8 +966,6 @@ typedef struct _TMX_SOURCE_RECORD typedef struct _TMX_TARGET_RECORD { LONG lRecordLen = 0; - // USHORT usSourceTagTable; - // USHORT usTargetTagTable; USHORT usTarget = 0; USHORT usClb = 0; void reset() @@ -1203,9 +1201,9 @@ class TagReplacer TagReplacer() { - sourceTagList.resize(50); - targetTagList.resize(50); - requestTagList.resize(50); + sourceTagList.reserve(50); + targetTagList.reserve(50); + requestTagList.reserve(50); reset(); } }; @@ -1921,6 +1919,10 @@ class EqfMemory //: public TMX_CLB USHORT Load(); + int sign(){ + return TmBtree.EQFNTMSign(stTmSign); + } + /*! \brief Info structure for an open translation memory */ // typedef struct _OPENEDMEMORY @@ -1972,13 +1974,14 @@ class EqfMemory //: public TMX_CLB TMX_EXT_OUT_W *pTmDelOut // ptr to output struct ); - USHORT FindTargetAndDelete(PTMX_RECORD pTmRecord, + USHORT FindTargetAndDelete( + std::vector& TmRecord, OtmProposal &TmDel, TMX_EXT_OUT_W *pTmExtOut, PULONG pulKey); USHORT FindTargetByKeyAndDelete( - PTMX_RECORD pTmRecord, + std::vector& pTmRecord, OtmProposal &pTmDel, PTMX_SENTENCE pSentence, TMX_EXT_OUT_W *pTmExtOut, @@ -2219,23 +2222,19 @@ class EqfMemory //: public TMX_CLB \returns 0 or error code in case of errors */ - int setGlobalMemoryOptions( - void *pvGlobalMemoryOptions); std::string strLastError; int iLastError = 0; - void *pvGlobalMemoryOptions = nullptr; // pointert to global memory options to be used for global memory proposals public: - char szPlugin[256]; // name of the plugin controlling this memory - char szName[256]; // name of the memory - char szDescription[256]; // description of the memory - char szFullPath[256]; // full path to memory file(s) (if applicable only) - char szFullDataFilePath[256]; // full path to memory file(s) (if applicable only) - char szFullIndexFilePath[256]; // full path to memory file(s) (if applicable only) - char szSourceLanguage[MAX_LANG_LENGTH + 1]; // memory source language - char szOwner[256]; // ID of the memory owner - char szDescrMemoryType[256]; // descriptive name of the memory type + std::string szName; // name of the memory + std::string szDescription; // description of the memory + std::string szFullPath; // full path to memory file(s) (if applicable only) + std::string szFullDataFilePath; // full path to memory file(s) (if applicable only) + std::string szFullIndexFilePath; // full path to memory file(s) (if applicable only) + std::string szSourceLanguage; // memory source language + std::string szOwner; // ID of the memory owner + std::string szDescrMemoryType; // descriptive name of the memory type unsigned long ulSize = 0; // size of the memory BOOL fEnabled = 0; // memory-is-enabled flag @@ -2298,7 +2297,7 @@ class EqfMemory //: public TMX_CLB \param pszMarkup markup table name \returns original or modified error return code */ - int handleError(int iRC, char *pszMemName, char *pszMarkup = "OTMXUXLF"); + int handleError(int iRC, const char *pszMemName, char *pszMarkup = "OTMXUXLF"); /// newTM code public: @@ -3524,10 +3523,11 @@ VOID BuildVotes(PTMX_SENTENCE); VOID Vote(PTMX_TERM_TOKEN, PTMX_SENTENCE, USHORT); USHORT CheckCompactArea(PTMX_SENTENCE, EqfMemory *); -VOID FillTmRecord(PTMX_SENTENCE, - PTMX_RECORD, PTMX_TARGET_CLB, USHORT); +VOID FillTmRecord(PTMX_SENTENCE, + std::vector& TmRecord, // filled tm record returned + PTMX_TARGET_CLB, USHORT); -USHORT DetermineTmRecord(EqfMemory *, PTMX_SENTENCE, PULONG); +USHORT DetermineTmRecord(EqfMemory *, PTMX_SENTENCE, std::vector& pulSids ); VOID FillTargetRecord(PTMX_SENTENCE, // PTMX_TAGTABLE_RECORD, PSZ_W, USHORT, PTMX_TARGET_RECORD *, PTMX_TARGET_CLB); @@ -3656,7 +3656,6 @@ struct MEM_ORGANIZE_IDA PDDEMEMORG pDDEMemOrg; // ptr to batch memory organize data USHORT usRC; // return code / error code LONG NextTask; // next task in non-DDE batch mode - PSZ pszNameList; // pointer to list of TMs being organized PSZ pszActiveName; // points to current name in pszNameList CHAR szBuffer[2048]; // general purpose buffer BOOL fFirstGet; // TRUE = this is the first get access @@ -3743,11 +3742,7 @@ class EqfMemoryPlugin : public OtmPlugin unsigned short usAccessMode, bool fReorganizeOnly); - /*! \brief Close a memory - \param pMemory pointer to memory object - */ - int closeMemory( - EqfMemory *pMemory); + /*! \brief set description of a memory \param pszName name of the memory @@ -3947,8 +3942,7 @@ class TMManager \param pMemory pointer to the memory object being closes \returns 0 when successful or a error return code */ - int closeMemory( - EqfMemory *pMemory); + /*! \brief Show error message for the last error \param pszPlugin plugin-name or NULL if not available or memory object name is used diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index c6218e68..022d15d9 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -60,42 +60,11 @@ OtmMemoryServiceWorker* OtmMemoryServiceWorker::getInstance() return &_instance; } -/*! \brief Data area for the processing of the openMemory function -*/ -typedef struct _OPENMEMORYDATA -{ - char szMemory[260]; - wchar_t szError[512]; -} OPENMEMORYDATA, *POPENMEMORYDATA; - -/*! \brief Data area for the processing of the createMemory function -*/ -typedef struct _CLOSEMEMORYDATA -{ - char szMemory[260]; - wchar_t szError[512]; -} CLOSEMEMORYDATA, *PCLOSEMEMORYDATA; - - - - - -int OtmMemoryServiceWorker::init(){ - return verifyAPISession(); -} - - /*! \brief Verify OpenTM2 API session - \returns 0 if successful or an error code in case of failures - */ -int OtmMemoryServiceWorker::verifyAPISession() -{ - if ( this->hSession != 0 ){ - return( 0 ); - } +OtmMemoryServiceWorker::OtmMemoryServiceWorker(){ T5LOG( T5DEBUG) << "Initializing API Session"; this->iLastRC = EqfStartSession( &(this->hSession) ); - T5LOG( T5DEBUG) << "[verifyAPISession] EqfStartSession ret: " << this->iLastRC; + T5LOG( T5DEBUG) << "EqfStartSession ret: " << this->iLastRC; if ( this->iLastRC != 0 ) { T5LOG(T5ERROR) << "OpenTM2 API session could not be started, the return code is" << this->iLastRC ; @@ -105,15 +74,29 @@ int OtmMemoryServiceWorker::verifyAPISession() xercesc::XMLPlatformUtils::Initialize(); } catch (const xercesc::XMLException& toCatch) { - toCatch; + //toCatch; T5LOG(T5ERROR) << ":: cant init Xercesc"; - return( ERROR_NOT_READY ); + throw; + //return( ERROR_NOT_READY ); } } - - return( this->iLastRC ); } +/*! \brief Data area for the processing of the openMemory function +*/ +typedef struct _OPENMEMORYDATA +{ + char szMemory[260]; + wchar_t szError[512]; +} OPENMEMORYDATA, *POPENMEMORYDATA; + +/*! \brief Data area for the processing of the createMemory function +*/ +typedef struct _CLOSEMEMORYDATA +{ + char szMemory[260]; + wchar_t szError[512]; +} CLOSEMEMORYDATA, *PCLOSEMEMORYDATA; @@ -441,7 +424,6 @@ void reorganizeMemoryProcess( void* pvData ) case MEM_END_ORGANIZE: pData->fComplete = TRUE; EQFMemOrganizeEnd( pCommArea); - if ( pRIDA->pszNameList ) UtlAlloc( (PVOID *)pRIDA->pszNameList, 0L, 0L, NOMSG ); break; } /* endswitch */ _rc_ = pRIDA->usRC; diff --git a/source/RestAPI/OtmMemoryServiceWorker.h b/source/RestAPI/OtmMemoryServiceWorker.h index 9280c0cc..efa2371f 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.h +++ b/source/RestAPI/OtmMemoryServiceWorker.h @@ -30,31 +30,25 @@ #define SHUTDOWN_CALLED_FROM_MAIN -2 class OtmMemoryServiceWorker /*! \brief This class provides translation memory related services*/ -{ +{ + OtmMemoryServiceWorker(); public: /*! \brief This static method returns a pointer to the OtmMemoryService object. The first call of the method creates the OtmMemoryService instance. */ static OtmMemoryServiceWorker* getInstance(); + - int init(); - - /* change the status of a memory in our memory list at the end of an import \param pszMemory name of the memory \param */ void importDone( char *pszMemory, int iRC, char *pszError ); - //std::unique_ptr replaceString(std::wstring&& src_data, std::wstring&& trg_data, std::wstring&& req_data, int* rc); - /*! \brief Verify OpenTM2 API session - \returns 0 if successful or an error code in case of failures - */ - int verifyAPISession(); - /*! \brief OpenTM2 API session handle*/ HSESSION hSession = 0; + /*! \brief Last return code*/ int iLastRC; diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index 8b597cfb..bb357d96 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -465,7 +465,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { // Start HTTPServer mainloop in a separate thread std::thread t([&]() { server.start(); }); - OtmMemoryServiceWorker::getInstance()->init(); + OtmMemoryServiceWorker::getInstance();//to init instance std::string runDate = getTimeStr(); Properties::GetInstance()->set_anyway(KEY_RUN_DATE, runDate); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index c6dda165..3947b05f 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -98,7 +98,6 @@ USHORT EQFMemOrganizeStart // Close the Translation memory which should be organized. if ( pRIDA->pMemTemp != NULL ) { - TMManager::GetInstance()->closeMemory( pRIDA->pMemTemp.get() ); pRIDA->pMemTemp = NULL; std::string strMsg; @@ -311,7 +310,7 @@ VOID EQFMemOrganizeProcess // everything in the termination process was ok // end message only in GUI single organize mode - if ( !pRIDA->fBatch && !pRIDA->pszNameList) + if ( !pRIDA->fBatch) { CHAR szNumber1[10]; // buffer for character string CHAR szNumber2[10]; // buffer for character string @@ -378,15 +377,14 @@ VOID EQFMemOrganizeEnd pRIDA->pMem->TmBtree.fb.data = std::move(pRIDA->pMemTemp->TmBtree.fb.data); //get signature record and add to control block pRIDA->pMem->TmBtree.resetLookupTable(); - USHORT usLen = sizeof( TMX_SIGN ); - USHORT usRc = pRIDA->pMem->TmBtree.EQFNTMSign((PCHAR) &(pRIDA->pMem->stTmSign), &usLen ); + USHORT usRc = pRIDA->pMem->sign(); pRIDA->pMem->TmBtree.fb.Flush(true); //FilesystemHelper::DeleteFile(FilesystemHelper::GetTmiPath(pRIDA->pMem->szName)); pRIDA->pMem->InBtree.fb.data = std::move(pRIDA->pMemTemp->InBtree.fb.data); pRIDA->pMem->InBtree.resetLookupTable(); - usLen = sizeof( TMX_SIGN ); + //usLen = sizeof( TMX_SIGN ); //usRc = pRIDA->pMem->InBtree.EQFNTMSign((PCHAR) &(pRIDA->pMem->stTmSign), &usLen ); pRIDA->pMem->InBtree.fb.Flush(true); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp index 0dcf7750..363c9837 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp @@ -134,7 +134,6 @@ USHORT EQFMemExportStart( PPROCESSCOMMAREA pCommArea, //--- Return codes are for testing purposes only if ( pExportIDA->pMem != NULL) { - TMManager::GetInstance()->closeMemory( pExportIDA->pMem ); pExportIDA->pMem = NULL; } /* endif */ pExportIDA->invalidXmlSegments = 0; @@ -264,14 +263,8 @@ USHORT EQFMemExportProcess ( PPROCESSCOMMAREA pCommArea, //--- set the progress indicator to 100 percent pCommArea->usComplete = 100; - if ( !pExportIDA->pszNameList ) - { - pReplString[2] = strcpy(szNumber, std::to_string( pExportIDA->ulSegmentCounter ).c_str()); - T5LOG( T5INFO) <<"EQFMemExportProcess::MESSAGE_MEM_EXPORT_COMPLETED"; - //T5LOG( T5ERROR) <<" MESSAGE_MEM_EXPORT_COMPLETED, MB_OK, 3, - // &(pReplString[0]), EQF_INFO, - // pExportIDA->hwndErrMsg ); - } /* endif */ + pReplString[2] = strcpy(szNumber, std::to_string( pExportIDA->ulSegmentCounter ).c_str()); + T5LOG( T5INFO) <<"EQFMemExportProcess::MESSAGE_MEM_EXPORT_COMPLETED"; //--- Issue message WM_EQF_MEMEXPORT_END pExportIDA->NextTask = MEM_END_EXPORT; @@ -382,10 +375,6 @@ USHORT EQFMemExportEnd ( PPROCESSCOMMAREA pCommArea, delete pExportIDA->lExternalExportHandle ; } - - if ( pExportIDA->pstMemInfo ) UtlAlloc( (PVOID *)&pExportIDA->pstMemInfo, 0L, 0L, NOMSG ); - if ( pExportIDA->pstSegment ) UtlAlloc( (PVOID *)&pExportIDA->pstSegment, 0L, 0L, NOMSG ); - //if ( pExportIDA->hmodMemExtExport ) DosFreeModule( &(pExportIDA->hmodMemExtExport) ); } //--- Check if the termination was due to a @@ -436,195 +425,7 @@ T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 12 WinPostMsg( pExport #endif //TO_BE_REPLACED_WITH_LINUX_CODE } /* endif */ - // continue with next TM if any - if ( pExportIDA->pszNameList ) { - pExportIDA->pszActiveName += strlen(pExportIDA->pszActiveName) + 1; - - if ( *pExportIDA->pszActiveName ) - { - BOOL fStartExport = FALSE; - TMManager *pFactory = TMManager::GetInstance(); - - do - { - pFactory->splitObjName( pExportIDA->pszActiveName, pExportIDA->szPlugin, sizeof(pExportIDA->szPlugin), pExportIDA->szMemName, sizeof(pExportIDA->szMemName) ); - - // build outfile name and get overwrite permission for existing files - UtlSplitFnameFromPath( pExportIDA->ControlsIda.szPathContent ); - strcat( pExportIDA->ControlsIda.szPathContent, BACKSLASH_STR ); - strcat( pExportIDA->ControlsIda.szPathContent, pExportIDA->szMemName ); - strcat( pExportIDA->ControlsIda.szPathContent, ".TMX" ); - - - if ( UtlFileExist( pExportIDA->ControlsIda.szPathContent ) ) - { - if ( pExportIDA->usYesToAllMode == MBID_EQF_YESTOALL ) - { - // we are already in yes-to-all mode so do nothing and continue export - fStartExport = TRUE; - } - else - { - USHORT usMBCode; - PSZ pszParm = pExportIDA->ControlsIda.szPathContent; - usMBCode = UtlError( ERROR_FILE_EXISTS_ALREADY, MB_EQF_YESTOALL, 1, - &pszParm, EQF_QUERY ); - if ( usMBCode == MBID_EQF_YESTOALL ) - { - // switch to yes-to-all mode and start export - pExportIDA->usYesToAllMode = MBID_EQF_YESTOALL; - fStartExport = TRUE; - } - else if ( usMBCode == MBID_CANCEL ) - { - // stop all further processing - *pExportIDA->pszActiveName = EOS; - } - else if ( usMBCode == MBID_NO ) - { - // skip this file by removing it from the name list - PSZ pszCurrent = pExportIDA->pszActiveName; - PSZ pszNext = pszCurrent + strlen(pszCurrent) + 1; - while ( *pszNext ) - { - while ( *pszNext ) *pszCurrent++ = *pszNext++; // copy string - *pszCurrent++ = *pszNext++; // copy string enddelimiter - } /* endwhile */ - *pszCurrent = EOS; // add end-of-list delimiter - } - else - { - // continue export - fStartExport = TRUE; - } /* endif */ - } /* endif */ - } - else - { - // output file does not exist, continue export - fStartExport = TRUE; - } /* endif */ - } while ( *pExportIDA->pszActiveName && !fStartExport ); - } /* endif */ - - - if ( pExportIDA->pszActiveName[0] ) - { - // prepare export of next TM - } - else - { - // show termination message - if ( SHORT1FROMMP2( mp2 ) == 0 ) - { - PSZ pszTMNames = NULL; // buffer for TM name list - - // compute size of required buffer - PSZ pszNext = pExportIDA->pszNameList; - int iSize = 5; - while ( *pszNext ) - { - int iLen = strlen(pszNext) + 1; - iSize += iLen + 5; - pszNext += iLen; - } /* endwhile */ - - // allocate buffer - UtlAlloc( (PVOID *)&pszTMNames, 0L, iSize, ERROR_STORAGE ); - - // fill buffer with TM names - if ( pszTMNames ) - { - PSZ pszNext = pExportIDA->pszNameList; - PSZ pszCurPos = pszTMNames; - while ( *pszNext ) - { - int iNameLen = 0; - - // skip any plugin name in the memory name - PSZ pszNamePos = strchr( pszNext, ':' ); - pszNamePos = (pszNamePos == NULL) ? pszNext : pszNamePos + 1; - - iNameLen = strlen( pszNamePos ); - *pszCurPos++ = '\"'; - strcpy( pszCurPos, pszNamePos ); - pszNext = pszNamePos + iNameLen + 1; - pszCurPos += iNameLen; - *pszCurPos++ = '\"'; - if ( *pszNext ) - { - // add delimiters for following TM name - *pszCurPos++ = ','; - *pszCurPos++ = ' '; - } /* endif */ - } /* endwhile */ - } /* endif */ - UtlError ( MESSAGE_MEM_EXP_COMPLETED_MUL, MB_OK, 1, &pszTMNames, EQF_INFO ); - if ( pszTMNames ) UtlAlloc( (PVOID *)&pszTMNames, 0L, 0L, NOMSG ); - } /* endif */ - } /* endif */ - } /* endif */ - - if ( pExportIDA->pszNameList && pExportIDA->pszActiveName[0] ) - { - BOOL fIsNew = FALSE; - BOOL fOK = TRUE; - int iRC = 0; - - // restart export with active TM - pExportIDA->fFirstExtract = TRUE; - pExportIDA->fEOF = FALSE; - pExportIDA->ulSegmentCounter = 0L; - pExportIDA->hFile = NULLHANDLE; - - TMManager *pFactory = TMManager::GetInstance(); - T5LOG(T5FATAL) << "TEMPORARY_COMMENTED CODE pExportIDA->pMem = pFactory->openMemory( pExportIDA->szPlugin, pExportIDA->szMemName, EXCLUSIVE, &iRC );"; - #ifdef TEMPORARY_COMMENTED - pExportIDA->pMem = pFactory->openMemory( pExportIDA->szPlugin, pExportIDA->szMemName, EXCLUSIVE, &iRC ); - #endif - if ( pExportIDA->pMem == NULL ) - { - pFactory->showLastError( pExportIDA->szPlugin, pExportIDA->szMemName, pExportIDA->pMem, pExportIDA->hwndErrMsg ); - fOK = FALSE; - } - - if ( fOK ) - { - USHORT usDosRc, usAction; - - - usDosRc = UtlOpen( pExportIDA->ControlsIda.szPathContent, &(pExportIDA->hFile), &usAction, - 0L, FILE_NORMAL, FILE_TRUNCATE | FILE_CREATE, - (USHORT)(OPEN_ACCESS_WRITEONLY | OPEN_SHARE_DENYWRITE | OPEN_FLAGS_WRITE_THROUGH), - 0L, TRUE ); - if ( usDosRc ) - { - pExportIDA->hFile = NULLHANDLE; - fOK = FALSE; - if ( pExportIDA->pMem != NULL) - { - TMManager::GetInstance()->closeMemory( pExportIDA->pMem ); - pExportIDA->pMem = NULL; - } /* endif */ - } /* endif */ - } /* endif */ - - if ( !fOK ) - { - // stop processing - pExportIDA->pszActiveName[0] = EOS; - } /* endif */ - } /* endif */ - - if ( pExportIDA->pszNameList && pExportIDA->pszActiveName[0] ) - { - // restart TM export - } - else - { - //--- Free data areas - if ( pExportIDA->pszNameList ) UtlAlloc( (PVOID *)&pExportIDA->pszNameList, 0L, 0L, NOMSG ); if ( (hwndErrMsg == HWND_FUNCIF) && pExportIDA->fDataCorrupted ) { @@ -723,15 +524,6 @@ USHORT MemExportStart( PPROCESSCOMMAREA pCommArea, if ( (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF) ) { - // allocate memory and segment data area - if ( !UtlAlloc( (PVOID *)&pExportIDA->pstMemInfo, 0L, sizeof(MEMEXPIMPINFO), ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING,ERROR_NOT_ENOUGH_MEMORY); - } - else if ( !UtlAlloc( (PVOID *)&pExportIDA->pstSegment, 0L, sizeof(MEMEXPIMPSEG), ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ // load external memory export module EqfPluginWrapper::init(); @@ -739,30 +531,30 @@ USHORT MemExportStart( PPROCESSCOMMAREA pCommArea, // call start entry point if ( !usRc ) { - memset( pExportIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); + memset( &pExportIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); // get name of first markup table if ( pExportIDA->pMem->getNumOfMarkupNames() >= 1 ) { - pExportIDA->pMem->getMarkupName( 0, pExportIDA->pstMemInfo->szFormat, sizeof(pExportIDA->pstMemInfo->szFormat) ); + pExportIDA->pMem->getMarkupName( 0, pExportIDA->pstMemInfo.szFormat, sizeof(pExportIDA->pstMemInfo.szFormat) ); } /* end */ - strcpy( pExportIDA->pstMemInfo->szName, pExportIDA->szMemName ); - pExportIDA->pstMemInfo->responseHandler = pCommArea->responseHandler; - pExportIDA->pMem->getDescription( pExportIDA->pstMemInfo->szDescription, sizeof(pExportIDA->pstMemInfo->szDescription) ); - pExportIDA->pMem->getSourceLanguage( pExportIDA->pstMemInfo->szSourceLang, sizeof(pExportIDA->pstMemInfo->szSourceLang) ); - pExportIDA->pstMemInfo->fUTF16 = (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) ; - pExportIDA->pstMemInfo->fNoCRLF = (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF); + strcpy( pExportIDA->pstMemInfo.szName, pExportIDA->szMemName ); + pExportIDA->pstMemInfo.responseHandler = pCommArea->responseHandler; + pExportIDA->pMem->getDescription( pExportIDA->pstMemInfo.szDescription, sizeof(pExportIDA->pstMemInfo.szDescription) ); + pExportIDA->pMem->getSourceLanguage( pExportIDA->pstMemInfo.szSourceLang, sizeof(pExportIDA->pstMemInfo.szSourceLang) ); + pExportIDA->pstMemInfo.fUTF16 = (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) ; + pExportIDA->pstMemInfo.fNoCRLF = (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF); - T5LOG( T5INFO) <<"MemExportStart::calling external function, mem name = " << pExportIDA->pstMemInfo->szName << "; source lang = "<< pExportIDA->pstMemInfo->szSourceLang<< - "; markup = " << pExportIDA->pstMemInfo->szFormat; + T5LOG( T5INFO) <<"MemExportStart::calling external function, mem name = " << pExportIDA->pstMemInfo.szName << "; source lang = "<< pExportIDA->pstMemInfo.szSourceLang<< + "; markup = " << pExportIDA->pstMemInfo.szFormat; //usRc = EXTMEMEXPORTSTART( ); { pExportIDA->lExternalExportHandle = new CTMXExportImport; pExportIDA->lExternalExportHandle->m_xw.bufQueue = pCommArea->pBufQueue; - pExportIDA->lExternalExportHandle->WriteHeader( pExportIDA->ControlsIda.szPathContent, pExportIDA->pstMemInfo ); + pExportIDA->lExternalExportHandle->WriteHeader( pExportIDA->ControlsIda.szPathContent, &pExportIDA->pstMemInfo ); } } /* endif */ } @@ -893,37 +685,37 @@ USHORT MemExportProcess ( PMEM_EXPORT_IDA pExportIDA ) // pointer to the export // reset first extract flag (in non-TMX mode this is done in MemExportWriteFile) pExportIDA->fFirstExtract = FALSE; - memset( pExportIDA->pstSegment, 0, sizeof(MEMEXPIMPSEG) ); + memset( &pExportIDA->pstSegment, 0, sizeof(MEMEXPIMPSEG) ); - pExportIDA->pstSegment->fValid = TRUE; - pExportIDA->pstSegment->usTranslationFlag = 0; - if ( pExportIDA->pProposal->getType() == OtmProposal::eptMachine ) pExportIDA->pstSegment->usTranslationFlag = 1; - if ( pExportIDA->pProposal->getType() == OtmProposal::eptGlobalMemory ) pExportIDA->pstSegment->usTranslationFlag = 2; + pExportIDA->pstSegment.fValid = TRUE; + pExportIDA->pstSegment.usTranslationFlag = 0; + if ( pExportIDA->pProposal->getType() == OtmProposal::eptMachine ) pExportIDA->pstSegment.usTranslationFlag = 1; + if ( pExportIDA->pProposal->getType() == OtmProposal::eptGlobalMemory ) pExportIDA->pstSegment.usTranslationFlag = 2; - pExportIDA->pstSegment->lSegNum = pExportIDA->pProposal->getSegmentId(); - pExportIDA->pstSegment->lTime = pExportIDA->pProposal->getUpdateTime(); - //pExportIDA->pMem->getSourceLanguage( pExportIDA->pstSegment->szSourceLang, sizeof(pExportIDA->pstSegment->szSourceLang) ); - pExportIDA->pProposal->getOriginalSourceLanguage( pExportIDA->pstSegment->szSourceLang, sizeof(pExportIDA->pstSegment->szSourceLang) ); + pExportIDA->pstSegment.lSegNum = pExportIDA->pProposal->getSegmentId(); + pExportIDA->pstSegment.lTime = pExportIDA->pProposal->getUpdateTime(); + //pExportIDA->pMem->getSourceLanguage( pExportIDA->pstSegment.szSourceLang, sizeof(pExportIDA->pstSegment.szSourceLang) ); + pExportIDA->pProposal->getOriginalSourceLanguage( pExportIDA->pstSegment.szSourceLang, sizeof(pExportIDA->pstSegment.szSourceLang) ); - pExportIDA->pProposal->getTargetLanguage( pExportIDA->pstSegment->szTargetLang, sizeof(pExportIDA->pstSegment->szTargetLang) ); - pExportIDA->pProposal->getAuthor( pExportIDA->pstSegment->szAuthor, sizeof(pExportIDA->pstSegment->szAuthor) ); - pExportIDA->pProposal->getMarkup( pExportIDA->pstSegment->szFormat, sizeof(pExportIDA->pstSegment->szFormat) ); - pExportIDA->pProposal->getDocName( pExportIDA->pstSegment->szDocument, sizeof(pExportIDA->pstSegment->szDocument) ); + pExportIDA->pProposal->getTargetLanguage( pExportIDA->pstSegment.szTargetLang, sizeof(pExportIDA->pstSegment.szTargetLang) ); + pExportIDA->pProposal->getAuthor( pExportIDA->pstSegment.szAuthor, sizeof(pExportIDA->pstSegment.szAuthor) ); + pExportIDA->pProposal->getMarkup( pExportIDA->pstSegment.szFormat, sizeof(pExportIDA->pstSegment.szFormat) ); + pExportIDA->pProposal->getDocName( pExportIDA->pstSegment.szDocument, sizeof(pExportIDA->pstSegment.szDocument) ); - pExportIDA->pProposal->getSource( pExportIDA->pstSegment->szSource, sizeof(pExportIDA->pstSegment->szSource) / sizeof(CHAR_W) ); - pExportIDA->pProposal->getTarget( pExportIDA->pstSegment->szTarget, sizeof(pExportIDA->pstSegment->szTarget) / sizeof(CHAR_W) ); - pExportIDA->pProposal->getContext( pExportIDA->pstSegment->szContext, sizeof(pExportIDA->pstSegment->szContext) / sizeof(CHAR_W) ); - pExportIDA->pProposal->getAddInfo( pExportIDA->pstSegment->szAddInfo, sizeof(pExportIDA->pstSegment->szAddInfo) / sizeof(CHAR_W) ); - pExportIDA->pProposal->getInternalKey( pExportIDA->pstSegment->szInternalKey, sizeof(pExportIDA->pstSegment->szInternalKey) ); + pExportIDA->pProposal->getSource( pExportIDA->pstSegment.szSource, sizeof(pExportIDA->pstSegment.szSource) / sizeof(CHAR_W) ); + pExportIDA->pProposal->getTarget( pExportIDA->pstSegment.szTarget, sizeof(pExportIDA->pstSegment.szTarget) / sizeof(CHAR_W) ); + pExportIDA->pProposal->getContext( pExportIDA->pstSegment.szContext, sizeof(pExportIDA->pstSegment.szContext) / sizeof(CHAR_W) ); + pExportIDA->pProposal->getAddInfo( pExportIDA->pstSegment.szAddInfo, sizeof(pExportIDA->pstSegment.szAddInfo) / sizeof(CHAR_W) ); + pExportIDA->pProposal->getInternalKey( pExportIDA->pstSegment.szInternalKey, sizeof(pExportIDA->pstSegment.szInternalKey) ); auto ll = T5Logger::GetInstance()->suppressLogging(); - bool fValidXml = IsValidXml( pExportIDA->pstSegment->szSource); + bool fValidXml = IsValidXml( pExportIDA->pstSegment.szSource); if(fValidXml){ - fValidXml = IsValidXml( pExportIDA->pstSegment->szTarget); + fValidXml = IsValidXml( pExportIDA->pstSegment.szTarget); T5Logger::GetInstance()->desuppressLogging(ll); if(fValidXml){ - usRc = pExportIDA->lExternalExportHandle->WriteSegment( pExportIDA->pstSegment ); + usRc = pExportIDA->lExternalExportHandle->WriteSegment( &pExportIDA->pstSegment ); if(!usRc){ pExportIDA->segmentsExported++; if(pExportIDA->numOfRequestedSegmentsForExport > 0 && @@ -968,7 +760,7 @@ USHORT MemExportProcess ( PMEM_EXPORT_IDA pExportIDA ) // pointer to the export //--- An unrecoverable memory DB error occurred TMManager *pFactory = TMManager::GetInstance(); usRc = MEM_DB_ERROR; - pFactory->showLastError( pExportIDA->szPlugin, pExportIDA->szMemName, pExportIDA->pMem, pExportIDA->hwndErrMsg ); + pFactory->showLastError( "", pExportIDA->szMemName, pExportIDA->pMem, pExportIDA->hwndErrMsg ); if ( pExportIDA->fBatch ) { pExportIDA->usRC = UtlGetDDEErrorCode( pExportIDA->hwndErrMsg ); @@ -1722,8 +1514,6 @@ USHORT FCTDATA::MemFuncPrepExport { if ( pIDA->pMem != NULL) { - TMManager *pFactory = TMManager::GetInstance(); - pFactory->closeMemory( pIDA->pMem ); pIDA->pMem = NULL; } /* endif */ if ( pIDA->hFile ) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 5508ff05..512c18df 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -242,18 +242,7 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, { fOK = FALSE; } - else - { - /* - // Create full path to the memory database format table - Properties::GetInstance()->get_value( KEY_OTM_DIR, pLIDA->szFullFtabPath, MAX_EQF_PATH ); - strcat(pLIDA->szFullFtabPath, "/TABLE/"); - strcat(pLIDA->szFullFtabPath, MEM_FORMAT_TABLE); - - // Load the format table and allocate storage for it - fOK = ( TALoadTagTableHwnd( MEM_FORMAT_TABLE, &pLIDA->pFormatTable, - TRUE, TRUE, pLIDA->hwndErrMsg ) == NO_ERROR);//*/ - } /* endif */ + } /* endif */ // If fOK. Allocate storage for the tokenlist and move address in @@ -286,10 +275,6 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // special handling for import in TMX format if ( fOK && ((pLIDA->usImpMode == MEM_FORMAT_TMX) || (pLIDA->usImpMode == MEM_FORMAT_TMX_TRADOS)) ) { - // allocate memory and segment data area - fOK = UtlAlloc( (PVOID *)&pLIDA->pstMemInfo, 0L, sizeof(MEMEXPIMPINFO), ERROR_STORAGE ); - if ( fOK ) fOK = UtlAlloc( (PVOID *)&pLIDA->pstSegment, 0L, sizeof(MEMEXPIMPSEG), ERROR_STORAGE ); - // call start entry point if ( fOK ) { @@ -302,23 +287,23 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, pLIDA->hFile = NULL; } /* endif */ - memset( pLIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); - strcpy( pLIDA->pstMemInfo->szName, pLIDA->szMemName ); - pLIDA->mem->getSourceLanguage( pLIDA->pstMemInfo->szSourceLang, sizeof(pLIDA->pstMemInfo->szSourceLang) ); + memset( &pLIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); + strcpy( pLIDA->pstMemInfo.szName, pLIDA->szMemName ); + pLIDA->mem->getSourceLanguage( pLIDA->pstMemInfo.szSourceLang, sizeof(pLIDA->pstMemInfo.szSourceLang) ); if ( pLIDA->usImpMode == MEM_FORMAT_TMX_TRADOS ) { - pLIDA->pstMemInfo->fCleanRTF = TRUE; - pLIDA->pstMemInfo->fTagsInCurlyBracesOnly = TRUE; + pLIDA->pstMemInfo.fCleanRTF = TRUE; + pLIDA->pstMemInfo.fTagsInCurlyBracesOnly = TRUE; } memset(&(pLIDA->TMXImport), 0 ,sizeof(pLIDA->TMXImport)); - fOK = (pLIDA->TMXImport.StartImport(pLIDA->szFilePath, pLIDA->pstMemInfo, pImportData ) == 0); + fOK = (pLIDA->TMXImport.StartImport(pLIDA->szFilePath, &pLIDA->pstMemInfo, pImportData ) == 0); pLIDA->mem->getSourceLanguage( szMemSourceLang, sizeof(szMemSourceLang) ); // check if memory source lanuage matchs source language of imported file - if ( pLIDA->pstMemInfo->szSourceLang[0] && (strcmp( pLIDA->pstMemInfo->szSourceLang, szMemSourceLang ) != 0) ) + if ( pLIDA->pstMemInfo.szSourceLang[0] && (strcmp( pLIDA->pstMemInfo.szSourceLang, szMemSourceLang ) != 0) ) { - pReplAddr[0] = pLIDA->pstMemInfo->szSourceLang; + pReplAddr[0] = pLIDA->pstMemInfo.szSourceLang; pReplAddr[1] = szMemSourceLang; T5LOG(T5INFO) << "::ERROR_MEM_DIFFERENT_SOURCE_LANG_IMPORT::" << pReplAddr[0] << "; szMemSourceLang = " << pReplAddr[1]; @@ -328,7 +313,7 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // update memory description with description of imported memory if ( fOK ) { - pLIDA->mem->setDescription( pLIDA->pstMemInfo->szDescription ); + pLIDA->mem->setDescription( pLIDA->pstMemInfo.szDescription ); } /* endif */ } /* endif */ @@ -339,30 +324,25 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // special handling for import in XLIFF MT format if ( fOK && ((pLIDA->usImpMode == MEM_FORMAT_XLIFF_MT)) ) { - // allocate memory and segment data area - fOK = UtlAlloc( (PVOID *)&pLIDA->pstMemInfo, 0L, sizeof(MEMEXPIMPINFO), ERROR_STORAGE ); - if ( fOK ) fOK = UtlAlloc( (PVOID *)&pLIDA->pstSegment, 0L, sizeof(MEMEXPIMPSEG), ERROR_STORAGE ); - - // call start entry point if ( fOK ) { char szMemSourceLang[MAX_LANG_LENGTH]; - memset( pLIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); - strcpy( pLIDA->pstMemInfo->szName, pLIDA->szMemName ); - pLIDA->mem->getSourceLanguage( pLIDA->pstMemInfo->szSourceLang, sizeof(pLIDA->pstMemInfo->szSourceLang) ); + memset( &pLIDA->pstMemInfo, 0, sizeof(MEMEXPIMPINFO) ); + strcpy( pLIDA->pstMemInfo.szName, pLIDA->szMemName ); + pLIDA->mem->getSourceLanguage( pLIDA->pstMemInfo.szSourceLang, sizeof(pLIDA->pstMemInfo.szSourceLang) ); - fOK = (pLIDA->TMXImport.StartImport(pLIDA->szFilePath, pLIDA->pstMemInfo, pImportData ) == 0); + fOK = (pLIDA->TMXImport.StartImport(pLIDA->szFilePath, &pLIDA->pstMemInfo, pImportData ) == 0); pLIDA->mem->getSourceLanguage( szMemSourceLang, sizeof(szMemSourceLang) ); // check if memory source lanuage matchs source language of imported file - if ( pLIDA->pstMemInfo->szSourceLang[0] && (strcmp( pLIDA->pstMemInfo->szSourceLang, szMemSourceLang ) != 0) ) + if ( pLIDA->pstMemInfo.szSourceLang[0] && (strcmp( pLIDA->pstMemInfo.szSourceLang, szMemSourceLang ) != 0) ) { USHORT usMBCode; - pReplAddr[0] = pLIDA->pstMemInfo->szSourceLang; + pReplAddr[0] = pLIDA->pstMemInfo.szSourceLang; pReplAddr[1] = szMemSourceLang; T5LOG(T5INFO) << "::ERROR_MEM_DIFFERENT_SOURCE_LANG_IMPORT::", pReplAddr[0] ; @@ -371,7 +351,7 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // update memory description with description of imported memory if ( fOK ) { - pLIDA->mem->setDescription( pLIDA->pstMemInfo->szDescription ); + pLIDA->mem->setDescription( pLIDA->pstMemInfo.szDescription ); } /* endif */ } /* endif */ @@ -504,9 +484,9 @@ static USHORT MemLoadProcess( MEM_LOAD_DLG_IDA* pLIDA, ImportStatusDetails* usRc = pLIDA->TMXImport.ImportNext( MEMINSERTSEGMENT, (LONG)pLIDA, pImportData); // handle any error conditions - if ( pLIDA->pstMemInfo->fError ) + if ( pLIDA->pstMemInfo.fError ) { - PSZ pszMsg = pLIDA->pstMemInfo->szError; + PSZ pszMsg = pLIDA->pstMemInfo.szError; T5LOG(T5ERROR) << "::ERROR_MEMIMP_ERROR::MEM_SEG_SYN_ERR::" << pszMsg ; LOG_AND_SET_RC(usRc, T5INFO, MEM_SEG_SYN_ERR); } /* endif */ @@ -590,7 +570,6 @@ USHORT MemFuncPrepImport /* it will be freed here. */ /*******************************************************************/ memset(pLoadIDA, 0, sizeof(*pLoadIDA)); - //fOK = UtlAllocHwnd( (PVOID *)&pLoadIDA, 0L, (LONG)sizeof(MEM_LOAD_DLG_IDA), ERROR_STORAGE, HWND_FUNCIF ); if ( fOK ) { // use def.target lang. from system preferences of TM pLoadIDA->ulOemCP = 1; @@ -663,30 +642,12 @@ USHORT MemFuncPrepImport /*******************************************************************/ if ( fOK ) { - strcpy( pLoadIDA->szMemName, pData->mem.get()->szName ); + strcpy( pLoadIDA->szMemName, pData->mem.get()->szName.c_str() ); }else if(!fOkFailHandled){ fOkFailHandled = true; T5LOG(T5ERROR) << "MemFuncPrepImport::prepare segment match ID prefix fails"; } /* endif */ - /*******************************************************************/ - /* Open the TM */ - /*******************************************************************/ - // - //if ( fOK ) - //{ - // int iRC = 0; - // TMManager *pFactory = TMManager::GetInstance(); - // pLoadIDA->pMem = pFactory->openMemory( NULL, pszMemName, EXCLUSIVE, &iRC ); - // if ( iRC != 0 || pLoadIDA->pMem == nullptr ) - // { - // fOK = FALSE; - // pFactory->showLastError( NULL, pszMemName, pLoadIDA->pMem, (HWND)HWND_FUNCIF ); - // } /* end */ - //}else if(!fOkFailHandled){ - // fOkFailHandled = true; - // T5LOG(T5ERROR) << "MemFuncPrepImport::Check if there is a TM with the given name fails"; - //} /* endif */ /*******************************************************************/ /* Check if input file exists */ @@ -721,12 +682,6 @@ USHORT MemFuncPrepImport }else{ pLoadIDA->hFile = nullptr; } - //if ( usDosRc ) //--- error from open - //{ - // //--- Reset file handle to NULL and set fOk to FALSE - // pLoadIDA->hFile = NULLHANDLE; - // fOK = FALSE; - //}/* endif */ }else if(!fOkFailHandled){ fOkFailHandled = true; T5LOG(T5ERROR) << "MemFuncPrepImport::Check if input file exists fails"; @@ -752,17 +707,10 @@ USHORT MemFuncPrepImport T5LOG(T5ERROR) << "MemFuncPrepImport::fOk in false, making Cleanup"; if ( pLoadIDA ) { - //if ( pLoadIDA->pMem ) - //{ - // TMManager *pFactory = TMManager::GetInstance(); - // pFactory->closeMemory( pLoadIDA->pMem ); - // pLoadIDA->pMem = NULL; - //} /* endif */ if ( pLoadIDA->hFile && pLoadIDA->hFile ) { UtlClose( pLoadIDA->hFile, FALSE ); } /* endif */ - //UtlAlloc( (PVOID *) &pLoadIDA, 0L, 0L, NOMSG ); } /* endif */ } /* endif */ @@ -819,7 +767,6 @@ USHORT MemFuncImportProcess { usPhase = MEM_IMPORT_TASK; } /* endif */ - //UtlAlloc( (PVOID *) &pLIDA, 0L, 0L, NOMSG ); break; case MEM_IMPORT_TASK: @@ -851,7 +798,7 @@ USHORT MemFuncImportProcess pLIDA = (MEM_LOAD_DLG_IDA*)&pData->MemLoadIda; if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); pData->usMemLoadRC = usRC = usRc? usRc : UtlQueryUShort( QS_LASTERRORMSGID ); - strncpy(pData->szError, pLIDA->pstMemInfo->szError, sizeof(pData->szError)); + strncpy(pData->szError, pLIDA->pstMemInfo.szError, sizeof(pData->szError)); usPhase = MEM_END_IMPORT; break; } /* end switch */ @@ -866,9 +813,6 @@ USHORT MemFuncImportProcess ( pLIDA->usImpMode == MEM_FORMAT_XLIFF_MT ) ) { pLIDA->TMXImport.WriteEnd(); - - if ( pLIDA->pstMemInfo ) UtlAlloc( (PVOID *)&pLIDA->pstMemInfo, 0L, 0L, NOMSG ); - if ( pLIDA->pstSegment) UtlAlloc( (PVOID *)&pLIDA->pstSegment, 0L, 0L, NOMSG ); } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index e76ac1fa..076962ce 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -59,32 +59,18 @@ USHORT EqfMemory::TmtXDelSegm TMX_EXT_OUT_W* pTmDelOut //ptr to output struct ) { - //TMX_SENTENCE Sentence(std::make_unique(pTmDelIn->szSource)); // ptr to sentence structure - ULONG ulKey; // tm record key - BOOL fOK; // success indicator + BOOL fOK = true; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound = 0; // compact area hits ULONG ulLen = 0; // length indication CHAR szString[MAX_EQF_PATH]; // character string - PULONG pulSids = NULL; // ptr to sentence ids + std::vector pulSids; // ptr to sentence ids PULONG pulSidStart = NULL; // ptr to sentence ids std::vector pTmRecord; // space for user data pTmRecord.resize(TMX_REC_SIZE); - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *) &(pulSids), 0L, (LONG)(MAX_INDEX_LEN * sizeof(ULONG)), - NOMSG ); - if ( fOK ) - pulSidStart = pulSids; - } /* endif */ - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - + if ( !usRc ) { //build tag table path @@ -124,10 +110,9 @@ USHORT EqfMemory::TmtXDelSegm usRc = DetermineTmRecord( this, TmDelIn.pInputSentence, pulSids ); if ( !usRc ) { - while ( *pulSids ) + ulLen = TMX_REC_SIZE; + for (ULONG ulKey : pulSids ) { - ulKey = *pulSids; - ulLen = TMX_REC_SIZE; usRc = TmBtree.EQFNTMGet(ulKey, //tm record key pTmRecord ); //length @@ -135,17 +120,12 @@ USHORT EqfMemory::TmtXDelSegm { //find target record and delete, if the target record was the //only target in the tm record, delete the entire record - usRc = FindTargetAndDelete( toTmxRecord(pTmRecord), TmDelIn, pTmDelOut, &ulKey ); - if ( usRc == SEG_NOT_FOUND ) - { - //get next tm record - pulSids++; - } - else + usRc = FindTargetAndDelete( pTmRecord, TmDelIn, pTmDelOut, &ulKey ); + if ( usRc != SEG_NOT_FOUND ) { //target record was found and deleted or an error occured //so don't try other sids - *pulSids = 0; + break; } /* endif */ } /* endif */ } /* endwhile */ @@ -157,8 +137,6 @@ USHORT EqfMemory::TmtXDelSegm } /* endif */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); pTmDelOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); pTmDelOut->stPrefixOut.usTmtXRc = usRc; @@ -199,7 +177,7 @@ USHORT EqfMemory::TmtXDelSegmByKey //find target record and delete, if the target record was the //only target in the tm record, delete the entire record - usRc = FindTargetByKeyAndDelete( toTmxRecord(pTmRecord), + usRc = FindTargetByKeyAndDelete( pTmRecord, TmDelIn, pSentence, pTmDelOut, &ulKey ); } /* endif */ } /* endif */ @@ -265,11 +243,12 @@ USHORT EqfMemory::TmtXDelSegmByKey //| get next sentence key | // ----------------------------------------------------------------------------+ USHORT EqfMemory::FindTargetAndDelete( - PTMX_RECORD pTmRecord, + std::vector& TmRecord, OtmProposal& TmDel, TMX_EXT_OUT_W * pTmExtOut, PULONG pulKey ) { + PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); BOOL fOK = FALSE; //success indicator BOOL fStop = FALSE; //indicates whether to leave loop or not PBYTE pByte; //position ptr @@ -448,10 +427,8 @@ USHORT EqfMemory::FindTargetAndDelete( /* by a new translation, we will not remove */ /* the key (only get rid of any target data) */ /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( - *pulKey, - (PBYTE)pTmRecord, - RECLEN(pTmRecord) ); + usRc = TmBtree.EQFNTMUpdate( *pulKey, TmRecord); + //RECLEN(pTmRecord) ); } /* endif */ //leave while loop @@ -493,13 +470,14 @@ USHORT EqfMemory::FindTargetAndDelete( } USHORT EqfMemory::FindTargetByKeyAndDelete( - PTMX_RECORD pTmRecord, + std::vector& TmRecord, OtmProposal& TmDel, PTMX_SENTENCE pSentence, TMX_EXT_OUT_W * pTmExtOut, PULONG pulKey ) { USHORT usRc = 0; + PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record PTMX_TARGET_CLB pTargetClb; //ptr to target CLB @@ -674,10 +652,8 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( /* by a new translation, we will not remove */ /* the key (only get rid of any target data) */ /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( - *pulKey, - (PBYTE)pTmRecord, - RECLEN(pTmRecord) ); + usRc = TmBtree.EQFNTMUpdate( *pulKey, TmRecord); + ///RECLEN(pTmRecord) ); } /* endif */ if ( ! usRc ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index e653e2e1..d506b900 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -65,7 +65,7 @@ USHORT EqfMemory::TmtXExtract ) { USHORT usRc = NO_ERROR; //return code - BOOL fOK; //success indicator + BOOL fOK = true; //success indicator PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record ULONG ulStartKey; //start tm key @@ -79,11 +79,6 @@ USHORT EqfMemory::TmtXExtract //allocate 32K for tm record //allocate pString - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - if ( !usRc ) { SHORT sRetries = MAX_RETRY_COUNT; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index b50c69dd..a85b7ea3 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -643,8 +643,7 @@ USHORT GetExactMatch ) { BOOL fOK = TRUE; //success indicator - PULONG pulSids = NULL; //ptr to sentence ids - PULONG pulSidStart = NULL; //ptr to sentence ids + std::vector ulSids; //ptr to sentence ids USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator ULONG ulKey; //tm record key @@ -653,97 +652,74 @@ USHORT GetExactMatch ULONG ulStrippedParm = pGetIn->ulParm & ~(GET_RESPECTCRLF | GET_IGNORE_PATH | GET_NO_GENERICREPLACE | GET_ALWAYS_WITH_TAGS | GET_IGNORE_COMMENT); + ulSids.reserve(MAX_INDEX_LEN+5); - - //allocate for sentence ids - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pulSids), 0L, - (LONG)((MAX_INDEX_LEN + 5) * sizeof(ULONG)), - NOMSG ); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + usRc = DetermineTmRecord( pTmClb, pSentence, ulSids ); + if ( usRc == NO_ERROR ) { - pulSidStart = pulSids; - usRc = DetermineTmRecord( pTmClb, pSentence, pulSids ); - if ( usRc == NO_ERROR ) + //get tm record(s) + for(ULONG ulKey: ulSids) { - //get tm record(s) - while ( (*pulSids) && (usRc == NO_ERROR) ) - { - ulKey = *pulSids; - T5LOG( T5INFO) << "GetExactMatch: EQFNTMGET of record #" <TmBtree.EQFNTMGet( ulKey, pTmRecord ); + T5LOG( T5INFO) << "GetExactMatch: EQFNTMGET of record #" <TmBtree.EQFNTMGet( ulKey, pTmRecord ); - if ( usRc == NO_ERROR ) + if ( usRc == NO_ERROR ) + { + //compare tm record data with data passed in the get in structure + T5LOG( T5INFO) << "GetExactMatch: ExactTest of record " << ulKey; + usRc = ExactTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pSentence, + pstMatchTable, &usMatchEntries, ulKey ); + T5LOG( T5INFO) << "GetExactMatch: ExactTest found " << usMatchEntries << " matching entries" ; + } /* endif */ + if ( usRc == NO_ERROR ) + { + //nr of matches found + (*pusMatchesFound) = usMatchEntries; + }else{ + break; + } + } /* endwhile */ + /****************************************************************/ + /* check if we are only looking for an exact match and we really*/ + /* found more than one... */ + /* store the availability of additional flags for later use */ + /****************************************************************/ + SetAvailFlags( pTmGetOut, *pusMatchesFound); + + if ( !usRc && (*pusMatchesFound > 1 ) && (ulStrippedParm & GET_EXACT) ) + { + /*************************************************************/ + /* there is one exact exact match available - forget about */ + /* any 'normal' matches */ + /*************************************************************/ + if ( pstMatchTable->usMatchLevel >= EQUAL_EQUAL ) { - //compare tm record data with data passed in the get in structure - T5LOG( T5INFO) << "GetExactMatch: ExactTest of record " << ulKey; - usRc = ExactTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pSentence, - pstMatchTable, &usMatchEntries, ulKey ); - T5LOG( T5INFO) << "GetExactMatch: ExactTest found " << usMatchEntries << " matching entries" ; - - if ( !usRc ) + if ( pstMatchTable->usMatchLevel == (EQUAL_EQUAL + 2) ) { - //nr of matches found - (*pusMatchesFound) = usMatchEntries; - - //get next tm record - pulSids++; + /*********************************************************/ + /* we have an exact exact ... */ + /*********************************************************/ + *pusMatchesFound = 1; + } + else + { + /*********************************************************/ + /* we have two or more exact one's. */ + /*********************************************************/ + USHORT i; // index value + for ( i=1; i< *pusMatchesFound ;i++ ) + { + if ((pstMatchTable+i)->usMatchLevel < EQUAL_EQUAL) + { + *pusMatchesFound = i; + break; + } /* endif */ + } /* endfor */ } /* endif */ } /* endif */ - } /* endwhile */ - /****************************************************************/ - /* check if we are only looking for an exact match and we really*/ - /* found more than one... */ - /* store the availability of additional flags for later use */ - /****************************************************************/ - SetAvailFlags( pTmGetOut, *pusMatchesFound); - - if ( !usRc && (*pusMatchesFound > 1 ) && (ulStrippedParm & GET_EXACT) ) - { - /*************************************************************/ - /* there is one exact exact match available - forget about */ - /* any 'normal' matches */ - /*************************************************************/ - if ( pstMatchTable->usMatchLevel >= EQUAL_EQUAL ) - { - if ( pstMatchTable->usMatchLevel == (EQUAL_EQUAL + 2) ) - { - /*********************************************************/ - /* we have an exact exact ... */ - /*********************************************************/ - *pusMatchesFound = 1; - } - else - { - /*********************************************************/ - /* we have two or more exact one's. */ - /*********************************************************/ - USHORT i; // index value - for ( i=1; i< *pusMatchesFound ;i++ ) - { - if ((pstMatchTable+i)->usMatchLevel < EQUAL_EQUAL) - { - *pusMatchesFound = i; - break; - } /* endif */ - } /* endfor */ - } /* endif */ - } /* endif */ - } /* endif */ } /* endif */ } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); - - if ( usRc ) - { - ERREVENT2( GETEXACTMATCH_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ return( usRc ); } @@ -1615,7 +1591,6 @@ BOOL TMFuzzynessEx // allocate required buffers fOK = UtlAlloc((PVOID *)&pInBuf, 0L, 64000L, NOMSG ); - //if ( fOK ) fOK = UtlAlloc((PVOID *)&pTokBuf, 0L, 64000 /*(LONG)TOK_BUFFER_SIZE*/, NOMSG ); // load tag table if ( fOK ) @@ -1642,7 +1617,6 @@ BOOL TMFuzzynessEx if ( pFuzzyTgt ) UtlAlloc( (PVOID *)&pFuzzyTgt, 0L, 0L, NOMSG ); if ( pFuzzyTok ) UtlAlloc( (PVOID *)&pFuzzyTok, 0L, 0L, NOMSG ); if ( pInBuf ) UtlAlloc( (PVOID *)&pInBuf, 0L, 0L, NOMSG ); - //if ( pTokBuf ) UtlAlloc( (PVOID *)&pTokBuf, 0L, 0L, NOMSG ); if ( pTable ) TAFreeTagTable( pTable ); if(*pusWords > *pusDiffs){ @@ -2997,14 +2971,14 @@ USHORT FillMatchEntry pTempMatch = pTempStart; //end criteria are all sentence ids in index key or only one - //sentence id left in pulSids + //sentence id left in ulSids for ( j = 0; (j < usMaxEntries) && (lMatchEntries < usMaxSentences); j++, pIndexEntry++ ) { ULONG ulIndexKey = NTMKEY(*pIndexEntry); // reset to begin of matching table pTempMatch = pTempStart; - //before adding sentence id check if already in pulsids as the + //before adding sentence id check if already in ulSids as the //respective tm record need only be checked once #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFillMatchFill1Time) ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index db74227f..07a872a7 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -263,46 +263,16 @@ SHORT BTREE::EQFNTMOpen USHORT usOpenFlags // Read Only or Read/Write ) { - //PBTREE pBTIda; // pointer to BTRee structure - SHORT sRc = 0; // return code + SHORT sRc = 0; // return code - DEBUGEVENT( EQFNTMOPEN_LOC, FUNCENTRY_EVENT, 0 ); - - { - /*****************************************************************/ - /* check if same dictionary is already open else return index */ - /* of next free slot... */ - /*****************************************************************/ - sRc = QDAMCheckDict( (PSZ)fb.fileName .c_str()); - if ( !sRc ) - { - /***************************************************************/ - /* check if dictionary is locked */ - /***************************************************************/ - if ( ! usDictNum ) - { - SHORT RetryCount; // retry counter for in-use condition - RetryCount = MAX_RETRY_COUNT; - do - { - sRc = QDAMDictOpenLocal( 20, usOpenFlags ); - - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - } - else - { - sRc = ( usOpenFlags & ASD_LOCKED ) ? BTREE_DICT_LOCKED : sRc; - } /* endif */ - } /* endif */ - } /* endif */ - - if ( sRc != NO_ERROR ) - { - ERREVENT( EQFNTMOPEN_LOC, ERROR_EVENT, sRc ); - } /* endif */ - - DEBUGEVENT( EQFNTMOPEN_LOC, FUNCEXIT_EVENT, 0 ); + SHORT RetryCount = MAX_RETRY_COUNT; // retry counter for in-use condition + RetryCount = MAX_RETRY_COUNT; + do + { + sRc = QDAMDictOpenLocal( 20, usOpenFlags ); + + } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && + (RetryCount > 0) ); /* enddo */ return ( sRc ); } @@ -400,17 +370,17 @@ SHORT BTREE::EQFNTMOpen SHORT BTREE::EQFNTMSign ( - PCHAR pUserData, // pointer to user data - PUSHORT pusLen // length of user data + TMX_SIGN& sign // pointer to user data ) { SHORT sRc; // function return code SHORT RetryCount; // retry counter for in-use condition - + long signSize = sizeof(sign) ; RetryCount = MAX_RETRY_COUNT; do { - sRc = QDAMDictSignLocal( pUserData, pusLen ); + USHORT signSize = sizeof(sign) ; + sRc = QDAMDictSignLocal((PCHAR)&sign, &signSize); if ( sRc == BTREE_IN_USE ) { RetryCount--; @@ -437,16 +407,7 @@ ULONG BTREE::GetNumOfSavedRecords()const{ int BTREE::initLookupTable(){ - LookupTable_V3.reserve(MIN_NUMBER_OF_LOOKUP_ENTRIES); - return 0; -} - -int BTREE::checkLookupTableAndRealocate(int number){ - if ( number >= LookupTable_V3.size() ) - { - T5LOG(T5DEBUG) << "resizing LookupTable number = " << number; - LookupTable_V3.resize(number+10); - } + //LookupTable.reserve(MIN_NUMBER_OF_LOOKUP_ENTRIES); return 0; } @@ -490,8 +451,7 @@ SHORT BTREE::EQFNTMInsert ( PULONG pulKey, // pointer to key - PBYTE pData, // pointer to user data - ULONG ulLen // length of user data + std::vector& pData // pointer to user data ) { SHORT sRc = 0; // return code @@ -546,26 +506,9 @@ BTREE::EQFNTMInsert /********************************************************************/ /* call QDAMDictInsert to do the dirty work of inserting entry... */ /********************************************************************/ - if ( !sRc ) - { - SHORT RetryCount; // retry counter for in-use condition - RetryCount = MAX_RETRY_COUNT; - do - { - sRc = QDAMDictInsertLocal( (PWCHAR)pulKey, pData, ulLen ); - if ( sRc == BTREE_IN_USE ) - { - RetryCount--; - UtlWait( MAX_WAIT_TIME ); - } /* endif */ - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - } /* endif */ - if ( sRc ) - { - ERREVENT( EQFNTMINSERT_LOC, INTFUNCFAILED_EVENT, sRc ); - } /* endif */ + sRc = QDAMDictInsertLocal( (PWCHAR)pulKey, pData); + return sRc; } /* end of function EQFNTMInsert */ @@ -621,7 +564,7 @@ BTREE::EQFNTMInsert //------------------------------------------------------------------------------ VOID BTREE::QDAMUpdateList_V3 ( - PBTREEBUFFER_V3 pRecord + std::shared_ptr& pRecord ) { USHORT i; // index @@ -731,7 +674,7 @@ VOID BTREE::QDAMUpdateList_V3 VOID BTREE::QDAMFreeFromList_V3 ( PRECPARAM pRecTemp, - PBTREEBUFFER_V3 pRecord + std::shared_ptr& pRecord ) { USHORT i; // index @@ -801,7 +744,7 @@ VOID BTREE::QDAMFreeFromList_V3 //------------------------------------------------------------------------------ SHORT BTREE::QDAMFreeRecord_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, RECTYPE recType // data or key record ) { @@ -882,7 +825,7 @@ SHORT BTREE::QDAMFreeRecord_V3 //------------------------------------------------------------------------------ ULONG BTREE::QDAMGetrecDataLen_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid // key number ) { @@ -932,9 +875,9 @@ ULONG BTREE::QDAMGetrecDataLen_V3 //------------------------------------------------------------------------------ VOID QDAMCopyDataTo_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT i, - PBTREEBUFFER_V3 pNew, + std::shared_ptr& pNew, SHORT j, USHORT usVersion // version of BTREE ) @@ -1054,15 +997,15 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 SHORT sRc = 0; PUSHORT pusOffset; // offset pointer SHORT i; // index - PBTREEBUFFER_V3 pRecord; - PBTREEBUFFER_V3 pNew; // temporary record + std::shared_ptr pRecord; + std::shared_ptr pNew; // temporary record PBTREEHEADER pHeader; // pointer to header ULONG ulDataLen; // length of data USHORT usNum; // number of record // read in record recParam.usNum = recParam.usNum + usFirstDataBuffer; // adjust for offset - sRc = QDAMReadRecord_V3( recParam.usNum, &pRecord, FALSE ); + sRc = QDAMReadRecord_V3( recParam.usNum, pRecord, FALSE ); if ( ! sRc ) { BTREELOCKRECORD( pRecord ); // lock eecord. @@ -1083,7 +1026,7 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 usNum = NEXT( pRecord ); while ( !sRc && usNum ) { - sRc = QDAMReadRecord_V3( usNum, &pNew, FALSE ); + sRc = QDAMReadRecord_V3( usNum, pNew, FALSE ); if ( !sRc ) { if ( TYPE(pNew) != DATA_NEXTNODE ) @@ -1130,7 +1073,7 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 if ( !NEXT(pRecord) && (pRecord->contents.header.usWasteSize > MAXWASTESIZE) ) { // get temp record - pNew = &BTreeTempBuffer_V3; + pNew = BTreeTempBuffer_V3; pHeader = &pNew->contents.header; pHeader->usOccupied = 0; pHeader->usFilled = sizeof(BTREEHEADER ); @@ -1195,7 +1138,7 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 //------------------------------------------------------------------------------ VOID QDAMSetrecData_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid, // key number RECPARAM recData, // data pointer USHORT usVersion // version of database @@ -1269,7 +1212,7 @@ VOID QDAMSetrecData_V3 //------------------------------------------------------------------------------ RECPARAM QDAMGetrecKey_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid // key number ) { @@ -1335,8 +1278,7 @@ RECPARAM QDAMGetrecKey_V3 SHORT BTREE::QDAMDictUpdateLocal ( PCHAR_W pKey, // key string - PBYTE pUserData, // user data - ULONG ulLen // user data length + std::vector& pData ) { SHORT sRc = 0; // return code @@ -1369,10 +1311,11 @@ SHORT BTREE::QDAMDictUpdateLocal } /* endif */ { - PBTREEBUFFER_V3 pRecord = NULL; // pointer to record + std::shared_ptr pRecord; // pointer to record if ( !sRc ) { - if ( ulLen == 0 ) + //if ( ulLen == 0 ) + if(pData.empty()) { sRc = BTREE_DATA_RANGE; } @@ -1381,7 +1324,7 @@ SHORT BTREE::QDAMDictUpdateLocal //memcpy( chHeadTerm, pKey, sizeof(ULONG)); // save data QDAMDictUpdStatus(); - sRc = QDAMFindRecord_V3( pKey, &pRecord ); + sRc = QDAMFindRecord_V3( pKey, pRecord ); } /* endif */ } /* endif */ @@ -1404,13 +1347,13 @@ SHORT BTREE::QDAMDictUpdateLocal // set new data value if ( recOldKey.usNum && recOldData.usNum ) { - sRc = QDAMAddToBuffer_V3( pUserData, ulLen, &recData ); + sRc = QDAMAddToBuffer_V3( pData, &recData ); if ( !sRc ) { - if( (ulLen > TMX_REC_SIZE) && (T5Logger::GetInstance()->CheckLogLevel(T5DEBUG))){ - T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << ulLen; + if( (pData.size() > TMX_REC_SIZE) && (T5Logger::GetInstance()->CheckLogLevel(T5DEBUG))){ + T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << pData.size(); } - recData.ulLen = ulLen; + recData.ulLen = pData.size(); QDAMSetrecData_V3( pRecord, i, recData, usVersion ); sRc = QDAMWriteRecord_V3( pRecord ); } /* endif */ @@ -1467,59 +1410,45 @@ SHORT BTREE::QDAMDictUpdateLocal int BTREE::resetLookupTable(){ int rc = 0; - for (int i=0; i < LookupTable_V3.size(); i++ ) - { - if ( LookupTable_V3[i].pBuffer ) - { - UtlAlloc( (PVOID *)&(LookupTable_V3[i].pBuffer), 0L, 0L, NOMSG ); - (usNumberOfAllocatedBuffers)--; - } /* endif */ - } /* endfor */ - LookupTable_V3.resize(0); + LookupTable.clear(); usNumberOfAllocatedBuffers = 0; return rc; } -int BTREE::allocateNewLookupTableBuffer(int number, PLOOKUPENTRY_V3& pLEntry, PBTREEBUFFER_V3& pBuffer){ +int BTREE::allocateNewLookupTableBuffer(int number, std::shared_ptr& pLEntry){ /********************************************************************/ /* Allocate space for a new buffer and let pBuffer point to it */ /********************************************************************/ int sRc = 0; - //if ( !LookupTable_V3 || ( number >= usNumberOfLookupEntries )) - if(number >= LookupTable_V3.size()) + if(LookupTable.count(number)) { SET_AND_LOG(sRc,BTREE_LOOKUPTABLE_CORRUPTED); } else { - //pLEntry = LookupTable_V3 + number; - pLEntry = &LookupTable_V3[number]; - - /* Safety-Check: is the lookup table entry (ptr. to buffer) for the - record to read NULL ? */ - if ( pLEntry->pBuffer ) - { - /* ptr. isn't NULL: this should never occur */ - SET_AND_LOG(sRc, BTREE_LOOKUPTABLE_CORRUPTED); - } - else + LookupTable[number] = std::make_shared(); + pLEntry = LookupTable[number]; { /* Allocate memory for a buffer */ - UtlAlloc( (PVOID *)&(pLEntry->pBuffer), 0L, (LONG) BTREE_BUFFER_V3 , NOMSG ); - if ( pLEntry->pBuffer ) - { - (usNumberOfAllocatedBuffers)++; - pBuffer = pLEntry->pBuffer; - } - else - { - sRc = BTREE_NO_ROOM; - } /* endif */ + //pLEntry->pBuffer.resize(BTREE_REC_SIZE_V3); + (usNumberOfAllocatedBuffers)++; } /* endif */ } /* endif */ return sRc; } + +int BTREE::deallocateLookupTableBuffer(int number){ + if(LookupTable.count(number)) + { + usNumberOfAllocatedBuffers--; + LookupTable.erase(number); + return 0; + } + + return 1; +} + //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ @@ -1549,26 +1478,10 @@ SHORT BTREE::EQFNTMUpdate ( ULONG ulKey, // key value - PBYTE pData, // pointer to user data - ULONG ulLen // length of user data + std::vector& pData // pointer to user data ) { - SHORT sRc = 0; // success indicator - - SHORT RetryCount = MAX_RETRY_COUNT; // retry counter for in-use condition - do - { - sRc = QDAMDictUpdateLocal( (PSZ_W) &ulKey, pData, ulLen ); - if ( sRc == BTREE_IN_USE ) - { - RetryCount--; - UtlWait( MAX_WAIT_TIME ); - } /* endif */ - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - - - return sRc; + return QDAMDictUpdateLocal( (PSZ_W) &ulKey, pData ); } /* end of function EQFNTMUpdate */ @@ -1640,7 +1553,7 @@ SHORT BTREE::QDAMDictExactLocal T5LOG(T5FATAL) << ":: BTREE_CORRUPTED"; sRc = BTREE_CORRUPTED; }else { - PBTREEBUFFER_V3 pRecord = NULL; + std::shared_ptr pRecord; SHORT sRetries = MAX_RETRY_COUNT; do { @@ -1651,7 +1564,7 @@ SHORT BTREE::QDAMDictExactLocal if ( !sRc ) { /* Locate the Leaf node that contains the appropriate key */ - sRc = QDAMFindRecord_V3( pKey, &pRecord ); + sRc = QDAMFindRecord_V3( pKey, pRecord ); } /* endif */ if ( !sRc ) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index dd9839ef..5230eeb3 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -195,11 +195,7 @@ USHORT EqfMemory::LoadMem() USHORT usRc1 = NO_ERROR; //return value ULONG ulLen = 0; //length indicator - //if(!usRc) { - //TmBtree.fb.fileName = pTmOpenIn->stTmOpen.szDataName; - //InBtree.fb.fileName = pTmOpenIn->stTmOpen.szIndexName; - //call open function for data file usRc1 = TmBtree.EQFNTMOpen((USHORT)(usAccessMode | ASD_FORCE_WRITE) ); if ( (usRc1 == NO_ERROR) || (usRc1 == BTREE_CORRUPTED) ) @@ -207,7 +203,7 @@ USHORT EqfMemory::LoadMem() //get signature record and add to control block USHORT usLen = sizeof( TMX_SIGN ); - usRc = TmBtree.EQFNTMSign((PCHAR) &(stTmSign), &usLen ); + usRc = sign(); if ( usRc == NO_ERROR ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index ecdb0845..a6e7fa8c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -736,22 +736,15 @@ USHORT EqfMemory::AddToTm PULONG pulNewKey // sid of newly added tm record ) { - PTMX_RECORD pTmRecord = NULL; // ptr to tm record + std::vector TmRecord; // ptr to tm record PTMX_TARGET_CLB pTargetClb = NULL; // ptr to target ctl block USHORT usRc = NO_ERROR; // return code - BOOL fOK; // success indicator + BOOL fOK = true; // success indicator USHORT usAddDataLen = 0; - //allocate 32K for tm record - fOK = UtlAlloc( (PVOID *) &(pTmRecord), 0L, (LONG) TMX_REC_SIZE, NOMSG ); + usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - //allocate target control block record - if ( fOK ) - { - usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - - fOK = UtlAlloc( (PVOID *) &pTargetClb, 0L, (LONG)(sizeof(TMX_TARGET_CLB)+usAddDataLen), NOMSG ); - } /* endif */ + fOK = UtlAlloc( (PVOID *) &pTargetClb, 0L, (LONG)(sizeof(TMX_TARGET_CLB)+usAddDataLen), NOMSG ); if ( !fOK ) { @@ -766,9 +759,10 @@ USHORT EqfMemory::AddToTm { USHORT usSrcLang = 0; usRc = NTMGetIDFromName( TmProposal.szSourceLanguage, NULL, (USHORT)LANG_KEY, &usSrcLang ); + TmRecord.resize(TMX_REC_SIZE); //fill tm record to add to database FillTmRecord ( TmProposal.pInputSentence, // ptr to sentence struct for source info - pTmRecord, // filled tm record returned + TmRecord, // filled tm record returned pTargetClb, usSrcLang ); //add new tm record to database @@ -776,8 +770,7 @@ USHORT EqfMemory::AddToTm usRc = TmBtree.EQFNTMInsert(//ptr to tm structure pulNewKey, //to be allocated in funct - (PBYTE)pTmRecord, //pointer to tm record - pTmRecord->lRecordLen); //length + TmRecord); //pointer to tm record if(!usRc){ TmProposal.currentInternalKey.setInternalKey(*pulNewKey, 1); } @@ -786,7 +779,6 @@ USHORT EqfMemory::AddToTm } /* endif */ //release memory - UtlAlloc( (PVOID *) &(pTmRecord), 0L, 0L, NOMSG); UtlAlloc( (PVOID *) &(pTargetClb), 0L, 0L, NOMSG); return( usRc ); @@ -824,10 +816,7 @@ USHORT EqfMemory::AddToTm VOID FillTmRecord ( PTMX_SENTENCE pSentence, // ptr to sentence struct for source info - //PTMX_TAGTABLE_RECORD pTagRecord, // ptr to target string tag table - //PSZ_W pNormString, // ptr to target normalized string - //USHORT usNormLen, // length of target normalized string - PTMX_RECORD pTmRecord, // filled tm record returned + std::vector& TmRecord, // filled tm record returned PTMX_TARGET_CLB pTargetClb, // ptr to target control block USHORT usSrcLangId ) @@ -835,6 +824,7 @@ VOID FillTmRecord PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to start of source structure PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to start of target structure PBYTE pTarget; //ptr to target record + PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); ULONG ulSrcLen = wcslen(pSentence->pStrings->getGenericTagStrC()); ULONG ulTrgLen = wcslen(pSentence->pStrings->getGenericTargetStrC()); @@ -973,11 +963,6 @@ USHORT EqfMemory::FillClb //get id of target language, call usRc = NTMGetIDFromName( TmProposal.szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTrgLang ); - //if ( !usRc ) - //{ - // usRc = NTMGetIDFromName( pTmClb, TmProposal.szSourceLanguage, NULL, (USHORT)LANG_KEY, &usSrcLang ); - //} /* endif */ - //get id of file name, call if ( !usRc ) { @@ -1102,8 +1087,8 @@ USHORT EqfMemory::UpdateTmIndex toIndexRecord(pIndexRecord)->stIndexEntry = NTMINDEX(pSentence->usActVote,ulSidKey); usRc = InBtree.EQFNTMInsert( &ulKey, - pIndexRecord.data(), //pointer to index - toIndexRecord(pIndexRecord)->usRecordLen ); //length + pIndexRecord); //pointer to index + //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) @@ -1112,7 +1097,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMInsert(&ulKey, pIndexRecord.data(), toIndexRecord(pIndexRecord)->usRecordLen ); + usRc = InBtree.EQFNTMInsert(&ulKey, pIndexRecord);//, toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ @@ -1183,8 +1168,8 @@ USHORT EqfMemory::UpdateTmIndex usRc = InBtree.EQFNTMUpdate( ulKey, - pIndexRecord.data(), //pointer to index - toIndexRecord(pIndexRecord)->usRecordLen ); //length + pIndexRecord);//.data(), //pointer to index + //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) { @@ -1192,7 +1177,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMUpdate(ulKey, pIndexRecord.data(), toIndexRecord(pIndexRecord)->usRecordLen ); + usRc = InBtree.EQFNTMUpdate(ulKey, pIndexRecord);//.data(), toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ if ( !usRc ) @@ -1263,15 +1248,13 @@ USHORT DetermineTmRecord ( EqfMemory* pTmClb, // ptr to tm control block PTMX_SENTENCE pSentence, // ptr to sentence structure - PULONG pulSids // ptr to tm record + std::vector& ulSids // ptr to tm record ) { USHORT usRc = NO_ERROR; // return code ULONG ulLen = 0; // length paramter - USHORT usSid = 0; // number of sentence ids found USHORT usPos; // position in pulSids PULONG pulVotes; // pointer to votes - PULONG pulSidStart; // po ter to votes USHORT i, j; // index in for loop USHORT usMaxEntries = 0; // nr of index entries in index record ULONG ulKey; // index key @@ -1279,7 +1262,6 @@ USHORT DetermineTmRecord BOOL fOk = true; // success indicator PTMX_INDEX_ENTRY pIndexEntry; // pointer to index entry structure - pulSidStart = pulSids; //for all votes add the index to the corresponding list pulVotes = pSentence->pulVotes.data(); @@ -1319,18 +1301,16 @@ USHORT DetermineTmRecord { if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) { - *pulSids = NTMKEY(*pIndexEntry); - usSid++; - pulSids++; + ulSids.push_back(NTMKEY(*pIndexEntry)); } /* endif */ } /* endfor */ - if ( (usSid > 1) ) + if ( (ulSids.size() > 1) ) { //there is more than one sentence to check so go through other //indices until all have been tried or usSid has been reduced to 1 pulVotes++; - for ( i = 0; (i < pSentence->usActVote-1) && (usSid > 1); + for ( i = 0; (i < pSentence->usActVote-1) && (ulSids.size() > 1); i++, pulVotes++ ) { if ( usRc == NO_ERROR ) @@ -1347,7 +1327,8 @@ USHORT DetermineTmRecord usMaxEntries = (USHORT)((pIndexRecord.size() - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; - pulSids = pulSidStart; +#ifdef OLD_CODE + PULONG pulSids = ulSidStart; usPos = 0; //end criteria are all sentence ids in index key or only one @@ -1382,6 +1363,24 @@ USHORT DetermineTmRecord } /* endif */ } /* endif */ } /* endfor */ +#else + // End criteria: all sentence IDs in index key or only one sentence ID left in pulSids + for (j = 0; (j < usMaxEntries) && (ulSids.size() > 1); j++, pIndexEntry++) { + if (NTMVOTES(*pIndexEntry) == static_cast(pSentence->usActVote)) { + // Before adding sentence ID, check if already in pulSids, as the respective TM record needs only be checked once + while ((NTMKEY(*pIndexEntry) > ulSids.front()) && (ulSids.size() > 1)) { + // Remove SID from pulSids and decrease SID counter + ulSids.erase(ulSids.begin()); + } + + if (NTMKEY(*pIndexEntry) == ulSids.front()) { + // Move on to the next position in pulSids + ulSids.erase(ulSids.begin()); + usPos++; + } + } + } +#endif } else { @@ -1410,19 +1409,13 @@ USHORT DetermineTmRecord if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ std::string msg = __func__ + std::string(":: Matching Sids: "); pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; - while ( *pulSidStart ) + for(auto sid: ulSids) { - msg += std::to_string(*pulSidStart) + "; "; - pulSidStart++; + msg += std::to_string(sid) + "; "; } T5LOG( T5DEBUG) << msg; } - if ( usRc ) - { - ERREVENT2( DETERMINETMRECORD_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ - return( usRc ); } @@ -1467,73 +1460,55 @@ USHORT EqfMemory::UpdateTmRecord { BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indication to leave while loop - PULONG pulSids = NULL; //ptr to sentence ids - PULONG pulSidStart = NULL; //ptr to sentence ids + std::vector ulSids; //ptr to sentence ids + int iSidCount=0; USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator ULONG ulKey; //tm record key std::vector TmRecord; //pointer to tm record TmRecord.resize(TMX_REC_SIZE); + ulSids.reserve(MAX_INDEX_LEN + 5); - - //allocate for sentence ids - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *) &(pulSids), 0L, - (LONG)((MAX_INDEX_LEN + 5) * sizeof(ULONG)), - NOMSG ); - } /* endif */ - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + usRc = DetermineTmRecord( this, TmProposal.pInputSentence, ulSids ); + if ( usRc == NO_ERROR ) { - pulSidStart = pulSids; - usRc = DetermineTmRecord( this, TmProposal.pInputSentence, pulSids ); - if ( usRc == NO_ERROR ) + //get tm record(s) + //while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) + for(ULONG ulKey: ulSids) { - //get tm record(s) - while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) + iSidCount ++; + usRc = TmBtree.EQFNTMGet( ulKey, //tm record key + TmRecord); //pointer to tm record data + + if ( usRc == NO_ERROR ) { - ulKey = *pulSids; - usRc = TmBtree.EQFNTMGet( ulKey, //tm record key - TmRecord); //pointer to tm record data + //compare tm record data with data passed in the get in structure + usRc = ComparePutData( TmRecord, TmProposal, &ulKey ); - if ( usRc == NO_ERROR ) + if ( usRc == SOURCE_STRING_ERROR ) { - //compare tm record data with data passed in the get in structure - usRc = ComparePutData( TmRecord, TmProposal, &ulKey ); - - if ( usRc == SOURCE_STRING_ERROR ) - { - //get next tm record - pulSids++; - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - } - else if ( usRc == NO_ERROR ) - { - //TmProposal.recordKey = ulKey; - //new target record was added or an existing one was successfully - //replaced so don't try other sids - fStop = TRUE; - } /* endif */ + //get next tm record + LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); + } + else + { + //TmProposal.recordKey = ulKey; + //new target record was added or an existing one was successfully + //replaced so don't try other sids + fStop = usRc == NO_ERROR; + break; } /* endif */ - } /* endwhile */ - - if ( !(*pulSids) && !fStop ) - { - //issue message that tm needs to be organized if pulsid is empty and - //no get was successful - LOG_AND_SET_RC(usRc, T5INFO, ERROR_ADD_TO_TM); } /* endif */ - } /* endif */ - } /* endif */ + } /* endwhile */ - //release memory - UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); + if ( (iSidCount == ulSids.size()) && !fStop ) + { + //issue message that tm needs to be organized if pulsid is empty and + //no get was successful + LOG_AND_SET_RC(usRc, T5INFO, ERROR_ADD_TO_TM); + } /* endif */ + } /* endif */ return( usRc ); } @@ -1577,8 +1552,8 @@ USHORT EqfMemory::UpdateTmRecordByInternalKey { BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indication to leave while loop - PULONG pulSids = NULL; //ptr to sentence ids - PULONG pulSidStart = NULL; //ptr to sentence ids + std::vector ulSids; //ptr to sentence ids + int iSidCount = 0; USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator ULONG ulKey; //tm record key @@ -1586,65 +1561,48 @@ USHORT EqfMemory::UpdateTmRecordByInternalKey std::vector TmRecord; // space for user data TmRecord.resize(TMX_REC_SIZE); + ulSids.reserve(MAX_INDEX_LEN + 5); - //allocate for sentence ids - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *) &(pulSids), 0L, - (LONG)((MAX_INDEX_LEN + 5) * sizeof(ULONG)), - NOMSG ); - } /* endif */ - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + usRc = DetermineTmRecord( this, TmProposal.pInputSentence, ulSids ); + if ( usRc == NO_ERROR ) { - pulSidStart = pulSids; - usRc = DetermineTmRecord( this, TmProposal.pInputSentence, pulSids ); - if ( usRc == NO_ERROR ) + //get tm record(s) + //while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) + for(ULONG ulKey: ulSids) { - //get tm record(s) - while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) + iSidCount++; + memset( &TmRecord, 0, TmRecord.size() ); + usRc = TmBtree.EQFNTMGet(ulKey, //tm record key + TmRecord); //pointer to tm record data + + if ( usRc == NO_ERROR ) { - ulKey = *pulSids; - memset( &TmRecord, 0, TmRecord.size() ); - usRc = TmBtree.EQFNTMGet(ulKey, //tm record key - TmRecord); //pointer to tm record data - + //compare tm record data with data passed in the get in structure + usRc = ComparePutData( TmRecord, TmProposal, &ulKey ); - if ( usRc == NO_ERROR ) + if ( usRc == SOURCE_STRING_ERROR ) { - //compare tm record data with data passed in the get in structure - usRc = ComparePutData( TmRecord, TmProposal, &ulKey ); - - if ( usRc == SOURCE_STRING_ERROR ) - { - //get next tm record - pulSids++; - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - } - else if ( usRc == NO_ERROR ) - { - //new target record was added or an existing one was successfully - //replaced so don't try other sids - fStop = TRUE; - } /* endif */ + //get next tm record + LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); + } + else + { + //new target record was added or an existing one was successfully + //replaced so don't try other sids + fStop = usRc == NO_ERROR; + break; } /* endif */ - } /* endwhile */ - - if ( !(*pulSids) && !fStop ) - { - //issue message that tm needs to be organized if pulsid is empty and - //no get was successful - LOG_AND_SET_RC(usRc, T5INFO, ERROR_ADD_TO_TM); } /* endif */ - } /* endif */ - } /* endif */ + } /* endfor */ + + if ( iSidCount == ulSids.size() && !fStop ) + { + //issue message that tm needs to be organized if pulsid is empty and + //no get was successful + LOG_AND_SET_RC(usRc, T5INFO, ERROR_ADD_TO_TM); + } /* endif */ + } /* endif */ - //release memory - UtlAlloc( (PVOID *) &pulSidStart, 0L, 0L, NOMSG ); return( usRc ); } @@ -1949,7 +1907,7 @@ USHORT EqfMemory::ComparePutData // update TM record if required if ( fUpdate ) { - usRc = TmBtree.EQFNTMUpdate( *pulKey, pTmRecord.data(), RECLENB(pTmRecord) ); + usRc = TmBtree.EQFNTMUpdate( *pulKey, pTmRecord);//.data(), RECLENB(pTmRecord) ); } /* endif */ if ( fStop ) @@ -2078,8 +2036,8 @@ USHORT EqfMemory::AddTmTarget( //add updated tm record to database usRc = TmBtree.EQFNTMUpdate( *pulKey, - pTmRecord.data(), - RECLENB(pTmRecord) ); + pTmRecord);//.data(), + //RECLENB(pTmRecord) ); } /* endif */ } /* endif */ } /* endif */ @@ -2596,10 +2554,9 @@ USHORT TMLoopAndDelTargetClb while ( ( lLeftTgtLen <= RECLEN(pTMXTgtRec) ) && (RECLEN(pTMXTgtRec) != 0) && !fDel ) { - NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTgtRec), 4713); - lLeftTgtLen -= RECLEN(pTMXTgtRec); + NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTgtRec), 4713); + lLeftTgtLen -= RECLEN(pTMXTgtRec); - { pClb = (PTMX_TARGET_CLB)NTRecPos((PBYTE)pTMXTgtRec, REC_CLB); lLeftClbLen = RECLEN(pTMXTgtRec) - pTMXTgtRec->usClb; @@ -2634,13 +2591,12 @@ USHORT TMLoopAndDelTargetClb pClb = NEXTTARGETCLB(pClb); } /* endif */ } /* endwhile */ - } /* endif */ - //position at next target - if ( !fDel ) - { - pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)pTMXTgtRec, REC_NEXTTARGET); - } /* endif */ - } /* endwhile */ + //position at next target + if ( !fDel ) + { + pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)pTMXTgtRec, REC_NEXTTARGET); + } /* endif */ + } /* endwhile */ return(usRc); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 90e1e44a..46de8855 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -62,6 +62,20 @@ void toLower(std::vector& vec) { }); } +#include +#include + +std::vector toLowerVector(const std::vector& input) { + std::vector result; + result.reserve(input.size()); // Optimize memory allocation + + for (unsigned char c : input) { + result.push_back(c == '\0' ? '\0' : std::tolower(c)); + } + + return result; +} + // #include "EQFCMPR.H" /**********************************************************************/ /* prototypes for internal functions */ @@ -373,9 +387,7 @@ USHORT EqfMemory::NTMGetIDFromNameEx( DEBUGEVENT2(NTMGETIDFROMNAME_LOC, INFO_EVENT, 2, TM_GROUP, pszLongName); usRc = TmBtree.EQFNTMUpdate(LONGNAME_KEY, - (PBYTE)LongNames.pszBuffer.data(), - //LongNames.ulBufUsed); - LongNames.pszBuffer.size()); + LongNames.pszBuffer); } /* endif */ } /* endif */ } /* endif */ @@ -859,10 +871,10 @@ EqfMemory::NTMGetPointersToTable( // ----------------------------------------------------------------------------+ USHORT EqfMemory::NTMCreateLongNameTable() { - LongNames.pszBuffer.resize(LONGNAMETABLE_ENTRIES); + LongNames.pszBuffer.reserve(LONGNAMETABLE_ENTRIES); LongNames.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); - LongNamesCaseIgnore.pszBuffer.resize(LONGNAMETABLE_ENTRIES); + LongNamesCaseIgnore.pszBuffer.reserve(LONGNAMETABLE_ENTRIES); LongNamesCaseIgnore.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); return 0; @@ -900,6 +912,8 @@ USHORT EqfMemory::NTMReadLongNameTable() if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) { +T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be deleted" ; +#ifdef TEMPORARY_COMMENTED // table is tersed... pNewArea.resize(pTerseHeader->usDataSize); // allocate buffer for untersed data @@ -942,6 +956,7 @@ USHORT EqfMemory::NTMReadLongNameTable() // pNewArea = NULL; } /* endif */ // if ( pNewArea != NULL ) UtlAlloc( (PVOID *)&pNewArea, 0L, 0L, NOMSG ); +#endif } /* endif */ } /* endif */ @@ -976,8 +991,7 @@ USHORT EqfMemory::NTMReadLongNameTable() // make copy of long name array for case ignore search LongNamesCaseIgnore.stTableEntry = LongNames.stTableEntry; - LongNamesCaseIgnore.pszBuffer = LongNames.pszBuffer; - toLower(LongNamesCaseIgnore.pszBuffer); + LongNamesCaseIgnore.pszBuffer = toLowerVector(LongNames.pszBuffer); LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); @@ -993,6 +1007,21 @@ USHORT EqfMemory::NTMReadLongNameTable() return (usRC); } /* end of function NTMReadLongNameTable */ +int tmxTableToBuffer(PTMX_TABLE pTable, std::vector& buff){ + buff.clear(); + int occupSize = (pTable->ulMaxEntries + 1) * sizeof(TMX_TABLE_ENTRY); + buff.resize(occupSize);; + TMX_TABLE_OLD *table = (TMX_TABLE_OLD *)buff.data(); + //pOldTable = pstTMTable->stTmTableEntry; + table->ulMaxEntries = pTable->ulMaxEntries; + if (table->ulMaxEntries) + { + memcpy(&table->stTmTableEntry, + &pTable->stTmTableEntry[0], sizeof(TMX_TABLE_ENTRY) * table->ulMaxEntries); + } + return 0; +} + //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ @@ -1099,21 +1128,10 @@ USHORT EqfMemory::NTMAddNameToTable( { if (usTableType != LANGGROUP_KEY) { - int occupSize = (pstTMTable->ulMaxEntries + 1) * sizeof(TMX_TABLE_ENTRY); - std::vector pOldTable; - pOldTable.resize(occupSize);; - TMX_TABLE_OLD *table = (TMX_TABLE_OLD *)pOldTable.data(); - //pOldTable = pstTMTable->stTmTableEntry; - table->ulMaxEntries = pstTMTable->ulMaxEntries; - if (table->ulMaxEntries) - { - memcpy(&table->stTmTableEntry, - &pstTMTable->stTmTableEntry[0], sizeof(TMX_TABLE_ENTRY) * table->ulMaxEntries); - } - usRc = TmBtree.EQFNTMUpdate( - (ULONG)usTableType, - (PBYTE)table, - occupSize); + std::vector pOldTable; + tmxTableToBuffer(pstTMTable, pOldTable); + usRc = TmBtree.EQFNTMUpdate((LONG)usTableType,pOldTable); + //(PBYTE)table, occupSize); } /* endif */ } /* endif */ } /* endif */ @@ -1161,56 +1179,39 @@ int NTMLongNameTableCompCaseIgnore( const void *pEntry2 // input ) { - /********************************************************************/ - /* compare the long names of the passed table entries using strcmp */ - /********************************************************************/ - // return( _stricmp( ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName, - // ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName )); - return (strcasecmp(((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName, - ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName)); + PSZ lname1 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName; + PSZ lname2 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName; + return strcmp(lname1,lname2);//string should be in lower case here + //return strcasecmp(lname1,lname2); } /* end of function NTMCompNames */ //+----------------------------------------------------------------------------+ -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMLoadNameTable | -//+----------------------------------------------------------------------------+ -//|Description: This function loads a TM name table from the database. | -//| If the table is tersed it is uncompressed. | -// ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMLoadNameTable( - ULONG ulTableKey, // key of table record - PTMX_TABLE pTMTable, // ptr to table data pointer - PULONG pulSize // ptr to buffer for size of table data -) -{ - USHORT usRc = NO_ERROR; // function return code - std::vector pOldTable; - // call to obtain exact length of record - *pulSize = 0; - usRc = TmBtree.EQFNTMGet(ulTableKey, pOldTable); +USHORT bytesToOldTable(ULONG ulTableKey, std::vector oldTmxTable, PTMX_TABLE pTMTable) +{ + USHORT usRc = 0; // handle tersed name tables if (usRc == NO_ERROR) { - PTERSEHEADER pTerseHeader = (PTERSEHEADER)pOldTable.data(); + PTERSEHEADER pTerseHeader = (PTERSEHEADER)oldTmxTable.data(); std::vector pNewArea; // ptr to unterse data area if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) { + T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be deleted" ; + #ifdef TEMPORARY_COMMENTED // table is tersed... pNewArea.resize(pTerseHeader->usDataSize); - // allocate buffer for untersed data - + // allocate buffer for untersed data + // unterse data if (usRc == NO_ERROR) { ULONG ulNewLen = 0; - memcpy(pNewArea.data(), (PBYTE)pOldTable.data() + sizeof(TERSEHEADER), + memcpy(pNewArea.data(), (PBYTE)oldTmxTable.data() + sizeof(TERSEHEADER), *pulSize - sizeof(TERSEHEADER)); T5LOG(T5ERROR) << ":: TEMPORARY_COMMENTED temcom_id = 51 in NTMLoadNameTable"; @@ -1239,6 +1240,7 @@ USHORT EqfMemory::NTMLoadNameTable( // avoid free of untersed data area // pNewArea = NULL; } /* endif */ + #endif } /* endif */ } /* endif */ @@ -1247,8 +1249,8 @@ USHORT EqfMemory::NTMLoadNameTable( // 0xA0 under Windows (due to OemToAnsi) but should be 0xFF if ((usRc == NO_ERROR) && (ulTableKey == LANG_KEY)) { - LONG lRest = *pulSize; - PBYTE pbTemp = (PBYTE)pOldTable.data(); + LONG lRest = oldTmxTable.size();; + PBYTE pbTemp = (PBYTE)oldTmxTable.data(); while (lRest) { if (*pbTemp == 0xA0) @@ -1262,14 +1264,40 @@ USHORT EqfMemory::NTMLoadNameTable( if (usRc == NO_ERROR) { - pTMTable->ulMaxEntries = ((PTMX_TABLE_OLD)pOldTable.data())->ulMaxEntries; + pTMTable->ulMaxEntries = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; pTMTable->stTmTableEntry.clear(); pTMTable->stTmTableEntry.resize(pTMTable->ulMaxEntries + 1); if (pTMTable->ulMaxEntries) { - memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)pOldTable.data())->stTmTableEntry, pTMTable->ulMaxEntries * sizeof(TMX_TABLE_ENTRY)); + memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, pTMTable->ulMaxEntries * sizeof(TMX_TABLE_ENTRY)); } } + return usRc; +} + + +//+----------------------------------------------------------------------------+ +//|Internal function | +//+----------------------------------------------------------------------------+ +//|Function name: NTMLoadNameTable | +//+----------------------------------------------------------------------------+ +//|Description: This function loads a TM name table from the database. | +//| If the table is tersed it is uncompressed. | +// ----------------------------------------------------------------------------+ +USHORT EqfMemory::NTMLoadNameTable( + ULONG ulTableKey, // key of table record + PTMX_TABLE pTMTable, // ptr to table data pointer + PULONG pulSize // ptr to buffer for size of table data +) +{ + USHORT usRc = NO_ERROR; // function return code + std::vector pOldTable; + + // call to obtain exact length of record + *pulSize = 0; + usRc = TmBtree.EQFNTMGet(ulTableKey, pOldTable); + + if(!usRc) usRc = bytesToOldTable(ulTableKey, pOldTable, pTMTable); // return to caller return (usRc); diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index a4eb038d..791769ea 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -110,32 +110,28 @@ void EqfMemory::setDescription //if ( ) { BOOL fOK = TRUE; - PTMX_SIGN pTmSign = new(TMX_SIGN); + TMX_SIGN pTmSign ; // get current signature record USHORT usSignLen = sizeof(TMX_SIGN); - USHORT usRc = TmBtree.EQFNTMSign((PCHAR)pTmSign, &usSignLen ); + USHORT usRc = sign(); fOK = (usRc == NO_ERROR); // update description field if ( fOK ) { - strncpy( pTmSign->szDescription, pszBuffer, sizeof(pTmSign->szDescription)-1 ); - pTmSign->szDescription[sizeof(pTmSign->szDescription)-1] = EOS; + strncpy( stTmSign.szDescription, pszBuffer, sizeof(stTmSign.szDescription)-1 ); + stTmSign.szDescription[sizeof(stTmSign.szDescription)-1] = EOS; - //trigger plugin to set description - //pMemoryPlugin->setDescription(szName,pTmSign->szDescription); } /* endif */ // re-write signature record if ( fOK ) { - usRc = TmBtree.QDAMDictUpdSignLocal(pTmSign); + usRc = TmBtree.QDAMDictUpdSignLocal(&stTmSign); fOK = (usRc == NO_ERROR); } /* endif */ - // free any allocated buffer - delete ( pTmSign ); } /* endif */ return; } @@ -155,7 +151,8 @@ int EqfMemory::getName int iSize ) { - return( CopyToBuffer( this->szName, pszBuffer, iSize ) ); + this->szName.assign( pszBuffer, pszBuffer+iSize ); + return 0; } /*! \brief Get the name of the memory @@ -230,8 +227,7 @@ int EqfMemory::RewriteCompactTable(){ USHORT usRc = 0; if ( bCompactChanged ) { - usRc = TmBtree.EQFNTMUpdate( COMPACT_KEY, - bCompact.data(), MAX_COMPACT_SIZE-1 ); + usRc = TmBtree.EQFNTMUpdate( COMPACT_KEY, bCompact ); if ( !usRc ) { bCompactChanged = FALSE; @@ -315,7 +311,7 @@ int EqfMemory::getNextProposal } else { - handleError( iRC, this->szName); + handleError( iRC, this->szName.c_str()); if ( iRC == BTREE_CORRUPTED ) iRC = ERROR_ENTRYISCORRUPTED; } @@ -649,9 +645,9 @@ OtmProposal::eProposalType FlagToProposalType( USHORT usTranslationFlag ) \param pszMarkup markup table name \returns original or modified error return code */ -int EqfMemory::handleError( int iRC, char *pszMemName, char *pszMarkup ) +int EqfMemory::handleError( int iRC,const char *pszMemName, char *pszMarkup ) { - return( EqfMemoryPlugin::handleError( iRC, pszMemName, pszMarkup, (PSZ)TmBtree.fb.fileName.c_str(), this->strLastError, this->iLastError ) ); + return( EqfMemoryPlugin::handleError( iRC, (PSZ)pszMemName, pszMarkup, (PSZ)TmBtree.fb.fileName.c_str(), this->strLastError, this->iLastError ) ); } /*! \brief Set or clear the pointer to a loaded global memory option file @@ -663,14 +659,6 @@ int EqfMemory::handleError( int iRC, char *pszMemName, char *pszMarkup ) \returns 0 or error code in case of errors */ -int EqfMemory::setGlobalMemoryOptions -( - void *pvGlobalMemoryOptionsIn -) -{ - this->pvGlobalMemoryOptions = pvGlobalMemoryOptionsIn; - return( 0 ); -} @@ -683,14 +671,12 @@ EqfMemory::EqfMemory(const std::string& tmName){ readOnlyCnt = std::make_shared(0); writeCnt = std::make_shared(0); - - pvGlobalMemoryOptions = NULL; int rc = NTMCreateLongNameTable(); if(rc) T5LOG(T5ERROR) << "NTMCreateLongNameTable returned non-zero, but " << rc; TmBtree.fb.fileName = FilesystemHelper::GetTmdPath(tmName); InBtree.fb.fileName = FilesystemHelper::GetTmiPath(tmName); - strcpy(szName, tmName.c_str()); + szName = tmName; } int EqfMemory::ReloadFromDisk(){ diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp index 88b410a9..a883ebb2 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp @@ -108,17 +108,6 @@ EqfMemory* EqfMemoryPlugin::initTM( } /* end */ -/*! \brief Close a memory - \param pMemory pointer to memory object -*/ -int EqfMemoryPlugin::closeMemory( - EqfMemory *pMemory -) -{ - T5LOG(T5ERROR) << "unimplemented funtion"; - return( 0 ); -} - ///*! \brief set description of a memory // \param pszName name of the memory diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 0c2e2b99..b9e116f1 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -187,24 +187,6 @@ int convertUTCTimeToLong( char *pszDateTime, PLONG plTime ) #include -/* -std::unique_ptr replaceString(std::wstring&& src_data, std::wstring&& trg_data, std::wstring&& req_data, int* rc){ - std::unique_ptr response; - *rc = 0; - try { - *rc = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if(*rc == 0){ - response = EncodingHelper::ReplaceOriginalTagsWithPlaceholders(std::move(src_data), std::move(trg_data), std::move(req_data) ); - } - } - //catch (const XMLException& toCatch) { - catch(...){ - *rc = 400; - //return( ERROR_NOT_READY ); - } - return std::move(response); -}//*/ - @@ -496,8 +478,8 @@ int RequestData::run(){ return 0; } - int res = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if(!res) res = parseJSON(); + int res = 0; + res = parseJSON(); if(!res) res = checkData(); if(!res) res = prepareData(); @@ -621,9 +603,6 @@ int CreateMemRequestData::createNewEmptyMemory(){ return TM_LIST_MUTEX_TIMEOUT_FAILED; } } - //if(mem != nullptr){ - // mem.reset(); - //} return 0; } @@ -1725,7 +1704,7 @@ int ReorganizeRequestData::execute(){ pRIDA->pMem = mem; pRIDA->memRef = memRef; pRIDA->usRC = NO_ERROR; - strcpy( pRIDA->szMemName, mem->szName ); + strcpy( pRIDA->szMemName, mem->szName.c_str() ); pRIDA->fBatch = TRUE; pRIDA->hwndErrMsg = HWND_FUNCIF; pRIDA->NextTask = MEM_START_ORGANIZE; @@ -1734,7 +1713,6 @@ int ReorganizeRequestData::execute(){ pData->fComplete = FALSE; pData->sLastFunction = FCT_EQFORGANIZEMEM; pData->pvMemOrganizeCommArea = pCommArea; - pRIDA->pszNameList = 0; if(mem->importDetails == nullptr){ mem->importDetails = new ImportStatusDetails; } @@ -1776,11 +1754,6 @@ int ReorganizeRequestData::execute(){ int DeleteEntriesReorganizeRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "can't verifyAPISession", INTERNAL_SERVER_ERROR); - } /* endif */ if ( strMemName.empty() ) { @@ -1868,7 +1841,7 @@ int DeleteEntriesReorganizeRequestData::execute(){ pRIDA->pMem = mem; pRIDA->memRef = memRef; pRIDA->usRC = NO_ERROR; - strcpy( pRIDA->szMemName, mem->szName ); + strcpy( pRIDA->szMemName, mem->szName.c_str() ); pRIDA->fBatch = TRUE; pRIDA->hwndErrMsg = HWND_FUNCIF; pRIDA->NextTask = MEM_START_ORGANIZE; @@ -1876,8 +1849,7 @@ int DeleteEntriesReorganizeRequestData::execute(){ // enable organize process if OK pData->fComplete = FALSE; pData->sLastFunction = FCT_EQFORGANIZEMEM; - pData->pvMemOrganizeCommArea = pCommArea; - pRIDA->pszNameList = 0; + pData->pvMemOrganizeCommArea = pCommArea; if(mem->importDetails == nullptr){ mem->importDetails = new ImportStatusDetails; } @@ -2850,11 +2822,6 @@ int ExportRequestData::parseJSON(){ int ExportRequestData::checkData(){ T5LOG( T5INFO) <<"::getMem::=== getMem request, memory = " << strMemName << "; format = " << requestAcceptHeader; - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn(_rc_ , "::getMem::Error: no valid API session" ); - } /* endif */ if ( strMemName.empty() ) { @@ -3018,12 +2985,6 @@ int ExportRequestData::ExportTmx(){ } int UpdateEntryRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( BAD_REQUEST, "can't verifyAPISession", BAD_REQUEST); - } /* endif */ - if ( strMemName.empty() ) { return buildErrorReturn( BAD_REQUEST, "Missing name of memory", BAD_REQUEST); @@ -3431,11 +3392,6 @@ int FuzzySearchRequestData::reset(){ } int FuzzySearchRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "can't verifyAPISession", BAD_REQUEST); - } /* endif */ if ( strMemName.empty() ) { @@ -3563,7 +3519,6 @@ int FuzzySearchRequestData::execute(){ GetIn.usConvert = MEM_OUTPUT_ASIS; GetIn.usRequestedMatches = (USHORT)vProposals.size(); GetIn.ulParm = GET_EXACT; - GetIn.pvGMOptList = mem->pvGlobalMemoryOptions; if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ auto str = EncodingHelper::convertToUTF8(GetIn.szSource); @@ -3701,11 +3656,6 @@ int MultiFuzzySearchRequestData::execute(){ } int ConcordanceExtendedSearchRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "can't verifyAPISession", INTERNAL_SERVER_ERROR); - } /* endif */ if ( strMemName.empty() ) { @@ -4030,11 +3980,7 @@ int ConcordanceExtendedSearchRequestData::execute() int ConcordanceSearchRequestData::parseJSON(){ - _rc_ = OtmMemoryServiceWorker::getInstance()->verifyAPISession(); - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "can't verifyAPISession", INTERNAL_SERVER_ERROR); - } /* endif */ + if ( strMemName.empty() ) { diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index acb656a9..b583c74c 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -349,32 +349,36 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem _rc_ = NewMem->TmBtree.QDAMDictCreateLocal( &(NewMem->stTmSign),FIRST_KEY, keepInRamOnly); } + + std::vector buffer; + buffer.reserve(TMX_TABLE_SIZE); + if ( _rc_ == NO_ERROR ) { //insert initialized record to tm data file ulKey = AUTHOR_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)&NewMem->Authors, TMX_TABLE_SIZE ); + tmxTableToBuffer(&NewMem->Authors,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey,buffer); } if ( _rc_ == NO_ERROR ) { ulKey = FILE_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)&NewMem->FileNames, TMX_TABLE_SIZE ); + tmxTableToBuffer(&NewMem->FileNames,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { ulKey = TAGTABLE_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)&NewMem->TagTables, TMX_TABLE_SIZE ); + tmxTableToBuffer(&NewMem->TagTables,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { ulKey = LANG_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)&NewMem->Languages, TMX_TABLE_SIZE ); + tmxTableToBuffer(&NewMem->Languages,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) @@ -386,7 +390,7 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem ulKey = COMPACT_KEY; _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - NewMem->bCompact.data(), size); + NewMem->bCompact); } /* endif */ @@ -394,10 +398,10 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem if ( _rc_ == NO_ERROR ) { ulKey = LONGNAME_KEY; + NewMem->LongNames.pszBuffer.push_back(0);//to have some size // write long document name buffer area to the database _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, - (PBYTE)NewMem->LongNames.pszBuffer.data(), - NewMem->LongNames.pszBuffer.size() ); + NewMem->LongNames.pszBuffer ); } /* endif */ @@ -432,37 +436,7 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem \param pMemory pointer to memory object beign closed \returns 0 when successful or error code */ -int TMManager::closeMemory -( - EqfMemory *pMemory -) -{ - int iRC = 0; - if ( pMemory == NULL ){ - T5LOG(T5ERROR) <<"TMManager::closeMemory, pMemory is NULL"; - return( -1 ); - } - - // build memory object name - char* pszObjName = NULL; - UtlAlloc( (PVOID *)&pszObjName, 0L, MAX_LONGFILESPEC + MAX_LONGFILESPEC + 2, NOMSG ); - strcpy( pszObjName, EqfMemoryPlugin::GetInstance()->getName() ); - strcat( pszObjName, ":" ); - pMemory->getName( pszObjName + strlen(pszObjName), MAX_LONGFILESPEC ); - - // close the memory - iRC = EqfMemoryPlugin::GetInstance()->closeMemory( pMemory ); - - // send a properties changed msg to memory handler - EqfSend2Handler( MEMORYHANDLER, WM_EQFN_PROPERTIESCHANGED, MP1FROMSHORT( PROP_CLASS_MEMORY ), MP2FROMP( pszObjName )); - - if ( pszObjName != NULL ) { - UtlAlloc( (PVOID *)&pszObjName, 0L, 0L, NOMSG ); - } - - return( iRC ); -} diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 6101f272..fea4f55c 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -962,7 +962,7 @@ SHORT QDAMDosRC2BtreeRC SHORT BTREE::QDAMWRecordToDisk_V3 ( - PBTREEBUFFER_V3 pBuffer // pointer to buffer to write + std::shared_ptr& pBuffer // pointer to buffer to write ) { LONG lOffset = RECORDNUM(pBuffer) * (long)BTREE_REC_SIZE_V3; // offset in file @@ -1055,7 +1055,7 @@ SHORT BTREE::QDAMWRecordToDisk_V3 SHORT BTREE::QDAMReadRecordFromDisk_V3 ( USHORT usNumber, - PBTREEBUFFER_V3 * ppReadBuffer, + std::shared_ptr& pBuffer, BOOL fNewRec // allow new records flag ) { @@ -1063,13 +1063,11 @@ SHORT BTREE::QDAMReadRecordFromDisk_V3 PBTREEHEADER pHeader; // pointer to header LONG lOffset; // file offset to be set ULONG ulNewOffset; // new position - PBTREEBUFFER_V3 pBuffer = NULL; SHORT sRc = 0; // return code - PLOOKUPENTRY_V3 pLEntry = NULL; DEBUGEVENT2( QDAMREADRECORDFROMDISK_LOC, FUNCENTRY_EVENT, usNumber, DB_GROUP, "" ); int resRead = 0; - sRc = allocateNewLookupTableBuffer(usNumber, pLEntry, pBuffer); + sRc = allocateNewLookupTableBuffer(usNumber, pBuffer); if ( !sRc ) { @@ -1089,7 +1087,7 @@ SHORT BTREE::QDAMReadRecordFromDisk_V3 { pBuffer->usRecordNumber = usNumber; // pBuffer->ulCheckSum = QDAMComputeCheckSum( pBuffer ); - *ppReadBuffer = pBuffer; + //*ppReadBuffer = pBuffer; if ( resRead == FilesystemHelper::FILEHELPER_WARNING_FILE_IS_SMALLER_THAN_REQUESTED) { /*************************************************************/ @@ -1130,12 +1128,8 @@ SHORT BTREE::QDAMReadRecordFromDisk_V3 /********************************************************************/ if ( sRc ) { - if ( (pLEntry != NULL) && (pLEntry->pBuffer != NULL) ) - { - UtlAlloc( (PVOID *)&(pLEntry->pBuffer), 0L, 0L, NOMSG ); - (usNumberOfAllocatedBuffers)--; - } /* endif */ - *ppReadBuffer = nullptr; + deallocateLookupTableBuffer(usNumber); + pBuffer.reset(); } /* endif */ if ( sRc ) @@ -1250,54 +1244,36 @@ SHORT BTREE::QDAMReadRecordFromDisk_V3 SHORT BTREE::QDAMReadRecord_V3 ( USHORT usNumber, - PBTREEBUFFER_V3 * ppReadBuffer, + std::shared_ptr& pReadBuffer, BOOL fNewRec ) { USHORT i; SHORT sRc = 0; // return code BOOL fMemOK = FALSE; - PLOOKUPENTRY_V3 pLEntry = nullptr; //PACCESSCTRTABLEENTRY pACTEntry; DEBUGEVENT2( QDAMREADRECORD_LOC, FUNCENTRY_EVENT, usNumber, DB_GROUP, "" ); - *ppReadBuffer = NULL; INC_READ_COUNT; - sRc = checkLookupTableAndRealocate(usNumber); - - - if ( !sRc ) + if(LookupTable.count(usNumber)) { /******************************************************************/ /* If record is in memory set ppReadBuffer */ /* else call QDAMReadRecordFromDisk to read it into memory */ /******************************************************************/ - //pLEntry = LookupTable_V3 + usNumber; - pLEntry = &LookupTable_V3[usNumber]; - - if ( pLEntry->pBuffer ) - { - /* Safety-Check: is rec.number = number of rec. to read ? */ - if ( (pLEntry->pBuffer)->usRecordNumber == usNumber ) - { - /* Record is already in memory */ - *ppReadBuffer = pLEntry->pBuffer; - } - else - { - /* This should never occur ! */ - SET_AND_LOG(sRc,BTREE_LOOKUPTABLE_CORRUPTED) <<"This should never occur"; - } /* endif */ - } - else + pReadBuffer = LookupTable[usNumber]; + + /* Safety-Check: is rec.number = number of rec. to read ? */ + if ( pReadBuffer->usRecordNumber != usNumber ) { - /* Record isn't in memory -> read it from disk */ - sRc = QDAMReadRecordFromDisk_V3( usNumber, ppReadBuffer, fNewRec ); - //pACTEntry = &AccessCtrTable[usNumber]; - //pACTEntry->ulAccessCounter = 0L; - pLEntry->ulAccessCounter = 0; + pReadBuffer.reset(); + /* This should never occur ! */ + LOG_AND_SET_RC(sRc, T5ERROR, BTREE_LOOKUPTABLE_CORRUPTED); } /* endif */ + }else { + /* Record isn't in memory -> read it from disk */ + sRc = QDAMReadRecordFromDisk_V3( usNumber, pReadBuffer, fNewRec ); } /* endif */ /****************************************************************************/ @@ -1311,33 +1287,20 @@ SHORT BTREE::QDAMReadRecord_V3 { if ( ulReadRecCalls >= MAX_READREC_CALLS ) { - for ( i=0; !sRc && (i < LookupTable_V3.size()); i++ ) + //for ( i=0; !sRc && (i < LookupTable.size()); i++ ) + for(auto entry: LookupTable) { - pLEntry = &LookupTable_V3[i]; - //pACTEntry = &AccessCtrTable[i]; - if ( pLEntry->pBuffer && !((pLEntry->pBuffer)->fLocked) && (i!=usNumber) - && (pLEntry->ulAccessCounterfLocked) && (entry.second->usRecordNumber != usNumber))//&& (entry.second->usNumber!=usNumber)) { - /* write buffer and free allocated space */ - //if ( (pLEntry->pBuffer)->fNeedToWrite ) - { - sRc = QDAMWRecordToDisk_V3( pLEntry->pBuffer); - } /* endif */ + + sRc = QDAMWRecordToDisk_V3(entry.second); if ( !sRc ) { - UtlAlloc( (PVOID *)&(pLEntry->pBuffer), 0L, 0L , NOMSG ); - usNumberOfAllocatedBuffers--; + // usNumberOfAllocatedBuffers--; } /* endif */ } /* endif */ - if ( pLEntry->ulAccessCounterulAccessCounter = 0L; - } - else - { - pLEntry->ulAccessCounter -= MAX_READREC_CALLS; - } /* endif */ + } /* endfor */ ulReadRecCalls = 0L; } @@ -1347,12 +1310,6 @@ SHORT BTREE::QDAMReadRecord_V3 } /* endif */ } /* endif */ - if ( !sRc ) - { - /* Increase the access counter for the record just read by ACCESSBONUSPOINTS */ - pLEntry = &LookupTable_V3[usNumber]; - pLEntry->ulAccessCounter += ACCESSBONUSPOINTS; - } /* endif */ return ( sRc ); } @@ -1509,26 +1466,18 @@ SHORT BTREE::QDAMDictFlushLocal() } else { - //if ( pBT->LookupTable_V3 != NULL ) - { - BOOL fRecordWritten = FALSE; - for (int i=0; !sRc && (i < LookupTable_V3.size()); i++ ) - { - PLOOKUPENTRY_V3 pLEntry = &LookupTable_V3[i]; - if ( pLEntry->pBuffer /*&& (pLEntry->pBuffer)->fNeedToWrite*/ ) - { - sRc = QDAMWRecordToDisk_V3( pLEntry->pBuffer); - fRecordWritten = TRUE; - } /* endif */ - } /* endfor */ - } /* endif */ - } + BOOL fRecordWritten = FALSE; + for(auto& entry: LookupTable){ + //if(!entry->pBuffer.empty() /*&& (pLEntry->pBuffer)->fNeedToWrite*/){ + sRc = QDAMWRecordToDisk_V3( entry.second); + if(sRc){ + return sRc; + } + fRecordWritten = TRUE; + //} - if ( sRc ) - { - ERREVENT2( QDAMDICTFLUSHLOCAL_LOC, INTFUNCFAILED_EVENT, sRc, DB_GROUP, "" ); - } /* endif */ - DEBUGEVENT2( QDAMDICTFLUSHLOCAL_LOC, FUNCEXIT_EVENT, 0, DB_GROUP, "" ); + } + } return sRc; } @@ -1646,74 +1595,9 @@ SHORT BTREE::QDAMDictClose() //------------------------------------------------------------------------------ -//------------------------------------------------------------------------------ -// Internal function -//------------------------------------------------------------------------------ -// Function name: QDAMDestroy Destroy dictionary -//------------------------------------------------------------------------------ -// Function call: QDAMDestroy( PBTREE ); -// -//------------------------------------------------------------------------------ -// Description: Destroy the file if something went wrong during the -// create -// -//------------------------------------------------------------------------------ -// Parameters: PBTREE pointer to btree structure -// -//------------------------------------------------------------------------------ -// Returncode type: SHORT -//------------------------------------------------------------------------------ -// Returncodes: 0 no error happened -// BTREE_INVALID incorrect pointer -// BTREE_DISK_FULL disk full condition encountered -// BTREE_WRITE_ERROR write error to disk -// BTREE_CORRUPTED dictionary is corrupted -// BTREE_CLOSE_ERROR error closing dictionary -// -//------------------------------------------------------------------------------ -// Function flow: if ptr to BTree does not exist -// set RC = BTREE_INVALID -// else -// if filename exists -// call QDAMDictClose -// delete the file 'filename' -// endif -// endif -//------------------------------------------------------------------------------ - -SHORT QDAMDestroy -( - PBTREE pBT // pointer to generic structure -) -{ - SHORT sRc = 0; // return code - - CHAR chName[ MAX_EQF_PATH ]; // file name - - if ( !pBT ) - { - sRc = BTREE_INVALID; - } - else if (!pBT->fb.fileName.empty() ) - { - // - // QDAMDictClose frees up all of the memory associated with a B-tree, - // we need to save the filename as we'll be deleteing the file by name - // - strcpy(chName, pBT->fb.fileName.c_str()); - sRc = pBT->QDAMDictClose( ); - if ( chName[0] ) - { - UtlDelete( chName, 0L, FALSE); - } - } - return ( sRc ); -} - - SHORT BTREE::QDAMWriteRecord_V3 ( - PBTREEBUFFER_V3 pBuffer + std::shared_ptr& pBuffer ) { SHORT sRc = 0; // return code @@ -1869,11 +1753,11 @@ SHORT BTREE::QDAMAllocKeyRecords //if ( pBT->bRecSizeVersion == BTREE_V3 ) { - PBTREEBUFFER_V3 pRecord; + std::shared_ptr pRecord; while ( usNum-- && !sRc ) { usNextFreeRecord++; - sRc = QDAMReadRecord_V3( usNextFreeRecord, &pRecord, TRUE ); + sRc = QDAMReadRecord_V3( usNextFreeRecord, pRecord, TRUE ); if ( !sRc ) { @@ -2076,7 +1960,7 @@ SHORT BTREE::QDAMDictCreateLocal /* write an empty record buffer to the file before writing the */ /* header */ /****************************************************************/ - BTREEBUFFER_V3 pbuffer; + BTREEBUFFER pbuffer; memset((PVOID) &pbuffer, 0, BTREE_REC_SIZE_V3); sRc = fb.Write((PVOID)&pbuffer,BTREE_REC_SIZE_V3, 0); } @@ -2095,8 +1979,8 @@ SHORT BTREE::QDAMDictCreateLocal /*******************************************************************/ /* Write out an empty root node */ /*******************************************************************/ - PBTREEBUFFER_V3 pRecord; - sRc = QDAMReadRecord_V3( usFirstNode, &pRecord, TRUE ); + std::shared_ptr pRecord; + sRc = QDAMReadRecord_V3( usFirstNode, pRecord, TRUE ); if ( !sRc ) { TYPE(pRecord) = ROOT_NODE | LEAF_NODE | DATA_KEYNODE; @@ -2130,20 +2014,6 @@ SHORT BTREE::QDAMDictCreateLocal break; } /* endswitch */ - if ( !sRc ) - { - /****************************************************************/ - /* add this dictionary to our list ... */ - /****************************************************************/ - sRc = QDAMAddDict( (PSZ)fb.fileName.c_str(), this ); - } - else - { - // leave the return code from create - // file will not be destroyed if create failed, since then - // filename is not set - QDAMDestroy( this ); - } /* endif */ return( sRc ); } @@ -2303,7 +2173,7 @@ VOID BTREE::QDAMDictUpdStatus() PWCHAR QDAMGetszKey_V3 ( - PBTREEBUFFER_V3 pRecord, // active record + std::shared_ptr& pRecord, // active record USHORT i, // get data term USHORT usVersion // version of database ) @@ -2343,7 +2213,7 @@ PWCHAR QDAMGetszKey_V3 RECPARAM QDAMGetrecData_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid, // key number USHORT usVersion // version of database ) @@ -2427,7 +2297,7 @@ RECPARAM QDAMGetrecData_V3 SHORT BTREE::QDAMFindRecord_V3 ( PWCHAR pKey, - PBTREEBUFFER_V3 * ppRecord + std::shared_ptr& pRecord ) { SHORT sResult; @@ -2439,19 +2309,19 @@ SHORT BTREE::QDAMFindRecord_V3 SHORT sRc; // return code memset(&recData, 0, sizeof(recData)); - sRc = QDAMReadRecord_V3( usFirstNode, ppRecord, FALSE ); - while ( !sRc && !IS_LEAF( *ppRecord )) + sRc = QDAMReadRecord_V3( usFirstNode, pRecord, FALSE ); + while ( !sRc && !IS_LEAF( pRecord )) { - BTREELOCKRECORD( *ppRecord ); + BTREELOCKRECORD( pRecord ); sLow = 0; // start here - sHigh = (SHORT) OCCUPIED( *ppRecord) -1 ; // counting starts at zero + sHigh = (SHORT) OCCUPIED(pRecord) -1 ; // counting starts at zero while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( *ppRecord, sMid, usVersion ); + pKey2 = QDAMGetszKey_V3( pRecord, sMid, usVersion ); if ( pKey2 == nullptr ) { sRc = BTREE_CORRUPTED; @@ -2481,14 +2351,14 @@ SHORT BTREE::QDAMFindRecord_V3 { sHigh = 0; } /* endif */ - recData = QDAMGetrecData_V3( *ppRecord, sHigh, usVersion ); + recData = QDAMGetrecData_V3( pRecord, sHigh, usVersion ); } /* endif */ - BTREEUNLOCKRECORD( *ppRecord ); // unlock previous record. + BTREEUNLOCKRECORD( pRecord ); // unlock previous record. if ( !sRc ) { - sRc = QDAMReadRecord_V3( recData.usNum, ppRecord, FALSE ); + sRc = QDAMReadRecord_V3( recData.usNum, pRecord, FALSE ); } /* endif */ } /* endwhile */ return( sRc ); @@ -2543,12 +2413,14 @@ SHORT BTREE::QDAMFindRecord_V3 BOOL BTREE::QDAMTerseData ( - PUCHAR pData, // pointer to data + std::vector& pData, // pointer to data PULONG pulLen // length of the string ) { + BOOL fShorter = FALSE; // new string is not shorter + T5LOG(T5FATAL) <<"TEMPORARY COMMENTED FUNCTION"; + #ifdef TEMPORARY_COMMENTED ULONG ulLen ; // length of string - BOOL fShorter = FALSE; // new string is not shorter USHORT usCurByte = 0; // currently processed byte PUCHAR pTempData; // pointer to temp data area PUCHAR pEndData; // pointer to end data area @@ -2645,6 +2517,7 @@ BOOL BTREE::QDAMTerseData } /* endif */ #if defined(MEASURE) ulTerseEnd += (pGlobInfoSeg->msecs -ulBeg ); +#endif #endif return ( fShorter ); } @@ -2692,11 +2565,11 @@ BOOL BTREE::QDAMTerseData // update ptr to next free key buffer // endif // endif -// reset info in *ppRecord +// reset info in pRecord //------------------------------------------------------------------------------ SHORT BTREE::QDAMNewRecord_V3 ( - PBTREEBUFFER_V3 * ppRecord, + std::shared_ptr& pRecord, RECTYPE recType // data or key record ) { @@ -2705,15 +2578,15 @@ SHORT BTREE::QDAMNewRecord_V3 // Try and use a record within the file if there are any. This saves // on disk usage // - *ppRecord = NULL; // in case of error + pRecord = NULL; // in case of error if ( recType == DATAREC ) { if ( usFreeDataBuffer ) { - sRc = QDAMReadRecord_V3( usFreeDataBuffer, ppRecord, FALSE ); + sRc = QDAMReadRecord_V3( usFreeDataBuffer, pRecord, FALSE ); if ( ! sRc ) { - usFreeDataBuffer = NEXT( *ppRecord ); + usFreeDataBuffer = NEXT( pRecord ); } /* endif */ } else @@ -2725,7 +2598,7 @@ SHORT BTREE::QDAMNewRecord_V3 else { usNextFreeRecord++; - sRc = QDAMReadRecord_V3( usNextFreeRecord, ppRecord, TRUE ); + sRc = QDAMReadRecord_V3( usNextFreeRecord, pRecord, TRUE ); } } } @@ -2738,18 +2611,18 @@ SHORT BTREE::QDAMNewRecord_V3 if ( !sRc ) { - sRc = QDAMReadRecord_V3( usFreeKeyBuffer, ppRecord, FALSE ); + sRc = QDAMReadRecord_V3( usFreeKeyBuffer, pRecord, FALSE ); if ( ! sRc ) { - usFreeKeyBuffer = NEXT( *ppRecord ); + usFreeKeyBuffer = NEXT( pRecord ); } /* endif */ } /* endif */ } /* endif */ - if ( *ppRecord ) + if ( pRecord ) { - NEXT(*ppRecord) = 0; // reset information - PREV(*ppRecord) = 0; // reset information + NEXT(pRecord) = 0; // reset information + PREV(pRecord) = 0; // reset information } /* endif */ if ( sRc ) @@ -2818,9 +2691,8 @@ SHORT BTREE::QDAMNewRecord_V3 //------------------------------------------------------------------------------ SHORT BTREE::QDAMAddToBuffer_V3 ( - PBYTE pData, // pointer to data - ULONG ulDataLen, // length of data to be filled in - PRECPARAM precReturn // pointer to return code + std::vector& Data, // pointer to data + PRECPARAM precReturn // pointer to return code ) { SHORT sRc = 0; // success indicator @@ -2833,8 +2705,8 @@ SHORT BTREE::QDAMAddToBuffer_V3 BOOL fFit = FALSE; // indicator if record found USHORT i; // index PBTREEHEADER pHeader; // record header - PBTREEBUFFER_V3 pRecord = NULL; // buffer record - //PBTREEBUFFER_V3 pTempRecord; // buffer record + std::shared_ptr pRecord; // buffer record + //std::shared_ptr pTempRecord; // buffer record CHAR chNodeType; // type of node BOOL fTerse = FALSE; // data are not tersed USHORT usLastPos; @@ -2846,28 +2718,17 @@ SHORT BTREE::QDAMAddToBuffer_V3 memset(&recStart, 0, sizeof(recStart)); memset(&TempRecord[0], 0, sizeof(TempRecord)); + PBYTE pData = nullptr; + ULONG ulDataLen = Data.size(); + /*******************************************************************/ /* Enlarge pTempRecord area if it is not large enough to contain */ /* the data for this record */ /*******************************************************************/ - if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)) + + if ( (Data.size() + sizeof(ULONG)) > TempRecord.size() ) { - T5LOG( T5DEBUG) << ":: data len = " << ulDataLen ; - //CHAR szTemp[8]; - //sprintf( szTemp, "%ld", ulDataLen ); - //INFOEVENT2( QDAMADDTOBUFFER_LOC, STATE_EVENT, 4711, DB_GROUP, szTemp ); - } - if ( (ulDataLen + sizeof(ULONG)) > TempRecord.size() ) - { - TempRecord.resize(ulDataLen + sizeof(ULONG)); - //if ( UtlAlloc( (PVOID *)&(pTempRecord), ulTempRecSize, ulDataLen + sizeof(ULONG), NOMSG ) ) - //{ - // ulTempRecSize = ulDataLen + sizeof(ULONG); - // } - //else - //{ - // sRc = BTREE_NO_ROOM; - //} /* endif */ + TempRecord.resize(Data.size() + sizeof(ULONG)); } /* endif */ usLenFieldSize = sizeof(ULONG); @@ -2878,7 +2739,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 switch ( fTerse ) { case BTREE_TERSE_HUFFMAN : - fTerse = QDAMTerseData(pData, &ulDataLen ); + fTerse = QDAMTerseData(Data, &ulDataLen ); break; default : fTerse = FALSE; @@ -2891,15 +2752,16 @@ SHORT BTREE::QDAMAddToBuffer_V3 // copy them if not yet done..... if ( !fTerse ) { - memcpy( &TempRecord[0], pData, ulDataLen ); + //memcpy( &TempRecord[0], pData, ulDataLen ); + TempRecord = Data; } /* endif */ - if ( !sRc && (ulDataLen <= BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER)) ) + if ( !sRc && (Data.size() <= BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER)) ) { pRecTemp = pRecParam; i = 0; // find slot in list - usFilled = (USHORT)(BTREE_REC_SIZE_V3 - ulDataLen - 2 * usLenFieldSize); + usFilled = (USHORT)(BTREE_REC_SIZE_V3 - Data.size() - 2 * usLenFieldSize); while ( !fFit && i < MAX_LIST ) { if ( pRecTemp->usOffset == 0 || pRecTemp->usOffset > usFilled ) // will not fit @@ -2919,7 +2781,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 recStart.usNum = pRecTemp->usNum; // get record, copy data and write them - sRc = QDAMReadRecord_V3( recStart.usNum, &pRecord, FALSE ); + sRc = QDAMReadRecord_V3( recStart.usNum, pRecord, FALSE ); if ( !sRc ) { BTREELOCKRECORD( pRecord ); @@ -2932,7 +2794,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 { // fill in key data usLastPos = pRecord->contents.header.usLastFilled; - usLastPos = usLastPos - (USHORT)(ulDataLen + usLenFieldSize); + usLastPos = usLastPos - (USHORT)(Data.size() + usLenFieldSize); pData = pRecord->contents.uchData + usLastPos; // insert reference *(PULONG) pData = ulDataLen; @@ -3008,7 +2870,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 // get record and store start position memset( pRecTemp, 0 , sizeof(RECPARAM)); - sRc = QDAMNewRecord_V3( &pRecord, DATAREC ); + sRc = QDAMNewRecord_V3( pRecord, DATAREC ); // fill it up to length or record size if ( ! sRc ) { @@ -3022,9 +2884,9 @@ SHORT BTREE::QDAMAddToBuffer_V3 /* get a new record, anchor it and fill data from end */ /* new record will be predecessor of already allocated rec.*/ /***********************************************************/ - PBTREEBUFFER_V3 pTempRecBuff; - //sRc = QDAMNewRecord_V3( (PBTREEBUFFER_V3*)&TempRecord[0], DATAREC ); - sRc = QDAMNewRecord_V3( &pTempRecBuff, DATAREC ); + std::shared_ptr pTempRecBuff; + //sRc = QDAMNewRecord_V3( (PBTREEBUFFER) &TempRecord[0], DATAREC ); + sRc = QDAMNewRecord_V3( pTempRecBuff, DATAREC ); if ( !sRc ) { @@ -3070,7 +2932,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 /* toggle for adressing purposes */ /* now: pRecord will be the free one for later filling */ /*********************************************************/ - //pRecord = (PBTREEBUFFER_V3)&TempRecord[0]; + //pRecord = (PBTREEBUFFER)&TempRecord[0]; pRecord = pTempRecBuff; BTREELOCKRECORD( pRecord ); } /* endif */ @@ -3199,7 +3061,7 @@ SHORT QDAMCaseCompare SHORT BTREE::QDAMLocateKey_V3 ( - PBTREEBUFFER_V3 pRecord, // record to be dealt with + std::shared_ptr& pRecord, // record to be dealt with PWCHAR pKey, // key to be searched PSHORT psKeyPos, // located key SEARCHTYPE searchType, // search type @@ -3356,7 +3218,7 @@ SHORT BTREE::QDAMLocateKey_V3 SHORT QDAMFirstEntry_V3 ( PBTREE pBTIda, - PBTREEBUFFER_V3 * ppRecord // pointer to pointer of record + std::shared_ptr& pRecord // pointer to pointer of record ) { SHORT sRc = 0; @@ -3364,17 +3226,17 @@ SHORT QDAMFirstEntry_V3 PBTREE pBT = pBTIda; // read in root record - sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, ppRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, pRecord, FALSE ); if ( !sRc ) { - while (!sRc && !IS_LEAF(*ppRecord)) + while (!sRc && !IS_LEAF(pRecord)) { - recData = QDAMGetrecData_V3( *ppRecord, 0, pBT->usVersion); - sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, ppRecord , FALSE ); + recData = QDAMGetrecData_V3( pRecord, 0, pBT->usVersion); + sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, pRecord , FALSE ); } /* endwhile */ if ( !sRc ) { - pBTIda->usCurrentRecord = RECORDNUM(*ppRecord); + pBTIda->usCurrentRecord = RECORDNUM(pRecord); pBT->usFirstLeaf = pBTIda->usCurrentRecord; // determine first leaf pBTIda->sCurrentIndex = 0; } /* endif */ @@ -3446,7 +3308,7 @@ SHORT QDAMFirstEntry_V3 SHORT QDAMFindParent_V3 ( PBTREE pBTIda, - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, PUSHORT pusParent // record number of parent ) { @@ -3459,7 +3321,7 @@ SHORT QDAMFindParent_V3 PCHAR_W pKey ; // pointer to search key CHAR_W chKey[HEADTERM_SIZE]; // key to be found SHORT sRc; // return code - PBTREEBUFFER_V3 pTempRec; // temp. record buffer + std::shared_ptr pTempRec; // temp. record buffer USHORT usRecNum; // passed record number PBTREE pBT = pBTIda; @@ -3477,7 +3339,7 @@ SHORT QDAMFindParent_V3 memcpy( chKey, pKey, sizeof(ULONG)); } /* endif */ - sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, &pTempRec, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, pTempRec, FALSE ); while ( !sRc && !IS_LEAF( pTempRec ) && (usRecNum != RECORDNUM(pTempRec)) ) { @@ -3529,7 +3391,7 @@ SHORT QDAMFindParent_V3 if ( !sRc ) { - sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, &pTempRec, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, pTempRec, FALSE ); } /* endif */ } /* endwhile */ return( sRc ); @@ -3563,9 +3425,9 @@ SHORT QDAMFindParent_V3 VOID QDAMCopyKeyTo_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT i, - PBTREEBUFFER_V3 pNew, + std::shared_ptr& pNew, SHORT j, USHORT usVersion // version of database ) @@ -3634,18 +3496,17 @@ VOID QDAMCopyKeyTo_V3 VOID QDAMReArrangeKRec_V3 ( PBTREE pBTIda, - PBTREEBUFFER_V3 pRecord + std::shared_ptr& pRecord ) { - PBTREEBUFFER_V3 pNew; PBTREEHEADER pHeader; // pointer to header PUSHORT pusOffset; SHORT i; SHORT j; PBTREE pBT = pBTIda; + std::shared_ptr pNew = pBT->BTreeTempBuffer_V3; // get temp record - pNew = &pBT->BTreeTempBuffer_V3; pHeader = &pNew->contents.header; pHeader->usOccupied = 0; pHeader->usFilled = sizeof(BTREEHEADER ); @@ -3763,16 +3624,14 @@ VOID QDAMReArrangeKRec_V3 SHORT QDAMSplitNode_V3 ( PBTREE pBTIda, // pointer to generic structure - PBTREEBUFFER_V3 *record, // pointer to pointer to node + std::shared_ptr& record, // pointer to pointer to node PWCHAR pKey // new key ) { SHORT i,j; - PBTREEBUFFER_V3 newRecord; - PBTREEBUFFER_V3 child; - PBTREEBUFFER_V3 parent = NULL; + std::shared_ptr newRecord, child, parent, pRecTemp; + USHORT usParent; // number of parent - PBTREEBUFFER_V3 pRecTemp; // temporary buffer RECPARAM recKey; // position/offset for key RECPARAM recData; // position/offset for data PWCHAR pParentKey; // new key to be inserted @@ -3789,27 +3648,27 @@ SHORT QDAMSplitNode_V3 memset( &recKey, 0, sizeof( recKey ) ); memset( &recData, 0, sizeof( recData ) ); - pRecTemp = *record; + pRecTemp = record; BTREELOCKRECORD( pRecTemp ); // if root needs to be split do it first - if (IS_ROOT(*record)) + if (IS_ROOT(record)) { - sRc = pBTIda->QDAMNewRecord_V3( &newRecord, KEYREC ); + sRc = pBTIda->QDAMNewRecord_V3( newRecord, KEYREC ); if ( newRecord ) { BTREELOCKRECORD( newRecord ); /* We can't simply split a root node, since only one is allowed */ /* so a new root is created to hold the records */ - pBT->usFirstNode = PARENT(*record) = RECORDNUM(newRecord); + pBT->usFirstNode = PARENT(record) = RECORDNUM(newRecord); TYPE(newRecord) = ROOT_NODE | INNER_NODE | DATA_KEYNODE; PARENT(newRecord) = PREV(newRecord) = NEXT(newRecord) = 0L; - TYPE(*record) &= ~ROOT_NODE; - recData.usNum = RECORDNUM(*record); + TYPE(record) &= ~ROOT_NODE; + recData.usNum = RECORDNUM(record); recData.usOffset = 0; //pParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); - pParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); + pParentKey = QDAMGetszKey_V3( record, 0, pBT->usVersion ); if ( pParentKey ) { @@ -3828,7 +3687,7 @@ SHORT QDAMSplitNode_V3 // update usFirstLeaf information if ( !sRc ) { - sRc = QDAMFirstEntry_V3( pBTIda, &newRecord ); + sRc = QDAMFirstEntry_V3( pBTIda, newRecord ); } /* endif */ if ( !sRc ) { @@ -3840,13 +3699,13 @@ SHORT QDAMSplitNode_V3 if ( !sRc ) { // allocate space for the new record - sRc = pBTIda->QDAMNewRecord_V3( &newRecord, KEYREC ); + sRc = pBTIda->QDAMNewRecord_V3( newRecord, KEYREC ); if ( newRecord ) { BTREELOCKRECORD(newRecord); - if ( NEXT(*record) ) + if ( NEXT(record) ) { - sRc = pBTIda->QDAMReadRecord_V3( NEXT(*record), &child, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( NEXT(record), child, FALSE ); if ( !sRc ) { PREV(child) = RECORDNUM(newRecord); @@ -3857,15 +3716,15 @@ SHORT QDAMSplitNode_V3 if ( !sRc ) { /* Adjust Sibling pointers */ - NEXT(newRecord) = NEXT(*record); - PREV(newRecord) = RECORDNUM(*record); - NEXT(*record) = RECORDNUM(newRecord); - PARENT(newRecord) = PARENT(*record); - TYPE(newRecord) = (CHAR)(TYPE(*record) & ~ROOT_NODE); // don't copy Root bit + NEXT(newRecord) = NEXT(record); + PREV(newRecord) = RECORDNUM(record); + NEXT(record) = RECORDNUM(newRecord); + PARENT(newRecord) = PARENT(record); + TYPE(newRecord) = (CHAR)(TYPE(record) & ~ROOT_NODE); // don't copy Root bit // Decide where to split the record //pParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); - pParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); + pParentKey = QDAMGetszKey_V3( record, (SHORT)(OCCUPIED(record)/2), pBT->usVersion ); if ( pParentKey ) { @@ -3875,7 +3734,7 @@ SHORT QDAMSplitNode_V3 /***********************************************************/ if ( fCompare ) { - pParentKey = QDAMGetszKey_V3(*record, (SHORT)(OCCUPIED(*record)-MINFREEKEYS), pBT->usVersion); + pParentKey = QDAMGetszKey_V3(record, (SHORT)(OCCUPIED(record)-MINFREEKEYS), pBT->usVersion); fCompare = (NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0 ) ; if ( fCompare ) { @@ -3883,20 +3742,20 @@ SHORT QDAMSplitNode_V3 } else { - usFreeKeys = OCCUPIED( *record )/2; + usFreeKeys = OCCUPIED( record )/2; } /* endif */ } else { - pParentKey = QDAMGetszKey_V3( *record,MINFREEKEYS, pBT->usVersion); + pParentKey = QDAMGetszKey_V3( record,MINFREEKEYS, pBT->usVersion); fCompare = (NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0 ) ; if ( fCompare ) { - usFreeKeys = OCCUPIED( *record )/2; + usFreeKeys = OCCUPIED( record )/2; } else { - usFreeKeys = OCCUPIED( *record ) -MINFREEKEYS; + usFreeKeys = OCCUPIED( record ) -MINFREEKEYS; } /* endif */ } /* endif */ } @@ -3913,26 +3772,26 @@ SHORT QDAMSplitNode_V3 // ELSE case... if ( !sRc ) { - i = (SHORT) (OCCUPIED( *record ) - usFreeKeys); + i = (SHORT) (OCCUPIED( record ) - usFreeKeys); j = 0; - pusOffset = (PUSHORT) (*record)->contents.uchData; - while ( i < (SHORT) OCCUPIED( *record )) + pusOffset = (PUSHORT) (record)->contents.uchData; + while ( i < (SHORT) OCCUPIED( record )) { - QDAMCopyKeyTo_V3( *record, i, newRecord, j, pBT->usVersion ); + QDAMCopyKeyTo_V3( record, i, newRecord, j, pBT->usVersion ); *(pusOffset+i) = 0 ; // mark it as deleted j++; i++; } /* Adjust count of keys */ - OCCUPIED(*record) = OCCUPIED(*record) - usFreeKeys; + OCCUPIED(record) = OCCUPIED(record) - usFreeKeys; OCCUPIED(newRecord) = usFreeKeys; - QDAMReArrangeKRec_V3( pBTIda, *record ); + QDAMReArrangeKRec_V3( pBTIda, record ); /* Insert pointer to new record into the parent node */ /* due to the construction parent MUST be the same */ - sRc = QDAMFindParent_V3( pBTIda, *record, &usParent ); + sRc = QDAMFindParent_V3( pBTIda, record, &usParent ); if ( !sRc && usParent ) { - sRc = pBTIda->QDAMReadRecord_V3( usParent, &parent, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( usParent, parent, FALSE ); } /* endif */ } /* endif */ @@ -3963,8 +3822,8 @@ SHORT QDAMSplitNode_V3 { if ( NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0) { - sRc = pBTIda->QDAMWriteRecord_V3( *record); - *record = newRecord; // add to new record + sRc = pBTIda->QDAMWriteRecord_V3( record); + record = newRecord; // add to new record } else { @@ -4220,7 +4079,7 @@ SHORT QDAMGetszData_V3 ) { SHORT sRc = 0; - PBTREEBUFFER_V3 pRecord; // pointer to record + std::shared_ptr pRecord; // pointer to record PBTREEHEADER pHeader; // pointer to header PCHAR pTempData = NULL; // pointer to data pointer LONG lLen = 0; // length of string @@ -4240,7 +4099,7 @@ SHORT QDAMGetszData_V3 recDataParam.usNum = recDataParam.usNum + pBT->usFirstDataBuffer; // get record and copy data - sRc = pBTIda->QDAMReadRecord_V3( recDataParam.usNum, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( recDataParam.usNum, pRecord, FALSE ); if ( !sRc ) { BTREELOCKRECORD( pRecord ); @@ -4324,7 +4183,7 @@ SHORT QDAMGetszData_V3 fRecLocked = FALSE; if ( usNum ) { - sRc = pBTIda->QDAMReadRecord_V3( usNum, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( usNum, pRecord, FALSE ); if ( !sRc && TYPE( pRecord ) != DATA_NEXTNODE ) { sRc = BTREE_CORRUPTED; @@ -4444,7 +4303,7 @@ SHORT QDAMFindChild_V3 PBTREE pBTIda, PCHAR_W pKey, USHORT usNode, - PBTREEBUFFER_V3 * ppRecord + std::shared_ptr& pRecord ) { SHORT sResult; @@ -4457,19 +4316,19 @@ SHORT QDAMFindChild_V3 PBTREE pBT = pBTIda; memset(&recData, 0, sizeof(recData)); - sRc = pBTIda->QDAMReadRecord_V3( usNode, ppRecord, FALSE ); - if ( !sRc && !IS_LEAF( *ppRecord )) + sRc = pBTIda->QDAMReadRecord_V3( usNode, pRecord, FALSE ); + if ( !sRc && !IS_LEAF( pRecord )) { - BTREELOCKRECORD( *ppRecord ); + BTREELOCKRECORD( pRecord ); sLow = 0; // start here - sHigh = (SHORT) OCCUPIED( *ppRecord) -1 ; // counting starts at zero + sHigh = (SHORT) OCCUPIED( pRecord) -1 ; // counting starts at zero while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( *ppRecord, sMid, pBT->usVersion ); + pKey2 = QDAMGetszKey_V3( pRecord, sMid, pBT->usVersion ); if ( pKey2 == NULL ) { sRc = BTREE_CORRUPTED; @@ -4500,14 +4359,14 @@ SHORT QDAMFindChild_V3 { sHigh = 0; } /* endif */ - recData = QDAMGetrecData_V3( *ppRecord, sHigh, pBT->usVersion ); + recData = QDAMGetrecData_V3( pRecord, sHigh, pBT->usVersion ); } /* endif */ - BTREEUNLOCKRECORD( *ppRecord ); // unlock previous record. + BTREEUNLOCKRECORD( pRecord ); // unlock previous record. if ( !sRc ) { - sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, ppRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, pRecord, FALSE ); } /* endif */ } /* endif */ return( sRc ); @@ -4522,8 +4381,8 @@ SHORT QDAMChangeKey_V3 PWCHAR pNewKey // find new key ) { - PBTREEBUFFER_V3 pRecord; // buffer for record - PBTREEBUFFER_V3 pNewRecord = NULL; // buffer for new record + std::shared_ptr pRecord; // buffer for record + std::shared_ptr pNewRecord; // buffer for new record SHORT i = 0; // index SHORT sRc = 0; // return code SHORT sNearKey; @@ -4531,14 +4390,14 @@ SHORT QDAMChangeKey_V3 RECPARAM recKey; // record parameter descrip. - sRc = pBTIda->QDAMReadRecord_V3( usNode, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( usNode, pRecord, FALSE ); if ( !sRc ) { BTREELOCKRECORD(pRecord); if ( !sRc ) { /* Locate the Leaf node that contains the appropriate key */ - sRc = QDAMFindChild_V3( pBTIda, pNewKey, usNode, &pNewRecord ); + sRc = QDAMFindChild_V3( pBTIda, pNewKey, usNode, pNewRecord ); } /* endif */ if ( !sRc ) { @@ -4638,14 +4497,14 @@ size_t BTREE::GetFileSize()const{ //------------------------------------------------------------------------------ SHORT BTREE::QDAMInsertKey_V3 ( - PBTREEBUFFER_V3 pRecord, // record where key is to be inserted + std::shared_ptr& pRecord, // record where key is to be inserted PWCHAR pKey, RECPARAM recKey, // position/offset for key RECPARAM recData // position/offset for data ) { SHORT i = 0; - PBTREEBUFFER_V3 pTempRec; + std::shared_ptr pTempRec; PWCHAR pCompKey = NULL; // key to be compared with PWCHAR pOldKey; // old key at first position PWCHAR pNewKey; // new key at first position @@ -4686,7 +4545,7 @@ SHORT BTREE::QDAMInsertKey_V3 { BTREEUNLOCKRECORD( pRecord ); fRecLocked = FALSE; - sRc = QDAMSplitNode_V3( this, &pRecord, pKey ); + sRc = QDAMSplitNode_V3( this, pRecord, pKey ); if ( !sRc ) { // SplitNode may have passed a new record back @@ -4697,7 +4556,7 @@ SHORT BTREE::QDAMInsertKey_V3 /* inserting must be changed */ if ( !IS_LEAF( pRecord ) ) { - sRc = QDAMReadRecord_V3( recData.usNum, &pTempRec, FALSE ); + sRc = QDAMReadRecord_V3( recData.usNum,pTempRec, FALSE ); if ( ! sRc ) { PARENT( pTempRec ) = RECORDNUM( pRecord ); @@ -5116,54 +4975,7 @@ SHORT BTREE::QDAMDictOpenLocal QDAMDictClose(); } /* endif */ ASDLOG(); - - if ( !sRc && /*fWrite &&*/ !header.fOpen) - { - //fOpen = TRUE; // set open flag - } /* endif */ - - #ifdef TEMPORARY_COMMENTED - // get the corruption flag but still go ahead - nec. for Organize - // ignore the corruption flag for shared resources (here we allow - // open of our database more than once) - if ( !sRc && header.fOpen && - !(pBT->usOpenFlags & (ASD_SHARED | ASD_NOOPENCHECK)) ) - { - sRc = BTREE_CORRUPTED; - - /**************************************************************/ - /* if in Write Mode opened disable write in case of Corruption*/ - /* and reset it to Read/Only mode */ - /**************************************************************/ - if ( fWrite ) - { - i = UtlSetFHandState( pBT->fb.file, OPEN_ACCESS_READONLY, FALSE ); - if ( i ) // handle could not be set read/only - { - pBT->fCorrupted = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ - #endif //TEMPORARY_COMMENTED - ASDLOG(); } - else - { - sRc = QDAMDosRC2BtreeRC( sRc, BTREE_OPEN_ERROR, usOpenFlags ); - } /* endif */ - - /*******************************************************************/ - /* set BTREE pointer in case it was changed or freed */ - /*******************************************************************/ - //*ppBTIda = pBTIda; - - /*******************************************************************/ - /* add the dictionary to the open list ... */ - /*******************************************************************/ - if ( !sRc || (sRc == BTREE_CORRUPTED) ) - { - QDAMAddDict( (PSZ)fb.fileName.c_str(), this ); - } /* endif */ if ( sRc ) { @@ -5357,8 +5169,7 @@ SHORT BTREE::QDAMDictOpenLocal SHORT BTREE::QDAMDictInsertLocal ( PWCHAR pKey, // pointer to key data - PBYTE pData, // pointer to user data - ULONG ulLen // length of user data in bytes + std::vector& pData // pointer to user data ) { SHORT sRc = 0; // return code @@ -5373,7 +5184,7 @@ SHORT BTREE::QDAMDictInsertLocal { sRc = BTREE_CORRUPTED; } - else if ( ulLen == 0) + else if ( pData.empty()) { sRc = BTREE_DATA_RANGE; } @@ -5386,26 +5197,26 @@ SHORT BTREE::QDAMDictInsertLocal sRc = BTREE_ENTRY_LOCKED; } /* endif */ - PBTREEBUFFER_V3 pNode = NULL; + std::shared_ptr pNode; if ( !sRc ) { usKeyLen = (USHORT) sizeof(ULONG); //memcpy( (PBYTE)chHeadTerm, (PBYTE)pKey, usKeyLen);//+sizeof(TMWCHAR) ); // save current data QDAMDictUpdStatus (); - sRc = QDAMFindRecord_V3( pKey, &pNode ); + sRc = QDAMFindRecord_V3( pKey, pNode ); } /* endif */ if ( !sRc ) { BTREELOCKRECORD( pNode ); - sRc = QDAMAddToBuffer_V3( pData, ulLen, &recData ); + sRc = QDAMAddToBuffer_V3( pData, &recData ); if ( !sRc ) { - if(ulLen > TMX_REC_SIZE && T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << ulLen; + if(pData.size() > TMX_REC_SIZE && T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ + T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << pData.size(); } - recData.ulLen = ulLen; + recData.ulLen = pData.size(); sRc = QDAMInsertKey_V3 ( pNode, pKey, recKey, recData); } /* endif */ BTREEUNLOCKRECORD( pNode ); @@ -5812,25 +5623,25 @@ BTREE::QDAMCheckDict SHORT QDAMValidateIndex_V3 ( PBTREE pBTIda, - PBTREEBUFFER_V3 * ppRecord + std::shared_ptr& pRecord ) { SHORT sRc = 0; // set return code USHORT usRecord; if ( pBTIda->sCurrentIndex < 0 || - pBTIda->sCurrentIndex >= (SHORT) OCCUPIED(*ppRecord)) + pBTIda->sCurrentIndex >= (SHORT) OCCUPIED(pRecord)) { if ( pBTIda->sCurrentIndex < 0 ) { - if ( PREV( *ppRecord )) + if ( PREV( pRecord )) { - usRecord = PREV(*ppRecord); - sRc = pBTIda->QDAMReadRecord_V3( usRecord, ppRecord, FALSE ); + usRecord = PREV(pRecord); + sRc = pBTIda->QDAMReadRecord_V3( usRecord, pRecord, FALSE ); if (!sRc ) { pBTIda->usCurrentRecord = usRecord; - pBTIda->sCurrentIndex = (SHORT) (OCCUPIED(*ppRecord) - 1); + pBTIda->sCurrentIndex = (SHORT) (OCCUPIED(pRecord) - 1); } /* endif */ } else @@ -5840,12 +5651,12 @@ SHORT QDAMValidateIndex_V3 sRc = BTREE_EOF_REACHED; } /* endif */ } - else if ( pBTIda->sCurrentIndex >= (SHORT) OCCUPIED(*ppRecord)) + else if ( pBTIda->sCurrentIndex >= (SHORT) OCCUPIED(pRecord)) { - if ( NEXT(*ppRecord) ) + if ( NEXT(pRecord) ) { - pBTIda->usCurrentRecord = NEXT(*ppRecord); - sRc = pBTIda->QDAMReadRecord_V3( pBTIda->usCurrentRecord, ppRecord, FALSE ); + pBTIda->usCurrentRecord = NEXT(pRecord); + sRc = pBTIda->QDAMReadRecord_V3( pBTIda->usCurrentRecord, pRecord, FALSE ); pBTIda->sCurrentIndex = 0; } else @@ -5865,7 +5676,7 @@ SHORT QDAMValidateIndex_V3 RECPARAM QDAMGetrecKey_V3 ( - PBTREEBUFFER_V3 pRecord, + std::shared_ptr& pRecord, SHORT sMid // key number ); @@ -5928,7 +5739,7 @@ SHORT QDAMNext_V3 ) { SHORT sRc = 0; - PBTREEBUFFER_V3 pRecord; + std::shared_ptr pRecord; PBTREE pBT = pBTIda; memset( precBTree, 0, sizeof(RECPARAM)); @@ -5941,18 +5752,18 @@ SHORT QDAMNext_V3 // if usCurrentRecord = 0 we are at EOF if ( !pBTIda->usCurrentRecord ) { - sRc = QDAMFirstEntry_V3( pBTIda, &pRecord ); + sRc = QDAMFirstEntry_V3( pBTIda, pRecord ); } else { pBTIda->sCurrentIndex ++; - sRc = pBTIda->QDAMReadRecord_V3( pBTIda->usCurrentRecord, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( pBTIda->usCurrentRecord, pRecord, FALSE ); } /* endif */ if ( !sRc ) { - sRc = QDAMValidateIndex_V3( pBTIda, &pRecord ); + sRc = QDAMValidateIndex_V3( pBTIda, pRecord ); if ( !sRc ) { precBTree->usOffset = pBTIda->sCurrentIndex; @@ -6020,12 +5831,12 @@ SHORT QDAMGetszKeyParam_V3 ) { PCHAR pData = NULL; - PBTREEBUFFER_V3 pRecord; // active record + std::shared_ptr pRecord; // active record SHORT sRc = 0; // return code PBTREE pBT = pBTIda; USHORT usLen; - sRc = pBTIda->QDAMReadRecord_V3( recKey.usNum, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3( recKey.usNum, pRecord, FALSE ); /*******************************************************************/ /* check length */ @@ -6112,7 +5923,7 @@ SHORT QDAMFirst_V3 ) { SHORT sRc = 0; - PBTREEBUFFER_V3 pRecord; + std::shared_ptr pRecord; PBTREE pBT = pBTIda; memset( precBTree, 0, sizeof(RECPARAM)); @@ -6129,7 +5940,7 @@ SHORT QDAMFirst_V3 } else { - sRc = pBTIda->QDAMReadRecord_V3(pBTIda->usCurrentRecord, &pRecord, FALSE ); + sRc = pBTIda->QDAMReadRecord_V3(pBTIda->usCurrentRecord, pRecord, FALSE ); if ( !sRc ) { /***********************************************************************/ diff --git a/source/opentm2/core/utilities/UtlAlloc.cpp b/source/opentm2/core/utilities/UtlAlloc.cpp index e158d3aa..36a5c1ae 100755 --- a/source/opentm2/core/utilities/UtlAlloc.cpp +++ b/source/opentm2/core/utilities/UtlAlloc.cpp @@ -100,24 +100,6 @@ BOOL UtlAlloc //| USHORT usMsgNo, | //| HWND hwndOwner ); | //+----------------------------------------------------------------------------+ -BOOL UtlAllocHwnd -( - PVOID *ppStorage, // pointer to allocated memory area - LONG lOldLength, // old length of storage area - LONG lNewLength, // length of area to be allocated - USHORT usMessageNo, // message to be displayed when error - HWND hwnd ) // handle for UtlError Hwnd call -{ - BOOL fOK = UtlAlloc( ppStorage, lOldLength, lNewLength, NOMSG ); - - hwnd; - usMessageNo; - if ( !fOK && (usMessageNo != NOMSG )) - { - T5LOG(T5ERROR) << "::rc = " << usMessageNo; - } /* endif */ - return( fOK ); -} /* end of UtlAllocHwnd */ PVOID UtlIntAlloc ( @@ -157,24 +139,6 @@ USHORT UtlIntFree ULONG ulActLength = *((PULONG) pStorage ); ULONG ulActLength2 = *((PULONG)(((PBYTE) pOldStorage) + ulActLength)); - ulLength; - /********************************************************************/ - /* check if someone tried to play KAMIKAZE with our memory, */ - /* fuck him.... */ - /********************************************************************/ - if ( ulActLength != ulActLength2 ) - { - /******************************************************************/ - /* display error message - someone killed us... */ - /******************************************************************/ - //usRC = ERROR_INTERNAL; - //T5LOG(T5FATAL) << ":: someone tried to play KAMIKAZE with our memory ulActLength != ulActLength2, ulActLength = "<< - // ulActLength << "; ulActLength2 = " << ulActLength2; - //UtlError( ERROR_INTERNAL, MB_CANCEL, 0, NULL, INTERNAL_ERROR ); - } - /******************************************************************/ - /* free the momory */ - /******************************************************************/ free( pStorage ); return usRC; From 39c79889c79ff0b5dad8a26acb7209aac7b40bfd Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 8 Apr 2025 17:30:28 +0300 Subject: [PATCH 04/62] * fixed crash with filename table when openning tm created in 0.7 * utlAlloc refactoring --- include/EQF.H | 6 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp | 8 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 4 +- .../core/EqfMemoryPlugin/EqfMemory.cpp | 26 +--- source/opentm2/core/tm.cpp | 2 +- source/opentm2/core/utilities/EQFUTFIL.cpp | 128 ++---------------- source/opentm2/core/utilities/EQFUTLNG.cpp | 11 +- .../OtmMarkupTablePlugin.cpp | 6 +- 8 files changed, 25 insertions(+), 166 deletions(-) diff --git a/include/EQF.H b/include/EQF.H index 92fa9aa6..1fc46c46 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -3642,9 +3642,9 @@ typedef ULONG HPRINT, *PHPRINT; ULONG UtlLongToDateString( LONG, PSZ, ULONG ); ULONG UtlLongToTimeStringW( LONG, PSZ_W, ULONG ); ULONG UtlLongToDateStringW( LONG, PSZ_W, ULONG ); - BOOL UtlLoadFile( PSZ, PVOID *, USHORT *, BOOL, BOOL ); - BOOL UtlLoadFileL( PSZ, PVOID *, ULONG *, BOOL, BOOL ); - BOOL UtlLoadFileHwnd( PSZ, PVOID *, ULONG *, BOOL, BOOL, HWND ); + BOOL UtlLoadFile( PSZ, std::vector& , USHORT *, BOOL, BOOL ); + BOOL UtlLoadFileL( PSZ, std::vector& , ULONG *, BOOL, BOOL ); + BOOL UtlLoadFileHwnd( PSZ, std::vector& , ULONG *, BOOL, BOOL, HWND ); USHORT UtlCheckSpaceForFile( PSZ, USHORT, LONG, PSZ *, LONG *, BOOL ); USHORT UtlCheckSpaceForFileEx( PSZ, USHORT, LONG, PSZ *, LONG *, BOOL, PUSHORT ); ULONG64 UtlQueryFreeSpace( CHAR, BOOL ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp index 363c9837..fe234982 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp @@ -1391,15 +1391,15 @@ USHORT FCTDATA::MemFuncPrepExport UtlSetUShort( QS_LASTERRORMSGID, NO_ERROR ); // allocate storage for the organize process communication area - fOK = UtlAllocHwnd( (PVOID *)&pCommArea, 0L, + fOK = UtlAlloc( (PVOID *)&pCommArea, 0L, (LONG)sizeof(PROCESSCOMMAREA), - ERROR_STORAGE, HWND_FUNCIF ); + ERROR_STORAGE ); // allocate storage for the MEM_EXPORT_IDA if ( fOK ) { - fOK = UtlAllocHwnd( (PVOID *)&pIDA, 0L, (LONG)sizeof(MEM_EXPORT_IDA), - ERROR_STORAGE, HWND_FUNCIF ); + fOK = UtlAlloc( (PVOID *)&pIDA, 0L, (LONG)sizeof(MEM_EXPORT_IDA), + ERROR_STORAGE ); } /* endif */ if (fOK) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 512c18df..21f3f219 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -250,9 +250,9 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, // If an error occurred set fOK to FALSE and issue appropriate message. if ( fOK ) { - fOK = UtlAllocHwnd( (PVOID *)&pLIDA->pTokenList, 0L, + fOK = UtlAlloc( (PVOID *)&pLIDA->pTokenList, 0L, (LONG)(NUMB_OF_TOKENS * sizeof(TOKENENTRY)), - ERROR_STORAGE, pLIDA->hwndErrMsg ); + ERROR_STORAGE ); if ( fOK ) { // Initialize pTokenEntry and pTokenEntryWork diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 791769ea..3929f8be 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -13,8 +13,7 @@ Copyright Notice: #define INCL_EQF_TP #define INCL_EQF_TM #define INCL_EQF_DAM -//#define INCL_EQF_ANALYSIS // analysis functions -//#include // General Translation Manager include file + #include "ThreadingWrapper.h" #include "tm.h" #include "../../core/pluginmanager/PluginManager.h" @@ -24,7 +23,6 @@ Copyright Notice: #include "../../core/utilities/FilesystemHelper.h" #include // Private header file of QDAM -//#include "EQFRDICS.H" // remote dictionary functions #include "LanguageFactory.H" // activate the folllowing define to activate logging @@ -427,8 +425,6 @@ int OtmProposal::SetProposalKey ) { currentInternalKey.setInternalKey(ulKey, usTargetNum); - //recordKey = ulKey; - //targetKey = usTargetNum; return( 0 ); } @@ -497,8 +493,6 @@ int EqfMemory::ExtOutToOtmProposal Proposal.setUpdateTime( pExtOut->lTargetTime ); Proposal.setContextRanking( 0 ); Proposal.setOriginalSourceLanguage( pExtOut->szOriginalSourceLanguage ); - //Proposal.recordKey = pExtOut->ulRecKey; - //Proposal.targetKey = pExtOut->usTargetKey Proposal.currentInternalKey.setInternalKey(pExtOut->ulRecKey, pExtOut->usTargetKey); return( iRC ); @@ -697,24 +691,6 @@ int EqfMemory::FlushFilebuffers(MutexTimeout& tmLockTimeout){ int EqfMemory::UnloadFromRAM(){ - //if(TmBtree.fb.file == nullptr){ - //file is in mem - // return -1; - //} - //if(InBtree.fb.file == nullptr){ - //file is in mem - // return -1; - //} - //TmBtree.fb.WriteToFile(); - //InBtree.fb.WriteToFile(); - - //FlushFilebuffers(); - //TmBtree.fb.data.clear(); - //TmBtree.fb.data.shrink_to_fit(); - - //InBtree.fb.data.clear(); - //InBtree.fb.data.shrink_to_fit(); - //fOpen = false; return 0; } diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index b583c74c..7299aab0 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -398,7 +398,7 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem if ( _rc_ == NO_ERROR ) { ulKey = LONGNAME_KEY; - NewMem->LongNames.pszBuffer.push_back(0);//to have some size + NewMem->LongNames.pszBuffer.resize(sizeof(TMX_RECORD));//to have some size // write long document name buffer area to the database _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, NewMem->LongNames.pszBuffer ); diff --git a/source/opentm2/core/utilities/EQFUTFIL.cpp b/source/opentm2/core/utilities/EQFUTFIL.cpp index 28ef7789..a388b493 100755 --- a/source/opentm2/core/utilities/EQFUTFIL.cpp +++ b/source/opentm2/core/utilities/EQFUTFIL.cpp @@ -183,7 +183,7 @@ USHORT UtlGetLANDriveList( PBYTE pszList ) //| return ok flag | //+----------------------------------------------------------------------------+ BOOL UtlLoadFile(PSZ pszFilename, // name of file to be loaded - PVOID *ppLoadedFile, // return pointer to loaded file + std::vector& ppLoadedFile, // return pointer to loaded file USHORT *pusBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. @@ -199,7 +199,7 @@ BOOL UtlLoadFile(PSZ pszFilename, // name of file to be loaded } BOOL UtlLoadFileL(PSZ pszFilename, // name of file to be loaded - PVOID *ppLoadedFile, // return pointer to loaded file + std::vector& ppLoadedFile, // return pointer to loaded file ULONG *pulBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. @@ -212,7 +212,7 @@ BOOL UtlLoadFileL(PSZ pszFilename, // name of file to be loaded BOOL UtlLoadFileHwnd ( PSZ pszFilename, // name of file to be loaded - PVOID *ppLoadedFile, // return pointer to loaded file + std::vector& ppLoadedFile, // return pointer to loaded file ULONG *pulBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. @@ -260,15 +260,9 @@ BOOL UtlLoadFileHwnd // If previous call OK allocate storage if required if ( !usDosRc && (ulSize > 0L) ) { - if ( *ppLoadedFile == NULL ) - { - usMessage = ( fMsg ) ? ERROR_STORAGE : NOMSG; - fOK = UtlAllocHwnd( (PVOID *)ppLoadedFile, // Allocate storage for input data - 0L, - get_max( (LONG) MIN_ALLOC, ulSize ), - usMessage, hwnd ); - fStorage = TRUE; - } /* endif */ + usMessage = ( fMsg ) ? ERROR_STORAGE : NOMSG; + ppLoadedFile.resize( get_max( (LONG) MIN_ALLOC, ulSize )); + fStorage = TRUE; } else { @@ -280,7 +274,7 @@ BOOL UtlLoadFileHwnd // If fOK. Load the input data in storage and close it if load was OK if (fOK) { - usDosRc = UtlReadL( hInputfile , *ppLoadedFile, + usDosRc = UtlReadL( hInputfile , ppLoadedFile.data(), ulSize, pulBytesRead, FALSE ); fOK = (usDosRc == 0 ); @@ -300,7 +294,7 @@ BOOL UtlLoadFileHwnd // Free storage allocation if allocated and something went wrong if ( !fOK && fStorage ) { - UtlAlloc( (PVOID *)ppLoadedFile, 0L, 0L, NOMSG ); + ppLoadedFile.clear(); } /* endif */ if ( !fOK ) // display error message if not ok @@ -800,113 +794,7 @@ BOOL UtlGetCheckedEqfDrives //| open the file | //| initialize control block variables | //+----------------------------------------------------------------------------+ -USHORT UtlBufOpen -( - PBUFCB *ppBufCB, - PSZ pszFile, - ULONG ulBufSize, - ULONG ulReadWrite, - BOOL fMsg -) -{ - return( UtlBufOpenHwnd( ppBufCB, pszFile, ulBufSize, ulReadWrite, fMsg, - NULLHANDLE ) ); -} - -USHORT UtlBufOpenHwnd -( - PBUFCB *ppBufCB, - PSZ pszFile, - ULONG ulBufSize, - ULONG ulReadWrite, - BOOL fMsg, - HWND hwnd -) -{ - USHORT usOpenAction; // action performed by UtlOpen - USHORT usRC = 0; // return code of function - USHORT usOpenFlag; // File open flag - USHORT usOpenMode; // File open mode - PBUFCB pBufCB; // buffered input/output control block - ULONG ulFilePos; // new file position - - /********************************************************************/ - /* Allocate control block / input output buffer */ - /********************************************************************/ - UtlAllocHwnd( (PVOID *)&pBufCB, 0L, (sizeof(BUFCB) + ulBufSize ), - ERROR_STORAGE, hwnd ); - if ( !pBufCB ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - /********************************************************************/ - /* Open the output file */ - /********************************************************************/ - if ( !usRC ) - { - switch (ulReadWrite) - { - case FILE_CREATE: - { - usOpenFlag = FILE_TRUNCATE | FILE_CREATE ; - usOpenMode = OPEN_ACCESS_READWRITE | OPEN_SHARE_DENYWRITE; - } - break; - case FILE_APPEND: - { - usOpenFlag = FILE_OPEN; - usOpenMode = OPEN_ACCESS_READWRITE | OPEN_SHARE_DENYWRITE; - } - break; - default: - { - usOpenFlag = FILE_TRUNCATE | FILE_OPEN; - usOpenMode = OPEN_ACCESS_READONLY | OPEN_SHARE_DENYWRITE; - } - break; - } /* endswitch */ - - usRC = UtlOpenHwnd( pszFile, - &pBufCB->hFile, - &usOpenAction, 0L, - FILE_NORMAL, - usOpenFlag, - usOpenMode, - 0L, - fMsg, hwnd ); - } /* endif */ - if ( !usRC && (ulReadWrite == FILE_APPEND) ) - { - usRC = UtlChgFilePtrHwnd( pBufCB->hFile, 0L, FILE_END, &ulFilePos, fMsg, hwnd ); - } /* endif */ - - - /********************************************************************/ - /* Initialize control block variables */ - /********************************************************************/ - if ( !usRC ) - { - pBufCB->ulUsed = 0; - pBufCB->ulSize = ulBufSize; - pBufCB->ulProcessed = 0; - strcpy( pBufCB->szFileName, pszFile ); - if ( ulReadWrite != FILE_CREATE ) - { - usRC = UtlGetFileSizeHwnd( pBufCB->hFile, - &(pBufCB->ulRemaining), - fMsg, hwnd ); - pBufCB->fWrite = ( ulReadWrite == FILE_APPEND ); - } - else - { - pBufCB->fWrite = TRUE; - } /* endif */ - *ppBufCB = pBufCB; - } /* endif */ - return( usRC ); -} /* end of function UtlBufOpen */ //+----------------------------------------------------------------------------+ //|Internal function | diff --git a/source/opentm2/core/utilities/EQFUTLNG.cpp b/source/opentm2/core/utilities/EQFUTLNG.cpp index 4134ed46..114101c2 100755 --- a/source/opentm2/core/utilities/EQFUTLNG.cpp +++ b/source/opentm2/core/utilities/EQFUTLNG.cpp @@ -48,7 +48,7 @@ BOOL UtlQuerySysLangFile { CHAR szFileName[MAX_EQF_PATH]; // buffer for file name BOOL fOK = TRUE; // function return code - PSZ pszLangFileBuffer = NULL; // buffer for in-memory copy of language file + std::vector pszLangFileBuffer; // buffer for in-memory copy of language file PSZ pszCurLine; // ptr to current line in file BOOL fFound = FALSE; ULONG ulLength; // length of language file @@ -76,7 +76,7 @@ BOOL UtlQuerySysLangFile /********************************************************************/ /* load system language file into memory */ /********************************************************************/ - fOK = UtlLoadFileL( szFileName, (PVOID *)&pszLangFileBuffer, + fOK = UtlLoadFileL( szFileName, pszLangFileBuffer, &ulLength, FALSE, FALSE ); /********************************************************************/ @@ -98,7 +98,7 @@ BOOL UtlQuerySysLangFile /********************************************************************/ if ( fOK ) { - pszCurLine = pszLangFileBuffer; + pszCurLine = (PSZ) pszLangFileBuffer.data(); pszLangFileBuffer[ulLength-1] = EOS; // force end string delimiter @@ -255,11 +255,6 @@ BOOL UtlQuerySysLangFile } /* endwhile */ } /* endif */ - /********************************************************************/ - /* cleanup */ - /********************************************************************/ - if ( pszLangFileBuffer ) - UtlAlloc( (PVOID *)&pszLangFileBuffer, 0L, 0L, NOMSG ); return( fOK && fFound ); } /* end of function TWBQuerySysLangFile */ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp index a84195f2..77e415fe 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp @@ -979,20 +979,20 @@ void PerformPendingUpdates( char *pszBasePath ) usRC = UtlFindFirst( szSearch, &hSearch, 0, &stFile, sizeof(stFile), &usCount, 0L, FALSE ); while ( (usRC == NO_ERROR) && usCount ) { - PPROPFOLDER pProp = NULL; + std::vector data; ULONG ulSize = 0; UtlMakeEQFPath( szSource, NULC, PROPERTY_PATH, NULL ); strcat( szSource, BACKSLASH_STR ); strcat( szSource, RESBUFNAME(stFile) ); - if ( UtlLoadFileL( szSource, (PVOID *)&pProp, &ulSize, FALSE, FALSE ) ) + if ( UtlLoadFileL( szSource, data, &ulSize, FALSE, FALSE ) ) { + PPROPFOLDER pProp = (PPROPFOLDER)data.data(); if ( pProp->fDisabled_UserExitRefresh ) { pProp->fDisabled_UserExitRefresh = FALSE; UtlWriteFile( szSource, ulSize, pProp ); } /* endif */ - UtlAlloc( (PVOID *)&pProp, 0, 0, NOMSG ); } /* endif */ // next property file From 4158c850b2be4eac550924838d053922e49a0766 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 9 Apr 2025 15:30:31 +0300 Subject: [PATCH 05/62] * Refactored dataReference in function to protect from typos * refactored record key to be ulong * code cleanup --- include/EQF.H | 6 +- include/EQFQDAMI.H | 53 +- include/EQFTA.H | 35 +- include/lowlevelotmdatastructs.h | 43 +- include/tm.h | 28 +- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 24 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 14 +- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 58 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 177 ++-- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 55 +- .../core/EqfMemoryPlugin/TMPluginWrapper.cpp | 224 +--- source/opentm2/core/tagtable/EQFTOKEN.cpp | 991 ++++++++---------- source/opentm2/core/tm.cpp | 12 +- source/opentm2/core/utilities/EQFDICT.cpp | 374 +++---- source/opentm2/core/utilities/EQFUTFIL.cpp | 6 +- 15 files changed, 818 insertions(+), 1282 deletions(-) diff --git a/include/EQF.H b/include/EQF.H index 1fc46c46..5c3840a3 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -3642,9 +3642,9 @@ typedef ULONG HPRINT, *PHPRINT; ULONG UtlLongToDateString( LONG, PSZ, ULONG ); ULONG UtlLongToTimeStringW( LONG, PSZ_W, ULONG ); ULONG UtlLongToDateStringW( LONG, PSZ_W, ULONG ); - BOOL UtlLoadFile( PSZ, std::vector& , USHORT *, BOOL, BOOL ); - BOOL UtlLoadFileL( PSZ, std::vector& , ULONG *, BOOL, BOOL ); - BOOL UtlLoadFileHwnd( PSZ, std::vector& , ULONG *, BOOL, BOOL, HWND ); + BOOL UtlLoadFile( PSZ, BytesRef , USHORT *, BOOL, BOOL ); + BOOL UtlLoadFileL( PSZ, BytesRef , ULONG *, BOOL, BOOL ); + BOOL UtlLoadFileHwnd( PSZ, BytesRef , ULONG *, BOOL, BOOL, HWND ); USHORT UtlCheckSpaceForFile( PSZ, USHORT, LONG, PSZ *, LONG *, BOOL ); USHORT UtlCheckSpaceForFileEx( PSZ, USHORT, LONG, PSZ *, LONG *, BOOL, PUSHORT ); ULONG64 UtlQueryFreeSpace( CHAR, BOOL ); diff --git a/include/EQFQDAMI.H b/include/EQFQDAMI.H index a6472ddb..bcf6f1d1 100755 --- a/include/EQFQDAMI.H +++ b/include/EQFQDAMI.H @@ -347,7 +347,7 @@ typedef struct _DICTSIGN SHORT QDAMDictInsert ( PBTREE pBT, // pointer to binary tree struct - PWCHAR pKey, // pointer to key data + ULONG ulKey, // pointer to key data PBYTE pData, // pointer to user data ULONG ulLen // length of user data ); @@ -392,7 +392,7 @@ typedef struct _DICTINSERT SHORT QDAMDictExact ( PBTREE pBT, // pointer to btree struct - PWCHAR pKey, // key to be searched for + ULONG ulKey, // key to be searched for PBYTE pchBuffer, // space for user data PULONG pulLength, // in/out length of returned user data USHORT usSeachSubType @@ -438,7 +438,7 @@ SHORT QDAMDictExact SHORT QDAMDictSubStr ( PBTREE pBT, // pointer to btree struct - PWCHAR pKey, // key to be searched for + ULONG ulKey, // key to be searched for PBYTE pchBuffer, // space for key data PULONG pulLength, // in/out length of returned key data PBYTE pchUserData, // space for user data @@ -484,7 +484,7 @@ SHORT QDAMDictSubStr SHORT QDAMDictEquiv ( PBTREE pBT, // pointer to btree struct - PWCHAR pKey, // key to be searched for + ULONG ulKey, // key to be searched for PBYTE pchBuffer, // space for key data PULONG pulLength, // in/out length of returned key data PBYTE pchUserData, // space for user data @@ -573,7 +573,7 @@ SHORT QDAMDictLockEntry SHORT QDAMDictFirst ( PBTREE pBT, - PWCHAR pKeyData, // pointer to space for key data + ULONG ulKeyData, // pointer to space for key data PULONG pulKeyLen, // length of space for key data PBYTE pUserData, // pointer to space for user data PULONG pulUserLen // length of space for user data @@ -781,7 +781,7 @@ SHORT QDAMDictNumber ( PBTREE pBT, // pointer to generic structure ULONG ulTermNum, // term number - PWCHAR pKey, // pointer to space for term + ULONG ulKey, // pointer to space for term PULONG pulKeyLen, // length of key PBYTE pData, // pointer to data PULONG pulLen // length of data @@ -894,7 +894,7 @@ typedef struct _DICTTIME SHORT QDAMDictNumEntriesLocal ( PBTREE, PULONG ); - SHORT QDAMDictFirstLocal ( PBTREE, PWCHAR, PULONG, std::vector& ); + SHORT QDAMDictFirstLocal ( PBTREE, ULONG, BytesRef ); SHORT QDAMInsertKey_V3( PBTREE, std::shared_ptr&, PWCHAR, RECPARAM, RECPARAM ); @@ -902,14 +902,14 @@ typedef struct _DICTTIME SHORT QDAMFindChild_V3 ( PBTREE, PWCHAR, USHORT, std::shared_ptr& ); - SHORT QDAMChangeKey_V3(PBTREE, USHORT, PWCHAR, PWCHAR ); + SHORT QDAMChangeKey_V3(PBTREE, USHORT, ULONG&, ULONG& ); SHORT QDAMNewRecord_V3( PBTREE, std::shared_ptr&, RECTYPE ); SHORT QDAMDeleteDataFromBuffer ( PBTREE, RECPARAM ); - SHORT QDAMSplitNode_V3( PBTREE, std::shared_ptr&, PWCHAR ); + SHORT QDAMSplitNode_V3( PBTREE, std::shared_ptr&, ULONG& ); - SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, PWCHAR, PULONG ); - SHORT QDAMGetszData_V3 ( PBTREE, RECPARAM, std::vector&, CHAR ); + SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, ULONG& ); + SHORT QDAMGetszData_V3 ( PBTREE, RECPARAM, BytesRef, CHAR ); BOOL QDAMDeleteKey_V3( PBTREE, std::shared_ptr&, PCHAR); SHORT QDAMDestroy ( PBTREE ); @@ -921,10 +921,10 @@ typedef struct _DICTTIME SHORT QDAMPrev_V3( PBTREE, PRECPARAM, PRECPARAM, PRECPARAM ); SHORT QDAMHeaderFirst ( PBTREE, PRECPARAM, PRECPARAM, PRECPARAM ); SHORT QDAMHeaderNext ( PBTREE, PRECPARAM, PRECPARAM,PRECPARAM ); - SHORT QDAMUnTerseData ( PBTREE, std::vector& , ULONG ); + SHORT QDAMUnTerseData ( PBTREE, BytesRef , ULONG ); SHORT QDAMValidateIndex_V3( PBTREE, std::shared_ptr& ); - PSZ_W QDAMGetszKey_V3 ( std::shared_ptr&, USHORT, USHORT ); + ULONG QDAMGetszKey_V3 ( std::shared_ptr&, USHORT ); RECPARAM QDAMGetrecData_V3 ( std::shared_ptr&, SHORT, USHORT ); VOID QDAMSetrecData_V3 ( std::shared_ptr&, SHORT, RECPARAM, USHORT ); RECPARAM QDAMGetrecKey_V3 ( std::shared_ptr&, SHORT ); @@ -1245,13 +1245,6 @@ void DAMLINK // <> keys are unequal //+----------------------------------------------------------------------------+ -SHORT NTMKeyCompare -( - PVOID pBTIda, // pointer to tree structure - PVOID pulKey1, // pointer to first key - PVOID pulKey2 // pointer to second key -); - /**********************************************************************/ /* validate the passed BTree pointer */ /**********************************************************************/ @@ -1310,26 +1303,6 @@ USHORT QDamWildCardList ULONG // in - size of buffer ); -SHORT QDAMDictNextWildLocal -( - PBTREE pBTIda, - PSZ_W pPattern, // search pattern or compound - BOOL fCompound, // compound search flag - PBYTE pKeyData, // pointer to space for key data - PULONG pulKeyLen // length of space for key data -); - -BOOL QDAMMatchCompound -( - PSZ_W pKey, - PSZ_W pCompound -); - -BOOL QDAMMatchWild -( - PUCHAR pKey, - PUCHAR pPattern -); ULONG QDAMComputeCheckSum_V3 (std::shared_ptr& pRecord ); BOOL QDAMCheckCheckSum_V3( std::shared_ptr& pRecord, SHORT sLocation ); diff --git a/include/EQFTA.H b/include/EQFTA.H index 989f51f7..b74669b6 100755 --- a/include/EQFTA.H +++ b/include/EQFTA.H @@ -23,9 +23,10 @@ /**********************************************************************/ typedef struct _STARTSTOP { - USHORT usStart; // start offset - USHORT usStop; // stop offset - USHORT usType; // type of entry + USHORT usStart = 0; // start offset + USHORT usStop = 0; // stop offset + USHORT usType = 0; // type of entry + void reset(){memset(this, 0, sizeof(*this));} } STARTSTOP, *PSTARTSTOP; @@ -94,30 +95,8 @@ BOOL EQFUnSegRevMark2 USHORT usTrackDocNum, // not 0, TVT tracking document num PSZ pszConversion // ptr to conversion name or NULL ); -USHORT TACreateProtectTable -( - PSZ pszSegment, // ptr to text of segment being processed - PVOID pVoidTable, // ptr to tag table (PLOADEDTABLE) - USHORT usColPos, // column position of first char in segment - PTOKENENTRY pTokBuffer, // buffer used temporarly for tokens - USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pfnUserExit, // ptr to user exit function - ULONG ulCP -); -USHORT TACreateProtectTableW -( - PSZ_W pszSegmentW, // ptr to text of segment being processed - PVOID pVoidTable, // ptr to tag table - USHORT usColPos, // column position of first char in segment - std::vector& pTokBuffer, // buffer used temporarly for tokens - USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function - PFN pvUserExitW, - ULONG ulCP -); + #define CREATEPROTTABLE_MARKATTR 0x02 #define CREATEPROTTABLE_NOTRANSLVAR 0x04 @@ -130,8 +109,8 @@ USHORT TACreateProtectTableWEx USHORT usColPos, // column position of first char in segment std::vector& pTokBuffer, // buffer used temporarly for tokens USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function + BytesRef pStartStop, // ptr to caller's start/stop table ptr + //PFN pvUserExit, // ptr to user exit function PFN pvUserExitW, ULONG ulCP, // ASCII cp fitting to Segmenttext int iMode // mode for function diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index d7b1fbbf..5d350d74 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -15,6 +15,21 @@ #include #include "ThreadingWrapper.h" + +using BytesRef = std::vector&; +template +inline void pushBytes(std::vector& buffer, const T& value) { + const BYTE* dataPtr = reinterpret_cast(&value); + buffer.insert(buffer.end(), dataPtr, dataPtr + sizeof(T)); +} + +inline void pushBytes(std::vector& buffer, const void* start, const void* end) { + const BYTE* beginPtr = static_cast(start); + const BYTE* endPtr = static_cast(end); + + buffer.insert(buffer.end(), beginPtr, endPtr); +} + class ChunkBuffer{ //long m_bytesCollecedInChunk = 0; //long m_bytesSend = 0; @@ -485,7 +500,7 @@ struct TMX_TABLE typedef TMX_TABLE * PTMX_TABLE; -int tmxTableToBuffer(PTMX_TABLE pTable, std::vector& buff); +int tmxTableToBuffer(PTMX_TABLE pTable, BytesRef buff); @@ -726,8 +741,8 @@ struct BTREE SHORT EQFNTMInsert ( - PULONG pulKey, // pointer to key - std::vector& pData // pointer to user data + ULONG& pulKey, // pointer to key + BytesRef pData // pointer to user data ); //+----------------------------------------------------------------------------+ @@ -756,7 +771,7 @@ struct BTREE EQFNTMUpdate ( ULONG ulKey, // key value - std::vector& pData // pointer to user data + BytesRef pData // pointer to user data ); @@ -782,7 +797,7 @@ struct BTREE EQFNTMGet ( ULONG ulKey, // key to be searched for - std::vector& uchBuff // space for user data + BytesRef uchBuff // space for user data ); @@ -891,17 +906,17 @@ struct BTREE ULONG QDAMGetrecDataLen_V3 ( std::shared_ptr&, SHORT ); SHORT QDAMDeleteDataFromBuffer_V3( RECPARAM recParam); - SHORT QDAMDictUpdateLocal ( PWCHAR, std::vector& ); + SHORT QDAMDictUpdateLocal ( PWCHAR, BytesRef ); - SHORT QDAMDictExactLocal ( PWCHAR, std::vector&, USHORT ); + SHORT QDAMDictExactLocal ( ULONG&, BytesRef, USHORT ); SHORT QDAMDictCreateLocal ( TMX_SIGN*, ULONG, bool keepInRamOnly = false); - SHORT QDAMDictInsertLocal ( PWCHAR, std::vector& pData ); - BOOL QDAMDictLockStatus ( PWCHAR ); + SHORT QDAMDictInsertLocal ( ULONG&, BytesRef pData ); + BOOL QDAMDictLockStatus ( ULONG ); VOID QDAMDictUpdStatus (); - SHORT QDAMFindRecord_V3( PWCHAR, std::shared_ptr&); + SHORT QDAMFindRecord_V3( ULONG, std::shared_ptr&); //------------------------------------------------------------------------------ @@ -1015,7 +1030,7 @@ struct BTREE ); - SHORT QDAMAddToBuffer_V3( std::vector&, PRECPARAM); + SHORT QDAMAddToBuffer_V3( BytesRef, PRECPARAM); SHORT QDAMWriteHeader (); SHORT QDAMDictUpdSignLocal @@ -1101,7 +1116,7 @@ SHORT QDAMNewRecord_V3 SHORT QDAMInsertKey_V3 ( std::shared_ptr& pRecord, // record where key is to be inserted - PWCHAR pKey, + ULONG& ulKey, RECPARAM recKey, // position/offset for key RECPARAM recData // position/offset for data ); @@ -1111,13 +1126,13 @@ size_t GetFileSize()const; BOOL QDAMTerseData ( - std::vector& , + BytesRef , PULONG pulLen // length of the string ); SHORT QDAMLocateKey_V3( - std::shared_ptr&, PWCHAR, PSHORT, SEARCHTYPE, PSHORT); + std::shared_ptr&, ULONG&, PSHORT, SEARCHTYPE, PSHORT); };//BTREE diff --git a/include/tm.h b/include/tm.h index 34d0156f..ea5da289 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1056,12 +1056,16 @@ typedef struct _TMX_INDEX_RECORD TMX_INDEX_ENTRY stIndexEntry; } TMX_INDEX_RECORD, *PTMX_INDEX_RECORD; -inline TMX_RECORD *toTmxRecord(std::vector &buffer) +inline TMX_RECORD *toTmxRecord(BytesRef buffer) { return reinterpret_cast(buffer.data()); } -inline TMX_INDEX_RECORD *toIndexRecord(std::vector &buffer) +inline TMX_TAGTABLE_RECORD* toTmxTagtableRecord(BytesRef buffer){ + return reinterpret_cast(buffer.data()); +} + +inline TMX_INDEX_RECORD *toIndexRecord(BytesRef buffer) { return reinterpret_cast(buffer.data()); } @@ -1522,8 +1526,7 @@ struct TMX_SENTENCE PSZ_W pAddString = nullptr; std::vector pTermTokens; - std::vector pTagRecord; - std::vector pTagEntryList; + std::vector pTagRecord;//TMX_TAGTABLE_RECORD USHORT usActVote = 0; std::vector pulVotes; // std::unique_ptr pPropString = nullptr; @@ -1560,7 +1563,8 @@ struct TMX_SENTENCE } bool fOK = true; pulVotes.reserve(ABS_VOTES / sizeof(pulVotes[0])); - pTagRecord.reserve(2 * TOK_SIZE / sizeof(pTagRecord[0])); + pTagRecord.reserve(2 * TOK_SIZE); + pTagRecord.resize(sizeof(TMX_TAGTABLE_RECORD)); pTermTokens.reserve(TOK_SIZE / sizeof(pTermTokens[0])); } @@ -1975,13 +1979,13 @@ class EqfMemory //: public TMX_CLB ); USHORT FindTargetAndDelete( - std::vector& TmRecord, + BytesRef TmRecord, OtmProposal &TmDel, TMX_EXT_OUT_W *pTmExtOut, PULONG pulKey); USHORT FindTargetByKeyAndDelete( - std::vector& pTmRecord, + BytesRef pTmRecord, OtmProposal &pTmDel, PTMX_SENTENCE pSentence, TMX_EXT_OUT_W *pTmExtOut, @@ -2006,15 +2010,15 @@ class EqfMemory //: public TMX_CLB PUSHORT pusID, // intput USHORT usTableType); - USHORT FillClb(PTMX_TARGET_CLB, OtmProposal &); + USHORT FillClb(BytesRef, OtmProposal &); USHORT ComparePutData( - std::vector &ppTmRecord, // ptr to ptr of tm record data buffer + BytesRef ppTmRecord, // ptr to ptr of tm record data buffer OtmProposal &TmProposal, // pointer to get in data PULONG pulKey // tm key ); USHORT AddTmTarget( OtmProposal &TmProposal, // pointer to get in data - std::vector& pTmRecord, // pointer to tm record data pointer + BytesRef pTmRecord, // pointer to tm record data pointer PULONG pulKey); // tm key /*! \brief Store the supplied proposal in the memory @@ -3524,8 +3528,8 @@ VOID Vote(PTMX_TERM_TOKEN, PTMX_SENTENCE, USHORT); USHORT CheckCompactArea(PTMX_SENTENCE, EqfMemory *); VOID FillTmRecord(PTMX_SENTENCE, - std::vector& TmRecord, // filled tm record returned - PTMX_TARGET_CLB, USHORT); + BytesRef TmRecord, // filled tm record returned + BytesRef, USHORT); USHORT DetermineTmRecord(EqfMemory *, PTMX_SENTENCE, std::vector& pulSids ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 076962ce..d33eb42f 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -79,16 +79,12 @@ USHORT EqfMemory::TmtXDelSegm strcat( szString, TmDelIn.szMarkup ); strcat( szString, EXT_OF_FORMAT ); - //remember start of norm string - why? - //Sentence.pNormStringStart = Sentence.pStrings->getNormStrC(); - - //tokenize source segment, resuting in normalized string and tag table record usRc = TokenizeSource( TmDelIn.pInputSentence, szString, TmDelIn.szSourceLanguage); // set the tag table ID in the tag record (this can't be done in TokenizeSource anymore) - TmDelIn.pInputSentence->pTagRecord.data()->usTagTableId = 1; + toTmxTagtableRecord(TmDelIn.pInputSentence->pTagRecord)->usTagTableId = 1; } @@ -96,7 +92,6 @@ USHORT EqfMemory::TmtXDelSegm if ( !usRc ) { //set pNormString to beginning of string - //Sentence.pNormString = Sentence.pStrings->getNormStrC(); HashSentence( TmDelIn.pInputSentence ); } /* endif */ @@ -105,7 +100,7 @@ USHORT EqfMemory::TmtXDelSegm if ( !usRc ) { usMatchesFound = CheckCompactArea( TmDelIn.pInputSentence, this ); - if ( usMatchesFound == TmDelIn.pInputSentence->usActVote ) //all hash triples found + if ( usMatchesFound == TmDelIn.pInputSentence->pulVotes.size() ) //all hash triples found { usRc = DetermineTmRecord( this, TmDelIn.pInputSentence, pulSids ); if ( !usRc ) @@ -243,7 +238,7 @@ USHORT EqfMemory::TmtXDelSegmByKey //| get next sentence key | // ----------------------------------------------------------------------------+ USHORT EqfMemory::FindTargetAndDelete( - std::vector& TmRecord, + BytesRef TmRecord, OtmProposal& TmDel, TMX_EXT_OUT_W * pTmExtOut, PULONG pulKey ) @@ -350,9 +345,6 @@ USHORT EqfMemory::FindTargetAndDelete( pString.clear(); ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - //tokenize target string in del structure - //usRc = TokenizeTarget( pTmDel->szTarget, pNormString, &pTagRecord, &lTagAlloc, pTmDel->szTagTable, &usNormLen, this ); - if ( !usRc ) { //compare target strings @@ -363,15 +355,9 @@ USHORT EqfMemory::FindTargetAndDelete( //position at target tag table record pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //pByte += pTMXTargetRecord->usTargetTagTable; - //pTMXTargetTagTable = (PTMX_TAGTABLE_RECORD)pByte; //compare tag table records - if (true - //|| //we use the same tag tables - // !memcmp( pTMXTargetTagTable, pTagRecord, - // RECLEN(pTMXTargetTagTable) ) - ) + if (true ) { //identical target tag table as in del structure so //check segment id and file name in control block @@ -470,7 +456,7 @@ USHORT EqfMemory::FindTargetAndDelete( } USHORT EqfMemory::FindTargetByKeyAndDelete( - std::vector& TmRecord, + BytesRef TmRecord, OtmProposal& TmDel, PTMX_SENTENCE pSentence, TMX_EXT_OUT_W * pTmExtOut, diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index a85b7ea3..f7356738 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -78,7 +78,7 @@ void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ); USHORT FillMatchTable( EqfMemory*, std::wstring&, PLONG, PTMX_TARGET_RECORD, PTMX_TARGET_CLB, PTMX_MATCH_TABLE_W, PUSHORT, BOOL, PUSHORT, PUSHORT, PUSHORT, - ULONG, USHORT, PTMX_GET_W, PTMX_TAGTABLE_RECORD, USHORT, USHORT, int, int, USHORT); + ULONG, USHORT, PTMX_GET_W, BytesRef, USHORT, USHORT, int, int, USHORT); /**********************************************************************/ /* Prototypes for Static functions... */ @@ -358,7 +358,7 @@ USHORT EqfMemory::TmtXGet if ( usRc == NO_ERROR ) { //usRc = NTMGetIDFromName( pTmGetIn->szTagTable, NULL, (USHORT)TAGTABLE_KEY, &Sentence.pTagRecord->usTagTableId ); - Sentence.pTagRecord.data()->usTagTableId = 1; + toTmxTagtableRecord(Sentence.pTagRecord)->usTagTableId = 1; } } /* endif */ @@ -648,7 +648,7 @@ USHORT GetExactMatch ULONG ulLen; //length indicator ULONG ulKey; //tm record key USHORT usMatchEntries = 0; //nr of found matches - std::vector& pTmRecord = pTmClb->pvTmRecord; + BytesRef pTmRecord = pTmClb->pvTmRecord; ULONG ulStrippedParm = pGetIn->ulParm & ~(GET_RESPECTCRLF | GET_IGNORE_PATH | GET_NO_GENERICREPLACE | GET_ALWAYS_WITH_TAGS | GET_IGNORE_COMMENT); @@ -1421,7 +1421,7 @@ USHORT ExactTest pusMatchEntries, &pSentence->usActVote, &pSentence->usActVote, ulKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord.data(), usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId ); + pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId ); } else { @@ -1677,7 +1677,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct ULONG ulKeyNum, // key number USHORT usTgtNum, // target number PTMX_GET_W pGetIn, // input structure - PTMX_TAGTABLE_RECORD pTagRecord, + BytesRef pTagRecord, USHORT usTranslationFlag, USHORT usContextRanking, int iWords, int iDiffs, USHORT usSrcLangId ) @@ -2314,7 +2314,7 @@ USHORT GetFuzzyMatch GetElapsedTime( &(pTmClb->lOtherTime) ); #endif - std::vector& pTmRecord = pTmClb->pvTmRecord; + BytesRef pTmRecord = pTmClb->pvTmRecord; //allocate for match entry if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pMatchEntry), 0L, (LONG)(ABS_VOTES * sizeof(TMX_MATCHENTRY)), NOMSG ); @@ -2840,7 +2840,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block pstMatchTable, &usFuzzy, fTagTableEqual, pusMatchesFound, pusTmMaxVotes, pusOverlaps, ulKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord.data(), usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId ); + pGetIn, pSentence->pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId ); } /* endif */ } /* endif */ //position at next target diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index 07a872a7..f6d47003 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -450,8 +450,8 @@ void BTREE::freeLookupTable(){ SHORT BTREE::EQFNTMInsert ( - PULONG pulKey, // pointer to key - std::vector& pData // pointer to user data + ULONG& ulKey, // pointer to key + BytesRef pData // pointer to user data ) { SHORT sRc = 0; // return code @@ -467,15 +467,15 @@ BTREE::EQFNTMInsert /********************************************************************/ if ( !sRc ) { - if ( *pulKey == NTMREQUESTNEWKEY ) + if ( ulKey == NTMREQUESTNEWKEY ) { /******************************************************************/ /* find next free key and anchor new value in file ... */ /******************************************************************/ - ULONG ulKey; + ULONG Key; //ulKey = *pulKey = ++(pvi->ulNextKey); - ulKey = *pulKey = (chCollate.ulNextKey)++; - if ( ulKey > 0xFFFFFF ) + Key = ulKey = (chCollate.ulNextKey)++; + if ( Key > 0xFFFFFF ) { sRc = BTREE_NUMBER_RANGE; } @@ -485,7 +485,7 @@ BTREE::EQFNTMInsert /* force update of header (only from time to time to avoid */ /* too much performance degration)... */ /**************************************************************/ - if ( (ulKey & 0x020) ) + if ( (Key & 0x020) ) { sRc = QDAMWriteHeader(); } /* endif */ @@ -496,7 +496,7 @@ BTREE::EQFNTMInsert /****************************************************************/ /* check if key is in valid range ... */ /****************************************************************/ - if ( *pulKey > NTMSTARTKEY( this ) ) + if ( ulKey > NTMSTARTKEY( this ) ) { sRc = BTREE_NUMBER_RANGE; } /* endif */ @@ -507,7 +507,7 @@ BTREE::EQFNTMInsert /* call QDAMDictInsert to do the dirty work of inserting entry... */ /********************************************************************/ - sRc = QDAMDictInsertLocal( (PWCHAR)pulKey, pData); + sRc = QDAMDictInsertLocal( ulKey, pData); return sRc; @@ -1275,10 +1275,10 @@ RECPARAM QDAMGetrecKey_V3 // return sRc // //------------------------------------------------------------------------------ -SHORT BTREE::QDAMDictUpdateLocal +SHORT BTREE::EQFNTMUpdate ( - PCHAR_W pKey, // key string - std::vector& pData + ULONG ulKey, // key string + BytesRef pData ) { SHORT sRc = 0; // return code @@ -1305,7 +1305,7 @@ SHORT BTREE::QDAMDictUpdateLocal /*******************************************************************/ /* check if entry is locked .... */ /*******************************************************************/ - if ( !sRc && QDAMDictLockStatus( pKey ) ) + if ( !sRc && QDAMDictLockStatus( ulKey ) ) { sRc = BTREE_ENTRY_LOCKED; } /* endif */ @@ -1320,11 +1320,9 @@ SHORT BTREE::QDAMDictUpdateLocal sRc = BTREE_DATA_RANGE; } else - { - //memcpy( chHeadTerm, pKey, sizeof(ULONG)); // save data - + { QDAMDictUpdStatus(); - sRc = QDAMFindRecord_V3( pKey, pRecord ); + sRc = QDAMFindRecord_V3( ulKey, pRecord ); } /* endif */ } /* endif */ @@ -1332,7 +1330,7 @@ SHORT BTREE::QDAMDictUpdateLocal if ( !sRc ) { // find the key - sRc = QDAMLocateKey_V3( pRecord, pKey, &i, FEXACT, &sNearKey ) ; + sRc = QDAMLocateKey_V3( pRecord, ulKey, &i, FEXACT, &sNearKey ) ; if ( !sRc ) { if ( i != -1) @@ -1474,15 +1472,7 @@ int BTREE::deallocateLookupTableBuffer(int number){ //|Function flow: call QDAMDictUpdateLocal | //| return success indicator.. | // ----------------------------------------------------------------------------+ -SHORT -BTREE::EQFNTMUpdate -( - ULONG ulKey, // key value - std::vector& pData // pointer to user data -) -{ - return QDAMDictUpdateLocal( (PSZ_W) &ulKey, pData ); -} /* end of function EQFNTMUpdate */ + //------------------------------------------------------------------------------ @@ -1538,8 +1528,8 @@ BTREE::EQFNTMUpdate SHORT BTREE::QDAMDictExactLocal ( - PWCHAR pKey, // key to be searched for - std::vector& pchBuffer, // space for user data + ULONG& ulKey, // key to be searched for + BytesRef pchBuffer, // space for user data USHORT usSearchSubType // special hyphenation lookup flag ) { @@ -1564,12 +1554,12 @@ SHORT BTREE::QDAMDictExactLocal if ( !sRc ) { /* Locate the Leaf node that contains the appropriate key */ - sRc = QDAMFindRecord_V3( pKey, pRecord ); + sRc = QDAMFindRecord_V3( ulKey, pRecord ); } /* endif */ if ( !sRc ) { - sRc = QDAMLocateKey_V3(pRecord, pKey, &i, FEXACT, &sNearKey); + sRc = QDAMLocateKey_V3(pRecord, ulKey, &i, FEXACT, &sNearKey); if ( !sRc ) { if ( i != -1 ) @@ -1578,8 +1568,6 @@ SHORT BTREE::QDAMDictExactLocal sCurrentIndex = i; usCurrentRecord = RECORDNUM( pRecord ); - //memcpy( chHeadTerm, pKey, sizeof(ULONG) ); // save data - recData = QDAMGetrecData_V3( pRecord, i, usVersion ); sRc = QDAMGetszData_V3( this, recData, pchBuffer, DATA_NODE ); @@ -1637,7 +1625,7 @@ SHORT BTREE::EQFNTMGet ( ULONG ulKey, // key to be searched for - std::vector& pchBuffer // space for user data + BytesRef pchBuffer // space for user data ) { SHORT sRc = 0; // return code @@ -1652,7 +1640,7 @@ BTREE::EQFNTMGet BOOL _fCorrupted = fCorrupted; fCorrupted = FALSE; - sRc = QDAMDictExactLocal( (PWCHAR) &ulKey, pchBuffer, FEXACT ); + sRc = QDAMDictExactLocal( ulKey, pchBuffer, FEXACT ); fCorrupted = _fCorrupted; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index a6e7fa8c..0729b302 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -45,7 +45,7 @@ PBYTE NTRecPos(PBYTE pStart, int iType); USHORT NTMAdjustAddDataInTgtCLB ( - std::vector& pTmRecordStart, + BytesRef pTmRecordStart, OtmProposal& TmProposal, PTMX_TARGET_CLB *ppClb, PTMX_TARGET_RECORD *ppTMXTargetRecord, @@ -54,7 +54,7 @@ USHORT NTMAdjustAddDataInTgtCLB USHORT TMLoopAndDelTargetClb ( - std::vector& pTmRecord, + BytesRef pTmRecord, OtmProposal& TmProposal, USHORT usPutLang, USHORT usPutFile, @@ -212,7 +212,7 @@ USHORT EqfMemory::TmtXReplace if ( !usRc ) { HashSentence( TmProposal.pInputSentence ); - TmProposal.pInputSentence->pTagRecord[0].usTagTableId = 1; + toTmxTagtableRecord(TmProposal.pInputSentence->pTagRecord)->usTagTableId = 1; if(usRc){ T5LOG( T5WARNING) << ":: NTMGetIDFromName( tagtable ) returned " << usRc; @@ -374,7 +374,6 @@ VOID HashSentence pSentence->pulVotes.push_back(pTermTokens[i].usHash); pSentence->usActVote++; T5LOG(T5TRANSACTION) << "pSentence->pulVotes.size = " << pSentence->pulVotes.size() <<"; actVote = " << pSentence->usActVote; - } /* endfor */ } /* endif */ @@ -471,35 +470,41 @@ BuildVotes { PTMX_TERM_TOKEN pTermTokens; // pointer to active token USHORT usIndex = 0; - PTMX_TERM_TOKEN pLastTerm; + PTMX_TERM_TOKEN pLastTerm = pSentence->pTermTokens.data() + pSentence->pTermTokens.size()-1; //run through list of tokens and build tuples pTermTokens = pSentence->pTermTokens.data(); - - while ( pTermTokens->usLength && (pSentence->usActVote < ABS_VOTES) ) + int index = 0 , listLen = pSentence->pTermTokens.size(); + while (// (indexusLength && (pSentence->usActVote < ABS_VOTES) ) { Vote( pTermTokens, pSentence, 0 ); // ABC, BCD, CDE, DEF, EFG pTermTokens++; + index++; } /* endwhile */ - pLastTerm = pTermTokens; - pLastTerm --; pTermTokens = pSentence->pTermTokens.data(); - - while ( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) + index = 0; + while ( //(indexusLength && (pSentence->usActVote < MAX_VOTES) ) { Vote( pTermTokens, pSentence, 1 ); // ABD, BCE, CDF, DEG pTermTokens++; + index++; } /* endwhile */ - pTermTokens = pSentence->pTermTokens.data(); - while ( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) + pTermTokens = pSentence->pTermTokens.data(); + index = 0; + while ( //indexusLength && (pSentence->usActVote < MAX_VOTES) ) { Vote( pTermTokens, pSentence, 2 ); // ACD, BDE, CEF, DFG pTermTokens++; + index++; } /* endwhile */ pTermTokens = pSentence->pTermTokens.data(); + index = 0; if( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { Vote (pTermTokens, pSentence, 4 ); // 1 * AAC @@ -514,6 +519,7 @@ BuildVotes /********************************************************************/ /* use 1st token as triple */ /********************************************************************/ + pTermTokens = pSentence->pTermTokens.data(); if( pTermTokens->usLength && (pSentence->usActVote < MAX_VOTES) ) { @@ -529,10 +535,12 @@ BuildVotes Vote (pTermTokens, pSentence, 7 ); // 1 * GGG } /* endif */ - usIndex = 0; + index = 0; pTermTokens = pSentence->pTermTokens.data(); - while ((usIndex < 3) && pTermTokens->usLength - && (pSentence->usActVote < MAX_VOTES) ) + while ((index < 3)// && index < listLen) + && pTermTokens->usLength + && (pSentence->usActVote < MAX_VOTES) + ) { Vote (pTermTokens, pSentence, 3 ); // 3 * AAB if ((pLastTerm-1)->usLength @@ -540,7 +548,7 @@ BuildVotes { Vote (pLastTerm-1, pSentence, 5 ); // 3 * FGG } /* endif */ - usIndex++; + index++; } /* endwhile */ } /* end of function BuildVotes */ @@ -570,6 +578,7 @@ Vote ULONG ulVote; //actual vote USHORT i; //index values BOOL fGo = FALSE; + int indexInList = pTermTokens - pSentence->pTermTokens.data(); // create all possible 3 tupels out of a sequence of 4 consecutive tokens //for dealing correctly with the tail... @@ -600,7 +609,7 @@ Vote pSentence->usActVote++; //#ifdef DEBUG - T5LOG(T5TRANSACTION) << "pulVodet.size()=" << pSentence->pulVotes.size() << "; actVote = " << pSentence->usActVote; + T5LOG(T5TRANSACTION) << "pulVodet.size()=" << pSentence->pulVotes.size() << "; actVote = " << pSentence->usActVote << "; indexInList = " << indexInList; //#endif } /* endif */ } /* end of function Vote */ @@ -737,50 +746,40 @@ USHORT EqfMemory::AddToTm ) { std::vector TmRecord; // ptr to tm record - PTMX_TARGET_CLB pTargetClb = NULL; // ptr to target ctl block + std::vector TargetClb; // ptr to target ctl block USHORT usRc = NO_ERROR; // return code BOOL fOK = true; // success indicator USHORT usAddDataLen = 0; usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - fOK = UtlAlloc( (PVOID *) &pTargetClb, 0L, (LONG)(sizeof(TMX_TARGET_CLB)+usAddDataLen), NOMSG ); + TargetClb.resize(sizeof(TMX_TARGET_CLB)+usAddDataLen); - if ( !fOK ) + usRc = FillClb( TargetClb, TmProposal ); + if ( usRc == NO_ERROR ) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - if ( usRc == NO_ERROR ) - { - usRc = FillClb( pTargetClb, TmProposal ); - if ( usRc == NO_ERROR ) - { - USHORT usSrcLang = 0; - usRc = NTMGetIDFromName( TmProposal.szSourceLanguage, NULL, (USHORT)LANG_KEY, &usSrcLang ); - TmRecord.resize(TMX_REC_SIZE); - //fill tm record to add to database - FillTmRecord ( TmProposal.pInputSentence, // ptr to sentence struct for source info - TmRecord, // filled tm record returned - pTargetClb, usSrcLang ); - - //add new tm record to database - *pulNewKey = NTMREQUESTNEWKEY; - - usRc = TmBtree.EQFNTMInsert(//ptr to tm structure - pulNewKey, //to be allocated in funct - TmRecord); //pointer to tm record - if(!usRc){ - TmProposal.currentInternalKey.setInternalKey(*pulNewKey, 1); - } - } /* endif */ - } /* endif */ + USHORT usSrcLang = 0; + usRc = NTMGetIDFromName( TmProposal.szSourceLanguage, NULL, (USHORT)LANG_KEY, &usSrcLang ); + TmRecord.resize(TMX_REC_SIZE); + //fill tm record to add to database + FillTmRecord ( TmProposal.pInputSentence, // ptr to sentence struct for source info + TmRecord, // filled tm record returned + TargetClb, usSrcLang ); + + //add new tm record to database + *pulNewKey = NTMREQUESTNEWKEY; + + if(toTmxRecord(TmRecord)->lRecordLen > 0){ + TmRecord.resize(toTmxRecord(TmRecord)->lRecordLen); + } + usRc = TmBtree.EQFNTMInsert(//ptr to tm structure + *pulNewKey, //to be allocated in funct + TmRecord); //pointer to tm record + if(!usRc){ + TmProposal.currentInternalKey.setInternalKey(*pulNewKey, 1); + } } /* endif */ - //release memory - UtlAlloc( (PVOID *) &(pTargetClb), 0L, 0L, NOMSG); - return( usRc ); } @@ -816,11 +815,12 @@ USHORT EqfMemory::AddToTm VOID FillTmRecord ( PTMX_SENTENCE pSentence, // ptr to sentence struct for source info - std::vector& TmRecord, // filled tm record returned - PTMX_TARGET_CLB pTargetClb, // ptr to target control block + BytesRef TmRecord, // filled tm record returned + BytesRef TargetClb, // ptr to target control block USHORT usSrcLangId ) { + PTMX_TARGET_CLB pTargetClb = (PTMX_TARGET_CLB)TargetClb.data(); PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to start of source structure PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to start of target structure PBYTE pTarget; //ptr to target record @@ -857,36 +857,7 @@ VOID FillTmRecord pTMXTargetRecord->usTarget = sizeof(*pTMXTargetRecord); pTarget += pTMXTargetRecord->usTarget; - - //set source tag table offset - //pTMXTargetRecord->usSourceTagTable = sizeof(TMX_TARGET_RECORD); - - //position pointer for source tag table - //pTarget += pTMXTargetRecord->usSourceTagTable; - - //copy source tag table record to correct position - //memcpy( pTarget, pSentence->pTagRecord, RECLEN(pSentence->pTagRecord) ); - - //set target tag table start offset - //pTMXTargetRecord->usTargetTagTable = (USHORT)(pTMXTargetRecord->usSourceTagTable + - // RECLEN(pSentence->pTagRecord)); - //adjust target pointer for target tag table - //pTarget += RECLEN(pSentence->pTagRecord); - - //copy target tag table record to correct position - //memcpy( pTarget, pTagRecord, RECLEN(pTagRecord) ); - - //set target string start offset - //pTMXTargetRecord->usTarget = //(USHORT)(pTMXTargetRecord->usTargetTagTable + - //RECLEN(pTagRecord); - //adjust target pointer for target string - //pTarget += RECLEN(pTagRecord); - - //copy target string to correct position -//@@ memcpy( pTarget, pNormString, usNormLen ); - { - ulTrgLen = EQFUnicode2Compress( pTarget, pSentence->pStrings->getGenericTargetStrC(), ulTrgLen ); - } + ulTrgLen = EQFUnicode2Compress( pTarget, pSentence->pStrings->getGenericTargetStrC(), ulTrgLen ); //set target string control block start offset pTMXTargetRecord->usClb = pTMXTargetRecord->usTarget + @@ -951,10 +922,11 @@ VOID FillTmRecord //------------------------------------------------------------------------------ USHORT EqfMemory::FillClb ( - PTMX_TARGET_CLB pTargetClb, // ptr to target control block + BytesRef TargetClb, // ptr to target control block OtmProposal& TmProposal // ptr to put input structure ) { + PTMX_TARGET_CLB pTargetClb = (PTMX_TARGET_CLB) TargetClb.data(); USHORT usTrgLang,usSrcLang; USHORT usFile = 0; USHORT usAuthor = 0; // ids @@ -1061,33 +1033,27 @@ USHORT EqfMemory::UpdateTmIndex //for all votes add the index to the corresponding list pulVotes = pSentence->pulVotes.data(); - //allocate 32K for tm index record - pIndexRecord.resize(TMX_REC_SIZE); - - if ( !fOK ) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - for ( i = 0; i < pSentence->usActVote; i++, pulVotes++ ) + for ( i = 0; i < pSentence->pulVotes.size(); i++, pulVotes++ ) { if ( usRc == NO_ERROR ) { ulKey = (*pulVotes) & START_KEY; - memset( pIndexRecord.data(), 0, TMX_REC_SIZE ); usRc = InBtree.EQFNTMGet( ulKey, //index key pIndexRecord); //pointer to index record if ( usRc == BTREE_NOT_FOUND ) { + pIndexRecord.resize(sizeof( TMX_INDEX_RECORD )); //key is not in index file; add a new index entry toIndexRecord(pIndexRecord)->usRecordLen = sizeof( TMX_INDEX_RECORD ); - toIndexRecord(pIndexRecord)->stIndexEntry = NTMINDEX(pSentence->usActVote,ulSidKey); + toIndexRecord(pIndexRecord)->stIndexEntry = NTMINDEX(pSentence->pulVotes.size(),ulSidKey); + if( toIndexRecord(pIndexRecord)->usRecordLen > 0){ + pIndexRecord.resize( toIndexRecord(pIndexRecord)->usRecordLen); + } - usRc = InBtree.EQFNTMInsert( &ulKey, - pIndexRecord); //pointer to index + usRc = InBtree.EQFNTMInsert( ulKey, pIndexRecord); //pointer to index //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file @@ -1097,7 +1063,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMInsert(&ulKey, pIndexRecord);//, toIndexRecord(pIndexRecord)->usRecordLen ); + usRc = InBtree.EQFNTMInsert(ulKey, pIndexRecord);//, toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ @@ -1119,6 +1085,9 @@ USHORT EqfMemory::UpdateTmIndex //key is in index file; update index entry with new sid ULONG ulLen = toIndexRecord(pIndexRecord)->usRecordLen; + if( toIndexRecord(pIndexRecord)->usRecordLen > 0){ + pIndexRecord.resize( toIndexRecord(pIndexRecord)->usRecordLen + sizeof(ULONG)); + } //calculate number of entries in index record usIndexEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); @@ -1660,7 +1629,7 @@ DECLARE_bool(log_memmove_in_compareputdata); //------------------------------------------------------------------------------ USHORT EqfMemory::ComparePutData ( - std::vector& pTmRecord, // ptr to ptr of tm record data buffer + BytesRef pTmRecord, // ptr to ptr of tm record data buffer OtmProposal& TmProposal, // pointer to get in data PULONG pulKey // tm key ) @@ -1978,7 +1947,7 @@ USHORT EqfMemory::ComparePutData //------------------------------------------------------------------------------ USHORT EqfMemory::AddTmTarget( OtmProposal& TmProposal, //pointer to get in data - std::vector& pTmRecord, //pointer to tm record data pointer + BytesRef pTmRecord, //pointer to tm record data pointer PULONG pulKey ) //tm key { PTMX_TARGET_CLB pTargetClb = nullptr ; // ptr to target ctl block @@ -2000,7 +1969,7 @@ USHORT EqfMemory::AddTmTarget( { if ( usRc == NO_ERROR ) { - usRc = FillClb( pTargetClb, TmProposal ); + usRc = FillClb( vTargetClb, TmProposal ); if ( usRc == NO_ERROR ) { //fill target record @@ -2384,7 +2353,7 @@ LONG EQFCompress2Unicode( std::wstring& pOutput, PBYTE pTarget, ULONG ulLenComp USHORT NTMAdjustAddDataInTgtCLB ( - std::vector& pTmRecord, + BytesRef pTmRecord, OtmProposal& TmProposal, PTMX_TARGET_CLB *ppClb, PTMX_TARGET_RECORD *ppTMXTargetRecord, @@ -2524,7 +2493,7 @@ USHORT NTMAdjustAddDataInTgtCLB USHORT TMLoopAndDelTargetClb ( - std::vector& pTmRecord, + BytesRef pTmRecord, OtmProposal& TmProposal, USHORT usPutLang, USHORT usPutFile, diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 46de8855..f7f43c93 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -56,11 +56,16 @@ #include // for std::transform //#include // for std::tolower -void toLower(std::vector& vec) { +void toLower(BytesRef vec) { std::transform(vec.begin(), vec.end(), vec.begin(), [](unsigned char c) { return std::tolower(c); }); } +void toUpper(BytesRef vec) { + std::transform(vec.begin(), vec.end(), vec.begin(), [](unsigned char c) { + return std::toupper(c); + }); +} #include #include @@ -76,6 +81,18 @@ std::vector toLowerVector(const std::vector& input return result; } + +std::vector toUpperVector(const std::vector& input) { + std::vector result; + result.reserve(input.size()); // Optimize memory allocation + + for (unsigned char c : input) { + result.push_back(c == '\0' ? '\0' : std::toupper(c)); + } + + return result; +} + // #include "EQFCMPR.H" /**********************************************************************/ /* prototypes for internal functions */ @@ -224,8 +241,6 @@ USHORT EqfMemory::NTMGetIDFromNameEx( PTMX_TABLE pstTMTable = NULL; // ptr to table structure BOOL fLongName = FALSE; - // initialize ID - // initialize ID if (lOptions & NTMGETID_NOUPDATE_OPT) { @@ -235,10 +250,7 @@ USHORT EqfMemory::NTMGetIDFromNameEx( { *pusID = 0; } /* endif */ - if (pusAlternativeID) - *pusAlternativeID = NTMGETID_NOTFOUND_ID; - *pusID = 0; if (pusAlternativeID) *pusAlternativeID = NTMGETID_NOTFOUND_ID; @@ -368,7 +380,7 @@ USHORT EqfMemory::NTMGetIDFromNameEx( { LongNamesCaseIgnore.stTableEntry = LongNames.stTableEntry; LongNamesCaseIgnore.pszBuffer = LongNames.pszBuffer; - toLower(LongNamesCaseIgnore.pszBuffer); + toUpper(LongNamesCaseIgnore.pszBuffer); LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); //memcpy(LongNamesCaseIgnore.stTableEntry.data(), @@ -399,11 +411,13 @@ USHORT EqfMemory::NTMGetIDFromNameEx( /********************************************************************/ /* search name passed in pszName in passed table usTableType */ /********************************************************************/ - pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch(pszName, - pstTMTableEntries, - pstTMTable->ulMaxEntries, - sizeof(TMX_TABLE_ENTRY), - NTMCompNames); + if(pstTMTable->ulMaxEntries){ + pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch(pszName, + pstTMTableEntries, + pstTMTable->ulMaxEntries, + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); + } //--- if passed name found in table if (pstTMTableEntry) { @@ -991,7 +1005,7 @@ T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be // make copy of long name array for case ignore search LongNamesCaseIgnore.stTableEntry = LongNames.stTableEntry; - LongNamesCaseIgnore.pszBuffer = toLowerVector(LongNames.pszBuffer); + LongNamesCaseIgnore.pszBuffer = toUpperVector(LongNames.pszBuffer); LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); @@ -1007,10 +1021,10 @@ T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be return (usRC); } /* end of function NTMReadLongNameTable */ -int tmxTableToBuffer(PTMX_TABLE pTable, std::vector& buff){ +int tmxTableToBuffer(PTMX_TABLE pTable, BytesRef buff){ buff.clear(); - int occupSize = (pTable->ulMaxEntries + 1) * sizeof(TMX_TABLE_ENTRY); - buff.resize(occupSize);; + int occupSize = sizeof(LONG) + (pTable->ulMaxEntries + 1) * sizeof(TMX_TABLE_ENTRY); + buff.resize(occupSize); TMX_TABLE_OLD *table = (TMX_TABLE_OLD *)buff.data(); //pOldTable = pstTMTable->stTmTableEntry; table->ulMaxEntries = pTable->ulMaxEntries; @@ -1099,11 +1113,6 @@ USHORT EqfMemory::NTMAddNameToTable( if (pstTMTable->ulMaxEntries >= currentCap) { size_t newCap = currentCap + 10; - if (newCap >= pstTMTable->stTmTableEntry.max_size()) - { - T5LOG(T5FATAL) << "tried to set too big capacity for table " << usTableType << "; newCap = " << newCap; - } - // T5LOG(T5ERROR)<<"remove this log later: tableType = " << usTableType <<" changing capacity to "<< newCap; pstTMTable->stTmTableEntry.resize(newCap); pstTMTableEntries = &pstTMTable->stTmTableEntry[0]; } @@ -1181,7 +1190,7 @@ int NTMLongNameTableCompCaseIgnore( { PSZ lname1 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName; PSZ lname2 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName; - return strcmp(lname1,lname2);//string should be in lower case here + return strcasecmp(lname1,lname2);//string should be in lower case here //return strcasecmp(lname1,lname2); } /* end of function NTMCompNames */ @@ -1189,7 +1198,7 @@ int NTMLongNameTableCompCaseIgnore( -USHORT bytesToOldTable(ULONG ulTableKey, std::vector oldTmxTable, PTMX_TABLE pTMTable) +USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTable) { USHORT usRc = 0; // handle tersed name tables diff --git a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp index e82e88b2..359c0971 100755 --- a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp @@ -79,100 +79,6 @@ PSZ_W MADSkipName( PSZ_W pszName ); PSZ_W MADSkipWhitespace( PSZ_W pszData ); // -// Delete a key with all its atttributes and data -// -BOOL MADDeleteKey( HADDDATAKEY pKey ) -{ - PSZ_W pszCurPos = pKey; - BOOL fAttrAvailable = FALSE; - PSZ_W pszTagName = NULL; - PSZ_W pszTagNameEnd = NULL; - - // check key parameter - if ( (pszCurPos == NULL) || (*pszCurPos == 0) ) - return( FALSE ); - - // skip tag name - if ( *pszCurPos == L'<' ) pszCurPos++; - pszTagName = pszCurPos; - pszTagNameEnd = pszCurPos = MADSkipName( pszCurPos ); - do - { - fAttrAvailable = MADNextAttr( &pszCurPos ); - } while ( fAttrAvailable ); /* enddo */ - - // find end of key data - if ( (*pszCurPos == L'/') && (*(pszCurPos+1) == L'>') ) - { - // a selfcontained tag... so remove everything up to end of the tag - PSZ_W pszSource = pszCurPos + 2; - PSZ_W pszTarget = pKey; - while ( *pszSource ) *pszTarget++ = *pszSource++; - *pszTarget = 0; - return( TRUE ); - } - - // find ending tag - // note: this is a very simple approach which only looks for the ending tag and does not interpret the data in any way... - while ( *pszCurPos != 0 ) - { - // move to begin of next tag - while ( (*pszCurPos != 0) && (*pszCurPos != L'<') ) *pszCurPos++; - - // check for end tag start - if ( (*pszCurPos == L'<') && (*(pszCurPos+1) == L'/') ) - { - // check if we have the correct end tag - if ( MADCompareKey( pszCurPos + 2, pszTagName ) ) - { - // end tag found, now skip end tag name and closing pointy brace - pszCurPos = MADSkipName( pszCurPos + 2 ); - pszCurPos = MADSkipWhitespace( pszCurPos ); - if ( *pszCurPos == L'>' ) pszCurPos++; - - // remove all data between (and including) start tag and end tag - PSZ_W pszSource = pszCurPos; - PSZ_W pszTarget = pKey; - while ( *pszSource ) *pszTarget++ = *pszSource++; - *pszTarget = 0; - - // removal of tag has been completed - return( TRUE ); - } /* endif */ - } /* endif */ - pszCurPos++; - } /* endwhile */ - - return( FALSE ); -} - - -// Add a match segment ID to the additional data section -BOOL MADAddMatchSegID( PSZ_W pszAddData, PSZ_W pszMatchIDPrefix, ULONG ulNum, BOOL fForce ) -{ - HADDDATAKEY hKey; - BOOL fMatchIDAdded = FALSE; - - // find any existing match segment ID - hKey = MADSearchKey( pszAddData, MATCHSEGID_KEY ); - if ( (hKey != NULL) && fForce ) - { - MADDeleteKey( hKey ); - hKey = NULL; - } /* endif */ - - // add match segment ID if non exists or the existing has been deleted and there is enough room left in the AddInfo buffer - if ( hKey == NULL ) - { - int iCurLen = wcslen(pszAddData); - if ( (wcslen(pszMatchIDPrefix) + iCurLen + 23) < MAX_SEGMENT_SIZE ) - { - swprintf( pszAddData + iCurLen, wcslen(pszAddData) - iCurLen, L"", pszMatchIDPrefix, ulNum ); - fMatchIDAdded = TRUE; - } /* endif */ - } /* endif */ - return( fMatchIDAdded ); -} //+----------------------------------------------------------------------------+ @@ -319,20 +225,20 @@ USHORT EqfMemory::TokenizeSource { std::vector pTokenList; // ptr to token table BOOL fOK = true; // success indicator - PTMX_TAGENTRY pTagEntry = nullptr; // pointer to tag entries + TMX_TAGENTRY TagEntry ; // pointer to tag entries //PTMX_TERM_TOKEN pTermTokens = nullptr; // pointer to term tokens PLOADEDTABLE pTable = NULL; // pointer to tagtable - PTMX_TAGTABLE_RECORD pTagRecord; // pointer to record USHORT usLangId; // language id USHORT usRc = NO_ERROR; // returned value size_t sFilled; // counter USHORT usTagEntryLen; // length indicator - CHAR szString[MAX_FNAME]; // name without extension + std::string szString; // name without extension USHORT usStart; // position counter - PSTARTSTOP pStartStop = NULL; // ptr to start/stop table + std::vector pStartStop; // ptr to start/stop table int iIterations = 0; USHORT usAddEntries = 0; PSZ_W pStringToTokenize = pSentence->pStrings->getNormStrC();//getNormStrC();//getNpReplStrC() + szString.resize(MAX_FNAME); /********************************************************************/ /* normalize \r\n combinations in input string .. */ /********************************************************************/ @@ -370,30 +276,25 @@ USHORT EqfMemory::TokenizeSource //allocate 4K pTokenlist for TaTagTokenize pTokenList.reserve(TOK_SIZE/sizeof(pTokenList[0])); - if ( !fOK ) { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else - { - pTagRecord = pSentence->pTagRecord.data(); + //pTermTokens = pSentence->pTermTokens.data(); - pTagEntry = (PTMX_TAGENTRY) ((PBYTE)pTagRecord + sizeof(TMX_TAGTABLE_RECORD)); + //pTagEntry = (PTMX_TAGENTRY) ((PBYTE)pTagRecord + sizeof(TMX_TAGTABLE_RECORD)); - RECLEN(pTagRecord) = 0; - pTagRecord->usFirstTagEntry = (USHORT)((PBYTE)pTagEntry - (PBYTE)pTagRecord); + RECLEN(toTmxTagtableRecord(pSentence->pTagRecord)) = 0; + toTmxTagtableRecord(pSentence->pTagRecord)->usFirstTagEntry = pSentence->pTagRecord.size();// (USHORT)((PBYTE)pTagEntry - (PBYTE)pTagRecord); //get id of tag table, call - Utlstrccpy( szString, UtlGetFnameFromPath( pTagTableName ), DOT ); + Utlstrccpy( szString.data(), UtlGetFnameFromPath( pTagTableName ), DOT ); - pTagRecord->usTagTableId = 1; + toTmxTagtableRecord(pSentence->pTagRecord)->usTagTableId = 1; //get lang id of source lang for morphtokenize call usLangId = 0; if ( !usRc ) { //load tag table for tokenize function - usRc = TALoadTagTableExHwnd( szString, &pTable, FALSE, + usRc = TALoadTagTableExHwnd( szString.data(), &pTable, FALSE, TALOADUSEREXIT | TALOADPROTTABLEFUNC | TALOADCOMPCONTEXTFUNC, FALSE, NULLHANDLE ); @@ -403,11 +304,11 @@ USHORT EqfMemory::TokenizeSource LOG_AND_SET_RC(usRc, T5INFO, ERROR_TA_ACC_TAGTABLE); if ( usAction == MEMIMP_MRKUP_ACTION_RESET ) { CHAR *ptrMarkup ; - ptrMarkup = strstr( pTagTableName, szString ) ; + ptrMarkup = strstr( pTagTableName, szString.data() ) ; if ( ptrMarkup ) { strcpy( ptrMarkup, "OTMUTF8.TBL" ) ; - strcpy( szString, "OTMUTF8" ) ; - usRc = TALoadTagTableExHwnd( szString, &pTable, FALSE, + strcpy( szString.data(), "OTMUTF8" ) ; + usRc = TALoadTagTableExHwnd( szString.data(), &pTable, FALSE, TALOADUSEREXIT | TALOADPROTTABLEFUNC | TALOADCOMPCONTEXTFUNC, FALSE, NULLHANDLE ); @@ -428,46 +329,16 @@ USHORT EqfMemory::TokenizeSource usRc = TACreateProtectTableWEx( pStringToTokenize, pTable, 0, pTokenList, - TOK_SIZE, &pStartStop, - pTable->pfnProtTable, + TOK_SIZE, pStartStop, + //pTable->pfnProtTable, pTable->pfnProtTableW, 1, 0); - - #ifdef TO_BE_REMOVED - while ((iIterations < 10) && (usRc == EQFRS_AREA_TOO_SMALL)) - { - // (re)allocate token buffer - LONG lOldSize = (usAddEntries * sizeof(TOKENENTRY)) + (LONG)TOK_SIZE; - LONG lNewSize = ((usAddEntries+128) * sizeof(TOKENENTRY)) + (LONG)TOK_SIZE; - - if (UtlAlloc((PVOID *) &pTokenList, lOldSize, lNewSize, NOMSG) ) - { - usAddEntries += 128; - iIterations++; - } - else - { - iIterations = 10; // force end of loop - } /* endif */ - // retry tokenization - if (iIterations < 10 ) - { - usRc = TACreateProtectTableWEx( pStringToTokenize, - pTable, 0, - pTokenList, - (USHORT)lNewSize, &pStartStop, - pTable->pfnProtTable, - pTable->pfnProtTableW, 1, 0 ); - } /* endif */ - - } /* endwhile */ - #endif } /* endif */ if ( !usRc ) { USHORT usEntries = 0; // number of entries in start stop table - PSTARTSTOP pEntry = pStartStop; + PSTARTSTOP pEntry = (PSTARTSTOP)pStartStop.data(); while ( (pEntry->usStart != 0) || (pEntry->usStop != 0) || (pEntry->usType != 0) ) @@ -510,7 +381,7 @@ USHORT EqfMemory::TokenizeSource TermToken.usLength = pTerm->usLen; TermToken.usHash = 0; - pSentence->pTermTokens.emplace_back(TermToken); + pSentence->pTermTokens.push_back(TermToken); } /* endif */ pTerm++; } /* endwhile */ @@ -532,32 +403,33 @@ USHORT EqfMemory::TokenizeSource { // add tag data usTagEntryLen = sizeof(TMX_TAGENTRY) + (pEntry->usStop - pEntry->usStart + 1) * sizeof(CHAR_W); + sFilled = pSentence->pTagRecord.size(); + //sFilled = (PBYTE)pTagEntry - (PBYTE)pSentence->pTagRecord.data(); - - sFilled = (PBYTE)pTagEntry - (PBYTE)pSentence->pTagRecord.data(); - - if ( std::ceil((sFilled + usTagEntryLen) / static_cast(sizeof(TMX_TAGTABLE_RECORD))) // space in bytes/size of entry - <= pSentence->pTagRecord.size()) + //if ( std::ceil((sFilled + usTagEntryLen) / static_cast(sizeof(TMX_TAGTABLE_RECORD))) // space in bytes/size of entry + // <= pSentence->pTagRecord.size()) { //remember offset of pTagEntry - pSentence->pTagRecord.resize(pSentence->pTagRecord.size() + usTagEntryLen); - pTagEntry = (PTMX_TAGENTRY)((PBYTE)pSentence->pTagRecord.data() + sFilled); + //pSentence->pTagRecord.resize(pSentence->pTagRecord.size() + usTagEntryLen); + //pTagEntry = (PTMX_TAGENTRY)((PBYTE)pSentence->pTagRecord.data() + sFilled); } - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else { - pTagEntry->usOffset = pEntry->usStart; - pTagEntry->usTagLen = - (pEntry->usStop - pEntry->usStart + 1); - memcpy( &(pTagEntry->bData), - pStringToTokenize + pEntry->usStart, - pTagEntry->usTagLen * sizeof(CHAR_W)); - pTagEntry += usTagEntryLen; + //TagEntry.reset(); + //TagEntry.usOffset = pEntry->usStart; + pushBytes(pSentence->pTagRecord, pEntry->usStart); //offset + USHORT usTagLen = pEntry->usStop - pEntry->usStart + 1; + pushBytes(pSentence->pTagRecord, usTagLen);//TagEntry.usTagLen + PBYTE data = (PBYTE) &pStringToTokenize[pEntry->usStart]; + pushBytes(pSentence->pTagRecord, data, data + usTagLen * sizeof(CHAR_W)); + + //TagEntry.usTagLen = pEntry->usStop - pEntry->usStart + 1; + + //memcpy( &(TagEntry.bData), + // pStringToTokenize + pEntry->usStart, + // TagEntry.usTagLen * sizeof(CHAR_W)); + //pSentence->pTagRecord.insert(pSentence->pTagRecord.end(), TagEntry, sizeof(TagEntry)); + //pTagEntry += usTagEntryLen; } /* endif */ } /* endif */ } /* end default */ @@ -571,30 +443,24 @@ USHORT EqfMemory::TokenizeSource /* check if we filled at least one term token -- if not */ /* use a dummy token - just to get an exact match ... */ /********************************************************/ + TMX_TERM_TOKEN TermToken; + TermToken.reset(); if (pSentence->pTermTokens.empty()) //pTermTokens == pSentence->pTermTokens.data() ) { - TMX_TERM_TOKEN TermToken; TermToken.usOffset = 0; TermToken.usLength = (USHORT)UTF16strlenCHAR( pStringToTokenize ); TermToken.usHash = 0; - pSentence->pTermTokens.emplace_back(TermToken); - + pSentence->pTermTokens.push_back(TermToken); + TermToken.reset(); } /* endif */ - //fOK = CheckForAlloc( pSentence, &pTermTokens, 3 ); - //if ( !fOK ) - //{ - // LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - //} + pSentence->pTermTokens.push_back(TermToken); //set total tag record length - RECLEN(pTagRecord) = (PBYTE)pTagEntry - (PBYTE)pTagRecord; + RECLEN(toTmxTagtableRecord(pSentence->pTagRecord)) = pSentence->pTagRecord.size();//(PBYTE)pTagEntry - (PBYTE)pTagRecord; } /* endif */ } /* endif */ - //release allocated memory - if ( pStartStop ) UtlAlloc( (PVOID *) &pStartStop, 0L, 0L, NOMSG ); - //free tag table / decrement use count if ( pTable != NULL ) { diff --git a/source/opentm2/core/tagtable/EQFTOKEN.cpp b/source/opentm2/core/tagtable/EQFTOKEN.cpp index f262f4b0..dc3188b8 100755 --- a/source/opentm2/core/tagtable/EQFTOKEN.cpp +++ b/source/opentm2/core/tagtable/EQFTOKEN.cpp @@ -109,24 +109,14 @@ static CHAR_W chInputW[ MAX_CHINPUTW ]; #define TAG_END_CHAR "." // tag end character #define TAG_END_CHARW L"." -//PNODE TAFastCreateNodeTree( PTBTAGENTRY pTags, ULONG ulNoOfTags ); -//PNODE TAFastCreateTagTree( PLOADEDTABLE pLoadedTable, PTAGTABLE pTagTable ); -//PNODE TAFastCreateAttrTree( PLOADEDTABLE pLoadedTable, PTAGTABLE pTagTable ); -//BOOL TAFastAddNode ( PNODE *, PNODE, PULONG, PULONG ); -//int TATastTagCompare( const void *, const void * ); - -VOID TASetStartStopType(PSTARTSTOP pCurrent, USHORT usStart, USHORT usStop, USHORT usType ); USHORT TAFindQuote ( PTOKENENTRY pTok, USHORT usStartSearch, PCHAR_W pchQuote ); USHORT TAProtectedPartsInQuotedText ( PTOKENENTRY pTok, int iAttrStartOffset, - PSTARTSTOP * pStartStop, - PSTARTSTOP * pCurrent, + BytesRef pStartStop, std::vector& pAttrTokBuffer, PLOADEDTABLE pTagTable, - PULONG pulTableAlloc, - PULONG pulTableUsed, PUSHORT pusColPos, CHAR_W chQuote, LONG lAttrSize, @@ -135,10 +125,9 @@ USHORT TAProtectedPartsInQuotedText USHORT TAGotoNextStartStopEntry ( - PSTARTSTOP * ppStartStop, + BytesRef pStartStop, PSTARTSTOP * ppCurrent, - PULONG pulTableUsed, - PULONG pulTableAlloc + PULONG pulTableUsed ); static void TATagAddWSpace @@ -2087,12 +2076,11 @@ PNODEAREA TACreateNodeTree USHORT TASoSiProtectTable ( PSZ_W pszSegment, // ptr to text of segment being processed - PSTARTSTOP *ppStartStop // ptr to caller's start/stop table ptr + BytesRef pStartStop // ptr to caller's start/stop table ptr ) { USHORT usRC = NO_ERROR; // function return code - PSTARTSTOP pStartStop = *ppStartStop; // ptr to new start/stop table - PSTARTSTOP pCurrent = NULL; // ptr for start/stop table processing + std::vector pCurrent; // ptr for start/stop table processing PSTARTSTOP pAct, pActNew; // ptr for start/stop table processing PSZ_W pText = pszSegment; // ptr to segment USHORT usCountSOSI = 0; @@ -2115,7 +2103,7 @@ USHORT TASoSiProtectTable if ( usCountSOSI ) { - pAct = pStartStop; + pAct = (PSTARTSTOP)pStartStop.data(); while ( pAct->usStop ) { pAct++; @@ -2124,16 +2112,12 @@ USHORT TASoSiProtectTable usCountSOSI++; LONG size = (usCountSOSI * sizeof(STARTSTOP)) > MIN_ALLOC ? (usCountSOSI * sizeof(STARTSTOP)) : MIN_ALLOC; - if ( !UtlAlloc( (PVOID *)&pCurrent, 0L, size, NOMSG )) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - + pCurrent.resize(size); if ( !usRC ) { - pAct = pStartStop; - pActNew = pCurrent; + pAct = (PSTARTSTOP)pStartStop.data(); + pActNew = (PSTARTSTOP)pCurrent.data(); while ( pAct->usStop ) { memcpy( pActNew, pAct, sizeof(STARTSTOP)); @@ -2183,8 +2167,7 @@ USHORT TASoSiProtectTable // Add end element .. memcpy( pActNew, pAct, sizeof(STARTSTOP)); - UtlAlloc( (PVOID *) ppStartStop, 0L, 0L, NOMSG ); - *ppStartStop = pCurrent; + pStartStop = pCurrent; } /* endif */ } /* endif */ @@ -2195,7 +2178,7 @@ USHORT TASoSiProtectTable typedef EQF_BOOL (// /*APIENTRY*/ *PFNSTARTSTOPEXIT) ( PSZ, PSTARTSTOP *); typedef EQF_BOOL (// /*APIENTRY*/ -*PFNSTARTSTOPEXITW) ( PSZ_W, PSTARTSTOP *); +*PFNSTARTSTOPEXITW) ( PSZ_W, BytesRef); //------------------------------------------------------------------------------ // External function @@ -2221,22 +2204,6 @@ typedef EQF_BOOL (// /*APIENTRY*/ // error // ERROR_NOT_ENOUGH_MEMORY if memory allocation failed //------------------------------------------------------------------------------ -USHORT TACreateProtectTableW -( - PSZ_W pszSegment, // ptr to text of segment being processed - PVOID pVoidTable, // ptr to tag table - USHORT usColPos, // column position of first char in segment - std::vector& pTokBuffer, // buffer used temporarly for tokens - USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function - PFN pvUserExitW, - ULONG ulCP // ASCII cp fitting to Segmenttext -) -{ - return( TACreateProtectTableWEx( pszSegment, pVoidTable, usColPos, pTokBuffer, usTokBufferSize, - ppStartStop, pvUserExit, pvUserExitW, ulCP, 0 ) ); -} int memicmp(const void *buf1, const void *buf2, size_t count){ return memcmp(buf1, buf2, count); @@ -2249,20 +2216,17 @@ USHORT TACreateProtectTableWEx USHORT usColPos, // column position of first char in segment std::vector& pTokBuffer, // buffer used temporarly for tokens USHORT usTokBufferSize, // size of token buffer in bytes - PSTARTSTOP *ppStartStop, // ptr to caller's start/stop table ptr - PFN pvUserExit, // ptr to user exit function + BytesRef pStartStop, // ptr to caller's start/stop table ptr + //PFN pvUserExit, // ptr to user exit function PFN pvUserExitW, ULONG ulCP, // ASCII cp fitting to Segmenttext int iMode // mode for function ) { USHORT usRC = NO_ERROR; // function return code - PSTARTSTOP pStartStop = NULL; // ptr to new start/stop table - PSTARTSTOP pCurrent; // ptr for start/stop table processing + //PSTARTSTOP pCurrent; // ptr for start/stop table processing PSTARTSTOP pAct; // ptr for start/stop table processing BOOL fTag; // we are in tag processing mode - ULONG ulTableUsed = 0; - ULONG ulTableAlloc = 0; PLOADEDTABLE pTagTable; // ptr to loaded tag table PTOKENENTRY pTok; // ptr for token list processing PSZ_W pRest; // ptr to not tokenized data @@ -2281,6 +2245,8 @@ USHORT TACreateProtectTableWEx std::vector pAttrTokBuffer; // used temp.for translatable attributetokens LONG lAttrSize = 0L; PTOKENENTRY pNextTok; // ptr for next token in token list + + STARTSTOP current; /********************************************************************/ /* Check input parameters */ @@ -2289,9 +2255,7 @@ USHORT TACreateProtectTableWEx if ( (pszSegment == NULL) || (pTagTable == NULL) || - //(pTokBuffer == NULL) || - (usTokBufferSize == 0) || - (ppStartStop == NULL) ) + (usTokBufferSize == 0)) { LOG_AND_SET_RC(usRC, T5INFO, ERROR_INVALID_DATA); } /* endif */ @@ -2300,432 +2264,332 @@ USHORT TACreateProtectTableWEx /* Use user exit for start/stop table if there is one */ /********************************************************************/ if ( usRC == NO_ERROR ) + { + /******************************************************************/ + /* Create start/stop table using tokenizer */ + /* Set tag table variables */ + /********************************************************************/ + sNumTags = (SHORT)pTagTable->pTagTable->uNumTags; + pByte = (PBYTE)pTagTable->pTagTable; + pTag = OFFSETTOPOINTER(PTAG, pTagTable->pTagTable->stFixTag.uOffset ); + pAttribute = OFFSETTOPOINTER(PATTRIBUTE, + pTagTable->pTagTable->stAttribute.uOffset ); + + + /********************************************************************/ + /* Tokenize input segment */ + /********************************************************************/ + pTagTable = (PLOADEDTABLE)pVoidTable; + pRest = NULL; + TATagTokenizeW( pszSegment, + pTagTable, + TRUE, + &pRest, + &usColPos, + pTokBuffer); + usRC = ( pRest == NULL ) ? NO_ERROR : EQFRS_AREA_TOO_SMALL; + + TATagAddWSpace(pszSegment, pTokBuffer.data(), sNumTags); + + } + + /********************************************************************/ + /* Fill start/stop table */ + /********************************************************************/ + if ( usRC == NO_ERROR ) { - if (pvUserExitW ) - { - pfnUserExitW = (PFNSTARTSTOPEXITW) pvUserExitW; - if ( !pfnUserExitW( pszSegment, &pStartStop ) ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } - else if ( pvUserExit ) + current.reset(); + pTok = pTokBuffer.data(); + while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) { /****************************************************************/ - /* Let user exit create the start/stop table */ + /* Check for attributes or tags with translatable info */ /****************************************************************/ - pfnUserExit = (PFNSTARTSTOPEXIT)pvUserExit; - Unicode2ASCII( pszSegment, &chAsciiSeg[0], ulCP); - if ( !pfnUserExit( &chAsciiSeg[0], &pStartStop ) ) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } - else - { - - /******************************************************************/ - /* Create start/stop table using tokenizer */ - /******************************************************************/ - - /********************************************************************/ - /* Set tag table variables */ - /********************************************************************/ - if ( usRC == NO_ERROR ) - { - sNumTags = (SHORT)pTagTable->pTagTable->uNumTags; - pByte = (PBYTE)pTagTable->pTagTable; - pTag = OFFSETTOPOINTER(PTAG, pTagTable->pTagTable->stFixTag.uOffset ); - pAttribute = OFFSETTOPOINTER(PATTRIBUTE, - pTagTable->pTagTable->stAttribute.uOffset ); - } /* endif */ - - /********************************************************************/ - /* Tokenize input segment */ - /********************************************************************/ - if ( usRC == NO_ERROR ) + if ( ( (pTok->sTokenid >= sNumTags) && + (pAttribute[pTok->sTokenid - sNumTags].BitFlags.fTranslate) ) || + ( (pTok->sTokenid >= 0) && + (pTok->sTokenid < sNumTags) && + (pTag[pTok->sTokenid].BitFlags.fTranslate) ) ) { - pTagTable = (PLOADEDTABLE)pVoidTable; - pRest = NULL; - TATagTokenizeW( pszSegment, - pTagTable, - TRUE, - &pRest, - &usColPos, - pTokBuffer); - usRC = ( pRest == NULL ) ? NO_ERROR : EQFRS_AREA_TOO_SMALL; - } /* endif */ + /**************************************************************/ + /* Process data up to first quote or up to end of token */ + /* Complete protected start/stop entry in front of */ + /* quoted string */ + /**************************************************************/ + current.usStart = (USHORT)(pTok->pDataStringW - pszSegment); + current.usStop = TAFindQuote( pTok, (USHORT) (current.usStart-1), &chQuote); + current.usType = + (pTok->sTokenid >= sNumTags) ? PROTECTED_CHAR : TAGPROT_CHAR; + pushBytes(pStartStop, current); + current.reset(); - /********************************************************************/ - /* Count number of tokens in list to get start/stop list size */ - /********************************************************************/ - if ( usRC == NO_ERROR ) - { - pTok = pTokBuffer.data(); + /**************************************************************/ + /* Process quoted strings if any */ + /**************************************************************/ + while ( pTok->iLength && (usRC == NO_ERROR) ) + { // P016238: check whether quoted text contains protected parts! + pAttrTokBuffer.reserve(MAX_ATTR_TOKENS); + + if ( usRC == NO_ERROR) + { + usRC = TAProtectedPartsInQuotedText( pTok, + pTok->pDataStringW - pszSegment, + pStartStop, + pAttrTokBuffer, + pTagTable, + &usColPos, + chQuote, + lAttrSize / sizeof(TOKENENTRY), + ((iMode & CREATEPROTTABLE_MARKATTR) != 0) ); + } - while ( pTok->sTokenid != ENDOFLIST ) - { - /****************************************************************/ - /* Check for attributes and tags with translatable info */ - /****************************************************************/ - if ( ( (pTok->sTokenid >= sNumTags) && - (pAttribute[pTok->sTokenid - sNumTags].BitFlags.fTranslate) ) || - ( (pTok->sTokenid >= 0) && - (pTok->sTokenid < sNumTags) && - (pTag[pTok->sTokenid].BitFlags.fTranslate) ) ) + /************************************************************/ + /* Add entry for protected data up to next quote or up to */ + /* end of token */ + /************************************************************/ + if ( pTok->iLength && (usRC == NO_ERROR)) { - /************************************************************/ - /* for any quote in the token text leave room for one more */ - /* start/stop entry */ - /************************************************************/ - pszTemp = pTok->pDataStringW; - for ( usI = 0; usI < pTok->iLength; usI++, pszTemp++ ) - { - if ( (*pszTemp == QUOTE) || (*pszTemp == DBLQUOTE) ) - { - ulTableAlloc++; // leave room for one more entry - // P016238: leave room for 4 more entries assuming 2 inline tags - // per each translatable part of an attribute - ulTableAlloc = ulTableAlloc + 4; - } /* endif */ - } /* endfor */ + current.usStart = (USHORT)(pTok->pDataStringW - pszSegment); + current.usType = PROTECTED_CHAR; + + pTok->pDataStringW++; // assure quote is in pCurrent + pTok->iLength--; + current.usStop = TAFindQuote( pTok, (USHORT) (current.usStart), &chQuote); + pushBytes(pStartStop, current); + current.reset(); } /* endif */ - ulTableAlloc++; // leave room for token - pTok++; } /* endwhile */ - ulTableAlloc++; // leave room for table end delimiter - } /* endif */ - - // assure that ulTableAlloc * sizeof(STARTSTOP) is at least MIN_ALLOC - if (ulTableAlloc < 4 ) - { - ulTableAlloc = 4; } - - /********************************************************************/ - /* Allocate start/stop list */ - /********************************************************************/ - if ( usRC == NO_ERROR ) + else { - if ( !UtlAlloc( (PVOID *)&pStartStop, 0L, - (LONG) (ulTableAlloc * sizeof(STARTSTOP)), - NOMSG )) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - ulTableUsed = 0; - } /* endif */ - - TATagAddWSpace(pszSegment, pTokBuffer.data(), sNumTags); + current.usStart = (USHORT)(pTok->pDataStringW - pszSegment); - /********************************************************************/ - /* Fill start/stop table */ - /********************************************************************/ - if ( usRC == NO_ERROR ) - { - pCurrent = pStartStop; - pTok = pTokBuffer.data(); - while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) + if ( pTok->sTokenid == TEXT_TOKEN ) + { + current.usType = UNPROTECTED_CHAR; + } + else if (pTok->sTokenid >= sNumTags ) { - /****************************************************************/ - /* Check for attributes or tags with translatable info */ - /****************************************************************/ - if ( ( (pTok->sTokenid >= sNumTags) && - (pAttribute[pTok->sTokenid - sNumTags].BitFlags.fTranslate) ) || - ( (pTok->sTokenid >= 0) && - (pTok->sTokenid < sNumTags) && - (pTag[pTok->sTokenid].BitFlags.fTranslate) ) ) + // do not include translatable variables in protected text when working in CREATEPROTECTTABLE_NOTRANSLVAR mode + if ( (pTok->ClassId == CLS_TRANSLVAR) && (iMode & CREATEPROTTABLE_NOTRANSLVAR) ) { - /**************************************************************/ - /* Process data up to first quote or up to end of token */ - /* Complete protected start/stop entry in front of */ - /* quoted string */ - /**************************************************************/ - pCurrent->usStart = (USHORT)(pTok->pDataStringW - pszSegment); - pCurrent->usStop = TAFindQuote( pTok, (USHORT) (pCurrent->usStart-1), &chQuote); - pCurrent->usType = - (pTok->sTokenid >= sNumTags) ? PROTECTED_CHAR : TAGPROT_CHAR; - - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - &ulTableUsed, &ulTableAlloc); - - /**************************************************************/ - /* Process quoted strings if any */ - /**************************************************************/ - while ( pTok->iLength && (usRC == NO_ERROR) ) - { // P016238: check whether quoted text contains protected parts! - pAttrTokBuffer.reserve(MAX_ATTR_TOKENS); - - if ( usRC == NO_ERROR) - { - usRC = TAProtectedPartsInQuotedText( pTok, - pTok->pDataStringW - pszSegment, - &pStartStop, - &pCurrent, - pAttrTokBuffer, - pTagTable, - &ulTableAlloc, - &ulTableUsed, - &usColPos, - chQuote, - lAttrSize / sizeof(TOKENENTRY), - ((iMode & CREATEPROTTABLE_MARKATTR) != 0) ); - } - - /************************************************************/ - /* Add entry for protected data up to next quote or up to */ - /* end of token */ - /************************************************************/ - if ( pTok->iLength && (usRC == NO_ERROR)) - { - pCurrent->usStart = (USHORT)(pTok->pDataStringW - pszSegment); - pCurrent->usType = PROTECTED_CHAR; - - pTok->pDataStringW++; // assure quote is in pCurrent - pTok->iLength--; - pCurrent->usStop = TAFindQuote( pTok, (USHORT) (pCurrent->usStart), &chQuote); - - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - &ulTableUsed, &ulTableAlloc); - - } /* endif */ - } /* endwhile */ + current.usType = UNPROTECTED_CHAR; } else { - pCurrent->usStart = (USHORT)(pTok->pDataStringW - pszSegment); - - if ( pTok->sTokenid == TEXT_TOKEN ) + current.usType = PROTECTED_CHAR; + } /* endif */ + } + else + { + current.usType = TAGPROT_CHAR; + if (pTag[pTok->sTokenid].BitFlags.fTRNote ) + { + if (fTRNoteFound ) { - pCurrent->usType = UNPROTECTED_CHAR; + /****************************************************/ + /* this approach requires that a TRNote is within */ + /* one segment; also nesting of notes is not allowed*/ + /****************************************************/ + current.usType = TRNOTE_CHAR; } - else if (pTok->sTokenid >= sNumTags ) + else { - // do not include translatable variables in protected text when working in CREATEPROTECTTABLE_NOTRANSLVAR mode - if ( (pTok->ClassId == CLS_TRANSLVAR) && (iMode & CREATEPROTTABLE_NOTRANSLVAR) ) + if (memicmp((PBYTE)&pTagTable->chTrnote1TextW[0], + (PBYTE)(pTok->pDataStringW+pTok->iLength), + pTagTable->ulTRNote1Len) == 0 ) { - pCurrent->usType = UNPROTECTED_CHAR; + current.usType = TRNOTE_CHAR; + fTRNoteFound = TRUE; } - else - { - pCurrent->usType = PROTECTED_CHAR; - } /* endif */ - } - else - { - pCurrent->usType = TAGPROT_CHAR; - if (pTag[pTok->sTokenid].BitFlags.fTRNote ) + else if (memicmp((PBYTE)&pTagTable->chTrnote2TextW[0], + (PBYTE)(pTok->pDataStringW+pTok->iLength), + pTagTable->ulTRNote2Len) == 0 ) { - if (fTRNoteFound ) - { - /****************************************************/ - /* this approach requires that a TRNote is within */ - /* one segment; also nesting of notes is not allowed*/ - /****************************************************/ - pCurrent->usType = TRNOTE_CHAR; - } - else - { - if (memicmp((PBYTE)&pTagTable->chTrnote1TextW[0], - (PBYTE)(pTok->pDataStringW+pTok->iLength), - pTagTable->ulTRNote1Len) == 0 ) - { - pCurrent->usType = TRNOTE_CHAR; - fTRNoteFound = TRUE; - } - else if (memicmp((PBYTE)&pTagTable->chTrnote2TextW[0], - (PBYTE)(pTok->pDataStringW+pTok->iLength), - pTagTable->ulTRNote2Len) == 0 ) - { - pCurrent->usType = TRNOTE_CHAR; - fTRNoteFound = TRUE; - } /* endif */ - } /* endif */ + current.usType = TRNOTE_CHAR; + fTRNoteFound = TRUE; } /* endif */ } /* endif */ - pCurrent->usStop = pCurrent->usStart + pTok->iLength - 1; - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - &ulTableUsed, &ulTableAlloc); - } /* endif */ + } /* endif */ + current.usStop = current.usStart + pTok->iLength - 1; + pushBytes(pStartStop, current); + current.reset(); - pTok++; - } /* endwhile */ + } /* endif */ - if (usRC == NO_ERROR) - { - TASetStartStopType(pCurrent, 0,0,0 ); //terminate start/stoptabl. + pTok++; + } /* endwhile */ - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - &ulTableUsed, &ulTableAlloc); + if (usRC == NO_ERROR) + { + current.reset(); //terminate start/stoptabl. + pushBytes(pStartStop, (current)); - } - } /* endif */ + } + } /* endif */ + /********************************************************************/ + /* now loop true list again and join tags with their attributes */ + /* this is necessary to avoid splitting of tags and attributes */ + /* during the insert operation in the editor... */ + /********************************************************************/ + if ( usRC == NO_ERROR ) + { + PSTARTSTOP pCurrent = pAct = (PSTARTSTOP)pStartStop.data(); + pTok = pTokBuffer.data(); + pNextTok = pTok + 1; + if ( (pAct->usType == TAGPROT_CHAR) && TA_IS_TAG(pTok->sTokenid, sNumTags)) + { + pAct->usType = PROTECTED_CHAR; + fTag = TRUE; + } + else + { + fTag = FALSE; + } /* endif */ + if ( pCurrent->usType ) + { + pCurrent++; + if (pNextTok && pNextTok->pDataStringW && + (pCurrent->usStart >=(USHORT)(pNextTok->pDataStringW - pszSegment))) + { + pTok ++; + pNextTok++; + } - /********************************************************************/ - /* now loop true list again and join tags with their attributes */ - /* this is necessary to avoid splitting of tags and attributes */ - /* during the insert operation in the editor... */ - /********************************************************************/ - if ( usRC == NO_ERROR ) + while ( pCurrent->usType ) { - pAct = pCurrent = pStartStop; - pTok = pTokBuffer.data(); - pNextTok = pTok + 1; - if ( (pAct->usType == TAGPROT_CHAR) && TA_IS_TAG(pTok->sTokenid, sNumTags)) - { - pAct->usType = PROTECTED_CHAR; - fTag = TRUE; - } - else + switch ( pCurrent->usType ) { - fTag = FALSE; - } /* endif */ - if ( pCurrent->usType ) + case TAGPROT_CHAR: + if ( (pCurrent->usStart == (USHORT)(pTok->pDataStringW-pszSegment)) + && TA_IS_TAG(pTok->sTokenid, sNumTags)) + { + fTag = TRUE; + } + pAct++; + pAct->usStart = pCurrent->usStart; + pAct->usStop = pCurrent->usStop; + pAct->usType = PROTECTED_CHAR; + break; + case PROTECTED_CHAR: + if ( fTag && (pCurrent->usType == pAct->usType)) + { + /************************************************************/ + /* combine tag and attribute... */ + /************************************************************/ + pAct->usStop = pCurrent->usStop; + } + else + { + pAct++; + pAct->usStart = pCurrent->usStart; + pAct->usStop = pCurrent->usStop; + pAct->usType = pCurrent->usType; + } /* endif */ + break; + default : + pAct++; + pAct->usStart = pCurrent->usStart; + pAct->usStop = pCurrent->usStop; + pAct->usType = pCurrent->usType; + if ( ((pCurrent->usStart == (USHORT)(pTok->pDataStringW-pszSegment)) + && (pTok->sTokenid == TEXT_TOKEN)) || + (pCurrent->usType == TRNOTE_CHAR)) + { + fTag = FALSE; + } + // P016804: if text-token is from quoted text, do not reset!! + //fTag = FALSE; + break; + } /* endswitch */ + /******************************************************************/ + /* point to next one... */ + /******************************************************************/ + pCurrent++; + if (pNextTok && pNextTok->pDataStringW && + (pCurrent->usStart >=(USHORT)(pNextTok->pDataStringW - pszSegment))) { - pCurrent++; - if (pNextTok && pNextTok->pDataStringW && - (pCurrent->usStart >=(USHORT)(pNextTok->pDataStringW - pszSegment))) - { - pTok ++; - pNextTok++; - } + pTok ++; + pNextTok++; + } + } /* endwhile */ + /********************************************************************/ + /* add stopping mode.... */ + /********************************************************************/ + pAct++; + pAct->usStart = 0; + pAct->usStop = 0; + pAct->usType = 0; + } /* endif */ + + /********************************************************************/ + /* now loop thru list and combine the tokens of a translators note */ + /* into one start-stop token */ + /********************************************************************/ + if (fTRNoteFound ) + { + pAct = pCurrent =(PSTARTSTOP) pStartStop.data(); + if ( pAct->usType == TRNOTE_CHAR ) + { + fTRTag = TRUE; + } + else + { + fTRTag = FALSE; + } /* endif */ + if ( pCurrent->usType ) + { + pCurrent++; - while ( pCurrent->usType ) + while ( pCurrent->usType ) + { + switch ( pCurrent->usType ) { - switch ( pCurrent->usType ) - { - case TAGPROT_CHAR: - if ( (pCurrent->usStart == (USHORT)(pTok->pDataStringW-pszSegment)) - && TA_IS_TAG(pTok->sTokenid, sNumTags)) - { - fTag = TRUE; - } + case TRNOTE_CHAR : // end of TRNOTE of begin of next TRNOTE + if (fTRTag ) + { + pAct->usStop = pCurrent->usStop; + } + else + { pAct++; pAct->usStart = pCurrent->usStart; pAct->usStop = pCurrent->usStop; - pAct->usType = PROTECTED_CHAR; - break; - case PROTECTED_CHAR: - if ( fTag && (pCurrent->usType == pAct->usType)) - { - /************************************************************/ - /* combine tag and attribute... */ - /************************************************************/ - pAct->usStop = pCurrent->usStop; - } - else - { - pAct++; - pAct->usStart = pCurrent->usStart; - pAct->usStop = pCurrent->usStop; - pAct->usType = pCurrent->usType; - } /* endif */ - break; - default : + pAct->usType = pCurrent->usType; + } /* endif */ + fTRTag = !fTRTag; + break; + default: + if ( fTRTag ) + { + /************************************************************/ + /* combine tag and attribute... */ + /************************************************************/ + pAct->usStop = pCurrent->usStop; + } + else + { pAct++; pAct->usStart = pCurrent->usStart; pAct->usStop = pCurrent->usStop; pAct->usType = pCurrent->usType; - if ( ((pCurrent->usStart == (USHORT)(pTok->pDataStringW-pszSegment)) - && (pTok->sTokenid == TEXT_TOKEN)) || - (pCurrent->usType == TRNOTE_CHAR)) - { - fTag = FALSE; - } - // P016804: if text-token is from quoted text, do not reset!! - //fTag = FALSE; - break; - } /* endswitch */ - /******************************************************************/ - /* point to next one... */ - /******************************************************************/ - pCurrent++; - if (pNextTok && pNextTok->pDataStringW && - (pCurrent->usStart >=(USHORT)(pNextTok->pDataStringW - pszSegment))) - { - pTok ++; - pNextTok++; - } - } /* endwhile */ - /********************************************************************/ - /* add stopping mode.... */ - /********************************************************************/ - pAct++; - pAct->usStart = 0; - pAct->usStop = 0; - pAct->usType = 0; - } /* endif */ - + } /* endif */ + break; + } /* endswitch */ + /******************************************************************/ + /* point to next one... */ + /******************************************************************/ + pCurrent++; + } /* endwhile */ /********************************************************************/ - /* now loop thru list and combine the tokens of a translators note */ - /* into one start-stop token */ + /* add stopping mode.... */ /********************************************************************/ - if (fTRNoteFound ) - { - pAct = pCurrent = pStartStop; - if ( pAct->usType == TRNOTE_CHAR ) - { - fTRTag = TRUE; - } - else - { - fTRTag = FALSE; - } /* endif */ - if ( pCurrent->usType ) - { - pCurrent++; - - while ( pCurrent->usType ) - { - switch ( pCurrent->usType ) - { - case TRNOTE_CHAR : // end of TRNOTE of begin of next TRNOTE - if (fTRTag ) - { - pAct->usStop = pCurrent->usStop; - } - else - { - pAct++; - pAct->usStart = pCurrent->usStart; - pAct->usStop = pCurrent->usStop; - pAct->usType = pCurrent->usType; - } /* endif */ - fTRTag = !fTRTag; - break; - default: - if ( fTRTag ) - { - /************************************************************/ - /* combine tag and attribute... */ - /************************************************************/ - pAct->usStop = pCurrent->usStop; - } - else - { - pAct++; - pAct->usStart = pCurrent->usStart; - pAct->usStop = pCurrent->usStop; - pAct->usType = pCurrent->usType; - } /* endif */ - break; - } /* endswitch */ - /******************************************************************/ - /* point to next one... */ - /******************************************************************/ - pCurrent++; - } /* endwhile */ - /********************************************************************/ - /* add stopping mode.... */ - /********************************************************************/ - pAct++; - pAct->usStart = 0; - pAct->usStop = 0; - pAct->usType = 0; - } /* endif */ - } /* endif */ + pAct++; + pAct->usStart = 0; + pAct->usStop = 0; + pAct->usType = 0; } /* endif */ } /* endif */ } /* endif */ @@ -2735,37 +2599,20 @@ USHORT TACreateProtectTableWEx /********************************************************************/ if ( usRC == NO_ERROR ) { - *ppStartStop = pStartStop; if (IsDBCS_CP(ulCP)) { - usRC = TASoSiProtectTable( pszSegment, ppStartStop); + usRC = TASoSiProtectTable( pszSegment, pStartStop); } } else { - if ( pStartStop ) - { - UtlAlloc( (PVOID *)&pStartStop, 0L, 0L, NOMSG ); - } /* endif */ - *ppStartStop = NULL; + pStartStop.clear(); } /* endif */ return( usRC ); } -VOID TASetStartStopType -( - PSTARTSTOP pCurrent, // ptr to caller's current start/stop entry - USHORT usStart, - USHORT usStop, - USHORT usType -) -{ - pCurrent->usStart = usStart; - pCurrent->usStop = usStop; - pCurrent->usType = usType; - return; -} + USHORT TAFindQuote ( @@ -2796,23 +2643,29 @@ USHORT TAFindQuote USHORT TAGotoNextStartStopEntry ( - PSTARTSTOP * ppStartStop, - PSTARTSTOP * ppCurrent, - PULONG pulTableUsed, - PULONG pulTableAlloc + BytesRef pStartStop, + STARTSTOP& current, + PULONG pulTableUsed ) { - PSTARTSTOP pCurrent = *ppCurrent; - ULONG ulTableUsed = * pulTableUsed; - ULONG ulTableAlloc = *pulTableAlloc; - USHORT usRC = NO_ERROR; // function return code + //PSTARTSTOP pCurrent = *ppCurrent; + //ULONG ulTableUsed = * pulTableUsed; + + //pCurrent ++; + //ulTableUsed ++; + //if ( pStartStop.size()/sizeof(STARTSTOP) <= ulTableUsed + 1){ + // pStartStop.resize(sizeof(STARTSTOP)* (ulTableUsed + 10)); + // *ppCurrent = (PSTARTSTOP)pStartStop.data() + ulTableUsed; + //} + // set return values + //*ppCurrent = pCurrent; + //*pulTableUsed = ulTableUsed; - pCurrent ++; - ulTableUsed ++; // for tests only is pCurrent after add as above? - { PSTARTSTOP pStartStop = *ppStartStop; - pCurrent = pStartStop + ulTableUsed; - } + //{ PSTARTSTOP pStartStop = *ppStartStop; + // pCurrent = pStartStop + ulTableUsed; + //} + #ifdef TO_BE_REMOVED if ( ulTableAlloc <= ulTableUsed + 1) { LONG lOldLen; // old length @@ -2837,13 +2690,9 @@ USHORT TAGotoNextStartStopEntry *pulTableAlloc = ulTableAlloc; } } - if ( usRC == NO_ERROR) - { - // set return values - *ppCurrent = pCurrent; - *pulTableUsed = ulTableUsed; - } - return ( usRC ); + #endif + + return ( 0 ); } @@ -2851,101 +2700,93 @@ USHORT TAProtectedPartsInQuotedText ( PTOKENENTRY pTok, int iAttrStartOffset, - PSTARTSTOP * ppStartStop, - PSTARTSTOP * ppCurrent, + BytesRef pStartStop, std::vector& pAttrTokBuffer, PLOADEDTABLE pTagTable, - PULONG pulTableAlloc, - PULONG pulTableUsed, PUSHORT pusColPos, CHAR_W chQuote, LONG lAttrSize, BOOL fSpecialMode ) { - USHORT usRC = NO_ERROR; - CHAR_W chTempBuf[MAX_SEGMENT_SIZE]; - PTOKENENTRY pAttrTok; - PSZ_W pAttrStart; - USHORT usAttrLength = 0; - PSZ_W pRest; // ptr to not tokenized data - PSZ_W pTemp; - PSTARTSTOP pCurrent = *ppCurrent; - PSTARTSTOP pStartStop = *ppStartStop; - - - memset (&chTempBuf[0], 0, sizeof(CHAR_W) * (MAX_SEGMENT_SIZE)); - - UTF16strcat(chTempBuf, pTok->pDataStringW); - pAttrStart = &chTempBuf[0]; - pTemp = &chTempBuf[0]; - while ( pTemp && (*pTemp != chQuote) ) - { - pTemp++; - usAttrLength ++; - } /* endwhile */ - - if (*pTemp == chQuote ) - { - *pTemp = EOS; //investigate only text inside quotes! - } + USHORT usRC = NO_ERROR; + CHAR_W chTempBuf[MAX_SEGMENT_SIZE]; + PTOKENENTRY pAttrTok; + PSZ_W pAttrStart; + USHORT usAttrLength = 0; + PSZ_W pRest; // ptr to not tokenized data + PSZ_W pTemp; + STARTSTOP current; - if ( usRC == NO_ERROR ) - { - TATagTokenizeW( chTempBuf, - pTagTable, - TRUE, - &pRest, - pusColPos, - pAttrTokBuffer ); - if(pRest == nullptr){ - LOG_AND_SET_RC(usRC, T5INFO, NO_ERROR); - }else{ - LOG_AND_SET_RC(usRC, T5INFO, EQFRS_AREA_TOO_SMALL); - } - } /* endif */ + memset (&chTempBuf[0], 0, sizeof(CHAR_W) * (MAX_SEGMENT_SIZE)); - if ( usRC == NO_ERROR ) - { - pAttrTok = pAttrTokBuffer.data(); - while ( (pAttrTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) - { - pCurrent->usStart = (USHORT)(pAttrTok->pDataStringW - pAttrStart - + iAttrStartOffset); - pCurrent->usType = (pAttrTok->sTokenid == TEXT_TOKEN) ? - UNPROTECTED_CHAR : PROTECTED_CHAR; - if ( fSpecialMode ) - { - pCurrent->usType |= 0x8000; - } /* endif */ - pCurrent->usStop = pCurrent->usStart + pAttrTok->iLength - 1; + UTF16strcat(chTempBuf, pTok->pDataStringW); + pAttrStart = &chTempBuf[0]; + pTemp = &chTempBuf[0]; + while ( pTemp && (*pTemp != chQuote) ) + { + pTemp++; + usAttrLength ++; + } /* endwhile */ - usRC = TAGotoNextStartStopEntry( &pStartStop, &pCurrent, - pulTableUsed, pulTableAlloc); + if (*pTemp == chQuote ) + { + *pTemp = EOS; //investigate only text inside quotes! + } - pAttrTok++; - } /* endwhile */ - } /* endif */ + if ( usRC == NO_ERROR ) + { + TATagTokenizeW( chTempBuf, + pTagTable, + TRUE, + &pRest, + pusColPos, + pAttrTokBuffer ); + if(pRest == nullptr){ + LOG_AND_SET_RC(usRC, T5INFO, NO_ERROR); + }else{ + LOG_AND_SET_RC(usRC, T5INFO, EQFRS_AREA_TOO_SMALL); + } + } /* endif */ - /**********************************************************/ - /* Add data up to first quote */ - /**********************************************************/ - while ( pTok->iLength && (pTok->pDataStringW[0] != chQuote) ) + if ( usRC == NO_ERROR ) + { + pAttrTok = pAttrTokBuffer.data(); + while ( (pAttrTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR)) { - pTok->pDataStringW++; - pTok->iLength--; - usAttrLength --; + current.usStart = (USHORT)(pAttrTok->pDataStringW - pAttrStart + + iAttrStartOffset); + current.usType = (pAttrTok->sTokenid == TEXT_TOKEN) ? + UNPROTECTED_CHAR : PROTECTED_CHAR; + if ( fSpecialMode ) + { + current.usType |= 0x8000; + } /* endif */ + current.usStop = current.usStart + pAttrTok->iLength - 1; + pushBytes(pStartStop, current); + current.reset(); + + pAttrTok++; } /* endwhile */ - // CHECK: does this loop run as far as pTemp in chTempBuf? - if (usAttrLength != 0) - { - // this should not happen!! - usAttrLength = 0; - } + } /* endif */ + + /**********************************************************/ + /* Add data up to first quote */ + /**********************************************************/ + while ( pTok->iLength && (pTok->pDataStringW[0] != chQuote) ) + { + pTok->pDataStringW++; + pTok->iLength--; + usAttrLength --; + } /* endwhile */ + // CHECK: does this loop run as far as pTemp in chTempBuf? + if (usAttrLength != 0) + { + // this should not happen!! + usAttrLength = 0; + } - *ppCurrent = pCurrent; - *ppStartStop = pStartStop; - return (usRC); + return (usRC); } @@ -2995,27 +2836,27 @@ bool iswspace(wchar_t ch){ //*/ // P016804: - static void - TATagAddWSpace - ( - PSZ_W pszSegment, - PTOKENENTRY pTokBuffer, - SHORT sNumTags - ) - { - PTOKENENTRY pTok; - USHORT usRC = NO_ERROR; - PTOKENENTRY pNextTok; - USHORT usStart = 0; - USHORT usNextStart = 0; - PSZ_W pTmp = NULL; - USHORT usI = 0; - USHORT usNewLen = 0; - - pTok = pTokBuffer; - pNextTok = pTok + 1; - while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR) && (pNextTok->sTokenid != ENDOFLIST)) - { +static void +TATagAddWSpace +( + PSZ_W pszSegment, + PTOKENENTRY pTokBuffer, + SHORT sNumTags +) +{ + PTOKENENTRY pTok; + USHORT usRC = NO_ERROR; + PTOKENENTRY pNextTok; + USHORT usStart = 0; + USHORT usNextStart = 0; + PSZ_W pTmp = NULL; + USHORT usI = 0; + USHORT usNewLen = 0; + + pTok = pTokBuffer; + pNextTok = pTok + 1; + while ( (pTok->sTokenid != ENDOFLIST) && (usRC == NO_ERROR) && (pNextTok->sTokenid != ENDOFLIST)) + { if (TA_IS_TAG(pTok->sTokenid, sNumTags) || TA_IS_ATTRIBUTE(pTok->sTokenid, sNumTags)) { if (TA_IS_ATTRIBUTE(pNextTok->sTokenid, sNumTags)) @@ -3039,7 +2880,7 @@ bool iswspace(wchar_t ch){ } /* endif */ pTok ++; pNextTok++; - } /* endwhile */ - return; - } + } /* endwhile */ + return; +} diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index 7299aab0..3b3d07a1 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -358,27 +358,27 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem //insert initialized record to tm data file ulKey = AUTHOR_KEY; tmxTableToBuffer(&NewMem->Authors,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey,buffer); } if ( _rc_ == NO_ERROR ) { ulKey = FILE_KEY; tmxTableToBuffer(&NewMem->FileNames,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { ulKey = TAGTABLE_KEY; tmxTableToBuffer(&NewMem->TagTables,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { ulKey = LANG_KEY; tmxTableToBuffer(&NewMem->Languages,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) @@ -389,7 +389,7 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem NewMem->bCompact.resize(size); ulKey = COMPACT_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, + _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey, NewMem->bCompact); } /* endif */ @@ -400,7 +400,7 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem ulKey = LONGNAME_KEY; NewMem->LongNames.pszBuffer.resize(sizeof(TMX_RECORD));//to have some size // write long document name buffer area to the database - _rc_ = NewMem->TmBtree.EQFNTMInsert(&ulKey, + _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey, NewMem->LongNames.pszBuffer ); } /* endif */ diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index fea4f55c..be7f1d18 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -812,31 +812,19 @@ VOID BTREE::QDAMTerseInit //|Function flow: return (SHORT) (*pulKey1 - *pulKey2) | //+----------------------------------------------------------------------------+ -SHORT NTMKeyCompare -( - PVOID pBTIda, // pointer to tree structure - PVOID pulKey1, // pointer to first key - PVOID pulKey2 // pointer to second key +SHORT compKeys( ULONG ulKey1, // pointer to first key + ULONG ulKey2 // pointer to second key ) { - TMWCHAR ulKey1 = *((PWCHAR)pulKey1); - TMWCHAR ulKey2 = *((PWCHAR)pulKey2); - SHORT sRc; - pBTIda; // avoid compiler warnings - if (ulKey1 > ulKey2 ) { - sRc = 1; + return 1; } else if (ulKey1 < ulKey2 ) { - sRc = -1; + return -1; } - else - { - sRc = 0; - } /* endif */ - return sRc; + return 0; } @@ -1283,24 +1271,18 @@ SHORT BTREE::QDAMReadRecord_V3 /* set #calls of QDAMReadRecord = 0 */ /* else increment #calls of QDAMReadRecord */ /****************************************************************************/ + #ifdef TO_BE_REMOVED if ( !sRc ) { if ( ulReadRecCalls >= MAX_READREC_CALLS ) { //for ( i=0; !sRc && (i < LookupTable.size()); i++ ) - for(auto entry: LookupTable) + for(auto& entry: LookupTable) { if ( !(entry.second->fLocked) && (entry.second->usRecordNumber != usNumber))//&& (entry.second->usNumber!=usNumber)) { - - sRc = QDAMWRecordToDisk_V3(entry.second); - if ( !sRc ) - { - // usNumberOfAllocatedBuffers--; - } /* endif */ - } /* endif */ - - + sRc = QDAMWRecordToDisk_V3(entry.second); + } /* endif */ } /* endfor */ ulReadRecCalls = 0L; } @@ -1309,6 +1291,7 @@ SHORT BTREE::QDAMReadRecord_V3 ulReadRecCalls++; } /* endif */ } /* endif */ + #endif //TO_BE_REMOVED return ( sRc ); @@ -1466,15 +1449,11 @@ SHORT BTREE::QDAMDictFlushLocal() } else { - BOOL fRecordWritten = FALSE; for(auto& entry: LookupTable){ - //if(!entry->pBuffer.empty() /*&& (pLEntry->pBuffer)->fNeedToWrite*/){ sRc = QDAMWRecordToDisk_V3( entry.second); if(sRc){ return sRc; } - fRecordWritten = TRUE; - //} } } @@ -1748,39 +1727,36 @@ SHORT BTREE::QDAMAllocKeyRecords USHORT usNum ) { - SHORT sRc = 0; - PBTREEHEADER pHeader; + SHORT sRc = 0; + PBTREEHEADER pHeader; - //if ( pBT->bRecSizeVersion == BTREE_V3 ) - { - std::shared_ptr pRecord; - while ( usNum-- && !sRc ) + std::shared_ptr pRecord; + while ( usNum-- && !sRc ) + { + usNextFreeRecord++; + sRc = QDAMReadRecord_V3( usNextFreeRecord, pRecord, TRUE ); + + if ( !sRc ) { - usNextFreeRecord++; - sRc = QDAMReadRecord_V3( usNextFreeRecord, pRecord, TRUE ); + // add free record to the free list + NEXT( pRecord ) = usFreeKeyBuffer; + PREV( pRecord ) = 0; + usFreeKeyBuffer = RECORDNUM( pRecord ); + + // init this record + pHeader = &pRecord->contents.header; + pHeader->usNum = usFreeKeyBuffer; + pHeader->usOccupied = 0; + pHeader->usFilled = sizeof(BTREEHEADER ); + pHeader->usLastFilled = BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER ); - if ( !sRc ) - { - // add free record to the free list - NEXT( pRecord ) = usFreeKeyBuffer; - PREV( pRecord ) = 0; - usFreeKeyBuffer = RECORDNUM( pRecord ); - - // init this record - pHeader = &pRecord->contents.header; - pHeader->usNum = usFreeKeyBuffer; - pHeader->usOccupied = 0; - pHeader->usFilled = sizeof(BTREEHEADER ); - pHeader->usLastFilled = BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER ); - - /*************************************************************/ - /* force write */ - /*************************************************************/ - - sRc = QDAMWRecordToDisk_V3( pRecord); - } /* endif */ - } /* endwhile */ - } + /*************************************************************/ + /* force write */ + /*************************************************************/ + + sRc = QDAMWRecordToDisk_V3( pRecord); + } /* endif */ + } /* endwhile */ return sRc; } @@ -1923,7 +1899,6 @@ SHORT BTREE::QDAMDictCreateLocal sCurrentIndex = 0; usFirstNode=usFirstLeaf = 1; usCurrentRecord = 0; - //compare = NTMKeyCompare; usNextFreeRecord = 1; usFreeKeyBuffer = 0; usFreeDataBuffer = 0; @@ -1943,10 +1918,6 @@ SHORT BTREE::QDAMDictCreateLocal /* header... */ /******************************************************************/ chCollate = NtmVitalInfo; - /******************************************************************/ - /* new key compare routine .... */ - /******************************************************************/ - //compare = NTMKeyCompare; sRc = initLookupTable(); usNumberOfAllocatedBuffers = 0; @@ -2082,7 +2053,7 @@ SHORT BTREE::QDAMDictCreateLocal BOOL BTREE::QDAMDictLockStatus ( - PWCHAR pKey + ULONG ulKey ) { return( false ); @@ -2171,11 +2142,10 @@ VOID BTREE::QDAMDictUpdStatus() { } /* end of function QDAMDictUpdStatus */ -PWCHAR QDAMGetszKey_V3 +ULONG QDAMGetszKey_V3 ( std::shared_ptr& pRecord, // active record - USHORT i, // get data term - USHORT usVersion // version of database + USHORT i // get data term ) { PBYTE pData = nullptr, @@ -2187,6 +2157,7 @@ PWCHAR QDAMGetszKey_V3 pusOffset += i; // point to key int step = sizeof(USHORT ) + sizeof(RECPARAM); // get pointer to data + USHORT res = -1; if ( (PBYTE)pusOffset > pEndOfRec ) { @@ -2205,10 +2176,13 @@ PWCHAR QDAMGetszKey_V3 "; pEndOfRec = " << (long int)pEndOfRec; pData = nullptr; //ERREVENT2( QDAMGETSZKEY_LOC, INTFUNCFAILED_EVENT, 2, DB_GROUP, "" ); + }else{ + res = (USHORT)*pData; } /* endif */ } /* endif */ - return ( (PWCHAR)pData ); + + return ( res ); } RECPARAM QDAMGetrecData_V3 @@ -2296,7 +2270,7 @@ RECPARAM QDAMGetrecData_V3 //------------------------------------------------------------------------------ SHORT BTREE::QDAMFindRecord_V3 ( - PWCHAR pKey, + ULONG ulKey, std::shared_ptr& pRecord ) { @@ -2305,7 +2279,7 @@ SHORT BTREE::QDAMFindRecord_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - PWCHAR pKey2; // pointer to search key + ULONG ulKey2; // pointer to search key SHORT sRc; // return code memset(&recData, 0, sizeof(recData)); @@ -2321,14 +2295,15 @@ SHORT BTREE::QDAMFindRecord_V3 while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( pRecord, sMid, usVersion ); - if ( pKey2 == nullptr ) + ulKey2 = QDAMGetszKey_V3( pRecord, sMid ); + //if ( ulKey2 == nullptr ) + if(ulKey2 == -1) { sRc = BTREE_CORRUPTED; } else { - sResult = NTMKeyCompare(this, pKey, pKey2); + sResult = compKeys(ulKey, ulKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -2413,7 +2388,7 @@ SHORT BTREE::QDAMFindRecord_V3 BOOL BTREE::QDAMTerseData ( - std::vector& pData, // pointer to data + BytesRef pData, // pointer to data PULONG pulLen // length of the string ) { @@ -2691,7 +2666,7 @@ SHORT BTREE::QDAMNewRecord_V3 //------------------------------------------------------------------------------ SHORT BTREE::QDAMAddToBuffer_V3 ( - std::vector& Data, // pointer to data + BytesRef Data, // pointer to data PRECPARAM precReturn // pointer to return code ) { @@ -3011,58 +2986,10 @@ SHORT BTREE::QDAMAddToBuffer_V3 return sRc; } - -SHORT QDAMCaseCompare -( - PVOID pvBT, // pointer to tree structure - PVOID pKey1, // pointer to first key - PVOID pKey2, // pointer to second key - BOOL fIgnorePunctuation // ignore punctuation flag -) -{ - SHORT sDiff; - BYTE c; - PBTREE pBTIda = (PBTREE)pvBT; // pointer to tree structure - PBTREE pBT = pBTIda; - PBYTE pbKey1 = (PBYTE) pKey1; - PBYTE pbKey2 = (PBYTE) pKey2; - PBYTE pMap = pBT->chCaseMap; // pointer to mapping table - - while ( (c = *pbKey1) != 0 ) - { - /******************************************************************/ - /* ignore the following characters during matching: '/', '-', ' ' */ - /******************************************************************/ - if ( fIgnorePunctuation ) - { - while ( (c = *pbKey2) == ' ' || fIsPunctuation[c] ) - { - pbKey2++; - } /* endwhile */ - while ( (c = *pbKey1) == ' ' || fIsPunctuation[c] ) - { - pbKey1++; - } /* endwhile */ - } /* endif */ - - sDiff = *(pMap+c) - *(pMap + *pbKey2); - if ( !sDiff && *pbKey1 && *pbKey2 ) - { - pbKey1++; - pbKey2++; - } - else - { - return ( sDiff ); - } /* endif */ - } /* endwhile */ - return ( *(pMap + c) - *(pMap + *pbKey2)); -} - SHORT BTREE::QDAMLocateKey_V3 ( std::shared_ptr& pRecord, // record to be dealt with - PWCHAR pKey, // key to be searched + ULONG& ulKey, // key to be searched PSHORT psKeyPos, // located key SEARCHTYPE searchType, // search type PSHORT psNearPos // near position @@ -3073,7 +3000,7 @@ SHORT BTREE::QDAMLocateKey_V3 SHORT sResult; SHORT sMid = 0; // SHORT sRc = 0; // return value - PWCHAR pKey2; // pointer to key string + ULONG ulKey2; // pointer to key string BOOL fFound = FALSE; *psKeyPos = -1; // key not found @@ -3086,11 +3013,11 @@ SHORT BTREE::QDAMLocateKey_V3 while ( !fFound && sLow <= sHigh ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( pRecord, sMid, usVersion ); + ulKey2 = QDAMGetszKey_V3( pRecord, sMid ); - if ( pKey2 ) + if ( ulKey2 != -1 ) { - sResult = NTMKeyCompare(this, pKey, pKey2); + sResult = compKeys(ulKey,ulKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -3109,7 +3036,7 @@ SHORT BTREE::QDAMLocateKey_V3 /* matching substring */ /*********************************************************/ { - if (*((PULONG)pKey) == *((PULONG)pKey2)) + if (ulKey == ulKey2) { *psKeyPos = sMid; } @@ -3118,12 +3045,12 @@ SHORT BTREE::QDAMLocateKey_V3 // try with previous if ( sMid > sLow ) { - pKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid-1), usVersion ); - if ( pKey2 == NULL ) + ulKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid-1) ); + if ( ulKey2 == -1 ) { sRc = BTREE_CORRUPTED; } - else if ( *((PULONG)pKey) == *((PULONG)pKey2) ) + else if ( ulKey == ulKey2) { *psKeyPos = sMid-1 ; } /* endif */ @@ -3131,12 +3058,12 @@ SHORT BTREE::QDAMLocateKey_V3 // still not found if ( !sRc && *psKeyPos == -1 && sMid < sHigh ) { - pKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid+1), usVersion ); - if ( pKey2 == NULL ) + ulKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid+1) ); + if ( ulKey2 == -1 ) { sRc = BTREE_CORRUPTED; } - else if ( *((PULONG)pKey) == *((PULONG)pKey2) ) + else if (ulKey == ulKey2) { *psKeyPos = sMid+1 ; } /* endif */ @@ -3317,9 +3244,10 @@ SHORT QDAMFindParent_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - PCHAR_W pKey2; // pointer to search key - PCHAR_W pKey ; // pointer to search key - CHAR_W chKey[HEADTERM_SIZE]; // key to be found + ULONG ulKey2 = -1; // pointer to search key + ULONG ulKey = -1 ; // pointer to search key + //CHAR_W chKey[HEADTERM_SIZE]; // key to be found + ULONG ulChKey = -1; SHORT sRc; // return code std::shared_ptr pTempRec; // temp. record buffer USHORT usRecNum; // passed record number @@ -3329,14 +3257,14 @@ SHORT QDAMFindParent_V3 *pusParent = pBT->usFirstNode; // set parent to first node usRecNum = RECORDNUM( pRecord ); - pKey = QDAMGetszKey_V3( pRecord, 0, pBT->usVersion ); - if ( pKey == NULL ) + ulKey = QDAMGetszKey_V3( pRecord, 0 ); + if ( ulKey != -1 ) { sRc = BTREE_CORRUPTED; } else { - memcpy( chKey, pKey, sizeof(ULONG)); + ulChKey = ulKey; } /* endif */ sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, pTempRec, FALSE ); @@ -3353,15 +3281,14 @@ SHORT QDAMFindParent_V3 while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( pTempRec, sMid, pBT->usVersion ); - if ( pKey2 == NULL ) + ulKey2 = QDAMGetszKey_V3( pTempRec, sMid ); + if ( ulKey2 == -1 ) { sRc = BTREE_CORRUPTED; } else { - //sResult = (*pBT->compare)(pBTIda, chKey, pKey2); - sResult = NTMKeyCompare(pBTIda, chKey, pKey2); + sResult = compKeys(ulChKey,ulKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -3625,7 +3552,7 @@ SHORT QDAMSplitNode_V3 ( PBTREE pBTIda, // pointer to generic structure std::shared_ptr& record, // pointer to pointer to node - PWCHAR pKey // new key + ULONG& ulKey // new key ) { SHORT i,j; @@ -3634,7 +3561,7 @@ SHORT QDAMSplitNode_V3 USHORT usParent; // number of parent RECPARAM recKey; // position/offset for key RECPARAM recData; // position/offset for data - PWCHAR pParentKey; // new key to be inserted + ULONG ulParentKey = -1; // new key to be inserted PUSHORT pusOffset; // pointer to offset table BOOL fCompare; // indicator where to insert new key USHORT usFreeKeys = 0; // number of free keys required @@ -3667,14 +3594,15 @@ SHORT QDAMSplitNode_V3 recData.usNum = RECORDNUM(record); recData.usOffset = 0; - //pParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); - pParentKey = QDAMGetszKey_V3( record, 0, pBT->usVersion ); + //ulParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); + ulParentKey = QDAMGetszKey_V3( record, 0 ); - if ( pParentKey ) + //if ( ulParentKey ) + if(ulParentKey != -1) { // store key temporarily, because record with key data is // not locked. - sRc = pBTIda->QDAMInsertKey_V3( newRecord, pParentKey, recKey, recData); + sRc = pBTIda->QDAMInsertKey_V3( newRecord, ulParentKey, recKey, recData); // might be freed during insert BTREELOCKRECORD( pRecTemp ); } @@ -3723,19 +3651,19 @@ SHORT QDAMSplitNode_V3 TYPE(newRecord) = (CHAR)(TYPE(record) & ~ROOT_NODE); // don't copy Root bit // Decide where to split the record - //pParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); - pParentKey = QDAMGetszKey_V3( record, (SHORT)(OCCUPIED(record)/2), pBT->usVersion ); + //ulParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); + ulParentKey = QDAMGetszKey_V3( record, (SHORT)(OCCUPIED(record)/2) ); - if ( pParentKey ) + if ( ulParentKey != -1 ) { - fCompare = NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0 ; + fCompare = (compKeys(ulParentKey, ulKey) <= 0 ) ; /***********************************************************/ /* check in which part we will lay */ /***********************************************************/ if ( fCompare ) { - pParentKey = QDAMGetszKey_V3(record, (SHORT)(OCCUPIED(record)-MINFREEKEYS), pBT->usVersion); - fCompare = (NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0 ) ; + ulParentKey = QDAMGetszKey_V3(record, (SHORT)(OCCUPIED(record)-MINFREEKEYS)); + fCompare = (compKeys(ulParentKey,ulKey) <= 0 ) ; if ( fCompare ) { usFreeKeys = MINFREEKEYS; @@ -3747,8 +3675,8 @@ SHORT QDAMSplitNode_V3 } else { - pParentKey = QDAMGetszKey_V3( record,MINFREEKEYS, pBT->usVersion); - fCompare = (NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0 ) ; + ulParentKey = QDAMGetszKey_V3( record,MINFREEKEYS); + fCompare = (compKeys(ulParentKey,ulKey) <= 0) ; if ( fCompare ) { usFreeKeys = OCCUPIED( record )/2; @@ -3799,10 +3727,10 @@ SHORT QDAMSplitNode_V3 { recData.usNum = RECORDNUM( newRecord ); recData.usOffset = 0; - pParentKey = QDAMGetszKey_V3( newRecord,0, pBT->usVersion ); - if ( pParentKey ) + ulParentKey = QDAMGetszKey_V3( newRecord,0 ); + if ( ulParentKey != -1) { - sRc = pBTIda->QDAMInsertKey_V3( parent, pParentKey, recKey, recData); + sRc = pBTIda->QDAMInsertKey_V3( parent, ulParentKey, recKey, recData); } else { @@ -3817,10 +3745,10 @@ SHORT QDAMSplitNode_V3 { // newRecord->ulCheckSum = QDAMComputeCheckSum( newRecord ); // (*record)->ulCheckSum = QDAMComputeCheckSum( *record ); - pParentKey = QDAMGetszKey_V3( newRecord, 0, pBT->usVersion ); - if ( pParentKey ) + ulParentKey = QDAMGetszKey_V3( newRecord, 0 ); + if ( ulParentKey != -1) { - if ( NTMKeyCompare(pBTIda, pParentKey, pKey) <= 0) + if ( compKeys(ulParentKey,ulKey) <= 0) { sRc = pBTIda->QDAMWriteRecord_V3( record); record = newRecord; // add to new record @@ -3921,7 +3849,7 @@ SHORT QDAMSplitNode_V3 SHORT QDAMUnTerseData ( PBTREE pBTIda, // - std::vector& pData, // pointer to data + BytesRef pData, // pointer to data ULONG ulDataLen // data length (uncompressed) ) { @@ -4074,7 +4002,7 @@ SHORT QDAMGetszData_V3 ( PBTREE pBTIda, RECPARAM recDataParam, - std::vector& pData, + BytesRef pData, CHAR chType // type of record key/data ) { @@ -4301,7 +4229,7 @@ SHORT QDAMGetszData_V3 SHORT QDAMFindChild_V3 ( PBTREE pBTIda, - PCHAR_W pKey, + ULONG& ulKey, USHORT usNode, std::shared_ptr& pRecord ) @@ -4311,7 +4239,7 @@ SHORT QDAMFindChild_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - PCHAR_W pKey2; // pointer to search key + ULONG ulKey2; // pointer to search key SHORT sRc; // return code PBTREE pBT = pBTIda; @@ -4328,15 +4256,14 @@ SHORT QDAMFindChild_V3 while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - pKey2 = QDAMGetszKey_V3( pRecord, sMid, pBT->usVersion ); - if ( pKey2 == NULL ) + ulKey2 = QDAMGetszKey_V3( pRecord, sMid ); + if ( ulKey2 == -1 ) { sRc = BTREE_CORRUPTED; } else { - //sResult = (*pBT->compare)(pBTIda, pKey, pKey2); - sResult = NTMKeyCompare(pBTIda, pKey, pKey2); + sResult = compKeys(ulKey, ulKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -4377,8 +4304,8 @@ SHORT QDAMChangeKey_V3 ( PBTREE pBTIda, // ptr to tree structure USHORT usNode, // start node - PWCHAR pOldKey, // find old key - PWCHAR pNewKey // find new key + ULONG& ulOldKey, // find old key + ULONG& ulNewKey // find new key ) { std::shared_ptr pRecord; // buffer for record @@ -4397,23 +4324,23 @@ SHORT QDAMChangeKey_V3 if ( !sRc ) { /* Locate the Leaf node that contains the appropriate key */ - sRc = QDAMFindChild_V3( pBTIda, pNewKey, usNode, pNewRecord ); + sRc = QDAMFindChild_V3( pBTIda, ulNewKey, usNode, pNewRecord ); } /* endif */ if ( !sRc ) { // get the key description and insert the new key recKey.usNum = pNewRecord->contents.header.usNum; - sRc = pBTIda->QDAMLocateKey_V3( pNewRecord, pNewKey, &i, FEXACT, &sNearKey); + sRc = pBTIda->QDAMLocateKey_V3( pNewRecord, ulNewKey, &i, FEXACT, &sNearKey); if ( !sRc && i != -1 ) { recKey.usOffset = i; recKey.ulLen = 0; // init length - sRc = pBTIda->QDAMInsertKey_V3( pRecord, pNewKey, recKey, recKey); + sRc = pBTIda->QDAMInsertKey_V3( pRecord, ulNewKey, recKey, recKey); } /* endif */ } /* endif */ if ( !sRc ) { - sRc = pBTIda->QDAMLocateKey_V3( pRecord, pOldKey, &i, FEXACT, &sNearKey); + sRc = pBTIda->QDAMLocateKey_V3( pRecord, ulOldKey, &i, FEXACT, &sNearKey); } /* endif */ if (!sRc && i!= -1) @@ -4498,16 +4425,16 @@ size_t BTREE::GetFileSize()const{ SHORT BTREE::QDAMInsertKey_V3 ( std::shared_ptr& pRecord, // record where key is to be inserted - PWCHAR pKey, + ULONG& ulKey, RECPARAM recKey, // position/offset for key RECPARAM recData // position/offset for data ) { SHORT i = 0; std::shared_ptr pTempRec; - PWCHAR pCompKey = NULL; // key to be compared with - PWCHAR pOldKey; // old key at first position - PWCHAR pNewKey; // new key at first position + ULONG ulCompKey = -1; // key to be compared with + ULONG ulOldKey = -1; // old key at first position + ULONG ulNewKey = -1; // new key at first position BOOL fFound = FALSE; SHORT sKeyFound; // key found SHORT sNearKey; // key found @@ -4524,7 +4451,7 @@ SHORT BTREE::QDAMInsertKey_V3 recKey; // get rid of compiler warnings // check if key is already there -- duplicates will not be supported - sRc = QDAMLocateKey_V3( pRecord, pKey, &sKeyFound, FEXACT, &sNearKey); + sRc = QDAMLocateKey_V3( pRecord, ulKey, &sKeyFound, FEXACT, &sNearKey); BTREELOCKRECORD( pRecord ); fRecLocked = TRUE; if ( !sRc ) @@ -4545,7 +4472,7 @@ SHORT BTREE::QDAMInsertKey_V3 { BTREEUNLOCKRECORD( pRecord ); fRecLocked = FALSE; - sRc = QDAMSplitNode_V3( this, pRecord, pKey ); + sRc = QDAMSplitNode_V3( this, pRecord, ulKey ); if ( !sRc ) { // SplitNode may have passed a new record back @@ -4577,11 +4504,10 @@ SHORT BTREE::QDAMInsertKey_V3 pusOffset = (PUSHORT) pRecord->contents.uchData; while ( i > 0 && !fFound ) { - pCompKey = QDAMGetszKey_V3( pRecord, (SHORT)(i-1), usVersion ); - if ( pCompKey ) + ulCompKey = QDAMGetszKey_V3( pRecord, (SHORT)(i-1) ); + if ( ulCompKey != -1 ) { - if ( NTMKeyCompare(this, pKey, pCompKey) < 0 ) - //if ( (*(compare))(this, pKey, pCompKey) < 0 ) + if ( compKeys(ulKey, ulCompKey) < 0 ) { *(pusOffset+i) = *(pusOffset+i-1); i--; @@ -4615,7 +4541,7 @@ SHORT BTREE::QDAMInsertKey_V3 { PBYTE pTarget; pTarget = pData + sizeof(USHORT) + sizeof(RECPARAM); - memcpy(pTarget, (PBYTE) pKey, usKeyLen ); + memcpy(pTarget, &ulKey, usKeyLen ); } // set data information memcpy((PRECPARAM) (pData+sizeof(USHORT)), &recData, sizeof(RECPARAM)); @@ -4636,11 +4562,11 @@ SHORT BTREE::QDAMInsertKey_V3 { if ((i == 0) && (!IS_ROOT(pRecord))) { - pOldKey = QDAMGetszKey_V3( pRecord, 1, usVersion ); - pNewKey = QDAMGetszKey_V3( pRecord, 0, usVersion ); - if ( pOldKey && pNewKey ) + ulOldKey = QDAMGetszKey_V3( pRecord, 1 ); + ulNewKey = QDAMGetszKey_V3( pRecord, 0 ); + if ( ulOldKey != -1 && ulNewKey != -1 ) { - if ( pCompKey && PARENT(pRecord) ) + if ( ulCompKey != -1 && PARENT(pRecord) ) { /**********************************************************/ /* save old current record since ChangeKey will update it */ @@ -4648,7 +4574,7 @@ SHORT BTREE::QDAMInsertKey_V3 /**********************************************************/ _usCurrentRecord = usCurrentRecord; _sCurrentIndex = sCurrentIndex; - sRc = QDAMChangeKey_V3( this, PARENT(pRecord), pOldKey, pNewKey); + sRc = QDAMChangeKey_V3( this, PARENT(pRecord), ulOldKey, ulNewKey); usCurrentRecord = _usCurrentRecord; sCurrentIndex = _sCurrentIndex; @@ -5168,14 +5094,13 @@ SHORT BTREE::QDAMDictOpenLocal //------------------------------------------------------------------------------ SHORT BTREE::QDAMDictInsertLocal ( - PWCHAR pKey, // pointer to key data - std::vector& pData // pointer to user data + ULONG& ulKey, // pointer to key data + BytesRef pData // pointer to user data ) { SHORT sRc = 0; // return code RECPARAM recData; // offset/node of data storage RECPARAM recKey; // offset/node of key storage - USHORT usKeyLen; // length of the key memset( &recKey, 0, sizeof( recKey ) ); memset( &recData, 0, sizeof( recData ) ); @@ -5192,19 +5117,16 @@ SHORT BTREE::QDAMDictInsertLocal /*******************************************************************/ /* check if entry is locked .... */ /*******************************************************************/ - if ( !sRc && QDAMDictLockStatus( pKey ) ) + if ( !sRc && QDAMDictLockStatus( ulKey ) ) { sRc = BTREE_ENTRY_LOCKED; } /* endif */ std::shared_ptr pNode; if ( !sRc ) - { - usKeyLen = (USHORT) sizeof(ULONG); - //memcpy( (PBYTE)chHeadTerm, (PBYTE)pKey, usKeyLen);//+sizeof(TMWCHAR) ); // save current data - + { QDAMDictUpdStatus (); - sRc = QDAMFindRecord_V3( pKey, pNode ); + sRc = QDAMFindRecord_V3( ulKey, pNode ); } /* endif */ if ( !sRc ) @@ -5217,7 +5139,7 @@ SHORT BTREE::QDAMDictInsertLocal T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << pData.size(); } recData.ulLen = pData.size(); - sRc = QDAMInsertKey_V3 ( pNode, pKey, recKey, recData); + sRc = QDAMInsertKey_V3 ( pNode, ulKey, recKey, recData); } /* endif */ BTREEUNLOCKRECORD( pNode ); /****************************************************************/ @@ -5225,13 +5147,6 @@ SHORT BTREE::QDAMDictInsertLocal /****************************************************************/ lTime ++; } - - - if ( sRc ) - { - ERREVENT2( QDAMDICTINSERTLOCAL_LOC, INTFUNCFAILED_EVENT, sRc, DB_GROUP, "" ); - } /* endif */ - return sRc; } @@ -5826,8 +5741,7 @@ SHORT QDAMGetszKeyParam_V3 ( PBTREE pBTIda, // pointer to btree structure RECPARAM recKey, // active record - PCHAR_W pKeyData, // pointer to data - PULONG pulLen // length of data + ULONG& ulKeyData // pointer to data ) { PCHAR pData = NULL; @@ -5848,15 +5762,16 @@ SHORT QDAMGetszKeyParam_V3 // as data is now in Unicode the length of the key may be up to // HEADTERM_SIZE *2 !!! - if ( (usLen < (HEADTERM_SIZE + HEADTERM_SIZE)) && ( *pulLen > usLen ) ) + if ( (usLen < (HEADTERM_SIZE + HEADTERM_SIZE)) )//&& ( *pulLen > usLen ) ) { - *pulLen = usLen; + //*pulLen = usLen; pData += sizeof(USHORT ) + sizeof(RECPARAM); // get pointer to data - memcpy( pKeyData, pData, *pulLen ); + memcpy( &ulKeyData, pData, usLen ); } else { + ulKeyData = -1; sRc = BTREE_CORRUPTED; ERREVENT2( QDAMGETSZKEYPARAM_LOC, STATE_EVENT, 1, DB_GROUP, "" ); } @@ -6020,9 +5935,8 @@ SHORT QDAMFirst_V3 SHORT QDAMDictFirstLocal ( PBTREE pBTIda, - PCHAR_W pKeyData, // pointer to space for key data - PULONG pulKeyLen, // length of space for key data - std::vector& pUserData // pointer to space for user data + ULONG ulKeyData, // pointer to space for key data + BytesRef pUserData // pointer to space for user data ) { SHORT sRc = 0; // return code @@ -6045,8 +5959,6 @@ SHORT QDAMDictFirstLocal do { - ulKeyLen = *pulKeyLen; - if ( !sRc ) { sRc = QDAMFirst_V3( pBTIda, &recBTree,&recKey, &recData); @@ -6054,14 +5966,9 @@ SHORT QDAMDictFirstLocal if ( !sRc ) { - sRc = QDAMGetszKeyParam_V3( pBTIda, recKey, pKeyData, &ulKeyLen ); + sRc = QDAMGetszKeyParam_V3( pBTIda, recKey, ulKeyData ); } - if ( !sRc && ulKeyLen ) - { - ULONG ul = 0l; - // memcpy( pBTIda->chHeadTerm, &ul, sizeof(ULONG) ); - } /* endif */ - + if ( !sRc ) { sRc = QDAMGetszData_V3( pBTIda, recData, pUserData, DATA_NODE ); @@ -6075,7 +5982,6 @@ SHORT QDAMDictFirstLocal } while( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && (sRetries > 0)); - *pulKeyLen = ulKeyLen; } /* endif */ if ( sRc ) @@ -6139,7 +6045,7 @@ USHORT EQFNTMOrganizeIndex { ULONG ulKeyLen = sizeof(ULONG) + 1; // ulKeyBufSize; - sRc = QDAMDictFirstLocal( pbTree, (PCHAR_W)&ulKey, &ulKeyLen, pbData ); + sRc = QDAMDictFirstLocal( pbTree, ulKey, pbData ); } /* endif */ // cleanup diff --git a/source/opentm2/core/utilities/EQFUTFIL.cpp b/source/opentm2/core/utilities/EQFUTFIL.cpp index a388b493..c6145ae8 100755 --- a/source/opentm2/core/utilities/EQFUTFIL.cpp +++ b/source/opentm2/core/utilities/EQFUTFIL.cpp @@ -183,7 +183,7 @@ USHORT UtlGetLANDriveList( PBYTE pszList ) //| return ok flag | //+----------------------------------------------------------------------------+ BOOL UtlLoadFile(PSZ pszFilename, // name of file to be loaded - std::vector& ppLoadedFile, // return pointer to loaded file + BytesRef ppLoadedFile, // return pointer to loaded file USHORT *pusBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. @@ -199,7 +199,7 @@ BOOL UtlLoadFile(PSZ pszFilename, // name of file to be loaded } BOOL UtlLoadFileL(PSZ pszFilename, // name of file to be loaded - std::vector& ppLoadedFile, // return pointer to loaded file + BytesRef ppLoadedFile, // return pointer to loaded file ULONG *pulBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. @@ -212,7 +212,7 @@ BOOL UtlLoadFileL(PSZ pszFilename, // name of file to be loaded BOOL UtlLoadFileHwnd ( PSZ pszFilename, // name of file to be loaded - std::vector& ppLoadedFile, // return pointer to loaded file + BytesRef ppLoadedFile, // return pointer to loaded file ULONG *pulBytesRead, // length of loaded file BOOL fContinue, // continue in case of error. If this flag // is set fMsg must be set too. From df478907190e0460f3f8d1fa4596f686c6932047 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 9 Apr 2025 15:39:21 +0300 Subject: [PATCH 06/62] *changed record ey type frpm ULONG to LONG --- include/EQFQDAMI.H | 18 +- include/Proposal.h | 4 +- include/lowlevelotmdatastructs.h | 40 ++-- include/tm.h | 24 +- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 52 ++--- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 12 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 74 +++--- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 52 ++--- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 130 +++++------ .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 6 +- .../core/EqfMemoryPlugin/EqfMemory.cpp | 14 +- source/opentm2/core/tm.cpp | 26 +-- source/opentm2/core/utilities/EQFDICT.cpp | 218 ++++++++---------- 13 files changed, 315 insertions(+), 355 deletions(-) diff --git a/include/EQFQDAMI.H b/include/EQFQDAMI.H index bcf6f1d1..35befcc5 100755 --- a/include/EQFQDAMI.H +++ b/include/EQFQDAMI.H @@ -347,7 +347,7 @@ typedef struct _DICTSIGN SHORT QDAMDictInsert ( PBTREE pBT, // pointer to binary tree struct - ULONG ulKey, // pointer to key data + LONG lKey, // pointer to key data PBYTE pData, // pointer to user data ULONG ulLen // length of user data ); @@ -392,7 +392,7 @@ typedef struct _DICTINSERT SHORT QDAMDictExact ( PBTREE pBT, // pointer to btree struct - ULONG ulKey, // key to be searched for + LONG lKey, // key to be searched for PBYTE pchBuffer, // space for user data PULONG pulLength, // in/out length of returned user data USHORT usSeachSubType @@ -438,7 +438,7 @@ SHORT QDAMDictExact SHORT QDAMDictSubStr ( PBTREE pBT, // pointer to btree struct - ULONG ulKey, // key to be searched for + LONG lKey, // key to be searched for PBYTE pchBuffer, // space for key data PULONG pulLength, // in/out length of returned key data PBYTE pchUserData, // space for user data @@ -484,7 +484,7 @@ SHORT QDAMDictSubStr SHORT QDAMDictEquiv ( PBTREE pBT, // pointer to btree struct - ULONG ulKey, // key to be searched for + LONG lKey, // key to be searched for PBYTE pchBuffer, // space for key data PULONG pulLength, // in/out length of returned key data PBYTE pchUserData, // space for user data @@ -781,7 +781,7 @@ SHORT QDAMDictNumber ( PBTREE pBT, // pointer to generic structure ULONG ulTermNum, // term number - ULONG ulKey, // pointer to space for term + LONG lKey, // pointer to space for term PULONG pulKeyLen, // length of key PBYTE pData, // pointer to data PULONG pulLen // length of data @@ -894,7 +894,7 @@ typedef struct _DICTTIME SHORT QDAMDictNumEntriesLocal ( PBTREE, PULONG ); - SHORT QDAMDictFirstLocal ( PBTREE, ULONG, BytesRef ); + SHORT QDAMDictFirstLocal ( PBTREE, LONG&, BytesRef ); SHORT QDAMInsertKey_V3( PBTREE, std::shared_ptr&, PWCHAR, RECPARAM, RECPARAM ); @@ -902,13 +902,13 @@ typedef struct _DICTTIME SHORT QDAMFindChild_V3 ( PBTREE, PWCHAR, USHORT, std::shared_ptr& ); - SHORT QDAMChangeKey_V3(PBTREE, USHORT, ULONG&, ULONG& ); + SHORT QDAMChangeKey_V3(PBTREE, USHORT, LONG&, LONG& ); SHORT QDAMNewRecord_V3( PBTREE, std::shared_ptr&, RECTYPE ); SHORT QDAMDeleteDataFromBuffer ( PBTREE, RECPARAM ); - SHORT QDAMSplitNode_V3( PBTREE, std::shared_ptr&, ULONG& ); + SHORT QDAMSplitNode_V3( PBTREE, std::shared_ptr&, LONG& ); - SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, ULONG& ); + SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, LONG& ); SHORT QDAMGetszData_V3 ( PBTREE, RECPARAM, BytesRef, CHAR ); BOOL QDAMDeleteKey_V3( PBTREE, std::shared_ptr&, PCHAR); diff --git a/include/Proposal.h b/include/Proposal.h index d851516a..dc40007c 100644 --- a/include/Proposal.h +++ b/include/Proposal.h @@ -72,8 +72,8 @@ class OtmProposal int SetProposalKey ( - unsigned long ulKey, - unsigned short usTargetNum + const long& lKey, + const unsigned short& usTargetNum ); /* \brief set the proposal key diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 5d350d74..4902c218 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -304,8 +304,8 @@ typedef struct _RECPARAM /**********************************************************************/ typedef struct _TMVITALINFO { - ULONG ulStartKey = 0; // key to start with - ULONG ulNextKey = 0; // currently active key + LONG lStartKey = 0; // key to start with + LONG lNextKey = 0; // currently active key } NTMVITALINFO, * PNTMVITALINFO; @@ -741,7 +741,7 @@ struct BTREE SHORT EQFNTMInsert ( - ULONG& pulKey, // pointer to key + LONG& lKey, // pointer to key BytesRef pData // pointer to user data ); @@ -770,7 +770,7 @@ struct BTREE SHORT EQFNTMUpdate ( - ULONG ulKey, // key value + LONG lKey, // key value BytesRef pData // pointer to user data ); @@ -796,7 +796,7 @@ struct BTREE SHORT EQFNTMGet ( - ULONG ulKey, // key to be searched for + LONG lKey, // key to be searched for BytesRef uchBuff // space for user data ); @@ -824,8 +824,8 @@ struct BTREE SHORT EQFNTMGetNextNumber ( - PULONG pulStartKey, // return start key number - PULONG pulNextKey // return next key data + PLONG plStartKey, // return start key number + PLONG plNextKey // return next key data ); @@ -908,15 +908,15 @@ struct BTREE SHORT QDAMDeleteDataFromBuffer_V3( RECPARAM recParam); SHORT QDAMDictUpdateLocal ( PWCHAR, BytesRef ); - SHORT QDAMDictExactLocal ( ULONG&, BytesRef, USHORT ); + SHORT QDAMDictExactLocal ( LONG&, BytesRef, USHORT ); - SHORT QDAMDictCreateLocal ( TMX_SIGN*, ULONG, bool keepInRamOnly = false); + SHORT QDAMDictCreateLocal ( TMX_SIGN*, LONG, bool keepInRamOnly = false); - SHORT QDAMDictInsertLocal ( ULONG&, BytesRef pData ); - BOOL QDAMDictLockStatus ( ULONG ); + SHORT QDAMDictInsertLocal ( LONG&, BytesRef pData ); + BOOL QDAMDictLockStatus ( const LONG& ); VOID QDAMDictUpdStatus (); - SHORT QDAMFindRecord_V3( ULONG, std::shared_ptr&); + SHORT QDAMFindRecord_V3( LONG&, std::shared_ptr&); //------------------------------------------------------------------------------ @@ -1116,7 +1116,7 @@ SHORT QDAMNewRecord_V3 SHORT QDAMInsertKey_V3 ( std::shared_ptr& pRecord, // record where key is to be inserted - ULONG& ulKey, + LONG& ulKey, RECPARAM recKey, // position/offset for key RECPARAM recData // position/offset for data ); @@ -1132,7 +1132,7 @@ BOOL QDAMTerseData SHORT QDAMLocateKey_V3( - std::shared_ptr&, ULONG&, PSHORT, SEARCHTYPE, PSHORT); + std::shared_ptr&, LONG&, PSHORT, SEARCHTYPE, PSHORT); };//BTREE @@ -1141,8 +1141,8 @@ typedef BTREE ** PPBTREE; -#define NTMNEXTKEY( pBT ) pBT->chCollate.ulNextKey -#define NTMSTARTKEY( pBT ) pBT->chCollate.ulStartKey +#define NTMNEXTKEY( pBT ) pBT->chCollate.lNextKey +#define NTMSTARTKEY( pBT ) pBT->chCollate.lStartKey @@ -2457,12 +2457,12 @@ struct MEM_LOAD_DLG_IDA class TMCursor { - ulong _recordKey; + long _recordKey; ushort _targetKey; public: - int SplitProposalKeyIntoRecordAndTarget(char *pszKey, ulong *pulKey, ushort *pusTargetNum); + int SplitProposalKeyIntoRecordAndTarget(char *pszKey, long *plKey, ushort *pusTargetNum); int parseAndSetInternalKey(char* pszKey){ return SplitProposalKeyIntoRecordAndTarget(pszKey, &_recordKey, &_targetKey); } @@ -2479,7 +2479,7 @@ class TMCursor return setInternalKey(_recordKey+1, 1); } - int setInternalKey(ulong recordKey, ushort targetKey){ + int setInternalKey(const ulong recordKey, const ushort targetKey){ _recordKey = recordKey; _targetKey = targetKey; return 0; @@ -2494,7 +2494,7 @@ class TMCursor return std::to_string(_recordKey) + ":" + std::to_string(_targetKey) ; } - ulong getRecordKey()const { return _recordKey; } + long getRecordKey()const { return _recordKey; } ushort getTargetKey()const { return _targetKey; } }; diff --git a/include/tm.h b/include/tm.h index ea5da289..22c430fa 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1079,7 +1079,7 @@ typedef struct _TMX_TERM_TOKEN typedef struct _TMX_MATCHENTRY { - ULONG ulKey; + LONG lKey; USHORT usMaxVotes; USHORT usMatchVotes; BYTE cCount; @@ -1693,7 +1693,7 @@ typedef struct _TMX_MATCH_TABLE_W int iDiffs; USHORT usOverlaps; // temp field - nr of overlapping triples CHAR szTagTable[MAX_FNAME]; // tag table name of source - ULONG ulKey; // key of match + LONG lKey; // key of match USHORT usTargetNum; // number of target USHORT usDBIndex; // number of memory in current hierarchy //CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context info @@ -1982,14 +1982,14 @@ class EqfMemory //: public TMX_CLB BytesRef TmRecord, OtmProposal &TmDel, TMX_EXT_OUT_W *pTmExtOut, - PULONG pulKey); + LONG& lKey); USHORT FindTargetByKeyAndDelete( BytesRef pTmRecord, OtmProposal &pTmDel, PTMX_SENTENCE pSentence, TMX_EXT_OUT_W *pTmExtOut, - PULONG pulKey); + LONG& lKey); USHORT TokenizeSource(PTMX_SENTENCE, PSZ, PSZ); @@ -2014,12 +2014,12 @@ class EqfMemory //: public TMX_CLB USHORT ComparePutData( BytesRef ppTmRecord, // ptr to ptr of tm record data buffer OtmProposal &TmProposal, // pointer to get in data - PULONG pulKey // tm key + LONG& lKey // tm key ); USHORT AddTmTarget( OtmProposal &TmProposal, // pointer to get in data BytesRef pTmRecord, // pointer to tm record data pointer - PULONG pulKey); // tm key + LONG& lKey); // tm key /*! \brief Store the supplied proposal in the memory When the proposal aready exists it will be overwritten with the supplied data @@ -2070,7 +2070,7 @@ class EqfMemory //: public TMX_CLB char *pszKeyBuffer, int iKeyBufferSize); - USHORT AddToTm(OtmProposal &, PULONG); + USHORT AddToTm(OtmProposal &, LONG&); USHORT UpdateTmIndex(PTMX_SENTENCE, ULONG); void importDone(int iRC, char *pszError); @@ -2097,7 +2097,7 @@ class EqfMemory //: public TMX_CLB ); USHORT NTMLoadNameTable( - ULONG ulTableKey, // key of table record + LONG lTableKey, // key of table record PTMX_TABLE pTMTable, // ptr to table data pointer PULONG pulSize // ptr to buffer for size of table data ); @@ -2259,7 +2259,7 @@ class EqfMemory //: public TMX_CLB \returns 0 or error code in case of errors */ int SetProposalKey( - ULONG ulKey, + LONG lKey, USHORT usTargetNum, OtmProposal *pProposal); @@ -3531,7 +3531,7 @@ VOID FillTmRecord(PTMX_SENTENCE, BytesRef TmRecord, // filled tm record returned BytesRef, USHORT); -USHORT DetermineTmRecord(EqfMemory *, PTMX_SENTENCE, std::vector& pulSids ); +USHORT DetermineTmRecord(EqfMemory *, PTMX_SENTENCE, std::vector& pulSids ); VOID FillTargetRecord(PTMX_SENTENCE, // PTMX_TAGTABLE_RECORD, PSZ_W, USHORT, PTMX_TARGET_RECORD *, PTMX_TARGET_CLB); @@ -3542,7 +3542,7 @@ VOID DeleteOldestRecord(PTMX_RECORD, PULONG); USHORT GetExactMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, PTMX_GET_OUT_W); USHORT ExactTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_SENTENCE, - PTMX_MATCH_TABLE_W, PUSHORT, ULONG); + PTMX_MATCH_TABLE_W, PUSHORT, LONG); BOOL AddTagsToString(PSZ, PULONG, PTMX_TAGTABLE_RECORD, PSZ); BOOL AddTagsToStringW(PSZ_W, PLONG, PTMX_TAGTABLE_RECORD, PSZ_W); @@ -3552,7 +3552,7 @@ INT CompCountVotes(const void *, const void *); VOID CleanupTempMatch(PTMX_MATCHENTRY, PTMX_MATCHENTRY *, PUSHORT, PUSHORT); USHORT FillMatchEntry(EqfMemory *, PTMX_SENTENCE, PTMX_MATCHENTRY, PUSHORT); USHORT FuzzyTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, - PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, ULONG); + PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, LONG); USHORT GetFuzzyMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT); // tm delete segment prototypes diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index d33eb42f..bb4b9cb5 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -64,7 +64,7 @@ USHORT EqfMemory::TmtXDelSegm USHORT usMatchesFound = 0; // compact area hits ULONG ulLen = 0; // length indication CHAR szString[MAX_EQF_PATH]; // character string - std::vector pulSids; // ptr to sentence ids + std::vector plSids; // ptr to sentence ids PULONG pulSidStart = NULL; // ptr to sentence ids std::vector pTmRecord; // space for user data @@ -102,20 +102,20 @@ USHORT EqfMemory::TmtXDelSegm usMatchesFound = CheckCompactArea( TmDelIn.pInputSentence, this ); if ( usMatchesFound == TmDelIn.pInputSentence->pulVotes.size() ) //all hash triples found { - usRc = DetermineTmRecord( this, TmDelIn.pInputSentence, pulSids ); + usRc = DetermineTmRecord( this, TmDelIn.pInputSentence, plSids ); if ( !usRc ) { ulLen = TMX_REC_SIZE; - for (ULONG ulKey : pulSids ) + for (LONG lKey : plSids ) { - usRc = TmBtree.EQFNTMGet(ulKey, //tm record key + usRc = TmBtree.EQFNTMGet(lKey, //tm record key pTmRecord ); //length if ( usRc == NO_ERROR ) { //find target record and delete, if the target record was the //only target in the tm record, delete the entire record - usRc = FindTargetAndDelete( pTmRecord, TmDelIn, pTmDelOut, &ulKey ); + usRc = FindTargetAndDelete( pTmRecord, TmDelIn, pTmDelOut, lKey ); if ( usRc != SEG_NOT_FOUND ) { //target record was found and deleted or an error occured @@ -146,7 +146,7 @@ USHORT EqfMemory::TmtXDelSegmByKey ) { PTMX_SENTENCE pSentence = NULL; // ptr to sentence structure - ULONG ulKey=TmDelIn.nextInternalKey.getRecordKey(); // tm record key + LONG lKey=TmDelIn.nextInternalKey.getRecordKey(); // tm record key BOOL fOK; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound = 0; // compact area hits @@ -155,16 +155,10 @@ USHORT EqfMemory::TmtXDelSegmByKey std::vector pTmRecord; // space for user data pTmRecord.resize(TMX_REC_SIZE); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - // update TM databse if ( !usRc ) { - usRc = TmBtree.EQFNTMGet( ulKey, //tm record key + usRc = TmBtree.EQFNTMGet( lKey, //tm record key pTmRecord ); //pointer to tm record data if ( usRc == NO_ERROR ) @@ -173,7 +167,7 @@ USHORT EqfMemory::TmtXDelSegmByKey //find target record and delete, if the target record was the //only target in the tm record, delete the entire record usRc = FindTargetByKeyAndDelete( pTmRecord, - TmDelIn, pSentence, pTmDelOut, &ulKey ); + TmDelIn, pSentence, pTmDelOut, lKey ); } /* endif */ } /* endif */ @@ -241,7 +235,7 @@ USHORT EqfMemory::FindTargetAndDelete( BytesRef TmRecord, OtmProposal& TmDel, TMX_EXT_OUT_W * pTmExtOut, - PULONG pulKey ) + LONG& lKey ) { PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); BOOL fOK = FALSE; //success indicator @@ -259,18 +253,8 @@ USHORT EqfMemory::FindTargetAndDelete( USHORT usRc = NO_ERROR; //returned value from function USHORT usTarget = 0; //initialize counter - //allocate 4k for pTagRecord - if ( fOK ) - { - if ( fOK ) - lTagAlloc = (LONG)TOK_SIZE; - } - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + lTagAlloc = (LONG)TOK_SIZE; + { //position at beginning of source structure in tm record pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); @@ -399,8 +383,8 @@ USHORT EqfMemory::FindTargetAndDelete( pTmExtOut ); NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); - pTmExtOut->ulRecKey = *pulKey; - TmDel.currentInternalKey.setInternalKey(*pulKey, usTarget); + pTmExtOut->ulRecKey = lKey; + TmDel.currentInternalKey.setInternalKey(lKey, usTarget); pTmExtOut->usTargetKey = usTarget; TMDelTargetClb( pTmRecord, pTMXTargetRecord, pClb ); @@ -413,7 +397,7 @@ USHORT EqfMemory::FindTargetAndDelete( /* by a new translation, we will not remove */ /* the key (only get rid of any target data) */ /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( *pulKey, TmRecord); + usRc = TmBtree.EQFNTMUpdate( lKey, TmRecord); //RECLEN(pTmRecord) ); } /* endif */ @@ -460,7 +444,7 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( OtmProposal& TmDel, PTMX_SENTENCE pSentence, TMX_EXT_OUT_W * pTmExtOut, - PULONG pulKey ) + LONG& lKey ) { USHORT usRc = 0; PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); @@ -620,8 +604,8 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( pTmExtOut ); NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); - TmDel.currentInternalKey.setInternalKey(*pulKey, usTarget); - pTmExtOut->ulRecKey = *pulKey; + TmDel.currentInternalKey.setInternalKey(lKey, usTarget); + pTmExtOut->ulRecKey = lKey; pTmExtOut->usTargetKey = usTarget; //if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime ) @@ -638,7 +622,7 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( /* by a new translation, we will not remove */ /* the key (only get rid of any target data) */ /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( *pulKey, TmRecord); + usRc = TmBtree.EQFNTMUpdate( lKey, TmRecord); ///RECLEN(pTmRecord) ); } /* endif */ if ( ! usRc ) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index d506b900..109b8821 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -68,8 +68,8 @@ USHORT EqfMemory::TmtXExtract BOOL fOK = true; //success indicator PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record - ULONG ulStartKey; //start tm key - ULONG ulNextKey; //last tm key + LONG lStartKey; //start tm key + LONG lNextKey; //last tm key BOOL fSpecialMode = FALSE; //special mode flag ULONG ulOemCP = 1L; @@ -259,13 +259,13 @@ USHORT EqfMemory::TmtXExtract /********************************************************************/ if ( !usRc && !fSpecialMode ) { - TmBtree.EQFNTMGetNextNumber( &ulStartKey, &ulNextKey); - pTmExtOut->ulMaxEntries = (ulNextKey - ulStartKey); + TmBtree.EQFNTMGetNextNumber( &lStartKey, &lNextKey); + pTmExtOut->ulMaxEntries = (lNextKey - lStartKey); /******************************************************************/ /* return one matching entry (if any available) */ /******************************************************************/ LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - while ( (pTmExtIn->ulTmKey < ulNextKey) && (usRc == BTREE_NOT_FOUND) ) + while ( (pTmExtIn->ulTmKey < lNextKey) && (usRc == BTREE_NOT_FOUND) ) { usRc = TmBtree.EQFNTMGet(pTmExtIn->ulTmKey, pTmRecord ); @@ -293,7 +293,7 @@ USHORT EqfMemory::TmtXExtract /******************************************************************/ /* set end of file condition ... */ /******************************************************************/ - if ( (usRc == BTREE_NOT_FOUND) && (pTmExtIn->ulTmKey == ulNextKey) ) + if ( (usRc == BTREE_NOT_FOUND) && (pTmExtIn->ulTmKey == lNextKey) ) { //arrived at last tm record LOG_AND_SET_RC(usRc, T5INFO, BTREE_EOF_REACHED); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index f7356738..3b41ad61 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -78,7 +78,7 @@ void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ); USHORT FillMatchTable( EqfMemory*, std::wstring&, PLONG, PTMX_TARGET_RECORD, PTMX_TARGET_CLB, PTMX_MATCH_TABLE_W, PUSHORT, BOOL, PUSHORT, PUSHORT, PUSHORT, - ULONG, USHORT, PTMX_GET_W, BytesRef, USHORT, USHORT, int, int, USHORT); + LONG, USHORT, PTMX_GET_W, BytesRef, USHORT, USHORT, int, int, USHORT); /**********************************************************************/ /* Prototypes for Static functions... */ @@ -643,32 +643,32 @@ USHORT GetExactMatch ) { BOOL fOK = TRUE; //success indicator - std::vector ulSids; //ptr to sentence ids + std::vector lSids; //ptr to sentence ids USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator - ULONG ulKey; //tm record key + LONG lKey; //tm record key USHORT usMatchEntries = 0; //nr of found matches BytesRef pTmRecord = pTmClb->pvTmRecord; ULONG ulStrippedParm = pGetIn->ulParm & ~(GET_RESPECTCRLF | GET_IGNORE_PATH | GET_NO_GENERICREPLACE | GET_ALWAYS_WITH_TAGS | GET_IGNORE_COMMENT); - ulSids.reserve(MAX_INDEX_LEN+5); + lSids.reserve(MAX_INDEX_LEN+5); - usRc = DetermineTmRecord( pTmClb, pSentence, ulSids ); + usRc = DetermineTmRecord( pTmClb, pSentence, lSids ); if ( usRc == NO_ERROR ) { //get tm record(s) - for(ULONG ulKey: ulSids) + for(LONG lKey: lSids) { - T5LOG( T5INFO) << "GetExactMatch: EQFNTMGET of record #" <TmBtree.EQFNTMGet( ulKey, pTmRecord ); + T5LOG( T5INFO) << "GetExactMatch: EQFNTMGET of record #" <TmBtree.EQFNTMGet( lKey, pTmRecord ); if ( usRc == NO_ERROR ) { //compare tm record data with data passed in the get in structure - T5LOG( T5INFO) << "GetExactMatch: ExactTest of record " << ulKey; + T5LOG( T5INFO) << "GetExactMatch: ExactTest of record " << lKey; usRc = ExactTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pSentence, - pstMatchTable, &usMatchEntries, ulKey ); + pstMatchTable, &usMatchEntries, lKey ); T5LOG( T5INFO) << "GetExactMatch: ExactTest found " << usMatchEntries << " matching entries" ; } /* endif */ if ( usRc == NO_ERROR ) @@ -906,7 +906,7 @@ USHORT ExactTest PTMX_SENTENCE pSentence, //pointer to sentence structure PTMX_MATCH_TABLE_W pstMatchTable, //get out structure to be filled PUSHORT pusMatchEntries, //nr of entries in match entry structure - ULONG ulKeyNum // number of key record + LONG lKeyNum // number of key record ) { BOOL fOK = true; //success indicator @@ -1420,7 +1420,7 @@ USHORT ExactTest pstMatchTable, &usMatchLevel, TRUE, pusMatchEntries, &pSentence->usActVote, &pSentence->usActVote, - ulKeyNum, usTgtNum, + lKeyNum, usTgtNum, pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId ); } else @@ -1674,7 +1674,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct PUSHORT pusMatchEntries, //nr of entries in match table PUSHORT pusMaxVotes, //nr of tm source string votes PUSHORT pusOverlaps, //nr of overlapping triples - ULONG ulKeyNum, // key number + LONG lKeyNum, // key number USHORT usTgtNum, // target number PTMX_GET_W pGetIn, // input structure BytesRef pTagRecord, @@ -2037,7 +2037,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /****************************************************************/ /* fill in key and target number ... */ /****************************************************************/ - pstMatchTable->ulKey = ulKeyNum; + pstMatchTable->lKey = lKeyNum; pstMatchTable->usTargetNum = usTgtNum; @@ -2338,16 +2338,16 @@ USHORT GetFuzzyMatch if ( usRc == NO_ERROR ) { //get tm record(s) - while ( (pMatchEntry->ulKey) && (usRc == NO_ERROR)) // && + while ( (pMatchEntry->lKey) && (usRc == NO_ERROR)) // && //(*pusMatchesFound <= pGetIn->usRequestedMatches) ) { #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFuzzyOtherTime) ); #endif - T5LOG( T5INFO) << "GetFuzzyMatch: EQFNTMGET of record " << pMatchEntry->ulKey; + T5LOG( T5INFO) << "GetFuzzyMatch: EQFNTMGET of record " << pMatchEntry->lKey; - usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->ulKey, pTmRecord ); + usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->lKey, pTmRecord ); #ifdef MEASURETIME @@ -2362,7 +2362,7 @@ USHORT GetFuzzyMatch usRc = FuzzyTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pstMatchTable, &usMatchEntries, &usOverlaps, &pMatchEntry->usMaxVotes, &pSentence->usActVote, - pSentence, pMatchEntry->ulKey ); + pSentence, pMatchEntry->lKey ); if(usRc == SOURCE_LANG_DIFFERENT) { usRc = NO_ERROR; @@ -2491,7 +2491,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block PUSHORT pusTmMaxVotes, //nr of tm source triple PUSHORT pusGetMaxVotes, //nr of get source triples PTMX_SENTENCE pSentence, //ptr to sentence structure - ULONG ulKeyNum ) // record number + LONG lKeyNum ) // record number { PBYTE pByte; //position ptr PBYTE pSource; //position ptr @@ -2516,7 +2516,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block pGetIn->ulSrcOemCP = 1; } - T5LOG( T5INFO) << "FuzzyTest for record " < usNumMatches ); T5LOG(T5INFO) << "FuzzyTest: After ExactTest, fStringEqual = " <pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId ); } /* endif */ } /* endif */ @@ -2907,7 +2907,7 @@ USHORT FillMatchEntry USHORT i, j; // index in for loop USHORT usMaxEntries; // nr of index entries in index record ULONG lMatchEntries = 0; // nr of entries in pTempMatch - ULONG ulKey; // index key + LONG lKey; // index key USHORT usMaxSentences = MAX_INDEX_LEN * 4; LONG lMatchListSize = (LONG)((usMaxSentences+1) * sizeof(TMX_MATCHENTRY)); @@ -2950,11 +2950,11 @@ USHORT FillMatchEntry for ( i = 0; i < pSentence->usActVote; i++, pulVotes++ ) { - ulKey = (*pulVotes) & START_KEY; + lKey = (*pulVotes) & START_KEY; #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFillMatchOtherTime) ); #endif - usRc = pTmClb->InBtree.EQFNTMGet( ulKey, pIndexRecord ); + usRc = pTmClb->InBtree.EQFNTMGet( lKey, pIndexRecord ); #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFillMatchReadTime) ); @@ -2962,7 +2962,7 @@ USHORT FillMatchEntry if ( usRc == NO_ERROR ) { //calculate number of index entries in index record - T5LOG(T5INFO) <<"Processing index record "<usRecordLen; usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); @@ -2974,7 +2974,7 @@ USHORT FillMatchEntry //sentence id left in ulSids for ( j = 0; (j < usMaxEntries) && (lMatchEntries < usMaxSentences); j++, pIndexEntry++ ) { - ULONG ulIndexKey = NTMKEY(*pIndexEntry); + LONG lIndexKey = NTMKEY(*pIndexEntry); // reset to begin of matching table pTempMatch = pTempStart; @@ -2994,7 +2994,7 @@ USHORT FillMatchEntry while (high - low > 1) { probe = (high + low) / 2; - if (pTempStart[probe].ulKey > ulIndexKey) + if (pTempStart[probe].lKey > lIndexKey) high = probe; else low = probe; @@ -3007,7 +3007,7 @@ USHORT FillMatchEntry else { pTempMatch = pTempStart + low; - if ( pTempMatch->ulKey < ulIndexKey ) pTempMatch++; + if ( pTempMatch->lKey < lIndexKey ) pTempMatch++; } /* endif */ } @@ -3020,14 +3020,14 @@ USHORT FillMatchEntry #ifdef MEASURETIME GetElapsedTime( &(pTmClb->lFillMatchFill2Time) ); #endif - if ( ulIndexKey == pTempMatch->ulKey ) + if ( lIndexKey == pTempMatch->lKey ) { pTempMatch->cCount++; } - else if ( !pTempMatch->ulKey ) + else if ( !pTempMatch->lKey ) { //add new match entry - pTempMatch->ulKey = NTMKEY(*pIndexEntry); + pTempMatch->lKey = NTMKEY(*pIndexEntry); pTempMatch->usMaxVotes = NTMVOTES(*pIndexEntry); pTempMatch->cCount = 1; pTempMatch->usMatchVotes = pSentence->usActVote; @@ -3040,7 +3040,7 @@ USHORT FillMatchEntry { /*Fix*/ memmove( pTempMatch+1, pTempMatch, (lMatchEntries - (pTempMatch - pTempStart) ) * sizeof(TMX_MATCHENTRY) ); } /* enidf */ - pTempMatch->ulKey = ulIndexKey; + pTempMatch->lKey = lIndexKey; pTempMatch->usMaxVotes = NTMVOTES(*pIndexEntry); pTempMatch->cCount = 1; lMatchEntries++; @@ -3214,7 +3214,7 @@ VOID CleanupTempMatch( PTMX_MATCHENTRY pTempMatch, usTempMaxVotes = MAX_VOTES - 3; } /* endif */ - T5LOG( T5INFO) << "ulKey=" << pTempPos->ulKey << " usMaxVotes=" << pTempPos->usMaxVotes << " cCount=" << pTempPos->cCount; + T5LOG( T5INFO) << "ulKey=" << pTempPos->lKey << " usMaxVotes=" << pTempPos->usMaxVotes << " cCount=" << pTempPos->cCount; /******************************************************************/ /* check that threshold criteria is fulfilled for BOTH sources */ @@ -3244,7 +3244,7 @@ VOID CleanupTempMatch( PTMX_MATCHENTRY pTempMatch, PTMX_MATCHENTRY pTest = pTempMatch; for ( i = 0; i < usEntries; i++, pTest++ ) { - T5LOG( T5DEVELOP) << "ulKey=" << pTest->ulKey << " usMaxVotes=" << pTest->usMaxVotes << " cCount=" <cCount ; + T5LOG( T5DEVELOP) << "lKey=" << pTest->lKey << " usMaxVotes=" << pTest->usMaxVotes << " cCount=" <cCount ; } /* endfor */ } @@ -3257,7 +3257,7 @@ VOID CleanupTempMatch( PTMX_MATCHENTRY pTempMatch, { T5LOG( T5INFO) << "CleanupTempMatch: Highest ranked entries returned are:" ; memcpy( pMatchEntry, pTempPos, sizeof(TMX_MATCHENTRY )); - T5LOG(T5INFO) << "::ulKey=" << (pTempPos->ulKey) << " usMaxVotes=" << (pTempPos->usMaxVotes)<<" cCount="<< ( pTempPos->cCount ); + T5LOG(T5INFO) << "::ulKey=" << (pTempPos->lKey) << " usMaxVotes=" << (pTempPos->usMaxVotes)<<" cCount="<< ( pTempPos->cCount ); } /* endfor */ @@ -3270,7 +3270,7 @@ VOID CleanupTempMatch( PTMX_MATCHENTRY pTempMatch, PTMX_MATCHENTRY pTest = pTempMatch; for ( i = 0; i < usEntries; i++, pTest++ ) { - T5LOG( T5DEVELOP) << "ulKey="<<(pTest->ulKey)<<" usMaxVotes="<<(pTest->usMaxVotes)<<" cCount="<<(pTest->cCount); + T5LOG( T5DEVELOP) << "ulKey="<<(pTest->lKey)<<" usMaxVotes="<<(pTest->usMaxVotes)<<" cCount="<<(pTest->cCount); } /* endfor */ } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index f6d47003..e6e9941b 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -420,12 +420,12 @@ void BTREE::freeLookupTable(){ //+----------------------------------------------------------------------------+ //|Function name: EQFNTMInsert | //+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMInsert( pBTIda, &ulKey, pData, usLen ); | +//|Function call: sRc = EQFNTMInsert( pBTIda, &lKey, pData, usLen ); | //+----------------------------------------------------------------------------+ //|Description: insert a new key (ULONG) with data | //+----------------------------------------------------------------------------+ //|Parameters: PBTREE pBTIda, pointer to binary tree struct | -//| PULONG pulKey, pointer to key | +//| PULONG plKey, pointer to key | //| PBYTE pData, pointer to user data | //| USHORT usLen length of user data | //+----------------------------------------------------------------------------+ @@ -450,7 +450,7 @@ void BTREE::freeLookupTable(){ SHORT BTREE::EQFNTMInsert ( - ULONG& ulKey, // pointer to key + LONG& lKey, // pointer to key BytesRef pData // pointer to user data ) { @@ -467,14 +467,14 @@ BTREE::EQFNTMInsert /********************************************************************/ if ( !sRc ) { - if ( ulKey == NTMREQUESTNEWKEY ) + if ( lKey == NTMREQUESTNEWKEY ) { /******************************************************************/ /* find next free key and anchor new value in file ... */ /******************************************************************/ ULONG Key; - //ulKey = *pulKey = ++(pvi->ulNextKey); - Key = ulKey = (chCollate.ulNextKey)++; + //lKey = *plKey = ++(pvi->ulNextKey); + Key = lKey = (chCollate.lNextKey)++; if ( Key > 0xFFFFFF ) { sRc = BTREE_NUMBER_RANGE; @@ -496,7 +496,7 @@ BTREE::EQFNTMInsert /****************************************************************/ /* check if key is in valid range ... */ /****************************************************************/ - if ( ulKey > NTMSTARTKEY( this ) ) + if ( lKey > NTMSTARTKEY( this ) ) { sRc = BTREE_NUMBER_RANGE; } /* endif */ @@ -507,7 +507,7 @@ BTREE::EQFNTMInsert /* call QDAMDictInsert to do the dirty work of inserting entry... */ /********************************************************************/ - sRc = QDAMDictInsertLocal( ulKey, pData); + sRc = QDAMDictInsertLocal( lKey, pData); return sRc; @@ -1277,7 +1277,7 @@ RECPARAM QDAMGetrecKey_V3 //------------------------------------------------------------------------------ SHORT BTREE::EQFNTMUpdate ( - ULONG ulKey, // key string + LONG lKey, // key string BytesRef pData ) { @@ -1305,7 +1305,7 @@ SHORT BTREE::EQFNTMUpdate /*******************************************************************/ /* check if entry is locked .... */ /*******************************************************************/ - if ( !sRc && QDAMDictLockStatus( ulKey ) ) + if ( !sRc && QDAMDictLockStatus( lKey ) ) { sRc = BTREE_ENTRY_LOCKED; } /* endif */ @@ -1322,7 +1322,7 @@ SHORT BTREE::EQFNTMUpdate else { QDAMDictUpdStatus(); - sRc = QDAMFindRecord_V3( ulKey, pRecord ); + sRc = QDAMFindRecord_V3( lKey, pRecord ); } /* endif */ } /* endif */ @@ -1330,7 +1330,7 @@ SHORT BTREE::EQFNTMUpdate if ( !sRc ) { // find the key - sRc = QDAMLocateKey_V3( pRecord, ulKey, &i, FEXACT, &sNearKey ) ; + sRc = QDAMLocateKey_V3( pRecord, lKey, &i, FEXACT, &sNearKey ) ; if ( !sRc ) { if ( i != -1) @@ -1452,12 +1452,12 @@ int BTREE::deallocateLookupTableBuffer(int number){ //+----------------------------------------------------------------------------+ //|Function name: EQFNTMUpdate | //+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMUpdate( pBTIda, ulKey, pData, usLen ); | +//|Function call: sRc = EQFNTMUpdate( pBTIda, lKey, pData, usLen ); | //+----------------------------------------------------------------------------+ //|Description: update the data of an already inserted key | //+----------------------------------------------------------------------------+ //|Parameters: PBTREE pBTIda, pointer to binary tree struct | -//| ULONG ulKey, key value | +//| ULONG lKey, key value | //| PBYTE pData, pointer to user data | //| USHORT usLen length of user data | //+----------------------------------------------------------------------------+ @@ -1528,7 +1528,7 @@ int BTREE::deallocateLookupTableBuffer(int number){ SHORT BTREE::QDAMDictExactLocal ( - ULONG& ulKey, // key to be searched for + LONG& lKey, // key to be searched for BytesRef pchBuffer, // space for user data USHORT usSearchSubType // special hyphenation lookup flag ) @@ -1554,12 +1554,12 @@ SHORT BTREE::QDAMDictExactLocal if ( !sRc ) { /* Locate the Leaf node that contains the appropriate key */ - sRc = QDAMFindRecord_V3( ulKey, pRecord ); + sRc = QDAMFindRecord_V3( lKey, pRecord ); } /* endif */ if ( !sRc ) { - sRc = QDAMLocateKey_V3(pRecord, ulKey, &i, FEXACT, &sNearKey); + sRc = QDAMLocateKey_V3(pRecord, lKey, &i, FEXACT, &sNearKey); if ( !sRc ) { if ( i != -1 ) @@ -1606,12 +1606,12 @@ SHORT BTREE::QDAMDictExactLocal //+----------------------------------------------------------------------------+ //|Function name: EQFNTMGet | //+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMGet( pBTIda, ulKey, chData, &usLen ); | +//|Function call: sRc = EQFNTMGet( pBTIda, lKey, chData, &usLen ); | //+----------------------------------------------------------------------------+ //|Description: get the data string for the passed key | //+----------------------------------------------------------------------------+ //|Parameters: PBTREE pBTIda, pointer to btree struct | -//| ULONG ulKey, key to be searched for | +//| ULONG lKey, key to be searched for | //| PCHAR pchBuffer, space for user data | //| PUSHORT pusLength in/out length of returned user data | //+----------------------------------------------------------------------------+ @@ -1624,7 +1624,7 @@ SHORT BTREE::QDAMDictExactLocal SHORT BTREE::EQFNTMGet ( - ULONG ulKey, // key to be searched for + LONG lKey, // key to be searched for BytesRef pchBuffer // space for user data ) { @@ -1640,7 +1640,7 @@ BTREE::EQFNTMGet BOOL _fCorrupted = fCorrupted; fCorrupted = FALSE; - sRc = QDAMDictExactLocal( ulKey, pchBuffer, FEXACT ); + sRc = QDAMDictExactLocal( lKey, pchBuffer, FEXACT ); fCorrupted = _fCorrupted; @@ -1667,7 +1667,7 @@ BTREE::EQFNTMGet //+----------------------------------------------------------------------------+ //|Function name: EQFNTMGetMaxNumber | //+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMGetNextNumber( pBTIda, &ulKey, &ulNextFree );| +//|Function call: sRc = EQFNTMGetNextNumber( pBTIda, &lKey, &ulNextFree );| //+----------------------------------------------------------------------------+ //|Description: get the start key and the next free key ... | //+----------------------------------------------------------------------------+ @@ -1684,12 +1684,12 @@ BTREE::EQFNTMGet SHORT BTREE::EQFNTMGetNextNumber ( - PULONG pulStartKey, // return start key number - PULONG pulNextKey // return next key data + PLONG plStartKey, // return start key number + PLONG plNextKey // return next key data ) { - *pulStartKey = chCollate.ulStartKey; - *pulNextKey = chCollate.ulNextKey; + *plStartKey = chCollate.lStartKey; + *plNextKey = chCollate.lNextKey; return 0; } /* end of function EQFNTMGetNextNumber */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 0729b302..094100f1 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -173,7 +173,7 @@ USHORT EqfMemory::TmtXReplace PTMX_EXT_OUT_W pTmPutOut //ptr to output struct ) { - ULONG ulNewKey = 0; // sid of newly added tm record + LONG lNewKey = -1; // sid of newly added tm record BOOL fOK = TRUE; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound; // compact area hits @@ -231,17 +231,17 @@ USHORT EqfMemory::TmtXReplace if(usRc == NO_ERROR) { - //ulNewKey = TmProposal.recordKey; - ulNewKey = TmProposal.currentInternalKey.getRecordKey(); + //lNewKey = TmProposal.recordKey; + lNewKey = TmProposal.currentInternalKey.getRecordKey(); } //if no tm record fitted for update assume new and add to tm else if ( usRc == ERROR_ADD_TO_TM ) { - usRc = AddToTm(TmProposal, &ulNewKey ); + usRc = AddToTm(TmProposal, lNewKey ); //update index if ( !usRc ) { - usRc = UpdateTmIndex( TmProposal.pInputSentence, ulNewKey ); + usRc = UpdateTmIndex( TmProposal.pInputSentence, lNewKey ); if ( usRc ) fUpdateOfIndexFailed = TRUE; } /* endif */ @@ -250,12 +250,12 @@ USHORT EqfMemory::TmtXReplace else { //add new tm record to tm database - usRc = AddToTm(TmProposal, &ulNewKey ); + usRc = AddToTm(TmProposal, lNewKey ); //update index if ( !usRc ) { - usRc = UpdateTmIndex( TmProposal.pInputSentence, ulNewKey ); + usRc = UpdateTmIndex( TmProposal.pInputSentence, lNewKey ); if ( usRc ) fUpdateOfIndexFailed = TRUE; } /* endif */ @@ -714,7 +714,7 @@ USHORT CheckCompactArea // PTMX_SENTENCE pSentence // EqfMemory* pTmClb // PTMX_PUT pTmPut -// PULONG pulNewKey +// PULONG plNewKey //------------------------------------------------------------------------------ // Output parameter: // @@ -742,7 +742,7 @@ USHORT CheckCompactArea USHORT EqfMemory::AddToTm ( OtmProposal& TmProposal, // ptr to sentence structure - PULONG pulNewKey // sid of newly added tm record + LONG& lNewKey // sid of newly added tm record ) { std::vector TmRecord; // ptr to tm record @@ -767,16 +767,16 @@ USHORT EqfMemory::AddToTm TargetClb, usSrcLang ); //add new tm record to database - *pulNewKey = NTMREQUESTNEWKEY; + lNewKey = NTMREQUESTNEWKEY; if(toTmxRecord(TmRecord)->lRecordLen > 0){ TmRecord.resize(toTmxRecord(TmRecord)->lRecordLen); } usRc = TmBtree.EQFNTMInsert(//ptr to tm structure - *pulNewKey, //to be allocated in funct + lNewKey, //to be allocated in funct TmRecord); //pointer to tm record if(!usRc){ - TmProposal.currentInternalKey.setInternalKey(*pulNewKey, 1); + TmProposal.currentInternalKey.setInternalKey(lNewKey, 1); } } /* endif */ @@ -1025,7 +1025,7 @@ USHORT EqfMemory::UpdateTmIndex USHORT usIndexEntries; // nr of entries in index record PULONG pulVotes = NULL; // pointer to votes USHORT i; // index in for loop - ULONG ulKey; // index key + LONG lKey; // index key std::vector pIndexRecord; // pointer to index structure BOOL fOK = TRUE; // success indicator PBYTE pIndex; // position pointer @@ -1038,8 +1038,8 @@ USHORT EqfMemory::UpdateTmIndex { if ( usRc == NO_ERROR ) { - ulKey = (*pulVotes) & START_KEY; - usRc = InBtree.EQFNTMGet( ulKey, //index key + lKey = (*pulVotes) & START_KEY; + usRc = InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record if ( usRc == BTREE_NOT_FOUND ) @@ -1053,7 +1053,7 @@ USHORT EqfMemory::UpdateTmIndex pIndexRecord.resize( toIndexRecord(pIndexRecord)->usRecordLen); } - usRc = InBtree.EQFNTMInsert( ulKey, pIndexRecord); //pointer to index + usRc = InBtree.EQFNTMInsert( lKey, pIndexRecord); //pointer to index //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file @@ -1063,17 +1063,17 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMInsert(ulKey, pIndexRecord);//, toIndexRecord(pIndexRecord)->usRecordLen ); + usRc = InBtree.EQFNTMInsert(lKey, pIndexRecord);//, toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ if ( !usRc ) { //add the match(tuple) to compact area - ulKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); - *(bCompact.data() + (ulKey >> 3)) |= - 1 << ((BYTE)ulKey & 0x07); -// 1 << (USHORT)(ulKey & 0x07); @01M + lKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); + *(bCompact.data() + (lKey >> 3)) |= + 1 << ((BYTE)lKey & 0x07); +// 1 << (USHORT)(lKey & 0x07); @01M bCompactChanged = TRUE; } /* endif */ } @@ -1136,7 +1136,7 @@ USHORT EqfMemory::UpdateTmIndex toIndexRecord(pIndexRecord)->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); usRc = InBtree.EQFNTMUpdate( - ulKey, + lKey, pIndexRecord);//.data(), //pointer to index //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file @@ -1146,15 +1146,15 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMUpdate(ulKey, pIndexRecord);//.data(), toIndexRecord(pIndexRecord)->usRecordLen ); + usRc = InBtree.EQFNTMUpdate(lKey, pIndexRecord);//.data(), toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ if ( !usRc ) { //add the match(tuple) to compact area - ulKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); - *((PBYTE)bCompact.data() + (ulKey >> 3)) |= - 1 << ((BYTE)ulKey & 0x07); + lKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); + *((PBYTE)bCompact.data() + (lKey >> 3)) |= + 1 << ((BYTE)lKey & 0x07); bCompactChanged = TRUE; } /* endif */ } @@ -1217,7 +1217,7 @@ USHORT DetermineTmRecord ( EqfMemory* pTmClb, // ptr to tm control block PTMX_SENTENCE pSentence, // ptr to sentence structure - std::vector& ulSids // ptr to tm record + std::vector& lSids // ptr to tm record ) { USHORT usRc = NO_ERROR; // return code @@ -1226,7 +1226,7 @@ USHORT DetermineTmRecord PULONG pulVotes; // pointer to votes USHORT i, j; // index in for loop USHORT usMaxEntries = 0; // nr of index entries in index record - ULONG ulKey; // index key + LONG lKey; // index key std::vector pIndexRecord; // pointer to index structure BOOL fOk = true; // success indicator PTMX_INDEX_ENTRY pIndexEntry; // pointer to index entry structure @@ -1239,9 +1239,9 @@ USHORT DetermineTmRecord pIndexRecord.resize(TMX_REC_SIZE); - ulKey = (*pulVotes) & START_KEY; + lKey = (*pulVotes) & START_KEY; memset( pIndexRecord.data(), 0, pIndexRecord.size() ); - usRc = pTmClb->InBtree.EQFNTMGet( ulKey, //index key + usRc = pTmClb->InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record if ( usRc == NO_ERROR ) @@ -1270,23 +1270,23 @@ USHORT DetermineTmRecord { if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) { - ulSids.push_back(NTMKEY(*pIndexEntry)); + lSids.push_back(NTMKEY(*pIndexEntry)); } /* endif */ } /* endfor */ - if ( (ulSids.size() > 1) ) + if ( (lSids.size() > 1) ) { //there is more than one sentence to check so go through other //indices until all have been tried or usSid has been reduced to 1 pulVotes++; - for ( i = 0; (i < pSentence->usActVote-1) && (ulSids.size() > 1); + for ( i = 0; (i < pSentence->usActVote-1) && (lSids.size() > 1); i++, pulVotes++ ) { if ( usRc == NO_ERROR ) { - ulKey = (*pulVotes) & START_KEY; + lKey = (*pulVotes) & START_KEY; memset( pIndexRecord.data(), 0, pIndexRecord.size() ); - usRc = pTmClb->InBtree.EQFNTMGet( ulKey, //index key + usRc = pTmClb->InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record if ( usRc == NO_ERROR ) @@ -1334,17 +1334,17 @@ USHORT DetermineTmRecord } /* endfor */ #else // End criteria: all sentence IDs in index key or only one sentence ID left in pulSids - for (j = 0; (j < usMaxEntries) && (ulSids.size() > 1); j++, pIndexEntry++) { + for (j = 0; (j < usMaxEntries) && (lSids.size() > 1); j++, pIndexEntry++) { if (NTMVOTES(*pIndexEntry) == static_cast(pSentence->usActVote)) { // Before adding sentence ID, check if already in pulSids, as the respective TM record needs only be checked once - while ((NTMKEY(*pIndexEntry) > ulSids.front()) && (ulSids.size() > 1)) { + while ((NTMKEY(*pIndexEntry) > lSids.front()) && (lSids.size() > 1)) { // Remove SID from pulSids and decrease SID counter - ulSids.erase(ulSids.begin()); + lSids.erase(lSids.begin()); } - if (NTMKEY(*pIndexEntry) == ulSids.front()) { + if (NTMKEY(*pIndexEntry) == lSids.front()) { // Move on to the next position in pulSids - ulSids.erase(ulSids.begin()); + lSids.erase(lSids.begin()); usPos++; } } @@ -1378,7 +1378,7 @@ USHORT DetermineTmRecord if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ std::string msg = __func__ + std::string(":: Matching Sids: "); pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; - for(auto sid: ulSids) + for(auto sid: lSids) { msg += std::to_string(sid) + "; "; } @@ -1429,31 +1429,31 @@ USHORT EqfMemory::UpdateTmRecord { BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indication to leave while loop - std::vector ulSids; //ptr to sentence ids + std::vector lSids; //ptr to sentence ids int iSidCount=0; USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator - ULONG ulKey; //tm record key + LONG lKey; //tm record key std::vector TmRecord; //pointer to tm record TmRecord.resize(TMX_REC_SIZE); - ulSids.reserve(MAX_INDEX_LEN + 5); + lSids.reserve(MAX_INDEX_LEN + 5); - usRc = DetermineTmRecord( this, TmProposal.pInputSentence, ulSids ); + usRc = DetermineTmRecord( this, TmProposal.pInputSentence, lSids ); if ( usRc == NO_ERROR ) { //get tm record(s) //while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) - for(ULONG ulKey: ulSids) + for(LONG lKey: lSids) { iSidCount ++; - usRc = TmBtree.EQFNTMGet( ulKey, //tm record key + usRc = TmBtree.EQFNTMGet( lKey, //tm record key TmRecord); //pointer to tm record data if ( usRc == NO_ERROR ) { //compare tm record data with data passed in the get in structure - usRc = ComparePutData( TmRecord, TmProposal, &ulKey ); + usRc = ComparePutData( TmRecord, TmProposal, lKey ); if ( usRc == SOURCE_STRING_ERROR ) { @@ -1471,7 +1471,7 @@ USHORT EqfMemory::UpdateTmRecord } /* endif */ } /* endwhile */ - if ( (iSidCount == ulSids.size()) && !fStop ) + if ( (iSidCount == lSids.size()) && !fStop ) { //issue message that tm needs to be organized if pulsid is empty and //no get was successful @@ -1521,33 +1521,33 @@ USHORT EqfMemory::UpdateTmRecordByInternalKey { BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indication to leave while loop - std::vector ulSids; //ptr to sentence ids + std::vector lSids; //ptr to sentence ids int iSidCount = 0; USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator - ULONG ulKey; //tm record key + LONG lKey; //tm record key PTMX_RECORD pTmRecord = NULL; //pointer to tm record std::vector TmRecord; // space for user data TmRecord.resize(TMX_REC_SIZE); - ulSids.reserve(MAX_INDEX_LEN + 5); + lSids.reserve(MAX_INDEX_LEN + 5); - usRc = DetermineTmRecord( this, TmProposal.pInputSentence, ulSids ); + usRc = DetermineTmRecord( this, TmProposal.pInputSentence, lSids ); if ( usRc == NO_ERROR ) { //get tm record(s) //while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) - for(ULONG ulKey: ulSids) + for(LONG lKey: lSids) { iSidCount++; memset( &TmRecord, 0, TmRecord.size() ); - usRc = TmBtree.EQFNTMGet(ulKey, //tm record key + usRc = TmBtree.EQFNTMGet(lKey, //tm record key TmRecord); //pointer to tm record data if ( usRc == NO_ERROR ) { //compare tm record data with data passed in the get in structure - usRc = ComparePutData( TmRecord, TmProposal, &ulKey ); + usRc = ComparePutData( TmRecord, TmProposal, lKey ); if ( usRc == SOURCE_STRING_ERROR ) { @@ -1564,7 +1564,7 @@ USHORT EqfMemory::UpdateTmRecordByInternalKey } /* endif */ } /* endfor */ - if ( iSidCount == ulSids.size() && !fStop ) + if ( iSidCount == lSids.size() && !fStop ) { //issue message that tm needs to be organized if pulsid is empty and //no get was successful @@ -1631,7 +1631,7 @@ USHORT EqfMemory::ComparePutData ( BytesRef pTmRecord, // ptr to ptr of tm record data buffer OtmProposal& TmProposal, // pointer to get in data - PULONG pulKey // tm key + LONG& lKey // tm key ) { BOOL fOK = true; //success indicator @@ -1817,7 +1817,7 @@ USHORT EqfMemory::ComparePutData { T5LOG(T5ERROR) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize - << "; ulKey = " << *pulKey ; + << "; ulKey = " << lKey ; } memmove( (((PBYTE)pClb) + lNewClbLen), pClb, size); RECLENB(pTmRecord) += lNewClbLen; @@ -1876,22 +1876,22 @@ USHORT EqfMemory::ComparePutData // update TM record if required if ( fUpdate ) { - usRc = TmBtree.EQFNTMUpdate( *pulKey, pTmRecord);//.data(), RECLENB(pTmRecord) ); + usRc = TmBtree.EQFNTMUpdate( lKey, pTmRecord);//.data(), RECLENB(pTmRecord) ); } /* endif */ if ( fStop ) { - TmProposal.currentInternalKey.setInternalKey(*pulKey, targetKey); + TmProposal.currentInternalKey.setInternalKey(lKey, targetKey); //TmProposal.recordKey = *pulKey; //TmProposal.targetKey = targetKey; }else{ //all target records have been checked but nothing overlapped //so add new target record to end of tm record - usRc = AddTmTarget( TmProposal, pTmRecord, pulKey ); + usRc = AddTmTarget( TmProposal, pTmRecord, lKey ); if(!usRc) { - TmProposal.currentInternalKey.setInternalKey(*pulKey, 1); + TmProposal.currentInternalKey.setInternalKey(lKey, 1); } } /* endif */ } @@ -1948,7 +1948,7 @@ USHORT EqfMemory::ComparePutData USHORT EqfMemory::AddTmTarget( OtmProposal& TmProposal, //pointer to get in data BytesRef pTmRecord, //pointer to tm record data pointer - PULONG pulKey ) //tm key + LONG& lKey ) //tm key { PTMX_TARGET_CLB pTargetClb = nullptr ; // ptr to target ctl block PTMX_TARGET_RECORD pTargetRecord = nullptr; // ptr to target record @@ -2004,7 +2004,7 @@ USHORT EqfMemory::AddTmTarget( //add updated tm record to database usRc = TmBtree.EQFNTMUpdate( - *pulKey, + lKey, pTmRecord);//.data(), //RECLENB(pTmRecord) ); } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index f7f43c93..ecd0ed77 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -1294,7 +1294,7 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab //| If the table is tersed it is uncompressed. | // ----------------------------------------------------------------------------+ USHORT EqfMemory::NTMLoadNameTable( - ULONG ulTableKey, // key of table record + LONG lTableKey, // key of table record PTMX_TABLE pTMTable, // ptr to table data pointer PULONG pulSize // ptr to buffer for size of table data ) @@ -1304,9 +1304,9 @@ USHORT EqfMemory::NTMLoadNameTable( // call to obtain exact length of record *pulSize = 0; - usRc = TmBtree.EQFNTMGet(ulTableKey, pOldTable); + usRc = TmBtree.EQFNTMGet(lTableKey, pOldTable); - if(!usRc) usRc = bytesToOldTable(ulTableKey, pOldTable, pTMTable); + if(!usRc) usRc = bytesToOldTable(lTableKey, pOldTable, pTMTable); // return to caller return (usRc); diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 3929f8be..ab6a6ec2 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -420,11 +420,11 @@ int EqfMemory::getLanguage */ int OtmProposal::SetProposalKey ( - ULONG ulKey, - USHORT usTargetNum + const long& lKey, + const USHORT& usTargetNum ) { - currentInternalKey.setInternalKey(ulKey, usTargetNum); + currentInternalKey.setInternalKey(lKey, usTargetNum); return( 0 ); } @@ -442,7 +442,7 @@ std::string OtmProposal::getInternalKey()const{ int TMCursor::SplitProposalKeyIntoRecordAndTarget ( char *pszKey, - ULONG *pulKey, + LONG *plKey, USHORT *pusTargetNum ) { @@ -451,7 +451,7 @@ int TMCursor::SplitProposalKeyIntoRecordAndTarget if ( pszTarget != NULL ) { *pszTarget = '\0'; - *pulKey = atol( pszKey ); + *plKey = atol( pszKey ); *pusTargetNum = (USHORT)atoi( pszTarget + 1 ); } else @@ -524,13 +524,13 @@ int EqfMemory::MatchToOtmProposal pProposal->setSegmentId( pMatch->ulSegmentId ); pProposal->setType( FlagToProposalType( pMatch->usTranslationFlag ) ); pProposal->setUpdateTime( pMatch->lTargetTime ); - pProposal->SetProposalKey( pMatch->ulKey, pMatch->usTargetNum ); + pProposal->SetProposalKey( pMatch->lKey, pMatch->usTargetNum ); pProposal->setContextRanking( (int)pMatch->usContextRanking ); pProposal->setMemoryIndex( pMatch->usDBIndex ); pProposal->setWords( pMatch->iWords ); pProposal->setDiffs( pMatch->iDiffs ); pProposal->setOriginalSourceLanguage( pMatch->szOriginalSrcLanguage ); - pProposal->SetProposalKey(pMatch->ulKey, pMatch->usTargetNum); + pProposal->SetProposalKey(pMatch->lKey, pMatch->usTargetNum); switch ( pMatch->usMatchLevel ) { diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index 3b3d07a1..c069975c 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -317,7 +317,7 @@ TMManager* TMManager::GetInstance(){ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMemName, const std::string& strSrcLang, const std::string& strMemDescription, int& _rc_, bool keepInRamOnly){ std::shared_ptr NewMem; - ULONG ulKey = 0; + LONG lKey = 0; if ( _rc_ == NO_ERROR ) { T5LOG( T5DEBUG) << ":: create the memory" ; @@ -356,29 +356,29 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem if ( _rc_ == NO_ERROR ) { //insert initialized record to tm data file - ulKey = AUTHOR_KEY; + lKey = AUTHOR_KEY; tmxTableToBuffer(&NewMem->Authors,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); } if ( _rc_ == NO_ERROR ) { - ulKey = FILE_KEY; + lKey = FILE_KEY; tmxTableToBuffer(&NewMem->FileNames,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { - ulKey = TAGTABLE_KEY; + lKey = TAGTABLE_KEY; tmxTableToBuffer(&NewMem->TagTables,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { - ulKey = LANG_KEY; + lKey = LANG_KEY; tmxTableToBuffer(&NewMem->Languages,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey,buffer); + _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) @@ -388,8 +388,8 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem //memset( NewMem->bCompact, 0, size ); NewMem->bCompact.resize(size); - ulKey = COMPACT_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey, + lKey = COMPACT_KEY; + _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey, NewMem->bCompact); } /* endif */ @@ -397,10 +397,10 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem // add long document table record if ( _rc_ == NO_ERROR ) { - ulKey = LONGNAME_KEY; + lKey = LONGNAME_KEY; NewMem->LongNames.pszBuffer.resize(sizeof(TMX_RECORD));//to have some size // write long document name buffer area to the database - _rc_ = NewMem->TmBtree.EQFNTMInsert(ulKey, + _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey, NewMem->LongNames.pszBuffer ); } /* endif */ diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index be7f1d18..7934617c 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -809,18 +809,18 @@ VOID BTREE::QDAMTerseInit //| <> keys are unequal | //| | //+----------------------------------------------------------------------------+ -//|Function flow: return (SHORT) (*pulKey1 - *pulKey2) | +//|Function flow: return (SHORT) (*plKey1 - *plKey2) | //+----------------------------------------------------------------------------+ -SHORT compKeys( ULONG ulKey1, // pointer to first key - ULONG ulKey2 // pointer to second key +SHORT compKeys(const LONG& lKey1, // pointer to first key + const LONG& lKey2 // pointer to second key ) { - if (ulKey1 > ulKey2 ) + if (lKey1 > lKey2 ) { return 1; } - else if (ulKey1 < ulKey2 ) + else if (lKey1 < lKey2 ) { return -1; } @@ -1870,13 +1870,13 @@ SHORT BTREE::QDAMAllocKeyRecords SHORT BTREE::QDAMDictCreateLocal ( PTMX_SIGN pSign, - ULONG ulStartKey, // translation memory + LONG lStartKey, // translation memory bool keepInRamOnly ) { const SHORT sNumberOfKey = 20; NTMVITALINFO NtmVitalInfo; // structure to contain vital info for TM - NtmVitalInfo.ulStartKey = NtmVitalInfo.ulNextKey = ulStartKey; + NtmVitalInfo.lStartKey = NtmVitalInfo.lNextKey = lStartKey; SHORT i; SHORT sRc = 0; // return code USHORT usAction; // used in open of file @@ -2053,7 +2053,7 @@ SHORT BTREE::QDAMDictCreateLocal BOOL BTREE::QDAMDictLockStatus ( - ULONG ulKey + const LONG& lKey ) { return( false ); @@ -2270,7 +2270,7 @@ RECPARAM QDAMGetrecData_V3 //------------------------------------------------------------------------------ SHORT BTREE::QDAMFindRecord_V3 ( - ULONG ulKey, + LONG& lKey, std::shared_ptr& pRecord ) { @@ -2279,7 +2279,7 @@ SHORT BTREE::QDAMFindRecord_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - ULONG ulKey2; // pointer to search key + LONG lKey2; // pointer to search key SHORT sRc; // return code memset(&recData, 0, sizeof(recData)); @@ -2295,15 +2295,15 @@ SHORT BTREE::QDAMFindRecord_V3 while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - ulKey2 = QDAMGetszKey_V3( pRecord, sMid ); - //if ( ulKey2 == nullptr ) - if(ulKey2 == -1) + lKey2 = QDAMGetszKey_V3( pRecord, sMid ); + //if ( lKey2 == nullptr ) + if(lKey2 == -1) { sRc = BTREE_CORRUPTED; } else { - sResult = compKeys(ulKey, ulKey2); + sResult = compKeys(lKey, lKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -2989,7 +2989,7 @@ SHORT BTREE::QDAMAddToBuffer_V3 SHORT BTREE::QDAMLocateKey_V3 ( std::shared_ptr& pRecord, // record to be dealt with - ULONG& ulKey, // key to be searched + LONG& lKey, // key to be searched PSHORT psKeyPos, // located key SEARCHTYPE searchType, // search type PSHORT psNearPos // near position @@ -3000,7 +3000,7 @@ SHORT BTREE::QDAMLocateKey_V3 SHORT sResult; SHORT sMid = 0; // SHORT sRc = 0; // return value - ULONG ulKey2; // pointer to key string + LONG lKey2 = -1; // pointer to key string BOOL fFound = FALSE; *psKeyPos = -1; // key not found @@ -3013,11 +3013,11 @@ SHORT BTREE::QDAMLocateKey_V3 while ( !fFound && sLow <= sHigh ) { sMid = (sLow + sHigh)/2; - ulKey2 = QDAMGetszKey_V3( pRecord, sMid ); + lKey2 = QDAMGetszKey_V3( pRecord, sMid ); - if ( ulKey2 != -1 ) + if ( lKey2 != -1 ) { - sResult = compKeys(ulKey,ulKey2); + sResult = compKeys(lKey,lKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -3036,7 +3036,7 @@ SHORT BTREE::QDAMLocateKey_V3 /* matching substring */ /*********************************************************/ { - if (ulKey == ulKey2) + if (lKey == lKey2) { *psKeyPos = sMid; } @@ -3045,12 +3045,12 @@ SHORT BTREE::QDAMLocateKey_V3 // try with previous if ( sMid > sLow ) { - ulKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid-1) ); - if ( ulKey2 == -1 ) + lKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid-1) ); + if ( lKey2 == -1 ) { sRc = BTREE_CORRUPTED; } - else if ( ulKey == ulKey2) + else if ( lKey == lKey2) { *psKeyPos = sMid-1 ; } /* endif */ @@ -3058,12 +3058,12 @@ SHORT BTREE::QDAMLocateKey_V3 // still not found if ( !sRc && *psKeyPos == -1 && sMid < sHigh ) { - ulKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid+1) ); - if ( ulKey2 == -1 ) + lKey2 = QDAMGetszKey_V3( pRecord, (SHORT)(sMid+1) ); + if ( lKey2 == -1 ) { sRc = BTREE_CORRUPTED; } - else if (ulKey == ulKey2) + else if (lKey == lKey2) { *psKeyPos = sMid+1 ; } /* endif */ @@ -3244,10 +3244,10 @@ SHORT QDAMFindParent_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - ULONG ulKey2 = -1; // pointer to search key - ULONG ulKey = -1 ; // pointer to search key + LONG lKey2 = -1; // pointer to search key + LONG lKey = -1 ; // pointer to search key //CHAR_W chKey[HEADTERM_SIZE]; // key to be found - ULONG ulChKey = -1; + LONG lChKey = -1; SHORT sRc; // return code std::shared_ptr pTempRec; // temp. record buffer USHORT usRecNum; // passed record number @@ -3257,14 +3257,14 @@ SHORT QDAMFindParent_V3 *pusParent = pBT->usFirstNode; // set parent to first node usRecNum = RECORDNUM( pRecord ); - ulKey = QDAMGetszKey_V3( pRecord, 0 ); - if ( ulKey != -1 ) + lKey = QDAMGetszKey_V3( pRecord, 0 ); + if ( lKey != -1 ) { sRc = BTREE_CORRUPTED; } else { - ulChKey = ulKey; + lChKey = lKey; } /* endif */ sRc = pBTIda->QDAMReadRecord_V3( pBT->usFirstNode, pTempRec, FALSE ); @@ -3281,14 +3281,14 @@ SHORT QDAMFindParent_V3 while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - ulKey2 = QDAMGetszKey_V3( pTempRec, sMid ); - if ( ulKey2 == -1 ) + lKey2 = QDAMGetszKey_V3( pTempRec, sMid ); + if ( lKey2 == -1 ) { sRc = BTREE_CORRUPTED; } else { - sResult = compKeys(ulChKey,ulKey2); + sResult = compKeys(lChKey,lKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -3552,7 +3552,7 @@ SHORT QDAMSplitNode_V3 ( PBTREE pBTIda, // pointer to generic structure std::shared_ptr& record, // pointer to pointer to node - ULONG& ulKey // new key + LONG& lKey // new key ) { SHORT i,j; @@ -3561,7 +3561,7 @@ SHORT QDAMSplitNode_V3 USHORT usParent; // number of parent RECPARAM recKey; // position/offset for key RECPARAM recData; // position/offset for data - ULONG ulParentKey = -1; // new key to be inserted + LONG lParentKey = -1; // new key to be inserted PUSHORT pusOffset; // pointer to offset table BOOL fCompare; // indicator where to insert new key USHORT usFreeKeys = 0; // number of free keys required @@ -3569,9 +3569,6 @@ SHORT QDAMSplitNode_V3 SHORT sRc = 0; // success indicator - - //T5LOG(T5FATAL) << "called commented out function QDAMSplitNode_V3"; - //#ifdef TEMPORARY_COMMENTED memset( &recKey, 0, sizeof( recKey ) ); memset( &recData, 0, sizeof( recData ) ); @@ -3594,15 +3591,15 @@ SHORT QDAMSplitNode_V3 recData.usNum = RECORDNUM(record); recData.usOffset = 0; - //ulParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); - ulParentKey = QDAMGetszKey_V3( record, 0 ); + //lParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); + lParentKey = QDAMGetszKey_V3( record, 0 ); - //if ( ulParentKey ) - if(ulParentKey != -1) + //if ( lParentKey ) + if(lParentKey != -1) { // store key temporarily, because record with key data is // not locked. - sRc = pBTIda->QDAMInsertKey_V3( newRecord, ulParentKey, recKey, recData); + sRc = pBTIda->QDAMInsertKey_V3( newRecord, lParentKey, recKey, recData); // might be freed during insert BTREELOCKRECORD( pRecTemp ); } @@ -3651,19 +3648,19 @@ SHORT QDAMSplitNode_V3 TYPE(newRecord) = (CHAR)(TYPE(record) & ~ROOT_NODE); // don't copy Root bit // Decide where to split the record - //ulParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); - ulParentKey = QDAMGetszKey_V3( record, (SHORT)(OCCUPIED(record)/2) ); + //lParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); + lParentKey = QDAMGetszKey_V3( record, (SHORT)(OCCUPIED(record)/2) ); - if ( ulParentKey != -1 ) + if ( lParentKey != -1 ) { - fCompare = (compKeys(ulParentKey, ulKey) <= 0 ) ; + fCompare = (compKeys(lParentKey, lKey) <= 0 ) ; /***********************************************************/ /* check in which part we will lay */ /***********************************************************/ if ( fCompare ) { - ulParentKey = QDAMGetszKey_V3(record, (SHORT)(OCCUPIED(record)-MINFREEKEYS)); - fCompare = (compKeys(ulParentKey,ulKey) <= 0 ) ; + lParentKey = QDAMGetszKey_V3(record, (SHORT)(OCCUPIED(record)-MINFREEKEYS)); + fCompare = (compKeys(lParentKey,lKey) <= 0 ) ; if ( fCompare ) { usFreeKeys = MINFREEKEYS; @@ -3675,8 +3672,8 @@ SHORT QDAMSplitNode_V3 } else { - ulParentKey = QDAMGetszKey_V3( record,MINFREEKEYS); - fCompare = (compKeys(ulParentKey,ulKey) <= 0) ; + lParentKey = QDAMGetszKey_V3( record,MINFREEKEYS); + fCompare = (compKeys(lParentKey,lKey) <= 0) ; if ( fCompare ) { usFreeKeys = OCCUPIED( record )/2; @@ -3727,10 +3724,10 @@ SHORT QDAMSplitNode_V3 { recData.usNum = RECORDNUM( newRecord ); recData.usOffset = 0; - ulParentKey = QDAMGetszKey_V3( newRecord,0 ); - if ( ulParentKey != -1) + lParentKey = QDAMGetszKey_V3( newRecord,0 ); + if ( lParentKey != -1) { - sRc = pBTIda->QDAMInsertKey_V3( parent, ulParentKey, recKey, recData); + sRc = pBTIda->QDAMInsertKey_V3( parent, lParentKey, recKey, recData); } else { @@ -3745,10 +3742,10 @@ SHORT QDAMSplitNode_V3 { // newRecord->ulCheckSum = QDAMComputeCheckSum( newRecord ); // (*record)->ulCheckSum = QDAMComputeCheckSum( *record ); - ulParentKey = QDAMGetszKey_V3( newRecord, 0 ); - if ( ulParentKey != -1) + lParentKey = QDAMGetszKey_V3( newRecord, 0 ); + if ( lParentKey != -1) { - if ( compKeys(ulParentKey,ulKey) <= 0) + if ( compKeys(lParentKey,lKey) <= 0) { sRc = pBTIda->QDAMWriteRecord_V3( record); record = newRecord; // add to new record @@ -4229,7 +4226,7 @@ SHORT QDAMGetszData_V3 SHORT QDAMFindChild_V3 ( PBTREE pBTIda, - ULONG& ulKey, + LONG& lKey, USHORT usNode, std::shared_ptr& pRecord ) @@ -4239,7 +4236,7 @@ SHORT QDAMFindChild_V3 SHORT sHigh; // Far right SHORT sMid = 0; // Middle RECPARAM recData; // data structure - ULONG ulKey2; // pointer to search key + LONG lKey2 = -1; // pointer to search key SHORT sRc; // return code PBTREE pBT = pBTIda; @@ -4256,14 +4253,14 @@ SHORT QDAMFindChild_V3 while ( !sRc && (sLow <= sHigh) ) { sMid = (sLow + sHigh)/2; - ulKey2 = QDAMGetszKey_V3( pRecord, sMid ); - if ( ulKey2 == -1 ) + lKey2 = QDAMGetszKey_V3( pRecord, sMid ); + if ( lKey2 == -1 ) { sRc = BTREE_CORRUPTED; } else { - sResult = compKeys(ulKey, ulKey2); + sResult = compKeys(lKey, lKey2); if ( sResult < 0 ) { sHigh = sMid - 1; // Go left @@ -4304,8 +4301,8 @@ SHORT QDAMChangeKey_V3 ( PBTREE pBTIda, // ptr to tree structure USHORT usNode, // start node - ULONG& ulOldKey, // find old key - ULONG& ulNewKey // find new key + LONG& lOldKey, // find old key + LONG& lNewKey // find new key ) { std::shared_ptr pRecord; // buffer for record @@ -4324,23 +4321,23 @@ SHORT QDAMChangeKey_V3 if ( !sRc ) { /* Locate the Leaf node that contains the appropriate key */ - sRc = QDAMFindChild_V3( pBTIda, ulNewKey, usNode, pNewRecord ); + sRc = QDAMFindChild_V3( pBTIda, lNewKey, usNode, pNewRecord ); } /* endif */ if ( !sRc ) { // get the key description and insert the new key recKey.usNum = pNewRecord->contents.header.usNum; - sRc = pBTIda->QDAMLocateKey_V3( pNewRecord, ulNewKey, &i, FEXACT, &sNearKey); + sRc = pBTIda->QDAMLocateKey_V3( pNewRecord, lNewKey, &i, FEXACT, &sNearKey); if ( !sRc && i != -1 ) { recKey.usOffset = i; recKey.ulLen = 0; // init length - sRc = pBTIda->QDAMInsertKey_V3( pRecord, ulNewKey, recKey, recKey); + sRc = pBTIda->QDAMInsertKey_V3( pRecord, lNewKey, recKey, recKey); } /* endif */ } /* endif */ if ( !sRc ) { - sRc = pBTIda->QDAMLocateKey_V3( pRecord, ulOldKey, &i, FEXACT, &sNearKey); + sRc = pBTIda->QDAMLocateKey_V3( pRecord, lOldKey, &i, FEXACT, &sNearKey); } /* endif */ if (!sRc && i!= -1) @@ -4425,16 +4422,16 @@ size_t BTREE::GetFileSize()const{ SHORT BTREE::QDAMInsertKey_V3 ( std::shared_ptr& pRecord, // record where key is to be inserted - ULONG& ulKey, + LONG& lKey, RECPARAM recKey, // position/offset for key RECPARAM recData // position/offset for data ) { SHORT i = 0; std::shared_ptr pTempRec; - ULONG ulCompKey = -1; // key to be compared with - ULONG ulOldKey = -1; // old key at first position - ULONG ulNewKey = -1; // new key at first position + LONG lCompKey = -1; // key to be compared with + LONG lOldKey = -1; // old key at first position + LONG lNewKey = -1; // new key at first position BOOL fFound = FALSE; SHORT sKeyFound; // key found SHORT sNearKey; // key found @@ -4451,7 +4448,7 @@ SHORT BTREE::QDAMInsertKey_V3 recKey; // get rid of compiler warnings // check if key is already there -- duplicates will not be supported - sRc = QDAMLocateKey_V3( pRecord, ulKey, &sKeyFound, FEXACT, &sNearKey); + sRc = QDAMLocateKey_V3( pRecord, lKey, &sKeyFound, FEXACT, &sNearKey); BTREELOCKRECORD( pRecord ); fRecLocked = TRUE; if ( !sRc ) @@ -4472,7 +4469,7 @@ SHORT BTREE::QDAMInsertKey_V3 { BTREEUNLOCKRECORD( pRecord ); fRecLocked = FALSE; - sRc = QDAMSplitNode_V3( this, pRecord, ulKey ); + sRc = QDAMSplitNode_V3( this, pRecord, lKey ); if ( !sRc ) { // SplitNode may have passed a new record back @@ -4504,10 +4501,10 @@ SHORT BTREE::QDAMInsertKey_V3 pusOffset = (PUSHORT) pRecord->contents.uchData; while ( i > 0 && !fFound ) { - ulCompKey = QDAMGetszKey_V3( pRecord, (SHORT)(i-1) ); - if ( ulCompKey != -1 ) + lCompKey = QDAMGetszKey_V3( pRecord, (SHORT)(i-1) ); + if ( lCompKey != -1 ) { - if ( compKeys(ulKey, ulCompKey) < 0 ) + if ( compKeys(lKey, lCompKey) < 0 ) { *(pusOffset+i) = *(pusOffset+i-1); i--; @@ -4541,7 +4538,7 @@ SHORT BTREE::QDAMInsertKey_V3 { PBYTE pTarget; pTarget = pData + sizeof(USHORT) + sizeof(RECPARAM); - memcpy(pTarget, &ulKey, usKeyLen ); + memcpy(pTarget, &lKey, usKeyLen ); } // set data information memcpy((PRECPARAM) (pData+sizeof(USHORT)), &recData, sizeof(RECPARAM)); @@ -4562,11 +4559,11 @@ SHORT BTREE::QDAMInsertKey_V3 { if ((i == 0) && (!IS_ROOT(pRecord))) { - ulOldKey = QDAMGetszKey_V3( pRecord, 1 ); - ulNewKey = QDAMGetszKey_V3( pRecord, 0 ); - if ( ulOldKey != -1 && ulNewKey != -1 ) + lOldKey = QDAMGetszKey_V3( pRecord, 1 ); + lNewKey = QDAMGetszKey_V3( pRecord, 0 ); + if ( lOldKey != -1 && lNewKey != -1 ) { - if ( ulCompKey != -1 && PARENT(pRecord) ) + if ( lCompKey != -1 && PARENT(pRecord) ) { /**********************************************************/ /* save old current record since ChangeKey will update it */ @@ -4574,7 +4571,7 @@ SHORT BTREE::QDAMInsertKey_V3 /**********************************************************/ _usCurrentRecord = usCurrentRecord; _sCurrentIndex = sCurrentIndex; - sRc = QDAMChangeKey_V3( this, PARENT(pRecord), ulOldKey, ulNewKey); + sRc = QDAMChangeKey_V3( this, PARENT(pRecord), lOldKey, lNewKey); usCurrentRecord = _usCurrentRecord; sCurrentIndex = _sCurrentIndex; @@ -5094,7 +5091,7 @@ SHORT BTREE::QDAMDictOpenLocal //------------------------------------------------------------------------------ SHORT BTREE::QDAMDictInsertLocal ( - ULONG& ulKey, // pointer to key data + LONG& lKey, // pointer to key data BytesRef pData // pointer to user data ) { @@ -5117,7 +5114,7 @@ SHORT BTREE::QDAMDictInsertLocal /*******************************************************************/ /* check if entry is locked .... */ /*******************************************************************/ - if ( !sRc && QDAMDictLockStatus( ulKey ) ) + if ( !sRc && QDAMDictLockStatus( lKey ) ) { sRc = BTREE_ENTRY_LOCKED; } /* endif */ @@ -5126,7 +5123,7 @@ SHORT BTREE::QDAMDictInsertLocal if ( !sRc ) { QDAMDictUpdStatus (); - sRc = QDAMFindRecord_V3( ulKey, pNode ); + sRc = QDAMFindRecord_V3( lKey, pNode ); } /* endif */ if ( !sRc ) @@ -5139,7 +5136,7 @@ SHORT BTREE::QDAMDictInsertLocal T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << pData.size(); } recData.ulLen = pData.size(); - sRc = QDAMInsertKey_V3 ( pNode, ulKey, recKey, recData); + sRc = QDAMInsertKey_V3 ( pNode, lKey, recKey, recData); } /* endif */ BTREEUNLOCKRECORD( pNode ); /****************************************************************/ @@ -5741,7 +5738,7 @@ SHORT QDAMGetszKeyParam_V3 ( PBTREE pBTIda, // pointer to btree structure RECPARAM recKey, // active record - ULONG& ulKeyData // pointer to data + LONG& lKeyData // pointer to data ) { PCHAR pData = NULL; @@ -5767,11 +5764,11 @@ SHORT QDAMGetszKeyParam_V3 //*pulLen = usLen; pData += sizeof(USHORT ) + sizeof(RECPARAM); // get pointer to data - memcpy( &ulKeyData, pData, usLen ); + memcpy( &lKeyData, pData, usLen ); } else { - ulKeyData = -1; + lKeyData = -1; sRc = BTREE_CORRUPTED; ERREVENT2( QDAMGETSZKEYPARAM_LOC, STATE_EVENT, 1, DB_GROUP, "" ); } @@ -5935,7 +5932,7 @@ SHORT QDAMFirst_V3 SHORT QDAMDictFirstLocal ( PBTREE pBTIda, - ULONG ulKeyData, // pointer to space for key data + LONG& lKeyData, // pointer to space for key data BytesRef pUserData // pointer to space for user data ) { @@ -5944,7 +5941,7 @@ SHORT QDAMDictFirstLocal RECPARAM recKey; // ... for key value RECPARAM recData; // ... for user value PBTREE pBT = pBTIda; - ULONG ulKeyLen; + LONG lKeyLen; memset(&recData, 0, sizeof(recData)); memset(&recKey, 0, sizeof(recKey)); @@ -5966,7 +5963,7 @@ SHORT QDAMDictFirstLocal if ( !sRc ) { - sRc = QDAMGetszKeyParam_V3( pBTIda, recKey, ulKeyData ); + sRc = QDAMGetszKeyParam_V3( pBTIda, recKey, lKeyData ); } if ( !sRc ) @@ -6006,36 +6003,21 @@ USHORT EQFNTMOrganizeIndex ) { SHORT sRc = 0; // function return code - //PBTREE pbTree = (PBTREE)(*ppBTIda); - PCHAR_W pchKeyBuffer = NULL; // buffer for record keys - ULONG ulKeyBufSize = 0; // current size of key buffer (number of characters) + std::vector pchKeyBuffer; // buffer for record keys + ULONG lKeyBufSize = 256; // current size of key buffer (number of characters) std::vector pbData; // buffer for record data BOOL fNewIndexCreated = FALSE; // new-index-has-been-created flag CHAR szNewIndex[MAX_LONGPATH]; // buffer for new index name - PBTREE pBtreeOut = NULL; // structure for output BTREE USHORT usSigLen = 0; // length of signature record - ULONG ulKey; + LONG lKey = -1; // allocate buffer areas pbData.resize(MAX_INDEX_LEN * sizeof(LONG) * 4); - ulKeyBufSize = 256; - if ( !UtlAlloc( (PVOID *)&pchKeyBuffer, 0, ulKeyBufSize*sizeof(CHAR_W) , ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(sRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - - if ( !sRc ) - { - if ( !UtlAlloc( (PVOID *)&pBtreeOut, 0, sizeof(BTREE), ERROR_STORAGE ) ) - { - LOG_AND_SET_RC(sRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - } /* endif */ + pchKeyBuffer.resize(lKeyBufSize*sizeof(CHAR_W)); // get user data (signature record) if ( !sRc ) { - //T5LOG( T5WARNING) << "TEMPORARY HARDCODED EQFNTMOrganizeIndex:: usSigLen = (USHORT)ulDataBufSize => usSigLen = (SHORT)ulDataBufSize"); usSigLen = (USHORT)pbData.size(); sRc = pbTree->QDAMDictSignLocal( (PCHAR)pbData.data(), &usSigLen ); } /* endif */ @@ -6043,15 +6025,9 @@ USHORT EQFNTMOrganizeIndex // check if index is empty if ( !sRc ) { - ULONG ulKeyLen = sizeof(ULONG) + 1; // ulKeyBufSize; - - sRc = QDAMDictFirstLocal( pbTree, ulKey, pbData ); + sRc = QDAMDictFirstLocal( pbTree, lKey, pbData ); } /* endif */ - // cleanup - if ( pchKeyBuffer ) UtlAlloc( (PVOID *)&pchKeyBuffer, 0, 0, NOMSG ); - if ( pBtreeOut ) UtlAlloc( (PVOID *)pBtreeOut, 0, 0, NOMSG ); - // re-map some return codes.. switch ( sRc ) { From 3714050e495177429f031692266f3ab70871e417 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 14 Apr 2025 22:26:05 +0300 Subject: [PATCH 07/62] * refactored TERMLENOFFS lists workflow * big code cleanup --- include/EQF.H | 3 - include/EQFFUZZ.H | 15 +- include/EQFMORPH.H | 31 +- include/EQFTA.H | 13 - include/EQFTP.H | 22 +- include/EQFTPI.H | 4 +- include/lowlevelotmdatastructs.h | 16 +- include/tm.h | 13 +- .../opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp | 374 +--- .../opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp | 2 - .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 2 - .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 334 +-- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 11 +- .../EqfMemoryPlugin/GenericTagReplace.cpp | 1933 +---------------- .../core/EqfMemoryPlugin/TMPluginWrapper.cpp | 22 - source/opentm2/core/document/EQFTATAG.cpp | 23 +- source/opentm2/core/tagtable/EQFTOKEN.cpp | 6 +- source/opentm2/core/utilities/EQFMORPH.cpp | 374 +--- source/opentm2/core/utilities/UtlMisc.cpp | 162 -- 19 files changed, 178 insertions(+), 3182 deletions(-) diff --git a/include/EQF.H b/include/EQF.H index 5c3840a3..b986b938 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -3707,7 +3707,6 @@ SHORT UtlDriveType( CHAR chDrive ); VOID UtlSetHorzScrollingForLB(HWND hLB); VOID UtlSaveWindowPos( HWND hFrame, EQF_PSWP pSwp); -BOOL UtlAllocHwnd ( PVOID *, LONG, LONG, USHORT, HWND ); BOOL UtlInitUtils( HAB ); BOOL UtlRegisterEqfHelp( HAB ); BOOL UtlTerminateUtils( VOID ); @@ -5373,8 +5372,6 @@ ULONG GetOrgLangCP( PSZ pLanguage, BOOL fASCII ); //------------------------------------------------------------------------------ struct TERMLENOFFS { - // int iLength; // length of term - // int iOffset; // offset of term data in buffer USHORT iLength; // length of term USHORT iOffset; // offset of term data in buffer TERMLENOFFS(){iLength = iOffset = 0;} diff --git a/include/EQFFUZZ.H b/include/EQFFUZZ.H index 40adb478..b9b323a1 100755 --- a/include/EQFFUZZ.H +++ b/include/EQFFUZZ.H @@ -1,4 +1,4 @@ - +#include "lowlevelotmdatastructs.h" #ifndef _EQFFUZZ_H_ #define _EQFFUZZ_H_ @@ -31,9 +31,6 @@ typedef struct _LCSTOKEN SHORT sTotalLen; } LCSTOKEN, *PLSCTOKEN; - - - //static SHORT CompFuzzyTok ( PFUZZYTOK , PFUZZYTOK, BOOL ); @@ -43,10 +40,8 @@ SHORT CompFuzzyTok ( PFUZZYTOK , PFUZZYTOK, BOOL ); //static VOID MakeHashValue ( PULONG, USHORT, PSZ_W, PULONG ); - - //static -BOOL PrepareTokens ( PLOADEDTABLE, PBYTE, std::vector&, PSZ_W, SHORT, PFUZZYTOK *, ULONG ); +BOOL PrepareTokens ( PLOADEDTABLE, std::vector&, std::vector&, PSZ_W, SHORT, std::vector &, ULONG ); //static BOOL FuzzyReplace ( PSZ_W, PSZ_W, PSZ_W, PREPLLIST, PREPLLIST ); //static @@ -64,13 +59,11 @@ SHORT FindMiddleSnake( PFUZZYTOK, PFUZZYTOK, PFUZZYTOK, PFUZZYTOK, //static VOID LCS( LCSTOKEN, LCSTOKEN , BOOL); //static -BOOL EQFBTokCountDiff(PFUZZYTOK, PFUZZYTOK, - USHORT, USHORT, PUSHORT); //static - BOOL EQFBMarkModDelIns( PFUZZYTOK, PFUZZYTOK, PFUZZYTOK*, PFUZZYTOK *, + BOOL EQFBMarkModDelIns( std::vector&, std::vector&, std::vector&, std::vector&, USHORT, USHORT); //static -void EQFBSimplifyAndCountMarks (PFUZZYTOK, PUSHORT, PUSHORT); +void EQFBSimplifyAndCountMarks (std::vector&, PUSHORT, PUSHORT); /**********************************************************************/ /* macro to calculate the number of tokens in the list ... */ diff --git a/include/EQFMORPH.H b/include/EQFMORPH.H index 66688efd..3371c24e 100755 --- a/include/EQFMORPH.H +++ b/include/EQFMORPH.H @@ -182,11 +182,7 @@ USHORT MorphTokenizeW ( SHORT sLanguageID, // language ID PSZ_W pszInData, // pointer to input segment - PUSHORT pusBufferSize, // address of variable containing size of - // term list buffer - TERMLENOFFS** ppTermList, // address of caller's term list pointer - USHORT usListType, // type of term list MORPH_ZTERMLIST or - // MORPH_OFFSLIST + std::vector& ppTermList, // address of caller's term list pointer ULONG ulOemCP ); @@ -320,30 +316,11 @@ USHORT MorphAddTermToList2 ); - USHORT MorphAddTermToList2W ( - PSZ_W* ppList, // ptr to term list pointer - PULONG pulSize, // ptr to allocated size of term list - PULONG pulUsed, // ptr to used bytes in term list - PSZ_W pszTerm, // ptr to new term being added to list - USHORT usLength, // length of term - USHORT usOffs, // offset of term - ULONG ulFlags, // flags for the term - USHORT usListType // type of list MORPH_ZTERMLIST or -); - - - -USHORT MorphCopyTermListASCII2Unicode -( - PSZ pList, // ptr to ASCII termlist - USHORT usBufferSize, // size of allocated buffer - PSZ_W *ppListW, // ptr to term list pointer - PULONG pulSize, // ptr to allocated size of term list - PULONG pulUsed, // ptr to used bytes in term list - USHORT usListType, // type of list - ULONG ulOemCP + std::vector& pList, // ptr to term list pointer + int iLength, // length of term in # of w's + int iOffs // offset of term in # of w's??? ); diff --git a/include/EQFTA.H b/include/EQFTA.H index b74669b6..7404b24d 100755 --- a/include/EQFTA.H +++ b/include/EQFTA.H @@ -187,19 +187,6 @@ void TAFillWSList PSZ_W *ppWSList ); - -BOOL TAAdjustWhiteSpace -( - PSZ_W pszSourceSeg, // ptr to source segment data - PSZ_W pszTargetSeg, // ptr to target segment - PSZ_W *ppszNewTargetSeg, // ptr to ptr of output buffer - BOOL fLeadingWS, // process leading whitespace - BOOL fTrailingWS, // process trailing whitespace - PBOOL pfChanged, // TRUE if target has been changed - PSZ_W pWSList // list of whitespace chars (TBX-CLS_WHITESPACEs) -); - - VOID TABuildArchTM( HWND hwnd, PSZ pSelFolderName ); VOID ExportSegs( HWND hwnd, PSZ pSelFolderName ); diff --git a/include/EQFTP.H b/include/EQFTP.H index ae69e144..11e9198f 100755 --- a/include/EQFTP.H +++ b/include/EQFTP.H @@ -982,9 +982,11 @@ VOID EQFBVioSetNewDocSize ( PTBDOCUMENT ); // refresh doc size VOID ImeSetFont( PTBDOCUMENT, HWND, PLOGFONT ); -BOOL EQFBFindDiffEx( PVOID, PBYTE, std::vector&, PSZ_W, PSZ_W, SHORT , PVOID * , PVOID *, ULONG); +BOOL EQFBFindDiffEx( PVOID, std::vector& , std::vector&, + PSZ_W, PSZ_W, SHORT , + std::vector&, std::vector&, ULONG); -BOOL EQFBCountDiff( PVOID pvFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ); +BOOL EQFBCountDiff( std::vector& pvFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ); BOOL EQFBDoEmptySegCheck( PTBDOCUMENT ); // do empty segment handling @@ -1017,28 +1019,12 @@ int BiDitransform -typedef struct _FUZZYTOK -{ - PSZ_W pData; // pointer to data.. - ULONG ulHash; // hash value - USHORT usStart; // start offset - USHORT usStop; // end offset - SHORT sType; // type of entry /* KIT0857M */ - EQF_BOOL fConnected; // true if connected to pPropTokList - EQF_BOOL fNoTagSubst; // true = do not use for tag substitution -} FUZZYTOK, *PFUZZYTOK; - typedef struct _REPLLIST { PFUZZYTOK pSrcTok; // ptr to source token PFUZZYTOK pTgtTok; // ptr to target token } REPLLIST, *PREPLLIST; - - -USHORT FuzzyLCSReplList ( PFUZZYTOK, PFUZZYTOK, PREPLLIST *, - USHORT, USHORT, PUSHORT, BOOL); - SHORT StrNLFCmp ( PSZ_W, PSZ_W ); diff --git a/include/EQFTPI.H b/include/EQFTPI.H index 7e5ff5df..a7d90ced 100755 --- a/include/EQFTPI.H +++ b/include/EQFTPI.H @@ -2301,12 +2301,10 @@ BOOL EQFBDoCopyData ( PTBDOCUMENT,PSZ_W ); ( (c == LF) || \ ( (c == SOFTLF_CHAR) && (c2 != SOFTLF_CHAR) && (c0 != SOFTLF_CHAR)) ) -BOOL EQFBCallLCS(PFUZZYTOK, PFUZZYTOK, USHORT, USHORT, BOOL); - +BOOL EQFBCallLCS(std::vector&, std::vector&, USHORT, USHORT, BOOL); USHORT EQFBMisspelledHLType ( PTBDOCUMENT, PTBSEGMENT, PSZ_W ); - /**********************************************************************/ /* worker thread defines */ /**********************************************************************/ diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 4902c218..9f8b4b78 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -531,7 +531,6 @@ struct TMX_LONGNAME_TABLE }; BOOL UtlAlloc ( void **, long, long, unsigned short ); -BOOL UtlAllocHwnd ( void**, long, long, unsigned short, void * ); #include "Filebuffer.h" #include @@ -2281,8 +2280,8 @@ struct LOADEDTABLE CHAR szDescription[MAX_DESCRIPTION]; // format description USHORT usCharacterSet; // characterset to use for import/export BOOL fUseUnicodeForSegFile; // TRUE = store segmented files in Unicode format - PSZ_W pTagNamesW; - PSZ_W pAttributesW; + std::wstring pTagNamesW; + std::wstring pAttributesW; PVOID pTagTableW; PFN pfnCompareContext; // compare context info function BOOL fCompareContextLoadFailed; // TRUE = CompareContext function load failed @@ -2819,4 +2818,15 @@ enum FilterField{ */ int convertTimeToUTC( long lTime, char *pszDateTime ); +typedef struct _FUZZYTOK +{ + PSZ_W pData; // pointer to data.. + ULONG ulHash; // hash value + USHORT usStart; // start offset + USHORT usStop; // end offset + SHORT sType; // type of entry /* KIT0857M */ + EQF_BOOL fConnected; // true if connected to pPropTokList + EQF_BOOL fNoTagSubst; // true = do not use for tag substitution +} FUZZYTOK, *PFUZZYTOK; + #endif //_LOW_LEVEL_OTM_DATA_STRUCTS_INCLUDED_ diff --git a/include/tm.h b/include/tm.h index 22c430fa..1ec1d5ac 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1612,9 +1612,6 @@ typedef struct _TMX_SUBSTPROP PBYTE pTokSource; // token buffer for source tokens PBYTE pTokPropSource; // token buffer for proposal tokens PBYTE pTokPropTarget; // token buffer for proposal target .. - // PTMX_TAGTABLE_RECORD pTagsSource; // tag table record for source .. - // PTMX_TAGTABLE_RECORD pTagsPropSource; // tag table record for prop source - // PTMX_TAGTABLE_RECORD pTagsPropTarget; // tag table record for prop target CHAR_W chBuffer[MAX_SEGMENT_SIZE * 2]; // generic token buffer USHORT usTokenSource; // number of source tokens USHORT usTokenPropSource; // number of prop source tokens @@ -2589,15 +2586,11 @@ USHORT MorphAddTermToList2W( ULONG ulFlags // flags for the ter ); -BOOL NTMTagSubst // generic inline tagging for TM - ( - PTMX_SUBSTPROP pSubstProp, - ULONG ulSrcOemCP, - ULONG ulTgtOemCP); + BOOL NTMDocMatch(PSZ pszShort1, PSZ pszLong1, PSZ pszShort2, PSZ pszLong2); -void NTMFreeSubstProp(PTMX_SUBSTPROP pSubstProp); +void NTMFreeSubstProp(TMX_SUBSTPROP& SubstProp); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /*! tm.h Internal header file for Translation Memory functions @@ -3544,8 +3537,6 @@ USHORT GetExactMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, USHORT ExactTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_SENTENCE, PTMX_MATCH_TABLE_W, PUSHORT, LONG); -BOOL AddTagsToString(PSZ, PULONG, PTMX_TAGTABLE_RECORD, PSZ); -BOOL AddTagsToStringW(PSZ_W, PLONG, PTMX_TAGTABLE_RECORD, PSZ_W); INT CompCount(const void *, const void *); INT CompCountVotes(const void *, const void *); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp index 9eeeea81..7f296155 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp @@ -159,11 +159,11 @@ BOOL PrepareTokens ( PLOADEDTABLE pTagTable, - PBYTE pInBuf, + std::vector& pInBuf, std::vector& pTokBuf, PSZ_W pString, // pointer to string to be tokenized SHORT sLanguageId, // language ID - PFUZZYTOK * ppTransTokList, // resulting list of tokens + std::vector& pTransTokList, // resulting list of tokens ULONG ulOemCP ) { @@ -176,8 +176,7 @@ PrepareTokens CHAR_W chTemp; // temp character BOOL fOK = TRUE; // success indicator USHORT usRandomIndex; // index in random sequence - USHORT usListSize; // size of buffer - TERMLENOFFS* pTermList; // ptr to created term list + std::vector pTermList; // ptr to created term list TERMLENOFFS* pActTerm; // actual term USHORT usRC; // return code SHORT sNumTags; // number of tags /* KIT0857A */ @@ -203,29 +202,26 @@ PrepareTokens /* Rational: use input buffer for temporary list .... */ /* this is large enough, we can avoid any checking... */ /********************************************************************/ - pstCurrent = (PFUZZYTOK) pInBuf; // use input buffer for table + pstCurrent = pInBuf.data(); // use input buffer for table pTok = (PTOKENENTRY) pTokBuf.data(); pStart = pTok->pDataStringW; while ( (pTok->sTokenid != ENDOFLIST) ) { if ( pTok->sTokenid == TEXT_TOKEN ) { - usListSize = 0; - pTermList = NULL; - chTemp = *(pTok->pDataStringW+pTok->iLength); *(pTok->pDataStringW + pTok->iLength) = EOS; usRC = MorphTokenizeW( sLanguageId, pTok->pDataStringW, - &usListSize, &pTermList, - MORPH_OFFSLIST, ulOemCP ); + pTermList, + ulOemCP ); *(pTok->pDataStringW + pTok->iLength) = chTemp; - if ( pTermList ) + if ( pTermList.size() ) { - pActTerm = pTermList; - if ( pTermList->iLength ) + pActTerm = pTermList.data(); + if ( pActTerm->iLength ) { while ( pActTerm->iLength ) { @@ -254,10 +250,6 @@ PrepareTokens } /* endif */ } /* endif */ - /****************************************************************/ - /* free allocated resource ... */ - /****************************************************************/ - UtlAlloc( (PVOID *)&pTermList, 0L, 0L, NOMSG ); } else { @@ -289,7 +281,7 @@ PrepareTokens /* pstAct -- points to active one */ /* pstCurrent -- points to next one... */ /********************************************************************/ - pstCurrent = (PFUZZYTOK) pInBuf; // use input buffer for table + pstCurrent = pInBuf.data(); // use input buffer for table pstAct = pstCurrent; /********************************************************************/ /* set tag indication */ @@ -336,17 +328,7 @@ PrepareTokens /********************************************************************/ pstAct++; - /********************************************************************/ - /* allocate memory for start/stop table and copy list into it */ - /********************************************************************/ - *ppTransTokList = NULL; // init token list.. - ulLength = (ULONG)(((PBYTE) pstAct) - pInBuf); - - fOK = UtlAlloc( (PVOID *)ppTransTokList, 0L, ulLength, ERROR_STORAGE ); - if ( fOK ) - { - memcpy( *ppTransTokList, pInBuf, ulLength ); - } /* endif */ + pTransTokList = pInBuf; return( fOK ); } /* end of function @@ -1122,12 +1104,12 @@ TokStrCompare // in a fuzzy token list created using EQFBFindDiffEx BOOL EQFBCountDiff ( - PVOID pvFuzzyTok, + std::vector& pFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ) { - PFUZZYTOK pTest = (PFUZZYTOK)pvFuzzyTok; + PFUZZYTOK pTest = pFuzzyTok.data(); PFUZZYTOK pTestToken; // pointer to token lists PFUZZYTOK pToken; // pointer to token lists USHORT usLen = 0; @@ -1139,8 +1121,8 @@ BOOL EQFBCountDiff /* MODIFIED DELETED is changed to MODIFIED */ /* MODIFIED INSERTED is changed to MODIFIED MODIFIED */ /******************************************************************/ - pTestToken = (PFUZZYTOK)pvFuzzyTok; //resulting pointers - pToken = (PFUZZYTOK)pvFuzzyTok; //current pointers + pTestToken = pFuzzyTok.data(); //resulting pointers + pToken = pFuzzyTok.data(); //current pointers usPrevType = MARK_EQUAL; while ( pToken->ulHash ) @@ -1201,52 +1183,46 @@ BOOL EQFBFindDiffEx ( PVOID pTable, // pointer to loaded tagtable - PBYTE pInBuf, // pointer to input buffer + std::vector& pInBuf, // pointer to input buffer std::vector& pTokBuf, // pointer to temp token buffer PSZ_W pString1, // first string passed PSZ_W pString2, // second string SHORT sLanguageId, // language id to be used for pString1 - PVOID * ppFuzzyTok, // returned token list of pString2 - PVOID * ppFuzzyTgt, // returned token list of pString1 + std::vector& pFuzzyTok, // returned token list of pString2 + std::vector& pFuzzyTgt, // returned token list of pString1 ULONG ulOemCP ) { BOOL fOK = TRUE; // success indicator USHORT usLenStr1 = 0; // length of string 1 (in num. of tokens) USHORT usLenStr2 = 0; // length of string 2 (in num. of tokens) - PFUZZYTOK pTokenList2 = NULL; // pointer to token lists - PFUZZYTOK pTokenList1 = NULL; // pointer to token lists + std::vector pTokenList2, pTokenList1; // pointer to token lists PFUZZYTOK pToken; // pointer to token lists PLOADEDTABLE pTagTable = (PLOADEDTABLE) pTable; /******************************************************************/ /* prepare tokens for String1 and string 2 */ /******************************************************************/ - T5LOG( T5DEVELOP) <<"EQFBFindDiffEx::Preparing tokens for first string: "; - //std::wstring ws1 = removeTagsFromString(pString1); fOK = PrepareTokens( pTagTable, pInBuf, pTokBuf, pString1, - sLanguageId, &pTokenList1, ulOemCP ); + sLanguageId, pTokenList1, ulOemCP ); if ( fOK ) { - - T5LOG( T5DEVELOP) <<"EQFBFindDiffEx::Preparing tokens for second string: "; - //std::wstring ws2 = removeTagsFromString(pString2); fOK = PrepareTokens( pTagTable, pInBuf, pTokBuf, pString2, - sLanguageId, &pTokenList2, ulOemCP ); + sLanguageId, pTokenList2, ulOemCP ); } /* endif */ if (fOK ) { /********************************************************************/ /* get number of tokens in strings */ /********************************************************************/ - NUMBEROFTOKENS(usLenStr1, pTokenList1); - NUMBEROFTOKENS(usLenStr2, pTokenList2); + NUMBEROFTOKENS(usLenStr1, pTokenList1.data()); + NUMBEROFTOKENS(usLenStr2, pTokenList2.data()); /******************************************************************/ /* call LCS and compare tokens with strncmp */ @@ -1262,7 +1238,7 @@ EQFBFindDiffEx /****************************************************************/ /* adjust length to avoid non-marked spaces between marked tokens*/ /****************************************************************/ - pToken = pTokenList1; //current pointers + pToken = pTokenList1.data(); //current pointers while ( pToken->ulHash ) { if ( (pToken->usStop < (pToken+1)->usStart) ) @@ -1272,7 +1248,7 @@ EQFBFindDiffEx pToken++; } /* endwhile */ - pToken = pTokenList2; //current pointers + pToken = pTokenList2.data(); //current pointers while ( pToken->ulHash ) { if ( (pToken->usStop < (pToken+1)->usStart) ) @@ -1286,15 +1262,10 @@ EQFBFindDiffEx /* this area MUST be freed by the calling application */ /****************************************************************/ fOK = EQFBMarkModDelIns( pTokenList1, pTokenList2, - (PFUZZYTOK *)ppFuzzyTgt, (PFUZZYTOK *)ppFuzzyTok, + pFuzzyTgt, pFuzzyTok, usLenStr1, usLenStr2); } /* endif */ - /********************************************************************/ - /* free any allocated resources */ - /********************************************************************/ - UtlAlloc( (PVOID *)&pTokenList1, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID *)&pTokenList2, 0L, 0L, NOMSG ); return( fOK ); } /* end of function EQFBFindDiffEx */ @@ -1332,10 +1303,10 @@ EQFBFindDiffEx BOOL EQFBMarkModDelIns ( - PFUZZYTOK pTokenList1, - PFUZZYTOK pTokenList2, - PFUZZYTOK * ppFuzzy1, - PFUZZYTOK * ppFuzzy2, + std::vector& pTokenList1, + std::vector& pTokenList2, + std::vector& pFuzzy1, + std::vector& pFuzzy2, USHORT usLenStr1, USHORT usLenStr2 ) @@ -1347,49 +1318,46 @@ EQFBMarkModDelIns PFUZZYTOK pT1Token; // pointer to token lists 1 LONG lTokBufLen = (LONG)(usLenStr1 + usLenStr2 + 1); - LONG lTokBufSize = lTokBufLen * sizeof(FUZZYTOK); + pFuzzy2.resize(lTokBufLen); + pFuzzy1.resize(lTokBufLen); LONG lTokBufUsed = 0; - fOK = UtlAlloc( (PVOID *)ppFuzzy2, 0L, lTokBufSize, ERROR_STORAGE ); - if (fOK ) fOK = UtlAlloc( (PVOID *)ppFuzzy1, 0L, lTokBufSize, ERROR_STORAGE ); - - if ( fOK ) { - pTestToken = *ppFuzzy2; - pT1Token = *ppFuzzy1; + pTestToken = pFuzzy2.data(); + pT1Token = pFuzzy1.data(); usStringA = usStringB = 0; while ((usStringA < usLenStr1) && (usStringB < usLenStr2) ) { - if ( ((pTokenList2+usStringB)->sType == MARK_EQUAL) - && ((pTokenList1+usStringA)->sType == MARK_EQUAL) ) + if ( (pTokenList2[usStringB].sType == MARK_EQUAL) + && (pTokenList1[usStringA].sType == MARK_EQUAL) ) { - *pTestToken = *(pTokenList2+usStringB); + *pTestToken = pTokenList2[usStringB]; pTestToken++; lTokBufUsed++; - *pT1Token = *(pTokenList1+usStringA); + *pT1Token = pTokenList1[usStringA]; pT1Token++; usStringA++; usStringB++; } - else if ((pTokenList1+usStringA)->sType == MARK_EQUAL) + else if (pTokenList1[usStringA].sType == MARK_EQUAL) { - *pTestToken = *(pTokenList2+usStringB); + *pTestToken = pTokenList2[usStringB]; pTestToken->sType = MARK_INSERTED; pTestToken++; lTokBufUsed++; - *pT1Token = *(pTokenList2+usStringB); + *pT1Token = pTokenList2[usStringB]; pT1Token->sType = MARK_DELETED; pT1Token++; usStringB++; } - else if ((pTokenList2+usStringB)->sType == MARK_EQUAL) + else if (pTokenList2[usStringB].sType == MARK_EQUAL) { - *pTestToken = *(pTokenList1+usStringA); + *pTestToken = pTokenList1[usStringA]; pTestToken->sType = MARK_DELETED; pTestToken++; lTokBufUsed++; - *pT1Token = *(pTokenList1+usStringA); + *pT1Token = pTokenList1[usStringA]; pT1Token->sType = MARK_INSERTED; pT1Token++; @@ -1397,11 +1365,11 @@ EQFBMarkModDelIns } else { - *pTestToken = *(pTokenList2+usStringB); + *pTestToken = pTokenList2[usStringB]; pTestToken->sType = MARK_MODIFIED; pTestToken++; lTokBufUsed++; - *pT1Token = *(pTokenList1+usStringA); + *pT1Token = pTokenList1[usStringA]; pT1Token->sType = MARK_MODIFIED; pT1Token++; @@ -1412,11 +1380,11 @@ EQFBMarkModDelIns while ( usStringA < usLenStr1 ) { - *pTestToken = *(pTokenList1+usStringA); + *pTestToken = pTokenList1[usStringA]; pTestToken->sType = MARK_DELETED; pTestToken++; lTokBufUsed++; - *pT1Token = *(pTokenList1+usStringA); + *pT1Token = pTokenList1[usStringA]; pT1Token->sType = MARK_INSERTED; pT1Token++; @@ -1424,12 +1392,12 @@ EQFBMarkModDelIns } /* endwhile */ while ( usStringB < usLenStr2 ) { - *pTestToken = *(pTokenList2+usStringB); + *pTestToken = pTokenList2[usStringB]; pTestToken->sType = MARK_INSERTED; pTestToken++; lTokBufUsed++; - *pT1Token = *(pTokenList2+usStringB); + *pT1Token = pTokenList2[usStringB]; pT1Token->sType = MARK_DELETED; pT1Token++; usStringB++; @@ -2026,129 +1994,6 @@ LCS } /* end of function LCS */ -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: FuzzyLCSReplList | -//+----------------------------------------------------------------------------+ -//|Function call: FuzzyLCSReplList( PFUZZYTOK,PFUZZYTOK,PREPLLIST) | -//+----------------------------------------------------------------------------+ -//|Description: Will return a list of possible tokens to be replaced | -//| in a fuzzy match | -//+----------------------------------------------------------------------------+ -//|Parameters: PFUZZYTOK pPropTokList, pointer to token list | -//| PFUZZYTOK pTransTokList, pointer to token list | -//| PREPLLIST * ppReplaceList pointer to replace list | -//+----------------------------------------------------------------------------+ -//|Returncode type: USHORT | -//+----------------------------------------------------------------------------+ -//|Returncodes: 1 .. n number of equal tokens | -//| 0 error during allocation | -//+----------------------------------------------------------------------------+ -//|Function flow: set up pointer array | -//| return success indicator | -//+----------------------------------------------------------------------------+ - -USHORT -FuzzyLCSReplList -( - PFUZZYTOK pTokList1, // ptr to token list->TgtTok of ReplList - PFUZZYTOK pTokList2, // ptr to token list->SrcTOk ofReplList - PREPLLIST * ppReplaceList, // pointer to replace list - USHORT usLenStr1, // num of tokens in TokList1 - USHORT usLenStr2, // num of tokens in TokList2 - PUSHORT pusFuzzy, - BOOL fCompareAll -) -{ - PREPLLIST pReplaceList = NULL; // pointer to replace list - BOOL fOK = TRUE; // success indicator - USHORT usMinEqual = 0; // no of equal but not-connected tokens - USHORT usI = 0; - USHORT usStringA; - USHORT usStringB; - - /********************************************************************/ - /* call LCS and use TOkStrCompare so that tokens are compared */ - /* without regard of punctuation */ - /********************************************************************/ - fOK = EQFBCallLCS(pTokList1, pTokList2, usLenStr1, usLenStr2, fCompareAll); - - /********************************************************************/ - /* count equal and different words; nether change pTokList1/2 */ - /* since it is needed later on too */ - /********************************************************************/ - if (fOK && pusFuzzy ) - { - fOK = EQFBTokCountDiff(pTokList1, pTokList2, usLenStr1, usLenStr2, pusFuzzy); - } /* endif */ - /*****************************************************************/ - /* build ReplaceList */ - /*****************************************************************/ - if (fOK ) - { - if ( !pReplaceList ) - { - if ( ! UtlAlloc( (PVOID *)ppReplaceList, - 0L, (LONG)( (MAX_REPL+1) * sizeof(REPLLIST) ), - ERROR_STORAGE ) ) - { /************************************************************/ - /* error happened - reset pointer to stop loop */ - /************************************************************/ - usI = MAX_REPL; - fOK = FALSE; - } - else - { - pReplaceList = *ppReplaceList; - } /* endif */ - } /* endif */ - - usStringA = usStringB = 0; - while ((usStringA < usLenStr1) && (usStringB < usLenStr2) ) - { - if ( ((pTokList2+usStringB)->sType == MARK_EQUAL) - && ((pTokList1+usStringA)->sType == MARK_EQUAL) ) - { - if ( usMinEqual < MAX_REPL-1 ) - { - pReplaceList->pSrcTok = pTokList1 + usStringA; // store values - pReplaceList->pTgtTok = pTokList2 + usStringB; - (pTokList1+usStringA)->fConnected = TRUE; - (pTokList2+usStringB)->fConnected = TRUE; - pReplaceList++; - usMinEqual++; - } - else - { - usMinEqual++; - } /* endif */ - - usStringA++; usStringB++; - } - else if ((pTokList1+usStringA)->sType == MARK_EQUAL) - { - usStringB++; - } - else if ((pTokList2+usStringB)->sType == MARK_EQUAL) - { - usStringA++; - } - else - { - usStringA++; - usStringB++; - } /* endif */ - } /* endwhile */ - } /* endif */ - - if ( !fOK ) - { - usMinEqual = 0; - } /* endif */ // tokens - return usMinEqual; -} /* end of function FuzzyLCSReplList */ - //+----------------------------------------------------------------------------+ //|Internal function | @@ -2174,8 +2019,8 @@ FuzzyLCSReplList //+----------------------------------------------------------------------------+ BOOL EQFBCallLCS ( - PFUZZYTOK pTokenList1, // ptr to token list1 - PFUZZYTOK pTokenList2, // ptr to token list2 + std::vector& pTokenList1, // ptr to token list1 + std::vector& pTokenList2, // ptr to token list2 USHORT usLenStr1, USHORT usLenStr2, BOOL fCompareAll @@ -2185,61 +2030,47 @@ BOOL EQFBCallLCS PFUZZYTOK pTestToken; LCSTOKEN LCSString1; LCSTOKEN LCSString2; - PFUZZYTOK pBackList1; - PFUZZYTOK pBackList2 = NULL; + std::vector pBackList1; + std::vector pBackList2; SHORT k; - - - fOK = UtlAlloc( (PVOID *)&pBackList1, - 0L, - (LONG) (usLenStr1 +1) * sizeof(FUZZYTOK), - ERROR_STORAGE ); - if (fOK ) - { - fOK = UtlAlloc( (PVOID *)&pBackList2, - 0L, - (LONG) (usLenStr2 +1) * sizeof(FUZZYTOK), - ERROR_STORAGE ); - } /* endif */ - if ( fOK ) + pBackList1.resize(usLenStr1+1); + pBackList2.resize(usLenStr2+1); + { /******************************************************************/ /* fill backward tokenlists */ /******************************************************************/ - pTestToken = pTokenList1; + pTestToken = pTokenList1.data(); for ( k=(usLenStr1-1); k >= 0 ; k-- ) { - *(pBackList1+k) = *pTestToken; + pBackList1[k] = *pTestToken; pTestToken++; } /* endfor */ - pTestToken = pTokenList2; + pTestToken = pTokenList2.data(); for ( k = (usLenStr2-1); k >= 0 ; k-- ) { - *(pBackList2+k) = *pTestToken; + pBackList2[k] = *pTestToken; pTestToken++; } /* endfor */ /******************************************************************/ /* call recursive function to find shortest edit script */ /******************************************************************/ - LCSString1.pTokenList = pTokenList1; - LCSString1.pBackList = pBackList1; + LCSString1.pTokenList = pTokenList1.data(); + LCSString1.pBackList = pBackList1.data(); LCSString1.sStart = 0; LCSString1.sStop = usLenStr1; LCSString1.sTotalLen = usLenStr1; - LCSString2.pTokenList = pTokenList2; - LCSString2.pBackList = pBackList2; + LCSString2.pTokenList = pTokenList2.data(); + LCSString2.pBackList = pBackList2.data(); LCSString2.sStart = 0; LCSString2.sStop = usLenStr2; LCSString2.sTotalLen = usLenStr2; LCS ( LCSString1,LCSString2, fCompareAll); - UtlAlloc( (PVOID *)&pBackList1, 0L, 0L, NOMSG); - UtlAlloc( (PVOID *)&pBackList2, 0L, 0L, NOMSG); - } /* endif */ return fOK; } /* end of function EQFBCallLCS */ @@ -2269,49 +2100,6 @@ BOOL EQFBCallLCS //| return success indicator | //+----------------------------------------------------------------------------+ //static -BOOL EQFBTokCountDiff -( - PFUZZYTOK pTokenList1, // ptr to token list1 - PFUZZYTOK pTokenList2, // ptr to token list2 - USHORT usLenStr1, - USHORT usLenStr2, - PUSHORT pusFuzzy -) -{ - PFUZZYTOK pFuzzy1 = NULL; // ret. tokenlist of Strin2 - PFUZZYTOK pFuzzy2 = NULL; // ret. tokenlist of string 1 - USHORT usDiff = 0; // different words in String2 - USHORT usWords = 0; // total words in String2 - BOOL fOK; - - fOK = EQFBMarkModDelIns( pTokenList1, pTokenList2, - &pFuzzy1, &pFuzzy2,usLenStr1, usLenStr2); - - if (fOK ) - { - /******************************************************************/ - /* usWords : total words in combined string */ - /* usDiff: # of words not MARK_EQUAL in combined string */ - /******************************************************************/ - EQFBSimplifyAndCountMarks(pFuzzy2, &usDiff, &usWords); - } /* endif */ - - - UtlAlloc ((PVOID *) &pFuzzy1, 0L, 0L, NOMSG); - UtlAlloc ((PVOID *) &pFuzzy2, 0L, 0L, NOMSG); - - if (usDiff < usWords ) - { - *pusFuzzy = (usWords != 0) ? ((usWords - usDiff)*100 / usWords) : 100; - } - else - { - *pusFuzzy = 0; - } /* endif */ - - return (fOK); -} /* end of function EQFBTokCountDiff */ - //+----------------------------------------------------------------------------+ //|Internal function | @@ -2339,7 +2127,7 @@ BOOL EQFBTokCountDiff //static void EQFBSimplifyAndCountMarks ( - PFUZZYTOK pFuzzy, // ptr to token + std::vector& pFuzzy, // ptr to token PUSHORT pusDiff, // # of different words PUSHORT pusWords // total # of words ) @@ -2359,8 +2147,8 @@ void EQFBSimplifyAndCountMarks /* count all words and count all words which are not equal */ /*******************************************************************/ usDiff = 0; - pTestToken = pFuzzy; //resulting pointers - pToken = pFuzzy; //current pointers + pTestToken = pFuzzy.data(); //resulting pointers + pToken = pFuzzy.data(); //current pointers usPrevType = MARK_EQUAL; while ( pToken->ulHash ) @@ -2460,8 +2248,7 @@ NTMCompareBetweenTokens { PLOADEDTABLE pTable = NULL; // ptr to loaded tag table BOOL fOK = TRUE; // success indicator - PFUZZYTOK pTokenList2 = NULL; // pointer to token lists - PFUZZYTOK pTokenList1 = NULL; // pointer to token lists + std::vector pTokenList1, pTokenList2; // pointer to token lists PFUZZYTOK pstTok1; // pointer to token lists PFUZZYTOK pstTok2; // pointer to token lists PFUZZYTOK pstNextTok1; @@ -2469,7 +2256,7 @@ NTMCompareBetweenTokens SHORT sNumTags = 0; BOOL fStringEqual = TRUE; CHAR_W chW; - std::vector pInBufL; + std::vector pInBufL; std::vector pTokBuf; USHORT usI = 0; BOOL fTokIsTag = FALSE; @@ -2480,7 +2267,7 @@ NTMCompareBetweenTokens PSZ_W pTemp = NULL; // allocate required buffers - pInBufL.resize(50000); + pInBufL.resize(64000/sizeof(pInBufL[0])); pTokBuf.reserve(TOK_BUFFER_SIZE/sizeof(TOKENENTRY)); fOK = true; @@ -2505,16 +2292,16 @@ NTMCompareBetweenTokens /* prepare tokens for String1 and string 2 */ /******************************************************************/ fOK = PrepareTokens( pTable, - pInBufL.data(), + pInBufL, pTokBuf, - pD1, sLangID, &pTokenList1, ulSrcCP ); + pD1, sLangID, pTokenList1, ulSrcCP ); } if ( fOK) { fOK = PrepareTokens( pTable, - pInBufL.data(), + pInBufL, pTokBuf, - pD2, sLangID, &pTokenList2, ulSrcCP ); + pD2, sLangID, pTokenList2, ulSrcCP ); } // prereq is FUzzyness= 100%, i.e. all texttokens have been MARK_EQUAL in TMFuzzyness!! // check whether all tokens ( text + tag) are in both lists! @@ -2522,8 +2309,8 @@ NTMCompareBetweenTokens if (fOK) { sNumTags = (SHORT) pTable->pTagTable->uNumTags; - pstTok1 = pTokenList1; - pstTok2 = pTokenList2; + pstTok1 = pTokenList1.data(); + pstTok2 = pTokenList2.data(); while (fStringEqual && pstTok1->ulHash && pstTok2->ulHash) { @@ -2587,8 +2374,8 @@ NTMCompareBetweenTokens // Now we have to take care of the spaces between the texttokens and TagTokens! if ( fStringEqual ) { - pstTok1 = pTokenList1; - pstTok2 = pTokenList2; + pstTok1 = pTokenList1.data(); + pstTok2 = pTokenList2.data(); pstNextTok1 = pstTok1 + 1; pstNextTok2 = pstTok2 + 1; while ( pstTok1->ulHash && pstNextTok1->ulHash && @@ -2646,9 +2433,6 @@ NTMCompareBetweenTokens } /* endwhile */ } /* endif */ } /* endif if fOK */ - - if (pTokenList1) UtlAlloc( (PVOID *)&pTokenList1, 0L, 0L, NOMSG ); - if (pTokenList2) UtlAlloc( (PVOID *)&pTokenList2, 0L, 0L, NOMSG ); if ( pTable ) TAFreeTagTable( pTable); *pfStringEqual = fStringEqual; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp index fe234982..476cd919 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp @@ -19,8 +19,6 @@ //| "Q"=Quick-and-dirty ) | //+----------------------------------------------------------------------------+ //|Externals: CloseFile | -//| UtlDelete | -//| UtlAlloc | //| UtlDispatch | //| MemGetAddressOfProcessIDA | //| MemCreateProcess | diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 21f3f219..d331d9d6 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -20,8 +20,6 @@ //| "Q"=Quick-and-dirty ) | //+----------------------------------------------------------------------------+ //|Externals: | -//| | -//| UtlAlloc | //| UtlQFileInfo | //| UtlMakeEQFPath | //| UtlLoadFile | diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 3b41ad61..20420662 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -102,85 +102,6 @@ MakeHashValueW ); -static -BOOL NTMPrepareTokens -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pData, - PFUZZYTOK *ppTokData, - PUSHORT pusTokens, - PTMX_TAGTABLE_RECORD pTagRecord, - SHORT sLangID, - ULONG ulOemCP, - PLOADEDTABLE pTagTable -); - -static BOOL -NTMFuzzyReplace -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pSource, // source string - PSZ_W pProp, // proposal string - PSZ_W pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -); - -#ifdef ACTIVATE_NTMGenericDelete -static BOOL -NTMGenericDelete -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pSource, // source string - PSZ_W pProp, // proposal string - PSZ_W pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -); -#endif - -static BOOL NTMCheckTagPairs -( - PTMX_SUBSTPROP pSubstProp, - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList, // list of tokens to be replaced - BOOL fRespectLFs -); - - -static PTMX_REPLTAGPAIR -NTMFindTagPair -( - PFUZZYTOK pTempTok, - PSZ_W pTempTokData, - PTMX_SUBSTPROP pSubstProp, - PTMX_REPLTAGPAIR pCurTagPair, - BOOL fRespectLFs -); - - -static BOOL -NTMReplaceTags -( - PTMX_SUBSTPROP pSubstProp, - BOOL fRespectLFs -); - -static BOOL -NTMCopyTokData -( - PSZ_W pTempData, - SHORT sLen, - PSZ_W * ppNewData, - PLONG plNewLen -); - - -static PFUZZYTOK -NTMSplitAndAddTokens ( PTMX_SUBSTPROP, - PFUZZYTOK, PUSHORT, USHORT, SHORT, - USHORT, PSZ_W, SHORT, ULONG, PLOADEDTABLE ); - USHORT NTMCompareContext ( EqfMemory* pTmClb, // ptr to ctl block struct @@ -190,16 +111,6 @@ USHORT NTMCompareContext ); -//USHORT FillMatchEntryEx -//( -// EqfMemory* pTmClb, -// PTMX_SENTENCE pSentence, -// PTMX_MATCHENTRY pMatchEntry, -// PUSHORT pusMatchThreshold, -// PTMX_INDEX_RECORD pIndexRecord -//); - - //typedef USHORT (APIENTRY *PFNCOMPCONTEXT)( PSZ_W, PSZ_W, PUSHORT ); typedef USHORT ( *PFNCOMPCONTEXT)( PSZ_W, PSZ_W, PUSHORT ); @@ -246,18 +157,6 @@ static OtmProposal::eProposalType GetProposalTypeFromFlag( USHORT usTranslationF void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); -void NTMMakeTagTablename( PSZ pszMarkup, PSZ pszTagTableName ); - -BOOL NTMAlignTags( PFUZZYTOK pTokSource, PFUZZYTOK pTokTarget, PREPLLIST *ppReplaceList ); - -static BOOL NTMCheckAndDeleteTagPairs -( - PTMX_SUBSTPROP pSubstProp, - PREPLLIST pReplaceSourceList, // list of same tokens in source and prop - PREPLLIST pReplaceList, // toklist to be replaced in prop src+tgt - BOOL fRespectLFs -); - //+----------------------------------------------------------------------------+ //|External function | @@ -1563,10 +1462,7 @@ BOOL TMFuzzynessEx ) { BOOL fOK; - PFUZZYTOK pFuzzyTgt = NULL; - PFUZZYTOK pFuzzyTok = NULL; // returned token list - PSZ pInBuf = NULL; // buffer for EQFBFindDiffEx function - //PSZ pTokBuf = NULL; // buffer for EQFBFindDiffEx function + std::vector pFuzzyTok,pFuzzyTgt,pInBuf; // returned token list std::vector pTokBuf; PLOADEDTABLE pTable = NULL; // ptr to loaded tag table @@ -1590,7 +1486,7 @@ BOOL TMFuzzynessEx } /* endif */ // allocate required buffers - fOK = UtlAlloc((PVOID *)&pInBuf, 0L, 64000L, NOMSG ); + pInBuf.resize(64000/sizeof(pInBuf[0])); // load tag table if ( fOK ) @@ -1603,9 +1499,9 @@ BOOL TMFuzzynessEx // call function to evaluate the differences if ( fOK ) { - fOK = EQFBFindDiffEx( pTable, (PBYTE)pInBuf, pTokBuf, pszSource, - pszMatch, sLanguageId, (PVOID *)&pFuzzyTok, - (PVOID *)&pFuzzyTgt, ulOemCP ); + fOK = EQFBFindDiffEx( pTable, pInBuf, pTokBuf, pszSource, + pszMatch, sLanguageId, pFuzzyTok, + pFuzzyTgt, ulOemCP ); } /* endif */ if ( fOK ) @@ -1614,9 +1510,6 @@ BOOL TMFuzzynessEx } /* endif */ // free allocated buffers and lists - if ( pFuzzyTgt ) UtlAlloc( (PVOID *)&pFuzzyTgt, 0L, 0L, NOMSG ); - if ( pFuzzyTok ) UtlAlloc( (PVOID *)&pFuzzyTok, 0L, 0L, NOMSG ); - if ( pInBuf ) UtlAlloc( (PVOID *)&pInBuf, 0L, 0L, NOMSG ); if ( pTable ) TAFreeTagTable( pTable ); if(*pusWords > *pusDiffs){ @@ -1692,7 +1585,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct std::wstring pString; //pointer to temp string USHORT usCurrentEntry; //counter BOOL fFound = FALSE; //indicates if correct pos found - PTMX_SUBSTPROP pSubstProp = NULL; // tag substitution array + TMX_SUBSTPROP SubstProp; // tag substitution array BOOL fSubstAll = FALSE; PTMX_MATCH_TABLE_W pstMatchTableStart = pstMatchTable; //point to start of match table ULONG ulSrcOemCP = 0L; @@ -1711,28 +1604,20 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /********************************************************************/ if ( !(pGetIn->ulParm & GET_NO_GENERICREPLACE) ) { - fOK = UtlAlloc( (PVOID*)&pSubstProp, 0L, (LONG)sizeof(TMX_SUBSTPROP), NOMSG); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else { /****************************************************************/ /* add current tagging to source and translation of proposal */ /****************************************************************/ ulSrcOemCP = 1; - strcpy( pSubstProp->szSourceTagTable, pGetIn->szTagTable ); - strcpy( pSubstProp->szSourceLanguage, pGetIn->szSourceLanguage); - strcpy( pSubstProp->szTargetLanguage, pGetIn->szTargetLanguage ); - UTF16strcpy( pSubstProp->szSource, pGetIn->szSource ); + strcpy( SubstProp.szSourceTagTable, pGetIn->szTagTable ); + strcpy( SubstProp.szSourceLanguage, pGetIn->szSourceLanguage); + strcpy( SubstProp.szTargetLanguage, pGetIn->szTargetLanguage ); + UTF16strcpy( SubstProp.szSource, pGetIn->szSource ); pByte = (PBYTE)pTMXTargetRecord; MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); - //instead of lines above we now save not normalize string to btree - wcsncpy(pSubstProp->szPropSource, pSourceString.c_str(), *plSourceLen); + wcsncpy(SubstProp.szPropSource, pSourceString.c_str(), *plSourceLen); if ( usRc == NO_ERROR ) { PSZ_W pTarget; @@ -1747,21 +1632,21 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct { std::wstring output; lTargetLen = EQFCompress2Unicode(output, pByte, lTargetLen); - wcsncpy(pSubstProp->szPropTarget, output.c_str(), sizeof(pSubstProp->szPropTarget)/sizeof(pSubstProp->szPropTarget[0])); + wcsncpy(SubstProp.szPropTarget, output.c_str(), sizeof(SubstProp.szPropTarget)/sizeof(SubstProp.szPropTarget[0])); //position at target tag record pByte = (PBYTE)pTMXTargetRecord; } if ( usRc == NO_ERROR ) { //fill in the markup table - strcpy(pSubstProp->szPropTagTable, "OTMXUXLF"); + strcpy(SubstProp.szPropTagTable, "OTMXUXLF"); ulTgtOemCP = 1; //fill in the target language pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, (USHORT)LANG_KEY, - pSubstProp->szTargetLanguage, NULL ); + SubstProp.szTargetLanguage, NULL ); } /* endif */ } /* endif */ @@ -1889,9 +1774,9 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /**************************************************************/ /* do a substitution of the tags... */ /**************************************************************/ - *plSourceLen = UTF16strlenCHAR( pSubstProp->szPropSource ); + *plSourceLen = UTF16strlenCHAR( SubstProp.szPropSource ); if( *plSourceLen ) { - memcpy( pstMatchTable->szSource, pSubstProp->szPropSource, + memcpy( pstMatchTable->szSource, SubstProp.szPropSource, *plSourceLen * sizeof(CHAR_W)); } } @@ -1925,12 +1810,12 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /* do a substitution of the replaced target string and */ /* adjust match level */ /**************************************************************/ - lTargetLen = UTF16strlenCHAR( pSubstProp->szPropTarget ); - memcpy( pstMatchTable->szTarget, pSubstProp->szPropTarget, + lTargetLen = UTF16strlenCHAR( SubstProp.szPropTarget ); + memcpy( pstMatchTable->szTarget, SubstProp.szPropTarget, lTargetLen * sizeof(CHAR_W)); - ulLen = UTF16strlenCHAR(pSubstProp->szSource); - *plSourceLen = UTF16strlenCHAR(pSubstProp->szPropSource); + ulLen = UTF16strlenCHAR(SubstProp.szSource); + *plSourceLen = UTF16strlenCHAR(SubstProp.szPropSource); if (*plSourceLen > ulLen ) { ulLen = *plSourceLen; @@ -1942,19 +1827,19 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct BOOL fFuzzynessOK; /*Q!*/ fFuzzynessOK = TMFuzzynessEx( pGetIn->szTagTable, - pSubstProp->szPropSource, pSubstProp->szSource, + SubstProp.szPropSource, SubstProp.szSource, sLangID, &usFuzzy, ulSrcOemCP, &usWords, &usDiffs ); if ( pGetIn->ulParm & GET_RESPECTCRLF ) { - fEqual = (UtlCompIgnSpaceW( pSubstProp->szPropSource, - pSubstProp->szSource, 0 ) == 0); + fEqual = (UtlCompIgnSpaceW( SubstProp.szPropSource, + SubstProp.szSource, 0 ) == 0); } else { fEqual = (usFuzzy == 100); -// (UtlCompIgnWhiteSpace( pSubstProp->szPropSource, -// pSubstProp->szSource, +// (UtlCompIgnWhiteSpace( SubstProp.szPropSource, +// SubstProp.szSource, // ulLen) == 0); } /* endif */ @@ -2096,7 +1981,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /********************************************************************/ /* free any allocated resource */ /********************************************************************/ - NTMFreeSubstProp( pSubstProp ); + NTMFreeSubstProp( SubstProp ); return( usRc ); } @@ -2124,173 +2009,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct //|Function flow: add tags at the given offsets and output string with tags| // ----------------------------------------------------------------------------+ -BOOL AddTagsToStringW( PSZ_W pInString, //character string - PLONG plInStringLen, //length of pInString in # of w's - PTMX_TAGTABLE_RECORD pTMXTagTable, //ptr to tag table record - PSZ_W pOutString ) //output string -{ - PBYTE pByte; //record position pointer - USHORT usStringPos, j; //positioning counters - PTMX_TAGENTRY pTagEntry = NULL; //ptr to tag entries in tag table record - ULONG ulTagLen = 0; //length of all tags added to string - ULONG ulEnd = 0; //end length for for loop - USHORT usTagRecordPos = 0; //current length processed in tag record - BOOL fNoTagsLeft = FALSE; //indication of existence of tags - ULONG ulInputStringLen = *plInStringLen; - PSZ_W pOutStart = pOutString; // anchar start pointers - PSZ_W pInStart = pInString; // - BOOL fOK = TRUE; - //position at beginning of target record - pByte = (PBYTE)pTMXTagTable; - pByte += pTMXTagTable->usFirstTagEntry; - - fOK = (*plInStringLen < MAX_SEGMENT_SIZE ); - - if ( fOK && ( RECLEN(pTMXTagTable) > sizeof(TMX_TAGTABLE_RECORD)) ) - { - //there is tag info so go through all tag entries and add tags to outstring - pTagEntry = (PTMX_TAGENTRY)pByte; - - //start of for loop - usStringPos = 0; - //initial positioning st beginning of first tag entry - usTagRecordPos = sizeof(TMX_TAGTABLE_RECORD); - //initial stop criterium for end of for loop - ulEnd = pTagEntry->usOffset; - - while ( fOK && (*pInString || !fNoTagsLeft) ) - { - if ( ulEnd <= ulInputStringLen ) - { - for ( j = usStringPos; j < ulEnd; j++, pOutString++, pInString++ ) - { - *pOutString = *pInString; - } /* endfor */ - } - else - { - for ( j = usStringPos; j < ulEnd; j++, pOutString++, pInString++ ) - { - if ( j < ulInputStringLen ) - { - *pOutString = *pInString; - } - else - { - *pOutString = ' '; - --pInString; // we've gone 1 too far.. - } /* endif */ - - if ( pOutString >= - pOutStart + MAX_SEGMENT_SIZE + usStringPos - ulEnd ) - { - *pOutStart = EOS; - fOK = FALSE; - break; - } /* endif */ - } /* endfor */ - } /* endif */ - - if ( fOK && (j == ulEnd) && (!fNoTagsLeft) ) - { - /**************************************************************/ - /* Add check that tag length does not jeopardize our */ - /* output string... */ - /**************************************************************/ - if ( (pOutString+pTagEntry->usTagLen) < (pOutStart+MAX_SEGMENT_SIZE) - && (pTagEntry->usTagLen < MAX_SEGMENT_SIZE) ) - { - USHORT usTagLenCHAR = pTagEntry->usTagLen; - memcpy( pOutString, &pTagEntry->bData, usTagLenCHAR * sizeof(CHAR_W)); - pOutString += usTagLenCHAR; - ulInputStringLen += usTagLenCHAR; - //reset for loop start - usStringPos = j + usTagLenCHAR; - //update tag table record position counter - usTagRecordPos += sizeof(TMX_TAGENTRY) + usTagLenCHAR * sizeof(CHAR_W); - //remember length of tags added to string - ulTagLen += pTagEntry->usTagLen; - - //move to next tag entry only if there are still tags to add to string - if ( usTagRecordPos < RECLEN(pTMXTagTable) ) - { - USHORT usOldOffs = pTagEntry->usOffset; - pByte += sizeof(TMX_TAGENTRY) + pTagEntry->usTagLen * sizeof(CHAR_W); - pTagEntry = (PTMX_TAGENTRY)pByte; - //new end criterium for for loop - if ( usOldOffs == pTagEntry->usOffset ) - { - // something went wrong: two tag entries for the same offset - fOK = FALSE; - memcpy( pOutStart, pInStart, *plInStringLen * sizeof(CHAR_W)); - fNoTagsLeft = TRUE; - pInString = pInStart + *plInStringLen; - ulEnd = *plInStringLen + ulTagLen; - } - else if ( pTagEntry->usOffset < ulEnd ) - { - // tag data seems to be corrupted, ignore the tags - fOK = FALSE; - memcpy( pOutStart, pInStart, *plInStringLen * sizeof(CHAR_W)); - fNoTagsLeft = TRUE; - pInString = pInStart + *plInStringLen; - ulEnd = *plInStringLen + ulTagLen; - } - else - { - ulEnd = pTagEntry->usOffset; - } /* endif */ - } - else - { - fNoTagsLeft = TRUE; - //new end criterium for for loop, in this case end of string - ulEnd = *plInStringLen + ulTagLen; - } /* endif */ - } - else - { - // something got corrupted -- ignore the tags ... - fOK = FALSE; - memcpy( pOutStart, pInStart, *plInStringLen * sizeof(CHAR_W) ); - fNoTagsLeft = TRUE; - pInString = pInStart + *plInStringLen; - ulEnd = *plInStringLen + ulTagLen; - } /* endif */ - } /* endif */ - - if ( pOutString >= pOutStart + MAX_SEGMENT_SIZE ) - { - *pOutStart = EOS; - fOK = FALSE; - break; - } /* endif */ - } /* endwhile */ - - if ( *pOutStart ) - { - // Add the end of string - *pOutString = EOS; - } - } - else - { - //there are no tags so copy character string - if ( *plInStringLen < MAX_SEGMENT_SIZE ) - { - memcpy( pOutString, pInString, *plInStringLen * sizeof(CHAR_W)); - pOutString[*plInStringLen] = EOS; - } - else - { - fOK = FALSE; - *pOutStart = EOS; - } /* endif */ - } /* endif */ - - return fOK; - -} USHORT GetFuzzyMatch diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 094100f1..22a0a1d7 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -281,15 +281,8 @@ USHORT EqfMemory::TmtXReplace { // try to delete the segment from the memory - PTMX_EXT_OUT_W pstDelOut = NULL; - UtlAlloc( (PVOID *)&pstDelOut, 0L, sizeof (TMX_EXT_OUT_W), NOMSG ); - - if ( pstDelOut ) - { - TmtXDelSegm( TmProposal, pstDelOut ); - - UtlAlloc( (PVOID *)&pstDelOut, 0L, 0L, NOMSG ); - } /* endif */ + TMX_EXT_OUT_W DelOut; + TmtXDelSegm( TmProposal, &DelOut ); } /* endif */ return( usRc ); diff --git a/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp b/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp index 1afde7a8..3218e155 100755 --- a/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp @@ -35,88 +35,6 @@ MakeHashValueW PULONG pulHashVal // resulting hash value ); - -static -BOOL NTMPrepareTokens -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pData, - PFUZZYTOK *ppTokData, - PUSHORT pusTokens, - PTMX_TAGTABLE_RECORD pTagRecord, - SHORT sLangID, - ULONG ulOemCP -); - -static BOOL -NTMFuzzyReplace -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pSource, // source string - PSZ_W pProp, // proposal string - PSZ_W pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -); - -#ifdef ACTIVATE_NTMGenericDelete -static BOOL -NTMGenericDelete -( - PTMX_SUBSTPROP pSubstProp, - PSZ pSource, // source string - PSZ pProp, // proposal string - PSZ pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -); -#endif - - -static BOOL NTMCheckTagPairs -( - PTMX_SUBSTPROP pSubstProp, - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList, // list of tokens to be replaced - BOOL fRespectLFs -); - - -static PTMX_REPLTAGPAIR -NTMFindTagPair -( - PFUZZYTOK pTempTok, - PSZ_W pTempTokData, - PTMX_SUBSTPROP pSubstProp, - PTMX_REPLTAGPAIR pCurTagPair, - BOOL fRespectLFs -); - - -static BOOL -NTMReplaceTags -( - PTMX_SUBSTPROP pSubstProp, - BOOL fRespectLFs -); - -static BOOL -NTMCopyTokData -( - PSZ_W pTempData, - SHORT sLen, - PSZ_W * ppNewData, - PLONG plNewLen -); - - -static PFUZZYTOK -NTMSplitAndAddTokens ( PTMX_SUBSTPROP, - PFUZZYTOK, PUSHORT, USHORT, SHORT, - USHORT, PSZ_W, SHORT, ULONG); - -BOOL NTMAlignTags( PFUZZYTOK pTokSource, PFUZZYTOK pTokTarget, PREPLLIST *ppReplaceList ); - // helper function for token logging #ifdef INLINE_TAG_REPL_LOGGING @@ -169,534 +87,6 @@ void NTMListTokens void NTMMarkCRLF( PSZ_W pszSource, PSZ_W pszTarget ); #endif -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMTagSubst | -//+----------------------------------------------------------------------------+ -//|Function call: NTMTagSubst( pSubstProp ); | -//+----------------------------------------------------------------------------+ -//|Description: GENERIC TM: substitute tags if substitution of all | -//| is possible | -//+----------------------------------------------------------------------------+ -//|Parameters: PTMX_SUBSTPROP pSubstProp ptr to substitution struct | -//+----------------------------------------------------------------------------+ -//|Returncode type: USHORT | -//+----------------------------------------------------------------------------+ -//|Returncodes: 1 all tagging in proposal replaced | -//| 0 nothing replaced | -//+----------------------------------------------------------------------------+ -//|Function flow: Prepare a token list for current src, src of prop and | -//| tgt of prop | -//| call LCS for data pair src - src of prop | -//| call LCS for data pair src of prop - tgt of prop | -//| determine which tags can be replaced | -//| if so, | -//| if all tags can be replaced | -//| replace them | -//+----------------------------------------------------------------------------+ -BOOL NTMTagSubst -( - PTMX_SUBSTPROP pSubstProp, - ULONG ulSrcOemCP, - ULONG ulTgtOemCP -) -{ - BOOL fOK; - PREPLLIST pReplaceList = NULL; - PREPLLIST pReplaceSourceList = NULL; - PFUZZYTOK pCopyTokList1 = NULL; - PFUZZYTOK pCopyTokList2 = NULL; - PFUZZYTOK pTempList; - PBYTE pTokBuf = NULL; - USHORT k; //index variable - BOOL fConnected; // temp variable for connection state - BOOL fReplaced = FALSE; - BOOL fDelete = FALSE; - SHORT sSrcLangID = 0; - SHORT sTgtLangID = 0; - PSZ_W pInBuf; -#ifdef INLINE_TAG_REPL_LOGGING - static CHAR_W szSegBuf[4096]; -#endif - - // ignore LFs in inline tagging only for IBMDITA markup - BOOL fRespectLFs = (_stricmp( pSubstProp->szSourceTagTable, "IBMDITA" ) != 0); - - UtlAlloc((PVOID *) &pInBuf, 0L, (LONG) IO_BUFFER_SIZE * sizeof(CHAR_W), ERROR_STORAGE ); - UtlAlloc((PVOID *) &pTokBuf, 0L, (LONG) TOK_BUFFER_SIZE, ERROR_STORAGE ); - fOK = ( pInBuf && pTokBuf ) ; - -#ifdef INLINE_TAG_REPL_LOGGING - if ( T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP) ) - { - T5LOG(T5DEBUG) << "*** NTMTagSubst ***" ; - NTMMarkCRLF( pSubstProp->szSource, szSegBuf ); - T5LOG(T5DEBUG) << ""<< szSegBuf << ""; - NTMMarkCRLF( pSubstProp->szPropSource, szSegBuf ); - T5LOG(T5DEBUG) << ""<"; - NTMMarkCRLF( pSubstProp->szPropTarget, szSegBuf ); - T5LOG(T5DEBUG) << ""<"; - } /* endif */ - #endif - - - /******************************************************************/ - /* prepare token structure */ - /******************************************************************/ - if (fOK ) - { - fOK = (MorphGetLanguageID(pSubstProp->szSourceLanguage, &sSrcLangID )==MORPH_OK); - } /* endif */ - - if (fOK && (MorphGetLanguageID( pSubstProp->szTargetLanguage, - &sTgtLangID ) != MORPH_OK) ) - { - sTgtLangID = sSrcLangID ; - } /* endif */ - - if (fOK ) - { - fOK = NTMPrepareTokens( pSubstProp, pSubstProp->szSource, - (PFUZZYTOK *)&pSubstProp->pTokSource, - &pSubstProp->usTokenSource, - nullptr, //pSubstProp->pTagsSource, - sSrcLangID, ulSrcOemCP ); - #ifdef INLINE_TAG_REPL_LOGGING - if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)) - if ( fOK ) NTMListTokens( "Source", (PFUZZYTOK)pSubstProp->pTokSource ); - #endif - - } /* endif */ - - if ( fOK ) - { - fOK = NTMPrepareTokens( pSubstProp, pSubstProp->szPropSource, - (PFUZZYTOK *)&pSubstProp->pTokPropSource, - &pSubstProp->usTokenPropSource, - nullptr, //pSubstProp->pTagsPropSource, - sSrcLangID, ulSrcOemCP ); -#ifdef INLINE_TAG_REPL_LOGGING - if ( fOK ) NTMListTokens( "PropSource", (PFUZZYTOK)pSubstProp->pTokPropSource ); -#endif - } /* endif */ - - if ( fOK ) - { // P018276: use TgtOemCP!! (must fit to sTgtLangID) - fOK = NTMPrepareTokens( pSubstProp, pSubstProp->szPropTarget, - (PFUZZYTOK *)&pSubstProp->pTokPropTarget, - &pSubstProp->usTokenPropTarget, - nullptr,//pSubstProp->pTagsPropTarget, - sTgtLangID, ulTgtOemCP ); -#ifdef INLINE_TAG_REPL_LOGGING - if ( fOK ) NTMListTokens( "PropTarget", (PFUZZYTOK)pSubstProp->pTokPropTarget ); -#endif - } /* endif */ - - if ( fOK ) - { - /******************************************************************/ - /* temporary store values */ - /******************************************************************/ - UtlAlloc( (PVOID *) &pCopyTokList1, 0L, (LONG)(pSubstProp->usTokenPropSource+1)*sizeof(FUZZYTOK), NOMSG ); - UtlAlloc( (PVOID *) &pCopyTokList2, 0L, (LONG)(pSubstProp->usTokenPropTarget+1)*sizeof(FUZZYTOK), NOMSG ); - fOK = ( pCopyTokList1 && pCopyTokList2 ); - } /* endif */ - - - if ( fOK ) - { - memcpy( pCopyTokList1, pSubstProp->pTokPropSource, pSubstProp->usTokenPropSource * sizeof(FUZZYTOK)); - memcpy( pCopyTokList2, pSubstProp->pTokPropTarget, pSubstProp->usTokenPropTarget * sizeof(FUZZYTOK)); - - /******************************************************************/ - /* now do the alignments.... */ - /******************************************************************/ -//#define USEOLDALIGN - -#ifdef USEOLDALIGN - fOK = (BOOL)FuzzyLCSReplList( (PFUZZYTOK)pSubstProp->pTokPropSource, (PFUZZYTOK)pSubstProp->pTokPropTarget, &pReplaceList, - pSubstProp->usTokenPropSource, pSubstProp->usTokenPropTarget, NULL, TRUE ); -#else - fOK = NTMAlignTags( (PFUZZYTOK)pSubstProp->pTokPropSource, (PFUZZYTOK)pSubstProp->pTokPropTarget, &pReplaceList ); -#endif - /******************************************************************/ - /* restore original content, but keep fConnected... */ - /* The original content is modified in the FuzzyLCSReplList func. */ - /******************************************************************/ - pTempList = (PFUZZYTOK) pSubstProp->pTokPropSource; - for ( k=0; k < pSubstProp->usTokenPropSource+1; k++ ) - { - fConnected = pTempList->fConnected; - *pTempList = *(pCopyTokList1+k); - pTempList->fConnected = (EQF_BOOL)fConnected; - pTempList++; - } /* endfor */ - - pTempList = (PFUZZYTOK) pSubstProp->pTokPropTarget; - for ( k=0; k < pSubstProp->usTokenPropTarget+1; k++ ) - { - fConnected = pTempList->fConnected; - *pTempList = *(pCopyTokList2+k); - pTempList->fConnected = (EQF_BOOL)fConnected; - pTempList++; - } /* endfor */ - #ifdef INLINE_TAG_REPL_LOGGING - NTMListTokens( "Connected PropSource", (PFUZZYTOK)pSubstProp->pTokPropSource ); - NTMListTokens( "Connected PropTarget", (PFUZZYTOK)pSubstProp->pTokPropTarget ); - #endif - - /******************************************************************/ - /* free CopyTokList2 */ - /******************************************************************/ - UtlAlloc( (PVOID*)&pCopyTokList2, 0L, 0L, NOMSG); - } /* endif */ - - if ( fOK && pReplaceList ) - { - /******************************************************************/ - /* temporary store values */ - /******************************************************************/ - UtlAlloc( (PVOID *) &pCopyTokList2, 0L, (LONG)(pSubstProp->usTokenSource+1)*sizeof(FUZZYTOK), NOMSG ); - fOK = ( pCopyTokList2 != NULL ); - } /* endif */ - - if ( fOK && pReplaceList ) - { - memcpy( pCopyTokList2, pSubstProp->pTokSource, pSubstProp->usTokenSource * sizeof(FUZZYTOK)); - fOK = (BOOL)FuzzyLCSReplList( (PFUZZYTOK)pSubstProp->pTokPropSource, - (PFUZZYTOK)pSubstProp->pTokSource, - &pReplaceSourceList, - pSubstProp->usTokenPropSource, - pSubstProp->usTokenSource, - NULL, TRUE ); - // #endif - /******************************************************************/ - /* restore original content, but keep fConnected... */ - /******************************************************************/ - pTempList = (PFUZZYTOK) pSubstProp->pTokPropSource; - for ( k=0; k < pSubstProp->usTokenPropSource+1; k++ ) - { - fConnected = pTempList->fConnected; - *pTempList = *(pCopyTokList1+k); - pTempList->fConnected = (EQF_BOOL)fConnected; - pTempList++; - } /* endfor */ - - pTempList = (PFUZZYTOK) pSubstProp->pTokSource; - for ( k=0; k < pSubstProp->usTokenSource+1; k++ ) - { - fConnected = pTempList->fConnected; - *pTempList = *(pCopyTokList2+k); - pTempList->fConnected = (EQF_BOOL)fConnected; - pTempList++; - } /* endfor */ - #ifdef INLINE_TAG_REPL_LOGGING - T5LOG(T5DEBUG) << "after FuzzyLCSReplList on source and proposal source" ; - NTMListTokens( "Connected Source", (PFUZZYTOK)pSubstProp->pTokSource ); - NTMListTokens( "Connected PropSource", (PFUZZYTOK)pSubstProp->pTokPropSource ); - #endif - - } /* endif */ - - /********************************************************************/ - /* find out which tags can be replaced by each other */ - /********************************************************************/ - if ( fOK && pReplaceSourceList ) - { - T5LOG(T5DEBUG) << "replace source list exists" ; - - /*************************************************************/ - /* insert a dummy token at the beginning of the pReplaceSourcList */ - /* list to have an anchor where to start from... */ - /*************************************************************/ - memmove( pReplaceSourceList+1, pReplaceSourceList, sizeof( REPLLIST ) * MAX_REPL); - - pReplaceSourceList->pSrcTok = (PFUZZYTOK)pSubstProp->pTokPropSource; - pReplaceSourceList->pTgtTok = (PFUZZYTOK)pSubstProp->pTokSource; - - fReplaced = NTMFuzzyReplace ( pSubstProp, pSubstProp->szSource, - pSubstProp->szPropSource, - pSubstProp->szPropTarget, - pReplaceSourceList, pReplaceList - ); - - T5LOG(T5DEBUG) << "After NTMFuzzyReplace: fReplaced = " << fReplaced ; - -#ifdef ACTIVATE_NTMGenericDelete - fDelete = NTMGenericDelete ( pSubstProp, pSubstProp->szSource, - pSubstProp->szPropSource, - pSubstProp->szPropTarget, - pReplaceSourceList, pReplaceList ); -#endif - - if (fReplaced || fDelete) - { - /****************************************************************/ - /* check whether all tags can be replaced */ - /****************************************************************/ - - if (NTMCheckTagPairs(pSubstProp, pReplaceSourceList, pReplaceList, fRespectLFs )) - { - /**************************************************************/ - /* replace all tags possible */ - /**************************************************************/ - fReplaced = NTMReplaceTags(pSubstProp, fRespectLFs ); - #ifdef INLINE_TAG_REPL_LOGGING - if ( T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)) - { - T5LOG(T5DEBUG) << "After NTMReplaceTags: fReplaced = " << ( ( fReplaced ) ? "True" : "False" ); - if ( fReplaced ) - { - NTMMarkCRLF( pSubstProp->szPropSource, szSegBuf ); - T5LOG(T5DEBUG) << ""<"; - NTMMarkCRLF( pSubstProp->szPropTarget, szSegBuf ); - T5LOG(T5DEBUG) << ""<"; - } /* endif */ - } - #endif - } - else - { - fReplaced = FALSE; - T5LOG(T5DEBUG) << "After NTMCheckTagPairs: fReplaced = " << fReplaced ; - - } /* endif */ - } /* endif */ - } /* endif */ - - // add tagging function of generic inline tag replacement - if ( fOK && !fReplaced ) - { - BOOL fAddTags = TRUE; - PSZ_W pSegmentEndTags = NULL; // ptr to segment end tagging - PSZ_W pSegmentText = NULL; // ptr to segment text portion - - #define TOKEN_TYPE_TAG -7 - #define TOKEN_TYPE_TEXT -1 - - // 1. check that proposal does not have any inline tagging - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokPropSource; - while ( pToken->ulHash ) - { - if ( pToken->sType != TOKEN_TYPE_TEXT ) - { - fAddTags = FALSE; - } /* endif */ - pToken++; - } /*endwhile */ - } /* endif */ - - // 2. check that source has tagging only at the beginning and at the end of the segment - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokSource; - - // handle tags at segment start - if ( pToken->sType == TOKEN_TYPE_TAG ) - { - while ( pToken->ulHash && (pToken->sType == TOKEN_TYPE_TAG) ) - { - pToken++; - } /*endwhile */ - } - else - { - // no tags at segment start - fAddTags = FALSE; - } /* endif */ - - // handle text part of segment - if ( pToken->ulHash &&(pToken->sType == TOKEN_TYPE_TEXT) ) - { - pSegmentText = pToken->pData; // remember start of text portion - - while ( pToken->ulHash && (pToken->sType == TOKEN_TYPE_TEXT) ) - { - pToken++; - } /*endwhile */ - } - else - { - // no text portion following tags - fAddTags = FALSE; - } /* endif */ - - // handle tags at segment end - if ( pToken->ulHash &&(pToken->sType == TOKEN_TYPE_TAG) ) - { - pSegmentEndTags = pToken->pData; // remember start of tagging at segment end - - while ( pToken->ulHash && (pToken->sType == TOKEN_TYPE_TAG) ) - { - pToken++; - } /*endwhile */ - - if ( pToken->ulHash && (pToken->pData[0] != 0) ) - { - // somethig is following tagging at segment end - fAddTags = FALSE; - } /* endif */ - } - else - { - // no tagging at segment end - fAddTags = FALSE; - } /* endif */ - } /* endif */ - - // 3. check that proposal text and segment text is identical - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokSource; - PFUZZYTOK pPropToken = (PFUZZYTOK)pSubstProp->pTokPropSource; - while ( pToken->ulHash && (pToken->pData[0] != 0) && pPropToken->ulHash && (pPropToken->pData[0] != 0)) - { - // skip any tags - while ( (pToken->ulHash && (pToken->pData[0] != 0)) && (pToken->sType == TOKEN_TYPE_TAG) ) - { - pToken++; - } /*endwhile */ - - if ( !pToken->ulHash || (pToken->pData[0] == 0)) - { - // end of source segment - fAddTags = FALSE; - } - else if ( (pToken->sType == TOKEN_TYPE_TEXT) && (pPropToken->sType == TOKEN_TYPE_TEXT ) ) - { - if ( wcsncmp( pToken->pData, pPropToken->pData, pToken->usStop - pToken->usStart + 1 ) != 0 ) - { - // text does not match - fAddTags = FALSE; - } /* endif */ - } - else - { - // not same number of text tokens - fAddTags = FALSE; - } /* endif */ - - // next token - if ( pToken->ulHash ) pToken++; - if ( pPropToken->ulHash ) pPropToken++; - } /*endwhile */ - } /* endif */ - - // 4. check that source segment tagging is of correct type - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokSource; - - if ( (pToken->pData[0] != L'<') || (pSegmentEndTags[0] != L'<') || (pSegmentEndTags[1] != L'/') ) - { - // incorrect type of tagging - fAddTags = FALSE; - } - else - { - // test characters of start and end tag - PSZ_W pszStartTag = pToken->pData + 1; - PSZ_W pszEndTag = pSegmentEndTags + 2; - while ( fAddTags && isalpha(*pszEndTag) ) - //while ( fAddTags && iswalpha(*pszEndTag) ) - { - if ( *pszStartTag == *pszEndTag ) - { - pszStartTag++; - pszEndTag++; - } - else - { - // tag names do not match - fAddTags = FALSE; - } /* endif */ - } /*endwhile */ - } /* endif */ - } /* endif */ - - - // add the tags when every test succeeded - if ( fAddTags ) - { - PFUZZYTOK pToken = (PFUZZYTOK)pSubstProp->pTokSource; - PSZ_W pStartTag = pToken->pData; - int iStartTagLen = pSegmentText - pStartTag; - int iEndTagLen = wcslen( pSegmentEndTags ); - int iSourceTextLen = wcslen(pSubstProp->szPropSource); - int iTargetTextLen = wcslen(pSubstProp->szPropTarget); - - if ( ( (iStartTagLen + iSourceTextLen + iEndTagLen) < MAX_SEGMENT_SIZE) && - ( (iStartTagLen + iTargetTextLen + iEndTagLen) < MAX_SEGMENT_SIZE) ) - { - // change proposal source - memmove( pSubstProp->szPropSource + iStartTagLen, pSubstProp->szPropSource, (iSourceTextLen + 1) * sizeof(CHAR_W) ); - memcpy( pSubstProp->szPropSource, pStartTag, iStartTagLen * sizeof(CHAR_W) ); - wcscpy( pSubstProp->szPropSource + wcslen(pSubstProp->szPropSource), pSegmentEndTags ); - - // change proposal target - memmove( pSubstProp->szPropTarget + iStartTagLen, pSubstProp->szPropTarget, (iTargetTextLen+1) * sizeof(CHAR_W) ); - memcpy( pSubstProp->szPropTarget, pStartTag, iStartTagLen * sizeof(CHAR_W) ); - wcscpy( pSubstProp->szPropTarget + wcslen(pSubstProp->szPropTarget), pSegmentEndTags ); - - fReplaced = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* free allocated resources */ - /********************************************************************/ - if ( pCopyTokList1 ) - UtlAlloc( (PVOID*)&pCopyTokList1, 0L, 0L, NOMSG); - if ( pCopyTokList2 ) - UtlAlloc( (PVOID*)&pCopyTokList2, 0L, 0L, NOMSG); - if ( pSubstProp->pTokSource ) - UtlAlloc( (PVOID*)&pSubstProp->pTokSource, 0L, 0L, NOMSG); - if ( pSubstProp->pTokPropSource ) - UtlAlloc( (PVOID*)&pSubstProp->pTokPropSource, 0L, 0L, NOMSG); - if ( pSubstProp->pTokPropTarget ) - UtlAlloc( (PVOID*)&pSubstProp->pTokPropTarget, 0L, 0L, NOMSG); - if (pInBuf ) - UtlAlloc((PVOID *) &pInBuf, 0L, 0L, NOMSG ); - if (pTokBuf ) - UtlAlloc((PVOID *) &pTokBuf, 0L, 0L, NOMSG ); - if (pReplaceList ) - UtlAlloc((PVOID *) &pReplaceList, 0L, 0L, NOMSG); - if (pReplaceSourceList ) - UtlAlloc((PVOID *) &pReplaceSourceList, 0L, 0L, NOMSG); - return (fReplaced); - -} - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMPrepareTokens | -//+----------------------------------------------------------------------------+ -//|Function call: BOOL NTMPrepareTokens | -//| ( PTMX_SUBSTPROP pSubstProp, | -//| PSZ pData, | -//| PFUZZYTOK *ppTokData, | -//| PUSHORT pusToken, | -//| PTMX_TAGTABLE_RECORD pTagRecord, | -//| SHORT sLangID ) | -//+----------------------------------------------------------------------------+ -//|Description: | -//+----------------------------------------------------------------------------+ -//|Parameters: PTMX_SUBSTPROP pSubstProp, | -//| PSZ pData, | -//| PFUZZYTOK *ppTokData, | -//| PUSHORT pusToken, | -//| PTMX_TAGTABLE_RECORD pTagRecord, | -//| SHORT sLangID | -//+----------------------------------------------------------------------------+ -//|Returncode type: BOOL | -//+----------------------------------------------------------------------------+ -//|Function flow: | -//+----------------------------------------------------------------------------+ #define TAG_TOKEN -7 static @@ -713,363 +103,6 @@ BOOL isWhiteSpaceW( PSZ_W pData ) } -static -BOOL NTMPrepareTokens -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pData, - PFUZZYTOK *ppTokData, - PUSHORT pusToken, - PTMX_TAGTABLE_RECORD pTagRecord, - SHORT sLangID, - ULONG ulOemCP -) -{ - BOOL fOK = TRUE; - USHORT usStringPos; - USHORT usTagRecordPos; // in # of bytes - - PSZ_W pTokBuf = pSubstProp->chBuffer; - PFUZZYTOK pCurrent = (PFUZZYTOK)pTokBuf; - ULONG ulCurStop; - BOOL fEndReached = FALSE; - USHORT usRandomIndex; // index in random sequence - PTMX_TAGENTRY pTagEntry = NULL; //ptr to tag entries in tag table record - PBYTE pByte = (PBYTE)pTagRecord; - USHORT usMaxToksInBuffer = (sizeof(pSubstProp->chBuffer) / - sizeof(FUZZYTOK)) - 2; - - - if ( !pSubstProp->ulRandom[0] ) - { - /********************************************************************/ - /* random sequences, see e.g. the book of Wirth... */ - /********************************************************************/ - pSubstProp->ulRandom[0] = 0xABCDEF01; - for (usRandomIndex = 1; usRandomIndex < MAX_RANDOM; usRandomIndex++) - { - pSubstProp->ulRandom[usRandomIndex] = - pSubstProp->ulRandom[usRandomIndex - 1] * 5 + 0xABCDEF01; - } /* endfor*/ - } /* endif */ - - // usFirstTagEntry is in number of bytes! - pByte += pTagRecord->usFirstTagEntry; - - //there is tag info so go through all tag entries and add tags to outstring - pTagEntry = (PTMX_TAGENTRY)pByte; - - //start of for loop - usStringPos = 0; - //initial positioning st beginning of first tag entry - // in # of bytes - usTagRecordPos = sizeof(TMX_TAGTABLE_RECORD); - - /********************************************************************/ - /* add first entry */ - /********************************************************************/ - memset( pTokBuf, 0, sizeof( pSubstProp->chBuffer )); - - if ( usTagRecordPos < RECLEN(pTagRecord) ) - { - if (pTagEntry->usOffset ) - { - ulCurStop = pTagEntry->usOffset - 1; - pCurrent = NTMSplitAndAddTokens(pSubstProp, pCurrent, - &usMaxToksInBuffer, - 0, - (SHORT)TEXT_TOKEN, - (USHORT)ulCurStop, - pData, sLangID, ulOemCP); - } /* endif */ - } - else - { - ulCurStop = UTF16strlenCHAR(pData); - pCurrent = NTMSplitAndAddTokens(pSubstProp, pCurrent, - &usMaxToksInBuffer, - 0, - (SHORT)TEXT_TOKEN, - (USHORT)ulCurStop, - pData, sLangID, ulOemCP); - } /* endif */ - - - - while ( !fEndReached && (usTagRecordPos < RECLEN(pTagRecord) ) - && (usMaxToksInBuffer > 0) ) - { - ULONG ulNewStart; - /******************************************************************/ - /* fill in tag record */ - /******************************************************************/ - ulNewStart = pTagEntry->usOffset; - pCurrent = NTMSplitAndAddTokens(pSubstProp, pCurrent, - &usMaxToksInBuffer, - (USHORT)ulNewStart, - (SHORT)TAG_TOKEN, - (USHORT) (ulNewStart + pTagEntry->usTagLen-1), - pData, sLangID, ulOemCP); - - ulNewStart += pTagEntry->usTagLen; - - // pTagEntry->usTagLen is in # of w's, but usTagRecordPos counts bytes - // also pByte is a byte ptr - - usTagRecordPos += sizeof(TMX_TAGENTRY) - + sizeof(CHAR_W) * (pTagEntry->usTagLen); - - pByte += sizeof(TMX_TAGENTRY) + sizeof(CHAR_W) * (pTagEntry->usTagLen); - pTagEntry = (PTMX_TAGENTRY)pByte; - - if ( usTagRecordPos < RECLEN(pTagRecord) ) - { - ulCurStop = pTagEntry->usOffset -1; - } - else - { - ulCurStop = UTF16strlenCHAR( pData ); // we reached the end - fEndReached = TRUE; - } /* endif */ - /******************************************************************/ - /* split text record into word tokens */ - /* and fill into pCurrent struct */ - /******************************************************************/ - if (usMaxToksInBuffer > 0 ) - { - pCurrent = NTMSplitAndAddTokens(pSubstProp, pCurrent, - &usMaxToksInBuffer, - (USHORT)ulNewStart, - (SHORT)TEXT_TOKEN, - (USHORT)ulCurStop, - pData, sLangID, ulOemCP); - } /* endif */ - } /* endwhile */ - /********************************************************************/ - /* terminate start-stop table */ - /********************************************************************/ - if (usMaxToksInBuffer > 0 ) - { - memset(pCurrent, 0,sizeof(FUZZYTOK)); - - *pusToken = (USHORT)(pCurrent - (PFUZZYTOK) pTokBuf); - - pCurrent++; - - fOK = UtlAlloc( (PVOID *)ppTokData, 0L, (PBYTE)pCurrent - (PBYTE)pTokBuf, NOMSG); - if ( fOK ) - { - memcpy( *ppTokData, pTokBuf, ((PBYTE)pCurrent-(PBYTE)pTokBuf)); - } /* endif */ - } - else - { - fOK = FALSE; - *ppTokData = NULL; - *pusToken = 0; - } /* endif */ - - /********************************************************************/ - /* clean list of tokens to combine consecutive tags only intercepted*/ - /* by whitespaces returned as TEXT_TOKENS (KBT0492) */ - /********************************************************************/ - if ( fOK && (*pusToken > 2) ) - { - USHORT k = 0; - USHORT usNum = *pusToken; - PFUZZYTOK pNew; - CHAR_W chTemp; - pNew = pCurrent = (PFUZZYTOK) *ppTokData; - - while ( k <= usNum ) - { - switch ( pCurrent->sType ) - { - case TAG_TOKEN: - if ( k < usNum - 2 ) - { - PFUZZYTOK pNext = pCurrent+1; - USHORT usTextLen = (pNext->usStop - pNext->usStart + 1); - chTemp = *(pNext->pData + usTextLen); - *(pNext->pData + usTextLen) = EOS; - - if ( (pNext->sType == TEXT_TOKEN) && - ((pCurrent+2)->sType == TAG_TOKEN) && - isWhiteSpaceW( pNext->pData ) ) - { - /**********************************************************/ - /* combine tag - whitespace - tag sequence into one token */ - /**********************************************************/ - *(pNext->pData + usTextLen) = chTemp; - memcpy( pNew, pCurrent, sizeof( FUZZYTOK )); - - pNext = pCurrent + 2; - usTextLen = (pNext->usStop - pNext->usStart + 1); - chTemp = *(pNext->pData + usTextLen); - *(pNext->pData + usTextLen) = EOS; - MakeHashValueW ( pSubstProp->ulRandom, MAX_RANDOM, - pNew->pData, &pNew->ulHash ); - *(pNext->pData + usTextLen) = chTemp; - - pNew->usStop = (pCurrent+2)->usStop; - pNew++; pCurrent+=3; k+=3; - } - else - { - /**********************************************************/ - /* no special handling necessary */ - /**********************************************************/ - *(pNext->pData + usTextLen) = chTemp; - memcpy( pNew, pCurrent, sizeof( FUZZYTOK )); - pNew++; pCurrent++; k++; - } /* endif */ - } - else - { - /**********************************************************/ - /* no special handling necessary */ - /**********************************************************/ - memcpy( pNew, pCurrent, sizeof( FUZZYTOK )); - pNew++; pCurrent++; k++; - } /* endif */ - break; - case TEXT_TOKEN: - default: - memcpy( pNew, pCurrent, sizeof( FUZZYTOK )); - pNew++; pCurrent++; k++; - break; - } /* endswitch */ - } /* endwhile */ - - /******************************************************************/ - /* adjust length */ - /******************************************************************/ - *pusToken = (USHORT)( pNew- (PFUZZYTOK) *ppTokData ) -1; - - while (pNew < pCurrent ) - { - memset(pNew, 0,sizeof(FUZZYTOK)); - pNew++; - } /* endwhile */ - } /* endif */ - - return fOK; -} - -static PFUZZYTOK -NTMSplitAndAddTokens -( - PTMX_SUBSTPROP pSubstProp, - PFUZZYTOK pstCurrent, - PUSHORT pusMaxFreeToksInBuffer, - USHORT usStart, - SHORT sType, - USHORT usStop, - PSZ_W pString, - SHORT sLangID, - ULONG ulOemCP -) -{ - CHAR_W chTemp; - USHORT usListSize = 0; - TERMLENOFFS* pTermList = NULL; // ptr to created term list - TERMLENOFFS* pActTerm; // actual term - PSZ_W pCurWord; - USHORT usCurStart; - ULONG ulCurStop; - USHORT usRC; - - /********************************************************************/ - /* if token is text token, make one token for each word */ - /********************************************************************/ - if (*pusMaxFreeToksInBuffer > 0) - { - if ( sType == TEXT_TOKEN ) - { - chTemp = *(pString + (usStop+(USHORT)1)); - *(pString + (usStop+(USHORT)1)) = EOS; - - usRC = MorphTokenizeW( sLangID, pString+usStart, - &usListSize, &pTermList, - MORPH_OFFSLIST, ulOemCP ); - - *(pString + (usStop+(USHORT)1)) = chTemp; - - if ( pTermList ) - { - pActTerm = pTermList; - if ( pTermList->iLength>0 && (*pusMaxFreeToksInBuffer > 0)) - { - while ( pActTerm->iLength>0 && (*pusMaxFreeToksInBuffer > 0)) - { - pCurWord = pString + usStart + pActTerm->iOffset; - /**********************************************************/ - /* ignore the linefeeds in the matching */ - /**********************************************************/ - if ( *pCurWord != LF ) - { - usCurStart = (USHORT)(usStart + pActTerm->iOffset); - ulCurStop = usCurStart + pActTerm->iLength - 1; - - pstCurrent->pData = pCurWord; - pstCurrent->usStart = usCurStart; - pstCurrent->sType = sType; - pstCurrent->usStop = (USHORT)ulCurStop; - pstCurrent->fConnected = FALSE; - - chTemp = *(pString + (ulCurStop+(USHORT)1)); - *(pString + (ulCurStop+(USHORT)1)) = EOS; - MakeHashValueW ( pSubstProp->ulRandom, MAX_RANDOM, - pString + usCurStart, &pstCurrent->ulHash ); - *(pString + (ulCurStop+(USHORT)1)) = chTemp; - pstCurrent++; - (*pusMaxFreeToksInBuffer)--; - } /* endif */ - pActTerm++; - } /* endwhile */ - } - else - { - pstCurrent->pData = pString+usStart; - pstCurrent->usStart = usStart; - pstCurrent->sType = sType; - pstCurrent->usStop = usStop; - pstCurrent->fConnected = FALSE; - chTemp = *(pString + (usStop+(USHORT)1)); - *(pString + (usStop+(USHORT)1)) = EOS; - MakeHashValueW ( pSubstProp->ulRandom, MAX_RANDOM, - pString + usStart, &pstCurrent->ulHash ); - *(pString + (usStop+(USHORT)1)) = chTemp; - pstCurrent++; - (*pusMaxFreeToksInBuffer)--; - - } /* endif */ - } /* endif */ - /****************************************************************/ - /* free allocated resource ... */ - /****************************************************************/ - UtlAlloc( (PVOID *)&pTermList, 0L, 0L, NOMSG ); - } - else - { - pstCurrent->pData = pString+usStart; - pstCurrent->usStart = usStart; - pstCurrent->sType = sType; - pstCurrent->usStop = usStop; - pstCurrent->fConnected = FALSE; - chTemp = *(pString + (usStop+(USHORT)1)); - *(pString + (usStop+(USHORT)1)) = EOS; - MakeHashValueW( pSubstProp->ulRandom, MAX_RANDOM, - pString+usStart , &pstCurrent->ulHash ); - *(pString + (usStop+(USHORT)1)) = chTemp; - pstCurrent++; - (*pusMaxFreeToksInBuffer)--; - } /* endif */ - } /* endif */ - return (pstCurrent); - -} - //+----------------------------------------------------------------------------+ //|Internal function | //+----------------------------------------------------------------------------+ @@ -1156,305 +189,8 @@ MakeHashValueW //| if we found a fuzzy match, we have to replace it now..| //| endwhile | //+----------------------------------------------------------------------------+ -static BOOL -NTMFuzzyReplace -( - PTMX_SUBSTPROP pSubstProp, - PSZ_W pSource, // source string - PSZ_W pProp, // proposal string - PSZ_W pTrans, // translation string - PREPLLIST pReplPropSrc, // list of same tokens in source and prop - PREPLLIST pReplaceList // list of tokens to be replaced -) -{ - PFUZZYTOK pSrcTok; // source token - PFUZZYTOK pEndSrcTok; // search until you detect this token - PFUZZYTOK pEndTgtTok; // search until you detect this token - PFUZZYTOK pPropTok; // proposal token - ULONG ulTgtLen; // target length - PREPLLIST pTempRepl; // pointer to temp repl element - BOOL fFound; // match found - BOOL fReplaced = FALSE; // nothing replaced yet - BOOL fFirstInLoop = TRUE; // sep. handling 1st time - USHORT usNumOfPairs = 0; - BOOL fOK = TRUE; - - pSource; - pProp; - ulTgtLen = UTF16strlenCHAR( pTrans ); - - T5LOG(T5DEBUG) << "NTMFuzzyReplace" ; - pSubstProp->pTagPairs = (PTMX_REPLTAGPAIR) (pSubstProp->chBuffer); - /********************************************************************/ - /* replace found tokens in translation with original ones.... */ - /********************************************************************/ - while ( pReplPropSrc->pSrcTok ) - { - pSrcTok = pReplPropSrc->pTgtTok; - pPropTok = pReplPropSrc->pSrcTok; - -#ifdef INLINE_TAG_REPL_LOGGING - T5LOG(T5DEBUG) << "Testing token"; - NTMListToken( "SourceToken", pReplPropSrc->pSrcTok ); - NTMListToken( "TargetToken", pReplPropSrc->pTgtTok ); -#endif - - /******************************************************************/ - /* if next proposal token is part of the possible replacement list*/ - /* we've found one match ... */ - /* we have to compare until the next match between Source and */ - /* Proposal is detected or the end of the list .... */ - /******************************************************************/ - pEndSrcTok = (pReplPropSrc+1)->pTgtTok; - pEndTgtTok = (pReplPropSrc+1)->pSrcTok; - - if ( !pEndSrcTok ) - { - pEndSrcTok = pSrcTok; - while ( pEndSrcTok->ulHash ) - { - pEndSrcTok++; - } /* endwhile */ - } /* endif */ - - if ( !pEndTgtTok ) - { - pEndTgtTok = pPropTok; - while ( pEndTgtTok->ulHash ) - { - pEndTgtTok++; - } /* endwhile */ - } /* endif */ - /******************************************************************/ - /* adjustment for 1st time, earlier decreasing may affect */ - /* pEndSrcTok/pEndTgtTok, if only 1 token in segment */ - /******************************************************************/ - if ( fFirstInLoop ) /* @KIT1171A */ - { /* @KIT1171A */ - fFirstInLoop = FALSE; /* @KIT1171A */ - pSrcTok--; /* @KIT1171A */ - pPropTok--; /* @KIT1171A */ - } /* endif */ /* @KIT1171A */ - - while ( ++pSrcTok && ++pPropTok && - (pSrcTok < pEndSrcTok ) && (pPropTok < pEndTgtTok) ) - { - pTempRepl = pReplaceList; - fFound = FALSE; - /****************************************************************/ - /* only replace of tagging done here */ - /****************************************************************/ - if ((pSrcTok->sType == TAG_TOKEN ) && (pPropTok->sType == TAG_TOKEN)) - { - while ( pTempRepl->pSrcTok && !fFound ) - { - if ( pTempRepl->pSrcTok == pPropTok ) - { - fFound = TRUE; - pSubstProp->pTagPairs->pSrcTok = (PBYTE)pSrcTok; - pSubstProp->pTagPairs->pPropTok = (PBYTE)pPropTok; - pSubstProp->pTagPairs ++; - usNumOfPairs ++; - } - else - { - pTempRepl++; - } /* endif */ - } /* endwhile */ - } /* endif */ - - } /* endwhile */ - pReplPropSrc++; // point to next one - } /* endwhile */ - if (usNumOfPairs) - { - fReplaced = TRUE; - //allocate for pairs - fOK = UtlAlloc( (PVOID *) &(pSubstProp->pTagPairs), 0L, - (LONG)((usNumOfPairs+1) * sizeof(TMX_REPLTAGPAIR)), NOMSG ); - - if (fOK ) - { - //copy source string for later compare function - memcpy( pSubstProp->pTagPairs, pSubstProp->chBuffer, - (USHORT)( usNumOfPairs * sizeof(TMX_REPLTAGPAIR))); - } /* endif */ - } - else - { - pSubstProp->pTagPairs = NULL; - } /* endif */ - return( fReplaced ); -} /* end of function NTMFuzzyReplace */ - -#ifdef ACTIVATE_NTMGenericDelete - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMGenericDelete | -//+----------------------------------------------------------------------------+ -//|Function call: NTMGenericDelete( pSource, pProp, pTrans, | -//| pReplPropSrc, pReplaceList ); | -//+----------------------------------------------------------------------------+ -//|Description: delete tags | -//+----------------------------------------------------------------------------+ -//|Parameters: PSZ pSource, source string | -//| PSZ pProp, proposal string | -//| PSZ pTrans, translation string | -//| PREPLLIST pReplPropSrc, list of same tokens in source | -//| PREPLLIST pReplaceList list of tokens to be replaced | -//+----------------------------------------------------------------------------+ -//|Returncode type: VOID | -//+----------------------------------------------------------------------------+ -//|Prerequesits: pTrans is large enough to hold a total segment in length | -//| of MAX_SEGMENT size | -//+----------------------------------------------------------------------------+ -//|Function flow: loop thru the list of candidates | -//| endwhile | -//+----------------------------------------------------------------------------+ -static BOOL -NTMGenericDelete -( - PTMX_SUBSTPROP pSubstProp, - PSZ pSource, // source string - PSZ pProp, // proposal string - PSZ pTrans, // translation string - PREPLLIST pReplaceSource, // list of same toks in src & srcofprop - PREPLLIST pReplaceList // eq. toks in src & tgt of prop -) -{ - PFUZZYTOK pSrcTok; // source token - PFUZZYTOK pEndSrcTok; // search until you detect this token - PFUZZYTOK pEndTgtTok; // search until you detect this token - PFUZZYTOK pSrcOfPropTok; // proposal token - USHORT ulTgtLen; // target length - PREPLLIST pTempRepl; // pointer to temp repl element - BOOL fFound; // match found - BOOL fReplaced = FALSE; // nothing replaced yet - BOOL fFirstInLoop = TRUE; // sep. handling 1st time - USHORT usNumOfPairs = 0; - BOOL fOK = TRUE; - ulTgtLen = strlen( pTrans ); - pSubstProp->pDelTagPairs = (PTMX_REPLTAGPAIR) (pSubstProp->chBuffer); - /********************************************************************/ - /* replace found tokens in translation with original ones.... */ - /********************************************************************/ - while ( pReplaceSource->pSrcTok ) - { - pSrcTok = pReplaceSource->pTgtTok; // cur source - pSrcOfPropTok = pReplaceSource->pSrcTok; // src of prop - - /******************************************************************/ - /* if next proposal token is part of the possible replacement list*/ - /* we've found one match ... */ - /* we have to compare until the next match between Source and */ - /* Proposal is detected or the end of the list .... */ - /******************************************************************/ - pEndSrcTok = (pReplaceSource+1)->pTgtTok; - pEndTgtTok = (pReplaceSource+1)->pSrcTok; - - if ( !pEndSrcTok ) - { - pEndSrcTok = pSrcTok; - while ( pEndSrcTok->ulHash ) - { - pEndSrcTok++; - } /* endwhile */ - pEndSrcTok--; - /*****************************************************************/ - /* force that tags to be deleted at end of proposal are detected */ - /*****************************************************************/ - if (!fFirstInLoop ) - { - pSrcTok = pEndSrcTok; - pSrcTok--; - } /* endif */ - } /* endif */ - - if ( !pEndTgtTok ) - { - pEndTgtTok = pSrcOfPropTok; - while ( pEndTgtTok->ulHash ) - { - pEndTgtTok++; - } /* endwhile */ - pEndTgtTok--; - } /* endif */ - /******************************************************************/ - /* adjustment for 1st time, earlier decreasing may affect */ - /* pEndSrcTok/pEndTgtTok, if only 1 token in segment */ - /******************************************************************/ - if ( fFirstInLoop ) - { - fFirstInLoop = FALSE; - pSrcTok = pEndSrcTok; // force that they are equal - pSrcTok--; - pSrcOfPropTok = (PFUZZYTOK) pSubstProp->pTokPropSource; // nec if tag at begin - pSrcOfPropTok--; - } /* endif */ - - pSrcTok++; - fFound = TRUE; - - while ( pSrcTok && ++pSrcOfPropTok - && (pSrcTok == pEndSrcTok ) && (pSrcOfPropTok < pEndTgtTok) - && fFound ) - { - if (pSrcOfPropTok->sType == TAG_TOKEN ) - { - pTempRepl = pReplaceList; // eq toks in src&tgt of prop - fFound = FALSE; - /****************************************************************/ - /* only delete of tagging in src/tgt of proposal checked here */ - /****************************************************************/ - while ( pTempRepl->pSrcTok && !fFound ) - { - if ( pTempRepl->pSrcTok == pSrcOfPropTok ) - { - /************************************************************/ - /* tgt of proposal is eq to src of prop !! */ - /************************************************************/ - fFound = TRUE; - pSubstProp->pDelTagPairs->pSrcTok = (PBYTE)pTempRepl->pTgtTok; - pSubstProp->pDelTagPairs->pPropTok = (PBYTE)pSrcOfPropTok; - - pSubstProp->pDelTagPairs ++; - usNumOfPairs ++; - } - else - { - pTempRepl++; - } /* endif */ - } /* endwhile */ - } /* endif */ - - } /* endwhile */ - pReplaceSource++; // point to next one - } /* endwhile */ - if (usNumOfPairs) - { - fReplaced = TRUE; - //allocate for pairs - fOK = UtlAlloc( (PVOID *) &(pSubstProp->pDelTagPairs), 0L, - (LONG)((usNumOfPairs+1) * sizeof(TMX_REPLTAGPAIR)), NOMSG ); - - if (fOK ) - { - //copy source string for later compare function - memcpy( pSubstProp->pDelTagPairs, pSubstProp->chBuffer, - (USHORT)( usNumOfPairs * sizeof(TMX_REPLTAGPAIR))); - } /* endif */ - } - else - { - pSubstProp->pDelTagPairs = NULL; - } /* endif */ - return( fReplaced ); -} /* end of function NTMGenericDelete */ -#endif //reset replace tag pair used flags static void NTMResetUsedFlag( PTMX_REPLTAGPAIR pCurTagPair ) @@ -1474,569 +210,8 @@ static void NTMResetUsedFlag( PTMX_REPLTAGPAIR pCurTagPair ) } /* end of function NTMResetUsedFlag */ -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMCheckTagPairs | -//+----------------------------------------------------------------------------+ -//|Function call: NTMCheckTagPairs(pSubstProp) | -//+----------------------------------------------------------------------------+ -//|Description: check whether all tags in src and target of proposal | -//| can be replaced | -//+----------------------------------------------------------------------------+ -//|Parameters: PTMX_SUBSTPROP pSubstProp | -//+----------------------------------------------------------------------------+ -//|Returncode type: BOOL | -//+----------------------------------------------------------------------------+ -//|Prerequesits: - | -//+----------------------------------------------------------------------------+ -//|Function flow: | -//+----------------------------------------------------------------------------+ -static BOOL -NTMCheckTagPairs -( - PTMX_SUBSTPROP pSubstProp, - PREPLLIST pReplaceSourceList, // list of same tokens in source and prop - PREPLLIST pReplaceList, // toklist to be replaced in prop src+tgt - BOOL fRespectLFs -) -{ - BOOL fAllReplace = TRUE; - PTMX_REPLTAGPAIR pCurTagPair = pSubstProp->pTagPairs; - PFUZZYTOK pTempTok; - USHORT k; - PREPLLIST pReplTest = NULL; - PREPLLIST pReplProp = NULL; - USHORT usStart= 0; - /********************************************************************/ - /* tags which are same in both srces and in propsrc-tgt, need not */ - /* to be replaced. Hence for ease of use change sType to TEXT_TOKEN */ - /********************************************************************/ - - pReplTest = pReplaceSourceList; - pReplTest++; // 1st entry is dummy - pReplProp = pReplaceList; - while ( pReplTest->pSrcTok) - { - pTempTok = pReplTest->pSrcTok; // toks of src of prop - if (pTempTok->sType == TAG_TOKEN ) - { - usStart = pTempTok->usStart; - /****************************************************************/ - /* tag in src of prop which is equal to a tag in current src */ - /****************************************************************/ - while ( pReplProp->pSrcTok && - (pReplProp->pSrcTok->usStart <= usStart )) - { - if (pReplProp->pSrcTok == pTempTok ) - { - /************************************************************/ - /* tag equal in all three sentence!!! no replace nec. */ - /************************************************************/ - pTempTok->sType = TEXT_TOKEN; - pReplTest->pTgtTok->sType = TEXT_TOKEN; - pReplProp->pTgtTok->sType = TEXT_TOKEN; - } /* endif */ - pReplProp++; - } /* endwhile */ - - } /* endif */ - pReplTest++; - } /* endwhile */ - -#ifdef INLINE_TAG_REPL_LOGGING - PREPLLIST pRepl = pReplaceSourceList; - T5LOG(T5DEBUG) << "Replace source list" ; - while ( pRepl->pSrcTok) - { - NTMListToken( "SourceToken", pRepl->pSrcTok ); - NTMListToken( "TargetToken", pRepl->pTgtTok ); - pRepl++; - } /*endwhile */ - - T5LOG(T5DEBUG) << "Replace list" ; - pRepl = pReplaceList; - while ( pRepl->pSrcTok) - { - NTMListToken( "SourceToken", pRepl->pSrcTok ); - NTMListToken( "TargetToken", pRepl->pTgtTok ); - pRepl++; - } /*endwhile */ -#endif - - /********************************************************************/ - /* check all remaining tags in source of proposal */ - /********************************************************************/ - pTempTok = (PFUZZYTOK) pSubstProp->pTokPropSource; - k = 0; - NTMResetUsedFlag( pSubstProp->pTagPairs ); - while ( (kusTokenPropSource) && fAllReplace ) - { - if (pTempTok->sType == TAG_TOKEN ) - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropSource, - pSubstProp, pSubstProp->pTagPairs, fRespectLFs ); - - if (pCurTagPair == NULL ) - { - pCurTagPair = NTMFindTagPair (pTempTok, - pSubstProp->szPropSource, - pSubstProp, pSubstProp->pDelTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { - fAllReplace = FALSE; -#ifdef INLINE_TAG_REPL_LOGGING - - T5LOG(T5DEBUG) << "No tag pair found for" ; - NTMListToken( "Temp token", pTempTok ); -#endif - } /* endif */ - } - else - { - if ( pCurTagPair->fUsed ) - { - // this tag pair has been used already - fAllReplace = FALSE; - } - else - { - pCurTagPair->fUsed = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ - - pTempTok++; - k++; - } /* endwhile */ - - /********************************************************************/ - /* check all tags in target of proposal */ - /********************************************************************/ - pTempTok = (PFUZZYTOK) pSubstProp->pTokPropTarget; - k = 0; - NTMResetUsedFlag( pSubstProp->pTagPairs ); - while ( (kusTokenPropTarget) && fAllReplace ) - { - if (pTempTok->sType == TAG_TOKEN ) - { - pCurTagPair = NTMFindTagPair ( pTempTok, pSubstProp->szPropTarget, pSubstProp, pSubstProp->pTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { - pCurTagPair = NTMFindTagPair (pTempTok, pSubstProp->szPropTarget, pSubstProp, pSubstProp->pDelTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { -#ifdef INLINE_TAG_REPL_LOGGING - - T5LOG(T5DEBUG) << "No tag pair found in target" ; - NTMListToken( "Temp token", pTempTok ); -#endif - fAllReplace = FALSE; - } /* endif */ - } - else - { - if ( pCurTagPair->fUsed ) - { - // this tag pair has been used already - fAllReplace = FALSE; - } - else - { - pCurTagPair->fUsed = TRUE; - } /* endif */ - } /* endif */ - } /* endif */ - - pTempTok++; - k++; - } /* endwhile */ - - - return(fAllReplace); -} /* end of function NTMCheckTagPairs */ - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMReplaceTags | -//+----------------------------------------------------------------------------+ -//|Function call: NTMReplaceTags (pSubstProp) | -//+----------------------------------------------------------------------------+ -//|Description: build up new src and tgt of proposal with replaced | -//| tagging | -//+----------------------------------------------------------------------------+ -//|Parameters: PTMX_SUBSTPROP pSubstProp | -//+----------------------------------------------------------------------------+ -//|Returncode type: BOOL | -//+----------------------------------------------------------------------------+ -//|Function flow: replace tags in target of proposal | -//| replace tags in source of proposal | -//+----------------------------------------------------------------------------+ -static BOOL -NTMReplaceTags -( - PTMX_SUBSTPROP pSubstProp, - BOOL fRespectLFs -) -{ - BOOL fAllReplace = TRUE; - PTMX_REPLTAGPAIR pCurTagPair = pSubstProp->pTagPairs; - PFUZZYTOK pTempTok; - USHORT k; - PFUZZYTOK pCurPairSrcTok; - SHORT sLen; - PSZ_W pTempData; - LONG lNewLen = 0; - PSZ_W pNewData; - - /********************************************************************/ - /* replace all tags in target of proposal */ - /********************************************************************/ - pTempTok = (PFUZZYTOK) pSubstProp->pTokPropTarget; - k = 0; - lNewLen = 0; - pNewData = pSubstProp->chBuffer; - NTMResetUsedFlag( pSubstProp->pTagPairs ); - while ( (kusTokenPropTarget) && fAllReplace ) - { - if (pTempTok->sType == TAG_TOKEN ) - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropTarget, - pSubstProp, pSubstProp->pTagPairs, fRespectLFs ); - if (pCurTagPair != NULL ) - { - /****************************************************************/ - /* pTempTok points to a tag token; current pCurTagPair */ - /* contains tag from current active source segment */ - /* which replaces tag from TranslationMemory */ - /****************************************************************/ - pCurTagPair->fUsed = TRUE; - pCurPairSrcTok = (PFUZZYTOK) pCurTagPair->pSrcTok; - pTempData = pSubstProp->szSource + pCurPairSrcTok->usStart; - sLen = pCurPairSrcTok->usStop - pCurPairSrcTok->usStart +1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, &pNewData, &lNewLen); - - if (fAllReplace ) - { - pCurPairSrcTok->fConnected = TRUE; - } /* endif */ - - /****************************************************************/ - /* copy "space" between tagtoken in prop and next tok in prop */ - /****************************************************************/ - if ((pTempTok + 1)-> ulHash && (pTempTok+1)->usStart ) - { - if ( (pTempTok->usStop + 1 < (pTempTok+1)->usStart) ) - { - sLen = (pTempTok+1)->usStart - pTempTok->usStop - 1; - pTempData = pSubstProp->szPropTarget + pTempTok->usStop+1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, - &pNewData, &lNewLen); - } /* endif */ - } /* endif */ - } - else // check whether tag should be deleted - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropTarget, - pSubstProp, pSubstProp->pDelTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { - fAllReplace = FALSE; - } /* endif */ - } /*endif */ - - } - else - { - /****************************************************************/ - /* pTempTok points to a text token; copy it into target */ - /****************************************************************/ - pTempData = pSubstProp->szPropTarget + pTempTok->usStart; - if ((pTempTok + 1)-> ulHash && (pTempTok+1)->usStart ) - { - if ( (pTempTok->usStop < (pTempTok+1)->usStart) ) - { - /************************************************************/ - /* adjust so that spaces between words are copied too */ - /************************************************************/ - pTempTok->usStop = (pTempTok+1)->usStart - 1; - } /* endif */ - } - else - { - // we are at the end of the token list, ensure that we do not miss any trailing blanks - pTempTok->usStop = (USHORT)(pTempTok->usStart + wcslen( pTempData ) - 1); - } /* endif */ - sLen = pTempTok->usStop - pTempTok->usStart +1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, &pNewData, &lNewLen); - } /* endif */ - - pTempTok++; - k++; - } /* endwhile */ - - if (fAllReplace ) - { - *pNewData = EOS; - lNewLen++; - UTF16strcpy ( pSubstProp->szPropTarget, pSubstProp->chBuffer); - } /* endif */ - - /********************************************************************/ - /* replace all tags in source of proposal */ - /* build new string in chBuffer and copy it at the end in szPropSource*/ - /********************************************************************/ - pTempTok = (PFUZZYTOK) pSubstProp->pTokPropSource; - k = 0; - pNewData = pSubstProp->chBuffer; - lNewLen = 0; - NTMResetUsedFlag( pSubstProp->pTagPairs ); - while ( (kusTokenPropSource) && fAllReplace ) - { - if (pTempTok->sType == TAG_TOKEN ) - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropSource, - pSubstProp, pSubstProp->pTagPairs, fRespectLFs ); - if (pCurTagPair != NULL ) - { - /****************************************************************/ - /* pTempTok points to a tag token; current pCurTagPair */ - /* contains tag from current active source segment */ - /* which replaces tag from TranslationMemory */ - /****************************************************************/ - pCurTagPair->fUsed = TRUE; - pCurPairSrcTok = (PFUZZYTOK) pCurTagPair->pSrcTok; - pTempData = pSubstProp->szSource + pCurPairSrcTok->usStart; - sLen = pCurPairSrcTok->usStop - pCurPairSrcTok->usStart +1; - - fAllReplace = NTMCopyTokData ( pTempData, sLen, &pNewData, &lNewLen); - /****************************************************************/ - /* copy "space" between tagtoken in prop and next tok in prop */ - /****************************************************************/ - if ((pTempTok + 1)-> ulHash && (pTempTok+1)->usStart ) - { - if ( (pTempTok->usStop + 1 < (pTempTok+1)->usStart) ) - { - sLen = (pTempTok+1)->usStart - pTempTok->usStop - 1; - pTempData = pSubstProp->szPropSource + pTempTok->usStop+1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, - &pNewData, &lNewLen); - } /* endif */ - } /* endif */ - } - else - { - pCurTagPair = NTMFindTagPair ( pTempTok, - pSubstProp->szPropSource, - pSubstProp, pSubstProp->pDelTagPairs, fRespectLFs ); - if (pCurTagPair == NULL ) - { - fAllReplace = FALSE; - } /* endif */ - } /*endif */ - } - else - { - /****************************************************************/ - /* pTempTok points to a text token; copy it into target */ - /****************************************************************/ - pTempData = pSubstProp->szPropSource + pTempTok->usStart; - if ((pTempTok + 1)-> ulHash && (pTempTok+1)->usStart ) - { - if ( (pTempTok->usStop < (pTempTok+1)->usStart) ) - { - /************************************************************/ - /* adjust so that spaces between words are copied too */ - /************************************************************/ - pTempTok->usStop = (pTempTok+1)->usStart - 1; - } /* endif */ - } - else - { - // we are at the end of the token list, ensure that we do not miss any trailing blanks - pTempTok->usStop = (USHORT)(pTempTok->usStart + wcslen( pTempData ) - 1); - } /* endif */ - sLen = pTempTok->usStop - pTempTok->usStart +1; - fAllReplace = NTMCopyTokData ( pTempData, sLen, &pNewData, &lNewLen); - } /* endif */ - - pTempTok++; - k++; - } /* endwhile */ - - if (fAllReplace ) - { - *pNewData = EOS; - lNewLen++; - UTF16strcpy ( pSubstProp->szPropSource, pSubstProp->chBuffer); - /******************************************************************/ - /* after this strcpy ALL pCurTagPair->pSrcTok and ALL pSrcTok */ - /* which point on szPropSource are INVALID */ - /* Therefore replace in szPropTarget takes place before */ - /* replace in szPropSource !!! */ - /******************************************************************/ - } /* endif */ - return(fAllReplace); -} /* end of function NTMReplaceTags */ - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMFindTagPair | -//+----------------------------------------------------------------------------+ -//|Function call: NTMFindTagPair (pSubstProp) | -//+----------------------------------------------------------------------------+ -//|Description: find tag pair which has to be replaced by each other | -//+----------------------------------------------------------------------------+ -//|Parameters: PFUZZYTOK pTempTok, | -//| PSZ pTempTokData, | -//| PTMX_SUBSTPROP pSubstProp | -//+----------------------------------------------------------------------------+ -//|Returncode type: PTMX_REPLTAGPAIR | -//+----------------------------------------------------------------------------+ -//|Function flow: find tag pair | -//+----------------------------------------------------------------------------+ -static PTMX_REPLTAGPAIR -NTMFindTagPair -( - PFUZZYTOK pTempTok, - PSZ_W pTempTokData, - PTMX_SUBSTPROP pSubstProp, - PTMX_REPLTAGPAIR pCurTagPair, - BOOL fRespectLFs - -) -{ - PTMX_REPLTAGPAIR pFoundTagPair = NULL; - BOOL fFound = FALSE; - SHORT sLen; - PFUZZYTOK pCurPairPropTok; - PSZ_W pPairData; - PSZ_W pTempData; - - - pTempData = pTempTokData + pTempTok->usStart; // prop src or tgt - sLen = pTempTok->usStop - pTempTok->usStart +1; - if (pCurTagPair ) - { - pCurPairPropTok = (PFUZZYTOK) pCurTagPair->pPropTok; //src of proposal - - while (!fFound && pCurPairPropTok ) - { - if ( pCurTagPair->fUsed ) - { - // ignore this entry - } - else if (pCurPairPropTok == pTempTok ) - { - fFound = TRUE; - } - else - { - pPairData = pSubstProp->szPropSource + pCurPairPropTok->usStart; - if (sLen == (pCurPairPropTok->usStop -pCurPairPropTok->usStart + 1 )) - { - if ( fRespectLFs ) - { - if ( memcmp( (PBYTE)pTempData,(PBYTE)pPairData, sLen * sizeof(CHAR_W)) == 0) - { - fFound = TRUE; - } /* endif */ - } - else - { - CHAR_W chTemp1 = pTempTokData[pTempTok->usStop+1]; - CHAR_W chTemp2 = pSubstProp->szPropSource[pCurPairPropTok->usStop+1]; - pTempTokData[pTempTok->usStop+1] = 0; - pSubstProp->szPropSource[pCurPairPropTok->usStop+1] = 0; - - T5LOG(T5FATAL) << "TEMPORARY_COMMENTED in NTMFindTagPair::UtlCompIgnWhiteSpaceW"; - #ifdef TEMPORARY_COMMENTED - fFound = (UtlCompIgnWhiteSpaceW( pTempData, pPairData, 0 ) == 0 ); - #endif - pTempTokData[pTempTok->usStop+1] = chTemp1; - pSubstProp->szPropSource[pCurPairPropTok->usStop+1] = chTemp2; - } /* endif */ - } /* endif */ - } /* endif */ - if (!fFound ) - { - pCurTagPair++; - if (pCurTagPair ) - { - pCurPairPropTok = (PFUZZYTOK)pCurTagPair->pPropTok; - } - else - { - pCurPairPropTok = NULL; - } /* endif */ - } - else - { - pFoundTagPair = pCurTagPair; - } /* endif */ - } /* endwhile */ - - } /* endif */ - return(pFoundTagPair); -} /* end of function NTMFindTagPair */ - - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: NTMCopyTokData | -//+----------------------------------------------------------------------------+ -//|Function call: NTMCopyTokdata (pSubstProp) | -//+----------------------------------------------------------------------------+ -//|Description: copy data into the new buffer | -//+----------------------------------------------------------------------------+ -//|Parameters: PSZ pTempData, | -//| SHORT sLen, | -//| PSZ * ppNewData, | -//| PSHORT pulNewLen | -//+----------------------------------------------------------------------------+ -//|Returncode type: PTMX_REPLTAGPAIR | -//+----------------------------------------------------------------------------+ -//|Function flow: copy data | -//+----------------------------------------------------------------------------+ -static BOOL -NTMCopyTokData -( - PSZ_W pTempData, - SHORT sLen, - PSZ_W *ppNewData, - PLONG plNewLen - -) -{ - BOOL fAllReplace = TRUE; - PSZ_W pData; - - pData = *ppNewData; - - if (*plNewLen + sLen < MAX_SEGMENT_SIZE ) - { - memcpy((PBYTE)pData, (PBYTE)pTempData, sLen*sizeof(CHAR_W)); - pData += sLen; - *ppNewData = pData; - *(plNewLen) += sLen; - } - else - { - fAllReplace = FALSE; - } /*endif */ - - return (fAllReplace); -} - typedef struct _NTMGETMATCHLEVELDATA { CHAR szSegmentTagTable[MAX_EQF_PATH]; // tag table name for segment markup @@ -2056,102 +231,18 @@ static char szLastTargetLang[MAX_LANG_LENGTH] = ""; static ULONG ulLastSrcOemCP = 0; static ULONG ulLastTgtOemCP = 0; - - - - // free tag substitution data area -void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ) -{ - if ( pSubstProp ) - { - if ( pSubstProp->pTokSource ) - UtlAlloc( (PVOID*) &pSubstProp->pTokSource, 0L, 0L, NOMSG ); - if ( pSubstProp->pTokPropSource ) - UtlAlloc( (PVOID*) &pSubstProp->pTokPropSource, 0L, 0L, NOMSG ); - if ( pSubstProp->pTokPropTarget ) - UtlAlloc( (PVOID*) &pSubstProp->pTokPropTarget, 0L, 0L, NOMSG ); - if ( pSubstProp->pTagPairs ) - UtlAlloc( (PVOID*) &pSubstProp->pTagPairs, 0L, 0L, NOMSG ); - if ( pSubstProp->pDelTagPairs ) - UtlAlloc( (PVOID*) &pSubstProp->pDelTagPairs, 0L, 0L, NOMSG ); - UtlAlloc( (PVOID*)&pSubstProp, 0L, 0L, NOMSG); - } /* endif */ +void NTMFreeSubstProp( TMX_SUBSTPROP& SubstProp ) +{ + if ( SubstProp.pTokSource ) + UtlAlloc( (PVOID*) &SubstProp.pTokSource, 0L, 0L, NOMSG ); + if ( SubstProp.pTokPropSource ) + UtlAlloc( (PVOID*) &SubstProp.pTokPropSource, 0L, 0L, NOMSG ); + if ( SubstProp.pTokPropTarget ) + UtlAlloc( (PVOID*) &SubstProp.pTokPropTarget, 0L, 0L, NOMSG ); + if ( SubstProp.pTagPairs ) + UtlAlloc( (PVOID*) &SubstProp.pTagPairs, 0L, 0L, NOMSG ); + if ( SubstProp.pDelTagPairs ) + UtlAlloc( (PVOID*) &SubstProp.pDelTagPairs, 0L, 0L, NOMSG ); return; } /* end of function NTMFreeSubstProp */ - - -// function aligning tags between source and target of a proposal -BOOL NTMAlignTags -( - PFUZZYTOK pTokListSource, // list of proposal source token - PFUZZYTOK pTokListTarget, // list of proposal target token - PREPLLIST *ppReplaceList // adress of caller's replacement list pointer -) -{ - PREPLLIST pReplaceList = NULL; // pointer to replace list - int iSourceTags = 0; // number of tags in source token list - PFUZZYTOK pTokSource = pTokListSource; // pointer for source token processing - PFUZZYTOK pTokTarget = pTokListTarget; // pointer for target token processing - BOOL fOK = TRUE; // function return code - - // count number of tags in source of proposal - while ( pTokSource->ulHash ) - { - if ( pTokSource->sType == TAG_TOKEN ) - { - iSourceTags++; - } /* endif */ - pTokSource++; - } /*endwhile */ - - // allocate replacement list - int iSize = MAX_REPL > iSourceTags? MAX_REPL : iSourceTags; - iSize++; - fOK = UtlAlloc( (PVOID *)ppReplaceList, 0L, iSize * sizeof(REPLLIST), ERROR_STORAGE ); - pReplaceList = *ppReplaceList; - - // look for source tags in proposal target - if ( fOK ) - { - PREPLLIST pReplaceEntry = pReplaceList; - - pTokSource = pTokListSource; - pTokTarget = pTokListTarget; - while ( pTokSource->ulHash ) - { - if ( pTokSource->sType == TAG_TOKEN ) - { - BOOL fFound = FALSE; - PFUZZYTOK pTargetStart = pTokTarget; - do - { - if ( (pTokTarget->sType == TAG_TOKEN) && (pTokTarget->ulHash == pTokSource->ulHash) && !pTokTarget->fConnected ) - { - pReplaceEntry->pSrcTok = pTokSource; - pReplaceEntry->pTgtTok = pTokTarget; - pTokSource->fConnected = TRUE; - pTokTarget->fConnected = TRUE; - pReplaceEntry++; - fFound = TRUE; - } - else - { - // try next target token, wrap-around at end of list - pTokTarget++; - if ( !pTokTarget->ulHash ) - { - pTokTarget = pTokListTarget; - } /* endif */ - } /* endif */ - } while ( !fFound && (pTargetStart != pTokTarget) ); // while not found and not through list - } /* endif */ - pTokSource++; - } /*endwhile */ - - pTokTarget = pTokListTarget; - pReplaceList = NULL; // pointer to replace list - } /* endif */ - - return( fOK ); -} /* end of function NTMAlignTags */ diff --git a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp index 359c0971..93bc2bf9 100755 --- a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp @@ -128,28 +128,6 @@ BOOL NTMDocMatch } /* end of function NTMDocMatch */ - - -BOOL EQFBCallLCS -( - PFUZZYTOK pTokenList1, // ptr to token list1 - PFUZZYTOK pTokenList2, // ptr to token list2 - USHORT usLenStr1, - USHORT usLenStr2, - BOOL fCompareAll -); - -BOOL EQFBMarkModDelIns -( - PFUZZYTOK pTokenList1, - PFUZZYTOK pTokenList2, - PFUZZYTOK * ppFuzzy1, - PFUZZYTOK * ppFuzzy2, - USHORT usLenStr1, - USHORT usLenStr2 -); - - static BOOL CheckForAlloc ( diff --git a/source/opentm2/core/document/EQFTATAG.cpp b/source/opentm2/core/document/EQFTATAG.cpp index a03cb590..001f56bf 100755 --- a/source/opentm2/core/document/EQFTATAG.cpp +++ b/source/opentm2/core/document/EQFTATAG.cpp @@ -603,7 +603,7 @@ BOOL TATagsToUnicode USHORT i,j = 0; PSZ pCurName; - PWCHAR pTmpW; + std::wstring& pTmpW = pTable->pTagNamesW; PBYTE pByte; PTAG pTag; PSZ pTagNames; @@ -617,16 +617,16 @@ BOOL TATagsToUnicode usTagCount = pTagTable->uNumTags; usAttrCount = pTagTable->stAttribute.uNumber; pAttr = (PATTRIBUTE) (pByte + pTagTable->stAttribute.uOffset); + pTmpW.resize(usSize+1); - if ( UtlAlloc( (PVOID *)&pTmpW, 0L, usSize * sizeof(TMWCHAR), ERROR_STORAGE ) ) { - pTable->pTagTableW = pTmpW; + pTable->pTagTableW = pTmpW.data(); for ( i = 0; i < usTagCount; i++ ) { pCurName = pTag[i].uTagnameOffs + pTagNames; if (pCurName) { - ASCII2Unicode( pCurName, pTmpW + pTag[i].uTagnameOffs, ulCP ); + ASCII2Unicode( pCurName, pTmpW.data() + pTag[i].uTagnameOffs, ulCP ); } } /* endfor */ for ( i= 0; i < usTagCount; i++ ) @@ -634,7 +634,7 @@ BOOL TATagsToUnicode pCurName = pTag[i].uEndDelimOffs + pTagNames; if (pCurName) { - ASCII2Unicode( pCurName, pTmpW + pTag[i].uEndDelimOffs, ulCP ); + ASCII2Unicode( pCurName, pTmpW.data() + pTag[i].uEndDelimOffs, ulCP ); } } /* endfor */ @@ -643,7 +643,7 @@ BOOL TATagsToUnicode pCurName = pTagNames + pAttr[i].uStringOffs; if (pCurName) { - ASCII2Unicode(pCurName, pTmpW + pAttr[i].uStringOffs, ulCP ); + ASCII2Unicode(pCurName, pTmpW.data() + pAttr[i].uStringOffs, ulCP ); } } /* endfor */ for ( i= 0; i < usAttrCount; i++ ) @@ -651,7 +651,7 @@ BOOL TATagsToUnicode pCurName = pAttr[i].uEndDelimOffs + pTagNames; if (pCurName) { - ASCII2Unicode( pCurName, pTmpW + pAttr[i].uEndDelimOffs, ulCP ); + ASCII2Unicode( pCurName, pTmpW.data() + pAttr[i].uEndDelimOffs, ulCP ); } } /* endfor */ for ( i = 0; i < 27; i++ ) @@ -662,22 +662,17 @@ BOOL TATagsToUnicode pCurName = pTagNames + pAttr->uStringOffs; if ( pCurName) { - ASCII2Unicode( pCurName, pTmpW + pAttr->uStringOffs, ulCP); + ASCII2Unicode( pCurName, pTmpW.data() + pAttr->uStringOffs, ulCP); } pCurName = pTagNames + pAttr->uEndDelimOffs; if ( pCurName) { - ASCII2Unicode( pCurName, pTmpW + pAttr->uEndDelimOffs, ulCP); + ASCII2Unicode( pCurName, pTmpW.data() + pAttr->uEndDelimOffs, ulCP); } pAttr++; } /* endfor */ } /* endfor */ - pTable->pTagNamesW = pTmpW; - } - else - { - fOK = FALSE; } return fOK; diff --git a/source/opentm2/core/tagtable/EQFTOKEN.cpp b/source/opentm2/core/tagtable/EQFTOKEN.cpp index dc3188b8..a604614f 100755 --- a/source/opentm2/core/tagtable/EQFTOKEN.cpp +++ b/source/opentm2/core/tagtable/EQFTOKEN.cpp @@ -565,7 +565,7 @@ BOOL TATagTokenizeW // only for blank end delimiters... if ( *pszTest == L' ') { - PSZ_W pszEndDelim = pTable->pTagNamesW + pTag[NewToken.sTokenid].uEndDelimOffs; + PSZ_W pszEndDelim = pTable->pTagNamesW.data() + pTag[NewToken.sTokenid].uEndDelimOffs; // find first non-blank character while ( *pszTest == L' ' ) @@ -1453,7 +1453,7 @@ PNODEAREA TACreateTagTree pByte = (PBYTE) pTagTable; pTag = (PTAG) ( pByte + pTagTable->stFixTag.uOffset); //pTagNames = (PSZ)( pByte + pTagTable-> uTagNames); - pTagNamesW = pLoadedTable->pTagNamesW; + pTagNamesW = pLoadedTable->pTagNamesW.data(); ulNoOfTags = pTagTable->stFixTag.uNumber; for ( usI = 0; usI < 27; usI++ ) { @@ -1569,7 +1569,7 @@ PNODEAREA TACreateAttrTree pByte = (PBYTE) pTagTable; // pTagNames = (PSZ)( pByte + pTagTable-> uTagNames); - pTagNamesW = pLoadedTable->pTagNamesW; + pTagNamesW = pLoadedTable->pTagNamesW.data(); ulNoOfTags = pTagTable->stAttribute.uNumber; for ( usI = 0; usI < 27; usI++ ) { diff --git a/source/opentm2/core/utilities/EQFMORPH.cpp b/source/opentm2/core/utilities/EQFMORPH.cpp index 3bd02ccb..88e3be54 100755 --- a/source/opentm2/core/utilities/EQFMORPH.cpp +++ b/source/opentm2/core/utilities/EQFMORPH.cpp @@ -72,12 +72,8 @@ USHORT TokenizeW ( PVOID pvLangCB, // IN : ptr to language control block PSZ_W pszInData, // IN : ptr to data being tokenized - PULONG pulTermListSize, // IN/OUT: address of variable - // containing size of term list buffer - TERMLENOFFS**ppTermList, // IN/OUT: address of term list pointer - USHORT usListType // IN: type of term list MORPH_ZTERMLIST, - // MORPH_OFFSLIST, MORPH_FLAG_OFFSLIST, - // or MORPH_FLAG_ZTERMLIST + std::vector& pTermList // IN/OUT: address of term list pointer + // MORPH_OFFSLIST ); void MorphCheckSpellInstance( PLANGCB pLangCB ); USHORT MorphWStrings2TermList( vector &vStrings, PVOID *ppTermList, PULONG pulTermListSize ); @@ -246,20 +242,12 @@ USHORT TokenizeW ( PVOID pvLangCB, // IN : ptr to language control block PSZ_W pszInData, // IN : ptr to data being tokenized - PULONG pulTermListSize, // IN/OUT: address of variable - // containing size of term list buffer - TERMLENOFFS** ppTermList, // IN/OUT: address of term list pointer - USHORT usListType // IN: type of term list MORPH_ZTERMLIST, - // MORPH_OFFSLIST, MORPH_FLAG_OFFSLIST, - // or MORPH_FLAG_ZTERMLIST + std::vector& pTermList // IN/OUT: address of term list pointer + // MORPH_OFFSLIST ) { USHORT usReturn = 0; // return code ULONG ulTermBufUsed = 0; // amount of space used in term buffer - BOOL fOffsList; // TRUE = return a offset/length list - - fOffsList = (usListType == MORPH_OFFSLIST) || - (usListType == MORPH_FLAG_OFFSLIST); /********************************************************************/ /* Work on input data */ @@ -309,15 +297,6 @@ USHORT TokenizeW std::string ps_In = EncodingHelper::convertToUTF8(pszInData); for (size_t j = 0; j < vSentenceList.size(); j++) { - if (!usReturn) - { - if ( (usListType == MORPH_FLAG_ZTERMLIST) || - (usListType == MORPH_FLAG_OFFSLIST) ) - { - usReturn = MorphAddTermToList2W( (PSZ_W *)ppTermList, pulTermListSize, &ulTermBufUsed, L" ", 1, 0, TF_NEWSENTENCE, usListType ); - } /* endif */ - } /* endif */ - // split segment/sentences into words OtmMorph::TERMLIST vTermList; T5LOG(T5DEBUG) << "pszInData = " << pszInData <<"; ps_In =" <& pTermList, // address of caller's term list pointer ULONG ulOemCP // CP of language of sLangID!! ) { USHORT usRC = MORPH_OK; // function return code PLCB pLCB = NULL; // pointer to language control block - ULONG ulTermBufUsed = 0; // number of bytes used in term buffer - ULONG ulBufferSize = *pusBufferSize; -#ifdef MEASURE_TOKENIZE_TIME - static int iMeasureCount = 0; - static _int64 iTotalTime = 0; - LARGE_INTEGER iStartTime; - LARGE_INTEGER iEndTime; -#endif - - -#ifdef MEASURE_TOKENIZE_TIME - QueryPerformanceCounter( &iStartTime ); -#endif - - ulOemCP; /********************************************************************/ /* Check input data */ /********************************************************************/ - if ( (pszInData == NULL) || - (pusBufferSize == NULL) || - (ppTermList == NULL) || - ((*ppTermList == NULL) && (*pusBufferSize != 0) ) ) + if (pszInData == NULL) { LOG_AND_SET_RC(usRC, T5INFO, MORPH_INV_PARMS); } /* endif */ @@ -717,82 +672,14 @@ USHORT MorphTokenizeW { if ( *pszInData != EOS ) { -#ifdef LOG_TOKENIZE - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - auto str = EncodingHelper::convertToUTF8(pszInData); - T5LOG( T5DEVELOP) << "::Tokenizing \"" << str << "\"" ; - } -#endif - - usRC = TokenizeW( pLCB->pvLangCB, pszInData, &ulBufferSize, ppTermList, usListType ); - -#ifdef LOG_TOKENIZE - - if ( usRC == MORPH_OK ) - { - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - if ( usListType == MORPH_FLAG_OFFSLIST ) - { - PINT psData = (PINT)*ppTermList; - INT sStart, sLen; - LONG lFlag; - do - { - lFlag = *(PLONG)psData; - psData += 2; - sLen = *psData++; - sStart = *psData++; - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - T5LOG( T5DEVELOP) << ":: Start=" << sStart << " Len=" << sLen << " Flag=" << lFlag; - } - } while ( (sStart != 0) || (sLen != 0) || (lFlag != 0) ); - } - else if ( usListType == MORPH_OFFSLIST ) - { - //PINT psData = (PINT)*ppTermList; - //INT sStart, sLen; - PUSHORT psData = (PUSHORT)*ppTermList; - USHORT sStart, sLen; - do - { - sLen = *psData++; - sStart = *psData++; - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - T5LOG( T5DEVELOP) << "::Start=" << sStart << ", Len=" << sLen; - } - } while ( (sStart != 0) || (sLen != 0) ); - } /* endif */ - } - } - else - { - T5LOG(T5ERROR) << ":: TokenizeW failed with rc=" << usRC ; - } /* endif */ - -#endif - + usRC = TokenizeW( pLCB->pvLangCB, pszInData, pTermList ); } else { - usRC = MorphAddTermToList2W( (PSZ_W *)ppTermList, &ulBufferSize, &ulTermBufUsed, L" ", 0, 0, 0L, usListType ); + usRC = MorphAddTermToList2W( pTermList, 0, 0 ); } /* endif */ } /* endif */ - *pusBufferSize = (USHORT)ulBufferSize; - -#ifdef MEASURE_TOKENIZE_TIME - QueryPerformanceCounter( &iEndTime ); - - iTotalTime += (iEndTime.QuadPart - iStartTime.QuadPart); - iMeasureCount++; - if ( iMeasureCount == 100000 ) - { - char szMsg[60]; - sprintf( szMsg, "Time for %ld MorphTokenizeWCalls is %I64d ticks", iMeasureCount, iTotalTime ); - MessageBox( HWND_DESKTOP, szMsg, "Info", MB_OK ); - } -#endif - return( usRC ); } /* endof MorphTokenizeW */ @@ -1132,239 +1019,16 @@ USHORT MorphAddTermToList2W USHORT MorphAddTermToList2W ( - PSZ_W* ppList, // ptr to term list pointer - PULONG pulSize, // alloc. size of term list in # of w's - PULONG pulUsed, // used # of w's in term list - PSZ_W pszTerm, // ptr to new term being added to list - USHORT usLength, // length of term in # of w's - USHORT usOffs, // offset of term in # of w's??? - ULONG ulFlags, // flags describing the term - USHORT usListType // type of list + std::vector& pList, // ptr to term list pointer + int iLength, // length of term in # of w's + int iOffs // offset of term in # of w's??? ) { - USHORT usRC = MORPH_OK; // function return code - ULONG ulDataLenBytes = 0; // length of data to add in # of bytes - ULONG ulDataLen = 0; // length of data to add in w's - BOOL fOK = TRUE; // internal OK flag - - PBYTE pList = (PBYTE)*ppList; // ptr to term list - PSZ_W *pAnchor = NULL; // ptr to anchor for list - - /********************************************************************/ - /* position to last block for large term lists */ - /********************************************************************/ - if ( usListType == MORPH_LARGE_ZTERMLIST ) - { - PSZ_W pNextList; - - // use caller's list pointer as default anchor - pAnchor = ppList; - - if ( pList != NULL ) - { - pNextList = *((PSZ_W *)pList); - - while ( pNextList != NULL ) - { - pAnchor = (PSZ_W *)pList; - pList = (PBYTE)pNextList; - pNextList = *((PSZ_W *)pList); - } /* endwhile */ - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* get required size for new entry */ - /********************************************************************/ - switch ( usListType ) - { - case MORPH_OFFSLIST : - /******************************************************************/ - /* space for offset and length of term plus end of term list */ - /* indicator which is a zero offset and length value */ - /******************************************************************/ - //ulDataLenBytes = 4 * sizeof(CHAR_W); - ulDataLenBytes = 4 * sizeof(USHORT); - //ulDataLenBytes = 4 * sizeof(int); - break; - - case MORPH_ZTERMLIST : - /******************************************************************/ - /* space for term, term delimiter and end of term list indicator */ - /******************************************************************/ - ulDataLenBytes = (usLength + 1 + 1)*sizeof(CHAR_W); - break; - - case MORPH_FLAG_OFFSLIST : - /******************************************************************/ - /* space for flag, offset and length of term plus end of term list*/ - /* indicator which is a zero flag, offset and length value */ - /******************************************************************/ - ulDataLenBytes = (2 * sizeof(ULONG)) + (4 * sizeof(CHAR_W)); - break; - - case MORPH_FLAG_ZTERMLIST : - /******************************************************************/ - /* space for flag, term, term delimiter and end of term list */ - /* indicator which is a empty flag and a term delimiter */ - /******************************************************************/ - ulDataLenBytes = sizeof(ULONG) + (usLength + 1)*sizeof(CHAR_W) - + sizeof(ULONG) + sizeof(CHAR_W); - break; - - case MORPH_LARGE_ZTERMLIST : - /******************************************************************/ - /* space for term, term delimiter and end of term list indicator */ - /******************************************************************/ - ulDataLenBytes = (usLength + 1 + 1)*sizeof(CHAR_W); - break; - - default : - fOK = FALSE; - break; - } /* endswitch */ - - /********************************************************************/ - /* enlarge term list if necessary */ - /********************************************************************/ - ulDataLen = ulDataLenBytes / sizeof(CHAR_W); // # of w's needed - if ( fOK && ((*pulUsed + ulDataLen) > *pulSize) ) - { - LONG lOldLen = 0L; - LONG max = TERMLIST_INCR > ulDataLen? TERMLIST_INCR : ulDataLen ; - LONG lNewLen = ((LONG)*pulSize) + max ; - - LONG lMaxLen = (usListType == MORPH_LARGE_ZTERMLIST) ? - MORPH_LARGE_ZTERMLIST_LEN : MAX_ALLOC; - lNewLen = lNewLen * sizeof(CHAR_W); // len in # of bytes - lOldLen = (*pulSize) * sizeof(CHAR_W); - if ( lNewLen >= lMaxLen ) - { - // for large termlists allocate another buffer else set error condition - if ( usListType == MORPH_LARGE_ZTERMLIST ) - { - lNewLen = (LONG)sizeof(PSZ_W) + - (LONG)sizeof(CHAR_W) * max; - *pulSize = 0; // no size yet - lOldLen = 0L; - pAnchor = (PSZ_W *)pList; // use current list as anchor - } - } /* endif */ - - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *)&pList, lOldLen, lNewLen, NOMSG ); - if ( fOK ) - { // set caller's termlist pointer - if ( usListType == MORPH_LARGE_ZTERMLIST ) - { - *pAnchor = (PSZ_W)pList; - if (*pulSize == 0 ) // new block??? - { - *pulUsed = sizeof(PSZ_W)/ sizeof(CHAR_W); // leave room for pointer to next block - } /* endif */ - *pulSize = lNewLen / sizeof(CHAR_W); - } - else - { - *pulSize = lNewLen / sizeof(CHAR_W); //@@+= max( TERMLIST_INCR, ulDataLength );?? - *ppList = (PSZ_W)pList; - } /* endif */ - } - else - { - LOG_AND_SET_RC(usRC, T5INFO, MORPH_NO_MEMORY); - - // free the term list in case of erros - if ( usListType == MORPH_LARGE_ZTERMLIST ) - { - pList = (PBYTE)*ppList; - while ( pList != NULL ) - { - PSZ_W pNextList = *((PSZ_W *)pList); - UtlAlloc( (PVOID *)&pList, 0L, 0L, NOMSG ); - pList = (PBYTE)pNextList; - } /* endwhile */ - } - else - { - UtlAlloc( (PVOID *)ppList, 0L, 0L, NOMSG ); - } /* endif */ - *ppList = NULL; - } /* endif */ - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* add term to term list (pList is a PBYTE ptr */ - /********************************************************************/ - if ( fOK ) - { - ULONG ulByteUsedInList = (*pulUsed) * sizeof(CHAR_W); - switch ( usListType ) - { - case MORPH_OFFSLIST : - *((PUSHORT)(pList + ulByteUsedInList)) = usLength; - ulByteUsedInList += sizeof(USHORT); - - *((PUSHORT)(pList + ulByteUsedInList)) = usOffs; - *pulUsed += 1; - ulByteUsedInList += sizeof(USHORT); - break; - - case MORPH_LARGE_ZTERMLIST : - case MORPH_ZTERMLIST : - if ( usLength ) - { - memcpy( (PBYTE)(pList + ulByteUsedInList), - (PBYTE) pszTerm, usLength*sizeof(CHAR_W) ); - *pulUsed += usLength; - ulByteUsedInList += usLength * sizeof(CHAR_W); - } /* endif */ - *((PSZ_W)(pList + ((*pulUsed) * sizeof(CHAR_W)))) = EOS; /* 10-11-16 */ - *pulUsed += 1; - ulByteUsedInList += sizeof(CHAR_W); - break; - - case MORPH_FLAG_OFFSLIST : - *((PULONG)(pList + ulByteUsedInList)) = ulFlags; - *pulUsed += sizeof(ULONG)/ sizeof(CHAR_W); - ulByteUsedInList += sizeof(ULONG); - - *((PUSHORT)(pList + ulByteUsedInList)) = usLength; - *pulUsed += 1;//sizeof(USHORT)/ sizeof(CHAR_W); - ulByteUsedInList += sizeof(USHORT); - - *((PUSHORT)(pList + ulByteUsedInList)) = usOffs; - *pulUsed += 1;//sizeof(USHORT) / sizeof(CHAR_W); - ulByteUsedInList += sizeof(USHORT); - break; - - case MORPH_FLAG_ZTERMLIST : - *((PULONG)(pList + ulByteUsedInList)) = ulFlags; - *pulUsed += sizeof(ULONG)/ sizeof(CHAR_W); - ulByteUsedInList += sizeof(ULONG); - - if ( usLength ) - { - memcpy( pList + ulByteUsedInList, - (PBYTE)pszTerm, usLength*sizeof(CHAR_W) ); - *pulUsed += usLength; - ulByteUsedInList += usLength * sizeof(CHAR_W); - } /* endif */ - *((PSZ_W)(pList + ulByteUsedInList)) = EOS; /* 10-11-16 */ - *pulUsed += 1; - break; - - default : - break; - } /* endswitch */ - } /* endif */ - - /********************************************************************/ - /* return usRC to calling function */ - /********************************************************************/ - return( usRC ); + TERMLENOFFS data; + data.iLength = iLength; + data.iOffset = iOffs; + pList.emplace_back(data); + return( 0 ); } /* end of function MorphAddTermToList2W */ diff --git a/source/opentm2/core/utilities/UtlMisc.cpp b/source/opentm2/core/utilities/UtlMisc.cpp index 2a392805..cd501cb2 100755 --- a/source/opentm2/core/utilities/UtlMisc.cpp +++ b/source/opentm2/core/utilities/UtlMisc.cpp @@ -616,165 +616,3 @@ static BOOL TACheckForWS return(fIsWS); } - - -BOOL TAAdjustWhiteSpace -( - PSZ_W pszSourceSeg, // ptr to source segment data - PSZ_W pszTargetSeg, // ptr to target segment - PSZ_W *ppszNewTargetSeg, // ptr to ptr of output buffer - BOOL fLeadingWS, // process leading whitespace flag - BOOL fTrailingWS, // process trailing whitespace flag - PBOOL pfChanged, // TRUE if target has been changed - PSZ_W pWSList -) -{ - BOOL fOK = TRUE; // function return code - PSZ_W pszTargetWhiteSpace=NULL;// points to last whitespace in target - PSZ_W pszSourceWhiteSpace=NULL;// points to last whitespace in source - PSZ_W pszTempSegW = NULL; - int iAdjustLen = 0; // length of adjusted leading whitespace - - // initialize callers change flag - *pfChanged = FALSE; - - // allocate buffer for temporary segment - fOK = UtlAlloc( (PVOID *)&pszTempSegW, 0L, (2L*(LONG)MAX_SEGMENT_SIZE * sizeof(CHAR_W)), - ERROR_STORAGE ); - - // handle leading whitespace - if ( fOK && fLeadingWS ) - { - CHAR_W chFirstNonWSSource; - CHAR_W chFirstNonWSTarget; - PSZ_W pszNonWSSource = pszSourceSeg; - PSZ_W pszNonWSTarget = pszTargetSeg; - - - // position to first non-whitespace character in source and target - while ( TACheckForWS(*pszNonWSSource, pWSList) ) pszNonWSSource++; - while ( TACheckForWS(*pszNonWSTarget, pWSList) ) pszNonWSTarget++; - - // temporarly terminate segment at found locations - chFirstNonWSSource = *pszNonWSSource; - *pszNonWSSource = EOS; - chFirstNonWSTarget = *pszNonWSTarget; - *pszNonWSTarget = EOS; - - // compare leading whitespace and adjust if necessary - if ( UTF16strcmp( pszSourceSeg, pszTargetSeg ) != 0 ) - { - // copy whitespace of source segment to our temp seg buffer - UTF16strcpy( pszTempSegW, pszSourceSeg ); - - iAdjustLen = UTF16strlenCHAR( pszTempSegW ); - - // restore end of leading whitespace - *pszNonWSSource = chFirstNonWSSource; - *pszNonWSTarget = chFirstNonWSTarget; - - // copy data of target segment to our temp seg buffer - UTF16strcat( pszTempSegW, pszNonWSTarget ); - - // set caller's changed flag - *pfChanged = TRUE; - } - else - { - iAdjustLen = UTF16strlenCHAR( pszTargetSeg ); - - // restore end of leading whitespace - *pszNonWSSource = chFirstNonWSSource; - *pszNonWSTarget = chFirstNonWSTarget; - } /* endif */ - } /* endif */ - - // if nothing has been changed copy complete target segment to - // our temp seg buffer - if ( fOK && !*pfChanged ) - { - UTF16strcpy( pszTempSegW, pszTargetSeg ); - } /* endif */ - - // locate trailing whitespace in target segment - if ( fOK && fTrailingWS ) - { - int iLen; - - iLen = UTF16strlenCHAR(pszTempSegW); - if ( iLen > iAdjustLen ) - { - pszTargetWhiteSpace = pszTempSegW + iLen - 1; - while ( (iLen > iAdjustLen) && TACheckForWS(*pszTargetWhiteSpace, pWSList) ) - { - pszTargetWhiteSpace--; - iLen--; - } /* endwhile */ - pszTargetWhiteSpace++; // point to first byte of whitespace - } /* endif */ - } /* endif */ - - // locate trailing whitespace in source segment - if ( fOK && fTrailingWS ) - { - int iLen; - - iLen = UTF16strlenCHAR(pszSourceSeg); - if ( iLen > iAdjustLen ) - { - pszSourceWhiteSpace = pszSourceSeg + iLen - 1; - while ( (iLen > iAdjustLen) && TACheckForWS(*pszSourceWhiteSpace, pWSList) ) - { - pszSourceWhiteSpace--; - iLen--; - } /* endwhile */ - pszSourceWhiteSpace++; // point to first byte of whitespace - } /* endif */ - } /* endif */ - - // setup new target if trailing whitespace differs - if ( fOK && fTrailingWS && pszSourceWhiteSpace && pszTargetWhiteSpace && - UTF16strcmp( pszSourceWhiteSpace, pszTargetWhiteSpace ) != 0 ) - { - // build new target segment - *pszTargetWhiteSpace = EOS; - UTF16strcat( pszTempSegW, pszSourceWhiteSpace ); - - // set caller's changed flag - *pfChanged = TRUE; - } /* endif */ - - // return modified temp segment to caller - if ( fOK && *pfChanged ) - { - LONG lNewLen = UTF16strlenCHAR(pszTempSegW) + 1; - - // if changed segment exceeds maximum segment size... - if ( lNewLen > MAX_SEGMENT_SIZE ) - { - // .. ignore the changes - *pfChanged = FALSE; - } - else - { - if ( *ppszNewTargetSeg == NULL ) - { - LONG lNewLen = UTF16strlenCHAR(pszTempSegW) + 1; - // GQ: allocate segment at least large enough for the :NONE. tag! - fOK = UtlAlloc( (PVOID *)ppszNewTargetSeg, 0L, - get_max( lNewLen * sizeof(CHAR_W), (sizeof(EMPTY_TAG)+4)), ERROR_STORAGE ); - } /* endif */ - - if ( fOK ) - { - UTF16strcpy( *ppszNewTargetSeg, pszTempSegW ); - } /* endif */ - } /* endif */ - } /* endif */ - - // cleanup - if ( pszTempSegW ) UtlAlloc( (PVOID *)&pszTempSegW, 0L, 0L, NOMSG ); - - return( fOK ); -} /* end of function TAAdjustWhiteSpace */ - From 4a6dbc4954edd171dab67067df75a6099e0366d2 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 14 Apr 2025 22:30:25 +0300 Subject: [PATCH 08/62] * refactored TMX_SENTENCE struct * fixed reading from tm !!! saving new cbl to tm is not working!!! --- include/tm.h | 72 ++++++++++--------- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 7 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 14 ++-- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 2 + .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 46 ++---------- .../core/EqfMemoryPlugin/EqfMemory.cpp | 5 +- 6 files changed, 59 insertions(+), 87 deletions(-) diff --git a/include/tm.h b/include/tm.h index 1ec1d5ac..34495eb7 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1056,19 +1056,43 @@ typedef struct _TMX_INDEX_RECORD TMX_INDEX_ENTRY stIndexEntry; } TMX_INDEX_RECORD, *PTMX_INDEX_RECORD; +#define SAFE_CAST + inline TMX_RECORD *toTmxRecord(BytesRef buffer) { + #ifdef SAFE_CAST + if(buffer.empty()) + return nullptr; + #endif return reinterpret_cast(buffer.data()); } inline TMX_TAGTABLE_RECORD* toTmxTagtableRecord(BytesRef buffer){ + #ifdef SAFE_CAST + if(buffer.empty()) + return nullptr; + #endif return reinterpret_cast(buffer.data()); } inline TMX_INDEX_RECORD *toIndexRecord(BytesRef buffer) { + #ifdef SAFE_CAST + if(buffer.empty()) + return nullptr; + #endif return reinterpret_cast(buffer.data()); } + +inline TMX_TARGET_CLB *toTmxTargetClb(BytesRef buffer) +{ + #ifdef SAFE_CAST + if(buffer.empty()) + return nullptr; + #endif + return reinterpret_cast(buffer.data()); +} + typedef struct _TMX_TERM_TOKEN { USHORT usOffset; @@ -1521,31 +1545,28 @@ class RequestTagReplacer : public StringVariants struct TMX_SENTENCE { - // std::unique_ptr pStrings = nullptr; - StringTagVariants *pStrings = nullptr; - - PSZ_W pAddString = nullptr; + // ADDINFO + std::array pAddString; // segment context + std::array szAddInfo; // additional segment information + std::array szContext; // additional segment information + std::array szAuthorName; // author name of target' + std::array szFileName; // where source comes from name+ext + std::array szTagTable; // tag table name + std::array szSourceLanguage; // language name of source + std::array szTargetLanguage; // language name of target + + std::unique_ptr pStrings, pPropString; std::vector pTermTokens; std::vector pTagRecord;//TMX_TAGTABLE_RECORD USHORT usActVote = 0; std::vector pulVotes; - // std::unique_ptr pPropString = nullptr; - StringTagVariants *pPropString = nullptr; - std::vector pPropTermTokens; // buffer for Termtokens - // ADDINFO - CHAR szSourceLanguage[MAX_LANG_LENGTH]; // language name of source - CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language name of target - CHAR szAuthorName[MAX_FILESPEC]; // author name of target + USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - CHAR szFileName[MAX_FILESPEC]; // where source comes from name+ext LONG_FN szLongName; // name of source file (long name or EOS) ULONG ulSourceSegmentId; // seg. num. of source sentence from analysis - CHAR szTagTable[MAX_FNAME]; // tag table name TIME_L lTime; // time stamp - CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import // end ADDINFO @@ -1555,11 +1576,11 @@ struct TMX_SENTENCE { if (w_trg.empty()) { - pStrings = new StringTagVariants(std::move(w_src)); //;std::make_unique (std::move(w_src)); + pStrings = std::make_unique(std::move(w_src)); } else { - pStrings = new StringTagVariants(std::move(w_src), std::move(w_trg)); // std::make_unique (std::move(w_src), std::move(w_trg)); + pStrings = std::make_unique(std::move(w_src),std::move(w_trg)); } bool fOK = true; pulVotes.reserve(ABS_VOTES / sizeof(pulVotes[0])); @@ -1568,21 +1589,8 @@ struct TMX_SENTENCE pTermTokens.reserve(TOK_SIZE / sizeof(pTermTokens[0])); } - TMX_SENTENCE(wchar_t *w_src) : TMX_SENTENCE(std::move(w_src), L"") {} + explicit TMX_SENTENCE(const wchar_t* w_src) : TMX_SENTENCE(std::wstring(w_src), L"") {} - ~TMX_SENTENCE() - { - if (pStrings) - { - delete pStrings; - pStrings = nullptr; - } - if (pPropString) - { - delete pPropString; - pPropString = nullptr; - } - } bool wasParsedSuccessfully() const { return pStrings && pStrings->isParsed(); } }; using PTMX_SENTENCE = TMX_SENTENCE *; @@ -2084,7 +2092,7 @@ class EqfMemory //: public TMX_CLB size_t GetExpectedRAMSize() const; USHORT ExtractRecordV6( - PTMX_RECORD pTmRecord, + BytesRef pTmRecord, PTMX_EXT_IN_W pTmExtIn, TMX_EXT_OUT_W *pTmExtOut); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 109b8821..3d1751c7 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -271,7 +271,7 @@ USHORT EqfMemory::TmtXExtract if ( usRc == NO_ERROR ) { - usRc = ExtractRecordV6( toTmxRecord(pTmRecord), pTmExtIn, pTmExtOut ); + usRc = ExtractRecordV6(pTmRecord, pTmExtIn, pTmExtOut ); } /****************************************************************/ /* setup new starting point (do this even in the case we are */ @@ -316,12 +316,13 @@ USHORT EqfMemory::TmtXExtract USHORT EqfMemory::ExtractRecordV6 ( - PTMX_RECORD pTmRecord, + BytesRef pTmRecordData, PTMX_EXT_IN_W pTmExtIn, TMX_EXT_OUT_W * pTmExtOut ) { USHORT usRc = 0; + PTMX_RECORD pTmRecord = toTmxRecord(pTmRecordData); PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record PTMX_TARGET_CLB pTargetClb; //ptr to target CLB @@ -676,8 +677,6 @@ USHORT EqfMemory::FillExtStructure pstExt->lTargetTime = pTMXTargetClb->lTime; // fill in any segment context info - pstExt->szContext[0] = 0; - pstExt->szAddInfo[0] = 0; if ( pTMXTargetClb->usAddDataLen >= MAX_ADD_DATA_LEN ) { // target CLB info seems to be corrupted diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 20420662..dfec6fc3 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -243,11 +243,7 @@ USHORT EqfMemory::TmtXGet Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); strcat( szString, "/TABLE/"); strcat( szString, pTmGetIn->szTagTable ); - strcat( szString, EXT_OF_FORMAT ); - - //remember start of norm string - //pSentence->pNormStringStart = pSentence->pNormString; - + strcat( szString, EXT_OF_FORMAT ); //tokenize source segment, resuting in normalized string and //tag table record @@ -2266,8 +2262,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block //copy source string for fill matchtable //T5LOG(T5TRANSACTION) << "pTmRecord: lRecordLen="<< pTmRecord->lRecordLen <<"; usFirstTargetRecord=" << pTmRecord->usFirstTargetRecord <<"; usSourceRecord=" << pTmRecord->usSourceRecord; ulSourceLen = EQFCompress2Unicode( pString, pSource, ulSourceLen ); - if(pSentence->pPropString) delete pSentence->pPropString; - pSentence->pPropString = new StringTagVariants(pString.c_str());//std::make_unique(pString); + pSentence->pPropString = std::make_unique(pString.c_str()); if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ auto str = EncodingHelper::convertToUTF8(pString); @@ -2392,13 +2387,12 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block if ( fTestCLB && (pTMXTargetClb->bTranslationFlag == TRANSLFLAG_GLOBMEM) && (pGetIn->pvGMOptList != NULL) && pTMXTargetClb->usAddDataLen ) { USHORT usAddDataLen = 0; - if ( !pSentence->pAddString ) UtlAlloc( (PVOID *) &pSentence->pAddString, 0L, (LONG) MAX_SEGMENT_SIZE * sizeof(CHAR_W), NOMSG ); std::wstring addDataOut; usAddDataLen = NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, addDataOut); - wcsncpy(pSentence->szAddInfo, addDataOut.c_str(), MAX_SEGMENT_SIZE); + wcsncpy(pSentence->szAddInfo.data(), addDataOut.c_str(), MAX_SEGMENT_SIZE); if ( usAddDataLen ) { - GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pSentence->pAddString ); + GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pSentence->pAddString.data() ); switch ( GobMemOpt ) { case GM_SUBSTITUTE_OPT: usModifiedTranslationFlag = TRANSLFLAG_NORMAL; break; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index e6e9941b..96b29d68 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -1537,6 +1537,8 @@ SHORT BTREE::QDAMDictExactLocal SHORT sNearKey; // nearest key position SHORT sRc = 0; // return code RECPARAM recData; // point to data structure + + pchBuffer.clear(); if ( fCorrupted ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 22a0a1d7..d4a6ddd0 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -761,9 +761,9 @@ USHORT EqfMemory::AddToTm //add new tm record to database lNewKey = NTMREQUESTNEWKEY; - - if(toTmxRecord(TmRecord)->lRecordLen > 0){ - TmRecord.resize(toTmxRecord(TmRecord)->lRecordLen); + size_t size = toTmxRecord(TmRecord)->lRecordLen; + if(size > 0){ + TmRecord.resize(size); } usRc = TmBtree.EQFNTMInsert(//ptr to tm structure lNewKey, //to be allocated in funct @@ -813,7 +813,7 @@ VOID FillTmRecord USHORT usSrcLangId ) { - PTMX_TARGET_CLB pTargetClb = (PTMX_TARGET_CLB)TargetClb.data(); + PTMX_TARGET_CLB pTargetClb = toTmxTargetClb(TargetClb); PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to start of source structure PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to start of target structure PBYTE pTarget; //ptr to target record @@ -822,7 +822,7 @@ VOID FillTmRecord ULONG ulTrgLen = wcslen(pSentence->pStrings->getGenericTargetStrC()); //position source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(TmRecord.data() + sizeof(TMX_RECORD)); pTMXSourceRecord->reset(); //source structure offset in tm record pTmRecord->usSourceRecord = (USHORT)((PBYTE)pTMXSourceRecord - (PBYTE)pTmRecord); @@ -838,8 +838,7 @@ VOID FillTmRecord RECLEN(pTMXSourceRecord) = sizeof( TMX_SOURCE_RECORD ) + ulSrcLen; //position target structure in tm record - pTarget = (PBYTE)(pTmRecord+1); - pTarget += RECLEN(pTMXSourceRecord); + pTarget = TmRecord.data() + sizeof(TMX_RECORD) + RECLEN(pTMXSourceRecord); //first target offset in tm record pTmRecord->usFirstTargetRecord = (USHORT)(pTarget - (PBYTE)pTmRecord); @@ -847,7 +846,7 @@ VOID FillTmRecord //position to start of target structure pTMXTargetRecord = ((PTMX_TARGET_RECORD)pTarget); pTMXTargetRecord->reset(); - pTMXTargetRecord->usTarget = sizeof(*pTMXTargetRecord); + pTMXTargetRecord->usTarget = sizeof(TMX_TARGET_RECORD); pTarget += pTMXTargetRecord->usTarget; ulTrgLen = EQFUnicode2Compress( pTarget, pSentence->pStrings->getGenericTargetStrC(), ulTrgLen ); @@ -864,10 +863,7 @@ VOID FillTmRecord //size of target record RECLEN(pTMXTargetRecord) = sizeof(TMX_TARGET_RECORD) + - //RECLEN(pSentence->pTagRecord) + - //RECLEN(pTagRecord) + ulTrgLen + - //ulSrcLen + TARGETCLBLEN(pTargetClb); //size of entire tm record RECLEN(pTmRecord) = sizeof( TMX_RECORD ) + @@ -1042,9 +1038,6 @@ USHORT EqfMemory::UpdateTmIndex toIndexRecord(pIndexRecord)->usRecordLen = sizeof( TMX_INDEX_RECORD ); toIndexRecord(pIndexRecord)->stIndexEntry = NTMINDEX(pSentence->pulVotes.size(),ulSidKey); - if( toIndexRecord(pIndexRecord)->usRecordLen > 0){ - pIndexRecord.resize( toIndexRecord(pIndexRecord)->usRecordLen); - } usRc = InBtree.EQFNTMInsert( lKey, pIndexRecord); //pointer to index //toIndexRecord(pIndexRecord)->usRecordLen ); //length @@ -2060,38 +2053,15 @@ VOID FillTargetRecord //position to start of target structure pTarget = (PBYTE)pTMXTargetRecord; - //set source tag table offset - //pTMXTargetRecord->usSourceTagTable = sizeof(TMX_TARGET_RECORD); - - //position pointer for source tag table - //pTarget += pTMXTargetRecord->usSourceTagTable; - - //copy source tag table record to correct position - //memcpy( pTarget, pSentence->pTagRecord, RECLEN(pSentence->pTagRecord) ); - - //set target tag table start offset - //pTMXTargetRecord->usTargetTagTable = (USHORT)(pTMXTargetRecord->usSourceTagTable + - // RECLEN(pSentence->pTagRecord)); - - //adjust target pointer for target tag table - //pTarget += RECLEN(pSentence->pTagRecord); - - //copy target tag table record to correct position - //memcpy( pTarget, pTagRecord, RECLEN(pTagRecord) ); - //set target string start offset pTMXTargetRecord->usTarget = sizeof(*pTMXTargetRecord); pTarget += pTMXTargetRecord->usTarget; - //(USHORT)(pTMXTargetRecord->usTargetTagTable + RECLEN(pTagRecord); - //adjust target pointer for target string - //pTarget += RECLEN(pTagRecord); //copy target string to correct position { ULONG ulTempLen = usNormLen; ulTempLen = EQFUnicode2Compress( pTarget, pNormString, ulTempLen ); usNormLen = (USHORT)ulTempLen; } -// memcpy( pTarget, pNormString, usNormLen ); //set target string control block start offset pTMXTargetRecord->usClb = pTMXTargetRecord->usTarget + usNormLen; @@ -2104,8 +2074,6 @@ VOID FillTargetRecord //size of target record RECLEN(pTMXTargetRecord) = sizeof(TMX_TARGET_RECORD) + - //RECLEN(pSentence->pTagRecord) + - //RECLEN(pTagRecord) + usNormLen + TARGETCLBLEN(pTargetClb); *ppTMXTargetRecord = pTMXTargetRecord; diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index ab6a6ec2..1db849e8 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -485,8 +485,9 @@ int EqfMemory::ExtOutToOtmProposal Proposal.setMarkup( pExtOut->szTagTable ); Proposal.setTargetLanguage( pExtOut->szTargetLanguage ); Proposal.setSourceLanguage( stTmSign.szSourceLanguage); - Proposal.setAddInfo( pExtOut->szAddInfo.data() ); - Proposal.setContext( pExtOut->szContext.data() ); + + if(!pExtOut->szAddInfo.empty()) Proposal.setAddInfo( pExtOut->szAddInfo.data() ); + if(!pExtOut->szContext.empty()) Proposal.setContext( pExtOut->szContext.data() ); Proposal.setDocName( pExtOut->szLongName ); Proposal.setSegmentId( pExtOut->ulSourceSegmentId ); Proposal.setType( FlagToProposalType( pExtOut->usTranslationFlag ) ); From 4541b99a41ef7164b0f42bd099383b4ad5abc034 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 16 Apr 2025 14:17:47 +0300 Subject: [PATCH 09/62] * code cleanup --- .../opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index 3947b05f..8cdcf24e 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -18,27 +18,9 @@ #include "LogWrapper.h" #include "tm.h" -// EQF.H is included by otmmemory.h -// #include // General Translation Manager include file EQF:H #include // batch mode definitions #define INCL_EQFMEM_DLGIDAS -//#include "EQFMEM.ID" // Translation Memory IDs -//#include "EQFCOLW.ID" // column width IDs - -BOOL EQFTMMaintain( PSZ ); - - - -//#define EqfRemoveObject( flg, hwnd) (BOOL)WinSendMsg( EqfQueryObjectManager(),\ -// WM_EQF_REMOVEOBJECT, \ -// MP1FROMSHORT( flg ), \ -// MP2FROMHWND(hwnd) ) - -bool EqfRemoveObject(auto flg, auto hwnd){ - LOG_UNIMPLEMENTED_FUNCTION; - return true; -} // ================ Handle the message WM_EQF_MEMORGANIZE_START ======================= @@ -120,11 +102,6 @@ USHORT EQFMemOrganizeStart else { pRIDA->NextTask = MEM_ORGANIZE_TASK; - //if(pRIDA->pMem->importDetails == nullptr){ - // pRIDA->pMem->importDetails = new ImportStatusDetails; - //} - //pRIDA->pMem->importDetails->reset(); - //pRIDA->pMem->importDetails->fReorganize = true; } return iRC; } /* end of function EQFMemOrganizeStart */ @@ -141,18 +118,8 @@ NTMCloseOrganize ( PMEM_ORGANIZE_IDA pRIDA, //pointer to organize IDA /* close the original TM */ if ( pRIDA->pMem != NULL ) { - int use_count = pRIDA->pMem.use_count(); - - //if(use_count != 2)//1) - //{ - // T5LOG(T5WARNING) << ":: use_count for tm for reorganize is not 2, but " << use_count; - //} - //TMManager::GetInstance()->CloseTM(strMemName); - //pRIDA->pMem = nullptr; - // replace original memory with the organized one usURc = TMManager::GetInstance()->ReplaceMemory( strMemName, strTempMemName, outputMessage, pRIDA->tmListTimeout ); - } /* endif */ return usURc; @@ -197,8 +164,6 @@ VOID EQFMemOrganizeProcess }catch(...) { pRIDA->pProposal->nextInternalKey.moveToNextRecord(); - //pRIDA->pMem->ulNextKey++; - //pRIDA->pMem->usNextTarget = 1; iRC = EqfMemory::ERROR_ENTRYISCORRUPTED; } @@ -242,7 +207,6 @@ VOID EQFMemOrganizeProcess else { // write proposal to output memory - //bool fFilterPassed = true; bool fSkipSegment = !pRIDA->m_reorganizeFilters.empty(); // if there are no filters - no need to filter out segment //if no filters - import segment @@ -251,12 +215,10 @@ VOID EQFMemOrganizeProcess bool result = filter.check(*pRIDA->pProposal); if(!result){ fSkipSegment = false; - //fFilterPassed = false; break; } } - //if(fFilterPassed == false){ if( fSkipSegment ){ pRIDA->pMem->importDetails->filteredSegments++; }else{ @@ -373,7 +335,6 @@ VOID EQFMemOrganizeEnd PMEM_ORGANIZE_IDA pRIDA = (PMEM_ORGANIZE_IDA)pCommArea->pUserIDA; if(nullptr != pRIDA->pMemTemp){ - //FilesystemHelper::DeleteFile(FilesystemHelper::GetTmdPath(pRIDA->pMem->szName)); pRIDA->pMem->TmBtree.fb.data = std::move(pRIDA->pMemTemp->TmBtree.fb.data); //get signature record and add to control block pRIDA->pMem->TmBtree.resetLookupTable(); @@ -381,11 +342,8 @@ VOID EQFMemOrganizeEnd pRIDA->pMem->TmBtree.fb.Flush(true); - //FilesystemHelper::DeleteFile(FilesystemHelper::GetTmiPath(pRIDA->pMem->szName)); pRIDA->pMem->InBtree.fb.data = std::move(pRIDA->pMemTemp->InBtree.fb.data); pRIDA->pMem->InBtree.resetLookupTable(); - //usLen = sizeof( TMX_SIGN ); - //usRc = pRIDA->pMem->InBtree.EQFNTMSign((PCHAR) &(pRIDA->pMem->stTmSign), &usLen ); pRIDA->pMem->InBtree.fb.Flush(true); } From c30baf7e250ade9dc754c84fd95ff5a1a8823e3b Mon Sep 17 00:00:00 2001 From: orestdrag Date: Thu, 17 Apr 2025 12:48:33 +0300 Subject: [PATCH 10/62] code cleanup --- source/RestAPI/OtmMemoryServiceWorker.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index 022d15d9..a0f763ac 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -56,7 +56,6 @@ void importMemoryProcess( void *pvData ); OtmMemoryServiceWorker* OtmMemoryServiceWorker::getInstance() { static OtmMemoryServiceWorker _instance; - return &_instance; } @@ -172,10 +171,6 @@ int convertDateTimeStringToLong( char *pszDateTime, PLONG plTime ) return( 0 ); } - - - - template void AddObjToJson(std::stringstream& ss, const char* key, T value, bool fAddSeparator){ ss << "{ \"" << key << "\" : "; @@ -194,15 +189,6 @@ void AddObjToJson(std::stringstream& ss, const char* key, T value, bool fAddSepa ss << "\n"; } -/*! \brief Saves all open and modified memories - \returns http return code0 if successful or an error code in case of failures - -int OtmMemoryServiceWorker::closeAll(){ - int rc = TMManager::GetInstance()->closeAll(); - return rc; -}//*/ - - std::string printTime(time_t time){ char buff[255]; From a7e42d14c623b0266d33990a644832ee1db871af Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 23 Apr 2025 15:24:23 +0300 Subject: [PATCH 11/62] * fixed removing previous target when saving new targe * code cleanup --- include/EQFPRO00.H | 1 - include/EQFQDAMI.H | 2 +- include/lowlevelotmdatastructs.h | 22 +-- include/tm.h | 5 +- source/opentm2/OTMFUNC.cpp | 9 +- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 8 +- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 15 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 139 +++++------------- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 59 +++----- .../core/EqfMemoryPlugin/EqfMemory.cpp | 6 +- .../core/EqfMemoryPlugin/EqfMemoryPlugin.cpp | 1 - source/opentm2/core/utilities/EQFDICT.cpp | 12 +- source/opentm2/core/utilities/EQFPRO00.cpp | 14 +- 14 files changed, 93 insertions(+), 204 deletions(-) diff --git a/include/EQFPRO00.H b/include/EQFPRO00.H index 8626502c..bcb87ba2 100755 --- a/include/EQFPRO00.H +++ b/include/EQFPRO00.H @@ -188,7 +188,6 @@ SHORT FreePropHnd( PPROP_IDA pIda, PPROPHND hprop); PPROPHND FindPropHnd( PPROPHND ptop, PPROPHND hprop); PPROPCNTL LoadPropFile( PPROP_IDA pIda, PSZ pName, PSZ pPath, USHORT Acc); PPROPCNTL CreatePropFile( PPROP_IDA pIda, PSZ pName, PSZ pPath, USHORT Class, BOOL fOverwriteExisting ); -VOID DropPropFile( PPROP_IDA pIda, PPROPCNTL pcntl); PPROPCNTL FindPropCntl( PPROPCNTL ptop, PSZ pszName, PSZ pszPath); USHORT GetSysProp( PPROP_IDA pIda); USHORT PutItAway( PPROPCNTL pcntl); diff --git a/include/EQFQDAMI.H b/include/EQFQDAMI.H index 35befcc5..befe04d7 100755 --- a/include/EQFQDAMI.H +++ b/include/EQFQDAMI.H @@ -924,7 +924,7 @@ typedef struct _DICTTIME SHORT QDAMUnTerseData ( PBTREE, BytesRef , ULONG ); SHORT QDAMValidateIndex_V3( PBTREE, std::shared_ptr& ); - ULONG QDAMGetszKey_V3 ( std::shared_ptr&, USHORT ); + LONG QDAMGetszKey_V3 ( std::shared_ptr&, USHORT ); RECPARAM QDAMGetrecData_V3 ( std::shared_ptr&, SHORT, USHORT ); VOID QDAMSetrecData_V3 ( std::shared_ptr&, SHORT, RECPARAM, USHORT ); RECPARAM QDAMGetrecKey_V3 ( std::shared_ptr&, SHORT ); diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 9f8b4b78..d8b71773 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -468,11 +468,18 @@ typedef LHANDLE HTM; //table entry structure -typedef struct _TMX_TABLE_ENTRY +struct TMX_TABLE_ENTRY { CHAR szName[MAX_LANG_LENGTH]; USHORT usId = 0; -} TMX_TABLE_ENTRY, * PTMX_TABLE_ENTRY; + + TMX_TABLE_ENTRY() + { + szName[0] = '\0'; + } +}; +typedef TMX_TABLE_ENTRY * PTMX_TABLE_ENTRY; + // name table structure (TM version 5 and up) constexpr int NUM_OF_TMX_TABLE_ENTRIES = (BTREE_REC_SIZE_V3 - sizeof(ULONG)) / sizeof(TMX_TABLE_ENTRY); @@ -485,16 +492,11 @@ typedef TMX_TABLE_OLD * PTMX_TABLE_OLD; struct TMX_TABLE { - //ULONG ulAllocSize = 0; - ULONG ulMaxEntries = 0; - //std::vector stTmTableEntry;//[NUM_OF_TMX_TABLE_ENTRIES]; + ULONG ulMaxEntries1 = 0; std::vector stTmTableEntry; TMX_TABLE(){ - stTmTableEntry.resize(NUM_OF_TMX_TABLE_ENTRIES); - for(int i=0;ilMagicWord = FCTDATA_IDENTIFIER; @@ -195,12 +194,6 @@ USHORT EqfStartSession // Initialize error handler to allow handling of error messages UtlInitError( NULLHANDLE, HWND_FUNCIF, (HWND)&(pData->LastMessage), pData->szMsgFile ); - - //if ( !fContinue ) - { - // T5LOG(T5ERROR) << "EqfStartSession():: fContinue is false"; - // usRC = 1; - } /* endif */ } /* endif */ // set directory strings and main drive diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index bb4b9cb5..3ddb3e08 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -387,7 +387,7 @@ USHORT EqfMemory::FindTargetAndDelete( TmDel.currentInternalKey.setInternalKey(lKey, usTarget); pTmExtOut->usTargetKey = usTarget; - TMDelTargetClb( pTmRecord, pTMXTargetRecord, pClb ); + TMDelTargetClb( TmRecord, pTMXTargetRecord, pClb ); //add updated tm record to database /**********************************************/ @@ -612,7 +612,7 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( if(pTargetClb->ulSegmId != TmDel.lSegmentId){ return SEGMENT_ID_NOT_EQUAL; } else { - TMDelTargetClb( pTmRecord, pTMXTargetRecord, pTargetClb ); + TMDelTargetClb( TmRecord, pTMXTargetRecord, pTargetClb ); //add updated tm record to database /**********************************************/ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 3d1751c7..19551b51 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -110,6 +110,8 @@ USHORT EqfMemory::TmtXExtract /********************************************************************/ if ( !usRc && fSpecialMode) { + T5LOG(T5ERROR) << "SpecialMode is temporary commented"; + #ifdef TEMPORARY_COMMENTED PTMX_TABLE pTable = NULL; // table containing the requested info ULONG ulMaxEntries = 0; // last + 1 entry in table @@ -120,19 +122,15 @@ USHORT EqfMemory::TmtXExtract { case MEM_OUTPUT_TAGTABLES : pTable = &TagTables; - ulMaxEntries = pTable->ulMaxEntries; break; case MEM_OUTPUT_AUTHORS : pTable = &Authors; - ulMaxEntries = pTable->ulMaxEntries; break; case MEM_OUTPUT_DOCUMENTS : pTable = &FileNames; - ulMaxEntries = pTable->ulMaxEntries; break; case MEM_OUTPUT_LANGUAGES : pTable = &Languages; - ulMaxEntries = pTable->ulMaxEntries; break; case MEM_OUTPUT_LONGNAMES : // different handling for long names, no setting of pTable @@ -141,7 +139,6 @@ USHORT EqfMemory::TmtXExtract case MEM_OUTPUT_ALLDOCS : // different handling for long names, no setting of pTable pTable = &FileNames; - ulMaxEntries = pTable->ulMaxEntries; break; default : LOG_AND_SET_RC(usRc, T5INFO, BTREE_INVALID); @@ -252,6 +249,7 @@ USHORT EqfMemory::TmtXExtract pTmExtOut->usTranslationFlag = TRANSLFLAG_NORMAL; } /* endif */ } /* endif */ +#endif// TEMPORARY_COMMENTED } /* endif */ /********************************************************************/ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index 5230eeb3..a42dfdda 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -166,6 +166,9 @@ USHORT EqfMemory::Load(){ return usRC; } + +DECLARE_int32(triplesthreshold); + USHORT EqfMemory::LoadMem() { //TimedMutexGuard l(tmMutex); @@ -389,16 +392,6 @@ USHORT EqfMemory::LoadMem() } /* endswitch */ } /* endif */ - // just for debugging purposes: check if short and long name table have the same numbe rof entries - if ( usTempRc == NO_ERROR ) - { - if ( LongNames.stTableEntry.size() != FileNames.ulMaxEntries ) - { - // this is strange... - int iSetBreakPointHere = 1; - } - } /* endif */ - if ( usTempRc != NO_ERROR ) { usRc = usTempRc; @@ -407,7 +400,7 @@ USHORT EqfMemory::LoadMem() } /* endif */ //add threshold to control block - usThreshold = 33;//pTmOpenIn->stTmOpen.usThreshold; + usThreshold = FLAGS_triplesthreshold;//pTmOpenIn->stTmOpen.usThreshold; if ( (usRc == NO_ERROR) || diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index d4a6ddd0..bab2ffbc 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -338,6 +338,9 @@ VOID HashSentence //while ( pTermTokens->usLength ) for(auto& token: pSentence->pTermTokens) { + if(token.usLength == 0 && token.usOffset == 0){ + break;//end-of-list-token + } pNormOffset = pSentence->pStrings->getNormStrC() + token.usOffset; token.usHash = HashTupelW( pNormOffset, token.usLength ); @@ -350,8 +353,6 @@ VOID HashSentence } //max nr of hashes built usCount++; - //pTermTokens++; - } /* endwhile */ /********************************************************************/ @@ -1217,46 +1218,43 @@ USHORT DetermineTmRecord BOOL fOk = true; // success indicator PTMX_INDEX_ENTRY pIndexEntry; // pointer to index entry structure - //for all votes add the index to the corresponding list pulVotes = pSentence->pulVotes.data(); - //allocate 32K for tm index record - pIndexRecord.resize(TMX_REC_SIZE); - - lKey = (*pulVotes) & START_KEY; - memset( pIndexRecord.data(), 0, pIndexRecord.size() ); usRc = pTmClb->InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record - if ( usRc == NO_ERROR ) { //calculate number of index entries in index record ulLen = toIndexRecord(pIndexRecord)->usRecordLen; - usMaxEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); - - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - std::string msg = __func__ + std::string(":: Number Entries: ") + toStr(usMaxEntries).c_str() +"; Entries:"; - pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; - for (j=0 ; jstIndexEntry; + for (j=0 ; jstIndexEntry; for ( j = 0; j < usMaxEntries; j++, pIndexEntry++ ) { if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) { - lSids.push_back(NTMKEY(*pIndexEntry)); + lSids.push_back(NTMKEY(*pIndexEntry)); } /* endif */ } /* endfor */ @@ -1413,7 +1411,6 @@ USHORT EqfMemory::UpdateTmRecord OtmProposal& TmProposal //pointer to get in data ) { - BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indication to leave while loop std::vector lSids; //ptr to sentence ids int iSidCount=0; @@ -1421,8 +1418,7 @@ USHORT EqfMemory::UpdateTmRecord ULONG ulLen; //length indicator LONG lKey; //tm record key std::vector TmRecord; //pointer to tm record - - TmRecord.resize(TMX_REC_SIZE); + lSids.reserve(MAX_INDEX_LEN + 5); usRc = DetermineTmRecord( this, TmProposal.pInputSentence, lSids ); @@ -1500,67 +1496,6 @@ USHORT EqfMemory::UpdateTmRecord // else try next sentence key //------------------------------------------------------------------------------ -USHORT EqfMemory::UpdateTmRecordByInternalKey -( - OtmProposal& TmProposal -) -{ - BOOL fOK = true; //success indicator - BOOL fStop = FALSE; //indication to leave while loop - std::vector lSids; //ptr to sentence ids - int iSidCount = 0; - USHORT usRc = NO_ERROR; //return code - ULONG ulLen; //length indicator - LONG lKey; //tm record key - PTMX_RECORD pTmRecord = NULL; //pointer to tm record - - std::vector TmRecord; // space for user data - TmRecord.resize(TMX_REC_SIZE); - lSids.reserve(MAX_INDEX_LEN + 5); - - usRc = DetermineTmRecord( this, TmProposal.pInputSentence, lSids ); - if ( usRc == NO_ERROR ) - { - //get tm record(s) - //while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) - for(LONG lKey: lSids) - { - iSidCount++; - memset( &TmRecord, 0, TmRecord.size() ); - usRc = TmBtree.EQFNTMGet(lKey, //tm record key - TmRecord); //pointer to tm record data - - if ( usRc == NO_ERROR ) - { - //compare tm record data with data passed in the get in structure - usRc = ComparePutData( TmRecord, TmProposal, lKey ); - - if ( usRc == SOURCE_STRING_ERROR ) - { - //get next tm record - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - } - else - { - //new target record was added or an existing one was successfully - //replaced so don't try other sids - fStop = usRc == NO_ERROR; - break; - } /* endif */ - } /* endif */ - } /* endfor */ - - if ( iSidCount == lSids.size() && !fStop ) - { - //issue message that tm needs to be organized if pulsid is empty and - //no get was successful - LOG_AND_SET_RC(usRc, T5INFO, ERROR_ADD_TO_TM); - } /* endif */ - } /* endif */ - - - return( usRc ); -} DECLARE_bool(ignore_newer_target_exists_check); DECLARE_bool(log_memmove_in_compareputdata); @@ -1778,8 +1713,8 @@ USHORT EqfMemory::ComparePutData size_t pClbOffs = (PBYTE)pClb - pTmRecord.data(); size_t pTMXTargetRecordOffs = (PBYTE)pTMXTargetRecord - pTmRecord.data(); pTmRecord.resize(ulNewSize); - pClb = (PTMX_TARGET_CLB) pTmRecord.data() + pClbOffs; - pTMXTargetRecord = (PTMX_TARGET_RECORD) pTmRecord.data() + pTMXTargetRecordOffs; + pClb = (PTMX_TARGET_CLB) &pTmRecord[pClbOffs]; + pTMXTargetRecord = (PTMX_TARGET_RECORD) &pTmRecord[pTMXTargetRecordOffs]; } /* endif */ // make room at pCLB for a new CLB and adjust TM record @@ -1868,8 +1803,6 @@ USHORT EqfMemory::ComparePutData if ( fStop ) { TmProposal.currentInternalKey.setInternalKey(lKey, targetKey); - //TmProposal.recordKey = *pulKey; - //TmProposal.targetKey = targetKey; }else{ //all target records have been checked but nothing overlapped //so add new target record to end of tm record @@ -2112,17 +2045,16 @@ VOID FillTargetRecord /**********************************************************************/ BOOL TMDelTargetClb ( - PTMX_RECORD pTmRecord, // ptr to TM record + BytesRef TmRecord, // ptr to TM record PTMX_TARGET_RECORD pTargetRecord, // ptr to target record within TM record PTMX_TARGET_CLB pTargetClb // ptr to target control record ) { BOOL fTargetRemoved = FALSE; - PBYTE pTmEnd; + PBYTE pTmEnd = TmRecord.data(); PBYTE pEndTarget; USHORT usTargetCLBLen; - pTmEnd = (PBYTE)pTmRecord; - pTmEnd += RECLEN(pTmRecord); + pTmEnd += RECLEN(toTmxRecord(TmRecord)); // check if there are more than one CLBs in current target record usTargetCLBLen = TARGETCLBLEN(pTargetClb); @@ -2135,7 +2067,7 @@ BOOL TMDelTargetClb pEndTarget = (PBYTE)pTargetRecord + RECLEN(pTargetRecord); //calculate new length of TM record - RECLEN(pTmRecord) -= RECLEN(pTargetRecord); + RECLEN(toTmxRecord(TmRecord)) -= RECLEN(pTargetRecord); //remove the current target record memmove( pTargetRecord, pEndTarget, pTmEnd - pEndTarget ); @@ -2144,8 +2076,8 @@ BOOL TMDelTargetClb fTargetRemoved = TRUE; // initialize last target record - pTmEnd = (PBYTE)pTmRecord; - pTmEnd += RECLEN(pTmRecord); + pTmEnd = TmRecord.data(); + pTmEnd += RECLEN(toTmxRecord(TmRecord)); pTargetRecord = (PTMX_TARGET_RECORD)pTmEnd; RECLEN(pTargetRecord) = 0; } @@ -2154,7 +2086,7 @@ BOOL TMDelTargetClb // delete only current target CLB //calculate new length of tm record - RECLEN(pTmRecord) -= usTargetCLBLen; + RECLEN(toTmxRecord(TmRecord)) -= usTargetCLBLen; RECLEN(pTargetRecord) -= usTargetCLBLen; //remove the current target CLB @@ -2259,8 +2191,9 @@ LONG EQFCompress2Unicode( std::wstring& pOutput, PBYTE pTarget, ULONG ulLenComp BYTE b = *pTarget++; if ( b == 0 ) // no compression { - lLen = ulLenComp / sizeof(CHAR_W); - pOutput.insert(pOutput.begin(), pTarget, pTarget + lLen); + lLen = ulLenComp / sizeof(CHAR_W); + PSZ_W pWCharTarget = (PSZ_W) pTarget; + pOutput.insert(pOutput.begin(), pWCharTarget, pWCharTarget + lLen); pOutput[ lLen ] = EOS; } else if (b == BOCU_COMPRESS) @@ -2482,7 +2415,7 @@ USHORT TMLoopAndDelTargetClb pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); if(pTargetKey) *pTargetKey = 0; - while ( ( lLeftTgtLen <= RECLEN(pTMXTgtRec) ) && (RECLEN(pTMXTgtRec) != 0) && !fDel ) + while ( lLeftTgtLen>0 && ( lLeftTgtLen <= RECLEN(pTMXTgtRec) ) && (RECLEN(pTMXTgtRec) != 0) && !fDel ) { NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTgtRec), 4713); lLeftTgtLen -= RECLEN(pTMXTgtRec); @@ -2505,7 +2438,7 @@ USHORT TMLoopAndDelTargetClb lLeftClbLen -= TARGETCLBLEN(pClb); // loop over all CLBs of this target record and remove // any CLB for the current segment - fTgtRemoved = TMDelTargetClb( toTmxRecord(pTmRecord), pTMXTgtRec, pClb); + fTgtRemoved = TMDelTargetClb( pTmRecord, pTMXTgtRec, pClb); fDel = TRUE; } else diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index ecd0ed77..01d61534 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -313,7 +313,7 @@ USHORT EqfMemory::NTMGetIDFromNameEx( { pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, LongNamesCaseIgnore.stTableEntry.data(), - LongNames.stTableEntry.size(), + LongNamesCaseIgnore.stTableEntry.size(), sizeof(TMX_LONGNAME_TABLE_ENTRY), NTMLongNameTableCompCaseIgnore); } /* endif */ @@ -352,7 +352,7 @@ USHORT EqfMemory::NTMGetIDFromNameEx( TMX_LONGNAME_TABLE_ENTRY Entry; // ptr to table entry LongNames.pszBuffer.push_back(*pusID); - Entry.longNameStartOffset = LongNames.stTableEntry.size(); + Entry.longNameStartOffset = LongNames.pszBuffer.size();//LongNames.stTableEntry.size(); LongNames.pszBuffer.insert(LongNames.pszBuffer.end(), pszLongName, pszLongName + strlen(pszLongName)); LongNames.pszBuffer.push_back(0);//as USHORT separator LongNames.pszBuffer.push_back(0); @@ -383,10 +383,6 @@ USHORT EqfMemory::NTMGetIDFromNameEx( toUpper(LongNamesCaseIgnore.pszBuffer); LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); - //memcpy(LongNamesCaseIgnore.stTableEntry.data(), - // LongNames.stTableEntry.data(), - // LongNames.stTableEntry.size() * sizeof(TMX_LONGNAME_TABLE_ENTRY)); - qsort(LongNamesCaseIgnore.stTableEntry.data(), LongNamesCaseIgnore.stTableEntry.size(), sizeof(TMX_LONGNAME_TABLE_ENTRY), @@ -411,10 +407,10 @@ USHORT EqfMemory::NTMGetIDFromNameEx( /********************************************************************/ /* search name passed in pszName in passed table usTableType */ /********************************************************************/ - if(pstTMTable->ulMaxEntries){ + if(pstTMTable->stTmTableEntry.size()){ pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch(pszName, pstTMTableEntries, - pstTMTable->ulMaxEntries, + pstTMTable->stTmTableEntry.size(), sizeof(TMX_TABLE_ENTRY), NTMCompNames); } @@ -602,7 +598,7 @@ USHORT EqfMemory::NTMGetNameFromID(PUSHORT pusID, // intput else { ULONG ulI; - for (ulI = 0; ulI < pstTMTable->ulMaxEntries && !fFound; ulI++) + for (ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) { if (*pusID == pstTMTableEntries[ulI].usId) { @@ -682,7 +678,7 @@ PSZ EqfMemory::NTMFindNameForID( else { ULONG ulI; - for (ulI = 0; ulI < pstTMTable->ulMaxEntries && !fFound; ulI++) + for (ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) { if (*pusID == pstTMTableEntries[ulI].usId) { @@ -1023,11 +1019,11 @@ T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be int tmxTableToBuffer(PTMX_TABLE pTable, BytesRef buff){ buff.clear(); - int occupSize = sizeof(LONG) + (pTable->ulMaxEntries + 1) * sizeof(TMX_TABLE_ENTRY); + int occupSize = sizeof(LONG) + (pTable->stTmTableEntry.size() + 1) * sizeof(TMX_TABLE_ENTRY); buff.resize(occupSize); TMX_TABLE_OLD *table = (TMX_TABLE_OLD *)buff.data(); //pOldTable = pstTMTable->stTmTableEntry; - table->ulMaxEntries = pTable->ulMaxEntries; + table->ulMaxEntries = pTable->stTmTableEntry.size();//pTable->ulMaxEntries; if (table->ulMaxEntries) { memcpy(&table->stTmTableEntry, @@ -1109,26 +1105,20 @@ USHORT EqfMemory::NTMAddNameToTable( /* will not exceed the maximum size of a QDAM record (32K) */ /*************************************************************/ - const size_t currentCap = pstTMTable->stTmTableEntry.size(); - if (pstTMTable->ulMaxEntries >= currentCap) - { - size_t newCap = currentCap + 10; - pstTMTable->stTmTableEntry.resize(newCap); - pstTMTableEntries = &pstTMTable->stTmTableEntry[0]; - } + TMX_TABLE_ENTRY& newEntry = pstTMTable->stTmTableEntry.back(); + *pusID = newEntry.usId = pstTMTable->stTmTableEntry.size();//++pstTMTable->ulMaxEntries; + strncpy(newEntry.szName, pszName, MAX_LANG_LENGTH - 1); + pstTMTable->stTmTableEntry.push_back(TMX_TABLE_ENTRY()); + if (usRc == NO_ERROR) { /*************************************************************/ /* insert name and id to table and sort table */ /*************************************************************/ - int pos = pstTMTable->ulMaxEntries; - *pusID = ++pstTMTable->ulMaxEntries; - strncpy(pstTMTableEntries[pos].szName, - pszName, MAX_LANG_LENGTH - 1); - pstTMTableEntries[pos].usId = *pusID; - qsort(pstTMTableEntries, - pstTMTable->ulMaxEntries, + + qsort(pstTMTable->stTmTableEntry.data(), + pstTMTable->stTmTableEntry.size(), sizeof(TMX_TABLE_ENTRY), NTMCompNames); @@ -1273,13 +1263,12 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab if (usRc == NO_ERROR) { - pTMTable->ulMaxEntries = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; + pTMTable->ulMaxEntries1 = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; pTMTable->stTmTableEntry.clear(); - pTMTable->stTmTableEntry.resize(pTMTable->ulMaxEntries + 1); - if (pTMTable->ulMaxEntries) - { - memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, pTMTable->ulMaxEntries * sizeof(TMX_TABLE_ENTRY)); - } + pTMTable->stTmTableEntry.resize((((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries + 1)); + //if (pTMTable->stTmTableEntry.size()) + memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, (pTMTable->stTmTableEntry.size()-1) * sizeof(TMX_TABLE_ENTRY)); + } return usRc; } @@ -1373,7 +1362,7 @@ USHORT EqfMemory::NTMCreateLangGroupTable() // allocate language-ID-to-group-ID-table - LONG lSize = Languages.ulMaxEntries > 100L ? Languages.ulMaxEntries : 100L; + LONG lSize = Languages.stTmTableEntry.size() > 100L ? Languages.stTmTableEntry.size() : 100L; if (lSize <= 0) lSize = 1; psLangIdToGroupTable.resize(lSize); @@ -1384,8 +1373,8 @@ USHORT EqfMemory::NTMCreateLangGroupTable() if (usRC == NO_ERROR) { int i = 0; - while ((usRC == NO_ERROR) && - (i < (int)Languages.ulMaxEntries)) + while ((usRC == NO_ERROR) && (i < (int)Languages.stTmTableEntry.size()) + ) { usRC = NTMAddLangGroup(Languages.stTmTableEntry[i].szName, Languages.stTmTableEntry[i].usId); diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 1db849e8..7a9ecfcb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -43,7 +43,7 @@ EqfMemory::~EqfMemory() */ int EqfMemory::getNumOfMarkupNames() { - return( (int)(TagTables.ulMaxEntries) ); + return( (int)(TagTables.stTmTableEntry.size()) ); } /*! \brief Get markup name at position n [n = 0.. GetNumOfMarkupNames()-1] @@ -59,7 +59,7 @@ int EqfMemory::getMarkupName int iSize ) { - if ( (iPos >= 0) && (iPos < (int)(TagTables.ulMaxEntries)) ) + if ( (iPos >= 0) && (iPos < (int)(TagTables.stTmTableEntry.size())) ) { return( CopyToBuffer( TagTables.stTmTableEntry[iPos].szName, pszBuffer, iSize ) ); } @@ -398,7 +398,7 @@ int EqfMemory::getLanguage int iSize ) { - if ( (iPos >= 0) && (iPos < (int)(Languages.ulMaxEntries)) ) + if ( (iPos >= 0) && (iPos < (int)(Languages.stTmTableEntry.size())) ) { return( CopyToBuffer( Languages.stTmTableEntry[iPos].szName, pszBuffer, iSize ) ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp index a883ebb2..7c5d3e9c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp @@ -289,7 +289,6 @@ int EqfMemoryPlugin::handleError( int iRC, char *pszMemName, char *pszMarkup, ch UtlGetMsgTxt( ERROR_DISK_FULL_MSG, pszErrorTextBuffer, 1, pReplAddr ); } break; - case DB_FULL: case BTREE_LOOKUPTABLE_TOO_SMALL: UtlGetMsgTxt( ERROR_MEM_DB_FULL, pszErrorTextBuffer, 1, pReplAddr ); diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 7934617c..afdd3b2d 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -2142,7 +2142,7 @@ VOID BTREE::QDAMDictUpdStatus() { } /* end of function QDAMDictUpdStatus */ -ULONG QDAMGetszKey_V3 +LONG QDAMGetszKey_V3 ( std::shared_ptr& pRecord, // active record USHORT i // get data term @@ -2156,8 +2156,8 @@ ULONG QDAMGetszKey_V3 PUSHORT pusOffset = (PUSHORT) pRecord->contents.uchData; pusOffset += i; // point to key - int step = sizeof(USHORT ) + sizeof(RECPARAM); // get pointer to data - USHORT res = -1; + constexpr int step = sizeof(USHORT ) + sizeof(RECPARAM); // get pointer to data + LONG res = -1; if ( (PBYTE)pusOffset > pEndOfRec ) { @@ -2175,9 +2175,8 @@ ULONG QDAMGetszKey_V3 T5LOG(T5ERROR) << "QDAMGetszKey_V3:: data pointer is out of range , pusOffset = " << (long int)pusOffset << "; pEndOfRec = " << (long int)pEndOfRec; pData = nullptr; - //ERREVENT2( QDAMGETSZKEY_LOC, INTFUNCFAILED_EVENT, 2, DB_GROUP, "" ); }else{ - res = (USHORT)*pData; + res = *((PLONG)pData); } /* endif */ } /* endif */ @@ -2203,9 +2202,6 @@ RECPARAM QDAMGetrecData_V3 pData += sizeof(USHORT ); // get pointer to datarec memcpy( &recData, (PRECPARAM) pData, sizeof( RECPARAM )); - //memcpy( &recData.usNum, (PUSHORT) pData, sizeof( USHORT ) ); - //memcpy( &recData.usOffset,(PUSHORT) (pData+2), sizeof( USHORT ) ); - //memcpy( &recData.ulLen, (PULONG) (pData+8), sizeof( PULONG ) ); return ( recData ); } diff --git a/source/opentm2/core/utilities/EQFPRO00.cpp b/source/opentm2/core/utilities/EQFPRO00.cpp index 89acfe71..be83e214 100755 --- a/source/opentm2/core/utilities/EQFPRO00.cpp +++ b/source/opentm2/core/utilities/EQFPRO00.cpp @@ -300,19 +300,7 @@ USHORT PutItAway( PPROPCNTL pcntl) } -/*! - Drop properties file from memory -*/ -VOID DropPropFile( PPROP_IDA pIda, PPROPCNTL pcntl) -{ - UtlAlloc( (PVOID *)&pcntl->pszFname, 0L, 0L, NOMSG ); - if( pcntl->Plug.Fw == NULP) - pIda->LastCntl = (PPROPCNTL)pcntl->Plug.Bw; - if( pcntl->Plug.Bw == NULP) - pIda->TopCntl = (PPROPCNTL)pcntl->Plug.Fw; - UtlPlugOut( &pcntl->Plug); - UtlAlloc( (PVOID *)&pcntl, 0L, 0L, NOMSG ); -} + /*! Find Properties Control block From c68e7ed9e752f41c07acdf87cb9befff8f4fbd2f Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 23 Apr 2025 15:24:56 +0300 Subject: [PATCH 12/62] * skipped empty lang --- source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 01d61534..ae00a619 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -1376,8 +1376,10 @@ USHORT EqfMemory::NTMCreateLangGroupTable() while ((usRC == NO_ERROR) && (i < (int)Languages.stTmTableEntry.size()) ) { - usRC = NTMAddLangGroup(Languages.stTmTableEntry[i].szName, - Languages.stTmTableEntry[i].usId); + jf(Languages.stTmTableEntry[i].szName[0]){ + usRC = NTMAddLangGroup(Languages.stTmTableEntry[i].szName, + Languages.stTmTableEntry[i].usId); + } i++; } /* endwhile */ } /* endif */ From 7b0cd515a2ca1507aab9a9247ccbbade2f2a3f7f Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 10 Jun 2025 14:39:45 +0300 Subject: [PATCH 13/62] * fixed tm tables sort, init and adding new elements * code cleanup --- include/EQFQDAMI.H | 1 - include/lowlevelotmdatastructs.h | 22 +- include/tm.h | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 261 +++--------------- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 37 +-- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 3 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 15 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 105 +++---- .../core/EqfMemoryPlugin/EqfMemory.cpp | 8 +- source/opentm2/core/utilities/EQFDICT.cpp | 22 +- 10 files changed, 117 insertions(+), 361 deletions(-) diff --git a/include/EQFQDAMI.H b/include/EQFQDAMI.H index df04500a..021f72e2 100755 --- a/include/EQFQDAMI.H +++ b/include/EQFQDAMI.H @@ -910,7 +910,6 @@ typedef struct _DICTTIME SHORT QDAMGetszKeyParam_V3( PBTREE, RECPARAM, LONG& ); - SHORT QDAMGetszData_V3 ( PBTREE, RECPARAM, BytesRef, CHAR ); BOOL QDAMDeleteKey_V3( PBTREE, std::shared_ptr&, PCHAR); SHORT QDAMDestroy ( PBTREE ); diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 5a00f70f..237d08b3 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -482,7 +482,8 @@ struct TMX_TABLE_ENTRY TMX_TABLE_ENTRY() { - szName[0] = '\0'; + memset(szName, 0, sizeof(szName)); + //szName[0] = '\0'; } }; typedef TMX_TABLE_ENTRY * PTMX_TABLE_ENTRY; @@ -499,11 +500,10 @@ typedef TMX_TABLE_OLD * PTMX_TABLE_OLD; struct TMX_TABLE { - ULONG ulMaxEntries1 = 0; std::vector stTmTableEntry; TMX_TABLE(){ stTmTableEntry.push_back(TMX_TABLE_ENTRY());//push_empty_entry - memset(&stTmTableEntry.back(), 0, sizeof(stTmTableEntry.back())); + memset(&stTmTableEntry.back(), 0, sizeof(TMX_TABLE_ENTRY)); } }; @@ -801,12 +801,12 @@ struct BTREE //+----------------------------------------------------------------------------+ // Returncodes: same as for QDAMDictExactLocal... //+----------------------------------------------------------------------------+ - SHORT - EQFNTMGet - ( - LONG lKey, // key to be searched for - BytesRef uchBuff // space for user data - ); + //SHORT + //EQFNTMGet + //( + //LONG lKey, // key to be searched for + //BytesRef uchBuff // space for user data + //); @@ -916,7 +916,7 @@ struct BTREE SHORT QDAMDeleteDataFromBuffer_V3( RECPARAM recParam); SHORT QDAMDictUpdateLocal ( PWCHAR, BytesRef, bool ); - SHORT QDAMDictExactLocal ( LONG&, BytesRef, USHORT ); + SHORT EQFNTMGet ( LONG&, BytesRef ); SHORT QDAMDictCreateLocal ( TMX_SIGN*, LONG, bool keepInRamOnly = false); @@ -1147,6 +1147,8 @@ BOOL QDAMTerseData SHORT QDAMLocateKey_V3( std::shared_ptr&, LONG&, PSHORT, SEARCHTYPE, PSHORT); + + SHORT QDAMGetszData_V3 ( RECPARAM, BytesRef ); };//BTREE diff --git a/include/tm.h b/include/tm.h index 50649359..9cb56a7a 100644 --- a/include/tm.h +++ b/include/tm.h @@ -82,7 +82,7 @@ struct TMX_EXT_OUT_W ULONG ulRecKey; USHORT usTargetKey; - ULONG ulTmKey; // tm record key + LONG lTmKey; // tm record key USHORT usNextTarget; // which target record to address next ULONG ulMaxEntries; // number of entries in tm data file }; @@ -1737,7 +1737,7 @@ using PTMX_GET_OUT_W = TMX_GET_OUT_W *; // structure TMX_EXT_IN typedef struct _TMX_EXT_IN { - ULONG ulTmKey; // tm get to get + LONG lTmKey; // tm get to get USHORT usConvert; // how the output is to appear USHORT usNextTarget; // which target record to address next } TMX_EXT_IN, *PTMX_EXT_IN; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index c5871355..198515fb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -64,7 +64,7 @@ USHORT EqfMemory::TmtXExtract TMX_EXT_OUT_W * pTmExtOut //ptr to output struct ) { - USHORT usRc = NO_ERROR; //return code + USHORT usRc = BTREE_NOT_FOUND; //return code BOOL fOK = true; //success indicator PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record @@ -74,230 +74,45 @@ USHORT EqfMemory::TmtXExtract ULONG ulOemCP = 1L; std::vector pTmRecord; // space for user data - pTmRecord.resize(2*TMX_REC_SIZE); - - //allocate 32K for tm record - //allocate pString - - if ( !usRc ) + //pTmRecord.resize(2*TMX_REC_SIZE); + + TmBtree.EQFNTMGetNextNumber( &lStartKey, &lNextKey); + pTmExtOut->ulMaxEntries = (lNextKey - lStartKey); + /******************************************************************/ + /* return one matching entry (if any available) */ + /******************************************************************/ + while ( (pTmExtIn->lTmKey < lNextKey) && (usRc == BTREE_NOT_FOUND) ) { - SHORT sRetries = MAX_RETRY_COUNT; + usRc = TmBtree.EQFNTMGet(pTmExtIn->lTmKey, pTmRecord ); + + if ( usRc == NO_ERROR ) { - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); // reset return code - /********************************************************************/ - /* Set special mode flag */ - /********************************************************************/ - if ( !usRc ) + usRc = ExtractRecordV6(pTmRecord, pTmExtIn, pTmExtOut ); + } + /****************************************************************/ + /* setup new starting point (do this even in the case we are */ + /* dealing with a corrupted TM ) */ + /****************************************************************/ + if ( usRc != NO_ERROR ) + { + if ( (usRc == BTREE_NOT_FOUND) || ( usRc == BTREE_CORRUPTED ) || (usRc == BTREE_BUFFER_SMALL) || ((usAccessMode & ASD_ORGANIZE) != 0) ) { - if ( (pTmExtIn->usConvert == MEM_OUTPUT_TAGTABLES) || - (pTmExtIn->usConvert == MEM_OUTPUT_AUTHORS) || - (pTmExtIn->usConvert == MEM_OUTPUT_LONGNAMES) || - (pTmExtIn->usConvert == MEM_OUTPUT_ALLDOCS) || - (pTmExtIn->usConvert == MEM_OUTPUT_DOCUMENTS) || - (pTmExtIn->usConvert == MEM_OUTPUT_LANGUAGES) ) - { - fSpecialMode = TRUE; - } - else - { - fSpecialMode = FALSE; - } /* endif */ + pTmExtIn->lTmKey ++; + pTmExtIn->usNextTarget = 1; + pTmExtOut->lTmKey = pTmExtIn->lTmKey; + pTmExtOut->usNextTarget = pTmExtIn->usNextTarget; + usRc = BTREE_NOT_FOUND; // reset error condition } /* endif */ + } /* endif */ + } /* endwhile */ - /********************************************************************/ - /* Special mode to get list of tag tables, authors, documents or */ - /* languages */ - /********************************************************************/ - if ( !usRc && fSpecialMode) - { - T5LOG(T5ERROR) << "SpecialMode is temporary commented"; - #ifdef TEMPORARY_COMMENTED - PTMX_TABLE pTable = NULL; // table containing the requested info - ULONG ulMaxEntries = 0; // last + 1 entry in table - - /******************************************************************/ - /* Address table containg the requested names */ - /******************************************************************/ - switch ( pTmExtIn->usConvert ) - { - case MEM_OUTPUT_TAGTABLES : - pTable = &TagTables; - break; - case MEM_OUTPUT_AUTHORS : - pTable = &Authors; - break; - case MEM_OUTPUT_DOCUMENTS : - pTable = &FileNames; - break; - case MEM_OUTPUT_LANGUAGES : - pTable = &Languages; - break; - case MEM_OUTPUT_LONGNAMES : - // different handling for long names, no setting of pTable - ulMaxEntries = LongNames.stTableEntry.size(); - break; - case MEM_OUTPUT_ALLDOCS : - // different handling for long names, no setting of pTable - pTable = &FileNames; - break; - default : - LOG_AND_SET_RC(usRc, T5INFO, BTREE_INVALID); - break; - } /* endswitch */ - - /******************************************************************/ - /* Extract requested names */ - /******************************************************************/ - if ( !usRc ) - { - PTMX_TABLE_ENTRY pEntry; // ptr to current entry - PSZ_W pszBuffer; // ptr to next free space in buffer - LONG lLen; // lenght of current name - LONG lRoomLeft; // space left in current buffer - int iteration; // iteration counter - PSZ pszName; // ptr to current name - - /****************************************************************/ - /* Initialize our output buffers */ - /****************************************************************/ - memset( pTmExtOut->szSource, NULC, - sizeof(pTmExtOut->szSource) ); - memset( pTmExtOut->szTarget, NULC, - sizeof(pTmExtOut->szTarget) ); - - /****************************************************************/ - /* Loop two times: first time to fill szSource, the second */ - /* time to fill the szTarget buffer */ - /****************************************************************/ - iteration = 0; - do - { - // set buffer and lRoomLeft variables - if ( iteration == 0 ) - { - // use szSource buffer - pszBuffer = pTmExtOut->szSource; - // leave room for NULC and ASCII expansion - lRoomLeft = (sizeof(pTmExtOut->szSource)/sizeof(CHAR_W))/2 - 1; - } - else - { - // use szTarget buffer - pszBuffer = pTmExtOut->szTarget; - // leave room for NULC and ASCII expansion - lRoomLeft = (sizeof(pTmExtOut->szTarget)/sizeof(CHAR_W))/2 - 1; - } /* endif */ - - while ( lRoomLeft && (pTmExtIn->usNextTarget < ulMaxEntries) ) - { - if ( pTmExtIn->usConvert == MEM_OUTPUT_LONGNAMES ) - { - pszName = LongNames.stTableEntry[pTmExtIn->usNextTarget].pszLongName; - } - else - { - pEntry = &pTable->stTmTableEntry[pTmExtIn->usNextTarget]; - pszName = pEntry->szName; - // check if there is a long name for the current document - if ( pTmExtIn->usConvert == MEM_OUTPUT_ALLDOCS ) - { - ULONG ulEntry = 0; - while ( (ulEntry < LongNames.stTableEntry.size()) && - (pEntry->usId != LongNames.stTableEntry[ulEntry].usId ) ) - { - ulEntry++; - } /* endwhile */ - if ( ulEntry < LongNames.stTableEntry.size() ) - { - pszName = LongNames.stTableEntry[ulEntry].pszLongName; - } /* endif */ - } /* endif */ - } /* endif */ - lLen = strlen(pszName); - - if ( lRoomLeft > (lLen + 1)) - { - ASCII2Unicode( pszName, pszBuffer, ulOemCP ); - pszBuffer += lLen; - *pszBuffer++ = X15; - lRoomLeft -= lLen + 1; - pTmExtIn->usNextTarget++; - } - else - { - lRoomLeft = 0; - } /* endif */ - } /* endwhile */ - - *pszBuffer = NULC; // terminate current buffer - iteration++; // continue with next buffer - } while ( (iteration <= 1) && (pTmExtIn->usNextTarget < ulMaxEntries) ); /* enddo */ - - /****************************************************************/ - /* Set flags in output structure */ - /****************************************************************/ - if ( pTmExtIn->usNextTarget < ulMaxEntries ) - { - // more names available for extract - pTmExtOut->usNextTarget = pTmExtIn->usNextTarget; - pTmExtOut->usTranslationFlag = TRANSLFLAG_MACHINE; - } - else - { - // all names have been extracted - pTmExtOut->usNextTarget = 0; - pTmExtOut->usTranslationFlag = TRANSLFLAG_NORMAL; - } /* endif */ - } /* endif */ -#endif// TEMPORARY_COMMENTED - } /* endif */ - - /********************************************************************/ - /* Normal mode of TMExtract */ - /********************************************************************/ - if ( !usRc && !fSpecialMode ) - { - TmBtree.EQFNTMGetNextNumber( &lStartKey, &lNextKey); - pTmExtOut->ulMaxEntries = (lNextKey - lStartKey); - /******************************************************************/ - /* return one matching entry (if any available) */ - /******************************************************************/ - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - while ( (pTmExtIn->ulTmKey < lNextKey) && (usRc == BTREE_NOT_FOUND) ) - { - usRc = TmBtree.EQFNTMGet(pTmExtIn->ulTmKey, pTmRecord ); - - if ( usRc == NO_ERROR ) - { - usRc = ExtractRecordV6(pTmRecord, pTmExtIn, pTmExtOut ); - } - /****************************************************************/ - /* setup new starting point (do this even in the case we are */ - /* dealing with a corrupted TM ) */ - /****************************************************************/ - if ( usRc != NO_ERROR ) - { - if ( (usRc == BTREE_NOT_FOUND) || ( usRc == BTREE_CORRUPTED ) || (usRc == BTREE_BUFFER_SMALL) || ((usAccessMode & ASD_ORGANIZE) != 0) ) - { - pTmExtIn->ulTmKey ++; - pTmExtIn->usNextTarget = 1; - pTmExtOut->ulTmKey = pTmExtIn->ulTmKey; - pTmExtOut->usNextTarget = pTmExtIn->usNextTarget; - usRc = BTREE_NOT_FOUND; // reset error condition - } /* endif */ - } /* endif */ - } /* endwhile */ - - /******************************************************************/ - /* set end of file condition ... */ - /******************************************************************/ - if ( (usRc == BTREE_NOT_FOUND) && (pTmExtIn->ulTmKey == lNextKey) ) - { - //arrived at last tm record - LOG_AND_SET_RC(usRc, T5INFO, BTREE_EOF_REACHED); - } /* endif */ - } /* endif */ - } + /******************************************************************/ + /* set end of file condition ... */ + /******************************************************************/ + if ( (usRc == BTREE_NOT_FOUND) && (pTmExtIn->lTmKey == lNextKey) ) + { + //arrived at last tm record + LOG_AND_SET_RC(usRc, T5INFO, BTREE_EOF_REACHED); } /* endif */ pTmExtOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); @@ -489,19 +304,19 @@ USHORT EqfMemory::ExtractRecordV6 pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); //if target exists - if ( ((RECLEN(pTMXTargetRecord) > 0) && (lLeftTgtLen > 0 ) ) + if ( ((lLeftTgtLen > 0 ) && (RECLEN(pTMXTargetRecord) > 0)) || (lLeftClbLen > 0) ) { //increase target count and leave tm record key number as before - pTmExtOut->ulTmKey = pTmExtIn->ulTmKey; + pTmExtOut->lTmKey = pTmExtIn->lTmKey; pTmExtOut->usNextTarget = pTmExtIn->usNextTarget + 1; } else { //no more target so get next tm record and initialize target count - pTmExtOut->ulTmKey = pTmExtIn->ulTmKey +1; + pTmExtOut->lTmKey = pTmExtIn->lTmKey +1; pTmExtOut->usNextTarget = 1; } /* endif */ } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index ca124eef..e2a7e267 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -1524,11 +1524,11 @@ int BTREE::deallocateLookupTableBuffer(int number){ // //------------------------------------------------------------------------------ -SHORT BTREE::QDAMDictExactLocal +SHORT BTREE::EQFNTMGet ( - LONG& lKey, // key to be searched for - BytesRef pchBuffer, // space for user data - USHORT usSearchSubType // special hyphenation lookup flag + LONG& lKey, // key to be searched for + BytesRef pchBuffer // space for user data + //USHORT usSearchSubType=FEXACT // special hyphenation lookup flag ) { SHORT i; @@ -1564,7 +1564,7 @@ SHORT BTREE::QDAMDictExactLocal usCurrentRecord = RECORDNUM( pRecord ); recData = QDAMGetrecData_V3( pRecord, i, usVersion ); - sRc = QDAMGetszData_V3( this, recData, pchBuffer, DATA_NODE ); + sRc = QDAMGetszData_V3( recData, pchBuffer ); } else @@ -1616,34 +1616,7 @@ SHORT BTREE::QDAMDictExactLocal //+----------------------------------------------------------------------------+ //|Function flow: call QDAMDictExactLocal to retrieve the data | // ----------------------------------------------------------------------------+ -SHORT -BTREE::EQFNTMGet -( - LONG lKey, // key to be searched for - BytesRef pchBuffer // space for user data -) -{ - SHORT sRc = 0; // return code - - SHORT RetryCount = MAX_RETRY_COUNT; // retry counter for in-use condition - do - { - sRc = QDAMDictExactLocal( lKey, pchBuffer, FEXACT ); - if ( sRc == BTREE_IN_USE ) - { - RetryCount--; - UtlWait( MAX_WAIT_TIME ); - } /* endif */ - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - if ( sRc && (sRc != BTREE_NOT_FOUND) ) - { - ERREVENT( EQFNTMGET_LOC, INTFUNCFAILED_EVENT, sRc ); - } /* endif */ - - return sRc; -} /* end of function EQFNTMGet */ //+----------------------------------------------------------------------------+ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index 8ea33a74..fe8c3963 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -261,7 +261,8 @@ USHORT EqfMemory::LoadMem() (usRc == VERSION_MISMATCH) ) { //get compact area and add to control block - USHORT usTempRc = TmBtree.EQFNTMGet( COMPACT_KEY, bCompact ); + LONG lKey = COMPACT_KEY; + USHORT usTempRc = TmBtree.EQFNTMGet( lKey, bCompact ); // in organize mode allow continue if compact area is corrupted if ( (usTempRc != NO_ERROR) && (usTempRc != VERSION_MISMATCH) ) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 80a35a27..4c18bbd8 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -212,13 +212,7 @@ USHORT EqfMemory::TmtXReplace if ( !usRc ) { HashSentence( TmProposal.pInputSentence.get() ); - toTmxTagtableRecord(TmProposal.pInputSentence->pTagRecord)->usTagTableId = 1; - - if(usRc){ - T5LOG( T5WARNING) << ":: NTMGetIDFromName( tagtable ) returned " << usRc; - } - - + toTmxTagtableRecord(TmProposal.pInputSentence->pTagRecord)->usTagTableId = 1; } /* endif */ if ( !usRc ) @@ -263,7 +257,7 @@ USHORT EqfMemory::TmtXReplace } /* endif */ if(!usRc){ - pTmPutOut->ulTmKey = TmProposal.currentInternalKey.getRecordKey(); + pTmPutOut->lTmKey = TmProposal.currentInternalKey.getRecordKey(); pTmPutOut->usTargetKey = TmProposal.currentInternalKey.getTargetKey(); RewriteCompactTable(); @@ -748,12 +742,11 @@ USHORT EqfMemory::AddToTm std::vector TmRecord; // ptr to tm record std::vector TargetClb; // ptr to target ctl block USHORT usRc = NO_ERROR; // return code - BOOL fOK = true; // success indicator USHORT usAddDataLen = 0; usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - TargetClb.resize(sizeof(TMX_TARGET_CLB)+usAddDataLen); + TargetClb.resize(sizeof(TMX_TARGET_CLB)+usAddDataLen + 1); usRc = FillClb( TargetClb, TmProposal ); if ( usRc == NO_ERROR ) @@ -1740,7 +1733,7 @@ USHORT EqfMemory::ComparePutData fStop = true; fOK = false; }else{ - if(FLAGS_log_memmove_in_compareputdata) + if(true || FLAGS_log_memmove_in_compareputdata) { T5LOG(T5ERROR) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 5cd3ba5c..cb2e6408 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -367,10 +367,10 @@ USHORT EqfMemory::NTMGetIDFromNameEx( } /* endif */ // sort long name array - if (usRc == NO_ERROR) + if (usRc == NO_ERROR && (LongNames.stTableEntry.size() > 1)) { qsort(LongNames.stTableEntry.data(), - LongNames.stTableEntry.size(), + LongNames.stTableEntry.size()-1, sizeof(TMX_LONGNAME_TABLE_ENTRY), NTMLongNameTableComp); } /* endif */ @@ -382,11 +382,13 @@ USHORT EqfMemory::NTMGetIDFromNameEx( LongNamesCaseIgnore.pszBuffer = LongNames.pszBuffer; toUpper(LongNamesCaseIgnore.pszBuffer); LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); - - qsort(LongNamesCaseIgnore.stTableEntry.data(), - LongNamesCaseIgnore.stTableEntry.size(), - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore); + + if( LongNamesCaseIgnore.stTableEntry.size() > 1){ + qsort(LongNamesCaseIgnore.stTableEntry.data(), + LongNamesCaseIgnore.stTableEntry.size()-1, + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); + } } /* endif */ // update TM record for long names @@ -909,7 +911,8 @@ USHORT EqfMemory::NTMReadLongNameTable() USHORT usRC = NO_ERROR; // function return code // call to obtain exact length of record - usRC = TmBtree.EQFNTMGet(LONGNAME_KEY, LongNames.pszBuffer); + LONG lKey = LONGNAME_KEY; + usRC = TmBtree.EQFNTMGet(lKey, LongNames.pszBuffer); if (usRC == NO_ERROR) { @@ -923,50 +926,6 @@ USHORT EqfMemory::NTMReadLongNameTable() if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) { T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be deleted" ; -#ifdef TEMPORARY_COMMENTED - // table is tersed... - pNewArea.resize(pTerseHeader->usDataSize); - // allocate buffer for untersed data - - // unterse data - if (usRC == NO_ERROR) - { - ULONG ulNewLen = 0; - - memcpy(pNewArea.data(), - LongNames.pszBuffer.data() + sizeof(TERSEHEADER), - LongNames.pszBuffer.size() - sizeof(TERSEHEADER)); - T5LOG(T5ERROR) << "::TEMPORARY_COMMENTED in NTMReadLongNameTable, fUtlHuffmanExpand"; -#ifdef TEMPORARY_COMMENTED - if (!fUtlHuffmanExpand((PUCHAR)pNewArea, pTerseHeader->usDataSize, - &ulNewLen)) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); // expand failed most likely to - // memory shortage - } /* endif */ -#endif - } /* endif */ - - // set table data and cleanup - if (usRC == NO_ERROR) - { - // set size of buffer area - //LongNames.data()->ulBufSize = (ULONG)pTerseHeader->usDataSize; - //LongNames.pszBuffer.resize(ulLen); - LongNames.pszBuffer = pNewArea; - //memcpy(LongNames.pszBuffer.data(), pNewArea.data(), pNewArea.size()); - - // free tersed data area - // UtlAlloc( (PVOID *)&LongNames.data()->pszBuffer, 0L, 0L, NOMSG ); - - // anchor untersed data area - //LongNames.data()->pszBuffer = (PSZ)pNewArea; - - // avoid free of untersed data area - // pNewArea = NULL; - } /* endif */ - // if ( pNewArea != NULL ) UtlAlloc( (PVOID *)&pNewArea, 0L, 0L, NOMSG ); -#endif } /* endif */ } /* endif */ @@ -994,10 +953,12 @@ T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be } /* endwhile */ // sort long name array - qsort(LongNames.stTableEntry.data(), - LongNames.stTableEntry.size(), + if( LongNames.stTableEntry.size() > 1){ + qsort(LongNames.stTableEntry.data(), + LongNames.stTableEntry.size()-1,// sizeof(TMX_LONGNAME_TABLE_ENTRY), NTMLongNameTableComp); + } // make copy of long name array for case ignore search LongNamesCaseIgnore.stTableEntry = LongNames.stTableEntry; @@ -1005,10 +966,12 @@ T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); - qsort(LongNamesCaseIgnore.stTableEntry.data(), - LongNamesCaseIgnore.stTableEntry.size(), - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore); + if( LongNamesCaseIgnore.stTableEntry.size() > 1){ + qsort(LongNamesCaseIgnore.stTableEntry.data(), + LongNamesCaseIgnore.stTableEntry.size()-1,// + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); + } } /* endif */ } /* endif */ @@ -1094,6 +1057,16 @@ USHORT EqfMemory::NTMAddNameToTable( usRc = NTMGetPointersToTable(usTableType, &pstTMTable, &pstTMTableEntries); + //bool isEmpty = true; + //for(auto& entry: pstTMTable->stTmTableEntry){ + // if(entry.usId || entry.szName[0] != '\0'){ + // isEmpty = false; + // break; + // } + //} + //if(isEmpty){ + // pstTMTable->stTmTableEntry.resize(1); + //} if (usRc == NO_ERROR) { @@ -1117,10 +1090,12 @@ USHORT EqfMemory::NTMAddNameToTable( /* insert name and id to table and sort table */ /*************************************************************/ - qsort(pstTMTable->stTmTableEntry.data(), - pstTMTable->stTmTableEntry.size(), - sizeof(TMX_TABLE_ENTRY), - NTMCompNames); + if( pstTMTable->stTmTableEntry.size() > 1){ + qsort(pstTMTable->stTmTableEntry.data(), + pstTMTable->stTmTableEntry.size()-1,//keep last entry empty + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); + } // update table record in TM QDAM file (if not in read-only mode) if (!(usAccessMode & ASD_READONLY)) @@ -1263,9 +1238,9 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab if (usRc == NO_ERROR) { - pTMTable->ulMaxEntries1 = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; + //pTMTable->ulMaxEntries1 = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; pTMTable->stTmTableEntry.clear(); - pTMTable->stTmTableEntry.resize((((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries + 1)); + pTMTable->stTmTableEntry.resize((((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries));// + 1)); //if (pTMTable->stTmTableEntry.size()) memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, (pTMTable->stTmTableEntry.size()-1) * sizeof(TMX_TABLE_ENTRY)); @@ -1515,7 +1490,7 @@ USHORT NtmStoreAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszNewData) // add new data to end of data area if (usNewLength != 0) { - PSZ_W pTarget = (PSZ_W)(((PBYTE)pData) + pCLB->usAddDataLen - 1); + PSZ_W pTarget = (PSZ_W)(((PBYTE)pData) + pCLB->usAddDataLen - 4); PUSHORT pTargetPShort = (PUSHORT)pTarget; *pTargetPShort++ = usDataID; *pTargetPShort++ = usNewLength; diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 37ff7f1e..4ce2ed16 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -269,7 +269,7 @@ int EqfMemory::getNextProposal memset( &TmExtIn, 0, sizeof(TMX_EXT_IN_W) ); memset( &TmExtOut, 0, sizeof(TMX_EXT_OUT_W) ); - TmExtIn.ulTmKey = Proposal.nextInternalKey.getRecordKey();//this->ulNextKey; + TmExtIn.lTmKey = Proposal.nextInternalKey.getRecordKey();//this->ulNextKey; TmExtIn.usNextTarget = Proposal.nextInternalKey.getTargetKey();//this->usNextTarget; TmExtIn.usConvert = MEM_OUTPUT_ASIS; Proposal.clear(); @@ -280,7 +280,7 @@ int EqfMemory::getNextProposal { if ( (piProgress != NULL) && (TmExtOut.ulMaxEntries != 0) ) { - *piProgress = (int)(( TmExtIn.ulTmKey - FIRST_KEY) * 100) / (int)TmExtOut.ulMaxEntries; + *piProgress = (int)(( TmExtIn.lTmKey - FIRST_KEY) * 100) / (int)TmExtOut.ulMaxEntries; } /* endif */ } /* endif */ @@ -289,14 +289,14 @@ int EqfMemory::getNextProposal ExtOutToOtmProposal( &TmExtOut, Proposal ); // set current proposal internal key ,which is used in updateProposal - Proposal.currentInternalKey.setInternalKey( TmExtIn.ulTmKey, TmExtIn.usNextTarget ); + Proposal.currentInternalKey.setInternalKey( TmExtIn.lTmKey, TmExtIn.usNextTarget ); }else{ Proposal.currentInternalKey.reset(); } /* endif */ if ( (iRC == 0) || (iRC == BTREE_CORRUPTED) || iRC == BTREE_BUFFER_SMALL || (usAccessMode & ASD_ORGANIZE) ) { - Proposal.nextInternalKey.setInternalKey(TmExtOut.ulTmKey, TmExtOut.usNextTarget); + Proposal.nextInternalKey.setInternalKey(TmExtOut.lTmKey, TmExtOut.usNextTarget); } /* endif */ if ( iRC == 0 ) diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 8fe89400..4ab38f96 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -3982,12 +3982,11 @@ SHORT QDAMUnTerseData // endif // //------------------------------------------------------------------------------ -SHORT QDAMGetszData_V3 +SHORT BTREE::QDAMGetszData_V3 ( - PBTREE pBTIda, RECPARAM recDataParam, - BytesRef pData, - CHAR chType // type of record key/data + BytesRef pData + //CHAR chType=DATA_NODE // type of record key/data ) { SHORT sRc = 0; @@ -4001,7 +4000,6 @@ SHORT QDAMGetszData_V3 LONG lFitLen; // free to be filled length USHORT usNum; // record number LONG lLZSSLen; - PBTREE pBT = pBTIda; BOOL fRecLocked = FALSE; // TRUE if BTREELOCKRECORD has been done USHORT usLenFieldSize; // size of record length field @@ -4009,14 +4007,14 @@ SHORT QDAMGetszData_V3 // get size of record length field usLenFieldSize = sizeof(ULONG); - recDataParam.usNum = recDataParam.usNum + pBT->usFirstDataBuffer; + recDataParam.usNum = recDataParam.usNum + usFirstDataBuffer; // get record and copy data - sRc = pBTIda->QDAMReadRecord_V3( recDataParam.usNum, pRecord, FALSE ); + sRc = QDAMReadRecord_V3( recDataParam.usNum, pRecord, FALSE ); if ( !sRc ) { BTREELOCKRECORD( pRecord ); fRecLocked = TRUE; - if ( TYPE( pRecord ) != chType ) + if ( TYPE( pRecord ) != DATA_NODE ) { {LOG_AND_SET_RC_W_INFO(sRc, T5ERROR, BTREE_CORRUPTED) << " if ( TYPE( pRecord ) != chType ), TYPE( pRecord )=" << toStr( TYPE( pRecord ));} } @@ -4092,7 +4090,7 @@ SHORT QDAMGetszData_V3 fRecLocked = FALSE; if ( usNum ) { - sRc = pBTIda->QDAMReadRecord_V3( usNum, pRecord, FALSE ); + sRc = QDAMReadRecord_V3( usNum, pRecord, FALSE ); if ( !sRc && TYPE( pRecord ) != DATA_NEXTNODE ) { sRc = BTREE_CORRUPTED; @@ -4121,11 +4119,11 @@ SHORT QDAMGetszData_V3 } /* endwhile */ if ( !sRc && fTerse ) { - switch ( pBT->fTerse ) + switch ( fTerse ) { case BTREE_TERSE_HUFFMAN : { - sRc = QDAMUnTerseData( pBTIda, pData, lTerseLen ); + sRc = QDAMUnTerseData( this, pData, lTerseLen ); } break; default : @@ -5936,7 +5934,7 @@ SHORT QDAMDictFirstLocal if ( !sRc ) { - sRc = QDAMGetszData_V3( pBTIda, recData, pUserData, DATA_NODE ); + sRc = pBTIda->QDAMGetszData_V3(recData, pUserData ); } /* endif */ if ( (sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED) ) From 117c65b4fc52ff6bd9589b447dbe00176f2653c3 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 10 Jun 2025 23:00:10 +0300 Subject: [PATCH 14/62] * fixed replacing segments with new segments whern required fields are identical * fixed crash when adding more than 2 subsegments in a row * code cleanup * fixed ASD_ORGANIZE always on --- include/EQFDASD.H | 8 ----- source/RestAPI/OTMMSJSONFactory.cpp | 15 ++++++++ source/RestAPI/ProxygenServer.cpp | 16 +-------- .../opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 35 ++++++++++--------- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 8 ++--- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 19 +++++----- .../core/EqfMemoryPlugin/EqfMemory.cpp | 1 - .../core/EqfMemoryPlugin/EqfMemoryPlugin.cpp | 4 ++- source/opentm2/core/utilities/EQFDICT.cpp | 1 - source/opentm2/core/utilities/UtlAlloc.cpp | 25 ------------- source/otmd.cpp | 2 ++ 14 files changed, 56 insertions(+), 84 deletions(-) diff --git a/include/EQFDASD.H b/include/EQFDASD.H index 23f8e683..cc50b09e 100755 --- a/include/EQFDASD.H +++ b/include/EQFDASD.H @@ -212,15 +212,7 @@ typedef enum _TERMTYPE #define STYLEPREFIX_UNDEFINED 0x2715 // -#define ASD_GUARDED 0x01 // Open dictionary in guarded mode -#define ASD_LOCKED 0x10 // Open dictionary and lock it -#define ASD_FORCE_WRITE 0x40 // force imm.write of any changed record -#define ASD_NOINDEX 0x80 // Do not process index dictionary - // Caution: this define should be used - // only for dictionary organize!!! #define ASD_ORGANIZE 0x100 // work in organize mode -#define ASD_NOOPENCHECK 0x200 // do not check QDAM header open flag -#define ASD_READONLY 0x400 // open in read only mode // AsdTranslate mode: return terms only diff --git a/source/RestAPI/OTMMSJSONFactory.cpp b/source/RestAPI/OTMMSJSONFactory.cpp index 33fa9004..b053e1de 100755 --- a/source/RestAPI/OTMMSJSONFactory.cpp +++ b/source/RestAPI/OTMMSJSONFactory.cpp @@ -13,6 +13,7 @@ #include "opentm2/core/utilities/EncodingHelper.h" #include +#include // Function to extract JSON objects from a JSON array string std::vector JSONFactory::extractJsonObjects(const std::string& jsonArray, const std::string& arrayKey, std::string& errMsg) { @@ -172,6 +173,20 @@ JSONFactory* JSONFactory::getInstance() return( 0 ); } +/* + std::wstring convertToWCharSafe( const std::string& str ) + { + std::wstring_convert, wchar_t> safe_converter; + try { + std::wstring w = safe_converter.from_bytes(str); + } catch (const std::range_error&) { + std::cerr << "Invalid UTF-8 detected. Replacing with placeholder.\n"; + std::wstring w = L"[INVALID_UTF8]"; + } + return w; +}//*/ + + /*! \brief convert a ASCII std::string to a UTF16 std::wstring \param strUTF8String string in UTF8 encoding \returns string converted to UTF16 diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index 4a5bdb6a..75d5e839 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -456,20 +456,6 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { //auto acceptorFactory = std::make_shared; server.bind({{addr, proxygen::HTTPServer::Protocol::HTTP}}); - //auto connectionEventCallback = std::make_shared(); - /* - for (auto &sock : server.getSockets()) { - // Note: setMaxConnections() is provided by folly::AsyncServerSocket. - //sock->onConnectionEnqueuedForAcceptorCallback(connectionEventCallback); - if (auto* serverSocket = dynamic_cast(sock)) { - // basePtr is actually an instance of AsyncServerSocket - serverSocket->setConnectionEventCallback(&connectionEventCallback); - } else { - // basePtr is not an instance of AsyncServerSocket - // Handle accordingly - } - }//*/ - // Start HTTPServer mainloop in a separate thread std::thread t([&]() { server.start(); }); OtmMemoryServiceWorker::getInstance();//to init instance @@ -499,7 +485,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { "\n Log level: " << uiLogLevel << "\n Triples threshold: " << uiThreshold << "\n Localhost only: " << fLocalHostOnly << - "\n Flags: " << listFlags(false) << + "\n Flags(*=non-default, !=custom flag from t5memory(and not from the libraries)): " << listFlags(false) << "\n\n\n\ |==================================================================|\n\ |-------------Setup is done -> waiting for requests...-------------|\n\ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index 6df0a0a1..025d4b70 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -110,8 +110,8 @@ USHORT EQFMemOrganizeStart // ================ Handle the message WM_EQF_MEMORGANIZE_PROCESS ===================== bool IsValidXml(std::wstring&& sentence); +DECLARE_int32(maxBadSegmentsIdsSaved); -DEFINE_int32(maxBadSegmentsIdsSaved, 20, "Defines how much segments(it's id's) that failed during import or reorganize would be saved to the status response"); VOID EQFMemOrganizeProcess ( diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 6ef38714..88c51370 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -153,7 +153,7 @@ USHORT EqfMemory::TmtXDelSegmByKey CHAR szString[MAX_EQF_PATH]; // character strin ULONG ulRecBufSize = TMX_REC_SIZE; // current size of record buffer std::vector pTmRecord; // space for user data - pTmRecord.resize(TMX_REC_SIZE); + //pTmRecord.resize(TMX_REC_SIZE); // update TM databse if ( !usRc ) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 198515fb..be7068da 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -70,7 +70,6 @@ USHORT EqfMemory::TmtXExtract PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record LONG lStartKey; //start tm key LONG lNextKey; //last tm key - BOOL fSpecialMode = FALSE; //special mode flag ULONG ulOemCP = 1L; std::vector pTmRecord; // space for user data @@ -150,7 +149,7 @@ USHORT EqfMemory::ExtractRecordV6 if ( !usRc ) { //check if record is empty - if ( ( RECLEN(pTmRecord) != 0) && + if ( (!pTmRecordData.empty()) && ( RECLEN(pTmRecord) != 0) && (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) { LONG lLeftTgtLen; // remaining target length @@ -254,22 +253,26 @@ USHORT EqfMemory::ExtractRecordV6 { lLeftTgtLen = 0; } /* endif */ - pByte += RECLEN(pTMXTargetRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - lLeftClbLen = RECLEN(pTMXTargetRecord) - + + if(lLeftClbLen > 0){//need to add this check in 0.7 + pByte += RECLEN(pTMXTargetRecord); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); + lLeftClbLen = RECLEN(pTMXTargetRecord) - pTMXTargetRecord->usClb; - // subtract size of current CLB - if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) - { - lLeftClbLen -= TARGETCLBLEN(pTargetClb); + // subtract size of current CLB + if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) + { + lLeftClbLen -= TARGETCLBLEN(pTargetClb); + } + else + { + // database is corrupted + lLeftClbLen = 0; + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ } - else - { - // database is corrupted - lLeftClbLen = 0; - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ + usTarget++; } /* endif */ } /* endwhile */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index fe8c3963..07576871 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -207,7 +207,7 @@ USHORT EqfMemory::LoadMem() { - usRc1 = TmBtree.EQFNTMOpen((USHORT)(usAccessMode | ASD_FORCE_WRITE) ); + usRc1 = TmBtree.EQFNTMOpen((usAccessMode) ); if ( (usRc1 == NO_ERROR) || (usRc1 == BTREE_CORRUPTED) ) { //get signature record and add to control block diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 4c18bbd8..d57950b0 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -1043,7 +1043,7 @@ USHORT EqfMemory::UpdateTmIndex //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file - if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) + if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL))// && (usAccessMode & ASD_LOCKED) ) { usRc = EQFNTMOrganizeIndex( &(InBtree), usAccessMode, START_KEY ); @@ -1126,7 +1126,7 @@ USHORT EqfMemory::UpdateTmIndex pIndexRecord);//.data(), //pointer to index //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file - if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL) && (usAccessMode & ASD_LOCKED) ) + if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL))// && (usAccessMode & ASD_LOCKED) ) { usRc = EQFNTMOrganizeIndex( &(InBtree), usAccessMode, START_KEY ); @@ -1733,9 +1733,9 @@ USHORT EqfMemory::ComparePutData fStop = true; fOK = false; }else{ - if(true || FLAGS_log_memmove_in_compareputdata) + if(FLAGS_log_memmove_in_compareputdata) { - T5LOG(T5ERROR) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " + T5LOG(T5TRANSACTION) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize << "; ulKey = " << lKey ; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index cb2e6408..24246b3d 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -605,7 +605,9 @@ USHORT EqfMemory::NTMGetNameFromID(PUSHORT pusID, // intput if (*pusID == pstTMTableEntries[ulI].usId) { fFound = TRUE; - strcpy(pszName, pstTMTableEntries[ulI].szName); + constexpr int sizeOfField = MAX_LANG_LENGTH ; + strncpy(pszName, pstTMTableEntries[ulI].szName, sizeOfField-1); + pszName[sizeOfField-1] = '\0'; } /* endif */ } /* endfor */ /**************************************************************/ @@ -1097,16 +1099,13 @@ USHORT EqfMemory::NTMAddNameToTable( NTMCompNames); } - // update table record in TM QDAM file (if not in read-only mode) - if (!(usAccessMode & ASD_READONLY)) + // update table record in TM QDAM file + if (usTableType != LANGGROUP_KEY) { - if (usTableType != LANGGROUP_KEY) - { - std::vector pOldTable; - tmxTableToBuffer(pstTMTable, pOldTable); - usRc = TmBtree.EQFNTMUpdate((LONG)usTableType,pOldTable); - //(PBYTE)table, occupSize); - } /* endif */ + std::vector pOldTable; + tmxTableToBuffer(pstTMTable, pOldTable); + usRc = TmBtree.EQFNTMUpdate((LONG)usTableType,pOldTable); + //(PBYTE)table, occupSize); } /* endif */ } /* endif */ } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 4ce2ed16..9aaf4875 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -271,7 +271,6 @@ int EqfMemory::getNextProposal TmExtIn.lTmKey = Proposal.nextInternalKey.getRecordKey();//this->ulNextKey; TmExtIn.usNextTarget = Proposal.nextInternalKey.getTargetKey();//this->usNextTarget; - TmExtIn.usConvert = MEM_OUTPUT_ASIS; Proposal.clear(); iRC = (int)TmtXExtract( &TmExtIn, &TmExtOut); diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp index 7c5d3e9c..69735dfa 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp @@ -99,7 +99,9 @@ EqfMemory* EqfMemoryPlugin::initTM( pMemory->setExpecedSizeInRAM(requiredMemory); pMemory->updateLastUseTime(); pMemory->eStatus = WAITING_FOR_LOADING_STATUS; - pMemory->usAccessMode |= ASD_ORGANIZE; + if(fReorganizeOnly){ + pMemory->usAccessMode |= ASD_ORGANIZE; + } //pMemory->fReorganizeOnly = fReorganizeOnly; }else{ LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 481126eb..a73fe2bf 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -4789,7 +4789,6 @@ SHORT BTREE::QDAMDictOpenLocal /****************************************************************/ /* check if immeadiate write of any changed necessary */ /****************************************************************/ - fGuard = ( usOpenFlags & ASD_FORCE_WRITE ); usOpenFlags = usOpenFlags; // open the file diff --git a/source/opentm2/core/utilities/UtlAlloc.cpp b/source/opentm2/core/utilities/UtlAlloc.cpp index 4bc22c4b..b5ade408 100755 --- a/source/opentm2/core/utilities/UtlAlloc.cpp +++ b/source/opentm2/core/utilities/UtlAlloc.cpp @@ -74,31 +74,6 @@ BOOL UtlAlloc return fSuccess; } /* end of function UtlAlloc */ -//+----------------------------------------------------------------------------+ -//|External Function | -//+----------------------------------------------------------------------------+ -//|Function Name: UtlAllocHwnd Interface for UtlAlloc | -//+----------------------------------------------------------------------------+ -//|Description: Call UtlAlloc and use given window handle for error | -//| messages. | -//+----------------------------------------------------------------------------+ -//|Function Call: BOOL UtlAllocHwnd( PVOID *ppMemory | -//| LONG lOldLength, | -//| LONG lNewLength, | -//| USHORT usMsgNo, | -//| HWND hwndOwner ); | -//+----------------------------------------------------------------------------+ -BOOL UtlAllocHwnd -( - PVOID *ppStorage, // pointer to allocated memory area - LONG lOldLength, // old length of storage area - LONG lNewLength, // length of area to be allocated - USHORT usMessageNo, // message to be displayed when error - HWND hwnd ) // handle for UtlError Hwnd call -{ - return UtlAlloc( ppStorage, lOldLength, lNewLength, NOMSG ); -} /* end of UtlAllocHwnd */ - /**********************************************************************/ /* get the correct task id ... */ diff --git a/source/otmd.cpp b/source/otmd.cpp index b7385efb..04bd15c0 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -305,6 +305,8 @@ std::string to_hex(unsigned long src){ return hex_value; } +DEFINE_int32(maxBadSegmentsIdsSaved, 20, "Defines how many segments(it's id's) that failed during import or reorganize would be saved to the status response"); + int proxygen_server_init(); int main(int argc, char* argv[]) { //FLAGS_disable_aslr = FLAGS_add_premade_socket = FLAGS_log_tcp_backog_events = true; From 44ec3a43cfa6da203c0c55a04d052ecefa6a24bd Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 11 Jun 2025 21:34:03 +0300 Subject: [PATCH 15/62] * fixed fuzzy search chashes in 0.7 --- include/tm.h | 9 -------- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 16 +------------- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 4 ++-- .../EqfMemoryPlugin/GenericTagReplace.cpp | 16 +------------- .../opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp | 21 +++++++++---------- .../opentm2/core/utilities/EncodingHelper.cpp | 4 ++-- 7 files changed, 17 insertions(+), 55 deletions(-) diff --git a/include/tm.h b/include/tm.h index 9cb56a7a..94b0534b 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1618,16 +1618,11 @@ typedef struct _TMX_SUBSTPROP char szSourceTagTable[MAX_FNAME]; // source tagtable char szTargetLanguage[MAX_LANG_LENGTH]; // target language char szPropTagTable[MAX_FNAME]; // tagtable for proposal - PBYTE pTokSource; // token buffer for source tokens - PBYTE pTokPropSource; // token buffer for proposal tokens - PBYTE pTokPropTarget; // token buffer for proposal target .. CHAR_W chBuffer[MAX_SEGMENT_SIZE * 2]; // generic token buffer USHORT usTokenSource; // number of source tokens USHORT usTokenPropSource; // number of prop source tokens USHORT usTokenPropTarget; // number of prop target tokens ULONG ulRandom[MAX_RANDOM]; // random sequence number - PTMX_REPLTAGPAIR pTagPairs; // tags to be replaced by each other - PTMX_REPLTAGPAIR pDelTagPairs; // tags to be deleted in prop } TMX_SUBSTPROP, *PTMX_SUBSTPROP; //======================================================================= @@ -2587,12 +2582,8 @@ USHORT MorphAddTermToList2W( ULONG ulFlags // flags for the ter ); - - BOOL NTMDocMatch(PSZ pszShort1, PSZ pszLong1, PSZ pszShort2, PSZ pszLong2); -void NTMFreeSubstProp(TMX_SUBSTPROP& SubstProp); - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /*! tm.h Internal header file for Translation Memory functions Copyright (c) 1990-2016, International Business Machines Corporation and others. All rights reserved. diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index dd354cb5..da27d99b 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -909,7 +909,7 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT pLIDA->pProposal->setAddInfo( pSegment->szAddInfo ); pLIDA->pProposal->setContext( pSegment->szContext ); pLIDA->pProposal->setSegmentId( pSegment->lSegNum ); - + pLIDA->pProposal->pInputSentence.reset(); pLIDA->pProposal->pInputSentence = std::make_unique(pSegment->szSource, pSegment->szTarget); if(pLIDA->pProposal->pInputSentence->wasParsedSuccessfully()){ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 207fb385..a4b5759f 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -155,9 +155,6 @@ static OtmProposal::eProposalType GetProposalTypeFromFlag( USHORT usTranslationF return ( OtmProposal::eptUndefined ); } - -void NTMFreeSubstProp( PTMX_SUBSTPROP pSubstProp ); - //+----------------------------------------------------------------------------+ //|External function | //+----------------------------------------------------------------------------+ @@ -1567,7 +1564,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct PSZ pSrcFileName = pGetIn->szFileName; // source file name PSZ pSrcLongName = pGetIn->szLongName; // long source file name PBYTE pByte; //position pointer - BOOL fOK; //success indicator + BOOL fOK = true; //success indicator USHORT usRc = NO_ERROR; //return code LONG lTargetLen; //length indicator LONG ulLen; //length indicator @@ -1899,9 +1896,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct pstMatchTable->usOverlaps = *pusOverlaps; pstMatchTable->usContextRanking = usContextRanking; - - pstMatchTable->szContext[0] = 0; - pstMatchTable->szAddInfo[0] = 0; if ( pTMXTargetClb->usAddDataLen ) { NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstMatchTable->szContext ); @@ -1956,9 +1950,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct } /* endif */ } /* endif */ } /* endif */ - - //release memory - UtlAlloc( (PVOID *) &pString, 0L, 0L, NOMSG ); } /* endif */ } /* endif */ @@ -1967,11 +1958,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct ERREVENT2( FILLMATCHTABLE_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); } /* endif */ - /********************************************************************/ - /* free any allocated resource */ - /********************************************************************/ - NTMFreeSubstProp( SubstProp ); - return( usRc ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index d57950b0..4821a07b 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -367,7 +367,7 @@ VOID HashSentence //pSentence->pulVotes[pSentence->usActVote] = pTermTokens[i].usHash; pSentence->pulVotes.push_back(pTermTokens[i].usHash); pSentence->usActVote++; - T5LOG(T5TRANSACTION) << "pSentence->pulVotes.size = " << pSentence->pulVotes.size() <<"; actVote = " << pSentence->usActVote; + //T5LOG(T5TRANSACTION) << "pSentence->pulVotes.size = " << pSentence->pulVotes.size() <<"; actVote = " << pSentence->usActVote; } /* endfor */ } /* endif */ @@ -603,7 +603,7 @@ Vote pSentence->usActVote++; //#ifdef DEBUG - T5LOG(T5TRANSACTION) << "pulVodet.size()=" << pSentence->pulVotes.size() << "; actVote = " << pSentence->usActVote << "; indexInList = " << indexInList; + //T5LOG(T5TRANSACTION) << "pulVotes.size()=" << pSentence->pulVotes.size() << "; actVote = " << pSentence->usActVote << "; indexInList = " << indexInList; //#endif } /* endif */ } /* end of function Vote */ diff --git a/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp b/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp index e14a5d4b..568073c8 100755 --- a/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp @@ -230,18 +230,4 @@ static char szLastTargetLang[MAX_LANG_LENGTH] = ""; static ULONG ulLastSrcOemCP = 0; static ULONG ulLastTgtOemCP = 0; -// free tag substitution data area -void NTMFreeSubstProp( TMX_SUBSTPROP& SubstProp ) -{ - if ( SubstProp.pTokSource ) - UtlAlloc( (PVOID*) &SubstProp.pTokSource, 0L, 0L, NOMSG ); - if ( SubstProp.pTokPropSource ) - UtlAlloc( (PVOID*) &SubstProp.pTokPropSource, 0L, 0L, NOMSG ); - if ( SubstProp.pTokPropTarget ) - UtlAlloc( (PVOID*) &SubstProp.pTokPropTarget, 0L, 0L, NOMSG ); - if ( SubstProp.pTagPairs ) - UtlAlloc( (PVOID*) &SubstProp.pTagPairs, 0L, 0L, NOMSG ); - if ( SubstProp.pDelTagPairs ) - UtlAlloc( (PVOID*) &SubstProp.pDelTagPairs, 0L, 0L, NOMSG ); - return; -} /* end of function NTMFreeSubstProp */ + diff --git a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp index b53a7c26..4a299c46 100755 --- a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp @@ -299,7 +299,7 @@ size_t str16len(const char16_t* source) // helper function for the access to the name mapping tables BOOL FindName( PNAMETABLE pTable, const char *pszName, char *pszValue, int iBufSize ); -void EscapeXMLChars( PSZ_W pszText, PSZ_W pszBuffer ); +void EscapeXMLChars( const wchar_t* pszText, wchar_t* pszBuffer ); BOOL CheckEntityW( PSZ_W pszText, int *piValue, int *piLen ); BOOL CheckEntity( PSZ pszText, int *piValue, int *piLen ); void CopyTextAndProcessMaskedEntitites( PSZ_W pszTarget, PSZ_W pszSource ); @@ -3186,15 +3186,13 @@ void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t lengt char16_t* pzChars = (char16_t*)chars; std::string sChars = EncodingHelper::convertToUTF8(pzChars); std::wstring wChars = EncodingHelper::convertToWChar(sChars.c_str()); - char* c_chars = (char*) sChars.c_str(); - wchar_t* w_chars = (wchar_t*) wChars.c_str(); - int iLength = length; int iwLength = wChars.length(); + int iCurLen = 0; if ( this->fTMXTagStarted ) { // close open TMX inline tag - int iCurLen = wcslen( pBuf->szData ); + iCurLen = wcslen( pBuf->szData ); if ( (iCurLen + 2) < DATABUFFERSIZE) { pBuf->szData[iCurLen++] = '>'; @@ -3213,16 +3211,16 @@ void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t lengt if ( iwLength > iwFree ) iwLength = iwFree; - wcsncpy( pBuf->szPropW + iwCurLength, w_chars, iwLength ); + wcsncpy( pBuf->szPropW + iwCurLength, wChars.c_str(), iwLength ); pBuf->szPropW[iwCurLength+iwLength] = 0; } else if ( fCatchData && (CurElement.ID == INVCHAR_ELEMENT) ) { // add invalid char - int iCurLen = wcslen( pBuf->szData ); + iCurLen = wcslen( pBuf->szData ); if ( (iCurLen + 1 + 1) < DATABUFFERSIZE) { - LONG lChar = _wtol( w_chars ); + LONG lChar = _wtol( wChars.c_str() ); if ( lChar != 0 ) { USHORT usChar = (USHORT)lChar; @@ -3234,11 +3232,11 @@ void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t lengt else if ( fCatchData && (fWithTagging || !CurElement.fInlineTagging) ) { // add data to current data buffer - int iCurLen = wcslen( pBuf->szData ); + iCurLen = wcslen( pBuf->szData ); if ( (iCurLen + length + 1) < DATABUFFERSIZE) { // escape any special characters in inline tags - EscapeXMLChars( w_chars, pBuf->szData + iCurLen ); + EscapeXMLChars( wChars.c_str(), pBuf->szData + iCurLen ); if(fCreateNormalizedStr){ wcscat(pBuf->szReplacedNpData, pBuf->szData + iCurLen); wcscat(pBuf->szNormalizedData, pBuf->szData + iCurLen); @@ -4967,8 +4965,9 @@ PSZ_W MADSkipWhitespace( PSZ_W pszData ) } // escape characters in segment data to form valid XML -void EscapeXMLChars( PSZ_W pszText, PSZ_W pszBuffer) +void EscapeXMLChars(const wchar_t* pszInput, wchar_t* pszBuffer) { + wchar_t* pszText = (wchar_t*)pszInput; while ( *pszText ) { if ( *pszText == L'\n' ) diff --git a/source/opentm2/core/utilities/EncodingHelper.cpp b/source/opentm2/core/utilities/EncodingHelper.cpp index d2ab2219..b5fcb908 100755 --- a/source/opentm2/core/utilities/EncodingHelper.cpp +++ b/source/opentm2/core/utilities/EncodingHelper.cpp @@ -23,11 +23,11 @@ struct deletable_facet : Facet{ ~deletable_facet(){}; }; -void EscapeXMLChars( wchar_t* pszText, wchar_t* pszBuffer ); +void EscapeXMLChars( const wchar_t* pszText, wchar_t* pszBuffer ); std::wstring EncodingHelper::EscapeXML( std::wstring input ){ wchar_t buff[4096]; - EscapeXMLChars((wchar_t*)input.c_str(), buff); + EscapeXMLChars(input.c_str(), buff); return buff; } From 9764f99c5d1263a4b2c4e04d7c2d15a1ebe46301 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Fri, 13 Jun 2025 15:50:17 +0300 Subject: [PATCH 16/62] * updated to v0.7.1 * fixed importtmx btree and xercesc crashes * fixed table loading crash --- source/CMakeLists.txt | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 96 +++++++++---------- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 5 +- .../opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp | 9 +- 4 files changed, 55 insertions(+), 57 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 96bbe5a2..d30e2451 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 0) +SET(APP_VERSION_MINOR 1) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 4821a07b..b32f45da 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -1563,7 +1563,6 @@ USHORT EqfMemory::ComparePutData PTMX_TARGET_CLB pClb = NULL; //ptr to target control block LONG lLen = 0; //length indicator USHORT usNormLen = 0; //length of normalized string - //PSZ_W pString = NULL; //pointer to character string std::wstring pString; USHORT usRc = NO_ERROR; //returned value from function BOOL fUpdate = FALSE; // TRUE = record has been updated @@ -1571,11 +1570,6 @@ USHORT EqfMemory::ComparePutData LONG lLeftClbLen; int delTargetKey = 0, targetKey = 0; - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else { BOOL fStringEqual; // indicate string equal USHORT usPutLang; // language id of target language @@ -1615,9 +1609,7 @@ USHORT EqfMemory::ComparePutData // get length of target block to work with //source strings equal - position at first target record - lLeftTgtLen = RECLENB(pTmRecord) - toTmxRecord(pTmRecord)->usFirstTargetRecord; pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); - pTMXTargetRecord = (PTMX_TARGET_RECORD)pStartTarget; fStop = (usRc != 0); //RJ: 04/01/22: P018830: @@ -1704,53 +1696,59 @@ USHORT EqfMemory::ComparePutData if ( !fStop ) { targetKey++; - // re-alloc record buffer if too small - USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - ULONG ulNewSize = RECLENB((pTmRecord)) + sizeof(TMX_TARGET_CLB) + usAddDataLen; - if ( ulNewSize > pTmRecord.size()) - { - size_t pClbOffs = (PBYTE)pClb - pTmRecord.data(); - size_t pTMXTargetRecordOffs = (PBYTE)pTMXTargetRecord - pTmRecord.data(); - pTmRecord.resize(ulNewSize); - pClb = (PTMX_TARGET_CLB) &pTmRecord[pClbOffs]; - pTMXTargetRecord = (PTMX_TARGET_RECORD) &pTmRecord[pTMXTargetRecordOffs]; - } /* endif */ + // re-alloc record buffer if too small + constexpr int additionalBytes = 13; + USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); // make room at pCLB for a new CLB and adjust TM record // length and target record length if ( fOK ) { - LONG lNewClbLen = sizeof(TMX_TARGET_CLB) + usAddDataLen; - //LONG size = RECLEN(pTmRecord) - ((PBYTE)pClb - (PBYTE)pTmRecord); - LONG size = TARGETCLBLEN(pClb); - - //try{ - if(size>=0) + LONG lNewClbLen = sizeof(TMX_TARGET_CLB) + usAddDataLen; + size_t occupiedSize = RECLENB((pTmRecord)) + sizeof( TMX_SOURCE_RECORD ) + sizeof(TMX_TARGET_RECORD) + additionalBytes; + if(lNewClbLen + occupiedSize >= pTmRecord.size()) { - size_t occupiedSize = RECLEN(toTmxRecord(pTmRecord)) + sizeof( TMX_SOURCE_RECORD ) + sizeof(TMX_TARGET_RECORD); - if(lNewClbLen + occupiedSize >= TMX_REC_SIZE){ - usRc = BTREE_NODE_IS_FULL; - fStop = true; - fOK = false; - }else{ - if(FLAGS_log_memmove_in_compareputdata) - { - T5LOG(T5TRANSACTION) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " - << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize - << "; ulKey = " << lKey ; - } - memmove( (((PBYTE)pClb) + lNewClbLen), pClb, size); - RECLENB(pTmRecord) += lNewClbLen; - RECLEN(pTMXTargetRecord) += lNewClbLen; + //remember offsets + size_t pStartTargetOffset = (PBYTE)pStartTarget - pTmRecord.data(); + size_t pClbOffset = (PBYTE)pClb - pTmRecord.data(); + size_t pTMXTargetRecordOffset = (PBYTE)pTMXTargetRecord - pTmRecord.data(); + size_t oldSize = pTmRecord.size(); + + pTmRecord.resize(lNewClbLen+occupiedSize); + if(FLAGS_log_memmove_in_compareputdata) + { + T5LOG(T5TRANSACTION) << "pStartTargetOffset=" <= TMX_REC_SIZE){ + usRc = BTREE_NODE_IS_FULL; + fStop = true; + fOK = false; + }else{ + if(FLAGS_log_memmove_in_compareputdata) + { + T5LOG(T5TRANSACTION) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " + << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize + << "; ulKey = " << lKey << "; TmRecord.size() = " << pTmRecord.size();// <<"; size before = " << size_before; + } + + memmove( (((PBYTE)pClb) + lNewClbLen), pClb, size); + RECLENB(pTmRecord) += lNewClbLen; + RECLEN(pTMXTargetRecord) += lNewClbLen; + } } /* endif */ // fill-in new target CLB @@ -1773,13 +1771,11 @@ USHORT EqfMemory::ComparePutData pClb->usAddDataLen = 0; NtmStoreAddData( pClb, ADDDATA_CONTEXT_ID, TmProposal.szContext ); NtmStoreAddData( pClb, ADDDATA_ADDINFO_ID, TmProposal.szAddInfo ); - } /* endif */ - if(fOK){ + fStop = TRUE; // avoid add of a new target record at end of outer loop fUpdate = TRUE; - }//else{ - // fOK = true; - //} + } /* endif */ + } /* endif */ } /* endif */ } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 24246b3d..85d123ed 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -1239,9 +1239,10 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab { //pTMTable->ulMaxEntries1 = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; pTMTable->stTmTableEntry.clear(); - pTMTable->stTmTableEntry.resize((((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries));// + 1)); + int numOfEntries = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; + pTMTable->stTmTableEntry.resize(numOfEntries);// + 1)); //if (pTMTable->stTmTableEntry.size()) - memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, (pTMTable->stTmTableEntry.size()-1) * sizeof(TMX_TABLE_ENTRY)); + memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, numOfEntries/*(pTMTable->stTmTableEntry.size()-1)*/ * sizeof(TMX_TABLE_ENTRY)); } return usRc; diff --git a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp index 4a299c46..9b943a56 100755 --- a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp @@ -3183,9 +3183,8 @@ void TMXParseHandler::endElement(const XMLCh* const name ) void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t length) { - char16_t* pzChars = (char16_t*)chars; - std::string sChars = EncodingHelper::convertToUTF8(pzChars); - std::wstring wChars = EncodingHelper::convertToWChar(sChars.c_str()); + char* utf8_chars = xercesc::XMLString::transcode(chars); + std::wstring wChars = EncodingHelper::convertToWChar(utf8_chars); int iwLength = wChars.length(); int iCurLen = 0; @@ -3201,7 +3200,7 @@ void TMXParseHandler::characters(const XMLCh* const chars, const XMLSize_t lengt this->fTMXTagStarted = FALSE; } /* endif */ - T5LOG( T5DEVELOP)<< "TMXParseHandler::characters " <fInlineTags = TRUE; } /* endif */ + + xercesc::XMLString::release(&utf8_chars); } void TMXParseHandler::fatalError(const SAXParseException& exception) From 805683a77a7ba7e8f33fa0cd0310ca0ae301fcbe Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 16 Jun 2025 22:32:43 +0300 Subject: [PATCH 17/62] * updated to 0.7.2 * fixed fuzzy crash connected to addInfo and context overflow and refactored code around * code cleanup --- include/EQF.H | 1 - include/Proposal.h | 4 ++-- source/CMakeLists.txt | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 6 ++--- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 4 ++-- .../core/EqfMemoryPlugin/EqfMemory.cpp | 9 ++++---- source/opentm2/core/utilities/EQFMORPH.cpp | 2 +- source/opentm2/core/utilities/OtmProposal.cpp | 22 +++++++++++++------ source/opentm2/core/utilities/UtlString.cpp | 8 ------- 10 files changed, 28 insertions(+), 32 deletions(-) diff --git a/include/EQF.H b/include/EQF.H index 69935fff..640729e6 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -5414,7 +5414,6 @@ void LogSysInfo( VOID ); PSZ_W UTF16strcpy( PSZ_W pszTarget, PSZ_W pszSource ); PSZ_W UTF16strcat( PSZ_W pszTarget, PSZ_W pszSource ); PSZ_W UTF16strncpy(PSZ_W pusTarget, PSZ_W pusSource, LONG lLen); -int UTF16strcmp( PSZ_W pszString1, PSZ_W pszString2 ); int UTF16strncmpL( PSZ_W pszString1, PSZ_W pszString2, LONG lLen ); int UTF16strnicmpL( PSZ_W pszString1, PSZ_W pszString2, LONG lLen ); int UTF16strncmp( PSZ_W pszString1, PSZ_W pszString2, USHORT usLen ); diff --git a/include/Proposal.h b/include/Proposal.h index f5f1f312..14c81e47 100644 --- a/include/Proposal.h +++ b/include/Proposal.h @@ -281,7 +281,7 @@ class OtmProposal /* \brief set the proposal context \param pszBuffer Pointer to buffer containing the proposal context */ - void setContext( wchar_t *pszBuffer ); + void setContext(const wchar_t *pszBuffer ); /* \brief get proposal context ranking \returns proposal context ranking @@ -308,7 +308,7 @@ class OtmProposal /* \brief set the proposal additional information \param pszBuffer Pointer to buffer containing the additional info */ - void setAddInfo( wchar_t *pszBuffer ); + void setAddInfo(const wchar_t *pszBuffer ); /* \brief set the proposal memory index \param iIndex new value for the memory index diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index d30e2451..3186a18e 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 1) +SET(APP_VERSION_MINOR 2) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp index 88b9ee65..93468d6f 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp @@ -663,7 +663,7 @@ BOOL FuzzyReplace /**************************************************************/ /* source and proposal different, so do a replace .... */ /**************************************************************/ - if ( UTF16strcmp(chSource, chProp) ) + if ( wcscmp(chSource, chProp) ) { TransferSource( chSource, chProp, chTrans ); fReplaced = TRUE; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 88c51370..7b1996d5 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -272,8 +272,7 @@ USHORT EqfMemory::FindTargetAndDelete( lSrcLen = ulLen; //compare source strings - //if ( !UTF16strcmp( pString, pSentence->pNormString ) ) - if ( !UTF16strcmp( pString.data(), TmDel.pInputSentence->pStrings->getGenericTagStrC() ) ) + if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTagStrC() ) ) { ULONG ulLeftTgtLen; // remaining target length /****************************************************************/ @@ -332,8 +331,7 @@ USHORT EqfMemory::FindTargetAndDelete( if ( !usRc ) { //compare target strings - //if ( !UTF16strcmp( pString, pNormString ) ) - if ( !UTF16strcmp( pString.data(), TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) + if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) { //target strings are equal so compare target tag records //position at target tag table record diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index b32f45da..2d7a572e 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -1598,7 +1598,7 @@ USHORT EqfMemory::ComparePutData //copy and compare source string //memset( pString.data(), 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); lLen = EQFCompress2Unicode( pString, pByte, lLen ); - fStringEqual = ! UTF16strcmp( pString.data(), TmProposal.pInputSentence->pStrings->getGenericTagStrC() ); + fStringEqual = ! wcscmp( pString.c_str(), TmProposal.pInputSentence->pStrings->getGenericTagStrC() ); if ( fStringEqual ) { @@ -1653,7 +1653,7 @@ USHORT EqfMemory::ComparePutData pString.data()[lLen] = EOS; //compare target strings and target tag record - if ( !UTF16strcmp( pString.data(), TmProposal.pInputSentence->pStrings->getGenericTargetStrC())) + if ( !wcscmp( pString.data(), TmProposal.pInputSentence->pStrings->getGenericTargetStrC())) { //target strings and target tag record are equal //position at first control block pClb = (PTMX_TARGET_CLB)NTRecPos(pStartTarget, REC_CLB); diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 9aaf4875..19e7269d 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -484,9 +484,8 @@ int EqfMemory::ExtOutToOtmProposal Proposal.setMarkup( pExtOut->szTagTable ); Proposal.setTargetLanguage( pExtOut->szTargetLanguage ); Proposal.setSourceLanguage( stTmSign.szSourceLanguage); - - if(!pExtOut->szAddInfo.empty()) Proposal.setAddInfo( pExtOut->szAddInfo.data() ); - if(!pExtOut->szContext.empty()) Proposal.setContext( pExtOut->szContext.data() ); + Proposal.setAddInfo( pExtOut->szAddInfo.c_str() ); + Proposal.setContext( pExtOut->szContext.c_str() ); Proposal.setDocName( pExtOut->szLongName ); Proposal.setSegmentId( pExtOut->ulSourceSegmentId ); Proposal.setType( FlagToProposalType( pExtOut->usTranslationFlag ) ); @@ -518,8 +517,8 @@ int EqfMemory::MatchToOtmProposal pProposal->setMarkup( pMatch->szTagTable ); pProposal->setTargetLanguage( pMatch->szTargetLanguage ); pProposal->setSourceLanguage( stTmSign.szSourceLanguage); - pProposal->setAddInfo( pMatch->szAddInfo.data() ); - pProposal->setContext( pMatch->szContext.data() ); + pProposal->setAddInfo( pMatch->szAddInfo.c_str() ); + pProposal->setContext( pMatch->szContext.c_str() ); pProposal->setDocName( pMatch->szLongName ); pProposal->setSegmentId( pMatch->ulSegmentId ); pProposal->setType( FlagToProposalType( pMatch->usTranslationFlag ) ); diff --git a/source/opentm2/core/utilities/EQFMORPH.cpp b/source/opentm2/core/utilities/EQFMORPH.cpp index a43a0e76..697677b3 100755 --- a/source/opentm2/core/utilities/EQFMORPH.cpp +++ b/source/opentm2/core/utilities/EQFMORPH.cpp @@ -1463,7 +1463,7 @@ SHORT MorphHashCompare { pUserPtr; // avoid compiler warning - return (SHORT)(UTF16strcmp( pEntry1->pszTerm, pEntry2->pszTerm )); + return (SHORT)(wcscmp( pEntry1->pszTerm, pEntry2->pszTerm )); } /* end of function MorphHashCompare */ //------------------------------------------------------------------------------ diff --git a/source/opentm2/core/utilities/OtmProposal.cpp b/source/opentm2/core/utilities/OtmProposal.cpp index 00000455..6ea75df7 100755 --- a/source/opentm2/core/utilities/OtmProposal.cpp +++ b/source/opentm2/core/utilities/OtmProposal.cpp @@ -527,10 +527,14 @@ int OtmProposal::getContext( wchar_t *pszBuffer, int iBufSize ) /* \brief set the proposal context \param pszBuffer Pointer to buffer containing the proposal context */ -void OtmProposal::setContext( wchar_t *pszBuffer ) +void OtmProposal::setContext(const wchar_t *pszBuffer ) { - wcsncpy( szContext, pszBuffer, OTMPROPOSAL_MAXSEGLEN ); - fFilled = 1; + szContext[0] = '\0'; + if(pszBuffer){ + wcsncpy( szContext, pszBuffer, OTMPROPOSAL_MAXSEGLEN ); + fFilled = 1; + } + szContext[OTMPROPOSAL_MAXSEGLEN] = '\0'; } /* \brief get proposal context ranking @@ -572,10 +576,14 @@ int OtmProposal::getAddInfo( wchar_t *pszBuffer, int iBufSize ) /* \brief set the proposal additional information \param pszBuffer Pointer to buffer containing the additional info */ -void OtmProposal::setAddInfo( wchar_t *pszBuffer ) -{ - wcsncpy( szAddInfo, pszBuffer, OTMPROPOSAL_MAXSEGLEN ); - fFilled = 1; +void OtmProposal::setAddInfo(const wchar_t *pszBuffer ) +{ + szAddInfo[0] = '\0'; + if(pszBuffer){ + wcsncpy( szAddInfo, pszBuffer, OTMPROPOSAL_MAXSEGLEN ); + fFilled = 1; + } + szAddInfo[OTMPROPOSAL_MAXSEGLEN] = '\0'; } diff --git a/source/opentm2/core/utilities/UtlString.cpp b/source/opentm2/core/utilities/UtlString.cpp index 3bb1231a..a13515d6 100755 --- a/source/opentm2/core/utilities/UtlString.cpp +++ b/source/opentm2/core/utilities/UtlString.cpp @@ -230,14 +230,6 @@ PSZ_W UTF16strcat( PSZ_W pszTarget, PSZ_W pszSource ) return( pszTarget ); } -/////////////////////////////////////////////////////////////////////////////// -/// UTF16strcmp compare two Unicode UTF16 strings /// -/////////////////////////////////////////////////////////////////////////////// - -int UTF16strcmp( PSZ_W pszString1, PSZ_W pszString2 ) -{ - return wcscmp(pszString1, pszString2); -} /////////////////////////////////////////////////////////////////////////////// /// UTF16strncmp compare two Unicode UTF16 strings up to usLen chars /// From 2e9a0108d3fc279c4d8387dcf0a661f6680a178e Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 17 Jun 2025 17:38:08 +0300 Subject: [PATCH 18/62] * updated to 0.7.3 - NOT STABLE * refactored TMX_MATCHENTRY code * code cleanup --- include/OTMAPI.H | 19 -- include/tm.h | 24 +- source/CMakeLists.txt | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 315 ++++++------------ 4 files changed, 116 insertions(+), 244 deletions(-) diff --git a/include/OTMAPI.H b/include/OTMAPI.H index d39c5f65..b60447fb 100755 --- a/include/OTMAPI.H +++ b/include/OTMAPI.H @@ -308,24 +308,6 @@ * EQF Typedefs *-----------------------------------------------------------------------------*/ -typedef struct _STEQFPCMD // Pipe Command/Response -{ - USHORT usCmd; // command code - ULONG ulParm1; // parameter 1 - USHORT usParm2; // parameter 2 - USHORT usParm3; // parameter 3 - USHORT usLen1; // length of buffer 1 - USHORT usLen2; // length of buffer 2 - USHORT usLen3; // length of buffer 3 - USHORT usLen4; // length of buffer 4 - UCHAR ucbBuffer [EQF_SEGLEN*8]; // buffer1/2/3/4 - PVOID pvMetaData; // ptr to segment meta data -} STEQFPCMD; - -typedef STEQFPCMD *PSTEQFPCMD; - - - typedef struct _STEQFGEN // generic EQF structure { HWND hwndTWBS; // handle of parent window @@ -361,7 +343,6 @@ typedef struct _STEQFGEN // generic EQF structure USHORT usRC; // return code CHAR_W szMsgBuffer[EQF_NAMELEN]; // message buffer PVOID pDoc; // reserved - PSTEQFPCMD pstEQFPCmd; // reserved pipe command struct ULONG flPropStyle; // window style of prop Window ULONG flDictStyle; // window style of dict.window ULONG flSrcStyle; // window style of prop.src.wnd. diff --git a/include/tm.h b/include/tm.h index 94b0534b..c11c0db6 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1103,12 +1103,21 @@ typedef struct _TMX_TERM_TOKEN typedef struct _TMX_MATCHENTRY { - LONG lKey; - USHORT usMaxVotes; - USHORT usMatchVotes; - BYTE cCount; + LONG lKey = 0; + USHORT usMaxVotes = 0; + USHORT usMatchVotes = 0; + BYTE cCount = 0; + } TMX_MATCHENTRY, *PTMX_MATCHENTRY; + +inline bool operator==(const TMX_MATCHENTRY& lhs, const TMX_MATCHENTRY& rhs) { + return lhs.lKey == rhs.lKey + && lhs.usMaxVotes == rhs.usMaxVotes + && lhs.usMatchVotes == rhs.usMatchVotes + && lhs.cCount == rhs.cCount; +} + // #pragma pack(show) // #pragma pack(push, 8) // #pragma pack(8) @@ -1672,6 +1681,7 @@ typedef struct _TMX_GET_W PVOID pvGMOptList; // ptr to a global memory option list or NULL bool fSourceLangIsPrefered; bool fTargetLangIsPrefered; + bool fForceFuzzySearch = false; } TMX_GET_W, *PTMX_GET_W; //======================================================================= @@ -1800,7 +1810,7 @@ class EqfMemory //: public TMX_CLB // fields for work area pointers of various subfunctions which are allocated // only once for performance reasons - PVOID pvTempMatchList = nullptr; // matchlist of FillMatchEntry function + std::vector pvTempMatchList; // matchlist of FillMatchEntry function std::vector pvIndexRecord; // index record area of FillMatchEntry function std::vector pvTmRecord; // buffer for memory record used by GetFuzzy and GetExact ULONG ulRecBufSize = 0; // current size of pvTMRecord; @@ -3532,8 +3542,8 @@ USHORT ExactTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_SENTENCE, INT CompCount(const void *, const void *); INT CompCountVotes(const void *, const void *); -VOID CleanupTempMatch(PTMX_MATCHENTRY, PTMX_MATCHENTRY *, PUSHORT, PUSHORT); -USHORT FillMatchEntry(EqfMemory *, PTMX_SENTENCE, PTMX_MATCHENTRY, PUSHORT); +VOID CleanupTempMatch(PTMX_MATCHENTRY, std::vector&, PUSHORT, PUSHORT); +USHORT FillMatchEntry(EqfMemory *, PTMX_SENTENCE, std::vector&, PUSHORT); USHORT FuzzyTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, LONG); USHORT GetFuzzyMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT); diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 3186a18e..5b54a786 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 2) +SET(APP_VERSION_MINOR 3) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index a4b5759f..b04dfebe 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -208,16 +208,6 @@ USHORT EqfMemory::TmtXGet CHAR szString[MAX_EQF_PATH]; // character string szString[0] = '\0'; - ULONG ulStrippedParm = pTmGetIn->ulParm & - ~(GET_RESPECTCRLF | GET_IGNORE_PATH | GET_NO_GENERICREPLACE | GET_ALWAYS_WITH_TAGS | GET_IGNORE_COMMENT); - -#ifdef MEASURETIME - LONG64 lDummy = 0; - fTimeLogging = TRUE; - GetElapsedTime( &lDummy ); -#endif - - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ auto str_source = EncodingHelper::convertToUTF8(pTmGetIn->szSource ); T5LOG(T5DEBUG) << "== Lookup in memory ==" << stTmSign.szName <<" == ;\tLookupSource = >>>" <szTagTable, NULL, (USHORT)TAGTABLE_KEY, &Sentence.pTagRecord->usTagTableId ); toTmxTagtableRecord(Sentence.pTagRecord)->usTagTableId = 1; } } /* endif */ @@ -1995,115 +1975,68 @@ USHORT GetFuzzyMatch PUSHORT pusMatchesFound //number of matches found ) { - BOOL fOK = TRUE; //success indicator USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator + std::vector pMatchEntries; //pointer to match entry structure PTMX_MATCHENTRY pMatchEntry = NULL; //pointer to match entry structure - PTMX_MATCHENTRY pMatchStart = NULL; //pointer to match entry structure USHORT usMatchEntries = 0; //nr of matches found USHORT usOverlaps = 0; //nr of overlapping triples - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lOtherTime) ); -#endif BytesRef pTmRecord = pTmClb->pvTmRecord; + + //fill pMatchEntry with the tm keys with the highest frequency + usRc = FillMatchEntry( pTmClb, pSentence, pMatchEntries, &pGetIn->usMatchThreshold ); + pMatchEntry = pMatchEntries.data(); - //allocate for match entry - if ( fOK ) fOK = UtlAlloc( (PVOID *) &(pMatchEntry), 0L, (LONG)(ABS_VOTES * sizeof(TMX_MATCHENTRY)), NOMSG ); - - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else + if ( usRc == NO_ERROR ) { - pMatchStart = pMatchEntry; - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyOtherTime) ); -#endif - //fill pMatchEntry with the tm keys with the highest frequency - usRc = FillMatchEntry( pTmClb, pSentence, pMatchEntry, &pGetIn->usMatchThreshold ); -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyFillMatchEntry) ); -#endif - if ( usRc == NO_ERROR ) + //get tm record(s) + while ( (pMatchEntry->lKey) && (usRc == NO_ERROR)) // && + //(*pusMatchesFound <= pGetIn->usRequestedMatches) ) { - //get tm record(s) - while ( (pMatchEntry->lKey) && (usRc == NO_ERROR)) // && - //(*pusMatchesFound <= pGetIn->usRequestedMatches) ) - { -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyOtherTime) ); -#endif - - T5LOG( T5INFO) << "GetFuzzyMatch: EQFNTMGET of record " << pMatchEntry->lKey; + T5LOG( T5INFO) << "GetFuzzyMatch: EQFNTMGET of record " << pMatchEntry->lKey; + usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->lKey, pTmRecord ); - usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->lKey, pTmRecord ); + if ( usRc == NO_ERROR ) + { + T5LOG( T5INFO) << "GetFuzzyMatch: MaxVotes=" << pMatchEntry->usMaxVotes << ", ActVote=" << pSentence->usActVote << " EQFNTMGET of record "; - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyGetTime) ); -#endif - if ( usRc == NO_ERROR ) + //compare tm record data with data passed in the get in structure + usOverlaps = std::min( pMatchEntry->usMaxVotes, pSentence->usActVote ); + usRc = FuzzyTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pstMatchTable, + &usMatchEntries, &usOverlaps, + &pMatchEntry->usMaxVotes, &pSentence->usActVote, + pSentence, pMatchEntry->lKey ); + if(usRc == SOURCE_LANG_DIFFERENT) { - T5LOG( T5INFO) << "GetFuzzyMatch: MaxVotes=" << pMatchEntry->usMaxVotes << ", ActVote=" << pSentence->usActVote << " EQFNTMGET of record "; - - //compare tm record data with data passed in the get in structure - usOverlaps = std::min( pMatchEntry->usMaxVotes, pSentence->usActVote ); - usRc = FuzzyTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pstMatchTable, - &usMatchEntries, &usOverlaps, - &pMatchEntry->usMaxVotes, &pSentence->usActVote, - pSentence, pMatchEntry->lKey ); - if(usRc == SOURCE_LANG_DIFFERENT) - { - usRc = NO_ERROR; - } - if ( !usRc ) - { - //nr of matches found - (*pusMatchesFound) = usMatchEntries; - - //get next tm record - pMatchEntry++; - }/* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyTestTime) ); -#endif - - } /* endif */ - } /* endwhile */ - - // limit matches to number of matches requested - if (*pusMatchesFound > pGetIn->usRequestedMatches) - { - // special handling for machine matches: ensure that at least one machine match is left in the truncated list - if ( (pstMatchTable[(*pusMatchesFound)-1].usTranslationFlag == TRANSLFLAG_MACHINE) && - (pstMatchTable[pGetIn->usRequestedMatches-1].usMatchLevel < 100)) + usRc = NO_ERROR; + } + if ( !usRc ) { - memcpy( pstMatchTable + (pGetIn->usRequestedMatches-1), pstMatchTable + (*pusMatchesFound - 1), sizeof(TMX_MATCH_TABLE_W) ); - } /* endif */ + //nr of matches found + (*pusMatchesFound) = usMatchEntries; - memset( &pstMatchTable[pGetIn->usRequestedMatches], 0, sizeof( TMX_MATCH_TABLE_W )); - *pusMatchesFound = pGetIn->usRequestedMatches; - } - - } /* endif */ - } /* endif */ + //get next tm record + pMatchEntry++; + }/* endif */ + } /* endif */ + } /* endwhile */ - //release memory - UtlAlloc( (PVOID *) &pMatchStart, 0L, 0L, NOMSG ); + // limit matches to number of matches requested + if (*pusMatchesFound > pGetIn->usRequestedMatches) + { + // special handling for machine matches: ensure that at least one machine match is left in the truncated list + if ( (pstMatchTable[(*pusMatchesFound)-1].usTranslationFlag == TRANSLFLAG_MACHINE) && + (pstMatchTable[pGetIn->usRequestedMatches-1].usMatchLevel < 100)) + { + memcpy( pstMatchTable + (pGetIn->usRequestedMatches-1), pstMatchTable + (*pusMatchesFound - 1), sizeof(TMX_MATCH_TABLE_W) ); + } /* endif */ - if ( usRc ) - { - ERREVENT2( GETFUZZYMATCH_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); + memset( &pstMatchTable[pGetIn->usRequestedMatches], 0, sizeof( TMX_MATCH_TABLE_W )); + *pusMatchesFound = pGetIn->usRequestedMatches; + } } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFuzzyOtherTime) ); -#endif - return( usRc ); } @@ -2583,7 +2516,7 @@ USHORT FillMatchEntry ( EqfMemory* pTmClb, PTMX_SENTENCE pSentence, - PTMX_MATCHENTRY pMatchEntry, + std::vector& pMatchEntry, PUSHORT pusMatchThreshold ) { @@ -2602,39 +2535,17 @@ USHORT FillMatchEntry USHORT usMaxSentences = MAX_INDEX_LEN * 4; LONG lMatchListSize = (LONG)((usMaxSentences+1) * sizeof(TMX_MATCHENTRY)); - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lOtherTime) ); -#endif - pTmClb->pvIndexRecord.reserve(TMX_REC_SIZE); - // allocate match list array if not done yet - if ( pTmClb->pvTempMatchList ) + if ( pTmClb->pvTempMatchList.empty() ) { - pTempMatch = (PTMX_MATCHENTRY)pTmClb->pvTempMatchList; - memset( pTempMatch , 0, lMatchListSize ); + pTmClb->pvTempMatchList.resize(lMatchListSize/sizeof(pTmClb->pvTempMatchList[0])); } - else - { - fOK = UtlAlloc( (PVOID *) &(pTempMatch), 0L, lMatchListSize, NOMSG ); - pTmClb->pvTempMatchList = pTempMatch; - } /* endif */ - if ( fOK ) - { - pTempStart = pTempMatch; - } - else - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ + pTempStart = pTempMatch = pTmClb->pvTempMatchList.data(); + memset( pTempMatch , 0, lMatchListSize ); -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchAllocTime) ); -#endif - if ( fOK ) { pulVotes = pSentence->pulVotes.data(); lMatchEntries = 0; @@ -2642,14 +2553,9 @@ USHORT FillMatchEntry for ( i = 0; i < pSentence->usActVote; i++, pulVotes++ ) { lKey = (*pulVotes) & START_KEY; -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchOtherTime) ); -#endif + usRc = pTmClb->InBtree.EQFNTMGet( lKey, pIndexRecord ); -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchReadTime) ); -#endif if ( usRc == NO_ERROR ) { //calculate number of index entries in index record @@ -2671,9 +2577,6 @@ USHORT FillMatchEntry pTempMatch = pTempStart; //before adding sentence id check if already in ulSids as the //respective tm record need only be checked once -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchFill1Time) ); -#endif // GQ: do a binary search for the key ( bsearch() cannot be used as we need the insert position) and // a linear search is far too slow @@ -2702,15 +2605,6 @@ USHORT FillMatchEntry } /* endif */ } - // below is the old search code - //while ( (ulIndexKey > pTempMatch->ulKey) && pTempMatch->ulKey ) - //{ - // pTempMatch++; - //} /* endwhile */ - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchFill2Time) ); -#endif if ( lIndexKey == pTempMatch->lKey ) { pTempMatch->cCount++; @@ -2741,22 +2635,17 @@ USHORT FillMatchEntry /**************************************************************/ /* make clean up if we are too near at the limit .. */ /**************************************************************/ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchFill3Time) ); -#endif if (lMatchEntries >= 2 * MAX_INDEX_LEN ) { - PTMX_MATCHENTRY pTemp1Start; + std::vector pTemp1Start; + pTemp1Start.resize(usMaxSentences+1); T5LOG( T5INFO) <<"Cleaning up..." ; - - fOK = UtlAlloc( (PVOID *) &(pTemp1Start), 0L, - (LONG)((usMaxSentences+1)* sizeof(TMX_MATCHENTRY)), NOMSG ); if ( fOK ) { USHORT k = 0; PTMX_MATCHENTRY pTemp1Match; - pTemp1Match = pTemp1Start; + pTemp1Match = pTemp1Start.data(); pTempMatch = pTempStart; for (k=0; klFillMatchFill4Time) ); -#endif - -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchFillTime) ); -#endif - } /* endif */ } /* endfor */ @@ -2800,13 +2680,9 @@ USHORT FillMatchEntry //sort the sentence keys by frequency discard those below the given //threshold - CleanupTempMatch( pTempMatch, &pMatchEntry, &pSentence->usActVote, pusMatchThreshold ); + CleanupTempMatch( pTempMatch, pMatchEntry, &pSentence->usActVote, pusMatchThreshold ); } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchCleanupTime) ); -#endif - //if the entry does not exist in the index file it is not to be regarded as //an error if ( usRc == BTREE_NOT_FOUND ) @@ -2814,14 +2690,6 @@ USHORT FillMatchEntry LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(pTmClb->lFillMatchAllocTime) ); -#endif - if ( usRc ) - { - ERREVENT2( FILLMATCHENTRY_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ - T5LOG( T5INFO) << "FillMatchEntry: lMatchEntries=" <usNumMatchesFound > 1 ) && (ulStrippedParm & GET_EXACT) ) + if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && (pTmGetIn->ulParm & GET_EXACT) ) { PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); int iNumExactSourceMatches = 0; // number of matches having exact the same source @@ -396,7 +396,7 @@ USHORT EqfMemory::TmtXGet /********************************************************************/ /* check if we have more than one equal tgt match in our proposals */ /********************************************************************/ - if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && (ulStrippedParm & GET_EXACT )) + if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && (pTmGetIn->ulParm & GET_EXACT )) { PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); PTMX_MATCH_TABLE_W pstActMatch; From bf6ac3e0299fa61b04255f2ad83e5138993dd07d Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 17 Jun 2025 18:14:47 +0300 Subject: [PATCH 20/62] * fuzzy refactoring and cleanup --- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 241 ++++-------------- 1 file changed, 48 insertions(+), 193 deletions(-) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 235b67d5..91cdd5ba 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -437,8 +437,6 @@ USHORT EqfMemory::TmtXGet pTmGetOut->stPrefixOut.usTmtXRc = usRc; - if ( usRc ) ERREVENT2( TMTXGET_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - // release access to any tag table loaded for segment markup // (may have been loaded for context handling) if ( pTagTable ) @@ -447,11 +445,6 @@ USHORT EqfMemory::TmtXGet pTagTable = NULL; } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(lOtherTime) ); -#endif - - T5LOG( T5INFO) <<"TmtXGet::Lookup complete, found "<usNumMatchesFound<<" matches" ; if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)) @@ -522,8 +515,6 @@ USHORT GetExactMatch USHORT usMatchEntries = 0; //nr of found matches BytesRef pTmRecord = pTmClb->pvTmRecord; - ULONG ulStrippedParm = pGetIn->ulParm & - ~(GET_RESPECTCRLF | GET_IGNORE_PATH | GET_NO_GENERICREPLACE | GET_ALWAYS_WITH_TAGS | GET_IGNORE_COMMENT); lSids.reserve(MAX_INDEX_LEN+5); usRc = DetermineTmRecord( pTmClb, pSentence, lSids ); @@ -558,7 +549,7 @@ USHORT GetExactMatch /****************************************************************/ SetAvailFlags( pTmGetOut, *pusMatchesFound); - if ( !usRc && (*pusMatchesFound > 1 ) && (ulStrippedParm & GET_EXACT) ) + if ( !usRc && (*pusMatchesFound > 1 ) && (pGetIn->ulParm & GET_EXACT) ) { /*************************************************************/ /* there is one exact exact match available - forget about */ @@ -851,24 +842,6 @@ USHORT ExactTest usGetFile = 1; } /* endif */ - if ( pGetIn->ulParm & GET_IGNORE_PATH ) - { - // get pointer to document name without path - if ( pGetIn->szLongName[0] != EOS ) - { - pszDocName = UtlGetFnameFromPath( pGetIn->szLongName ); - if ( pszDocName == NULL ) - { - pszDocName = pGetIn->szLongName; - } /* endif */ - } - else - { - pszDocName = pGetIn->szFileName; - } /* endif */ - } /* endif */ - - /****************************************************************/ /* get length of target block to work with */ /****************************************************************/ @@ -897,9 +870,6 @@ USHORT ExactTest /* update left target length */ /**************************************************************/ usTgtNum++; // target number - - //assert( ulLeftTgtLen >= RECLEN(pTMXTargetRecord) ); - lLeftTgtLen -= RECLEN(pTMXTargetRecord); //next check the target language @@ -913,16 +883,6 @@ USHORT ExactTest fTestCLB = FALSE; } - // check for comments when requested - if ( pGetIn->ulParm & GET_IGNORE_COMMENT ) - { - if ( NtmFindInAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, L"ulParm & GET_IGNORE_PATH ) - { - // we have to compare the real document names rather than comparing the document name IDs - PSZ pszCLBDocName = pTmClb->NTMFindNameForID( &(pClb->usFileId), (USHORT)FILE_KEY ); - if ( pszCLBDocName != NULL ) - { - PSZ pszName = UtlGetFnameFromPath( pszCLBDocName ); - if ( pszName == NULL ) - { - pszName = pszCLBDocName; - } /* endif */ - fMatchingDocName = stricmp( pszName, pszDocName ) == 0; - } - else - { - // could not access the document name, we have to compare the document name IDs - fMatchingDocName = ((pClb->usFileId == usGetFile) || (pClb->usFileId == usAlternateGetFile)); - } /* endif */ - } - else { // we can compare the document name IDs fMatchingDocName = ((pClb->usFileId == usGetFile) || (pClb->usFileId == usAlternateGetFile)); @@ -1558,69 +1498,52 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct ULONG ulTgtOemCP = 0L; TMManager *pFactory = TMManager::GetInstance(); SHORT sLangID = 0; - - // force fTag flag if proposal with inline tagging is requested - if ( pGetIn->ulParm & GET_ALWAYS_WITH_TAGS ) - { - fTag = TRUE; - } /* endif */ - - /********************************************************************/ - /* if tagtables are different, call substitution function */ - /********************************************************************/ - if ( !(pGetIn->ulParm & GET_NO_GENERICREPLACE) ) + /****************************************************************/ + /* add current tagging to source and translation of proposal */ + /****************************************************************/ + ulSrcOemCP = 1; + strcpy( SubstProp.szSourceTagTable, pGetIn->szTagTable ); + strcpy( SubstProp.szSourceLanguage, pGetIn->szSourceLanguage); + strcpy( SubstProp.szTargetLanguage, pGetIn->szTargetLanguage ); + UTF16strcpy( SubstProp.szSource, pGetIn->szSource ); + pByte = (PBYTE)pTMXTargetRecord; + + MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); + + wcsncpy(SubstProp.szPropSource, pSourceString.c_str(), *plSourceLen); + if ( usRc == NO_ERROR ) { - { - /****************************************************************/ - /* add current tagging to source and translation of proposal */ - /****************************************************************/ - ulSrcOemCP = 1; - strcpy( SubstProp.szSourceTagTable, pGetIn->szTagTable ); - strcpy( SubstProp.szSourceLanguage, pGetIn->szSourceLanguage); - strcpy( SubstProp.szTargetLanguage, pGetIn->szTargetLanguage ); - UTF16strcpy( SubstProp.szSource, pGetIn->szSource ); - pByte = (PBYTE)pTMXTargetRecord; + PSZ_W pTarget; - MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); + //position at target string + pByte = (PBYTE)pTMXTargetRecord; + pByte += pTMXTargetRecord->usTarget; - wcsncpy(SubstProp.szPropSource, pSourceString.c_str(), *plSourceLen); - if ( usRc == NO_ERROR ) - { - PSZ_W pTarget; - - //position at target string - pByte = (PBYTE)pTMXTargetRecord; - pByte += pTMXTargetRecord->usTarget; - - //calculate length of target string - lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; + //calculate length of target string + lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - { - std::wstring output; - lTargetLen = EQFCompress2Unicode(output, pByte, lTargetLen); - wcsncpy(SubstProp.szPropTarget, output.c_str(), sizeof(SubstProp.szPropTarget)/sizeof(SubstProp.szPropTarget[0])); - //position at target tag record - pByte = (PBYTE)pTMXTargetRecord; - } - if ( usRc == NO_ERROR ) - { - //fill in the markup table - strcpy(SubstProp.szPropTagTable, "OTMXUXLF"); - - ulTgtOemCP = 1; + { + std::wstring output; + lTargetLen = EQFCompress2Unicode(output, pByte, lTargetLen); + wcsncpy(SubstProp.szPropTarget, output.c_str(), sizeof(SubstProp.szPropTarget)/sizeof(SubstProp.szPropTarget[0])); + //position at target tag record + pByte = (PBYTE)pTMXTargetRecord; + } + if ( usRc == NO_ERROR ) + { + //fill in the markup table + strcpy(SubstProp.szPropTagTable, "OTMXUXLF"); - //fill in the target language - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, - (USHORT)LANG_KEY, - SubstProp.szTargetLanguage, NULL ); - } /* endif */ + ulTgtOemCP = 1; - } /* endif */ + //fill in the target language + pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, + (USHORT)LANG_KEY, + SubstProp.szTargetLanguage, NULL ); } /* endif */ - } - else - { + } /* endif */ + /********************************************************************/ /* if src of prop is empty ( i.e. only inline tagging), add tagging */ @@ -1796,19 +1719,8 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct SubstProp.szPropSource, SubstProp.szSource, sLangID, &usFuzzy, ulSrcOemCP, &usWords, &usDiffs ); - if ( pGetIn->ulParm & GET_RESPECTCRLF ) - { - fEqual = (UtlCompIgnSpaceW( SubstProp.szPropSource, - SubstProp.szSource, 0 ) == 0); - } - else - { - fEqual = (usFuzzy == 100); -// (UtlCompIgnWhiteSpace( SubstProp.szPropSource, -// SubstProp.szSource, -// ulLen) == 0); + fEqual = (usFuzzy == 100); - } /* endif */ // GQ 2004/08/04: do not set match level to 100 for machine translations!! // RJ 2004/08/16: assure that *pusMatchLevel is filled independently @@ -2143,12 +2055,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block } T5LOG( T5INFO) << "FuzzyTest for record " <" << str << "" ; } - if (pGetIn->ulParm & GET_RESPECTCRLF ) // if-else nec for P018279 - { - fRespectCRLFStringEqual = (UtlCompIgnSpaceW( pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 )== 0L); + //compare source strings + fStringEqual = ( UtlCompIgnWhiteSpaceW(pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L + || UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(),0) == 0L ) ; - fStringEqual = (UtlCompIgnWhiteSpaceW( pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L); - if (fStringEqual && !fRespectCRLFStringEqual) - { // there is a LF difference! - fStringEqual = fRespectCRLFStringEqual; - } - } - else - { //compare source strings - fStringEqual = ( UtlCompIgnWhiteSpaceW(pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L - || UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(),0) == 0L ) ; - } /* endif*/ T5LOG( T5INFO) << "FuzzyTest: After String compare, fStringEqual = " <szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTargetId, NTMGETID_NOUPDATE_OPT, NULL )) if ( pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTargetId)) { usTargetId = 1; // set default.. @@ -2265,7 +2159,6 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block //compare target language group IDs if(pGetIn->fTargetLangIsPrefered){ - //T5LOG(T5TRANSACTION) << "pTMXTargetClb->usLangId=" << pTMXTargetClb->usLangId <<"; usTargetId=" << usTargetId; if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] ) { T5LOG( T5INFO) <<"FuzzyTest: Wrong target language!" ; @@ -2281,17 +2174,6 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block //return SOURCE_LANG_DIFFERENT; fTestCLB = FALSE; } - } - - - // check for comments when requested - if ( pGetIn->ulParm & GET_IGNORE_COMMENT ) - { - if ( NtmFindInAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, L"= 100 ) if ( fNormStringEqual ) { - //PTMX_TAGTABLE_RECORD pTMXSourceTagTable; //ptr to source tag info - BOOL fStringEqual = FALSE; //compare source tag table records //position at source tag table record pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - if ( fOK ) { //calculate length of source string LONG lLenTmp = ulSourceLen; - - if ( !fOK ) - { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "fOK is false";} - } - else { - fStringEqual = FALSE; - fStringEqual = (UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(), 0 ) == 0 ); @@ -2397,7 +2258,6 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block // We can assume all Text tokens to be equal ( usFuzzy = 100), // but we do not know about tag tokens BOOL fTempStringEqual = FALSE; - BOOL fOK = FALSE; fOK = NTMCompareBetweenTokens( pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(), @@ -2433,7 +2293,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block } //fill get output structure - T5LOG( T5INFO) << "FuzzyTest: usFuzzy="< Date: Tue, 17 Jun 2025 18:36:55 +0300 Subject: [PATCH 21/62] * fuzzy refactoring --- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 185 ++++++------------ source/opentm2/core/utilities/EQFMORPH.cpp | 8 - 2 files changed, 60 insertions(+), 133 deletions(-) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 91cdd5ba..9a0d97c6 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -31,7 +31,6 @@ #include "../utilities/EncodingHelper.h" // import logging -//#ifdef MEASURETIME static LARGE_INTEGER liLast = { 0 }; static LARGE_INTEGER liFrequency = { 0 }; @@ -44,33 +43,6 @@ static int QueryPerformanceFrequency(LARGE_INTEGER* li){ #define ACTIVATE_NTMGenericDelete - -#ifdef MEASURETIME - void LogWritePerfTime( FILE *hLog, PLARGE_INTEGER pliStart, PLARGE_INTEGER pliEnd, char *pszText ); - - void LogWritePerfTime( FILE *hLog, PLARGE_INTEGER pliStart, PLARGE_INTEGER pliEnd, char *pszText ) - { - LARGE_INTEGER liFrequency; - - QueryPerformanceFrequency( &liFrequency ); - - if ( liFrequency.QuadPart != 0 ) - { - LONGLONG ldwDiff = pliEnd->QuadPart - pliStart->QuadPart; - DWORD dwTime = (DWORD)((ldwDiff * 1000) / liFrequency.QuadPart); - - if ( *pszText ) - { - fprintf( hLog, "Time for %-20s : %ld ms (%I64d ticks)\n", pszText, dwTime, ldwDiff ); - } - else - { - fprintf( hLog, "Total time : %ld ms (%I64d ticks)\n", dwTime, ldwDiff ); - } /* endif */ - } - } -#endif - void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ); @@ -234,116 +206,79 @@ USHORT EqfMemory::TmtXGet } } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(lTokenizeTime) ); -#endif if ( !usRc ) { - SHORT sRetries = MAX_RETRY_COUNT; - { - usRc = NO_ERROR; // reset return code - - if ( !usRc ) + HashSentence( &Sentence ); + usOverlaps = CheckCompactArea( &Sentence, this ); + T5LOG( T5INFO) << "TmtXGet: Checked compact area, usOverlaps=" << usOverlaps; + if ( usOverlaps == Sentence.usActVote ) //all hash triples found { - HashSentence( &Sentence ); - -#ifdef MEASURETIME - GetElapsedTime( &(lOtherTime) ); -#endif - - usOverlaps = CheckCompactArea( &Sentence, this ); - T5LOG( T5INFO) << "TmtXGet: Checked compact area, usOverlaps=" << usOverlaps; - if ( usOverlaps == Sentence.usActVote ) //all hash triples found - { -#ifdef MEASURETIME - GetElapsedTime( &(lOtherTime) ); -#endif - if ( pTmGetIn->ulParm & GET_EXACT ) - { - T5LOG( T5INFO) << "TmtXGet: Calling GetExactMatch" ; - //get exact matches only - usRc = GetExactMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound, pTmGetOut ); -#ifdef MEASURETIME - GetElapsedTime( &(lGetExactTime) ); -#endif - //if usNumMatchesFound is zero then try for fuzzies - if ( (pTmGetOut->usNumMatchesFound == 0) && (usRc == NO_ERROR) ) - { - T5LOG( T5INFO) << "TmtXGet: No exact matches found, trying GetFuzzyMatch" ; - usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); - T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; - } /* endif */ -#ifdef MEASURETIME - GetElapsedTime( &(lGetFuzzyTime) ); -#endif - } - else - { - T5LOG( T5INFO) << "TmtXGet: Calling GetFuzzyMatch" ; - - usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); - T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; - -#ifdef MEASURETIME - GetElapsedTime( &(lGetFuzzyTime) ); -#endif - } /* endif */ - } - else - { - BOOL fCheckForFuzzyMatches = FALSE; + if ( pTmGetIn->ulParm & GET_EXACT ) + { + T5LOG( T5INFO) << "TmtXGet: Calling GetExactMatch" ; + //get exact matches only + usRc = GetExactMatch( this, &Sentence, pTmGetIn, + pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound, pTmGetOut ); + + //if usNumMatchesFound is zero then try for fuzzies + if ( (pTmGetOut->usNumMatchesFound == 0) && (usRc == NO_ERROR) ) + { + T5LOG( T5INFO) << "TmtXGet: No exact matches found, trying GetFuzzyMatch" ; + usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, + pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); + T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; + } /* endif */ + } + else + { + T5LOG( T5INFO) << "TmtXGet: Calling GetFuzzyMatch" ; + usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, + pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); + T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; + } /* endif */ + } + else + { + BOOL fCheckForFuzzyMatches = FALSE; - // check if the overlap is good enough... + // check if the overlap is good enough... - // GQ 2016/04/05: new approach: (tentative fix for P403177) - // sentences with up to 8 (and incl) votes: always check fuzzyness - // sentences from 9 up to 12 votes: check fuzziness when 20% of votes are matching - // sentences with 12 or more votes: check fuzziness when pTmGetIn->usMatchThreshold (=33%) of votes are matching + // GQ 2016/04/05: new approach: (tentative fix for P403177) + // sentences with up to 8 (and incl) votes: always check fuzzyness + // sentences from 9 up to 12 votes: check fuzziness when 20% of votes are matching + // sentences with 12 or more votes: check fuzziness when pTmGetIn->usMatchThreshold (=33%) of votes are matching - // old code: - // if (( usOverlaps * 100 >= pSentence->usActVote * pTmGetIn->usMatchThreshold ) || (pSentence->usActVote <= 8)) + // old code: + // if (( usOverlaps * 100 >= pSentence->usActVote * pTmGetIn->usMatchThreshold ) || (pSentence->usActVote <= 8)) - // GQ 2016/05/12: disabled new approach as it causes a 12% decrease in the overall analysis performance - if ( Sentence.usActVote <= 8 ) - { - fCheckForFuzzyMatches = TRUE; - } - //else if ( pSentence->usActVote < 18 ) - //{ - // fCheckForFuzzyMatches = (usOverlaps * 100) >= (pSentence->usActVote * 20); - //} - else - { - fCheckForFuzzyMatches = (usOverlaps * 100) >= (Sentence.usActVote * pTmGetIn->usMatchThreshold); - } /* endif */ - - if ( fCheckForFuzzyMatches ) - { - T5LOG(T5INFO) << "TmtXGet: usOverlaps= "<stMatchTable, &pTmGetOut->usNumMatchesFound ); + if ( fCheckForFuzzyMatches ) + { + T5LOG(T5INFO) << "TmtXGet: usOverlaps= "<usNumMatchesFound<<" matches" ; - -#ifdef MEASURETIME - GetElapsedTime( &(lGetFuzzyTime) ); -#endif - } /* endif */ - } /* endif */ + //not all triples on in compact area so fuzzy match + T5LOG( T5INFO) << "TmtXGet: Calling GetFuzzyMatch" ; + usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); + T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned "<usNumMatchesFound<<" matches" ; + } /* endif */ } /* endif */ - } } /* endif */ // store indication if more matches are available diff --git a/source/opentm2/core/utilities/EQFMORPH.cpp b/source/opentm2/core/utilities/EQFMORPH.cpp index 697677b3..5d05f139 100755 --- a/source/opentm2/core/utilities/EQFMORPH.cpp +++ b/source/opentm2/core/utilities/EQFMORPH.cpp @@ -136,14 +136,6 @@ static MORPHLANGTYPE LangTypes[] = #define min( a, b ) ( a < b ) ? (a) : (b) #endif -#if defined(MEASURETIME) - #define INITTIME( pLCB ) pLCB->ulLastTime = pLCB->pInfoSeg->msecs - #define GETTIME( pLCB, ulTime ) \ - { \ - ulTime += pLCB->pInfoSeg->msecs - pLCB->ulLastTime;\ - pLCB->ulLastTime = pLCB->pInfoSeg->msecs; \ - } -#endif /**********************************************************************/ /* indicator for DBCS initialisation */ From ac830e3499fc15a5f1b99d31f3a92bd02f397257 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 18 Jun 2025 22:07:28 +0300 Subject: [PATCH 22/62] * fuzzy code refactoring * code cleanup and restructured --- include/EQF.H | 3 - include/EQFFUZZ.H | 2 +- include/EQFMORPH.H | 3 +- include/EQFTAG.H | 2 +- include/EQFTP.H | 2 +- include/tm.h | 4 - .../opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp | 46 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 1 - .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 583 ++++--- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 301 ++-- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 1485 ++++++++--------- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 437 +++-- source/opentm2/core/morph/OtmMorphICU.cpp | 33 +- source/opentm2/core/tagtable/EQFTOKEN.cpp | 170 +- source/opentm2/core/utilities/EQFMORPH.cpp | 8 +- source/opentm2/core/utilities/LogWrapper.h | 1 - .../OtmMarkupTablePlugin/OTMRTF/OTMRTF.H | 2 - 17 files changed, 1401 insertions(+), 1682 deletions(-) diff --git a/include/EQF.H b/include/EQF.H index 640729e6..f869d3ef 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -3712,8 +3712,6 @@ BOOL UtlRegisterEqfHelp( HAB ); BOOL UtlTerminateUtils( VOID ); USHORT UtlError ( SHORT, USHORT, USHORT, PSZ *, ERRTYPE ); USHORT UtlErrorHwnd ( SHORT, USHORT, USHORT, PSZ *, ERRTYPE, HWND ); -USHORT UtlErrorW ( SHORT, USHORT, USHORT, PSZ_W *, ERRTYPE, BOOL ); -USHORT UtlErrorHwndW ( SHORT, USHORT, USHORT, PSZ_W *, ERRTYPE, HWND, BOOL ); VOID UtlGetMsgTxt( SHORT, PSZ, USHORT, PSZ *); VOID UtlInitError ( HAB, HWND, HWND, PSZ ); @@ -5730,7 +5728,6 @@ NTMCompareBetweenTokens PSZ_W pD2, PSZ pszMarkup, SHORT sLangID, - ULONG ulSrcCP, PBOOL pfStringEqual ); diff --git a/include/EQFFUZZ.H b/include/EQFFUZZ.H index b9b323a1..297d37b8 100755 --- a/include/EQFFUZZ.H +++ b/include/EQFFUZZ.H @@ -41,7 +41,7 @@ SHORT CompFuzzyTok ( PFUZZYTOK , PFUZZYTOK, BOOL ); VOID MakeHashValue ( PULONG, USHORT, PSZ_W, PULONG ); //static -BOOL PrepareTokens ( PLOADEDTABLE, std::vector&, std::vector&, PSZ_W, SHORT, std::vector &, ULONG ); +BOOL PrepareTokens ( PLOADEDTABLE, std::vector&, std::vector&, PSZ_W, SHORT, std::vector & ); //static BOOL FuzzyReplace ( PSZ_W, PSZ_W, PSZ_W, PREPLLIST, PREPLLIST ); //static diff --git a/include/EQFMORPH.H b/include/EQFMORPH.H index 3371c24e..c03a501e 100755 --- a/include/EQFMORPH.H +++ b/include/EQFMORPH.H @@ -182,8 +182,7 @@ USHORT MorphTokenizeW ( SHORT sLanguageID, // language ID PSZ_W pszInData, // pointer to input segment - std::vector& ppTermList, // address of caller's term list pointer - ULONG ulOemCP + std::vector& ppTermList// address of caller's term list pointer ); diff --git a/include/EQFTAG.H b/include/EQFTAG.H index 09313575..8b2aee8f 100755 --- a/include/EQFTAG.H +++ b/include/EQFTAG.H @@ -224,7 +224,7 @@ VOID EQFTagTokenize(PSZ, TAGTABLE *, BOOL, PSZ *, USHORT *, TOKENENTRY *, USHORT //BOOL TATagTokenize (PSZ, PLOADEDTABLE, BOOL, PSZ *, USHORT *, TOKENENTRY *, USHORT, ULONG); -BOOL TATagTokenizeW(PSZ_W, PLOADEDTABLE, BOOL, PSZ_W *, USHORT *, std::vector&); +BOOL TATagTokenizeW(PSZ_W, PLOADEDTABLE, PSZ_W *, USHORT *, std::vector&); /**********************************************************************/ diff --git a/include/EQFTP.H b/include/EQFTP.H index 11e9198f..7d6c4900 100755 --- a/include/EQFTP.H +++ b/include/EQFTP.H @@ -984,7 +984,7 @@ VOID EQFBVioSetNewDocSize ( PTBDOCUMENT ); // refresh doc size BOOL EQFBFindDiffEx( PVOID, std::vector& , std::vector&, PSZ_W, PSZ_W, SHORT , - std::vector&, std::vector&, ULONG); + std::vector&, std::vector&); BOOL EQFBCountDiff( std::vector& pvFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ); diff --git a/include/tm.h b/include/tm.h index c11c0db6..6a60afd5 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1675,8 +1675,6 @@ typedef struct _TMX_GET_W ULONG ulParm; // for future use, xmp. GET_ONLY_MT_MATCHES, GET_ONLY_EXACT_MATCHES...) CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - ULONG ulSrcOemCP; - ULONG ulTgtOemCP; PVOID pvReplacementList; // ptr to a SGML-DITA replacement list or NULL PVOID pvGMOptList; // ptr to a global memory option list or NULL bool fSourceLangIsPrefered; @@ -2567,7 +2565,6 @@ BOOL TMFuzzynessEx( PSZ_W pszMatch, // found match SHORT sLanguageId, // language id to be used PUSHORT pusFuzzy, // fuzzyness - ULONG ulOemCP, PUSHORT pusWords, // number of words in segment PUSHORT pusDiffs // number of diffs in segment ); @@ -3512,7 +3509,6 @@ MRESULT MemImportCallBack(PPROCESSCOMMAREA, HWND, WINMSG, WPARAM, LPARAM); // returned ID if a name is not found in the table #define NTMGETID_NOTFOUND_ID 0xFFFF -USHORT TmtXGet(EqfMemory *, PTMX_GET_W, PTMX_GET_OUT_W); // tm put prototypes VOID HashSentence(PTMX_SENTENCE); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp index 93468d6f..dd9a8222 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp @@ -159,12 +159,11 @@ BOOL PrepareTokens ( PLOADEDTABLE pTagTable, - std::vector& pInBuf, - std::vector& pTokBuf, + std::vector& pInBuf, + std::vector& pTokBuf, PSZ_W pString, // pointer to string to be tokenized SHORT sLanguageId, // language ID - std::vector& pTransTokList, // resulting list of tokens - ULONG ulOemCP + std::vector& pTransTokList // resulting list of tokens ) { PCHAR_W pRest; // ptr to start of not-processed bytes @@ -190,7 +189,6 @@ PrepareTokens /********************************************************************/ TATagTokenizeW(pString, pTagTable, - TRUE, &pRest, &usColPos, pTokBuf ); @@ -212,9 +210,8 @@ PrepareTokens chTemp = *(pTok->pDataStringW+pTok->iLength); *(pTok->pDataStringW + pTok->iLength) = EOS; - usRC = MorphTokenizeW( sLanguageId, pTok->pDataStringW, - pTermList, - ulOemCP ); + usRC = MorphTokenizeW( sLanguageId, pTok->pDataStringW, + pTermList ); *(pTok->pDataStringW + pTok->iLength) = chTemp; @@ -1182,17 +1179,15 @@ BOOL EQFBFindDiffEx ( PVOID pTable, // pointer to loaded tagtable - std::vector& pInBuf, // pointer to input buffer - std::vector& pTokBuf, // pointer to temp token buffer + std::vector& pInBuf, // pointer to input buffer + std::vector& pTokBuf, // pointer to temp token buffer PSZ_W pString1, // first string passed PSZ_W pString2, // second string SHORT sLanguageId, // language id to be used for pString1 - std::vector& pFuzzyTok, // returned token list of pString2 - std::vector& pFuzzyTgt, // returned token list of pString1 - ULONG ulOemCP + std::vector& pFuzzyTok, // returned token list of pString2 + std::vector& pFuzzyTgt // returned token list of pString1 ) { - BOOL fOK = TRUE; // success indicator USHORT usLenStr1 = 0; // length of string 1 (in num. of tokens) USHORT usLenStr2 = 0; // length of string 2 (in num. of tokens) std::vector pTokenList2, pTokenList1; // pointer to token lists @@ -1202,18 +1197,18 @@ EQFBFindDiffEx /******************************************************************/ /* prepare tokens for String1 and string 2 */ /******************************************************************/ - fOK = PrepareTokens( pTagTable, + BOOL fOK = PrepareTokens( pTagTable, pInBuf, pTokBuf, pString1, - sLanguageId, pTokenList1, ulOemCP ); + sLanguageId, pTokenList1 ); if ( fOK ) { fOK = PrepareTokens( pTagTable, pInBuf, pTokBuf, pString2, - sLanguageId, pTokenList2, ulOemCP ); + sLanguageId, pTokenList2 ); } /* endif */ if (fOK ) { @@ -2241,7 +2236,6 @@ NTMCompareBetweenTokens PSZ_W pD2, PSZ pszMarkup, SHORT sLangID, - ULONG ulSrcCP, PBOOL pfStringEqual ) { @@ -2269,15 +2263,11 @@ NTMCompareBetweenTokens pInBufL.resize(64000/sizeof(pInBufL[0])); pTokBuf.reserve(TOK_BUFFER_SIZE/sizeof(TOKENENTRY)); - fOK = true; - // load tag table - if ( fOK ) - { - fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, - TALOADUSEREXIT | TALOADPROTTABLEFUNC, - FALSE, NULLHANDLE ) == 0 /*NO_ERROR*/); - } /* endif */ + fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, + TALOADUSEREXIT | TALOADPROTTABLEFUNC, + FALSE, NULLHANDLE ) == 0 /*NO_ERROR*/); + if (fOK && (pTable->pfnProtTableW || pTable->pfnProtTable)) { // protect-unprotect cannot be detected via TATagTokenize in PrepareTokens @@ -2293,14 +2283,14 @@ NTMCompareBetweenTokens fOK = PrepareTokens( pTable, pInBufL, pTokBuf, - pD1, sLangID, pTokenList1, ulSrcCP ); + pD1, sLangID, pTokenList1 ); } if ( fOK) { fOK = PrepareTokens( pTable, pInBufL, pTokBuf, - pD2, sLangID, pTokenList2, ulSrcCP ); + pD2, sLangID, pTokenList2 ); } // prereq is FUzzyness= 100%, i.e. all texttokens have been MARK_EQUAL in TMFuzzyness!! // check whether all tokens ( text + tag) are in both lists! diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index da27d99b..0a757b09 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -929,7 +929,6 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT { pLIDA->mem->importDetails->invalidSegmentsRCs[usRC] ++; pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter - //if( pLIDA->mem->importDetails->invalidSegments < 100){ if(pLIDA->mem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ pLIDA->mem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pSegment->lSegNo, usRC) ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 7b1996d5..62b7b68c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -238,14 +238,12 @@ USHORT EqfMemory::FindTargetAndDelete( LONG& lKey ) { PTMX_RECORD pTmRecord = toTmxRecord(TmRecord); - BOOL fOK = FALSE; //success indicator BOOL fStop = FALSE; //indicates whether to leave loop or not PBYTE pByte; //position ptr PBYTE pStartTarget; //position ptr PTMX_SOURCE_RECORD pTMXSourceRecord = NULL; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord = NULL; //ptr to target record PTMX_TARGET_CLB pClb = NULL; //ptr to target control block - LONG lTagAlloc; //allocate length ULONG ulLen = 0; //length indicator LONG lSrcLen = 0; std::wstring pString; //pointer to character string @@ -253,184 +251,171 @@ USHORT EqfMemory::FindTargetAndDelete( USHORT usRc = NO_ERROR; //returned value from function USHORT usTarget = 0; //initialize counter - lTagAlloc = (LONG)TOK_SIZE; - - { - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - //move pointer to source string - pByte = (PBYTE)(pTmRecord+1); - pByte += pTMXSourceRecord->usSource; + //move pointer to source string + pByte = (PBYTE)(pTmRecord+1); + pByte += pTMXSourceRecord->usSource; - //calculate length of source string - ulLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); + //calculate length of source string + ulLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); - //copy source string for later compare function -// memcpy( pString, pByte, ulLen ); - ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - lSrcLen = ulLen; + //copy source string for later compare function + ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); + lSrcLen = ulLen; - //compare source strings - if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTagStrC() ) ) - { - ULONG ulLeftTgtLen; // remaining target length - /****************************************************************/ - /* get length of target block to work with */ - /****************************************************************/ - assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - ulLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - - //source strings equal - //position at first target record - pByte = (PBYTE)(pTmRecord+1); - pByte += RECLEN(pTMXSourceRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pStartTarget = (PBYTE)pTMXTargetRecord; + //compare source strings + if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTagStrC() ) ) + { + ULONG ulLeftTgtLen; // remaining target length + /****************************************************************/ + /* get length of target block to work with */ + /****************************************************************/ + assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); + ulLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + + //source strings equal + //position at first target record + pByte = (PBYTE)(pTmRecord+1); + pByte += RECLEN(pTMXSourceRecord); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pStartTarget = (PBYTE)pTMXTargetRecord; - //source strings are identical so loop through target records - while ( ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0) && !fStop ) + //source strings are identical so loop through target records + while ( ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0) && !fStop ) + { + usTarget++; + /**************************************************************/ + /* update left target length */ + /**************************************************************/ + assert( ulLeftTgtLen >= RECLEN(pTMXTargetRecord) ); + ulLeftTgtLen -= RECLEN(pTMXTargetRecord); + + //next check the target language + //position at target control block + pByte += pTMXTargetRecord->usClb; + pClb = (PTMX_TARGET_CLB)pByte; + + //get id of target language in the put structure + usRc = NTMGetIDFromName( TmDel.szTargetLanguage, + NULL, + (USHORT)LANG_KEY, &usId ); + //compare target language ids + if ( (pClb->usLangId == usId) && !usRc ) { - usTarget++; - /**************************************************************/ - /* update left target length */ - /**************************************************************/ - assert( ulLeftTgtLen >= RECLEN(pTMXTargetRecord) ); - ulLeftTgtLen -= RECLEN(pTMXTargetRecord); - - //next check the target language - //position at target control block - pByte += pTMXTargetRecord->usClb; - pClb = (PTMX_TARGET_CLB)pByte; - - //get id of target language in the put structure - usRc = NTMGetIDFromName( TmDel.szTargetLanguage, - NULL, - (USHORT)LANG_KEY, &usId ); - //compare target language ids - if ( (pClb->usLangId == usId) && !usRc ) + //compare source tag table records + //position at source tag table record + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + + //source tag tables are identical + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pByte += pTMXTargetRecord->usTarget; + + //calculate length of target string + ulLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; + + //copy target string for later compare function + pString.clear(); + ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); + + //compare target strings + if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) { - //compare source tag table records - //position at source tag table record + //target strings are equal so compare target tag records + //position at target tag table record pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - { - //source tag tables are identical - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pByte += pTMXTargetRecord->usTarget; - - //calculate length of target string - ulLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; + //compare tag table records + //identical target tag table as in del structure so + //check segment id and file name in control block + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //copy target string for later compare function - pString.clear(); - ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); + //position at target control block + pByte += pTMXTargetRecord->usClb; + pClb = (PTMX_TARGET_CLB)pByte; - if ( !usRc ) + //get id of filename in the put structure + usRc = NTMGetIDFromName( TmDel.szDocName, + TmDel.szDocName, + (USHORT)FILE_KEY, &usId ); + if ( !usRc ) + { + ULONG ulLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + while ( ulLeftClbLen && !fStop ) { - //compare target strings - if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) + if ( + (pClb->usFileId == usId) && + (pClb->ulSegmId == TmDel.lSegmentId) && + (pClb->bTranslationFlag == (BYTE)ProposalTypeToFlag(TmDel.eType)) + ) { - //target strings are equal so compare target tag records - //position at target tag table record - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //if segment id and filename are equal then delete + //target CLB and any empty target record - //compare tag table records - if (true ) + //check that multiple flag isn't on + //if on leave while loop as though delete was carried out + if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime //|| true + ) { - //identical target tag table as in del structure so - //check segment id and file name in control block - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //position at target control block - pByte += pTMXTargetRecord->usClb; - pClb = (PTMX_TARGET_CLB)pByte; - - //get id of filename in the put structure - usRc = NTMGetIDFromName( TmDel.szDocName, - TmDel.szDocName, - (USHORT)FILE_KEY, &usId ); - if ( !usRc ) - { - ULONG ulLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - while ( ulLeftClbLen && !fStop ) - { - if ( - (pClb->usFileId == usId) && - (pClb->ulSegmId == TmDel.lSegmentId) && - (pClb->bTranslationFlag == (BYTE)ProposalTypeToFlag(TmDel.eType)) - ) - { - //if segment id and filename are equal then delete - //target CLB and any empty target record - - //check that multiple flag isn't on - //if on leave while loop as though delete was carried out - if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime //|| true - ) - { - //fill out the put structure as output of the extract function - usRc = FillExtStructure( pTMXTargetRecord, - pClb, - TmDel.pInputSentence->pStrings->getGenericTargetStr(), &lSrcLen, - pTmExtOut ); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); - pTmExtOut->ulRecKey = lKey; - TmDel.currentInternalKey.setInternalKey(lKey, usTarget); - pTmExtOut->usTargetKey = usTarget; - - TMDelTargetClb( TmRecord, pTMXTargetRecord, pClb ); - - //add updated tm record to database - /**********************************************/ - /* we usually should delete the record here */ - /* (if no translation is available for it) */ - /* BUT: since usually an untranslate is follow*/ - /* by a new translation, we will not remove */ - /* the key (only get rid of any target data) */ - /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( lKey, TmRecord, true); - } /* endif */ - - //leave while loop - fStop = TRUE; - } - else - { - usTarget++; - // try next target CLB - ulLeftClbLen -= TARGETCLBLEN(pClb); - pClb = NEXTTARGETCLB(pClb); - } /* endif */ - } /* endwhile */ - } /* endif */ + //fill out the put structure as output of the extract function + usRc = FillExtStructure( pTMXTargetRecord, + pClb, + TmDel.pInputSentence->pStrings->getGenericTargetStr(), &lSrcLen, + pTmExtOut ); + NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + pTmExtOut->szOriginalSourceLanguage, NULL ); + pTmExtOut->ulRecKey = lKey; + TmDel.currentInternalKey.setInternalKey(lKey, usTarget); + pTmExtOut->usTargetKey = usTarget; + + TMDelTargetClb( TmRecord, pTMXTargetRecord, pClb ); + + //add updated tm record to database + /**********************************************/ + /* we usually should delete the record here */ + /* (if no translation is available for it) */ + /* BUT: since usually an untranslate is follow*/ + /* by a new translation, we will not remove */ + /* the key (only get rid of any target data) */ + /**********************************************/ + usRc = TmBtree.EQFNTMUpdate( lKey, TmRecord, true); } /* endif */ + + //leave while loop + fStop = TRUE; + } + else + { + usTarget++; + // try next target CLB + ulLeftClbLen -= TARGETCLBLEN(pClb); + pClb = NEXTTARGETCLB(pClb); } /* endif */ - } /* endif */ + } /* endwhile */ } /* endif */ } /* endif */ + } /* endif */ - //position at next target - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target - pByte += RECLEN(pTMXTargetRecord); - //remember the end/beginning of record - pStartTarget = pByte; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - } /* endwhile */ - } /* endif */ + //position at next target + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //move pointer to end of target + pByte += RECLEN(pTMXTargetRecord); + //remember the end/beginning of record + pStartTarget = pByte; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + } /* endwhile */ + } /* endif */ - if ( !fStop ) - { - LOG_AND_SET_RC(usRc, T5INFO, SEG_NOT_FOUND); - } /* endif */ + if ( !fStop ) + { + LOG_AND_SET_RC(usRc, T5INFO, SEG_NOT_FOUND); } /* endif */ return( usRc ); @@ -455,80 +440,115 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( std::wstring pSourceString; //pointer to source string PBYTE pSource; //position ptr - - if ( !usRc ) + //check if record is empty + if ( ( RECLEN(pTmRecord) != 0) && + (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) { - //check if record is empty - if ( ( RECLEN(pTmRecord) != 0) && - (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) + LONG lLeftTgtLen; // remaining target length + /****************************************************************/ + /* get length of target block to work with */ + /****************************************************************/ + assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); + lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + //extract source string + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + + //move pointer to corresponding position + pSource = (PBYTE)(pTmRecord+1); + pSource += pTMXSourceRecord->usSource; + + //calculate length of source string + lSourceLen = (RECLEN(pTMXSourceRecord) - + sizeof(TMX_SOURCE_RECORD) ); + + //copy source string for fill matchtable + if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ) { - LONG lLeftTgtLen; // remaining target length - /****************************************************************/ - /* get length of target block to work with */ - /****************************************************************/ - assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - //extract source string - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - - //move pointer to corresponding position - pSource = (PBYTE)(pTmRecord+1); - pSource += pTMXSourceRecord->usSource; - - //calculate length of source string - lSourceLen = (RECLEN(pTMXSourceRecord) - - sizeof(TMX_SOURCE_RECORD) ); - - //copy source string for fill matchtable - if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ) + lSourceLen = EQFCompress2Unicode( pSourceString, pSource, lSourceLen ); + if ( lSourceLen < MAX_SEGMENT_SIZE && lSourceLen >=0 ) { + pSourceString[lSourceLen] = EOS; + } + else { - lSourceLen = EQFCompress2Unicode( pSourceString, pSource, lSourceLen ); - if ( lSourceLen < MAX_SEGMENT_SIZE && lSourceLen >=0 ) { - pSourceString[lSourceLen] = EOS; - } - else - { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); lSoureLen = " << lSourceLen;} - } + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); lSoureLen = " << lSourceLen;} + } + } + else + { + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< " ((RECLEN(pTmRecord) == 0) || (pTmRecord->usFirstTargetRecord >= RECLEN(pTmRecord)) ); reclen = " << RECLEN(pTmRecord) << ";usFirstTargetRecord="<usFirstTargetRecord;} + } /* endif */ + + if ( !usRc ) + { + //find target record specified in ExtIn structure + + //move pointer to first target + pByte = (PBYTE)(pTmRecord); + pByte += pTmRecord->usFirstTargetRecord; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); + if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ) + { + // target record is corrupted, continue with next TM record + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< "if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ); usClb = " << pTMXTargetRecord->usClb<<"; reclen = " << RECLEN(pTMXTargetRecord);} } else { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< " ((RECLEN(pTmRecord) == 0) || (pTmRecord->usFirstTargetRecord >= RECLEN(pTmRecord)) ); reclen = " << RECLEN(pTmRecord) << ";usFirstTargetRecord="<usFirstTargetRecord;} + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + lLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB } /* endif */ + usTarget = 1; //initialize counter - if ( !usRc ) + //loop until correct target is found + while ( (usTarget < TmDel.nextInternalKey.getTargetKey()) && (lLeftTgtLen > 0) && !usRc ) { - //find target record specified in ExtIn structure - - //move pointer to first target - pByte = (PBYTE)(pTmRecord); - pByte += pTmRecord->usFirstTargetRecord; + // position to first target CLB pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ) + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + + // subtract size of current CLB + if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) { - // target record is corrupted, continue with next TM record - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< "if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ); usClb = " << pTMXTargetRecord->usClb<<"; reclen = " << RECLEN(pTMXTargetRecord);} + lLeftClbLen -= TARGETCLBLEN(pTargetClb); } else { - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - lLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB - } /* endif */ - usTarget = 1; //initialize counter + // database is corrupted + lLeftClbLen = 0; + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ + + // loop over all target CLBs + while ( (usTarget < TmDel.nextInternalKey.getTargetKey()) && ( lLeftClbLen > 0 ) ) + { + usTarget++; + pTargetClb = NEXTTARGETCLB(pTargetClb); + lLeftClbLen -= TARGETCLBLEN(pTargetClb); + } /* endwhile */ -//NEW CODE - //loop until correct target is found - while ( (usTarget < TmDel.nextInternalKey.getTargetKey()) && (lLeftTgtLen > 0) && !usRc ) + // continue with next target if no match yet + if ( usTarget < TmDel.nextInternalKey.getTargetKey() ) { - // position to first target CLB + // position at next target + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //move pointer to end of target + if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) + { + lLeftTgtLen -= RECLEN(pTMXTargetRecord); + } + else + { + lLeftTgtLen = 0; + } /* endif */ + pByte += RECLEN(pTMXTargetRecord); pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - + pTMXTargetRecord->usClb; // subtract size of current CLB if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) { @@ -540,21 +560,51 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( lLeftClbLen = 0; LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ + usTarget++; + } /* endif */ + } /* endwhile */ - // loop over all target CLBs - while ( (usTarget < TmDel.nextInternalKey.getTargetKey()) && ( lLeftClbLen > 0 ) ) - { - usTarget++; - pTargetClb = NEXTTARGETCLB(pTargetClb); - lLeftClbLen -= TARGETCLBLEN(pTargetClb); - } /* endwhile */ + if ( !usRc && (usTarget == TmDel.nextInternalKey.getTargetKey()) ) + { + //position at start of target record + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - // continue with next target if no match yet - if ( usTarget < TmDel.nextInternalKey.getTargetKey() ) + //if target record exists + if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) + { + //check that multiple flag isn't on + //if on leave while loop as though delete was carried out + + //fill out the put structure as output of the extract function + usRc = FillExtStructure( pTMXTargetRecord, + pTargetClb, + pSourceString, &lSourceLen, + pTmExtOut ); + NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + pTmExtOut->szOriginalSourceLanguage, NULL ); + TmDel.currentInternalKey.setInternalKey(lKey, usTarget); + pTmExtOut->ulRecKey = lKey; + pTmExtOut->usTargetKey = usTarget; + + //if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime ) + if(pTargetClb->ulSegmId != TmDel.lSegmentId){ + return SEGMENT_ID_NOT_EQUAL; + } else { + TMDelTargetClb( TmRecord, pTMXTargetRecord, pTargetClb ); + + //add updated tm record to database + /**********************************************/ + /* we usually should delete the record here */ + /* (if no translation is available for it) */ + /* BUT: since usually an untranslate is follow*/ + /* by a new translation, we will not remove */ + /* the key (only get rid of any target data) */ + /**********************************************/ + usRc = TmBtree.EQFNTMUpdate( lKey, TmRecord, true); + } /* endif */ + if ( ! usRc ) { - // position at next target - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target + //check for another target if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) { lLeftTgtLen -= RECLEN(pTMXTargetRecord); @@ -565,81 +615,6 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( } /* endif */ pByte += RECLEN(pTMXTargetRecord); pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - // subtract size of current CLB - if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) - { - lLeftClbLen -= TARGETCLBLEN(pTargetClb); - } - else - { - // database is corrupted - lLeftClbLen = 0; - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ - usTarget++; - } /* endif */ - } /* endwhile */ - - if ( !usRc && (usTarget == TmDel.nextInternalKey.getTargetKey()) ) - { - //position at start of target record - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //if target record exists - if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) - { - //check that multiple flag isn't on - //if on leave while loop as though delete was carried out - - //fill out the put structure as output of the extract function - usRc = FillExtStructure( pTMXTargetRecord, - pTargetClb, - pSourceString, &lSourceLen, - pTmExtOut ); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); - TmDel.currentInternalKey.setInternalKey(lKey, usTarget); - pTmExtOut->ulRecKey = lKey; - pTmExtOut->usTargetKey = usTarget; - - //if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime ) - if(pTargetClb->ulSegmId != TmDel.lSegmentId){ - return SEGMENT_ID_NOT_EQUAL; - } else { - TMDelTargetClb( TmRecord, pTMXTargetRecord, pTargetClb ); - - //add updated tm record to database - /**********************************************/ - /* we usually should delete the record here */ - /* (if no translation is available for it) */ - /* BUT: since usually an untranslate is follow*/ - /* by a new translation, we will not remove */ - /* the key (only get rid of any target data) */ - /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( lKey, TmRecord, true); - } /* endif */ - if ( ! usRc ) - { - //check for another target - if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) - { - lLeftTgtLen -= RECLEN(pTMXTargetRecord); - } - else - { - lLeftTgtLen = 0; - } /* endif */ - pByte += RECLEN(pTMXTargetRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - } /* endif */ - } - else - { - //no more target so get next tm record and initialize target count - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ } else @@ -648,9 +623,13 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ } - } - - } /* endif */ + else + { + //no more target so get next tm record and initialize target count + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ + } + } return( usRc ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index be7068da..e75a417a 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -65,12 +65,10 @@ USHORT EqfMemory::TmtXExtract ) { USHORT usRc = BTREE_NOT_FOUND; //return code - BOOL fOK = true; //success indicator PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record LONG lStartKey; //start tm key LONG lNextKey; //last tm key - ULONG ulOemCP = 1L; std::vector pTmRecord; // space for user data //pTmRecord.resize(2*TMX_REC_SIZE); @@ -145,106 +143,152 @@ USHORT EqfMemory::ExtractRecordV6 std::wstring pSourceString; //pointer to source string PBYTE pSource; //position ptr - - if ( !usRc ) + //check if record is empty + if ( (!pTmRecordData.empty()) && ( RECLEN(pTmRecord) != 0) && + (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) { - //check if record is empty - if ( (!pTmRecordData.empty()) && ( RECLEN(pTmRecord) != 0) && - (pTmRecord->usFirstTargetRecord < RECLEN(pTmRecord)) ) + LONG lLeftTgtLen; // remaining target length + /****************************************************************/ + /* get length of target block to work with */ + /****************************************************************/ + assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); + lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + //extract source string + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + + //move pointer to corresponding position + pSource = (PBYTE)(pTmRecord+1); + pSource += pTMXSourceRecord->usSource; + + //calculate length of source string + lSourceLen = (RECLEN(pTMXSourceRecord) - + sizeof(TMX_SOURCE_RECORD) ); + + //copy source string for fill matchtable + if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ) { - LONG lLeftTgtLen; // remaining target length - /****************************************************************/ - /* get length of target block to work with */ - /****************************************************************/ - assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - //extract source string - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - - //move pointer to corresponding position - pSource = (PBYTE)(pTmRecord+1); - pSource += pTMXSourceRecord->usSource; - - //calculate length of source string - lSourceLen = (RECLEN(pTMXSourceRecord) - - sizeof(TMX_SOURCE_RECORD) ); - - //copy source string for fill matchtable - if ( lSourceLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ) + lSourceLen = EQFCompress2Unicode( pSourceString, pSource, lSourceLen ); + if ( lSourceLen < MAX_SEGMENT_SIZE && lSourceLen >=0 ) { + pSourceString[lSourceLen] = EOS; + } + else { - lSourceLen = EQFCompress2Unicode( pSourceString, pSource, lSourceLen ); - if ( lSourceLen < MAX_SEGMENT_SIZE && lSourceLen >=0 ) { -//// memcpy( pSourceString, pSource, ulSourceLen ); - pSourceString[lSourceLen] = EOS; - } - else - { - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( lSourceLen >= MAX_SEGMENT_SIZE || lSourceLen<0 ); lSourceLen" << lSourceLen;} - } + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( lSourceLen >= MAX_SEGMENT_SIZE || lSourceLen<0 ); lSourceLen" << lSourceLen;} + } + } + else + { + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<< "if ( lSourceLen >= MAX_SEGMENT_SIZE * sizeof(WCHAR) ); lSourceLen=" <usFirstTargetRecord; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); + if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ) + { + // target record is corrupted, continue with next TM record + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ); usClb=" << pTMXTargetRecord->usClb <<"; reclen="<= MAX_SEGMENT_SIZE * sizeof(WCHAR) ); lSourceLen=" <usClb; + lLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB } /* endif */ + usTarget = 1; //initialize counter - if ( !usRc ) + //loop until correct target is found + while ( (usTarget < pTmExtIn->usNextTarget) && (lLeftTgtLen > 0) && !usRc ) { - //find target record specified in ExtIn structure - - //move pointer to first target - pByte = (PBYTE)(pTmRecord); - pByte += pTmRecord->usFirstTargetRecord; + // position to first target CLB pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ) + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + + // subtract size of current CLB + if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) { - // target record is corrupted, continue with next TM record - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( pTMXTargetRecord->usClb >= RECLEN(pTMXTargetRecord) ); usClb=" << pTMXTargetRecord->usClb <<"; reclen="<usClb; - lLeftClbLen -= TARGETCLBLEN(pTargetClb); // subtract size of current CLB - } /* endif */ - usTarget = 1; //initialize counter + // database is corrupted + lLeftClbLen = 0; + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ - //loop until correct target is found - while ( (usTarget < pTmExtIn->usNextTarget) && (lLeftTgtLen > 0) && !usRc ) + // loop over all target CLBs + while ( (usTarget < pTmExtIn->usNextTarget) && ( lLeftClbLen > 0 ) ) { - // position to first target CLB - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; + usTarget++; + pTargetClb = NEXTTARGETCLB(pTargetClb); + lLeftClbLen -= TARGETCLBLEN(pTargetClb); + } /* endwhile */ - // subtract size of current CLB - if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) + // continue with next target if no match yet + if ( usTarget < pTmExtIn->usNextTarget ) + { + // position at next target + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //move pointer to end of target + if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) { - lLeftClbLen -= TARGETCLBLEN(pTargetClb); + lLeftTgtLen -= RECLEN(pTMXTargetRecord); } else { - // database is corrupted - lLeftClbLen = 0; - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ + lLeftTgtLen = 0; + } /* endif */ - // loop over all target CLBs - while ( (usTarget < pTmExtIn->usNextTarget) && ( lLeftClbLen > 0 ) ) - { - usTarget++; - pTargetClb = NEXTTARGETCLB(pTargetClb); - lLeftClbLen -= TARGETCLBLEN(pTargetClb); - } /* endwhile */ + if(lLeftClbLen > 0){//need to add this check in 0.7 + pByte += RECLEN(pTMXTargetRecord); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + // subtract size of current CLB + if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) + { + lLeftClbLen -= TARGETCLBLEN(pTargetClb); + } + else + { + // database is corrupted + lLeftClbLen = 0; + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ + } + + usTarget++; + } /* endif */ + } /* endwhile */ + + if ( !usRc && (usTarget == pTmExtIn->usNextTarget) ) + { + //position at start of target record + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - // continue with next target if no match yet - if ( usTarget < pTmExtIn->usNextTarget ) + //if target record exists + if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) + { + //fill out the put structure as output of the extract function + usRc = FillExtStructure( pTMXTargetRecord, + pTargetClb, + pSourceString, &lSourceLen, + pTmExtOut ); + NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + pTmExtOut->szOriginalSourceLanguage, NULL ); + if ( ! usRc ) { - // position at next target - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target + //check for another target if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) { lLeftTgtLen -= RECLEN(pTMXTargetRecord); @@ -253,81 +297,25 @@ USHORT EqfMemory::ExtractRecordV6 { lLeftTgtLen = 0; } /* endif */ + pByte += RECLEN(pTMXTargetRecord); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - if(lLeftClbLen > 0){//need to add this check in 0.7 - pByte += RECLEN(pTMXTargetRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - // subtract size of current CLB - if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) - { - lLeftClbLen -= TARGETCLBLEN(pTargetClb); - } - else - { - // database is corrupted - lLeftClbLen = 0; - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); - } /* endif */ + //if target exists + if ( ((lLeftTgtLen > 0 ) && (RECLEN(pTMXTargetRecord) > 0)) + || + (lLeftClbLen > 0) + ) + { + //increase target count and leave tm record key number as before + pTmExtOut->lTmKey = pTmExtIn->lTmKey; + pTmExtOut->usNextTarget = pTmExtIn->usNextTarget + 1; } - - usTarget++; - } /* endif */ - } /* endwhile */ - - if ( !usRc && (usTarget == pTmExtIn->usNextTarget) ) - { - //position at start of target record - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //if target record exists - if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) - { - //fill out the put structure as output of the extract function - usRc = FillExtStructure( pTMXTargetRecord, - pTargetClb, - pSourceString, &lSourceLen, - pTmExtOut ); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); - if ( ! usRc ) + else { - //check for another target - if (lLeftTgtLen >= RECLEN(pTMXTargetRecord)) - { - lLeftTgtLen -= RECLEN(pTMXTargetRecord); - } - else - { - lLeftTgtLen = 0; - } /* endif */ - pByte += RECLEN(pTMXTargetRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - //if target exists - if ( ((lLeftTgtLen > 0 ) && (RECLEN(pTMXTargetRecord) > 0)) - || - (lLeftClbLen > 0) - ) - { - //increase target count and leave tm record key number as before - pTmExtOut->lTmKey = pTmExtIn->lTmKey; - pTmExtOut->usNextTarget = pTmExtIn->usNextTarget + 1; - } - else - { - //no more target so get next tm record and initialize target count - pTmExtOut->lTmKey = pTmExtIn->lTmKey +1; - pTmExtOut->usNextTarget = 1; - } /* endif */ + //no more target so get next tm record and initialize target count + pTmExtOut->lTmKey = pTmExtIn->lTmKey +1; + pTmExtOut->usNextTarget = 1; } /* endif */ - } - else - { - //no more target so get next tm record and initialize target count - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ } else @@ -335,16 +323,19 @@ USHORT EqfMemory::ExtractRecordV6 //no more target so get next tm record and initialize target count LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ + } + else + { + //no more target so get next tm record and initialize target count + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ - } - else - { - //if record is empty, get next tm record and initialize target count - LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); } /* endif */ - } - + else + { + //if record is empty, get next tm record and initialize target count + LOG_AND_SET_RC(usRc, T5INFO, BTREE_NOT_FOUND); + } /* endif */ return usRc; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 9a0d97c6..5f281291 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -181,106 +181,87 @@ USHORT EqfMemory::TmtXGet szString[0] = '\0'; if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - auto str_source = EncodingHelper::convertToUTF8(pTmGetIn->szSource ); - T5LOG(T5DEBUG) << "== Lookup in memory ==" << stTmSign.szName <<" == ;\tLookupSource = >>>" <szSource); - T5LOG(T5INFO) << "-------------- Looking up:" << str << "" ; + auto str_source = EncodingHelper::convertToUTF8(pTmGetIn->szSource ); + T5LOG(T5DEBUG) << "== Lookup in memory ==" << stTmSign.szName <<" == ;\tLookupSource = >>>" <szSource); + T5LOG(T5INFO) << "-------------- Looking up:" << str << "" ; } - if ( !usRc ) + //build tag table path + Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); + strcat( szString, "/TABLE/"); + strcat( szString, pTmGetIn->szTagTable ); + strcat( szString, EXT_OF_FORMAT ); + + //tokenize source segment, resuting in normalized string and + //tag table record + usRc = TokenizeSource( &Sentence, szString, pTmGetIn->szSourceLanguage); + + // set the tag table ID in the tag record (this can't be done in TokenizeSource anymore) + if ( usRc == NO_ERROR ) { - //build tag table path - Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); - strcat( szString, "/TABLE/"); - strcat( szString, pTmGetIn->szTagTable ); - strcat( szString, EXT_OF_FORMAT ); - - //tokenize source segment, resuting in normalized string and - //tag table record - usRc = TokenizeSource( &Sentence, szString, pTmGetIn->szSourceLanguage); - - // set the tag table ID in the tag record (this can't be done in TokenizeSource anymore) - if ( usRc == NO_ERROR ) + toTmxTagtableRecord(Sentence.pTagRecord)->usTagTableId = 1; + + HashSentence( &Sentence ); + usOverlaps = CheckCompactArea( &Sentence, this ); + T5LOG( T5INFO) << "TmtXGet: Checked compact area, usOverlaps=" << usOverlaps; + + if ( usOverlaps == Sentence.usActVote ) //all hash triples found { - toTmxTagtableRecord(Sentence.pTagRecord)->usTagTableId = 1; + T5LOG( T5INFO) << "TmtXGet: Calling GetExactMatch" ; + //get exact matches only + usRc = GetExactMatch( this, &Sentence, pTmGetIn, + pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound, pTmGetOut ); + + //if usNumMatchesFound is zero then try for fuzzies + if ( (pTmGetOut->usNumMatchesFound == 0) && (usRc == NO_ERROR) ) + { + T5LOG( T5INFO) << "TmtXGet: No exact matches found, trying GetFuzzyMatch" ; + usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, + pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); + T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; + } /* endif */ } - } /* endif */ + else + { + BOOL fCheckForFuzzyMatches = FALSE; + // check if the overlap is good enough... + // GQ 2016/04/05: new approach: (tentative fix for P403177) + // sentences with up to 8 (and incl) votes: always check fuzzyness + // sentences from 9 up to 12 votes: check fuzziness when 20% of votes are matching + // sentences with 12 or more votes: check fuzziness when pTmGetIn->usMatchThreshold (=33%) of votes are matching - if ( !usRc ) - { - HashSentence( &Sentence ); - usOverlaps = CheckCompactArea( &Sentence, this ); - T5LOG( T5INFO) << "TmtXGet: Checked compact area, usOverlaps=" << usOverlaps; - if ( usOverlaps == Sentence.usActVote ) //all hash triples found - { + // old code: + // if (( usOverlaps * 100 >= pSentence->usActVote * pTmGetIn->usMatchThreshold ) || (pSentence->usActVote <= 8)) - if ( pTmGetIn->ulParm & GET_EXACT ) - { - T5LOG( T5INFO) << "TmtXGet: Calling GetExactMatch" ; - //get exact matches only - usRc = GetExactMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound, pTmGetOut ); - - //if usNumMatchesFound is zero then try for fuzzies - if ( (pTmGetOut->usNumMatchesFound == 0) && (usRc == NO_ERROR) ) - { - T5LOG( T5INFO) << "TmtXGet: No exact matches found, trying GetFuzzyMatch" ; - usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); - T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; - } /* endif */ - } - else - { - T5LOG( T5INFO) << "TmtXGet: Calling GetFuzzyMatch" ; - usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, - pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); - T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; - } /* endif */ + // GQ 2016/05/12: disabled new approach as it causes a 12% decrease in the overall analysis performance + if ( Sentence.usActVote <= 8 ) + { + fCheckForFuzzyMatches = TRUE; } - else + //else if ( pSentence->usActVote < 18 ) + //{ + // fCheckForFuzzyMatches = (usOverlaps * 100) >= (pSentence->usActVote * 20); + //} + else { - BOOL fCheckForFuzzyMatches = FALSE; - - // check if the overlap is good enough... - - - // GQ 2016/04/05: new approach: (tentative fix for P403177) - // sentences with up to 8 (and incl) votes: always check fuzzyness - // sentences from 9 up to 12 votes: check fuzziness when 20% of votes are matching - // sentences with 12 or more votes: check fuzziness when pTmGetIn->usMatchThreshold (=33%) of votes are matching - - // old code: - // if (( usOverlaps * 100 >= pSentence->usActVote * pTmGetIn->usMatchThreshold ) || (pSentence->usActVote <= 8)) - - // GQ 2016/05/12: disabled new approach as it causes a 12% decrease in the overall analysis performance - if ( Sentence.usActVote <= 8 ) - { - fCheckForFuzzyMatches = TRUE; - } - //else if ( pSentence->usActVote < 18 ) - //{ - // fCheckForFuzzyMatches = (usOverlaps * 100) >= (pSentence->usActVote * 20); - //} - else - { - fCheckForFuzzyMatches = (usOverlaps * 100) >= (Sentence.usActVote * pTmGetIn->usMatchThreshold); - } /* endif */ + fCheckForFuzzyMatches = (usOverlaps * 100) >= (Sentence.usActVote * pTmGetIn->usMatchThreshold); + } /* endif */ - if ( fCheckForFuzzyMatches ) - { - T5LOG(T5INFO) << "TmtXGet: usOverlaps= "<stMatchTable, &pTmGetOut->usNumMatchesFound ); - T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned "<usNumMatchesFound<<" matches" ; - } /* endif */ + //not all triples on in compact area so fuzzy match + usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); + T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned "<usNumMatchesFound<<" matches" ; } /* endif */ + } /* endif */ } /* endif */ + // store indication if more matches are available if ( pTmGetOut->usNumMatchesFound > 1 ) { @@ -333,29 +314,29 @@ USHORT EqfMemory::TmtXGet /********************************************************************/ if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && (pTmGetIn->ulParm & GET_EXACT )) { - PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); - PTMX_MATCH_TABLE_W pstActMatch; - USHORT i, j; // index values - INT wsDiff; - - for ( j = 0 ; j < pTmGetOut->usNumMatchesFound - 1 ;j++ ) - { - pstActMatch = &(pTmGetOut->stMatchTable[j]); - for ( i=j+1; i< pTmGetOut->usNumMatchesFound ; i++) - { - //if (UtlCompIgnWhiteSpaceW(pstActMatch->szTarget, (pstMatchTable+i)->szTarget, 0, &wsDiff) == 0L && wsDiff == 0) - if(wcscmp(pstActMatch->szTarget, (pstMatchTable+i)->szTarget) == 0) - { - /*************************************************************/ - /* delete this match... */ - /*************************************************************/ - ULONG ulLen = ( MAX_MATCHES - i - 1 ) * sizeof(TMX_MATCH_TABLE_W); - memmove( pstMatchTable+i, pstMatchTable+1+i, ulLen ); - pTmGetOut->usNumMatchesFound--; - i--; // reset i because we deleted the i-th one - } /* endif */ - } /* endfor */ - } /* endfor */ + PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); + PTMX_MATCH_TABLE_W pstActMatch; + USHORT i, j; // index values + INT wsDiff; + + for ( j = 0 ; j < pTmGetOut->usNumMatchesFound - 1 ;j++ ) + { + pstActMatch = &(pTmGetOut->stMatchTable[j]); + for ( i=j+1; i< pTmGetOut->usNumMatchesFound ; i++) + { + //if (UtlCompIgnWhiteSpaceW(pstActMatch->szTarget, (pstMatchTable+i)->szTarget, 0, &wsDiff) == 0L && wsDiff == 0) + if(wcscmp(pstActMatch->szTarget, (pstMatchTable+i)->szTarget) == 0) + { + /*************************************************************/ + /* delete this match... */ + /*************************************************************/ + ULONG ulLen = ( MAX_MATCHES - i - 1 ) * sizeof(TMX_MATCH_TABLE_W); + memmove( pstMatchTable+i, pstMatchTable+1+i, ulLen ); + pTmGetOut->usNumMatchesFound--; + i--; // reset i because we deleted the i-th one + } /* endif */ + } /* endfor */ + } /* endfor */ } /* endif */ /********************************************************************/ @@ -365,10 +346,8 @@ USHORT EqfMemory::TmtXGet pTmGetOut->usNumMatchesFound = pTmGetOut->usNumMatchesFound < pTmGetIn->usRequestedMatches ? pTmGetOut->usNumMatchesFound : pTmGetIn->usRequestedMatches; - { - int iTemp = sizeof( TMX_GET_OUT_W); - pTmGetOut->stPrefixOut.usLengthOutput = (USHORT)(iTemp); - } + int iTemp = sizeof( TMX_GET_OUT_W); + pTmGetOut->stPrefixOut.usLengthOutput = (USHORT)(iTemp); pTmGetOut->stPrefixOut.usTmtXRc = usRc; @@ -730,472 +709,457 @@ USHORT ExactTest pContextBuffer.reserve(MAX_SEGMENT_SIZE); } /* endif */ - - { - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - //move pointer to corresponding position - pByte = (PBYTE)(pTmRecord+1); - pByte += pTMXSourceRecord->usSource; + //move pointer to corresponding position + pByte = (PBYTE)(pTmRecord+1); + pByte += pTMXSourceRecord->usSource; - //calculate length of source string - ulLen = (RECLEN(pTMXSourceRecord) - sizeof(TMX_SOURCE_RECORD)); + //calculate length of source string + ulLen = (RECLEN(pTMXSourceRecord) - sizeof(TMX_SOURCE_RECORD)); - //copy source string for later compare function - ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - auto normalizedTmStr = std::make_unique(pString.c_str()); - //compare source strings - fStringEqual = UtlCompIgnWhiteSpaceW(normalizedTmStr->getNormStrC(), pSentence->pStrings->getNormStrC(), 0) == 0L; + //copy source string for later compare function + ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); + auto normalizedTmStr = std::make_unique(pString.c_str()); + //compare source strings + fStringEqual = UtlCompIgnWhiteSpaceW(normalizedTmStr->getNormStrC(), pSentence->pStrings->getNormStrC(), 0) == 0L; - if ( fStringEqual ) + if ( fStringEqual ) + { + USHORT usTgtNum = 0; // init target number + LONG lLeftTgtLen; // remaining target length + USHORT usGetLang; // id of target language + USHORT usGetFile; // id of target file + USHORT usAlternateGetFile; // alternate ID of target file + + USHORT usOldMatches = *pusMatchEntries; + + //get id of target language in the getin structure + if (pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, + NULL, + (USHORT)LANG_KEY, &usGetLang)) { - USHORT usTgtNum = 0; // init target number - LONG lLeftTgtLen; // remaining target length - USHORT usGetLang; // id of target language - USHORT usGetFile; // id of target file - USHORT usAlternateGetFile; // alternate ID of target file - - USHORT usOldMatches = *pusMatchEntries; - - - //get id of target language in the getin structure - if (pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, - NULL, - (USHORT)LANG_KEY, &usGetLang)) - { - usGetLang = 1; - } /* endif */ - - //get file name id of file name in the getin structure - if ( pTmClb->NTMGetIDFromNameEx( pGetIn->szFileName, - pGetIn->szLongName, - (USHORT)FILE_KEY, &usGetFile, - NTMGETID_NOUPDATE_OPT,//0, - &usAlternateGetFile )) - { - usGetFile = 1; - } /* endif */ + usGetLang = 1; + } /* endif */ - /****************************************************************/ - /* get length of target block to work with */ - /****************************************************************/ + //get file name id of file name in the getin structure + if ( pTmClb->NTMGetIDFromNameEx( pGetIn->szFileName, + pGetIn->szLongName, + (USHORT)FILE_KEY, &usGetFile, + NTMGETID_NOUPDATE_OPT,//0, + &usAlternateGetFile )) + { + usGetFile = 1; + } /* endif */ - assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); + /****************************************************************/ + /* get length of target block to work with */ + /****************************************************************/ - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; + assert( RECLEN(pTmRecord) >= pTmRecord->usFirstTargetRecord ); - //position at first target record - pByte = (PBYTE)(pTmRecord+1); - pByte += RECLEN(pTMXSourceRecord); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pStartTarget = (PBYTE)pTMXTargetRecord; + lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - //source strings are identical so loop through target records - while ( (usRc == NO_ERROR) && //(ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0)) ) - - ( lLeftTgtLen > 0 ) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && ( RECLEN(pTMXTargetRecord) > 0)) + //position at first target record + pByte = (PBYTE)(pTmRecord+1); + pByte += RECLEN(pTMXSourceRecord); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pStartTarget = (PBYTE)pTMXTargetRecord; + + //source strings are identical so loop through target records + while ( (usRc == NO_ERROR) && //(ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0)) ) + + ( lLeftTgtLen > 0 ) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && ( RECLEN(pTMXTargetRecord) > 0)) + { + BOOL fTestCLB = TRUE; + BOOL fMatchingDocName = FALSE; + USHORT usContextRanking = 0; // context ranking of this match + + /**************************************************************/ + /* update left target length */ + /**************************************************************/ + usTgtNum++; // target number + lLeftTgtLen -= RECLEN(pTMXTargetRecord); + + //next check the target language + //position at target control block + pByte += pTMXTargetRecord->usClb; + pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + + //compare target language group IDs + if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usGetLang] ) { - BOOL fTestCLB = TRUE; - BOOL fMatchingDocName = FALSE; - USHORT usContextRanking = 0; // context ranking of this match - - - /**************************************************************/ - /* update left target length */ - /**************************************************************/ - usTgtNum++; // target number - lLeftTgtLen -= RECLEN(pTMXTargetRecord); + fTestCLB = FALSE; + } - //next check the target language - //position at target control block - pByte += pTMXTargetRecord->usClb; - pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + // compare target table IDs + if ( fTestCLB ) + { + //position at source tag table record + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + fStringEqual=true; + + /**********************************************************/ + /* if tagging record is unequal than we have some */ + /* (slight) differences */ + /* i.e. we will create a fully qualified string and try */ + /* another compare... */ + /**********************************************************/ + LONG lLenTmp = ulLen; // len of pString in # of w's + fStringEqual = FALSE; + auto genericTagsTmSeg = std::make_unique(std::move(pString)); + fStringEqual = (UtlCompIgnWhiteSpaceW( + pSentence->pStrings->getNpReplStrC(), + genericTagsTmSeg->getNpReplStrC(), + 0 ) == 0 ); - //compare target language group IDs - if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usGetLang] ) + if ( usRc == NO_ERROR ) { - fTestCLB = FALSE; - } + if (fStringEqual) + { + fEqualFound = TRUE; + usEqual = EQUAL_EQUAL; + } + else + { + usEqual = EQUAL_EQUAL - 3; + } /* endif */ - // compare target table IDs - if ( fTestCLB ) - { - //position at source tag table record + //extract data from tm record + //target with tags(TRUE) + //compare file name and segment id in control block + //position at control block pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - fStringEqual=true; - - - /**********************************************************/ - /* if tagging record is unequal than we have some */ - /* (slight) differences */ - /* i.e. we will create a fully qualified string and try */ - /* another compare... */ - /**********************************************************/ - LONG lLenTmp = ulLen; // len of pString in # of w's + pByte += pTMXTargetRecord->usClb; + pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; + + // loop over CLBs and look for best matching entry + LONG lLeftClbLen; // left CLB entries in CLB list + PTMX_TARGET_CLB pClb; // pointer for CLB list processing + #define SEG_DOC_AND_CONTEXT_MATCH 8 + #define DOC_AND_CONTEXT_MATCH 7 + #define CONTEXT_MATCH 6 + #define SAME_SEG_AND_DOC_MATCH 5 + #define SAME_DOC_MATCH 4 + #define MULT_DOC_MATCH 3 + #define NORMAL_MATCH 2 + #define IGNORE_MATCH 1 + SHORT sCurMatch = 0; + + // loop over all target CLBs + pClb = pTMXTargetClb; + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + while ( ( lLeftClbLen > 0 ) && (sCurMatch < SAME_SEG_AND_DOC_MATCH) ) { - fStringEqual = FALSE; - auto genericTagsTmSeg = std::make_unique(std::move(pString)); - fStringEqual = (UtlCompIgnWhiteSpaceW( - pSentence->pStrings->getNpReplStrC(), - genericTagsTmSeg->getNpReplStrC(), - 0 ) == 0 ); - } /* endif */ + //T5LOG(T5TRANSACTION) << ":: lLeftClbLen = " << lLeftClbLen << "; sCurMatch = " << sCurMatch; + USHORT usTranslationFlag = pClb->bTranslationFlag; + USHORT usCurContextRanking = 0; // context ranking of this match + BOOL fIgnoreProposal = FALSE; - if ( usRc == NO_ERROR ) - { - if (fStringEqual) + // apply global memory option file on global memory proposals + if ( pClb->bTranslationFlag == TRANSLFLAG_GLOBMEM ) { - fEqualFound = TRUE; - usEqual = EQUAL_EQUAL; - } - - else + if ( (pGetIn->pvGMOptList != NULL) && pClb->usAddDataLen ) + { + USHORT usAddDataLen = NtmGetAddData( pClb, ADDDATA_ADDINFO_ID, pContextBuffer ); + if ( usAddDataLen ) + { + GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pContextBuffer.data() ); + switch ( GobMemOpt ) + { + case GM_SUBSTITUTE_OPT: usTranslationFlag = TRANSLFLAG_NORMAL; break; + case GM_HFLAG_OPT : usTranslationFlag = TRANSLFLAG_GLOBMEM; break; + case GM_HFLAGSTAR_OPT : usTranslationFlag = TRANSLFLAG_GLOBMEMSTAR; break; + case GM_EXCLUDE_OPT : fIgnoreProposal = TRUE; break; + } /* endswitch */ + } /* endif */ + } /* endif */ + if ( pClb == pTMXTargetClb ) + { + usTargetTranslationFlag = usTranslationFlag; + } /* endif */ + } /* endif */ + + // check context strings (if any) + if ( + (!fIgnoreProposal) + && pGetIn->szContext[0] + && pClb->usAddDataLen ) { - usEqual = EQUAL_EQUAL - 3; + USHORT usContextLen = NtmGetAddData( pClb, ADDDATA_CONTEXT_ID, pContextBuffer ); + if ( usContextLen != 0 ) + { + usCurContextRanking = NTMCompareContext( pTmClb, pGetIn->szTagTable, pGetIn->szContext, pContextBuffer.data() ); + } /* endif */ } /* endif */ - //extract data from tm record - //target with tags(TRUE) - //compare file name and segment id in control block - //position at control block - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pByte += pTMXTargetRecord->usClb; - pTMXTargetClb = (PTMX_TARGET_CLB)pByte; - usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; - - // loop over CLBs and look for best matching entry + // check for matching document names + // we can compare the document name IDs + fMatchingDocName = ((pClb->usFileId == usGetFile) || (pClb->usFileId == usAlternateGetFile)); + + if ( fIgnoreProposal ) { - LONG lLeftClbLen; // left CLB entries in CLB list - PTMX_TARGET_CLB pClb; // pointer for CLB list processing - #define SEG_DOC_AND_CONTEXT_MATCH 8 - #define DOC_AND_CONTEXT_MATCH 7 - #define CONTEXT_MATCH 6 - #define SAME_SEG_AND_DOC_MATCH 5 - #define SAME_DOC_MATCH 4 - #define MULT_DOC_MATCH 3 - #define NORMAL_MATCH 2 - #define IGNORE_MATCH 1 - SHORT sCurMatch = 0; - - // loop over all target CLBs - pClb = pTMXTargetClb; - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - while ( ( lLeftClbLen > 0 ) && (sCurMatch < SAME_SEG_AND_DOC_MATCH) ) + if ( sCurMatch == 0 ) { - //T5LOG(T5TRANSACTION) << ":: lLeftClbLen = " << lLeftClbLen << "; sCurMatch = " << sCurMatch; - USHORT usTranslationFlag = pClb->bTranslationFlag; - USHORT usCurContextRanking = 0; // context ranking of this match - BOOL fIgnoreProposal = FALSE; - - // apply global memory option file on global memory proposals - if ( pClb->bTranslationFlag == TRANSLFLAG_GLOBMEM ) - { - if ( (pGetIn->pvGMOptList != NULL) && pClb->usAddDataLen ) - { - USHORT usAddDataLen = NtmGetAddData( pClb, ADDDATA_ADDINFO_ID, pContextBuffer ); - if ( usAddDataLen ) - { - GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pContextBuffer.data() ); - switch ( GobMemOpt ) - { - case GM_SUBSTITUTE_OPT: usTranslationFlag = TRANSLFLAG_NORMAL; break; - case GM_HFLAG_OPT : usTranslationFlag = TRANSLFLAG_GLOBMEM; break; - case GM_HFLAGSTAR_OPT : usTranslationFlag = TRANSLFLAG_GLOBMEMSTAR; break; - case GM_EXCLUDE_OPT : fIgnoreProposal = TRUE; break; - } /* endswitch */ - } /* endif */ - } /* endif */ - if ( pClb == pTMXTargetClb ) - { - usTargetTranslationFlag = usTranslationFlag; - } /* endif */ - } /* endif */ - - // check context strings (if any) - if ( - (!fIgnoreProposal) - && pGetIn->szContext[0] - && pClb->usAddDataLen ) - { - USHORT usContextLen = NtmGetAddData( pClb, ADDDATA_CONTEXT_ID, pContextBuffer ); - if ( usContextLen != 0 ) - { - usCurContextRanking = NTMCompareContext( pTmClb, pGetIn->szTagTable, pGetIn->szContext, pContextBuffer.data() ); - } /* endif */ - } /* endif */ - - // check for matching document names + sCurMatch = IGNORE_MATCH; + } /* endif */ + } + else if ( usCurContextRanking == 100 ) + { + if ( fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)) ) + { + if ( sCurMatch < SEG_DOC_AND_CONTEXT_MATCH ) { - // we can compare the document name IDs - fMatchingDocName = ((pClb->usFileId == usGetFile) || (pClb->usFileId == usAlternateGetFile)); - } /* endif */ - - - if ( fIgnoreProposal ) + sCurMatch = SEG_DOC_AND_CONTEXT_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; + } + } + else if ( fMatchingDocName ) + { + if ( sCurMatch < DOC_AND_CONTEXT_MATCH ) { - if ( sCurMatch == 0 ) - { - sCurMatch = IGNORE_MATCH; - } /* endif */ + sCurMatch = DOC_AND_CONTEXT_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; } - else if ( usCurContextRanking == 100 ) + else if ( sCurMatch == DOC_AND_CONTEXT_MATCH ) { - if ( fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)) ) + // we have already a match of this type so check if context ranking + if ( usCurContextRanking > usContextRanking ) { - if ( sCurMatch < SEG_DOC_AND_CONTEXT_MATCH ) - { - sCurMatch = SEG_DOC_AND_CONTEXT_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - } - else if ( fMatchingDocName ) + pTMXTargetClb = pClb; // use newer target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; + } + // use time info to ensure that latest match is used + else if ( usCurContextRanking == usContextRanking ) { - if ( sCurMatch < DOC_AND_CONTEXT_MATCH ) + // GQ 2015-04-10 New approach: If we have an exact-exact match use this one, otherwise use timestamp for the comparism + BOOL fExactExactNewCLB = fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); + BOOL fExactExactExistingCLB = ((pTMXTargetClb->usFileId == usGetFile) || (pTMXTargetClb->usFileId == usAlternateGetFile)) && + (pTMXTargetClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pTMXTargetClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); + if ( fExactExactNewCLB && !fExactExactExistingCLB ) { - sCurMatch = DOC_AND_CONTEXT_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match + // use exact-exact CLB for match + pTMXTargetClb = pClb; usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; } - else if ( sCurMatch == DOC_AND_CONTEXT_MATCH ) - { - // we have already a match of this type so check if context ranking - if ( usCurContextRanking > usContextRanking ) - { - pTMXTargetClb = pClb; // use newer target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - // use time info to ensure that latest match is used - else if ( usCurContextRanking == usContextRanking ) - { - // GQ 2015-04-10 New approach: If we have an exact-exact match use this one, otherwise use timestamp for the comparism - BOOL fExactExactNewCLB = fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); - BOOL fExactExactExistingCLB = ((pTMXTargetClb->usFileId == usGetFile) || (pTMXTargetClb->usFileId == usAlternateGetFile)) && - (pTMXTargetClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pTMXTargetClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); - if ( fExactExactNewCLB && !fExactExactExistingCLB ) - { - // use exact-exact CLB for match - pTMXTargetClb = pClb; - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - else if ( (fExactExactNewCLB == fExactExactExistingCLB) && (pClb->lTime > pTMXTargetClb->lTime) ) - { - // use newer target CLB for match - pTMXTargetClb = pClb; - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - } /* endif */ - } /* endif */ - } - else - { - if ( sCurMatch < CONTEXT_MATCH ) + else if ( (fExactExactNewCLB == fExactExactExistingCLB) && (pClb->lTime > pTMXTargetClb->lTime) ) { - sCurMatch = CONTEXT_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match + // use newer target CLB for match + pTMXTargetClb = pClb; usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; } - else if ( sCurMatch == CONTEXT_MATCH ) - { - // we have already a match of this type so check if context ranking - if ( usCurContextRanking > usContextRanking ) - { - pTMXTargetClb = pClb; // use newer target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - // use time info to ensure that latest match is used - else if ( (usCurContextRanking == usContextRanking) && (pClb->lTime > pTMXTargetClb->lTime) ) - { - pTMXTargetClb = pClb; // use newer target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } /* endif */ - } /* endif */ - } /* endif */ - } - else if ( fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)) ) + } /* endif */ + } /* endif */ + } + else + { + if ( sCurMatch < CONTEXT_MATCH ) { - // same segment from same document available - sCurMatch = SAME_SEG_AND_DOC_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match - usContextRanking = usCurContextRanking; + sCurMatch = CONTEXT_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; } - else if ( fMatchingDocName ) - { - // segment from same document available - if ( sCurMatch < SAME_DOC_MATCH ) - { - sCurMatch = SAME_DOC_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - else if ( sCurMatch == SAME_DOC_MATCH ) - { - // we have already a match of this type so - // use time info to ensure that latest match is used - if ( pClb->lTime > pTMXTargetClb->lTime ) - { - pTMXTargetClb = pClb; // use newer target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } /* endif */ - } /* endif */ - } - else if ( pClb->bMultiple ) + else if ( sCurMatch == CONTEXT_MATCH ) { - // multiple target segment available - if ( sCurMatch < MULT_DOC_MATCH ) + // we have already a match of this type so check if context ranking + if ( usCurContextRanking > usContextRanking ) { - // no better match yet - sCurMatch = MULT_DOC_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match + pTMXTargetClb = pClb; // use newer target CLB for match usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; - } /* endif */ - } - else if ( usTranslationFlag == TRANSLFLAG_NORMAL ) - { - // a 'normal' memory match is available - if ( sCurMatch < NORMAL_MATCH ) + } + // use time info to ensure that latest match is used + else if ( (usCurContextRanking == usContextRanking) && (pClb->lTime > pTMXTargetClb->lTime) ) { - // no better match yet - sCurMatch = NORMAL_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match + pTMXTargetClb = pClb; // use newer target CLB for match usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; } /* endif */ } /* endif */ - - // continue with next target CLB - if ( sCurMatch < SAME_SEG_AND_DOC_MATCH ) + } /* endif */ + } + else if ( fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)) ) + { + // same segment from same document available + sCurMatch = SAME_SEG_AND_DOC_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usContextRanking = usCurContextRanking; + usTargetTranslationFlag = usTranslationFlag; + } + else if ( fMatchingDocName ) + { + // segment from same document available + if ( sCurMatch < SAME_DOC_MATCH ) + { + sCurMatch = SAME_DOC_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; + } + else if ( sCurMatch == SAME_DOC_MATCH ) + { + // we have already a match of this type so + // use time info to ensure that latest match is used + if ( pClb->lTime > pTMXTargetClb->lTime ) { - lLeftClbLen -= TARGETCLBLEN(pClb); - if (lLeftClbLen > 0) - { - usTgtNum++; - pClb = NEXTTARGETCLB(pClb); - } + pTMXTargetClb = pClb; // use newer target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; } /* endif */ - } /* endwhile */ + } /* endif */ + } + else if ( pClb->bMultiple ) + { + // multiple target segment available + if ( sCurMatch < MULT_DOC_MATCH ) + { + // no better match yet + sCurMatch = MULT_DOC_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; + } /* endif */ + } + else if ( usTranslationFlag == TRANSLFLAG_NORMAL ) + { + // a 'normal' memory match is available + if ( sCurMatch < NORMAL_MATCH ) + { + // no better match yet + sCurMatch = NORMAL_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; + } /* endif */ + } /* endif */ + // continue with next target CLB + if ( sCurMatch < SAME_SEG_AND_DOC_MATCH ) + { + lLeftClbLen -= TARGETCLBLEN(pClb); + if (lLeftClbLen > 0) + { + usTgtNum++; + pClb = NEXTTARGETCLB(pClb); + } + } /* endif */ + } /* endwhile */ + BOOL fNormalMatch = (usTargetTranslationFlag == TRANSLFLAG_NORMAL) || + (usTargetTranslationFlag == TRANSLFLAG_GLOBMEM) || + (usTargetTranslationFlag == TRANSLFLAG_GLOBMEMSTAR); + switch ( sCurMatch ) + { + case IGNORE_MATCH : + usMatchLevel = 0; + break; + case SAME_SEG_AND_DOC_MATCH : + usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; + break; + case SEG_DOC_AND_CONTEXT_MATCH : + usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; // exact-exact match with matching context + break; + case DOC_AND_CONTEXT_MATCH : + if ( usContextRanking == 100 ) { - BOOL fNormalMatch = (usTargetTranslationFlag == TRANSLFLAG_NORMAL) || - (usTargetTranslationFlag == TRANSLFLAG_GLOBMEM) || - (usTargetTranslationFlag == TRANSLFLAG_GLOBMEMSTAR); - switch ( sCurMatch ) - { - case IGNORE_MATCH : - usMatchLevel = 0; - break; - case SAME_SEG_AND_DOC_MATCH : - usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; - break; - case SEG_DOC_AND_CONTEXT_MATCH : - usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; // exact-exact match with matching context - break; - case DOC_AND_CONTEXT_MATCH : - if ( usContextRanking == 100 ) - { - // GQ 2015/05/09: treat 100% context matches as normal exact matches - // usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; - usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; - } - else - { - usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; - } /* endif */ - break; - case CONTEXT_MATCH : - if ( usContextRanking == 100 ) - { - // GQ 2015/05/09: treat 100% context matches as normal exact context matches - // usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; - // GQ 2016/10/24: treat 100% context matches as normal exact matches - usMatchLevel = fNormalMatch ? usEqual : usEqual-1; - } - else - { - usMatchLevel = fNormalMatch ? usEqual : usEqual-1; - } /* endif */ - break; - case SAME_DOC_MATCH : - usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; - break; - case MULT_DOC_MATCH : - usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; - break; - default : - usMatchLevel = fNormalMatch ? usEqual : usEqual-1; - break; - } /* endswitch */ + // GQ 2015/05/09: treat 100% context matches as normal exact matches + // usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; + usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; } - } - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + else + { + usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; + } /* endif */ + break; + case CONTEXT_MATCH : + if ( usContextRanking == 100 ) + { + // GQ 2015/05/09: treat 100% context matches as normal exact context matches + // usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; + // GQ 2016/10/24: treat 100% context matches as normal exact matches + usMatchLevel = fNormalMatch ? usEqual : usEqual-1; + } + else + { + usMatchLevel = fNormalMatch ? usEqual : usEqual-1; + } /* endif */ + break; + case SAME_DOC_MATCH : + usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; + break; + case MULT_DOC_MATCH : + usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; + break; + default : + usMatchLevel = fNormalMatch ? usEqual : usEqual-1; + break; + } /* endswitch */ + + + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - /**********************************************************/ - /* store only exact matches... */ - /**********************************************************/ - if ( usMatchLevel >= 100 ) - { - LONG lSourceLen = ulLen; - - // use CLB translation flag is ustargetTranslationFlag is not set - if ( usTargetTranslationFlag == (USHORT)-1 ) usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; - FillMatchTable( pTmClb, pString, &lSourceLen, pTMXTargetRecord, - pTMXTargetClb, - pstMatchTable, &usMatchLevel, TRUE, - pusMatchEntries, &pSentence->usActVote, - &pSentence->usActVote, - lKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId ); - } - else - { - /********************************************************/ - /* don't treat MT matches as exact ones.. */ - /********************************************************/ - } /* endif */ + /**********************************************************/ + /* store only exact matches... */ + /**********************************************************/ + if ( usMatchLevel >= 100 ) + { + LONG lSourceLen = ulLen; + + // use CLB translation flag is ustargetTranslationFlag is not set + if ( usTargetTranslationFlag == (USHORT)-1 ) usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; + FillMatchTable( pTmClb, pString, &lSourceLen, pTMXTargetRecord, + pTMXTargetClb, + pstMatchTable, &usMatchLevel, TRUE, + pusMatchEntries, &pSentence->usActVote, + &pSentence->usActVote, + lKeyNum, usTgtNum, + pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId ); + } + else + { + /********************************************************/ + /* don't treat MT matches as exact ones.. */ + /********************************************************/ } /* endif */ } /* endif */ + } /* endif */ - //position at next target - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target - pByte += RECLEN(pTMXTargetRecord); - //remember the end/beginning of record - pStartTarget = pByte; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - } /* endwhile */ + //position at next target + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //move pointer to end of target + pByte += RECLEN(pTMXTargetRecord); + //remember the end/beginning of record + pStartTarget = pByte; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + } /* endwhile */ - /****************************************************************/ - /* check if we found an exact match -- if not we only use best */ - /* fuzzy */ - /****************************************************************/ - if ( !fEqualFound ) + /****************************************************************/ + /* check if we found an exact match -- if not we only use best */ + /* fuzzy */ + /****************************************************************/ + if ( !fEqualFound ) + { + /**************************************************************/ + /* use only the best one... */ + /**************************************************************/ + if (*pusMatchEntries > usOldMatches ) { - /**************************************************************/ - /* use only the best one... */ - /**************************************************************/ - if (*pusMatchEntries > usOldMatches ) - { - *pusMatchEntries = usOldMatches + 1; - } /* endif */ + *pusMatchEntries = usOldMatches + 1; } /* endif */ } /* endif */ } /* endif */ @@ -1289,7 +1253,6 @@ void NTMRemoveAdditionalWhitepace #endif - BOOL TMFuzzynessEx ( PSZ pszMarkup, // markup table name @@ -1297,7 +1260,6 @@ BOOL TMFuzzynessEx PSZ_W pszMatch, // found match SHORT sLanguageId, // language id to be used PUSHORT pusFuzzy, // fuzzyness - ULONG ulOemCP, PUSHORT pusWords, // number of words in segment PUSHORT pusDiffs // number of diffs in segment ) @@ -1307,7 +1269,6 @@ BOOL TMFuzzynessEx std::vector pTokBuf; PLOADEDTABLE pTable = NULL; // ptr to loaded tag table - // fast exit if one or both strings are empty... if ( (*pszSource == EOS) || (*pszMatch == EOS) ) { @@ -1326,9 +1287,6 @@ BOOL TMFuzzynessEx return( TRUE ); } /* endif */ - // allocate required buffers - pInBuf.resize(64000/sizeof(pInBuf[0])); - // load tag table if ( fOK ) { @@ -1337,12 +1295,15 @@ BOOL TMFuzzynessEx FALSE, NULLHANDLE ) == NO_ERROR); } /* endif */ + // allocate required buffers + pInBuf.resize(64000/sizeof(pInBuf[0])); + // call function to evaluate the differences if ( fOK ) { fOK = EQFBFindDiffEx( pTable, pInBuf, pTokBuf, pszSource, pszMatch, sLanguageId, pFuzzyTok, - pFuzzyTgt, ulOemCP ); + pFuzzyTgt ); } /* endif */ if ( fOK ) @@ -1429,14 +1390,11 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct TMX_SUBSTPROP SubstProp; // tag substitution array BOOL fSubstAll = FALSE; PTMX_MATCH_TABLE_W pstMatchTableStart = pstMatchTable; //point to start of match table - ULONG ulSrcOemCP = 0L; - ULONG ulTgtOemCP = 0L; TMManager *pFactory = TMManager::GetInstance(); SHORT sLangID = 0; /****************************************************************/ /* add current tagging to source and translation of proposal */ /****************************************************************/ - ulSrcOemCP = 1; strcpy( SubstProp.szSourceTagTable, pGetIn->szTagTable ); strcpy( SubstProp.szSourceLanguage, pGetIn->szSourceLanguage); strcpy( SubstProp.szTargetLanguage, pGetIn->szTargetLanguage ); @@ -1468,9 +1426,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct { //fill in the markup table strcpy(SubstProp.szPropTagTable, "OTMXUXLF"); - - ulTgtOemCP = 1; - //fill in the target language pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, (USHORT)LANG_KEY, @@ -1652,7 +1607,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /*Q!*/ fFuzzynessOK = TMFuzzynessEx( pGetIn->szTagTable, SubstProp.szPropSource, SubstProp.szSource, - sLangID, &usFuzzy, ulSrcOemCP, &usWords, &usDiffs ); + sLangID, &usFuzzy, &usWords, &usDiffs ); fEqual = (usFuzzy == 100); @@ -1982,296 +1937,256 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block BOOL fNormStringEqual; // normalized strings are equal ??? BOOL fRespectCRLFStringEqual = 0L; + T5LOG( T5INFO) << "FuzzyTest for record " <szSourceLanguage, &sLangID ); - if (!pGetIn->ulSrcOemCP) - { - pGetIn->ulSrcOemCP = 1; + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); + if(pGetIn->fSourceLangIsPrefered == false){ + char recordSrcLang[MAX_LANG_LENGTH]; + pTmClb->NTMGetNameFromID( &pTMXSourceRecord->usLangId, + (USHORT)LANG_KEY, + recordSrcLang, NULL ); + if(strcasecmp(recordSrcLang, pGetIn->szSourceLanguage)){ + T5LOG( T5WARNING) <<":: source langs is different in record(" << recordSrcLang << ") and request (" << pGetIn->szSourceLanguage << ")"; + return SOURCE_LANG_DIFFERENT; + } } + + //move pointer to corresponding position + pSource = (PBYTE)(pTmRecord+1); + pSource += pTMXSourceRecord->usSource; - T5LOG( T5INFO) << "FuzzyTest for record " <fSourceLangIsPrefered == false){ - char recordSrcLang[MAX_LANG_LENGTH]; - pTmClb->NTMGetNameFromID( &pTMXSourceRecord->usLangId, - (USHORT)LANG_KEY, - recordSrcLang, NULL ); - if(strcasecmp(recordSrcLang, pGetIn->szSourceLanguage)){ - T5LOG( T5WARNING) <<":: source langs is different in record(" << recordSrcLang << ") and request (" << pGetIn->szSourceLanguage << ")"; - return SOURCE_LANG_DIFFERENT; - } - } - - //move pointer to corresponding position - pSource = (PBYTE)(pTmRecord+1); - pSource += pTMXSourceRecord->usSource; - - //calculate length of source string - ulSourceLen = (RECLEN(pTMXSourceRecord) - - sizeof(TMX_SOURCE_RECORD)); - - //copy source string for fill matchtable - //T5LOG(T5TRANSACTION) << "pTmRecord: lRecordLen="<< pTmRecord->lRecordLen <<"; usFirstTargetRecord=" << pTmRecord->usFirstTargetRecord <<"; usSourceRecord=" << pTmRecord->usSourceRecord; - ulSourceLen = EQFCompress2Unicode( pString, pSource, ulSourceLen ); - pSentence->pPropString = std::make_unique(pString.c_str()); - - if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ - auto str = EncodingHelper::convertToUTF8(pString); - T5LOG( T5INFO) << "::FuzzyTest:" << str << "" ; - } - - //compare source strings - fStringEqual = ( UtlCompIgnWhiteSpaceW(pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L - || UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(),0) == 0L ) ; + //copy source string for fill matchtable + ulSourceLen = EQFCompress2Unicode( pString, pSource, ulSourceLen ); + pSentence->pPropString = std::make_unique(pString.c_str()); + if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ + auto str = EncodingHelper::convertToUTF8(pString); + T5LOG( T5INFO) << "::FuzzyTest:" << str << "" ; + } - T5LOG( T5INFO) << "FuzzyTest: After String compare, fStringEqual = " <pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L + || UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(),0) == 0L ) ; + T5LOG( T5INFO) << "FuzzyTest: After String compare, fStringEqual = " < usNumMatches ); + T5LOG(T5INFO) << "FuzzyTest: After ExactTest, fStringEqual = " < usNumMatches ); - T5LOG(T5INFO) << "FuzzyTest: After ExactTest, fStringEqual = " <NTMGetIDFromName( pGetIn->szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTargetId)) { - USHORT usTgtNum = 0; // first target number - LONG lLeftTgtLen; // target length record... - USHORT usTagId; // tag table id - USHORT usTargetId; // target language id - USHORT usTagThreshold; // set dependent of fTagTableEqual - LONG lTempSrcLen; // temp. source length - usTagId = 1; // set default... - - // we have to update the memory language table to keep the language group table up-to-date... - if ( pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTargetId)) - { - usTargetId = 1; // set default.. - } /* endif */ - - - //strings are not equal, assume fuzzy - //position at the first target record - pByte = (PBYTE)pTmRecord; - pByte += pTmRecord->usFirstTargetRecord; - pTMXTargetRecord = (PTMX_TARGET_RECORD)pByte; - pStartTarget = (PBYTE)pTMXTargetRecord; - - //loop through target records - lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - - T5LOG( T5INFO) << "FuzzyTest: Checking targets, lLeftTgtLen = "< 0) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && (RECLEN(pTMXTargetRecord) != 0) ) - { - BOOL fTestCLB = TRUE; - USHORT usModifiedTranslationFlag = 0; - - /**************************************************************/ - /* update left target length */ - /**************************************************************/ - usTgtNum ++; // we are dealing with next target - - //assert( ulLeftTgtLen >= RECLEN(pTMXTargetRecord) ); + usTargetId = 1; // set default.. + } /* endif */ - lLeftTgtLen -= RECLEN(pTMXTargetRecord); + //strings are not equal, assume fuzzy + //position at the first target record + pByte = (PBYTE)pTmRecord; + pByte += pTmRecord->usFirstTargetRecord; + pTMXTargetRecord = (PTMX_TARGET_RECORD)pByte; + pStartTarget = (PBYTE)pTMXTargetRecord; - //check the target language - //position at target control block - pByte += pTMXTargetRecord->usClb; - pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + //loop through target records + lLeftTgtLen = RECLEN(pTmRecord) - pTmRecord->usFirstTargetRecord; - //compare target language group IDs - if(pGetIn->fTargetLangIsPrefered){ - if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] ) - { - T5LOG( T5INFO) <<"FuzzyTest: Wrong target language!" ; - fTestCLB = FALSE; - } - } else { - char recordTrgLang[MAX_LANG_LENGTH]; - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, - (USHORT)LANG_KEY, - recordTrgLang, NULL ); - if(strcasecmp(recordTrgLang, pGetIn->szTargetLanguage)){ - T5LOG( T5WARNING) <<":: target langs is different in record(" << recordTrgLang << ") and request (" << pGetIn->szTargetLanguage<< ")"; - //return SOURCE_LANG_DIFFERENT; - fTestCLB = FALSE; - } + T5LOG( T5INFO) << "FuzzyTest: Checking targets, lLeftTgtLen = "< 0) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && (RECLEN(pTMXTargetRecord) != 0) ) + { + BOOL fTestCLB = TRUE; + USHORT usModifiedTranslationFlag = 0; + + /**************************************************************/ + /* update left target length */ + /**************************************************************/ + usTgtNum ++; // we are dealing with next target + lLeftTgtLen -= RECLEN(pTMXTargetRecord); + + //check the target language + //position at target control block + pByte += pTMXTargetRecord->usClb; + pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + + //compare target language group IDs + if(pGetIn->fTargetLangIsPrefered){ + if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] ) + { + T5LOG( T5INFO) <<"FuzzyTest: Wrong target language!" ; + fTestCLB = FALSE; } + } else { + char recordTrgLang[MAX_LANG_LENGTH]; + pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, + (USHORT)LANG_KEY, + recordTrgLang, NULL ); + if(strcasecmp(recordTrgLang, pGetIn->szTargetLanguage)){ + T5LOG( T5WARNING) <<":: target langs is different in record(" << recordTrgLang << ") and request (" << pGetIn->szTargetLanguage<< ")"; + //return SOURCE_LANG_DIFFERENT; + fTestCLB = FALSE; + } + } - // preprocessing for global memory processing - usModifiedTranslationFlag = pTMXTargetClb->bTranslationFlag; - if ( fTestCLB && (pTMXTargetClb->bTranslationFlag == TRANSLFLAG_GLOBMEM) && (pGetIn->pvGMOptList != NULL) && pTMXTargetClb->usAddDataLen ) + // preprocessing for global memory processing + usModifiedTranslationFlag = pTMXTargetClb->bTranslationFlag; + if ( fTestCLB && (pTMXTargetClb->bTranslationFlag == TRANSLFLAG_GLOBMEM) && (pGetIn->pvGMOptList != NULL) && pTMXTargetClb->usAddDataLen ) + { + USHORT usAddDataLen = 0; + std::wstring addDataOut; + usAddDataLen = NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, addDataOut); + wcsncpy(pSentence->szAddInfo.data(), addDataOut.c_str(), MAX_SEGMENT_SIZE); + if ( usAddDataLen ) { - USHORT usAddDataLen = 0; - std::wstring addDataOut; - usAddDataLen = NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, addDataOut); - wcsncpy(pSentence->szAddInfo.data(), addDataOut.c_str(), MAX_SEGMENT_SIZE); - if ( usAddDataLen ) + GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pSentence->pAddString.data() ); + switch ( GobMemOpt ) { - GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pSentence->pAddString.data() ); - switch ( GobMemOpt ) - { - case GM_SUBSTITUTE_OPT: usModifiedTranslationFlag = TRANSLFLAG_NORMAL; break; - case GM_HFLAG_OPT : usModifiedTranslationFlag = TRANSLFLAG_GLOBMEM; break; - case GM_HFLAGSTAR_OPT : usModifiedTranslationFlag = TRANSLFLAG_GLOBMEMSTAR; break; - case GM_EXCLUDE_OPT : fTestCLB = FALSE;; break; - } /* endswitch */ - } /* endif */ - } /* endif */ - - T5LOG( T5INFO) << "FuzzyTest: fTestCLB="<<( fTestCLB ? "Yes" : "NO") ; - - // compare target table IDs - if ( fTestCLB ) + case GM_SUBSTITUTE_OPT: usModifiedTranslationFlag = TRANSLFLAG_NORMAL; break; + case GM_HFLAG_OPT : usModifiedTranslationFlag = TRANSLFLAG_GLOBMEM; break; + case GM_HFLAGSTAR_OPT : usModifiedTranslationFlag = TRANSLFLAG_GLOBMEMSTAR; break; + case GM_EXCLUDE_OPT : fTestCLB = FALSE;; break; + } /* endswitch */ + } /* endif */ + } /* endif */ + + T5LOG( T5INFO) << "FuzzyTest: fTestCLB="<<( fTestCLB ? "Yes" : "NO") ; + + // compare target table IDs + if ( fTestCLB ) + { + //compare target tag table ids + //position at target tag table record + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + +// as the strings doe not contain the inline tagging, we have + // to add the inline tags first + // if the tag tables are different this is omitted as the + // inline tags are suppressed in the proposal in such a case { - //compare target tag table ids - //position at target tag table record + LONG lLenTmp = ulSourceLen; pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + } /* endif */ - /**********************************************************/ - /* TAGS_EQUAL == base 98, TAGS_UNEQUAL == base 95 */ - /**********************************************************/ - usTagThreshold = TAGS_EQUAL; + //TODO: remove punctuation here + BOOL fFuzzynessOK = TMFuzzynessEx( pGetIn->szTagTable, + pSentence->pStrings->getNormStrC(), + pSentence->pPropString->getNormStrC(), + sLangID, &usFuzzy, + &usWords, &usDiffs); + /**********************************************************/ + /* additional comparison if normalized strings are equal */ + /**********************************************************/ +// if ( usFuzzy >= 100 ) + if ( fNormStringEqual ) + { + //compare source tag table records + //position at source tag table record pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - { - BOOL fFuzzynessOK; -// as the strings doe not contain the inline tagging, we have - // to add the inline tags first - // if the tag tables are different this is omitted as the - // inline tags are suppressed in the proposal in such a case + fStringEqual = (UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), + pSentence->pStrings->getGenericTagStrC(), + 0 ) == 0 ); + if (!fStringEqual && (usFuzzy >= 100) ) + { + // compare with respect to protected/unprotected parts + // We can assume all Text tokens to be equal ( usFuzzy = 100), + // but we do not know about tag tokens + BOOL fTempStringEqual = FALSE; + fOK = NTMCompareBetweenTokens( + pSentence->pPropString->getGenericTagStrC(), + pSentence->pStrings->getGenericTagStrC(), + pGetIn->szTagTable, + sLangID, + &fTempStringEqual); + if (fOK) + { + fStringEqual = fTempStringEqual; + } + } + /********************************************************/ + /* ensure that we are dealing with a fuzzy match */ + /********************************************************/ + if ( !fStringEqual ) + { + if ( usFuzzy > 3 ) { - LONG lLenTmp = ulSourceLen; - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - } /* endif */ - - //TODO: remove punctuation here - fFuzzynessOK = TMFuzzynessEx( pGetIn->szTagTable, - pSentence->pStrings->getNormStrC(), - pSentence->pPropString->getNormStrC(), - sLangID, &usFuzzy, - pGetIn->ulSrcOemCP, &usWords, &usDiffs); - - /**********************************************************/ - /* additional comparison if normalized strings are equal */ - /**********************************************************/ -// if ( usFuzzy >= 100 ) - if ( fNormStringEqual ) + usFuzzy -= 3; + } + else { - //compare source tag table records - //position at source tag table record - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - - { - //calculate length of source string - LONG lLenTmp = ulSourceLen; - { - fStringEqual = (UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), - pSentence->pStrings->getGenericTagStrC(), - 0 ) == 0 ); - if (!fStringEqual && (usFuzzy >= 100) ) - { - // compare with respect to protected/unprotected parts - // We can assume all Text tokens to be equal ( usFuzzy = 100), - // but we do not know about tag tokens - BOOL fTempStringEqual = FALSE; - fOK = NTMCompareBetweenTokens( - pSentence->pPropString->getGenericTagStrC(), - pSentence->pStrings->getGenericTagStrC(), - pGetIn->szTagTable, - sLangID, - pGetIn->ulSrcOemCP, &fTempStringEqual); - if (fOK) - { - fStringEqual = fTempStringEqual; - } - } - } /* endif */ - } /* endif */ - //} /* endif */ - /********************************************************/ - /* ensure that we are dealing with a fuzzy match */ - /********************************************************/ - if ( !fStringEqual ) - { - if ( usFuzzy > 3 ) - { - usFuzzy -= 3; - } - else - { - usFuzzy = 0; - } /* endif */ - - usFuzzy = std::min( (USHORT)99, usFuzzy ); - } /* endif */ - + usFuzzy = 0; } /* endif */ - } - //fill get output structure - T5LOG( T5INFO) << "FuzzyTest: usFuzzy="< TM_FUZZINESS_THRESHOLD ) + usFuzzy = std::min( (USHORT)99, usFuzzy ); + } /* endif */ + } /* endif */ + + //fill get output structure + T5LOG( T5INFO) << "FuzzyTest: usFuzzy="< TM_FUZZINESS_THRESHOLD ) + { + /********************************************************/ + /* give MT flag a little less fuzziness */ + /********************************************************/ + if ( usModifiedTranslationFlag == TRANSLFLAG_MACHINE ) { - /********************************************************/ - /* give MT flag a little less fuzziness */ - /********************************************************/ - if ( usModifiedTranslationFlag == TRANSLFLAG_MACHINE ) + if ( usFuzzy > 1 ) { - if ( usFuzzy > 1 ) - { - usFuzzy -= 1; - } - else - { - usFuzzy = 0; - } /* endif */ + usFuzzy -= 1; + } + else + { + usFuzzy = 0; } /* endif */ - if (usFuzzy == 100 && (pGetIn->ulParm & GET_RESPECTCRLF) && !fRespectCRLFStringEqual ) - { // P018279! - usFuzzy -= 1; - } - lTempSrcLen = ulSourceLen; - FillMatchTable( pTmClb, pString, &lTempSrcLen, pTMXTargetRecord, - pTMXTargetClb, - pstMatchTable, &usFuzzy, true, - pusMatchesFound, pusTmMaxVotes, pusOverlaps, - lKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId ); } /* endif */ + + lTempSrcLen = ulSourceLen; + FillMatchTable( pTmClb, pString, &lTempSrcLen, pTMXTargetRecord, + pTMXTargetClb, + pstMatchTable, &usFuzzy, true, + pusMatchesFound, pusTmMaxVotes, pusOverlaps, + lKeyNum, usTgtNum, + pGetIn, pSentence->pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId ); } /* endif */ - //position at next target - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //move pointer to end of target - pByte += RECLEN(pTMXTargetRecord); - //remember the beginning of next record - pStartTarget = pByte; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - }/* endwhile */ - } /* endif */ + } /* endif */ + //position at next target + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + //move pointer to end of target + pByte += RECLEN(pTMXTargetRecord); + //remember the beginning of next record + pStartTarget = pByte; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + }/* endwhile */ } /* endif */ return( usRc ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 2d7a572e..b7435af0 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -1570,259 +1570,252 @@ USHORT EqfMemory::ComparePutData LONG lLeftClbLen; int delTargetKey = 0, targetKey = 0; + BOOL fStringEqual; // indicate string equal + USHORT usPutLang; // language id of target language + USHORT usPutFile; // file id of new entry + + //get id of target language in the put structure + if (NTMGetIDFromName( TmProposal.szTargetLanguage, + NULL, (USHORT)LANG_KEY, &usPutLang )) { - BOOL fStringEqual; // indicate string equal - USHORT usPutLang; // language id of target language - USHORT usPutFile; // file id of new entry + usPutLang = 1; + } /* endif */ + + NTMGetIDFromName( TmProposal.szTargetAuthor, // get author ID + NULL, (USHORT)AUTHOR_KEY, &usAuthorId ); - //get id of target language in the put structure - if (NTMGetIDFromName( TmProposal.szTargetLanguage, - NULL, (USHORT)LANG_KEY, &usPutLang )) + usRc = NTMGetIDFromName( TmProposal.szDocName, // get file id + TmProposal.szDocName, (USHORT)FILE_KEY, &usPutFile ); + if ( !usRc ) + { + //position at beginning of source structure in tm record + pTMXSourceRecord = (PTMX_SOURCE_RECORD)(toTmxRecord(pTmRecord)+1); + pByte = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_SRCSTRING); + + //calculate length of source string + lLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); + //copy and compare source string + //memset( pString.data(), 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); + lLen = EQFCompress2Unicode( pString, pByte, lLen ); + fStringEqual = ! wcscmp( pString.c_str(), TmProposal.pInputSentence->pStrings->getGenericTagStrC() ); + + if ( fStringEqual ) { - usPutLang = 1; - } /* endif */ + BOOL fNewerTargetExists = FALSE; + LONG lLeftTgtLen; // remaining target length - NTMGetIDFromName( TmProposal.szTargetAuthor, // get author ID - NULL, (USHORT)AUTHOR_KEY, &usAuthorId ); + NTASSERTLEN(RECLENB(pTmRecord), toTmxRecord(pTmRecord)->usFirstTargetRecord, 4712) - usRc = NTMGetIDFromName( TmProposal.szDocName, // get file id - TmProposal.szDocName, (USHORT)FILE_KEY, &usPutFile ); - if ( !usRc ) - { - //position at beginning of source structure in tm record - pTMXSourceRecord = (PTMX_SOURCE_RECORD)(toTmxRecord(pTmRecord)+1); - pByte = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_SRCSTRING); - - //calculate length of source string - lLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); - //copy and compare source string - //memset( pString.data(), 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); - lLen = EQFCompress2Unicode( pString, pByte, lLen ); - fStringEqual = ! wcscmp( pString.c_str(), TmProposal.pInputSentence->pStrings->getGenericTagStrC() ); - - if ( fStringEqual ) - { - BOOL fNewerTargetExists = FALSE; - LONG lLeftTgtLen; // remaining target length + // get length of target block to work with + //source strings equal - position at first target record + pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); - NTASSERTLEN(RECLENB(pTmRecord), toTmxRecord(pTmRecord)->usFirstTargetRecord, 4712) + fStop = (usRc != 0); + //RJ: 04/01/22: P018830: + // loop through all target records + // delete entry if current segment has already been translated + TMLoopAndDelTargetClb(pTmRecord, TmProposal, usPutLang, usPutFile, usAuthorId, &fNewerTargetExists, &delTargetKey ); - // get length of target block to work with - //source strings equal - position at first target record - pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); + // recalc since record may have changed during delete above! + lLeftTgtLen = RECLENB(pTmRecord) - toTmxRecord(pTmRecord)->usFirstTargetRecord; + pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); + pTMXTargetRecord = (PTMX_TARGET_RECORD)pStartTarget; - fStop = (usRc != 0); - //RJ: 04/01/22: P018830: - // loop through all target records - // delete entry if current segment has already been translated - TMLoopAndDelTargetClb(pTmRecord, TmProposal, usPutLang, usPutFile, usAuthorId, &fNewerTargetExists, &delTargetKey ); + if ( fNewerTargetExists && !FLAGS_ignore_newer_target_exists_check) + { + fStop = TRUE; // do not continue update loop + } + else + { + //source strings are identical so loop through target records + // and add new entry + while ( fOK && (lLeftTgtLen > 0) && (lLeftTgtLen <= RECLEN(pTMXTargetRecord)) + && (RECLEN(pTMXTargetRecord) > 0) && !fStop ) + { + // check for valid target length + NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTargetRecord), 4713); + + // update target length + lLeftTgtLen -= RECLEN(pTMXTargetRecord); - // recalc since record may have changed during delete above! - lLeftTgtLen = RECLENB(pTmRecord) - toTmxRecord(pTmRecord)->usFirstTargetRecord; - pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); - pTMXTargetRecord = (PTMX_TARGET_RECORD)pStartTarget; + // next check the target language and target tag info + // position at first target control block and to source tag info + pClb = (PTMX_TARGET_CLB)NTRecPos((PBYTE)pTMXTargetRecord, REC_CLB); - if ( fNewerTargetExists && !FLAGS_ignore_newer_target_exists_check) - { - fStop = TRUE; // do not continue update loop - } - else - { - //source strings are identical so loop through target records - // and add new entry - while ( fOK && (lLeftTgtLen > 0) && (lLeftTgtLen <= RECLEN(pTMXTargetRecord)) - && (RECLEN(pTMXTargetRecord) > 0) && !fStop ) - { - // check for valid target length - NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTargetRecord), 4713); - - // update target length - lLeftTgtLen -= RECLEN(pTMXTargetRecord); - - // next check the target language and target tag info - // position at first target control block and to source tag info - pClb = (PTMX_TARGET_CLB)NTRecPos((PBYTE)pTMXTargetRecord, REC_CLB); - - // compare target language IDs and source tag record - if (pClb->usLangId == usPutLang) - { - // check if target string and target tag record are identical - pByte = NTRecPos(pStartTarget, REC_TGTSTRING); - lLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - lLen = EQFCompress2Unicode( pString, pByte, lLen ); - pString.data()[lLen] = EOS; - - //compare target strings and target tag record - if ( !wcscmp( pString.data(), TmProposal.pInputSentence->pStrings->getGenericTargetStrC())) - { //target strings and target tag record are equal - //position at first control block - pClb = (PTMX_TARGET_CLB)NTRecPos(pStartTarget, REC_CLB); - - // loop over all target CLBs - pTMXTargetRecord = (PTMX_TARGET_RECORD)pStartTarget; - lLeftClbLen = RECLEN(pTMXTargetRecord) - pTMXTargetRecord->usClb; - while ( lLeftClbLen > 0 && !fStop ) + // compare target language IDs and source tag record + if (pClb->usLangId == usPutLang) + { + // check if target string and target tag record are identical + pByte = NTRecPos(pStartTarget, REC_TGTSTRING); + lLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; + lLen = EQFCompress2Unicode( pString, pByte, lLen ); + pString.data()[lLen] = EOS; + + //compare target strings and target tag record + if ( !wcscmp( pString.data(), TmProposal.pInputSentence->pStrings->getGenericTargetStrC())) + { //target strings and target tag record are equal + //position at first control block + pClb = (PTMX_TARGET_CLB)NTRecPos(pStartTarget, REC_CLB); + + // loop over all target CLBs + pTMXTargetRecord = (PTMX_TARGET_RECORD)pStartTarget; + lLeftClbLen = RECLEN(pTMXTargetRecord) - pTMXTargetRecord->usClb; + while ( lLeftClbLen > 0 && !fStop ) + { + targetKey++; + if ( + (pClb->usFileId == usPutFile) && + (pClb->usAuthorId == usAuthorId) && + isAddDataIsTheSame(TmProposal, *pClb) + ) { - targetKey++; - if ( - (pClb->usFileId == usPutFile) && - (pClb->usAuthorId == usAuthorId) && - isAddDataIsTheSame(TmProposal, *pClb) - ) - { - // an identical segment already in record (update - // time if newer than existing one - pClb->lTime = TmProposal.lTargetTime; - pClb->ulSegmId = TmProposal.getSegmentId(); - pClb->usFileId = usPutFile; - pClb->bTranslationFlag = ProposalTypeToFlag(TmProposal.eType); - pClb->usAuthorId = usAuthorId; - fUpdate = TRUE; - fStop = TRUE; - - // adjust context part of target control block if necessary - NTMAdjustAddDataInTgtCLB( pTmRecord, TmProposal, &pClb, &pTMXTargetRecord, &lLeftClbLen, &fUpdate ); - } /* endif */ - - // continue with next target CLB - if ( !fStop ) - { - lLeftClbLen -= TARGETCLBLEN(pClb); - pClb = NEXTTARGETCLB(pClb); - } /* endif */ - } /* endwhile */ + // an identical segment already in record (update + // time if newer than existing one + pClb->lTime = TmProposal.lTargetTime; + pClb->ulSegmId = TmProposal.getSegmentId(); + pClb->usFileId = usPutFile; + pClb->bTranslationFlag = ProposalTypeToFlag(TmProposal.eType); + pClb->usAuthorId = usAuthorId; + fUpdate = TRUE; + fStop = TRUE; + + // adjust context part of target control block if necessary + NTMAdjustAddDataInTgtCLB( pTmRecord, TmProposal, &pClb, &pTMXTargetRecord, &lLeftClbLen, &fUpdate ); + } /* endif */ - // add new CLB if no matching CLB has been found + // continue with next target CLB if ( !fStop ) { - targetKey++; - // re-alloc record buffer if too small - constexpr int additionalBytes = 13; - USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - - // make room at pCLB for a new CLB and adjust TM record - // length and target record length - if ( fOK ) - { - LONG lNewClbLen = sizeof(TMX_TARGET_CLB) + usAddDataLen; - size_t occupiedSize = RECLENB((pTmRecord)) + sizeof( TMX_SOURCE_RECORD ) + sizeof(TMX_TARGET_RECORD) + additionalBytes; - if(lNewClbLen + occupiedSize >= pTmRecord.size()) - { - //remember offsets - size_t pStartTargetOffset = (PBYTE)pStartTarget - pTmRecord.data(); - size_t pClbOffset = (PBYTE)pClb - pTmRecord.data(); - size_t pTMXTargetRecordOffset = (PBYTE)pTMXTargetRecord - pTmRecord.data(); - size_t oldSize = pTmRecord.size(); - - pTmRecord.resize(lNewClbLen+occupiedSize); - if(FLAGS_log_memmove_in_compareputdata) - { - T5LOG(T5TRANSACTION) << "pStartTargetOffset=" <= TMX_REC_SIZE){ - usRc = BTREE_NODE_IS_FULL; - fStop = true; - fOK = false; - }else{ - if(FLAGS_log_memmove_in_compareputdata) - { - T5LOG(T5TRANSACTION) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " - << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize - << "; ulKey = " << lKey << "; TmRecord.size() = " << pTmRecord.size();// <<"; size before = " << size_before; - } - - memmove( (((PBYTE)pClb) + lNewClbLen), pClb, size); - RECLENB(pTmRecord) += lNewClbLen; - RECLEN(pTMXTargetRecord) += lNewClbLen; - } - } /* endif */ + lLeftClbLen -= TARGETCLBLEN(pClb); + pClb = NEXTTARGETCLB(pClb); + } /* endif */ + } /* endwhile */ - // fill-in new target CLB - if ( fOK ) + // add new CLB if no matching CLB has been found + if ( !fStop ) + { + targetKey++; + // re-alloc record buffer if too small + constexpr int additionalBytes = 13; + USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); + + // make room at pCLB for a new CLB and adjust TM record + // length and target record length + if ( fOK ) + { + LONG lNewClbLen = sizeof(TMX_TARGET_CLB) + usAddDataLen; + size_t occupiedSize = RECLENB((pTmRecord)) + sizeof( TMX_SOURCE_RECORD ) + sizeof(TMX_TARGET_RECORD) + additionalBytes; + if(lNewClbLen + occupiedSize >= pTmRecord.size()) { - //pClb->bMultiple = FALSE; - if ( TmProposal.lTargetTime ) + //remember offsets + size_t pStartTargetOffset = (PBYTE)pStartTarget - pTmRecord.data(); + size_t pClbOffset = (PBYTE)pClb - pTmRecord.data(); + size_t pTMXTargetRecordOffset = (PBYTE)pTMXTargetRecord - pTmRecord.data(); + size_t oldSize = pTmRecord.size(); + + pTmRecord.resize(lNewClbLen+occupiedSize); + if(FLAGS_log_memmove_in_compareputdata) { - pClb->lTime = TmProposal.lTargetTime; + T5LOG(T5TRANSACTION) << "pStartTargetOffset=" <= TMX_REC_SIZE){ + usRc = BTREE_NODE_IS_FULL; + fStop = true; + fOK = false; + }else{ + if(FLAGS_log_memmove_in_compareputdata) { - UtlTime( &(pClb->lTime) ); - } /* endif */ - pClb->ulSegmId = TmProposal.getSegmentId(); - pClb->usFileId = usPutFile; - pClb->bTranslationFlag = ProposalTypeToFlag(TmProposal.eType); - pClb->usAuthorId = usAuthorId; - pClb->usLangId = usPutLang; - pClb->usAddDataLen = 0; - NtmStoreAddData( pClb, ADDDATA_CONTEXT_ID, TmProposal.szContext ); - NtmStoreAddData( pClb, ADDDATA_ADDINFO_ID, TmProposal.szAddInfo ); - - fStop = TRUE; // avoid add of a new target record at end of outer loop - fUpdate = TRUE; + T5LOG(T5TRANSACTION) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " + << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize + << "; ulKey = " << lKey << "; TmRecord.size() = " << pTmRecord.size();// <<"; size before = " << size_before; + } + + memmove( (((PBYTE)pClb) + lNewClbLen), pClb, size); + RECLENB(pTmRecord) += lNewClbLen; + RECLEN(pTMXTargetRecord) += lNewClbLen; + } + } /* endif */ + + // fill-in new target CLB + if ( fOK ) + { + //pClb->bMultiple = FALSE; + if ( TmProposal.lTargetTime ) + { + pClb->lTime = TmProposal.lTargetTime; + } + else + { + UtlTime( &(pClb->lTime) ); } /* endif */ - + pClb->ulSegmId = TmProposal.getSegmentId(); + pClb->usFileId = usPutFile; + pClb->bTranslationFlag = ProposalTypeToFlag(TmProposal.eType); + pClb->usAuthorId = usAuthorId; + pClb->usLangId = usPutLang; + pClb->usAddDataLen = 0; + NtmStoreAddData( pClb, ADDDATA_CONTEXT_ID, TmProposal.szContext ); + NtmStoreAddData( pClb, ADDDATA_ADDINFO_ID, TmProposal.szAddInfo ); + + fStop = TRUE; // avoid add of a new target record at end of outer loop + fUpdate = TRUE; } /* endif */ + } /* endif */ } /* endif */ + } /* endif */ - //position at next target - if ( !fStop ) - { - pStartTarget = NTRecPos(pStartTarget, REC_NEXTTARGET); - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pStartTarget); - } /* endif */ - } /* endwhile */ - } /* endif */ + //position at next target + if ( !fStop ) + { + pStartTarget = NTRecPos(pStartTarget, REC_NEXTTARGET); + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pStartTarget); + } /* endif */ + } /* endwhile */ + } /* endif */ - // update TM record if required - if ( fUpdate ) - { - usRc = TmBtree.EQFNTMUpdate( lKey, pTmRecord);//.data(), RECLENB(pTmRecord) ); - } /* endif */ + // update TM record if required + if ( fUpdate ) + { + usRc = TmBtree.EQFNTMUpdate( lKey, pTmRecord);//.data(), RECLENB(pTmRecord) ); + } /* endif */ - if ( fStop ) - { - TmProposal.currentInternalKey.setInternalKey(lKey, targetKey); - }else{ - //all target records have been checked but nothing overlapped - //so add new target record to end of tm record - usRc = AddTmTarget( TmProposal, pTmRecord, lKey ); - - if(!usRc) - { - TmProposal.currentInternalKey.setInternalKey(lKey, 1); - } - } /* endif */ - } - else + if ( fStop ) { - //source strings are not equal so try another sid or if all have been - //tries add new tm record - LOG_AND_SET_RC(usRc, T5INFO, SOURCE_STRING_ERROR); + TmProposal.currentInternalKey.setInternalKey(lKey, targetKey); + }else{ + //all target records have been checked but nothing overlapped + //so add new target record to end of tm record + usRc = AddTmTarget( TmProposal, pTmRecord, lKey ); + + if(!usRc) + { + TmProposal.currentInternalKey.setInternalKey(lKey, 1); + } } /* endif */ + } + else + { + //source strings are not equal so try another sid or if all have been + //tries add new tm record + LOG_AND_SET_RC(usRc, T5INFO, SOURCE_STRING_ERROR); } /* endif */ } /* endif */ - if ( usRc ) - { - ERREVENT2( COMPAREPUTDATA_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ - return( usRc ); } diff --git a/source/opentm2/core/morph/OtmMorphICU.cpp b/source/opentm2/core/morph/OtmMorphICU.cpp index d906cc28..3403dfd7 100755 --- a/source/opentm2/core/morph/OtmMorphICU.cpp +++ b/source/opentm2/core/morph/OtmMorphICU.cpp @@ -97,31 +97,26 @@ int OtmMorphICU::tokenizeByTerm(const wchar_t* pText, TERMLIST& vResult) return OtmMorph::ERROR_PARAMETAR; } - { - std::lock_guard guard(this->MorphICUMutex); - - - - std::u16string vSection_u16 = EncodingHelper::toUtf16(pText); - UnicodeString text(vSection_u16.c_str()); + std::lock_guard guard(this->MorphICUMutex); + std::u16string vSection_u16 = EncodingHelper::toUtf16(pText); + UnicodeString text(vSection_u16.c_str()); - pWordBoundary->setText(text); - getResultFromIterator(pWordBoundary, vResult); + pWordBoundary->setText(text); + getResultFromIterator(pWordBoundary, vResult); - /* Force maximum length to 511, since temp buffers are 512 10-6-16 */ - for (auto it = begin (vResult); it != end (vResult); ++it) - { + /* Force maximum length to 511, since temp buffers are 512 10-6-16 */ + for (auto it = begin (vResult); it != end (vResult); ++it) + { if ( it->iLength > 511 ) it->iLength=511; - } + } #ifdef TOKENIZELOG - T5LOG(T5DEBUG) << "results:" ; - for (auto it = begin (vResult); it != end (vResult); ++it) - { - T5LOG(T5DEBUG) << " Offset="<iStartOffset <<" Len="<iLength<<" Flag=" << it->iTermType ; - } -#endif + T5LOG(T5DEBUG) << "results:" ; + for (auto it = begin (vResult); it != end (vResult); ++it) + { + T5LOG(T5DEBUG) << " Offset="<iStartOffset <<" Len="<iLength<<" Flag=" << it->iTermType ; } +#endif return OtmMorph::SUCCESS_RETURN; } diff --git a/source/opentm2/core/tagtable/EQFTOKEN.cpp b/source/opentm2/core/tagtable/EQFTOKEN.cpp index a604614f..71e80f44 100755 --- a/source/opentm2/core/tagtable/EQFTOKEN.cpp +++ b/source/opentm2/core/tagtable/EQFTOKEN.cpp @@ -422,12 +422,10 @@ CHAR fWhiteSpace[256] = BOOL TATagTokenizeW ( PWCHAR pszInput, // pointer to input data - PLOADEDTABLE pVoidTable, // pointer to loaded tag table - BOOL fComplete, // TRUE = no more buffers to follow + PLOADEDTABLE pTable, // pointer to loaded tag table PWCHAR *ppRest, // pointer to not processed data in buffer - PUSHORT pusColPos, // column position - std::vector& pTokBuf//, // buffer for created tokens - //USHORT usTokens // max entries in token buffer + PUSHORT pusColPos, // column position + std::vector& pTokBuf // buffer for created tokens ) { PWCHAR pszData; // pointer for input data processing @@ -435,7 +433,6 @@ BOOL TATagTokenizeW TOKENENTRY Token; // pointer to current token PWCHAR pszEnd; // pointer to end of data BOOL fFound = FALSE; // flag used in setting pRest - PLOADEDTABLE pTable = (PLOADEDTABLE)pVoidTable; // pointer to loaded tag table TOKSTATUS TokStatus; // tokenize status area TOKENENTRY NewToken; // buffer for new tokens MSSTACK MSStack[10]; @@ -443,8 +440,6 @@ BOOL TATagTokenizeW //PTOKENENTRY pLastTag = NULL; // pointer to last tag token int iLastTagPosInBuf = -1; USHORT usTagColPos = 0; // column position of last tag - USHORT usCurColPos; // current column position - BOOL fProcessingAttributes; // TRUE = attributes are processed PATTRIBUTE pAttr; // ptr to attribute array of tag table PBYTE pByte; // helper pointer SHORT sTags; // number of tags in tag table @@ -466,7 +461,6 @@ BOOL TATagTokenizeW pTagAddInfo = NULL; } /* endif */ - /*******************************************************************/ /* Initialize tokenize status area */ /*******************************************************************/ @@ -511,27 +505,20 @@ BOOL TATagTokenizeW Token.sTokenid = TEXT_TOKEN; // assume text data Token.pDataStringW = pszData; // remember start of data //usTokens--; // reserve space for end of list token - fProcessingAttributes = FALSE; // we are waiting for tags ... NewToken.reset(); while( fOK && *pszData)// && usTokens )// while not end of data and free tokens - { - USHORT usProcessFlag; // processing flag /* @KWT0010A */ - + { // the following statement has been disabled to fix PTM KAT0330 -// fProcessingAttributes = FALSE; // we are waiting for tags ... pszDataStart = pszData; - usCurColPos = TokStatus.usColPos; - usProcessFlag = (fComplete ) ? /* @KWT0010A */ - ( MATCH_SKIP_DATA|MATCH_DATA_COMPLETE) : MATCH_SKIP_DATA; + //usProcessFlag = (fComplete ) ? /* @KWT0010A */ + // ( MATCH_SKIP_DATA|MATCH_DATA_COMPLETE) : MATCH_SKIP_DATA; fFound = TAMatchTag( &TokStatus, &pszData, pTable->pNodeArea->pRootNode, &NewToken, - usProcessFlag); /* @KWT0010M */ + MATCH_SKIP_DATA|MATCH_DATA_COMPLETE); /* @KWT0010M */ if ( fFound ) { - fProcessingAttributes = FALSE; - /*************************************************************/ /* Build token for data up to start of recognized tag */ /*************************************************************/ @@ -613,7 +600,6 @@ BOOL TATagTokenizeW /**********************************************************/ if ( (NewToken.sTokenid >= 0) && pTag[NewToken.sTokenid].fAttr ) { - fProcessingAttributes = TRUE; // attributes may follow ... do { /***********************************************************/ @@ -629,7 +615,6 @@ BOOL TATagTokenizeW if ( !fWhiteSpace[NewToken.pDataStringW[NewToken.iLength-1]] ) { fFound = FALSE; - fProcessingAttributes = FALSE; // no more attributes } else { @@ -641,23 +626,12 @@ BOOL TATagTokenizeW /***********************************************************/ if ( fFound ) { - usCurColPos = TokStatus.usColPos; fFound = TAMatchTag( &TokStatus, &pszData, pTable->pAttrNodeArea->pRootNode, &NewToken, MATCH_SKIP_WHITESPACE | MATCH_QUOTED_STRINGS ); - // the following code has been disabled to fix KAT0330 - // if no attribute has been found the pszData pointer still - // points to the begin of the maybe attribute ... - // if ( !fFound && *pszData ) - // { - // /*******************************************************/ - // /* Attribute mismatch, but not end of text block */ - // /*******************************************************/ - // fProcessingAttributes = FALSE; // no more attributes - // } /* endif */ } /* endif */ if ( fFound)// && usTokens ) @@ -684,28 +658,16 @@ BOOL TATagTokenizeW /*************************************************************/ /* No tag in remaining data found */ /*************************************************************/ - if ( fComplete ) - { - /***********************************************************/ - /* Add remaining data as TEXT token */ - /***********************************************************/ - Token.sTokenid = TEXT_TOKEN; // text data - Token.pDataStringW = pszDataStart; // set start of data - Token.iLength = (USHORT)UTF16strlenCHAR(pszDataStart); - pszData = pszDataStart + Token.iLength; + /***********************************************************/ + /* Add remaining data as TEXT token */ + /***********************************************************/ + Token.sTokenid = TEXT_TOKEN; // text data + Token.pDataStringW = pszDataStart; // set start of data + Token.iLength = (USHORT)UTF16strlenCHAR(pszDataStart); + pszData = pszDataStart + Token.iLength; - pTokBuf.push_back(Token); - Token.reset(); - //usTokens--; - //pToken++; - } - else - { - /*************************************************************/ - /* Process remaining data the next time ... */ - /*************************************************************/ - pszData = pszDataStart + UTF16strlenCHAR(pszDataStart); - } /* endif */ + pTokBuf.push_back(Token); + Token.reset(); } /* endif */ } /* endwhile */ @@ -717,96 +679,6 @@ BOOL TATagTokenizeW // nothing was processed due to errors (memory allocation?) *ppRest = pszInput; } - else if ( //!usTokens || // end of token table reached - !fComplete ) // or not last block of data ??? - { - - TOKENENTRY* pLastToken = pTokBuf.size()>0? & pTokBuf[pTokBuf.size()-1] : nullptr; - // at least two tokens in buffer .... - if ( pTokBuf.size()>2)//(pToken != pTokBuf.data()) && ( pToken != (pTokBuf.data()+1)) ) // if tokens in buffer - { - /**************************************************************/ - /* There are tokens in the buffer, let's see how we have to */ - /* deal with them ... */ - /**************************************************************/ - - // go back to the last tag to allow tag attribute and whitespace - // processing in the next tokenization block (unless this tag - // is the first token in the token buffer or the remaining data - // is longer than twice the max segment size ) - int iRemainLength = UTF16strlenCHAR(pLastToken->pDataStringW + pLastToken->iLength); - if ( ( (iLastTagPosInBuf != pTokBuf.size()-1) && // more than one tag in buffer - (iRemainLength < (2*MAX_SEGMENT_SIZE)) ) //|| // rest is smaller than 2 segments - //(usTokens == 0) - ) // token buffer overflow - { - Token = pTokBuf[iLastTagPosInBuf]; - TokStatus.usColPos = usTagColPos; - *ppRest = Token.pDataStringW; - } - else if(pLastToken) - { - // continue right behind the last recognized token - *ppRest = pLastToken->pDataStringW + pLastToken->iLength; - if ( (*pszData == EOS) && (*ppRest >= pszData) ) - { - // no more data to follow, segment has been processed completely - *ppRest = NULL; - } /* endif */ - } /* endif */ - } - else - { - /**************************************************************/ - /* The token buffer is empty, no tags were recognized ... */ - /* In addition we came here if only one tag is recognized... */ - /* */ - /* Go back to the last LF and add that data as a TEXT token */ - /**************************************************************/ - PSZ_W pszAreaStart = NULL; - pszEnd = pszData; - pszData--; - if ( pTokBuf.empty() ) - { - // no token in buffer - pszAreaStart = pszInput; // set start of data - } - else - { - // one token in buffer - pszAreaStart = pLastToken->pDataStringW + pLastToken->iLength; - } /* endif */ - - while ( pszData > pszAreaStart ) - { - if ( *pszData == LF ) - { - break; // leave while loop - } - else - { - pszData--; // test previous character - } /* endif */ - } /* endwhile */ - - if ( pszData == pszInput ) // no linefeed found ??? - { - *ppRest = pszEnd - 1; // set pRest to end of data CHECK in DEBUG - } - else - { - *ppRest = pszData; // set pRest to last linefeed - } /* endif */ - Token.sTokenid = TEXT_TOKEN; // text data - Token.pDataStringW = pszAreaStart; // set start of data - Token.iLength = (USHORT)(*ppRest - pszAreaStart); - - //usTokens--; - //pToken++; - pTokBuf.push_back(Token); - Token.reset(); - } /* endif */ - } else { *ppRest = NULL; @@ -826,13 +698,13 @@ BOOL TATagTokenizeW //#define DEBUG_PRINT_TOKENS //#ifdef DEBUG_PRINT_TOKENS - for(auto& tok: pTokBuf){ - T5LOG(T5DEVELOP) << "pTokBuf[" << &tok - pTokBuf.data() <<"]: ClassId=" << tok.ClassId << "; iLength=" << tok.iLength << "; pDataStringW=" << (tok.pDataStringW? EncodingHelper::convertToUTF8(tok.pDataStringW): "" ) + if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ + for(auto& tok: pTokBuf){ + T5LOG(T5DEVELOP) << "pTokBuf[" << &tok - pTokBuf.data() <<"]: ClassId=" << tok.ClassId << "; iLength=" << tok.iLength << "; pDataStringW=" << (tok.pDataStringW? EncodingHelper::convertToUTF8(tok.pDataStringW): "" ) << "; sAddInfo=" << tok.sAddInfo << "; sTokenid=" << tok.sTokenid << "; usOrgId=" << tok.usOrgId; + } } - //#endif - /*******************************************************************/ /* cleanup */ /*******************************************************************/ @@ -2283,7 +2155,6 @@ USHORT TACreateProtectTableWEx pRest = NULL; TATagTokenizeW( pszSegment, pTagTable, - TRUE, &pRest, &usColPos, pTokBuffer); @@ -2738,7 +2609,6 @@ USHORT TAProtectedPartsInQuotedText { TATagTokenizeW( chTempBuf, pTagTable, - TRUE, &pRest, pusColPos, pAttrTokBuffer ); diff --git a/source/opentm2/core/utilities/EQFMORPH.cpp b/source/opentm2/core/utilities/EQFMORPH.cpp index 5d05f139..3f8a1128 100755 --- a/source/opentm2/core/utilities/EQFMORPH.cpp +++ b/source/opentm2/core/utilities/EQFMORPH.cpp @@ -624,10 +624,9 @@ USHORT BuildDict //------------------------------------------------------------------------------ USHORT MorphTokenizeW ( - SHORT sLanguageID, // language ID - PSZ_W pszInData, // pointer to input segment - std::vector& pTermList, // address of caller's term list pointer - ULONG ulOemCP // CP of language of sLangID!! + SHORT sLanguageID, // language ID + PSZ_W pszInData, // pointer to input segment + std::vector& pTermList // address of caller's term list pointer ) { USHORT usRC = MORPH_OK; // function return code @@ -672,7 +671,6 @@ USHORT MorphTokenizeW } /* endif */ return( usRC ); - } /* endof MorphTokenizeW */ //------------------------------------------------------------------------------ diff --git a/source/opentm2/core/utilities/LogWrapper.h b/source/opentm2/core/utilities/LogWrapper.h index f02434c4..601eaa2b 100755 --- a/source/opentm2/core/utilities/LogWrapper.h +++ b/source/opentm2/core/utilities/LogWrapper.h @@ -11,7 +11,6 @@ #ifdef _DEBUG_LOGS #define TOKENIZELOG #define SPELLFACTORY_LOGGING - //#define MEASURETIME #define SGMLDITA_LOGGING #define INLINE_TAG_REPL_LOGGING #define _NTGET_LOG diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/OTMRTF.H b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/OTMRTF.H index 7de2db99..012832ef 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/OTMRTF.H +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/OTMRTF.H @@ -1810,8 +1810,6 @@ typedef struct _PARSEDATA int iUCStatus; // current \uc setting BOOL fHiddenMode; // current hidden text mode in PreUnseg step BOOL fBidiOffMode; // TRUE = we are in BIDIOFF mode - ULONG ulSrcOemCP; - ULONG ulTgtOemCP; ULONG ulSrcAnsiCP; ULONG ulTgtAnsiCP; CHAR_W UTF16Buffer[MAX_SEG_SIZE+20]; // buffer for UTF16 conversion From a947b62ab6cf46889eb9263bb7e26863d90b8bd4 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 23 Jun 2025 16:27:52 +0300 Subject: [PATCH 23/62] * fuzzy refactoring --- include/EQFFUZZ.H | 4 +- include/EQFTP.H | 3 +- include/lowlevelotmdatastructs.h | 18 +- include/tm.h | 85 +++-- .../opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp | 290 ++++++++---------- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 14 +- source/opentm2/core/requestdata.cpp | 6 +- source/opentm2/core/tagtable/EQFTOKEN.cpp | 61 ++-- source/opentm2/core/utilities/EQFDICT.cpp | 2 +- source/opentm2/core/utilities/EQFUTERR.cpp | 93 +----- source/opentm2/core/utilities/UtlMisc.cpp | 3 +- 12 files changed, 211 insertions(+), 370 deletions(-) diff --git a/include/EQFFUZZ.H b/include/EQFFUZZ.H index 297d37b8..6aad5807 100755 --- a/include/EQFFUZZ.H +++ b/include/EQFFUZZ.H @@ -41,7 +41,7 @@ SHORT CompFuzzyTok ( PFUZZYTOK , PFUZZYTOK, BOOL ); VOID MakeHashValue ( PULONG, USHORT, PSZ_W, PULONG ); //static -BOOL PrepareTokens ( PLOADEDTABLE, std::vector&, std::vector&, PSZ_W, SHORT, std::vector & ); +BOOL PrepareTokens ( PLOADEDTABLE, PSZ_W, SHORT, std::vector & ); //static BOOL FuzzyReplace ( PSZ_W, PSZ_W, PSZ_W, PREPLLIST, PREPLLIST ); //static @@ -49,7 +49,7 @@ VOID TransferSource( PSZ_W, PSZ_W, PSZ_W ); //static SHORT TokStrCompare ( PFUZZYTOK, PFUZZYTOK); //static -PFUZZYTOK SplitTokens (PFUZZYTOK, USHORT, SHORT, int, PSZ_W); +void SplitTokens (std::vector&, USHORT, SHORT, int, PSZ_W); //static //static SHORT Snake ( PFUZZYTOK, PFUZZYTOK, SHORT, SHORT, SHORT, SHORT, BOOL); diff --git a/include/EQFTP.H b/include/EQFTP.H index 7d6c4900..bb6623de 100755 --- a/include/EQFTP.H +++ b/include/EQFTP.H @@ -982,8 +982,7 @@ VOID EQFBVioSetNewDocSize ( PTBDOCUMENT ); // refresh doc size VOID ImeSetFont( PTBDOCUMENT, HWND, PLOGFONT ); -BOOL EQFBFindDiffEx( PVOID, std::vector& , std::vector&, - PSZ_W, PSZ_W, SHORT , +BOOL EQFBFindDiffEx( PLOADEDTABLE, PSZ_W, PSZ_W, SHORT , std::vector&, std::vector&); BOOL EQFBCountDiff( std::vector& pvFuzzyTok, PUSHORT pusTokens, PUSHORT pusDiffs ); diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 237d08b3..56672310 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -1168,8 +1168,8 @@ typedef BTREE ** PPBTREE; //prefix for each output structures typedef struct _TMX_PREFIX_OUT { - USHORT usLengthOutput; //length of complete output structure - USHORT usTmtXRc; //function returncode + USHORT usLengthOutput = 0; //length of complete output structure + USHORT usTmtXRc = 0; //function returncode } TMX_PREFIX_OUT, * PTMX_PREFIX_OUT, XOUT, * PXOUT; @@ -2819,13 +2819,13 @@ enum FilterField{ typedef struct _FUZZYTOK { - PSZ_W pData; // pointer to data.. - ULONG ulHash; // hash value - USHORT usStart; // start offset - USHORT usStop; // end offset - SHORT sType; // type of entry /* KIT0857M */ - EQF_BOOL fConnected; // true if connected to pPropTokList - EQF_BOOL fNoTagSubst; // true = do not use for tag substitution + PSZ_W pData = nullptr; // pointer to data.. + ULONG ulHash = 0; // hash value + USHORT usStart = 0; // start offset + USHORT usStop = 0; // end offset + SHORT sType = 0; // type of entry /* KIT0857M */ + EQF_BOOL fConnected = 0; // true if connected to pPropTokList + EQF_BOOL fNoTagSubst = 0; // true = do not use for tag substitution } FUZZYTOK, *PFUZZYTOK; #endif //_LOW_LEVEL_OTM_DATA_STRUCTS_INCLUDED_ diff --git a/include/tm.h b/include/tm.h index 6a60afd5..985ce199 100644 --- a/include/tm.h +++ b/include/tm.h @@ -63,28 +63,27 @@ typedef CHAR LONG_FN[MAX_LONGFILESPEC]; struct TMX_EXT_OUT_W { - TMX_PREFIX_OUT stPrefixOut; // prefix of output buffer - - CHAR_W szSource[MAX_SEGMENT_SIZE]; // source sentence - CHAR_W szTarget[MAX_SEGMENT_SIZE]; // target sentence - CHAR szOriginalSourceLanguage[MAX_LANG_LENGTH]; // language name of the source - CHAR szTagTable[MAX_FNAME]; // tag table name - CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language name of target - CHAR szAuthorName[MAX_FILESPEC]; // author name of target - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - CHAR szFileName[MAX_FILESPEC]; // where source comes from name+ext - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSourceSegmentId; // seg. num. of source sentence from analysis - TIME_L lTargetTime; // time stamp of target + TMX_PREFIX_OUT stPrefixOut{}; // prefix of output buffer + + CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence + CHAR_W szTarget[MAX_SEGMENT_SIZE]{}; // target sentence + CHAR szOriginalSourceLanguage[MAX_LANG_LENGTH]{}; // language name of the source + CHAR szTagTable[MAX_FNAME]{}; // tag table name + CHAR szTargetLanguage[MAX_LANG_LENGTH]{}; // language name of target + CHAR szAuthorName[MAX_FILESPEC]{}; // author name of target + USHORT usTranslationFlag = 0; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ + CHAR szFileName[MAX_FILESPEC]{}; // where source comes from name+ext + LONG_FN szLongName{}; // name of source file (long name or EOS) + ULONG ulSourceSegmentId = 0; // seg. num. of source sentence from analysis + TIME_L lTargetTime = 0; // time stamp of target std::wstring szContext, szAddInfo; - //CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context - //CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - ULONG ulRecKey; - USHORT usTargetKey; + ULONG ulRecKey = 0; + USHORT usTargetKey = 0; - LONG lTmKey; // tm record key - USHORT usNextTarget; // which target record to address next - ULONG ulMaxEntries; // number of entries in tm data file + LONG lTmKey = 0; // tm record key + USHORT usNextTarget = 0; // which target record to address next + ULONG ulMaxEntries = 0; // number of entries in tm data file + TMX_EXT_OUT_W(){}; }; using PTMX_EXT_OUT_W = TMX_EXT_OUT_W *; @@ -1661,24 +1660,24 @@ typedef struct _TMX_PUT_W typedef struct _TMX_GET_W { - CHAR_W szSource[MAX_SEGMENT_SIZE]; // source sentence - CHAR szTagTable[MAX_FNAME]; // tag table name of source - CHAR szSourceLanguage[MAX_LANG_LENGTH]; // language of source - CHAR szFileName[MAX_FILESPEC]; // file name the source comes from - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSegmentId; // segment number from analysis - CHAR szAuthorName[MAX_LANG_LENGTH]; // author name - CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language of translation - USHORT usRequestedMatches; // number of requested matches - USHORT usMatchThreshold; // threshold for match level - USHORT usConvert; // how the output should be converted - ULONG ulParm; // for future use, xmp. GET_ONLY_MT_MATCHES, GET_ONLY_EXACT_MATCHES...) - CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context - CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - PVOID pvReplacementList; // ptr to a SGML-DITA replacement list or NULL - PVOID pvGMOptList; // ptr to a global memory option list or NULL - bool fSourceLangIsPrefered; - bool fTargetLangIsPrefered; + CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence + CHAR szTagTable[MAX_FNAME]{}; // tag table name of source + CHAR szSourceLanguage[MAX_LANG_LENGTH]{}; // language of source + CHAR szFileName[MAX_FILESPEC]{}; // file name the source comes from + LONG_FN szLongName{}; // name of source file (long name or EOS) + ULONG ulSegmentId = 0; // segment number from analysis + CHAR szAuthorName[MAX_LANG_LENGTH]{}; // author name + CHAR szTargetLanguage[MAX_LANG_LENGTH]{}; // language of translation + USHORT usRequestedMatches = 0; // number of requested matches + USHORT usMatchThreshold = 0; // threshold for match level + USHORT usConvert = 0; // how the output should be converted + ULONG ulParm = 0; // for future use, xmp. GET_ONLY_MT_MATCHES, GET_ONLY_EXACT_MATCHES...) + CHAR_W szContext[MAX_SEGMENT_SIZE]{}; // segment context + CHAR_W szAddInfo[MAX_SEGMENT_SIZE]{}; // additional segment information + PVOID pvReplacementList = nullptr; // ptr to a SGML-DITA replacement list or NULL + PVOID pvGMOptList = nullptr; // ptr to a global memory option list or NULL + bool fSourceLangIsPrefered = false; + bool fTargetLangIsPrefered = false; bool fForceFuzzySearch = false; } TMX_GET_W, *PTMX_GET_W; @@ -1712,13 +1711,13 @@ typedef struct _TMX_MATCH_TABLE_W USHORT usMatchInfo; // for future use: special info for match } TMX_MATCH_TABLE_W, *PTMX_MATCH_TABLE_W; -using TMX_GET_OUT_W = struct _TMX_GET_OUT_W : public TMX_EXT_OUT_W +struct TMX_GET_OUT_W : public TMX_EXT_OUT_W { - // TMX_PREFIX_OUT stPrefixOut; //prefix of Output buffer - USHORT usNumMatchesFound; // number of matches found - TMX_MATCH_TABLE_W stMatchTable[MAX_MATCHES]; // match structure - USHORT fsAvailFlags; // additional flags (more exact/fuzzy matches avail) + USHORT usNumMatchesFound = 0; // number of matches found + TMX_MATCH_TABLE_W stMatchTable[MAX_MATCHES]{}; // match structure + USHORT fsAvailFlags = 0; // additional flags (more exact/fuzzy matches avail) std::vector tokens; + TMX_GET_OUT_W(){} }; using PTMX_GET_OUT_W = TMX_GET_OUT_W *; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp index dd9a8222..f8f07780 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp @@ -154,24 +154,21 @@ struct random{ //| allocate memory for start/stop table and copy list into | //| return success indicator | //+----------------------------------------------------------------------------+ -//static BOOL PrepareTokens ( PLOADEDTABLE pTagTable, - std::vector& pInBuf, - std::vector& pTokBuf, PSZ_W pString, // pointer to string to be tokenized SHORT sLanguageId, // language ID std::vector& pTransTokList // resulting list of tokens ) { + std::vector pTokBuf; PCHAR_W pRest; // ptr to start of not-processed bytes USHORT usColPos = 0; // column pos used by EQFTagTokenize PTOKENENTRY pTok; // ptr for token table processing int iCurOffs = 0; // current offset into segment ULONG ulLength; // length of start/stop table - PFUZZYTOK pstCurrent; // ptr to entries of start/stop table CHAR_W chTemp; // temp character BOOL fOK = TRUE; // success indicator USHORT usRandomIndex; // index in random sequence @@ -180,10 +177,8 @@ PrepareTokens USHORT usRC; // return code SHORT sNumTags; // number of tags /* KIT0857A */ BOOL fTag; // currently in tagging /* KIT0857A */ - PFUZZYTOK pstAct; // ptr start/stop table /* KIT0857A */ PSZ_W pStart; - //initializeRandom(); /********************************************************************/ /* run TATagTokenize to find tokens .... */ /********************************************************************/ @@ -200,20 +195,19 @@ PrepareTokens /* Rational: use input buffer for temporary list .... */ /* this is large enough, we can avoid any checking... */ /********************************************************************/ - pstCurrent = pInBuf.data(); // use input buffer for table pTok = (PTOKENENTRY) pTokBuf.data(); pStart = pTok->pDataStringW; while ( (pTok->sTokenid != ENDOFLIST) ) { if ( pTok->sTokenid == TEXT_TOKEN ) { - chTemp = *(pTok->pDataStringW+pTok->iLength); - *(pTok->pDataStringW + pTok->iLength) = EOS; + chTemp = pTok->pDataStringW[pTok->iLength]; + pTok->pDataStringW[pTok->iLength] = EOS; usRC = MorphTokenizeW( sLanguageId, pTok->pDataStringW, pTermList ); - *(pTok->pDataStringW + pTok->iLength) = chTemp; + pTok->pDataStringW[pTok->iLength] = chTemp; if ( pTermList.size() ) { @@ -222,51 +216,49 @@ PrepareTokens { while ( pActTerm->iLength ) { - pString = pTok->pDataStringW + pActTerm->iOffset; + pString = &pTok->pDataStringW[pActTerm->iOffset]; /**********************************************************/ /* ignore the linefeeds and tabs in the matching */ /**********************************************************/ if ( (*pString != LF) && (*pString != 0x09)) { - pstCurrent = SplitTokens(pstCurrent, - (iCurOffs + pActTerm->iOffset), - TEXT_TOKEN, - pActTerm->iLength, - pString); + SplitTokens(pTransTokList, + (iCurOffs + pActTerm->iOffset), + TEXT_TOKEN, + pActTerm->iLength, + pString); } /* endif */ pActTerm++; } /* endwhile */ } else { - pstCurrent = SplitTokens(pstCurrent, - iCurOffs, - pTok->sTokenid, - pTok->iLength, - pTok->pDataStringW); + SplitTokens(pTransTokList, + iCurOffs, + pTok->sTokenid, + pTok->iLength, + pTok->pDataStringW); } /* endif */ } /* endif */ } else { - pstCurrent = SplitTokens(pstCurrent, - iCurOffs, - pTok->sTokenid, - pTok->iLength, - pTok->pDataStringW); + SplitTokens(pTransTokList, + iCurOffs, + pTok->sTokenid, + pTok->iLength, + pTok->pDataStringW); } /* endif */ /****************************************************************/ /* adjust current offset to point to new offset in string... */ /****************************************************************/ pTok++; - //pTokenBuf.emplace_back(Tok) iCurOffs = (USHORT)(pTok->pDataStringW - pStart); } /* endwhile */ // terminate start/stop table - memset( pstCurrent, 0, sizeof( FUZZYTOK )); - + pTransTokList.push_back(FUZZYTOK{}); /********************************************************************/ /* get number of tags ... */ @@ -278,8 +270,8 @@ PrepareTokens /* pstAct -- points to active one */ /* pstCurrent -- points to next one... */ /********************************************************************/ - pstCurrent = pInBuf.data(); // use input buffer for table - pstAct = pstCurrent; + PFUZZYTOK pstCurrent = pTransTokList.data(); // use input buffer for table + PFUZZYTOK pstAct = pstCurrent; /********************************************************************/ /* set tag indication */ /********************************************************************/ @@ -293,7 +285,7 @@ PrepareTokens { fTag = ( pstCurrent->sType >= 0 ); pstAct++; - memcpy( pstAct, pstCurrent, sizeof( FUZZYTOK )); + *pstAct = *pstCurrent; pstCurrent++; } else // it is an attribute @@ -310,7 +302,7 @@ PrepareTokens else { pstAct++; - memcpy( pstAct, pstCurrent, sizeof( FUZZYTOK )); + *pstAct = *pstCurrent; pstCurrent++; } /* endif */ } /* endif */ @@ -319,14 +311,12 @@ PrepareTokens /* put in end indication */ /********************************************************************/ pstAct++; - memcpy( pstAct, pstCurrent, sizeof( FUZZYTOK )); + *pstAct = *pstCurrent; /********************************************************************/ /* have to point to next one, because we use it as length parameter */ /********************************************************************/ pstAct++; - pTransTokList = pInBuf; - return( fOK ); } /* end of function */ @@ -351,30 +341,30 @@ PrepareTokens //| now deleted again (July 93) | //+----------------------------------------------------------------------------+ -//static -PFUZZYTOK SplitTokens - ( - PFUZZYTOK pstCurrent, +void SplitTokens +( + std::vector& FuzzyTokVec, USHORT usStart, SHORT sType, int usLength, PSZ_W pString - ) +) { CHAR_W chTemp; - pstCurrent->pData = pString; - pstCurrent->usStart = usStart; - pstCurrent->sType = sType; - pstCurrent->usStop = usStart + usLength - 1; - pstCurrent->fConnected = FALSE; - chTemp = *(pString + usLength); - *(pString + usLength) = EOS; - MakeHashValue ( random::getRandomVec(), MAX_RANDOM, - pString , &pstCurrent->ulHash ); - *(pString + usLength) = chTemp; - pstCurrent++; - return (pstCurrent); + FuzzyTokVec.push_back(FUZZYTOK{}); + auto& current = FuzzyTokVec.back(); + + current.pData = pString; + current.usStart = usStart; + current.sType = sType; + current.usStop = usStart + usLength - 1; + current.fConnected = FALSE; + chTemp = pString[usLength]; + pString[usLength] = EOS; + MakeHashValue ( random::getRandomVec(), MAX_RANDOM, + pString , ¤t.ulHash ); + pString[usLength] = chTemp; } /* end of function SplitTokens */ //+----------------------------------------------------------------------------+ @@ -398,7 +388,6 @@ PFUZZYTOK SplitTokens //| Characters not in the alphanumeric range are ignored.. | //| (Check in case of DBCS might be necessary...) | //+----------------------------------------------------------------------------+ -//static VOID MakeHashValue ( PULONG pulRandom, // array of random numbers for hashing @@ -469,7 +458,6 @@ VOID MakeHashValue //| ment string is of different length | //| endwhile | //+----------------------------------------------------------------------------+ -//static BOOL FuzzyReplace ( PSZ_W pSource, // source string @@ -792,8 +780,6 @@ static BOOL isNumber( PSZ_W pszText ) //| differences like in the representation of numbers ) | //| if something found to be edited do it during copying | //+----------------------------------------------------------------------------+ - -//static VOID TransferSource ( @@ -934,8 +920,6 @@ TransferSource //| memcmp both data strings | //| endif | //+----------------------------------------------------------------------------+ - -//static SHORT TokStrCompare ( @@ -1178,51 +1162,44 @@ BOOL EQFBCountDiff BOOL EQFBFindDiffEx ( - PVOID pTable, // pointer to loaded tagtable - std::vector& pInBuf, // pointer to input buffer - std::vector& pTokBuf, // pointer to temp token buffer - PSZ_W pString1, // first string passed - PSZ_W pString2, // second string + PLOADEDTABLE pTagTable, // pointer to loaded tagtable + PSZ_W pStringSrc, // first string passed + PSZ_W pStringMatch, // second string SHORT sLanguageId, // language id to be used for pString1 std::vector& pFuzzyTok, // returned token list of pString2 std::vector& pFuzzyTgt // returned token list of pString1 ) { - USHORT usLenStr1 = 0; // length of string 1 (in num. of tokens) - USHORT usLenStr2 = 0; // length of string 2 (in num. of tokens) - std::vector pTokenList2, pTokenList1; // pointer to token lists + USHORT usLenStrSrc = 0; // length of string 1 (in num. of tokens) + USHORT usLenStrMatch = 0; // length of string 2 (in num. of tokens) + std::vector pTokenListSrc, pTokenListMatch; // pointer to token lists PFUZZYTOK pToken; // pointer to token lists - PLOADEDTABLE pTagTable = (PLOADEDTABLE) pTable; /******************************************************************/ /* prepare tokens for String1 and string 2 */ /******************************************************************/ BOOL fOK = PrepareTokens( pTagTable, - pInBuf, - pTokBuf, - pString1, - sLanguageId, pTokenList1 ); + pStringSrc, + sLanguageId, pTokenListSrc ); if ( fOK ) { fOK = PrepareTokens( pTagTable, - pInBuf, - pTokBuf, - pString2, - sLanguageId, pTokenList2 ); + pStringMatch, + sLanguageId, pTokenListMatch ); } /* endif */ if (fOK ) { /********************************************************************/ /* get number of tokens in strings */ /********************************************************************/ - NUMBEROFTOKENS(usLenStr1, pTokenList1.data()); - NUMBEROFTOKENS(usLenStr2, pTokenList2.data()); + NUMBEROFTOKENS(usLenStrSrc, pTokenListSrc.data()); + NUMBEROFTOKENS(usLenStrMatch, pTokenListMatch.data()); /******************************************************************/ /* call LCS and compare tokens with strncmp */ /******************************************************************/ - fOK = EQFBCallLCS(pTokenList1, pTokenList2, - usLenStr1, usLenStr2, TRUE); + fOK = EQFBCallLCS(pTokenListSrc, pTokenListMatch, + usLenStrSrc, usLenStrMatch, TRUE); } /* endif */ /********************************************************************/ /* get number of tokens in strings */ @@ -1232,7 +1209,7 @@ EQFBFindDiffEx /****************************************************************/ /* adjust length to avoid non-marked spaces between marked tokens*/ /****************************************************************/ - pToken = pTokenList1.data(); //current pointers + pToken = pTokenListSrc.data(); //current pointers while ( pToken->ulHash ) { if ( (pToken->usStop < (pToken+1)->usStart) ) @@ -1242,7 +1219,7 @@ EQFBFindDiffEx pToken++; } /* endwhile */ - pToken = pTokenList2.data(); //current pointers + pToken = pTokenListMatch.data(); //current pointers while ( pToken->ulHash ) { if ( (pToken->usStop < (pToken+1)->usStart) ) @@ -1255,9 +1232,9 @@ EQFBFindDiffEx /* allocate space for fuzzy tokens ,.,,, */ /* this area MUST be freed by the calling application */ /****************************************************************/ - fOK = EQFBMarkModDelIns( pTokenList1, pTokenList2, + fOK = EQFBMarkModDelIns( pTokenListSrc, pTokenListMatch, pFuzzyTgt, pFuzzyTok, - usLenStr1, usLenStr2); + usLenStrSrc, usLenStrMatch); } /* endif */ return( fOK ); @@ -1293,7 +1270,6 @@ EQFBFindDiffEx //| MARK_MODIFIED, MARK_DELETED, MARK_INSERTED | //| return success indicator | //+----------------------------------------------------------------------------+ -//static BOOL EQFBMarkModDelIns ( @@ -1304,7 +1280,6 @@ EQFBMarkModDelIns USHORT usLenStr1, USHORT usLenStr2 ) - { BOOL fOK; PFUZZYTOK pTestToken; // pointer to token lists 2 @@ -1316,63 +1291,35 @@ EQFBMarkModDelIns pFuzzy1.resize(lTokBufLen); LONG lTokBufUsed = 0; + pTestToken = pFuzzy2.data(); + pT1Token = pFuzzy1.data(); + usStringA = usStringB = 0; + while ((usStringA < usLenStr1) && (usStringB < usLenStr2) ) { - pTestToken = pFuzzy2.data(); - pT1Token = pFuzzy1.data(); - usStringA = usStringB = 0; - while ((usStringA < usLenStr1) && (usStringB < usLenStr2) ) + if ( (pTokenList2[usStringB].sType == MARK_EQUAL) + && (pTokenList1[usStringA].sType == MARK_EQUAL) ) { - if ( (pTokenList2[usStringB].sType == MARK_EQUAL) - && (pTokenList1[usStringA].sType == MARK_EQUAL) ) - { - *pTestToken = pTokenList2[usStringB]; - pTestToken++; - lTokBufUsed++; - *pT1Token = pTokenList1[usStringA]; - pT1Token++; - - usStringA++; usStringB++; - } - else if (pTokenList1[usStringA].sType == MARK_EQUAL) - { - *pTestToken = pTokenList2[usStringB]; - pTestToken->sType = MARK_INSERTED; - pTestToken++; - lTokBufUsed++; - *pT1Token = pTokenList2[usStringB]; - pT1Token->sType = MARK_DELETED; - pT1Token++; - - usStringB++; - } - else if (pTokenList2[usStringB].sType == MARK_EQUAL) - { - *pTestToken = pTokenList1[usStringA]; - pTestToken->sType = MARK_DELETED; - pTestToken++; - lTokBufUsed++; - *pT1Token = pTokenList1[usStringA]; - pT1Token->sType = MARK_INSERTED; - pT1Token++; - - usStringA++; - } - else - { *pTestToken = pTokenList2[usStringB]; - pTestToken->sType = MARK_MODIFIED; pTestToken++; lTokBufUsed++; *pT1Token = pTokenList1[usStringA]; - pT1Token->sType = MARK_MODIFIED; pT1Token++; - usStringA++; - usStringB++; - } /* endif */ - } /* endwhile */ + usStringA++; usStringB++; + } + else if (pTokenList1[usStringA].sType == MARK_EQUAL) + { + *pTestToken = pTokenList2[usStringB]; + pTestToken->sType = MARK_INSERTED; + pTestToken++; + lTokBufUsed++; + *pT1Token = pTokenList2[usStringB]; + pT1Token->sType = MARK_DELETED; + pT1Token++; - while ( usStringA < usLenStr1 ) + usStringB++; + } + else if (pTokenList2[usStringB].sType == MARK_EQUAL) { *pTestToken = pTokenList1[usStringA]; pTestToken->sType = MARK_DELETED; @@ -1383,20 +1330,46 @@ EQFBMarkModDelIns pT1Token++; usStringA++; - } /* endwhile */ - while ( usStringB < usLenStr2 ) + } + else { *pTestToken = pTokenList2[usStringB]; - pTestToken->sType = MARK_INSERTED; + pTestToken->sType = MARK_MODIFIED; pTestToken++; lTokBufUsed++; - - *pT1Token = pTokenList2[usStringB]; - pT1Token->sType = MARK_DELETED; + *pT1Token = pTokenList1[usStringA]; + pT1Token->sType = MARK_MODIFIED; pT1Token++; + + usStringA++; usStringB++; - } /* endwhile */ - } /* endif */ + } /* endif */ + } /* endwhile */ + + while ( usStringA < usLenStr1 ) + { + *pTestToken = pTokenList1[usStringA]; + pTestToken->sType = MARK_DELETED; + pTestToken++; + lTokBufUsed++; + *pT1Token = pTokenList1[usStringA]; + pT1Token->sType = MARK_INSERTED; + pT1Token++; + + usStringA++; + } /* endwhile */ + while ( usStringB < usLenStr2 ) + { + *pTestToken = pTokenList2[usStringB]; + pTestToken->sType = MARK_INSERTED; + pTestToken++; + lTokBufUsed++; + + *pT1Token = pTokenList2[usStringB]; + pT1Token->sType = MARK_DELETED; + pT1Token++; + usStringB++; + } /* endwhile */ return( fOK ); } /* end of function EQFBMarkModDelIns */ @@ -1457,8 +1430,6 @@ EQFBMarkModDelIns //| endif | //| return length of shortest edit script | //+----------------------------------------------------------------------------+ - -//static SHORT FindMiddleSnake ( @@ -1797,10 +1768,7 @@ static long CompareTokenWS( PSZ_W pTok1, USHORT len1, PSZ_W pTok2, USHORT len2 ) //| both strings are identical. | //| Return the compounded value | //+----------------------------------------------------------------------------+ - -//static -SHORT -Snake +SHORT Snake ( PFUZZYTOK pFuzzyStr1, PFUZZYTOK pFuzzyStr2, @@ -1894,10 +1862,7 @@ Snake //| endif | //| endif | //+----------------------------------------------------------------------------+ - -//static -VOID -LCS +VOID LCS ( LCSTOKEN LCSStringA, LCSTOKEN LCSStringB, @@ -2118,7 +2083,6 @@ BOOL EQFBCallLCS //|Function flow: set up pointer array | //| return success indicator | //+----------------------------------------------------------------------------+ -//static void EQFBSimplifyAndCountMarks ( std::vector& pFuzzy, // ptr to token @@ -2228,9 +2192,7 @@ void EQFBSimplifyAndCountMarks //|Function flow: set up pointer array | //| return success indicator | //+----------------------------------------------------------------------------+ - -BOOL -NTMCompareBetweenTokens +BOOL NTMCompareBetweenTokens ( PSZ_W pD1, PSZ_W pD2, @@ -2249,8 +2211,6 @@ NTMCompareBetweenTokens SHORT sNumTags = 0; BOOL fStringEqual = TRUE; CHAR_W chW; - std::vector pInBufL; - std::vector pTokBuf; USHORT usI = 0; BOOL fTokIsTag = FALSE; BOOL fTok2IsTag = FALSE; @@ -2259,10 +2219,6 @@ NTMCompareBetweenTokens ULONG ulLen2 = 0; PSZ_W pTemp = NULL; - // allocate required buffers - pInBufL.resize(64000/sizeof(pInBufL[0])); - pTokBuf.reserve(TOK_BUFFER_SIZE/sizeof(TOKENENTRY)); - // load tag table fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, TALOADUSEREXIT | TALOADPROTTABLEFUNC, @@ -2280,17 +2236,11 @@ NTMCompareBetweenTokens /******************************************************************/ /* prepare tokens for String1 and string 2 */ /******************************************************************/ - fOK = PrepareTokens( pTable, - pInBufL, - pTokBuf, - pD1, sLangID, pTokenList1 ); + fOK = PrepareTokens( pTable, pD1, sLangID, pTokenList1 ); } if ( fOK) { - fOK = PrepareTokens( pTable, - pInBufL, - pTokBuf, - pD2, sLangID, pTokenList2 ); + fOK = PrepareTokens( pTable, pD2, sLangID, pTokenList2 ); } // prereq is FUzzyness= 100%, i.e. all texttokens have been MARK_EQUAL in TMFuzzyness!! // check whether all tokens ( text + tag) are in both lists! diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 0a757b09..1093744c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -399,7 +399,7 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, //+----------------------------------------------------------------------------+ //|Function name: MemLoadProcess | //+----------------------------------------------------------------------------+ -//|Function call: static USHORT MemLoadProcess( MEM_LOAD_DLG_IDA* pLIDA ) | +//|Function call: static USHORT MemLoadProcess( MEM_LOAD_DLG_IDA* pLIDA )| //+----------------------------------------------------------------------------+ //|Description: Import a segment to TM | //| read a block of data from a file if needed and add a | diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 5f281291..4727bf28 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -1264,9 +1264,7 @@ BOOL TMFuzzynessEx PUSHORT pusDiffs // number of diffs in segment ) { - BOOL fOK; - std::vector pFuzzyTok,pFuzzyTgt,pInBuf; // returned token list - std::vector pTokBuf; + std::vector pFuzzyTok,pFuzzyTgt; // returned token list PLOADEDTABLE pTable = NULL; // ptr to loaded tag table // fast exit if one or both strings are empty... @@ -1288,20 +1286,16 @@ BOOL TMFuzzynessEx } /* endif */ // load tag table - if ( fOK ) - { - fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, + BOOL fOK = (TALoadTagTableExHwnd( pszMarkup, &pTable, FALSE, TALOADUSEREXIT | TALOADPROTTABLEFUNC, FALSE, NULLHANDLE ) == NO_ERROR); - } /* endif */ - // allocate required buffers - pInBuf.resize(64000/sizeof(pInBuf[0])); + // call function to evaluate the differences if ( fOK ) { - fOK = EQFBFindDiffEx( pTable, pInBuf, pTokBuf, pszSource, + fOK = EQFBFindDiffEx( pTable, pszSource, pszMatch, sLanguageId, pFuzzyTok, pFuzzyTgt ); } /* endif */ diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 0175ba0b..371cbea1 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -3633,10 +3633,8 @@ int FuzzySearchRequestData::execute(){ std::vector vProposals(Data.iNumOfProposals); if ( _rc_ == NO_ERROR ) { - TMX_GET_W GetIn; - TMX_GET_OUT_W GetOut; - memset( &GetIn, 0, sizeof(TMX_GET_W) ); - memset( &GetOut, 0, sizeof(TMX_GET_OUT_W) ); + TMX_GET_W GetIn{}; + TMX_GET_OUT_W GetOut{}; mem->OtmProposalToGetIn( Data, &GetIn ); GetIn.usConvert = MEM_OUTPUT_ASIS; diff --git a/source/opentm2/core/tagtable/EQFTOKEN.cpp b/source/opentm2/core/tagtable/EQFTOKEN.cpp index 71e80f44..75b27bf5 100755 --- a/source/opentm2/core/tagtable/EQFTOKEN.cpp +++ b/source/opentm2/core/tagtable/EQFTOKEN.cpp @@ -437,7 +437,6 @@ BOOL TATagTokenizeW TOKENENTRY NewToken; // buffer for new tokens MSSTACK MSStack[10]; PTAG pTag; // pointer to first tag in tagtable - //PTOKENENTRY pLastTag = NULL; // pointer to last tag token int iLastTagPosInBuf = -1; USHORT usTagColPos = 0; // column position of last tag PATTRIBUTE pAttr; // ptr to attribute array of tag table @@ -674,46 +673,40 @@ BOOL TATagTokenizeW // if not processed completely or not last part of file // set pRest pointer to begin of last tag or (if no tags // are contained in data) to last linebreak. - if ( !fOK ) - { - // nothing was processed due to errors (memory allocation?) - *ppRest = pszInput; - } - else - { - *ppRest = NULL; - } /* endif */ - - // add end-of-list token - if ( fOK ) - { - Token.reset(); - Token.iLength = 0; - Token.sTokenid = ENDOFLIST; - pTokBuf.push_back(Token); - Token.reset(); - - *pusColPos = TokStatus.usColPos; - } /* endif */ + if ( fOK ) + { + *ppRest = NULL; + + // add end-of-list token + Token.reset(); + Token.iLength = 0; + Token.sTokenid = ENDOFLIST; + pTokBuf.push_back(Token); + Token.reset(); + *pusColPos = TokStatus.usColPos; + } + else + { + // nothing was processed due to errors (memory allocation?) + *ppRest = pszInput; + } /* endif */ - //#define DEBUG_PRINT_TOKENS - //#ifdef DEBUG_PRINT_TOKENS - if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ + if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ for(auto& tok: pTokBuf){ T5LOG(T5DEVELOP) << "pTokBuf[" << &tok - pTokBuf.data() <<"]: ClassId=" << tok.ClassId << "; iLength=" << tok.iLength << "; pDataStringW=" << (tok.pDataStringW? EncodingHelper::convertToUTF8(tok.pDataStringW): "" ) << "; sAddInfo=" << tok.sAddInfo << "; sTokenid=" << tok.sTokenid << "; usOrgId=" << tok.usOrgId; } - } + } - /*******************************************************************/ - /* cleanup */ - /*******************************************************************/ - if ( TokStatus.pRetryStack != NULL ) - { - UtlAlloc( (PVOID *)&TokStatus.pRetryStack, 0L, 0L, NOMSG ); - } /* endif */ + /*******************************************************************/ + /* cleanup */ + /*******************************************************************/ + if ( TokStatus.pRetryStack != NULL ) + { + UtlAlloc( (PVOID *)&TokStatus.pRetryStack, 0L, 0L, NOMSG ); + } /* endif */ - return( fOK ); + return( fOK ); } /* end of function TATagTokenize */ //------------------------------------------------------------------------------ diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index a73fe2bf..88a522ec 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -4774,7 +4774,7 @@ SHORT BTREE::QDAMDictOpenLocal ) { SHORT i; - BTREEHEADRECORD header; + //BTREEHEADRECORD header; SHORT sRc = 0; // return code USHORT usFlags; // set the open flags USHORT usAction; // return code from UtlOpen diff --git a/source/opentm2/core/utilities/EQFUTERR.cpp b/source/opentm2/core/utilities/EQFUTERR.cpp index d695868f..c46eff0c 100755 --- a/source/opentm2/core/utilities/EQFUTERR.cpp +++ b/source/opentm2/core/utilities/EQFUTERR.cpp @@ -133,99 +133,8 @@ USHORT UtlErrorHwnd //------------------------------------------------------------------------------ // Function flow: Call UtlErrorHwnd with a NULL window handle //------------------------------------------------------------------------------ -USHORT UtlErrorHwndW -( - SHORT sErrorNumber, // number of message - USHORT usMsgType, // type of message - USHORT usNoOfParms, // number of message parameters - PSZ_W *pParmTableW, // pointer to parameter table - ERRTYPE ErrorType, // type of error - HWND hwnd, // window handle - BOOL fUnicode // parameter table contains unicode strings -); -USHORT UtlErrorW -( - SHORT sErrorNumber, // number of message - USHORT usMsgType, // type of message - USHORT usNoOfParms, // number of message parameters - PSZ_W *pParmTableW, // pointer to parameter table - ERRTYPE ErrorType, // type of error - BOOL fUnicode // parameter table contains unicode strings -) -{ - return UtlErrorHwndW( sErrorNumber, // number of message - usMsgType, // type of message - usNoOfParms, // number of message parameters - pParmTableW, // pointer to parameter table - ErrorType, // type of error - NULLHANDLE, // window handle - fUnicode ); // parameter table contains unicode strings -} -USHORT UtlErrorHwndW -( - SHORT sErrorNumber, // number of message - USHORT usMsgType, // type of message - USHORT usNoOfParms, // number of message parameters - PSZ_W *pParmTableW, // pointer to parameter table - ERRTYPE ErrorType, // type of error - HWND hwnd, // window handle - BOOL fUnicode // parameter table contains unicode strings -) -{ - USHORT usRc = 0; - PSZ pParmTable[5]; - int i = 0; - - - if (!fUnicode) - { - for ( i=0; i= 128) - { - CHAR_W chW = pParmTableW[i][128]; - pParmTableW[i][128] = EOS; - UtlDirectUnicode2Ansi( pParmTableW[i], pParmTable[i], 0L ); - pParmTableW[i][128] = chW; - } - else - { - UtlDirectUnicode2Ansi( pParmTableW[i], pParmTable[i], 0L ); - } - } - - usRc = UtlErrorHwnd( sErrorNumber, - usMsgType, - usNoOfParms, - pParmTable, - ErrorType, - hwnd ); - UtlAlloc((PVOID *) &pData, 0L, 0L, NOMSG ); - } - - } - return( usRc ); -} + USHORT UtlError ( diff --git a/source/opentm2/core/utilities/UtlMisc.cpp b/source/opentm2/core/utilities/UtlMisc.cpp index cd501cb2..e65397d5 100755 --- a/source/opentm2/core/utilities/UtlMisc.cpp +++ b/source/opentm2/core/utilities/UtlMisc.cpp @@ -45,7 +45,6 @@ #define MAX_DISPATCH_MSGS 30 // number of msgs processed per dispatch static HWND hwndDlgs[MAX_DIALOGS]; // array for handles of modeless dialogs static SHORT sRegisteredDlgs = 0; // number of registered dialogs -static FILE *hFFSTFile = NULL; // UtlAlloc log file //static FARPROC lpfnOldButtonProc = NULL;// ptr to original button proc LONG FAR PASCAL EqfHelpButtonProc (HWND, UINT, WPARAM, LPARAM); @@ -580,7 +579,7 @@ BOOL UtlInitUtils( HAB hab ) //| FALSE if no change is required (in this | //| case no new target segment is allocated!) | //+----------------------------------------------------------------------------+ -//|Returncode type: BOOL FALSE in case of errors (=UtlAlloc failed) | +//|Returncode type: BOOL FALSE in case of errors | //| TRUE everything O.K. | //+----------------------------------------------------------------------------+ From 6961a5e9ed1bc9d30172d82f69553bd7bb75370f Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 23 Jun 2025 22:02:46 +0300 Subject: [PATCH 24/62] * fixed fuzzy crashes and calculations * fuzzy code cleanup --- include/EQFTA.H | 13 +- include/Proposal.h | 80 +++----- include/lowlevelotmdatastructs.h | 52 +++-- include/tm.h | 115 ++++++----- .../opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp | 1 + .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 15 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 193 +++++++++--------- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 50 +++-- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 8 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 19 +- .../core/EqfMemoryPlugin/EqfMemory.cpp | 20 +- .../core/EqfMemoryPlugin/EqfMemoryPlugin.cpp | 4 - source/opentm2/core/requestdata.cpp | 2 - source/opentm2/core/utilities/OtmProposal.cpp | 7 +- source/opentm2/core/utilities/UtlMisc.cpp | 17 -- source/opentm2/core/utilities/UtlRegistry.cpp | 4 +- 16 files changed, 289 insertions(+), 311 deletions(-) diff --git a/include/EQFTA.H b/include/EQFTA.H index 7404b24d..71859037 100755 --- a/include/EQFTA.H +++ b/include/EQFTA.H @@ -21,13 +21,14 @@ /**********************************************************************/ /* Structure for start/stop tables as used by TACreateStartStopTable */ /**********************************************************************/ -typedef struct _STARTSTOP +struct STARTSTOP { - USHORT usStart = 0; // start offset - USHORT usStop = 0; // stop offset - USHORT usType = 0; // type of entry - void reset(){memset(this, 0, sizeof(*this));} -} STARTSTOP, *PSTARTSTOP; + USHORT usStart = 0; // start offset + USHORT usStop = 0; // stop offset + USHORT usType = 0; // type of entry + void reset(){*this=STARTSTOP{};} +}; +using PSTARTSTOP = STARTSTOP*; // structure for commincation between analysis and MT functions diff --git a/include/Proposal.h b/include/Proposal.h index 14c81e47..4724b536 100644 --- a/include/Proposal.h +++ b/include/Proposal.h @@ -377,94 +377,91 @@ class OtmProposal //protected: /*! \brief ID of this proposal */ - char szId[OTMPROPOSAL_MAXNAMELEN]; + char szId[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Source string of memory proposal (UTF-16) */ - wchar_t szSource[OTMPROPOSAL_MAXSEGLEN]; + wchar_t szSource[OTMPROPOSAL_MAXSEGLEN]{}; //std::wstring strSource; /*! \brief Target string of memory proposal (UTF-16). */ //std::wstring strTarget; - wchar_t szTarget[OTMPROPOSAL_MAXSEGLEN]; + wchar_t szTarget[OTMPROPOSAL_MAXSEGLEN]{}; /*! \brief Name of document from which the proposal comes from. */ //std::string strDocName; - char szDocName[OTMPROPOSAL_MAXNAMELEN]; + char szDocName[OTMPROPOSAL_MAXNAMELEN]{}; /*! \briefCustom Id provided for the segment in request to track segments in response in multifuzzy*/ - char szCustomId[OTMPROPOSAL_MAXNAMELEN]; + char szCustomId[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Segment number within the document from which the proposal comes from. */ - long lSegmentId; + long lSegmentId{}; /*! \brief source language. */ //std::string strSourceLanguage; - char szSourceLanguage[OTMPROPOSAL_MAXNAMELEN]; - char szOriginalSourceLanguage[OTMPROPOSAL_MAXNAMELEN]; + char szSourceLanguage[OTMPROPOSAL_MAXNAMELEN]{}; + char szOriginalSourceLanguage[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief target language. */ //std::string strTargetLanguage; - char szTargetLanguage[OTMPROPOSAL_MAXNAMELEN]; + char szTargetLanguage[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief origin or type of the proposal. */ - eProposalType eType; + eProposalType eType{}; /*! \brief match type of the proposal. */ - eMatchType eMatch; + eMatchType eMatch{}; /*! \brief Author of the proposal. */ // std::string strTargetAuthor; - char szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]; + char szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Update time stamp of the proposal. */ - long lTargetTime; + long lTargetTime{}; /*! \brief Fuzziness of the proposal. */ - int iFuzziness, iDiffs, iWords; + int iFuzziness{}, iDiffs{}, iWords{}; /*! \brief Markup table (format) of the proposal. */ - char szMarkup[OTMPROPOSAL_MAXNAMELEN]; + char szMarkup[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Context information of the proposal */ //std::wstring strContext; - wchar_t szContext[OTMPROPOSAL_MAXSEGLEN+1]; + wchar_t szContext[OTMPROPOSAL_MAXSEGLEN+1]{}; /*! \brief Additional information of the proposal */ //std::wstring strAddInfo; - wchar_t szAddInfo[OTMPROPOSAL_MAXSEGLEN+1]; + wchar_t szAddInfo[OTMPROPOSAL_MAXSEGLEN+1]{}; /*! \brief Proposal data has been filled flag */ - bool fFilled; + bool fFilled{}; /*! \brief Index of memory when looking up in a list of memories */ - int iMemoryIndex; + int iMemoryIndex{}; /*! \brief ranking of the context information (0..100) */ - int iContextRanking; + int iContextRanking{}; /*! \brief list of replacement values */ - long pvReplacementList; + long pvReplacementList{}; //searchproposal std::unique_ptr pInputSentence = nullptr; - char szMemory[260]; - char szIsoSourceLang[MAX_LANG_LENGTH]; - char szIsoTargetLang[MAX_LANG_LENGTH]; - wchar_t szError[512]; - //char szType[256]; - //char szDateTime[40]; - char szSearchMode[40]; - char szSearchPos[80]; - int iNumOfProposals; - int iSearchTime; - wchar_t szSearchString[2050]; + char szMemory[260]{}; + char szIsoSourceLang[MAX_LANG_LENGTH]{}; + char szIsoTargetLang[MAX_LANG_LENGTH]{}; + wchar_t szError[512]{}; + char szSearchMode[40]{}; + char szSearchPos[80]{}; + int iNumOfProposals{}; + int iSearchTime{}; + wchar_t szSearchString[2050]{}; - // - TMCursor currentInternalKey; - TMCursor nextInternalKey; + TMCursor currentInternalKey{}; + TMCursor nextInternalKey{}; int moveInternalKeyToTheNextPosition() { currentInternalKey = nextInternalKey; @@ -478,22 +475,13 @@ class OtmProposal int setSequentialAccessKey( char *pszKey); -//otm_put - //USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - //ULONG ulSourceSegmentId; //seg. num. of source sentence from analysis - //TIME_L lTime; //time stamp - BOOL fMarkupChanged; // Markup does not exist, changed to OTMUTF8 during import -//otm_put end - - + BOOL fMarkupChanged{}; // Markup does not exist, changed to OTMUTF8 during import + /*! \brief is source lang is marked as prefered in languages.xml */ bool fIsoSourceLangIsPrefered = false; /*! \brief is target lang is marked as prefered in languages.xml */ bool fIsoTargetLangIsPrefered = false; - //searchproposal - - //char errorStr[1000]; }; #include "tm.h" diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 56672310..004ed4cd 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -295,15 +295,16 @@ typedef char * PSZ; typedef LONG TIME_L; // new typedef to avoid conflicts with Lotus -typedef struct _RECPARAM +struct RECPARAM { USHORT usNum = 0; // record number USHORT usOffset = 0; // record offset ULONG ulLen = 0; // record length void reset(){ - memset(this, 0 ,sizeof(*this)); + *this = RECPARAM{}; } -} RECPARAM, * PRECPARAM; +}; +using PRECPARAM = RECPARAM*; /**********************************************************************/ /* typedef used for all vital information on our new approach... */ @@ -410,8 +411,8 @@ typedef enum _RECTYPE #define FREE_SIZE_V3 (BTREE_REC_SIZE_V3 - sizeof(BTREEHEADER)) typedef struct _BTREERECORD_V3 { - BTREEHEADER header; // 16 bytes header - UCHAR uchData[ FREE_SIZE_V3 ] ; // free size to be used + BTREEHEADER header{}; // 16 bytes header + UCHAR uchData[ FREE_SIZE_V3 ]{} ; // free size to be used } BTREERECORD_V3, *PBTREERECORD_V3, **PPBTREERECORD_V3; /*****************************************************************************/ @@ -422,12 +423,11 @@ struct BTREEBUFFER { USHORT usRecordNumber=0; // index of rec in buffer BOOL fLocked=0; // Is the record locked ? - //BOOL fNeedToWrite; // Commit before reuse SHORT sUsed=0; // buffer used count ULONG ulCheckSum=0; // CheckSum of contents data - BTREERECORD_V3 contents; // data from disk + BTREERECORD_V3 contents{}; // data from disk - BTREEBUFFER(){memset(this, 0, sizeof(*this));} + //BTREEBUFFER(){memset(this, 0, sizeof(*this));} }; typedef struct _BTREEINDEX_V3 @@ -477,14 +477,8 @@ typedef LHANDLE HTM; //table entry structure struct TMX_TABLE_ENTRY { - CHAR szName[MAX_LANG_LENGTH]; + CHAR szName[MAX_LANG_LENGTH]{}; USHORT usId = 0; - - TMX_TABLE_ENTRY() - { - memset(szName, 0, sizeof(szName)); - //szName[0] = '\0'; - } }; typedef TMX_TABLE_ENTRY * PTMX_TABLE_ENTRY; @@ -502,8 +496,7 @@ struct TMX_TABLE { std::vector stTmTableEntry; TMX_TABLE(){ - stTmTableEntry.push_back(TMX_TABLE_ENTRY());//push_empty_entry - memset(&stTmTableEntry.back(), 0, sizeof(TMX_TABLE_ENTRY)); + stTmTableEntry.push_back(TMX_TABLE_ENTRY{});//push_empty_entry } }; @@ -1418,21 +1411,22 @@ class TMXParseHandler; //class SAXParser; struct LOADEDTABLE; -typedef struct _TOKENENTRY // entry in tokenlist : +struct TOKENENTRY // entry in tokenlist : { // !!!! Attention: below has to match TOKENENTRYSEG definition .... !!!! - SHORT sTokenid; // Tokenid - int iLength; // Length of data string - SHORT sAddInfo; // additional information from tag table - CHAR * pDataString; // pointer to data string - USHORT usOrgId; // original id - USHORT ClassId; // class id of token - CHAR_W * pDataStringW; // pointer to data string - Unicode + SHORT sTokenid{}; // Tokenid + int iLength{}; // Length of data string + SHORT sAddInfo{}; // additional information from tag table + CHAR * pDataString{}; // pointer to data string + USHORT usOrgId{}; // original id + USHORT ClassId{}; // class id of token + CHAR_W * pDataStringW{}; // pointer to data string - Unicode // !!!! Attention: above has to match TOKENENTRYSEG definition .... !!!! - void reset(){ memset(this, 0, sizeof(*this));} + void reset(){ *this = TOKENENTRY{};} -} TOKENENTRY, *PTOKENENTRY; +}; +using PTOKENENTRY = TOKENENTRY*; class ImportStatusDetails; @@ -2454,8 +2448,8 @@ struct MEM_LOAD_DLG_IDA class TMCursor { - long _recordKey; - ushort _targetKey; + long _recordKey{}; + ushort _targetKey{}; public: diff --git a/include/tm.h b/include/tm.h index 985ce199..61e9ea49 100644 --- a/include/tm.h +++ b/include/tm.h @@ -950,41 +950,43 @@ typedef struct _TMX_RECORD // structure of the source segment -typedef struct _TMX_SOURCE_RECORD +struct TMX_SOURCE_RECORD { - LONG lRecordLen; - USHORT usSource; - USHORT usLangId; + LONG lRecordLen{}; + USHORT usSource{}; + USHORT usLangId{}; void reset() { - memset(this, 0, sizeof(*this)); + *this = TMX_SOURCE_RECORD{}; } -} TMX_SOURCE_RECORD, *PTMX_SOURCE_RECORD; +}; +using PTMX_SOURCE_RECORD = TMX_SOURCE_RECORD*; // structure of the target segment -typedef struct _TMX_TARGET_RECORD +struct TMX_TARGET_RECORD { LONG lRecordLen = 0; USHORT usTarget = 0; USHORT usClb = 0; void reset() { - memset(this, 0, sizeof(*this)); + *this = TMX_TARGET_RECORD{}; } -} TMX_TARGET_RECORD, *PTMX_TARGET_RECORD; +}; +using PTMX_TARGET_RECORD = TMX_TARGET_RECORD*; // control block structure in target record typedef struct _TMX_TARGET_CLB { - TIME_L lTime; - TIME_L lUpdateTime; - ULONG ulSegmId; - USHORT usLangId; - USHORT usFileId; - USHORT usAuthorId; - USHORT usAddDataLen; // new for Major_version6: Length of following context and additional info data - BYTE bMultiple; - BYTE bTranslationFlag; + TIME_L lTime{}; + TIME_L lUpdateTime{}; + ULONG ulSegmId{}; + USHORT usLangId{}; + USHORT usFileId{}; + USHORT usAuthorId{}; + USHORT usAddDataLen{}; // new for Major_version6: Length of following context and additional info data + BYTE bMultiple{}; + BYTE bTranslationFlag{}; } TMX_TARGET_CLB, *PTMX_TARGET_CLB; // helper macros for working with TMX_TARGET_CLBs @@ -1018,7 +1020,7 @@ USHORT NtmGetAddDataLen(PTMX_TARGET_CLB pCLB, USHORT usDataID); USHORT NtmStoreAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszAddData); // retrieve specific data from the combined data area, returns length of retrieved data (incl. string end delimiter) -USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, std::wstring& pszBuffer); +USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, wchar_t* pszBuffer, PBYTE endOfMemRegion = 0); // find a string in a specific data area BOOL NtmFindInAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch); @@ -1092,22 +1094,23 @@ inline TMX_TARGET_CLB *toTmxTargetClb(BytesRef buffer) return reinterpret_cast(buffer.data()); } -typedef struct _TMX_TERM_TOKEN +struct TMX_TERM_TOKEN { - USHORT usOffset; - USHORT usLength; - USHORT usHash; - void reset() { memset(this, 0, sizeof(*this)); } -} TMX_TERM_TOKEN, *PTMX_TERM_TOKEN; + USHORT usOffset{}; + USHORT usLength{}; + USHORT usHash{}; + void reset() { *this = TMX_TERM_TOKEN{}; } +}; +using PTMX_TERM_TOKEN = TMX_TERM_TOKEN*; -typedef struct _TMX_MATCHENTRY +struct TMX_MATCHENTRY { LONG lKey = 0; USHORT usMaxVotes = 0; USHORT usMatchVotes = 0; BYTE cCount = 0; - -} TMX_MATCHENTRY, *PTMX_MATCHENTRY; +}; +using PTMX_MATCHENTRY = TMX_MATCHENTRY*; inline bool operator==(const TMX_MATCHENTRY& lhs, const TMX_MATCHENTRY& rhs) { @@ -1686,29 +1689,29 @@ typedef struct _TMX_GET_W typedef struct _TMX_MATCH_TABLE_W { - CHAR_W szSource[MAX_SEGMENT_SIZE]; // source sentence with tags - CHAR szFileName[MAX_FILESPEC]; // file name where the source comes from. - LONG_FN szLongName; // name of source file (long name or EOS) - ULONG ulSegmentId; // segment number from analysis - CHAR_W szTarget[MAX_SEGMENT_SIZE]; // target sentence with tags - CHAR szTargetLanguage[MAX_LANG_LENGTH]; // language of translation - CHAR szOriginalSrcLanguage[MAX_LANG_LENGTH]; // language of src of translation - USHORT usTranslationFlag; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ - CHAR szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]; // author name of target - TIME_L lTargetTime; // time stamp of target - USHORT usMatchLevel; // similarity of the source - int iWords; - int iDiffs; - USHORT usOverlaps; // temp field - nr of overlapping triples + CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence with tags + CHAR szFileName[MAX_FILESPEC]{}; // file name where the source comes from. + LONG_FN szLongName{}; // name of source file (long name or EOS) + ULONG ulSegmentId{}; // segment number from analysis + CHAR_W szTarget[MAX_SEGMENT_SIZE]{}; // target sentence with tags + CHAR szTargetLanguage[MAX_LANG_LENGTH]{}; // language of translation + CHAR szOriginalSrcLanguage[MAX_LANG_LENGTH]{}; // language of src of translation + USHORT usTranslationFlag{}; /* type of translation, 0 = human, 1 = machine, 2 = GobalMemory */ + CHAR szTargetAuthor[OTMPROPOSAL_MAXNAMELEN]{}; // author name of target + TIME_L lTargetTime{}; // time stamp of target + USHORT usMatchLevel{}; // similarity of the source + int iWords{}; + int iDiffs{}; + USHORT usOverlaps{}; // temp field - nr of overlapping triples CHAR szTagTable[MAX_FNAME]; // tag table name of source - LONG lKey; // key of match - USHORT usTargetNum; // number of target - USHORT usDBIndex; // number of memory in current hierarchy - //CHAR_W szContext[MAX_SEGMENT_SIZE]; // segment context info - //CHAR_W szAddInfo[MAX_SEGMENT_SIZE]; // additional segment information - std::wstring szContext, szAddInfo; - USHORT usContextRanking; // context ranking from user exit context processing - USHORT usMatchInfo; // for future use: special info for match + LONG lKey{}; // key of match + USHORT usTargetNum{}; // number of target + USHORT usDBIndex{}; // number of memory in current hierarchy + CHAR_W szContext[MAX_SEGMENT_SIZE]{}; // segment context info + CHAR_W szAddInfo[MAX_SEGMENT_SIZE]{}; // additional segment information + //std::wstring szContext, szAddInfo; + USHORT usContextRanking{}; // context ranking from user exit context processing + USHORT usMatchInfo{}; // for future use: special info for match } TMX_MATCH_TABLE_W, *PTMX_MATCH_TABLE_W; struct TMX_GET_OUT_W : public TMX_EXT_OUT_W @@ -1739,9 +1742,9 @@ using PTMX_GET_OUT_W = TMX_GET_OUT_W *; // structure TMX_EXT_IN typedef struct _TMX_EXT_IN { - LONG lTmKey; // tm get to get - USHORT usConvert; // how the output is to appear - USHORT usNextTarget; // which target record to address next + LONG lTmKey{}; // tm get to get + USHORT usConvert{}; // how the output is to appear + USHORT usNextTarget{}; // which target record to address next } TMX_EXT_IN, *PTMX_EXT_IN; typedef TMX_EXT_IN TMX_EXT_IN_W, *PTMX_EXT_IN_W; @@ -2055,7 +2058,7 @@ class EqfMemory //: public TMX_CLB // tm extract prototypes USHORT FillExtStructure(PTMX_TARGET_RECORD, PTMX_TARGET_CLB, - std::wstring&, PLONG, PTMX_EXT_OUT_W); + std::wstring&, PLONG, PTMX_EXT_OUT_W, PBYTE); int RewriteCompactTable(); @@ -3532,7 +3535,7 @@ VOID DeleteOldestRecord(PTMX_RECORD, PULONG); USHORT GetExactMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, PTMX_GET_OUT_W); USHORT ExactTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_SENTENCE, - PTMX_MATCH_TABLE_W, PUSHORT, LONG); + PTMX_MATCH_TABLE_W, PUSHORT, LONG, PBYTE); INT CompCount(const void *, const void *); @@ -3540,7 +3543,7 @@ INT CompCountVotes(const void *, const void *); VOID CleanupTempMatch(PTMX_MATCHENTRY, std::vector&, PUSHORT, PUSHORT); USHORT FillMatchEntry(EqfMemory *, PTMX_SENTENCE, std::vector&, PUSHORT); USHORT FuzzyTest(EqfMemory *, PTMX_RECORD, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT, - PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, LONG); + PUSHORT, PUSHORT, PUSHORT, PTMX_SENTENCE, LONG, PBYTE); USHORT GetFuzzyMatch(EqfMemory *, PTMX_SENTENCE, PTMX_GET_W, PTMX_MATCH_TABLE_W, PUSHORT); // tm delete segment prototypes diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp index f8f07780..924bc24c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFFUZZ.cpp @@ -204,6 +204,7 @@ PrepareTokens chTemp = pTok->pDataStringW[pTok->iLength]; pTok->pDataStringW[pTok->iLength] = EOS; + pTermList.clear(); usRC = MorphTokenizeW( sLanguageId, pTok->pDataStringW, pTermList ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 62b7b68c..63d1b9ca 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -324,11 +324,6 @@ USHORT EqfMemory::FindTargetAndDelete( //compare target strings if ( !wcscmp( pString.c_str(), TmDel.pInputSentence->pStrings->getGenericTargetStrC() ) ) { - //target strings are equal so compare target tag records - //position at target tag table record - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - //compare tag table records //identical target tag table as in del structure so //check segment id and file name in control block @@ -362,12 +357,13 @@ USHORT EqfMemory::FindTargetAndDelete( //if on leave while loop as though delete was carried out if ( !pClb->bMultiple || (BOOL)TmDel.lTargetTime //|| true ) - { + { + PBYTE endOfMemRegion = TmRecord.data() + TmRecord.size(); //fill out the put structure as output of the extract function usRc = FillExtStructure( pTMXTargetRecord, pClb, TmDel.pInputSentence->pStrings->getGenericTargetStr(), &lSrcLen, - pTmExtOut ); + pTmExtOut, endOfMemRegion ); NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); pTmExtOut->ulRecKey = lKey; @@ -509,7 +505,7 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); lLeftClbLen = RECLEN(pTMXTargetRecord) - pTMXTargetRecord->usClb; - + // subtract size of current CLB if ( lLeftClbLen >= TARGETCLBLEN(pTargetClb) ) { @@ -575,11 +571,12 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( //check that multiple flag isn't on //if on leave while loop as though delete was carried out + PBYTE endOfMemRegion = TmRecord.data() + TmRecord.size(); //fill out the put structure as output of the extract function usRc = FillExtStructure( pTMXTargetRecord, pTargetClb, pSourceString, &lSourceLen, - pTmExtOut ); + pTmExtOut, endOfMemRegion ); NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); TmDel.currentInternalKey.setInternalKey(lKey, usTarget); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index e75a417a..2611479e 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -279,11 +279,12 @@ USHORT EqfMemory::ExtractRecordV6 //if target record exists if ( ( lLeftTgtLen > 0) && ( RECLEN(pTMXTargetRecord) != 0) ) { + PBYTE endOfMemRegion = pTmRecordData.data() + pTmRecordData.size(); //fill out the put structure as output of the extract function usRc = FillExtStructure( pTMXTargetRecord, pTargetClb, pSourceString, &lSourceLen, - pTmExtOut ); + pTmExtOut, endOfMemRegion); NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); if ( ! usRc ) @@ -374,133 +375,131 @@ USHORT EqfMemory::FillExtStructure ( PTMX_TARGET_RECORD pTMXTargetRecord, //ptr to tm target PTMX_TARGET_CLB pTargetClb, // ptr to current target CLB - std::wstring& pSourceString, //ptr to source string + std::wstring& pSourceString, //ptr to source string PLONG plSourceLen, //length of source string - PTMX_EXT_OUT_W pstExt //extout ext struct + PTMX_EXT_OUT_W pstExt, //extout ext struct + PBYTE endOfMemRegion ) { PTMX_TARGET_CLB pTMXTargetClb = NULL; //ptr to target control block PTMX_TAGTABLE_RECORD pTMXTagTableRecord = NULL; //ptr to tag table record - PBYTE pByte; //position pointer BOOL fOK = true; //success indicator USHORT usRc = NO_ERROR; //return code LONG lTargetLen = 0; //length indicator std::wstring pTargetString; //pointer to target string + PBYTE pByte = (PBYTE)pTMXTargetRecord; + if(*plSourceLen > 0) { - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usSourceTagTable; - //pTMXTagTableRecord = (PTMX_TAGTABLE_RECORD)pByte; + //else copy string + memcpy( pstExt->szSource, pSourceString.c_str(), *plSourceLen * sizeof(CHAR_W) ); + pstExt->szSource[*plSourceLen] = EOS; + } /* endif */ - if(*plSourceLen > 0) - { - //else copy string - memcpy( pstExt->szSource, pSourceString.c_str(), *plSourceLen * sizeof(CHAR_W) ); - pstExt->szSource[*plSourceLen] = EOS; - } /* endif */ + //position at target string + pByte = (PBYTE)pTMXTargetRecord; + pByte += pTMXTargetRecord->usTarget; - if ( fOK ) - { - //position at target string - pByte = (PBYTE)pTMXTargetRecord; - pByte += pTMXTargetRecord->usTarget; + //calculate length of target string + lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; + fOK = ( lTargetLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); - //calculate length of target string - lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; - fOK = ( lTargetLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); - } /* endif */ - // GQ 2008/06/18 Check if target string data is valid - if ( fOK ) + // GQ 2008/06/18 Check if target string data is valid + if ( fOK ) + { + BYTE b = *pByte; + + if ( (b != 0) && (b != BOCU_COMPRESS) ) { - BYTE b = *pByte; - - if ( (b != 0) && (b != BOCU_COMPRESS) ) - { - fOK = FALSE; // invalid compression ID - } /* endif */ + fOK = FALSE; // invalid compression ID } /* endif */ + } /* endif */ - if ( fOK ) - { - //copy target string - lTargetLen = EQFCompress2Unicode( pTargetString, pByte, lTargetLen ); - if ( lTargetLen < MAX_SEGMENT_SIZE ){ - if( lTargetLen >= 0 ) - pTargetString[lTargetLen] = EOS; - } else { - fOK = FALSE; // invalid string length - } + if ( fOK ) + { + //copy target string + lTargetLen = EQFCompress2Unicode( pTargetString, pByte, lTargetLen ); + if ( lTargetLen < MAX_SEGMENT_SIZE ){ + if( lTargetLen >= 0 ) + pTargetString[lTargetLen] = EOS; + } else { + fOK = FALSE; // invalid string length + } - //position at target tag record - pByte = (PBYTE)pTMXTargetRecord; + //position at target tag record + pByte = (PBYTE)pTMXTargetRecord; - //fill in the tag table name - strcpy(pstExt->szTagTable, "OTMXUXLF"); + //fill in the tag table name + strcpy(pstExt->szTagTable, "OTMXUXLF"); - - if(lTargetLen >= 0){ - memcpy( pstExt->szTarget, pTargetString.c_str(), lTargetLen * sizeof(CHAR_W)); - pstExt->szTarget[lTargetLen] = EOS; - } - /* endif */ + + if(lTargetLen >= 0){ + memcpy( pstExt->szTarget, pTargetString.c_str(), lTargetLen * sizeof(CHAR_W)); + pstExt->szTarget[lTargetLen] = EOS; + } + /* endif */ - //position at target control block - pTMXTargetClb = pTargetClb; + //position at target control block + pTMXTargetClb = pTargetClb; - // in organize mode preset author and file name field with default - // values as here the corresponding tables may be corrupted - if ( usAccessMode & ASD_ORGANIZE ) - { - strcpy( pstExt->szFileName, OVERFLOW_NAME ); - pstExt->szLongName[0] = EOS; - strcpy( pstExt->szAuthorName, OVERFLOW_NAME ); - } /* endif */ + // in organize mode preset author and file name field with default + // values as here the corresponding tables may be corrupted + if ( usAccessMode & ASD_ORGANIZE ) + { + strcpy( pstExt->szFileName, OVERFLOW_NAME ); + pstExt->szLongName[0] = EOS; + strcpy( pstExt->szAuthorName, OVERFLOW_NAME ); + } /* endif */ - //fill in the target file name - NTMGetNameFromID( &pTMXTargetClb->usFileId, (USHORT)FILE_KEY, - pstExt->szFileName, pstExt->szLongName ); - //use overflow name if no document name available - if ( pstExt->szFileName[0] == EOS ) - { - strcpy( pstExt->szFileName, OVERFLOW_NAME ); - } /* endif */ + //fill in the target file name + NTMGetNameFromID( &pTMXTargetClb->usFileId, (USHORT)FILE_KEY, + pstExt->szFileName, pstExt->szLongName ); + //use overflow name if no document name available + if ( pstExt->szFileName[0] == EOS ) + { + strcpy( pstExt->szFileName, OVERFLOW_NAME ); + } /* endif */ - //fill in the target author - NTMGetNameFromID( &pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, - pstExt->szAuthorName, NULL ); + //fill in the target author + NTMGetNameFromID( &pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, + pstExt->szAuthorName, NULL ); - //fill in the target language - NTMGetNameFromID( &pTMXTargetClb->usLangId, (USHORT)LANG_KEY, - pstExt->szTargetLanguage, NULL ); - + //fill in the target language + NTMGetNameFromID( &pTMXTargetClb->usLangId, (USHORT)LANG_KEY, + pstExt->szTargetLanguage, NULL ); + - //fill in the segment id - pstExt->ulSourceSegmentId = pTMXTargetClb->ulSegmId; - //state whether machine translation - pstExt->usTranslationFlag = pTMXTargetClb->bTranslationFlag ; - //fill in target time - pstExt->lTargetTime = pTMXTargetClb->lTime; + //fill in the segment id + pstExt->ulSourceSegmentId = pTMXTargetClb->ulSegmId; + //state whether machine translation + pstExt->usTranslationFlag = pTMXTargetClb->bTranslationFlag ; + //fill in target time + pstExt->lTargetTime = pTMXTargetClb->lTime; - // fill in any segment context info - if ( pTMXTargetClb->usAddDataLen >= MAX_ADD_DATA_LEN ) - { - // target CLB info seems to be corrupted - fOK = FALSE; - } - else if ( pTMXTargetClb->usAddDataLen ) - { - NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstExt->szContext); - NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstExt->szAddInfo ); - } /* endif */ - } /* endif */ - if(fOK){ - LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); - }else{ - {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<<"fOk is false";} + // fill in any segment context info + if ( pTMXTargetClb->usAddDataLen >= MAX_ADD_DATA_LEN ) + { + // target CLB info seems to be corrupted + fOK = FALSE; } + else if ( pTMXTargetClb->usAddDataLen ) + { + wchar_t buff[MAX_SEGMENT_SIZE]; + buff[0]='\0'; + NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, buff, endOfMemRegion); + pstExt->szContext = buff; + buff[0]='\0'; + NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, buff, endOfMemRegion ); + pstExt->szAddInfo = buff; + } /* endif */ } /* endif */ + if(fOK){ + LOG_AND_SET_RC(usRc, T5INFO, NO_ERROR); + }else{ + {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED)<<"fOk is false";} + } return( usRc ); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 4727bf28..6271b592 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -50,7 +50,7 @@ void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ); USHORT FillMatchTable( EqfMemory*, std::wstring&, PLONG, PTMX_TARGET_RECORD, PTMX_TARGET_CLB, PTMX_MATCH_TABLE_W, PUSHORT, BOOL, PUSHORT, PUSHORT, PUSHORT, - LONG, USHORT, PTMX_GET_W, BytesRef, USHORT, USHORT, int, int, USHORT); + LONG, USHORT, PTMX_GET_W, BytesRef, USHORT, USHORT, int, int, USHORT, PBYTE); /**********************************************************************/ /* Prototypes for Static functions... */ @@ -444,8 +444,10 @@ USHORT GetExactMatch { //compare tm record data with data passed in the get in structure T5LOG( T5INFO) << "GetExactMatch: ExactTest of record " << lKey; + + PBYTE endOfMemRegion = pTmRecord.data() + pTmRecord.size(); usRc = ExactTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pSentence, - pstMatchTable, &usMatchEntries, lKey ); + pstMatchTable, &usMatchEntries, lKey, endOfMemRegion ); T5LOG( T5INFO) << "GetExactMatch: ExactTest found " << usMatchEntries << " matching entries" ; } /* endif */ if ( usRc == NO_ERROR ) @@ -683,7 +685,8 @@ USHORT ExactTest PTMX_SENTENCE pSentence, //pointer to sentence structure PTMX_MATCH_TABLE_W pstMatchTable, //get out structure to be filled PUSHORT pusMatchEntries, //nr of entries in match entry structure - LONG lKeyNum // number of key record + LONG lKeyNum, // number of key record + PBYTE endOfMemRegion ) { PBYTE pByte; //position ptr @@ -698,7 +701,9 @@ USHORT ExactTest BOOL fStringEqual = FALSE; // indicator for string equal BOOL fEqualFound = FALSE; // tagging equal ?? USHORT usEqual = 0; - std::wstring pContextBuffer, pString; //pointer to buffer for context processing + std::wstring pString; //pointer to buffer for context processing + wchar_t pContextBuffer[MAX_SEGMENT_SIZE]; + pContextBuffer[0] = '\0'; USHORT usTargetTranslationFlag = (USHORT)-1; // translation flag of target CLB (processed) PSZ pszDocName = NULL; // document name to check for equal document names in GET_IGNORE_PATH mode @@ -706,7 +711,7 @@ USHORT ExactTest //allocate pString and buffer for context processing (when necessary) if ( pGetIn->szContext[0] || (pGetIn->pvGMOptList != NULL) ) { - pContextBuffer.reserve(MAX_SEGMENT_SIZE); + //pContextBuffer.reserve(MAX_SEGMENT_SIZE); } /* endif */ //position at beginning of source structure in tm record @@ -869,7 +874,7 @@ USHORT ExactTest USHORT usAddDataLen = NtmGetAddData( pClb, ADDDATA_ADDINFO_ID, pContextBuffer ); if ( usAddDataLen ) { - GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pContextBuffer.data() ); + GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pContextBuffer ); switch ( GobMemOpt ) { case GM_SUBSTITUTE_OPT: usTranslationFlag = TRANSLFLAG_NORMAL; break; @@ -894,7 +899,7 @@ USHORT ExactTest USHORT usContextLen = NtmGetAddData( pClb, ADDDATA_CONTEXT_ID, pContextBuffer ); if ( usContextLen != 0 ) { - usCurContextRanking = NTMCompareContext( pTmClb, pGetIn->szTagTable, pGetIn->szContext, pContextBuffer.data() ); + usCurContextRanking = NTMCompareContext( pTmClb, pGetIn->szTagTable, pGetIn->szContext, pContextBuffer ); } /* endif */ } /* endif */ @@ -1127,7 +1132,7 @@ USHORT ExactTest pusMatchEntries, &pSentence->usActVote, &pSentence->usActVote, lKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId ); + pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId, endOfMemRegion ); } else { @@ -1369,7 +1374,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct BytesRef pTagRecord, USHORT usTranslationFlag, USHORT usContextRanking, - int iWords, int iDiffs, USHORT usSrcLangId ) + int iWords, int iDiffs, USHORT usSrcLangId, PBYTE endOfMemRegion) { PSZ pSrcFileName = pGetIn->szFileName; // source file name PSZ pSrcLongName = pGetIn->szLongName; // long source file name @@ -1486,10 +1491,17 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct else { //found the right position + fFound = TRUE; + //OLD CODE ULONG ulLen = ( MAX_MATCHES - usCurrentEntry - 1 ) * sizeof(TMX_MATCH_TABLE_W); memmove( pstMatchTable+1, pstMatchTable, ulLen ); memset( pstMatchTable, 0, sizeof(TMX_MATCH_TABLE_W) ); - fFound = TRUE; + //NEW CODE + //for (int i = MAX_MATCHES - 2; i >= usCurrentEntry; --i) { + // pstMatchTable[i + 1] = pstMatchTable[i]; // uses copy assignment operator + //} + //pstMatchTable[usCurrentEntry] = TMX_MATCH_TABLE_W{}; // reset the entry + } /* endif */ } else @@ -1674,8 +1686,8 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct if ( pTMXTargetClb->usAddDataLen ) { - NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstMatchTable->szContext ); - NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstMatchTable->szAddInfo ); + NtmGetAddData( pTMXTargetClb, ADDDATA_CONTEXT_ID, pstMatchTable->szContext, endOfMemRegion ); + NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, pstMatchTable->szAddInfo, endOfMemRegion ); } /* endif */ /****************************************************************/ @@ -1799,10 +1811,11 @@ USHORT GetFuzzyMatch //compare tm record data with data passed in the get in structure usOverlaps = std::min( pMatchEntry->usMaxVotes, pSentence->usActVote ); + PBYTE endOfMemRegion = pTmRecord.data() + pTmRecord.size(); usRc = FuzzyTest( pTmClb, toTmxRecord(pTmRecord), pGetIn, pstMatchTable, &usMatchEntries, &usOverlaps, &pMatchEntry->usMaxVotes, &pSentence->usActVote, - pSentence, pMatchEntry->lKey ); + pSentence, pMatchEntry->lKey, endOfMemRegion ); if(usRc == SOURCE_LANG_DIFFERENT) { usRc = NO_ERROR; @@ -1913,7 +1926,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block PUSHORT pusTmMaxVotes, //nr of tm source triple PUSHORT pusGetMaxVotes, //nr of get source triples PTMX_SENTENCE pSentence, //ptr to sentence structure - LONG lKeyNum ) // record number + LONG lKeyNum, PBYTE endOfMemRegion ) // record number { PBYTE pByte; //position ptr PBYTE pSource; //position ptr @@ -1978,7 +1991,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block { //string are equal so apply equal test USHORT usNumMatches = *pusMatchesFound; - usRc = ExactTest( pTmClb, pTmRecord, pGetIn, pSentence, pstMatchTable, pusMatchesFound, lKeyNum ); + usRc = ExactTest( pTmClb, pTmRecord, pGetIn, pSentence, pstMatchTable, pusMatchesFound, lKeyNum, endOfMemRegion ); fStringEqual = ( *pusMatchesFound > usNumMatches ); T5LOG(T5INFO) << "FuzzyTest: After ExactTest, fStringEqual = " <bTranslationFlag == TRANSLFLAG_GLOBMEM) && (pGetIn->pvGMOptList != NULL) && pTMXTargetClb->usAddDataLen ) { USHORT usAddDataLen = 0; - std::wstring addDataOut; + //std::wstring addDataOut; + wchar_t addDataOut[MAX_SEGMENT_SIZE]{}; usAddDataLen = NtmGetAddData( pTMXTargetClb, ADDDATA_ADDINFO_ID, addDataOut); - wcsncpy(pSentence->szAddInfo.data(), addDataOut.c_str(), MAX_SEGMENT_SIZE); + wcsncpy(pSentence->szAddInfo.data(), addDataOut, MAX_SEGMENT_SIZE); if ( usAddDataLen ) { GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pSentence->pAddString.data() ); @@ -2169,7 +2183,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block pstMatchTable, &usFuzzy, true, pusMatchesFound, pusTmMaxVotes, pusOverlaps, lKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId ); + pGetIn, pSentence->pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId, endOfMemRegion ); } /* endif */ } /* endif */ //position at next target diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index b7435af0..d4b4eb22 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -90,7 +90,7 @@ USHORT TMLoopAndDelTargetClb bool isAddDataIsTheSame(OtmProposal& TmProposal,TMX_TARGET_CLB& Clb) { - std::wstring szBuff; + wchar_t szBuff[MAX_SEGMENT_SIZE]{}; USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); if(Clb.usAddDataLen != usAddDataLen) @@ -99,11 +99,11 @@ bool isAddDataIsTheSame(OtmProposal& TmProposal,TMX_TARGET_CLB& Clb) if (Clb.usAddDataLen) { NtmGetAddData( &Clb, ADDDATA_CONTEXT_ID, szBuff ); - if(wcscmp(szBuff.c_str(),TmProposal.szContext)) + if(wcscmp(szBuff,TmProposal.szContext)) return false; - szBuff.clear(); + szBuff[0]='\0'; NtmGetAddData( &Clb, ADDDATA_ADDINFO_ID, szBuff ); - if(wcscmp(szBuff.c_str(),TmProposal.szAddInfo)) + if(wcscmp(szBuff,TmProposal.szAddInfo)) return false; } return true; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 85d123ed..d71c0fe4 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -1527,13 +1527,14 @@ USHORT NtmStoreAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszNewData) } /* end of function NtmStoreAddData */ // retrieve specific data from the combined data area, returns length of retrieved data (incl. string end delimiter) -USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, std::wstring& pszBuffer) +USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, wchar_t* pszBuffer, PBYTE endOfMemRegion) { USHORT usLength = 0; if (pCLB->usAddDataLen != 0) // do we have additional data ? { - PSZ_W pData = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); + PSZ_W pAddDataStart = (PSZ_W)(((PBYTE)pCLB) + sizeof(TMX_TARGET_CLB)); + PSZ_W pData = pAddDataStart; if (NTMIsAddData(pData)) // is it additional data? { @@ -1544,7 +1545,19 @@ USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, std::wstring& pszBuf PUSHORT pDataPUShort = (PUSHORT)pData; usLength = pDataPUShort[1]; pData++; - pszBuffer.assign(pData, usLength); + //int offset = pData - pAddDataStart; + //int leftToTheEnd = endOfMemRegion - (PBYTE)pData; + //if(pCLB->usAddDataLen == 20){ + // int a =0; + //} + //pszBuffer.assign(pData, usLength); + //try { + //pszBuffer = std::wstring(pData); //, usLength); + wcsncpy(pszBuffer, pData, usLength); + // pszBuffer.assign(pData, usLength-1); + //} catch (const std::exception& e) { + // T5LOG(T5ERROR) << "NtmGetAddData: std::wstring assign failed: " << e.what() <<"; usLength=" << usLength <<"; usDataID=" << usDataID; + //} } /* endif */ } else diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 19e7269d..4e9f0d15 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -149,7 +149,8 @@ int EqfMemory::getName int iSize ) { - this->szName.assign( pszBuffer, pszBuffer+iSize ); + //this->szName.assign( pszBuffer, pszBuffer+iSize ); + szName = std::string(pszBuffer, iSize); return 0; } @@ -198,8 +199,8 @@ int EqfMemory::putProposal ) { int iRC = 0; - TMX_EXT_OUT_W TmPutOut; - memset( &TmPutOut, 0, sizeof(TMX_EXT_OUT_W) ); + TMX_EXT_OUT_W TmPutOut{}; + //memset( &TmPutOut, 0, sizeof(TMX_EXT_OUT_W) ); if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ std::string author = Proposal.szTargetAuthor; std::string source = EncodingHelper::convertToUTF8(Proposal.pInputSentence->pStrings->getOriginalStrC()); @@ -263,11 +264,8 @@ int EqfMemory::getNextProposal ) { int iRC = 0; - TMX_EXT_IN_W TmExtIn; - TMX_EXT_OUT_W TmExtOut; - - memset( &TmExtIn, 0, sizeof(TMX_EXT_IN_W) ); - memset( &TmExtOut, 0, sizeof(TMX_EXT_OUT_W) ); + TMX_EXT_IN_W TmExtIn{}; + TMX_EXT_OUT_W TmExtOut{}; TmExtIn.lTmKey = Proposal.nextInternalKey.getRecordKey();//this->ulNextKey; TmExtIn.usNextTarget = Proposal.nextInternalKey.getTargetKey();//this->usNextTarget; @@ -517,8 +515,8 @@ int EqfMemory::MatchToOtmProposal pProposal->setMarkup( pMatch->szTagTable ); pProposal->setTargetLanguage( pMatch->szTargetLanguage ); pProposal->setSourceLanguage( stTmSign.szSourceLanguage); - pProposal->setAddInfo( pMatch->szAddInfo.c_str() ); - pProposal->setContext( pMatch->szContext.c_str() ); + pProposal->setAddInfo( pMatch->szAddInfo ); + pProposal->setContext( pMatch->szContext ); pProposal->setDocName( pMatch->szLongName ); pProposal->setSegmentId( pMatch->ulSegmentId ); pProposal->setType( FlagToProposalType( pMatch->usTranslationFlag ) ); @@ -585,7 +583,7 @@ int EqfMemory::OtmProposalToGetIn { int iRC = 0; - memset( pGetIn, 0, sizeof(pGetIn) ); + *pGetIn = TMX_GET_W{}; Proposal.getSource( pGetIn->szSource, sizeof(pGetIn->szSource) ); Proposal.getAuthor( pGetIn->szAuthorName, sizeof(pGetIn->szAuthorName) ); Proposal.getMarkup( pGetIn->szTagTable, sizeof(pGetIn->szTagTable) ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp index 69735dfa..d4188ab7 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp @@ -165,10 +165,6 @@ int tryStrCpy(char* dest, const char* src, const char* def){ } - - - - bool EqfMemoryPlugin::stopPlugin( bool fForce ) { diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 371cbea1..bb170244 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -3210,7 +3210,6 @@ int UpdateEntryRequestData::execute(){ } // update the memory TMX_EXT_OUT_W TmPutOut; // ptr to TMX_EXT_OUT_W structure - memset( &TmPutOut, 0, sizeof(TMX_EXT_OUT_W) ); if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ std::string source = EncodingHelper::convertToUTF8(Data.szSource); @@ -3431,7 +3430,6 @@ int DeleteEntryRequestData::execute(){ std::string errorStr; TMX_EXT_OUT_W TmPutOut; - memset( &TmPutOut, 0, sizeof(TmPutOut) ); if(Data.nextInternalKey.getRecordKey() && Data.nextInternalKey.getTargetKey() && Data.lSegmentId){ //internalKey = true; diff --git a/source/opentm2/core/utilities/OtmProposal.cpp b/source/opentm2/core/utilities/OtmProposal.cpp index 6ea75df7..37049760 100755 --- a/source/opentm2/core/utilities/OtmProposal.cpp +++ b/source/opentm2/core/utilities/OtmProposal.cpp @@ -25,7 +25,6 @@ int CopyToBuffer(const char *pszSource, char *pszBuffer, int iBufSize ); /*! \brief Constructors */ OtmProposal::OtmProposal() { - this->clear(); fFilled = 0; }; @@ -39,13 +38,9 @@ OtmProposal::~OtmProposal() void OtmProposal::clear()//bool clearKeys) { - //if(this->pInputSentence) delete pInputSentence; pInputSentence.reset(); TMCursor currentKey, nextKey; - //if(false == clearKeys){ } - memset( this, 0, sizeof(OtmProposal) ); - eType = OtmProposal::eptUndefined; - eMatch = OtmProposal::emtUndefined; + *this = OtmProposal{}; } OtmProposal::eProposalType getMemProposalType( char *pszType ) diff --git a/source/opentm2/core/utilities/UtlMisc.cpp b/source/opentm2/core/utilities/UtlMisc.cpp index e65397d5..8bca0f6f 100755 --- a/source/opentm2/core/utilities/UtlMisc.cpp +++ b/source/opentm2/core/utilities/UtlMisc.cpp @@ -467,23 +467,6 @@ T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 74 usRC = (USHORT)GetL return( usRC ); } /* end of function UtlFindAndShowConversionError */ -// function checking a given codepage value by doing a test conversion -BOOL UtlIsValidCP( ULONG ulCP ) -{ - // do a test conversion using the supplied codepage - CHAR_W c_W[10]; - LONG lBytesLeft = 0L; - LONG lRc = 0; - ULONG ulOutPut = 0; - - ulCP = ADJUSTCP( ulCP ); - - ulOutPut = UtlDirectAnsi2UnicodeBuf( "abcdefgh", c_W, 8, ulCP, FALSE, &lRc, &lBytesLeft); - - return( (ulOutPut != 0) && (ulCP != 0) ); -} /* end of function UtlIsValidCP */ - - /*! UtlInitUtils Initialization of utilities Description: General utilities initialization routine diff --git a/source/opentm2/core/utilities/UtlRegistry.cpp b/source/opentm2/core/utilities/UtlRegistry.cpp index 003cb8b8..8880944b 100755 --- a/source/opentm2/core/utilities/UtlRegistry.cpp +++ b/source/opentm2/core/utilities/UtlRegistry.cpp @@ -28,7 +28,6 @@ int GetIntFromRegistry( PSZ pszAppl, PSZ pszKey, int iDefault ) BOOL WriteStringToRegistry( PSZ pszAppl, PSZ pszKey, PSZ pszValue ) { int res = Properties::GetInstance()->set_anyway(pszKey, pszValue); - return res == PROPERTY_NO_ERRORS; } /* end of function WriteStringToRegistry */ @@ -36,7 +35,6 @@ BOOL WriteStringToRegistry( PSZ pszAppl, PSZ pszKey, PSZ pszValue ) BOOL WriteIntToRegistry( PSZ pszAppl, PSZ pszKey, int iValue ) { int res = Properties::GetInstance()->set_anyway(pszKey, iValue); - return res == PROPERTY_NO_ERRORS; - } /* end of function WriteIntToRegistry */ + From 4c050c9b67ff81ef59ffd753c697650e2bf20ea0 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 25 Jun 2025 20:28:13 +0300 Subject: [PATCH 25/62] * updated to 0.7.4 + implemented forcedfuzzy search + added forceFuzzy param to fuzzysearch json request * fixed xml issue for some segments in tm when doing fuzzy search * code refactoring and cleanup --- include/EQF.H | 1 - include/EQFOSWIN.H | 1 - include/EQFUTPRI.H | 2 - include/requestdata.h | 1 + include/tm.h | 2 - source/CMakeLists.txt | 2 +- source/RestAPI/ProxygenHandler.cpp | 4 - source/opentm2/OTMFUNC.cpp | 31 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 678 ++++++++---------- .../core/EqfMemoryPlugin/EqfMemoryPlugin.cpp | 1 - .../opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp | 23 +- source/opentm2/core/requestdata.cpp | 3 + source/opentm2/core/utilities/EQFUTFIL.cpp | 40 -- source/opentm2/core/utilities/UtlMATVal.cpp | 3 +- source/opentm2/core/utilities/UtlString.cpp | 21 - 15 files changed, 329 insertions(+), 484 deletions(-) diff --git a/include/EQF.H b/include/EQF.H index f869d3ef..6af728f9 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -3669,7 +3669,6 @@ typedef ULONG HPRINT, *PHPRINT; PSZ UtlMakeFullPath ( PSZ, PSZ, PSZ, PSZ, PSZ ); USHORT UtlWriteFile( PSZ pszFile, ULONG ulDataLength, PVOID pData); BOOL UtlIsEqfDrive( CHAR chDrive ); - BOOL UtlGetCheckedEqfDrives( PSZ pszDrives ); char* UtlGetFnameFromPath( const char* path); PSZ UtlMakeObjName( PSZ p, PSZ drive, PSZ path, PSZ fname, PSZ ext); VOID UtlMakeFNameAndPath( PSZ, PSZ, PSZ ); diff --git a/include/EQFOSWIN.H b/include/EQFOSWIN.H index e54dfe3a..4270ad0e 100755 --- a/include/EQFOSWIN.H +++ b/include/EQFOSWIN.H @@ -130,7 +130,6 @@ HWND WinQueryWindow ( HWND, SHORT, BOOL ); /* one might probably use NULL as default, but we use our UTL-function*/ /* which stores the anchor block for PM - we store the instead instead*/ /**********************************************************************/ -#define WinQueryAnchorBlock( a ) ((HAB) UtlQueryULong( QL_HAB )) /**********************************************************************/ /* resource strings */ /**********************************************************************/ diff --git a/include/EQFUTPRI.H b/include/EQFUTPRI.H index bb5bc61b..3925f036 100755 --- a/include/EQFUTPRI.H +++ b/include/EQFUTPRI.H @@ -187,8 +187,6 @@ INT_PTR CALLBACK UTLYESTOALLDLG LPARAM mp2 // second message parameter ); -VOID UtlReplString( SHORT sID, PSZ pszString ); - /**********************************************************************/ /* Private data for the rename object dialog */ /**********************************************************************/ diff --git a/include/requestdata.h b/include/requestdata.h index 65e470a1..b780eb07 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -478,6 +478,7 @@ class FuzzySearchRequestData: public RequestData{ protected: char szDateTime[100]; char szType[100]; + BOOL fForceFuzzy = 0; int parseJSON() override ; int checkData() override ; diff --git a/include/tm.h b/include/tm.h index 61e9ea49..9c8b94fe 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1823,7 +1823,6 @@ class EqfMemory //: public TMX_CLB LONG64 lOtherTime = 0; // time for other activities LONG64 lGetFuzzyTime = 0; // time for GetFuzzy LONG64 lFuzzyOtherTime = 0; // other time spent in GetFuzzy - LONG64 lFuzzyTestTime = 0; // FuzzyTest time spent in GetFuzzy LONG64 lFuzzyGetTime = 0; // NTMGet time spent in GetFuzzy LONG64 lFuzzyFillMatchEntry = 0; // FillMatchEntry time spent in GetFuzzy LONG64 lFillMatchAllocTime = 0; // FillMatchEntry: allocation time @@ -3798,7 +3797,6 @@ class EqfMemoryPlugin : public OtmPlugin std::string supplier; std::string descrType; char szBuffer[4000]; // general purpose buffer area - char szSupportedDrives[27]; // list of supported drives }; // enum TIMED_BOOL{ diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 5b54a786..9b9a199c 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 3) +SET(APP_VERSION_MINOR 4) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/ProxygenHandler.cpp b/source/RestAPI/ProxygenHandler.cpp index 5b0f4e03..e1b6f20f 100755 --- a/source/RestAPI/ProxygenHandler.cpp +++ b/source/RestAPI/ProxygenHandler.cpp @@ -270,10 +270,6 @@ void ProxygenHandler::onError(ProxygenError /*err*/) noexcept { delete this; } - - - - //#define LOG_SEND_RESPONSE 1 void ProxygenHandler::sendResponse()noexcept{ diff --git a/source/opentm2/OTMFUNC.cpp b/source/opentm2/OTMFUNC.cpp index 20ccf7b3..cac194b4 100755 --- a/source/opentm2/OTMFUNC.cpp +++ b/source/opentm2/OTMFUNC.cpp @@ -265,13 +265,6 @@ USHORT EqfStartSession UtlSetString( QST_ORGEQFDRIVES, pPropSys->szDriveList ); UtlSetString( QST_VALIDEQFDRIVES, pPropSys->szDriveList ); - - // force a refresh of QST_VALIDEQFDRIVES string - { - CHAR szDriveList[MAX_DRIVELIST]; - UtlGetCheckedEqfDrives( szDriveList ); - } - // set fuzziness limits if ( !pPropSys->lSmallLkupFuzzLevel ) pPropSys->lSmallLkupFuzzLevel = 3300; @@ -313,13 +306,7 @@ USHORT EqfStartSession // set caller's session handle if ( usRC == NO_ERROR ) { - - T5LOG( T5DEBUG) << "==EQFSTARTSESSION== Starting plugins..." ; - - - // initialie plugins - if ( usRC == NO_ERROR ) - { + T5LOG( T5DEBUG) << "==EQFSTARTSESSION== Starting plugins..." ; std::string strPluginPath = FilesystemHelper::GetOtmDir(); usRC = InitializePlugins( (PSZ)strPluginPath.c_str() ); // add return value for P402974 @@ -333,16 +320,14 @@ USHORT EqfStartSession T5LOG(T5ERROR) << "EqfSessioStart()::usRC = ERROR_PLUGIN_EXPIRED"; usRC = ERROR_PLUGIN_EXPIRED; } - // Add end - } - T5LOG( T5INFO) << " ...Plugins have been started" ; - { - char szBuf[10]; - int i = _getpid(); - sprintf( szBuf, "%ld", i ); - T5LOG( T5INFO) << "EqfStartSession: Process ID is " << szBuf; - } + T5LOG( T5INFO) << " ...Plugins have been started" ; + + char szBuf[10]; + int i = _getpid(); + sprintf( szBuf, "%ld", i ); + T5LOG( T5INFO) << "EqfStartSession: Process ID is " << szBuf; + *phSession = (LONG)pData; } else diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 6271b592..045b8fb4 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -210,13 +210,15 @@ USHORT EqfMemory::TmtXGet { T5LOG( T5INFO) << "TmtXGet: Calling GetExactMatch" ; //get exact matches only - usRc = GetExactMatch( this, &Sentence, pTmGetIn, + if(!pTmGetIn->fForceFuzzySearch){ + usRc = GetExactMatch( this, &Sentence, pTmGetIn, pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound, pTmGetOut ); + } //if usNumMatchesFound is zero then try for fuzzies if ( (pTmGetOut->usNumMatchesFound == 0) && (usRc == NO_ERROR) ) { - T5LOG( T5INFO) << "TmtXGet: No exact matches found, trying GetFuzzyMatch" ; + T5LOG( T5INFO) << "TmtXGet: No exact matches found or forced fuzzy match, trying GetFuzzyMatch" ; usRc = GetFuzzyMatch( this, &Sentence, pTmGetIn, pTmGetOut->stMatchTable, &pTmGetOut->usNumMatchesFound ); T5LOG( T5INFO) << "TmtXGet: GetFuzzyMatch returned " << pTmGetOut->usNumMatchesFound << " matches" ; @@ -265,12 +267,12 @@ USHORT EqfMemory::TmtXGet // store indication if more matches are available if ( pTmGetOut->usNumMatchesFound > 1 ) { - SetAvailFlags( pTmGetOut, pTmGetOut->usNumMatchesFound); + SetAvailFlags( pTmGetOut, pTmGetOut->usNumMatchesFound); } // if we have more than one exact match check if we have an exact match where the source // matchs exactly the source segment data - if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && (pTmGetIn->ulParm & GET_EXACT) ) + if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && !(pTmGetIn->fForceFuzzySearch) && (pTmGetIn->ulParm & GET_EXACT) ) { PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); int iNumExactSourceMatches = 0; // number of matches having exact the same source @@ -289,7 +291,7 @@ USHORT EqfMemory::TmtXGet // if we have exactly one exact match which exact matches the source segment we can discard // the other exact matches - if ( iNumExactSourceMatches == 1 ) + if ( iNumExactSourceMatches == 1 && !pTmGetIn->fForceFuzzySearch) { for ( i = 0; i < pTmGetOut->usNumMatchesFound; i++ ) { @@ -381,7 +383,6 @@ USHORT EqfMemory::TmtXGet } /* endif */ pTmGetOut->tokens = std::move(Sentence.tokens); - return( usRc ); } @@ -773,8 +774,7 @@ USHORT ExactTest pStartTarget = (PBYTE)pTMXTargetRecord; //source strings are identical so loop through target records - while ( (usRc == NO_ERROR) && //(ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0)) ) - + while ( (usRc == NO_ERROR) && //(ulLeftTgtLen && ( RECLEN(pTMXTargetRecord) != 0)) ) ( lLeftTgtLen > 0 ) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && ( RECLEN(pTMXTargetRecord) > 0)) { BOOL fTestCLB = TRUE; @@ -804,7 +804,6 @@ USHORT ExactTest //position at source tag table record pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - fStringEqual=true; /**********************************************************/ /* if tagging record is unequal than we have some */ @@ -813,333 +812,328 @@ USHORT ExactTest /* another compare... */ /**********************************************************/ LONG lLenTmp = ulLen; // len of pString in # of w's - fStringEqual = FALSE; auto genericTagsTmSeg = std::make_unique(std::move(pString)); fStringEqual = (UtlCompIgnWhiteSpaceW( pSentence->pStrings->getNpReplStrC(), genericTagsTmSeg->getNpReplStrC(), 0 ) == 0 ); - if ( usRc == NO_ERROR ) + if (fStringEqual) { - if (fStringEqual) - { - fEqualFound = TRUE; - usEqual = EQUAL_EQUAL; - } - else - { - usEqual = EQUAL_EQUAL - 3; - } /* endif */ + fEqualFound = TRUE; + usEqual = EQUAL_EQUAL; + } + else + { + usEqual = EQUAL_EQUAL - 3; + } /* endif */ - //extract data from tm record - //target with tags(TRUE) - //compare file name and segment id in control block - //position at control block - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - pByte += pTMXTargetRecord->usClb; - pTMXTargetClb = (PTMX_TARGET_CLB)pByte; - usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; - - // loop over CLBs and look for best matching entry - LONG lLeftClbLen; // left CLB entries in CLB list - PTMX_TARGET_CLB pClb; // pointer for CLB list processing - #define SEG_DOC_AND_CONTEXT_MATCH 8 - #define DOC_AND_CONTEXT_MATCH 7 - #define CONTEXT_MATCH 6 - #define SAME_SEG_AND_DOC_MATCH 5 - #define SAME_DOC_MATCH 4 - #define MULT_DOC_MATCH 3 - #define NORMAL_MATCH 2 - #define IGNORE_MATCH 1 - SHORT sCurMatch = 0; - - // loop over all target CLBs - pClb = pTMXTargetClb; - lLeftClbLen = RECLEN(pTMXTargetRecord) - - pTMXTargetRecord->usClb; - while ( ( lLeftClbLen > 0 ) && (sCurMatch < SAME_SEG_AND_DOC_MATCH) ) - { - //T5LOG(T5TRANSACTION) << ":: lLeftClbLen = " << lLeftClbLen << "; sCurMatch = " << sCurMatch; - USHORT usTranslationFlag = pClb->bTranslationFlag; - USHORT usCurContextRanking = 0; // context ranking of this match - BOOL fIgnoreProposal = FALSE; + //extract data from tm record + //target with tags(TRUE) + //compare file name and segment id in control block + //position at control block + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + pByte += pTMXTargetRecord->usClb; + pTMXTargetClb = (PTMX_TARGET_CLB)pByte; + usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; + + // loop over CLBs and look for best matching entry + LONG lLeftClbLen; // left CLB entries in CLB list + PTMX_TARGET_CLB pClb; // pointer for CLB list processing + #define SEG_DOC_AND_CONTEXT_MATCH 8 + #define DOC_AND_CONTEXT_MATCH 7 + #define CONTEXT_MATCH 6 + #define SAME_SEG_AND_DOC_MATCH 5 + #define SAME_DOC_MATCH 4 + #define MULT_DOC_MATCH 3 + #define NORMAL_MATCH 2 + #define IGNORE_MATCH 1 + SHORT sCurMatch = 0; + + // loop over all target CLBs + pClb = pTMXTargetClb; + lLeftClbLen = RECLEN(pTMXTargetRecord) - + pTMXTargetRecord->usClb; + while ( ( lLeftClbLen > 0 ) && (sCurMatch < SAME_SEG_AND_DOC_MATCH) ) + { + //T5LOG(T5TRANSACTION) << ":: lLeftClbLen = " << lLeftClbLen << "; sCurMatch = " << sCurMatch; + USHORT usTranslationFlag = pClb->bTranslationFlag; + USHORT usCurContextRanking = 0; // context ranking of this match + BOOL fIgnoreProposal = FALSE; - // apply global memory option file on global memory proposals - if ( pClb->bTranslationFlag == TRANSLFLAG_GLOBMEM ) + // apply global memory option file on global memory proposals + if ( pClb->bTranslationFlag == TRANSLFLAG_GLOBMEM ) + { + if ( (pGetIn->pvGMOptList != NULL) && pClb->usAddDataLen ) { - if ( (pGetIn->pvGMOptList != NULL) && pClb->usAddDataLen ) + USHORT usAddDataLen = NtmGetAddData( pClb, ADDDATA_ADDINFO_ID, pContextBuffer ); + if ( usAddDataLen ) { - USHORT usAddDataLen = NtmGetAddData( pClb, ADDDATA_ADDINFO_ID, pContextBuffer ); - if ( usAddDataLen ) + GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pContextBuffer ); + switch ( GobMemOpt ) { - GMMEMOPT GobMemOpt = GlobMemGetFlagForProposal( pGetIn->pvGMOptList, pContextBuffer ); - switch ( GobMemOpt ) - { - case GM_SUBSTITUTE_OPT: usTranslationFlag = TRANSLFLAG_NORMAL; break; - case GM_HFLAG_OPT : usTranslationFlag = TRANSLFLAG_GLOBMEM; break; - case GM_HFLAGSTAR_OPT : usTranslationFlag = TRANSLFLAG_GLOBMEMSTAR; break; - case GM_EXCLUDE_OPT : fIgnoreProposal = TRUE; break; - } /* endswitch */ - } /* endif */ - } /* endif */ - if ( pClb == pTMXTargetClb ) - { - usTargetTranslationFlag = usTranslationFlag; - } /* endif */ + case GM_SUBSTITUTE_OPT: usTranslationFlag = TRANSLFLAG_NORMAL; break; + case GM_HFLAG_OPT : usTranslationFlag = TRANSLFLAG_GLOBMEM; break; + case GM_HFLAGSTAR_OPT : usTranslationFlag = TRANSLFLAG_GLOBMEMSTAR; break; + case GM_EXCLUDE_OPT : fIgnoreProposal = TRUE; break; + } /* endswitch */ + } /* endif */ } /* endif */ - - // check context strings (if any) - if ( - (!fIgnoreProposal) - && pGetIn->szContext[0] - && pClb->usAddDataLen ) + if ( pClb == pTMXTargetClb ) { - USHORT usContextLen = NtmGetAddData( pClb, ADDDATA_CONTEXT_ID, pContextBuffer ); - if ( usContextLen != 0 ) - { - usCurContextRanking = NTMCompareContext( pTmClb, pGetIn->szTagTable, pGetIn->szContext, pContextBuffer ); - } /* endif */ - } /* endif */ + usTargetTranslationFlag = usTranslationFlag; + } /* endif */ + } /* endif */ + + // check context strings (if any) + if ( + (!fIgnoreProposal) + && pGetIn->szContext[0] + && pClb->usAddDataLen ) + { + USHORT usContextLen = NtmGetAddData( pClb, ADDDATA_CONTEXT_ID, pContextBuffer ); + if ( usContextLen != 0 ) + { + usCurContextRanking = NTMCompareContext( pTmClb, pGetIn->szTagTable, pGetIn->szContext, pContextBuffer ); + } /* endif */ + } /* endif */ - // check for matching document names - // we can compare the document name IDs - fMatchingDocName = ((pClb->usFileId == usGetFile) || (pClb->usFileId == usAlternateGetFile)); + // check for matching document names + // we can compare the document name IDs + fMatchingDocName = ((pClb->usFileId == usGetFile) || (pClb->usFileId == usAlternateGetFile)); - if ( fIgnoreProposal ) + if ( fIgnoreProposal ) + { + if ( sCurMatch == 0 ) { - if ( sCurMatch == 0 ) - { - sCurMatch = IGNORE_MATCH; - } /* endif */ - } - else if ( usCurContextRanking == 100 ) + sCurMatch = IGNORE_MATCH; + } /* endif */ + } + else if ( usCurContextRanking == 100 ) + { + if ( fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)) ) { - if ( fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)) ) + if ( sCurMatch < SEG_DOC_AND_CONTEXT_MATCH ) { - if ( sCurMatch < SEG_DOC_AND_CONTEXT_MATCH ) - { - sCurMatch = SEG_DOC_AND_CONTEXT_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } + sCurMatch = SEG_DOC_AND_CONTEXT_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; } - else if ( fMatchingDocName ) + } + else if ( fMatchingDocName ) + { + if ( sCurMatch < DOC_AND_CONTEXT_MATCH ) { - if ( sCurMatch < DOC_AND_CONTEXT_MATCH ) - { - sCurMatch = DOC_AND_CONTEXT_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - else if ( sCurMatch == DOC_AND_CONTEXT_MATCH ) - { - // we have already a match of this type so check if context ranking - if ( usCurContextRanking > usContextRanking ) - { - pTMXTargetClb = pClb; // use newer target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - // use time info to ensure that latest match is used - else if ( usCurContextRanking == usContextRanking ) - { - // GQ 2015-04-10 New approach: If we have an exact-exact match use this one, otherwise use timestamp for the comparism - BOOL fExactExactNewCLB = fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); - BOOL fExactExactExistingCLB = ((pTMXTargetClb->usFileId == usGetFile) || (pTMXTargetClb->usFileId == usAlternateGetFile)) && - (pTMXTargetClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pTMXTargetClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); - if ( fExactExactNewCLB && !fExactExactExistingCLB ) - { - // use exact-exact CLB for match - pTMXTargetClb = pClb; - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - else if ( (fExactExactNewCLB == fExactExactExistingCLB) && (pClb->lTime > pTMXTargetClb->lTime) ) - { - // use newer target CLB for match - pTMXTargetClb = pClb; - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } - } /* endif */ - } /* endif */ + sCurMatch = DOC_AND_CONTEXT_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; } - else + else if ( sCurMatch == DOC_AND_CONTEXT_MATCH ) { - if ( sCurMatch < CONTEXT_MATCH ) + // we have already a match of this type so check if context ranking + if ( usCurContextRanking > usContextRanking ) { - sCurMatch = CONTEXT_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match + pTMXTargetClb = pClb; // use newer target CLB for match usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; - } - else if ( sCurMatch == CONTEXT_MATCH ) + } + // use time info to ensure that latest match is used + else if ( usCurContextRanking == usContextRanking ) { - // we have already a match of this type so check if context ranking - if ( usCurContextRanking > usContextRanking ) + // GQ 2015-04-10 New approach: If we have an exact-exact match use this one, otherwise use timestamp for the comparism + BOOL fExactExactNewCLB = fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); + BOOL fExactExactExistingCLB = ((pTMXTargetClb->usFileId == usGetFile) || (pTMXTargetClb->usFileId == usAlternateGetFile)) && + (pTMXTargetClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pTMXTargetClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); + if ( fExactExactNewCLB && !fExactExactExistingCLB ) { - pTMXTargetClb = pClb; // use newer target CLB for match + // use exact-exact CLB for match + pTMXTargetClb = pClb; usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; - } - // use time info to ensure that latest match is used - else if ( (usCurContextRanking == usContextRanking) && (pClb->lTime > pTMXTargetClb->lTime) ) + } + else if ( (fExactExactNewCLB == fExactExactExistingCLB) && (pClb->lTime > pTMXTargetClb->lTime) ) { - pTMXTargetClb = pClb; // use newer target CLB for match + // use newer target CLB for match + pTMXTargetClb = pClb; usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; - } /* endif */ + } } /* endif */ - } /* endif */ - } - else if ( fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)) ) - { - // same segment from same document available - sCurMatch = SAME_SEG_AND_DOC_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match - usContextRanking = usCurContextRanking; - usTargetTranslationFlag = usTranslationFlag; + } /* endif */ } - else if ( fMatchingDocName ) + else { - // segment from same document available - if ( sCurMatch < SAME_DOC_MATCH ) + if ( sCurMatch < CONTEXT_MATCH ) { - sCurMatch = SAME_DOC_MATCH; + sCurMatch = CONTEXT_MATCH; pTMXTargetClb = pClb; // use this target CLB for match usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; } - else if ( sCurMatch == SAME_DOC_MATCH ) + else if ( sCurMatch == CONTEXT_MATCH ) { - // we have already a match of this type so + // we have already a match of this type so check if context ranking + if ( usCurContextRanking > usContextRanking ) + { + pTMXTargetClb = pClb; // use newer target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; + } // use time info to ensure that latest match is used - if ( pClb->lTime > pTMXTargetClb->lTime ) + else if ( (usCurContextRanking == usContextRanking) && (pClb->lTime > pTMXTargetClb->lTime) ) { pTMXTargetClb = pClb; // use newer target CLB for match usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; } /* endif */ } /* endif */ + } /* endif */ + } + else if ( fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)) ) + { + // same segment from same document available + sCurMatch = SAME_SEG_AND_DOC_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usContextRanking = usCurContextRanking; + usTargetTranslationFlag = usTranslationFlag; + } + else if ( fMatchingDocName ) + { + // segment from same document available + if ( sCurMatch < SAME_DOC_MATCH ) + { + sCurMatch = SAME_DOC_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; } - else if ( pClb->bMultiple ) + else if ( sCurMatch == SAME_DOC_MATCH ) { - // multiple target segment available - if ( sCurMatch < MULT_DOC_MATCH ) + // we have already a match of this type so + // use time info to ensure that latest match is used + if ( pClb->lTime > pTMXTargetClb->lTime ) { - // no better match yet - sCurMatch = MULT_DOC_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match + pTMXTargetClb = pClb; // use newer target CLB for match usTargetTranslationFlag = usTranslationFlag; usContextRanking = usCurContextRanking; } /* endif */ - } - else if ( usTranslationFlag == TRANSLFLAG_NORMAL ) + } /* endif */ + } + else if ( pClb->bMultiple ) + { + // multiple target segment available + if ( sCurMatch < MULT_DOC_MATCH ) { - // a 'normal' memory match is available - if ( sCurMatch < NORMAL_MATCH ) - { - // no better match yet - sCurMatch = NORMAL_MATCH; - pTMXTargetClb = pClb; // use this target CLB for match - usTargetTranslationFlag = usTranslationFlag; - usContextRanking = usCurContextRanking; - } /* endif */ + // no better match yet + sCurMatch = MULT_DOC_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; } /* endif */ - - // continue with next target CLB - if ( sCurMatch < SAME_SEG_AND_DOC_MATCH ) + } + else if ( usTranslationFlag == TRANSLFLAG_NORMAL ) + { + // a 'normal' memory match is available + if ( sCurMatch < NORMAL_MATCH ) { - lLeftClbLen -= TARGETCLBLEN(pClb); - if (lLeftClbLen > 0) - { - usTgtNum++; - pClb = NEXTTARGETCLB(pClb); - } + // no better match yet + sCurMatch = NORMAL_MATCH; + pTMXTargetClb = pClb; // use this target CLB for match + usTargetTranslationFlag = usTranslationFlag; + usContextRanking = usCurContextRanking; } /* endif */ - } /* endwhile */ + } /* endif */ - BOOL fNormalMatch = (usTargetTranslationFlag == TRANSLFLAG_NORMAL) || - (usTargetTranslationFlag == TRANSLFLAG_GLOBMEM) || - (usTargetTranslationFlag == TRANSLFLAG_GLOBMEMSTAR); - switch ( sCurMatch ) + // continue with next target CLB + if ( sCurMatch < SAME_SEG_AND_DOC_MATCH ) { - case IGNORE_MATCH : - usMatchLevel = 0; - break; - case SAME_SEG_AND_DOC_MATCH : - usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; - break; - case SEG_DOC_AND_CONTEXT_MATCH : - usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; // exact-exact match with matching context - break; - case DOC_AND_CONTEXT_MATCH : - if ( usContextRanking == 100 ) - { - // GQ 2015/05/09: treat 100% context matches as normal exact matches - // usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; - usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; - } - else - { - usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; - } /* endif */ - break; - case CONTEXT_MATCH : - if ( usContextRanking == 100 ) - { - // GQ 2015/05/09: treat 100% context matches as normal exact context matches - // usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; - // GQ 2016/10/24: treat 100% context matches as normal exact matches - usMatchLevel = fNormalMatch ? usEqual : usEqual-1; - } - else - { - usMatchLevel = fNormalMatch ? usEqual : usEqual-1; - } /* endif */ - break; - case SAME_DOC_MATCH : + lLeftClbLen -= TARGETCLBLEN(pClb); + if (lLeftClbLen > 0) + { + usTgtNum++; + pClb = NEXTTARGETCLB(pClb); + } + } /* endif */ + } /* endwhile */ + + BOOL fNormalMatch = (usTargetTranslationFlag == TRANSLFLAG_NORMAL) || + (usTargetTranslationFlag == TRANSLFLAG_GLOBMEM) || + (usTargetTranslationFlag == TRANSLFLAG_GLOBMEMSTAR); + switch ( sCurMatch ) + { + case IGNORE_MATCH : + usMatchLevel = 0; + break; + case SAME_SEG_AND_DOC_MATCH : + usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; + break; + case SEG_DOC_AND_CONTEXT_MATCH : + usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; // exact-exact match with matching context + break; + case DOC_AND_CONTEXT_MATCH : + if ( usContextRanking == 100 ) + { + // GQ 2015/05/09: treat 100% context matches as normal exact matches + // usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; - break; - case MULT_DOC_MATCH : + } + else + { usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; - break; - default : + } /* endif */ + break; + case CONTEXT_MATCH : + if ( usContextRanking == 100 ) + { + // GQ 2015/05/09: treat 100% context matches as normal exact context matches + // usMatchLevel = fNormalMatch ? usEqual+2 : usEqual-1; + // GQ 2016/10/24: treat 100% context matches as normal exact matches usMatchLevel = fNormalMatch ? usEqual : usEqual-1; - break; - } /* endswitch */ - - - pByte = pStartTarget; - pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); + } + else + { + usMatchLevel = fNormalMatch ? usEqual : usEqual-1; + } /* endif */ + break; + case SAME_DOC_MATCH : + usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; + break; + case MULT_DOC_MATCH : + usMatchLevel = fNormalMatch ? usEqual+1 : usEqual-1; + break; + default : + usMatchLevel = fNormalMatch ? usEqual : usEqual-1; + break; + } /* endswitch */ + + pByte = pStartTarget; + pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); - /**********************************************************/ - /* store only exact matches... */ - /**********************************************************/ - if ( usMatchLevel >= 100 ) - { - LONG lSourceLen = ulLen; - - // use CLB translation flag is ustargetTranslationFlag is not set - if ( usTargetTranslationFlag == (USHORT)-1 ) usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; - FillMatchTable( pTmClb, pString, &lSourceLen, pTMXTargetRecord, - pTMXTargetClb, - pstMatchTable, &usMatchLevel, TRUE, - pusMatchEntries, &pSentence->usActVote, - &pSentence->usActVote, - lKeyNum, usTgtNum, - pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId, endOfMemRegion ); - } - else - { - /********************************************************/ - /* don't treat MT matches as exact ones.. */ - /********************************************************/ - } /* endif */ + /**********************************************************/ + /* store only exact matches... */ + /**********************************************************/ + if ( usMatchLevel >= 100 ) + { + LONG lSourceLen = ulLen; + + // use CLB translation flag is ustargetTranslationFlag is not set + if ( usTargetTranslationFlag == (USHORT)-1 ) usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; + FillMatchTable( pTmClb, genericTagsTmSeg->getNormStr(), &lSourceLen, pTMXTargetRecord, + pTMXTargetClb, + pstMatchTable, &usMatchLevel, TRUE, + pusMatchEntries, &pSentence->usActVote, + &pSentence->usActVote, + lKeyNum, usTgtNum, + pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId, endOfMemRegion ); + } + else + { + /********************************************************/ + /* don't treat MT matches as exact ones.. */ + /********************************************************/ } /* endif */ } /* endif */ @@ -1379,7 +1373,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct PSZ pSrcFileName = pGetIn->szFileName; // source file name PSZ pSrcLongName = pGetIn->szLongName; // long source file name PBYTE pByte; //position pointer - BOOL fOK = true; //success indicator USHORT usRc = NO_ERROR; //return code LONG lTargetLen; //length indicator LONG ulLen; //length indicator @@ -1387,7 +1380,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct USHORT usCurrentEntry; //counter BOOL fFound = FALSE; //indicates if correct pos found TMX_SUBSTPROP SubstProp; // tag substitution array - BOOL fSubstAll = FALSE; PTMX_MATCH_TABLE_W pstMatchTableStart = pstMatchTable; //point to start of match table TMManager *pFactory = TMManager::GetInstance(); SHORT sLangID = 0; @@ -1546,32 +1538,12 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct if ( fFound ) { - if ( !fOK ) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } - else { - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usSourceTagTable; - if (fSubstAll ) - { - /**************************************************************/ - /* do a substitution of the tags... */ - /**************************************************************/ - *plSourceLen = UTF16strlenCHAR( SubstProp.szPropSource ); - if( *plSourceLen ) { - memcpy( pstMatchTable->szSource, SubstProp.szPropSource, - *plSourceLen * sizeof(CHAR_W)); - } - } - else - { - // copy normalized propsource string - if(*plSourceLen > 0 ){ - memcpy( pstMatchTable->szSource, pSourceString.c_str(), *plSourceLen * sizeof(CHAR_W)); - } - } /* endif */ + pByte = (PBYTE)pTMXTargetRecord; + // copy normalized propsource string + if(*plSourceLen > 0 ){ + memcpy( pstMatchTable->szSource, pSourceString.c_str(), *plSourceLen * sizeof(CHAR_W)); + } if ( usRc == NO_ERROR ) { @@ -1586,63 +1558,11 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct lTargetLen = EQFCompress2Unicode( pString, pByte, lTargetLen ); // now ulTargetLen is # of w's //position at target tag record - pByte = (PBYTE)pTMXTargetRecord; - //pByte += pTMXTargetRecord->usTargetTagTable; - - if (fSubstAll ) - { - /**************************************************************/ - /* do a substitution of the replaced target string and */ - /* adjust match level */ - /**************************************************************/ - lTargetLen = UTF16strlenCHAR( SubstProp.szPropTarget ); - memcpy( pstMatchTable->szTarget, SubstProp.szPropTarget, - lTargetLen * sizeof(CHAR_W)); - - ulLen = UTF16strlenCHAR(SubstProp.szSource); - *plSourceLen = UTF16strlenCHAR(SubstProp.szPropSource); - if (*plSourceLen > ulLen ) - { - ulLen = *plSourceLen; - } /* endif */ - - { - BOOL fEqual; - USHORT usFuzzy = 0, usWords = 0, usDiffs = 0; - BOOL fFuzzynessOK; - -/*Q!*/ fFuzzynessOK = TMFuzzynessEx( pGetIn->szTagTable, - SubstProp.szPropSource, SubstProp.szSource, - sLangID, &usFuzzy, &usWords, &usDiffs ); - - fEqual = (usFuzzy == 100); - - - // GQ 2004/08/04: do not set match level to 100 for machine translations!! - // RJ 2004/08/16: assure that *pusMatchLevel is filled independently - // of bMT flag set or not (P020111) - if ( fEqual && (usFuzzy == 100) /*&& !pTMXTargetClb->bMT */) - { - *pusMatchLevel = (usTranslationFlag == TRANSLFLAG_MACHINE) ? 99 : 100; - } /* endif */ -#ifdef SGMLDITA_LOGGING - T5LOG(T5INFO) << "new match level is "<< *pusMatchLevel <<", fuzziness is " << usFuzzy ; -#endif - } - } - else - { - memcpy( pstMatchTable->szTarget, pString.data(), lTargetLen * sizeof(CHAR_W)); - } /* endif */ + pByte = (PBYTE)pTMXTargetRecord; + memcpy( pstMatchTable->szTarget, pString.data(), lTargetLen * sizeof(CHAR_W)); if ( usRc == NO_ERROR ) { - //position at target control block - // should already be done!!! - // pByte = (PBYTE)pTMXTargetRecord; - // pByte += pTMXTargetRecord->usClb; - // pTMXTargetClb = (PTMX_TARGET_CLB)pByte; - //fill in the target file name pTmClb->NTMGetNameFromID( &pTMXTargetClb->usFileId, (USHORT)FILE_KEY, @@ -1654,13 +1574,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct pstMatchTable->szTargetAuthor, NULL ); //fill in the markup table - { - //pTmClb->NTMGetNameFromID( - // &(((PTMX_TAGTABLE_RECORD)p)->usTagTableId), - // (USHORT)TAGTABLE_KEY, - // pstMatchTable->szTagTable, NULL ); - strcpy(pstMatchTable->szTagTable, "OTMXUXLF"); - } + strcpy(pstMatchTable->szTagTable, "OTMXUXLF"); //fill in the target and original src language pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, @@ -1741,11 +1655,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct } /* endif */ } /* endif */ - if ( usRc ) - { - ERREVENT2( FILLMATCHTABLE_LOC, ERROR_EVENT, usRc, TM_GROUP, "" ); - } /* endif */ - return( usRc ); } @@ -1776,7 +1685,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct USHORT GetFuzzyMatch ( - EqfMemory* pTmClb, //ptr to ctl block struct + EqfMemory* pTmClb, //ptr to ctl block struct PTMX_SENTENCE pSentence, //ptr to sentence structure PTMX_GET_W pGetIn, //ptr to data in get in structure PTMX_MATCH_TABLE_W pstMatchTable, //get out output structure @@ -1940,7 +1849,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block ULONG ulSourceLen = 0; //length of normalized source string USHORT usRc = NO_ERROR; //return code USHORT usFuzzy = 0, usWords = 0, usDiffs = 0; //fuzzy match value - BOOL fStringEqual; // strings are equal ??? + BOOL fStringEqual = false; // strings are equal ??? BOOL fNormStringEqual; // normalized strings are equal ??? BOOL fRespectCRLFStringEqual = 0L; @@ -1977,25 +1886,28 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block T5LOG( T5INFO) << "::FuzzyTest:" << str << "" ; } - //compare source strings - fStringEqual = ( UtlCompIgnWhiteSpaceW(pSentence->pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L - || UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(),0) == 0L ) ; - T5LOG( T5INFO) << "FuzzyTest: After String compare, fStringEqual = " <pPropString->getNormStrC(), pSentence->pStrings->getNormStrC(), 0 ) == 0L + || UtlCompIgnWhiteSpaceW(pSentence->pPropString->getGenericTagStrC(), pSentence->pStrings->getGenericTagStrC(),0) == 0L ) ; + T5LOG( T5INFO) << "FuzzyTest: After String compare, fStringEqual = " < usNumMatches ); - T5LOG(T5INFO) << "FuzzyTest: After ExactTest, fStringEqual = " < usNumMatches ); + T5LOG(T5INFO) << "FuzzyTest: After ExactTest, fStringEqual = " <") + EncodingHelper::convertToUTF8(original) + std::string(""); + std::string originalSrc = EncodingHelper::convertToUTF8(original) ; + //std::string sanitizedOriginalSrc; + //std::remove_copy_if(originalSrc.begin(), originalSrc.end(), std::back_inserter(sanitizedOriginalSrc), + // [](char c) { return c == '\0'; }); + removeTrailingNull(originalSrc); + src = std::string("") + /*sanitizedOriginalSrc*/ originalSrc + std::string(""); + T5LOG( T5DEBUG) << ":: parsing request str = \'" << src << "\'"; xercesc::MemBufInputSource req_buff((const XMLByte *)src.c_str(), src.size(), "req_buff (in memory)"); parser.parse(req_buff); - if(parser.getErrorCount()){ + if(int errCount = parser.getErrorCount()){ char buff[512]; handler.GetErrorText(buff, sizeof(buff)); T5LOG(T5ERROR) << ":: error during parsing req("<< src <<") : " << buff; @@ -1755,7 +1767,12 @@ void StringTagVariants::parseTrg(){ }else{ handler.tagReplacer.activeSegment = TARGET_SEGMENT; //handler.fCreateNormalizedStr = false;//not needed for target - trg = std::string("") + EncodingHelper::convertToUTF8(originalTarget) + std::string(""); + std::string originalTrg = EncodingHelper::convertToUTF8(originalTarget); + //std::string sanitizedOriginalTrg; + //std::remove_copy_if(originalTrg.begin(), originalTrg.end(), std::back_inserter(sanitizedOriginalTrg), + // [](char c) { return c == '\0'; }); + removeTrailingNull(originalTrg); + trg = std::string("") + /*sanitizedOriginalTrg*/ originalTrg + std::string(""); xercesc::MemBufInputSource trg_buff((const XMLByte *)trg.c_str(), trg.size(), "trg_buff (in memory)"); parser.parse(trg_buff); diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index bb170244..f4eef785 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -3540,6 +3540,7 @@ int FuzzySearchRequestData::parseJSON(){ { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, + { L"forceFuzzy", JSONFactory::INT_PARM_TYPE, &fForceFuzzy, 0}, { L"", JSONFactory::ASCII_STRING_PARM_TYPE, NULL, 0 } }; _rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); @@ -3638,6 +3639,7 @@ int FuzzySearchRequestData::execute(){ GetIn.usConvert = MEM_OUTPUT_ASIS; GetIn.usRequestedMatches = (USHORT)vProposals.size(); GetIn.ulParm = GET_EXACT; + GetIn.fForceFuzzySearch = fForceFuzzy; if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ auto str = EncodingHelper::convertToUTF8(GetIn.szSource); @@ -3710,6 +3712,7 @@ int FuzzySearchRequestData::execute(){ json_factory.addParmToJSONW( strOutputParmsW, L"NumOfFoundProposals", Data.iNumOfProposals ); json_factory.addParmToJSONW( strOutputParmsW, L"searchedSrc", Data.szSource ); json_factory.addParmToJSONW( strOutputParmsW, L"customId", Data.szCustomId ); + json_factory.addParmToJSONW( strOutputParmsW, L"forceFuzzy", fForceFuzzy ); if(FLAGS_add_tokens_to_fuzzy){ json_factory.addParmToJSONW( strOutputParmsW, L"tokens", tokens ); diff --git a/source/opentm2/core/utilities/EQFUTFIL.cpp b/source/opentm2/core/utilities/EQFUTFIL.cpp index c6145ae8..10b87445 100755 --- a/source/opentm2/core/utilities/EQFUTFIL.cpp +++ b/source/opentm2/core/utilities/EQFUTFIL.cpp @@ -721,46 +721,6 @@ BOOL UtlIsEqfDrive //| copy checked EQF drives to supplied buffer | //| return ok flag | //+----------------------------------------------------------------------------+ -BOOL UtlGetCheckedEqfDrives -( - PSZ pszDrives // ptr to buffer for EQF drives -) -{ - BOOL fOK = TRUE; // function return code - CHAR szOrgDrives[MAX_DRIVELIST]; // buffer for original EQF drive letters - PSZ pszOrgDrive = szOrgDrives; // pointer for drive string processing - CHAR szValidDrives[MAX_DRIVELIST]; // buffer for valid EQF drive letters - PSZ pszValidDrive = szValidDrives;// pointer for drive string processing - - /*******************************************************************/ - /* Get list of MAT drives (source: system properties) */ - /*******************************************************************/ - UtlQueryString( QST_ORGEQFDRIVES, szOrgDrives, sizeof(szOrgDrives) ); - - /*******************************************************************/ - /* Check each of the drives in the drive list */ - /*******************************************************************/ - while ( *pszOrgDrive) - { - if ( UtlIsEqfDrive( *pszOrgDrive ) ) - { - *pszValidDrive++ = *pszOrgDrive; // add to valid drives - } /* endif */ - pszOrgDrive++; // test next drive - } /* endwhile */ - *pszValidDrive = EOS; // terminate valid drives list - - /*******************************************************************/ - /* Set list of checked MAT drives */ - /*******************************************************************/ - UtlReplString( QST_VALIDEQFDRIVES, szValidDrives ); - if ( pszDrives ) - { - strcpy( pszDrives, szValidDrives ); - } /* endif */ - - return( fOK ); -} /* end of UtlGetCheckedEqfDrives */ //+----------------------------------------------------------------------------+ //|Internal function | diff --git a/source/opentm2/core/utilities/UtlMATVal.cpp b/source/opentm2/core/utilities/UtlMATVal.cpp index 3e6a757a..468b1147 100755 --- a/source/opentm2/core/utilities/UtlMATVal.cpp +++ b/source/opentm2/core/utilities/UtlMATVal.cpp @@ -234,8 +234,7 @@ PSZ UtlMakeEQFPath USHORT usPathID, // ID of path PSZ pszFolder // folder name or NULL ) -{ - +{ pszPath = new char[MAX_EQF_PATH]; // no path info yet auto otmDir = filesystem_get_otm_dir(); diff --git a/source/opentm2/core/utilities/UtlString.cpp b/source/opentm2/core/utilities/UtlString.cpp index a13515d6..0eab3ca5 100755 --- a/source/opentm2/core/utilities/UtlString.cpp +++ b/source/opentm2/core/utilities/UtlString.cpp @@ -921,28 +921,7 @@ T5LOG( T5DEBUG) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 79 if ( (GetOEMCP() = // store copy of string in internal buffers // endif //+----------------------------------------------------------------------------+ -VOID UtlReplString( SHORT sID, PSZ pszString ) -{ - PSZ pszBuffer; // buffer for new string value - if ( (sID > QST_FIRST) && (sID < QST_LAST) ) - { - if ( UtlAlloc( (PVOID *)&pszBuffer, 0L, - (LONG) get_max( MIN_ALLOC, strlen(pszString)+1 ), - ERROR_STORAGE ) ) - { - USHORT usTask = UtlGetTask(); - strcpy( pszBuffer, pszString ); - - if ( UtiVar[usTask].pszQueryArea[sID] ) - { - UtlAlloc( (PVOID *)&(UtiVar[usTask].pszQueryArea[sID]), 0L, 0L, NOMSG ); - } /* endif */ - - UtiVar[usTask].pszQueryArea[sID] = pszBuffer; - } /* endif */ - } /* endif */ -} //ulLen in number of CHAR_W's which are in pszUni! ULONG UtlDirectUnicode2AnsiBuf( PSZ_W pszUni, PSZ pszAnsi, ULONG ulLen, LONG lBufLen, From 77a643bc0611a49106ceb1db0a578643a7b2c3af Mon Sep 17 00:00:00 2001 From: orestdrag Date: Thu, 26 Jun 2025 12:45:04 +0300 Subject: [PATCH 26/62] * fixed compilation issue --- include/tm.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/tm.h b/include/tm.h index 9c8b94fe..6a3a5eb0 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1120,6 +1120,10 @@ inline bool operator==(const TMX_MATCHENTRY& lhs, const TMX_MATCHENTRY& rhs) { && lhs.cCount == rhs.cCount; } +inline bool operator!=(const TMX_MATCHENTRY& lhs, const TMX_MATCHENTRY& rhs) { + return !(lhs == rhs); +} + // #pragma pack(show) // #pragma pack(push, 8) // #pragma pack(8) From ffed386ef5db76755aa39d96693e37a893547411 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 1 Jul 2025 14:20:28 +0300 Subject: [PATCH 27/62] * fixed longname tables * fixed comparison for the same segments * fixed string params optimizations for some functions * code cleanup --- include/Proposal.h | 3 +- include/lowlevelotmdatastructs.h | 66 ++-- include/tm.h | 12 +- source/RestAPI/OTMMSJSONFactory.cpp | 4 +- source/RestAPI/OtmMemoryServiceWorker.cpp | 6 +- source/RestAPI/ProxygenServer.cpp | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 55 ++- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 20 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 130 +++++--- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 315 +++++++++--------- .../core/EqfMemoryPlugin/EqfMemory.cpp | 7 +- .../opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp | 12 +- source/opentm2/core/morph/OtmMorphICU.cpp | 10 - source/opentm2/core/requestdata.cpp | 28 +- source/opentm2/core/tagtable/EQFTOKEN.cpp | 61 +--- source/opentm2/core/utilities/EQFDICT.cpp | 117 +------ .../opentm2/core/utilities/EncodingHelper.cpp | 57 +--- .../opentm2/core/utilities/EncodingHelper.h | 20 +- .../core/utilities/FilesystemHelper.cpp | 30 +- .../core/utilities/LanguageFactory.cpp | 4 +- source/opentm2/core/utilities/Utility.h | 1 - 21 files changed, 368 insertions(+), 594 deletions(-) diff --git a/include/Proposal.h b/include/Proposal.h index 4724b536..c042755d 100644 --- a/include/Proposal.h +++ b/include/Proposal.h @@ -395,7 +395,8 @@ class OtmProposal char szCustomId[OTMPROPOSAL_MAXNAMELEN]{}; /*! \brief Segment number within the document from which the proposal comes from. */ - long lSegmentId{}; + long lSegmentId{}; + bool fSegmentIdWasGenerated{};//if true, for import, reorganize, update, means that id wasn't provided. /*! \brief source language. */ //std::string strSourceLanguage; diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 004ed4cd..c7e680f5 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -366,7 +366,7 @@ typedef struct _BTREEHEADER /*****************************************************************************/ typedef struct _BTREEHEADRECORD { - CHAR chEQF[7]; // The type of file + CHAR chEQF[7]{}; // The type of file // BYTE bVersion; // version flag struct { @@ -377,19 +377,19 @@ typedef struct _BTREEHEADRECORD unsigned char f16kRec : 1; // TRUE = BTREE uses 16k Recs (V3 record size) } Flags; - bool fOpen; // open flag/corruption flag - USHORT usFirstNode; // first node record - USHORT usFirstLeaf; // first leaf record - RECPARAM DataRecList[ MAX_LIST ]; // last used data records - bool fTerse; // tersing requested - BYTE chCollate[COLLATE_SIZE]; // collating sequence to use - BYTE chCaseMap[COLLATE_SIZE]; // case mapping to be used - BYTE chEntryEncode[ ENTRYENCODE_LEN ]; // significant characters - USHORT usFreeKeyBuffer; // index of buffer to use - USHORT usFreeDataBuffer; // first data buffer chain - USHORT usFirstDataBuffer; // first data buffer - ULONG ulUpdCount; // last update counter - USHORT usNextFreeRecord; // Next record to expand to + bool fOpen=false; // open flag/corruption flag + USHORT usFirstNode=0; // first node record + USHORT usFirstLeaf=0; // first leaf record + RECPARAM DataRecList[ MAX_LIST ]{}; // last used data records + bool fTerse = false; // tersing requested + BYTE chCollate[COLLATE_SIZE]{}; // collating sequence to use + BYTE chCaseMap[COLLATE_SIZE]{}; // case mapping to be used + BYTE chEntryEncode[ ENTRYENCODE_LEN ]{}; // significant characters + USHORT usFreeKeyBuffer=0; // index of buffer to use + USHORT usFreeDataBuffer=0; // first data buffer chain + USHORT usFirstDataBuffer=0; // first data buffer + ULONG ulUpdCount=0; // last update counter + USHORT usNextFreeRecord=0; // Next record to expand to } BTREEHEADRECORD, PBTREEHEADRECORD ; @@ -504,17 +504,14 @@ typedef TMX_TABLE * PTMX_TABLE; int tmxTableToBuffer(PTMX_TABLE pTable, BytesRef buff); - - typedef char *PSZ; // table entry structure for long document name table typedef struct _TMX_LONGNAME_TABLE_ENTRY { - PSZ pszLongName; // ptr to long name in buffer area - int longNameStartOffset; - //std::string pszLongName; - USHORT usId; // ID for long name + PSZ pszLongName{}; // ptr to long name in buffer area + int longNameStartOffset{}; + USHORT usId{}; // ID for long name } TMX_LONGNAME_TABLE_ENTRY, * PTMX_LONGNAME_TABLE_ENTRY; @@ -526,9 +523,9 @@ struct TMX_LONGNAME_TABLE void restoreStrPointersAfterRealloc(){ for(auto& entry: stTableEntry){//during insertion to the pszBuffer, it could be reallocated, so we need to update poiners - entry.pszLongName = (PSZ) &(pszBuffer[entry.longNameStartOffset]); + if(entry.longNameStartOffset) + entry.pszLongName = (PSZ) &(pszBuffer[entry.longNameStartOffset]); }; - } }; @@ -566,40 +563,35 @@ typedef enum _SEARCHTYPE struct BTREE { - bool fGuard =false; // write every record - //BOOL fOpen=false; // open flag bool fTerse =false; // tersing requested ushort usFirstNode=0; // file pointer of record ushort usFirstLeaf=0; // file pointer of record ushort usNextFreeRecord = 0; // Next record to expand to - RECPARAM DataRecList[ MAX_LIST ]; // last used data records - unsigned char chEntryEncode[ ENTRYENCODE_LEN]; // significant characters - unsigned char bEncodeLen[COLLATE_SIZE]; // encoding table length - unsigned char chEncodeVal[COLLATE_SIZE]; // encoding table - unsigned char chDecode[ENTRYDECODE_LEN]; // decoding table + RECPARAM DataRecList[ MAX_LIST ]{}; // last used data records + unsigned char chEntryEncode[ ENTRYENCODE_LEN]{}; // significant characters + unsigned char bEncodeLen[COLLATE_SIZE]{}; // encoding table length + unsigned char chEncodeVal[COLLATE_SIZE]{}; // encoding table + unsigned char chDecode[ENTRYDECODE_LEN]{}; // decoding table NTMVITALINFO chCollate; - unsigned char chCaseMap[COLLATE_SIZE]; // case mapping to be used ushort usFreeKeyBuffer=0; // index of buffer to use ushort usFreeDataBuffer=0; // first data buffer chain ushort usFirstDataBuffer=0; // first data buffer std::shared_ptr BTreeTempBuffer_V3; // temporary V3 buffer long lTime=0; // time of last update/open ushort usVersion=0; // version identification... - char chEQF[7]; // The type of file + char chEQF[7]{}; // The type of file char bVersion=0; // version flag ushort usOpenFlags=0; // settings used for open HFILE fpDummy=nullptr; // dummy/lock semaphore file handle - //USHORT usNumberOfLookupEntries=0; // Number of allocated lookup-table-entries ushort usNumberOfAllocatedBuffers=0; // Number of allocated buffers unsigned long ulReadRecCalls=0; // Number of calls to QDAMReadRecord char bRecSizeVersion=0; // record size version flag char sCurrentIndex=0; // current sequence array ushort usCurrentRecord=0; // current sequence record - //wchar_t chHeadTerm[HEADTERM_SIZE]; // last active head term bool fLock =false; // head term is locked unsigned long ulNum=0; bool fDictLock =false; @@ -612,15 +604,7 @@ struct BTREE BTREE(){ TempRecord = std::vector(MAXDATASIZE, 0); // Ensure proper initialization for arrays and complex types - std::fill(std::begin(chEntryEncode), std::end(chEntryEncode), 0); - std::fill(std::begin(bEncodeLen), std::end(bEncodeLen), 0); - std::fill(std::begin(chEncodeVal), std::end(chEncodeVal), 0); - std::fill(std::begin(chDecode), std::end(chDecode), 0); - std::fill(std::begin(chCaseMap), std::end(chCaseMap), 0); - //std::fill(std::begin(chCollate), std::end(chCollate), 0); - std::fill(std::begin(chEQF), std::end(chEQF), '\0'); BTreeTempBuffer_V3 = std::make_shared(); - //std::fill(std::begin(chHeadTerm), std::end(chHeadTerm), L'\0'); } diff --git a/include/tm.h b/include/tm.h index 6a3a5eb0..a894b47b 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1478,11 +1478,8 @@ class StringVariants xercesc::XMLPScanToken saxToken; public: - StringVariants(std::wstring &&w_src, std::wstring &&w_trg) - { - original = std::move(w_src); - originalTarget = std::move(w_trg); - } + StringVariants(std::wstring &&w_src, std::wstring &&w_trg): + original(std::move(w_src)), originalTarget(std::move(w_trg)) { } bool isParsed() const { return fSuccess; } std::wstring &getOriginalSrcStr() { return original; } std::wstring &getOriginalTrgStr() { return original; } @@ -2261,6 +2258,11 @@ class EqfMemory //: public TMX_CLB USHORT NTMReadLongNameTable(); USHORT NTMWriteLongNameTable(); USHORT NTMDestroyLongNameTable(); + + USHORT AddToLongNameTables(const char* pszLongName,PUSHORT pusID); + USHORT WriteLongNameTablesToTheTM(); + USHORT SortLongNameTables(); + USHORT NTMCreateLangGroupTable(); USHORT NTMAddLangGroup(PSZ pszLang, USHORT sLangID); USHORT NTMOrganizeIndexFile(); diff --git a/source/RestAPI/OTMMSJSONFactory.cpp b/source/RestAPI/OTMMSJSONFactory.cpp index b053e1de..bb47aff0 100755 --- a/source/RestAPI/OTMMSJSONFactory.cpp +++ b/source/RestAPI/OTMMSJSONFactory.cpp @@ -193,7 +193,7 @@ JSONFactory* JSONFactory::getInstance() */ std::wstring convertToWChar( const std::string& strASCII ) { - std::wstring strUTF16 = EncodingHelper::convertToWChar(strASCII.c_str()); + std::wstring strUTF16 = EncodingHelper::convertToWChar(strASCII); return strUTF16; } @@ -844,7 +844,7 @@ int JSONFactory::extractString //iUTF8Len = WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iLen, 0, 0, 0, 0 ); //std::string strUTF8( iUTF8Len, '\0' ); //WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iLen, &strUTF8[0], iUTF8Len, 0, 0 ); - std::string strUTF8 = EncodingHelper::convertToUTF8(strUTF16.c_str()); + std::string strUTF8 = EncodingHelper::convertToUTF8(strUTF16); // add UTF8 character to target string for ( int i = 0; i < strUTF8.length(); i++ ) string[iTargetPos++] = strUTF8[i]; } diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index 60e6523c..58715fa1 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -288,7 +288,7 @@ int FilesystemHelper::WriteToFile( const std::string& pszFile, const std::string return( -1); } - auto file = FilesystemHelper::OpenFile(pszFile.c_str(), "w+", false); + auto file = FilesystemHelper::OpenFile(pszFile, "w+", false); FilesystemHelper::WriteToFile(file, &Data[0], Data.size()); FilesystemHelper::CloseFile(file); @@ -344,9 +344,9 @@ void importMemoryProcess( void* pvData ) // cleanup if(pData->fDeleteTmx && T5Logger::GetInstance()->CheckLogLevel(T5DEBUG) == false){ //for DEBUG and DEVELOP modes leave file in fs std::string tempFile = pData->szInFile; - FilesystemHelper::DeleteFile( tempFile.c_str(), true ); + FilesystemHelper::DeleteFile( tempFile, true ); tempFile += ".Temp"; - FilesystemHelper::DeleteFile(tempFile.c_str(), true); + FilesystemHelper::DeleteFile(tempFile, true); } #ifdef TIME_MEASURES diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index 75d5e839..ae8e21d4 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -362,7 +362,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { char szServiceName[100] = "t5memory"; - char szOtmDirPath[255] =""; + char szOtmDirPath[255]{}; unsigned int uiPort = 4040; int iWorkerThreads = 0; unsigned int uiTimeOut = 60000; @@ -403,7 +403,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { } Properties::GetInstance()->set_anyway(KEY_SERVICE_URL, szServiceName); - Properties::GetInstance()->set_anyway(KEY_OTM_DIR, FilesystemHelper::GetOtmDir().c_str()); + Properties::GetInstance()->set_anyway(KEY_OTM_DIR, FilesystemHelper::GetOtmDir()); Properties::GetInstance()->set_anyway(KEY_ALLOWED_RAM, uiAllowedRAM);// saving in megabytes to avoid int overflow // Properties::GetInstance()->set_anyway(KEY_TRIPLES_THRESHOLD, uiThreshold); Properties::GetInstance()->set_anyway(KEY_NUM_OF_THREADS, iWorkerThreads); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index 1113776d..f3793c61 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -459,49 +459,44 @@ BTREE::EQFNTMInsert /********************************************************************/ /* if user wants that we find an appropriate key, we have to do so..*/ /********************************************************************/ - if ( !sRc ) + if ( lKey == NTMREQUESTNEWKEY ) { - if ( lKey == NTMREQUESTNEWKEY ) + /******************************************************************/ + /* find next free key and anchor new value in file ... */ + /******************************************************************/ + ULONG Key = lKey = (chCollate.lNextKey)++; + if ( Key > 0xFFFFFF ) { - /******************************************************************/ - /* find next free key and anchor new value in file ... */ - /******************************************************************/ - ULONG Key; - //lKey = *plKey = ++(pvi->ulNextKey); - Key = lKey = (chCollate.lNextKey)++; - if ( Key > 0xFFFFFF ) - { - sRc = BTREE_NUMBER_RANGE; - } - else - { - /**************************************************************/ - /* force update of header (only from time to time to avoid */ - /* too much performance degration)... */ - /**************************************************************/ - if ( (Key & 0x020) ) - { - sRc = QDAMWriteHeader(); - } /* endif */ - } /* endif */ + sRc = BTREE_NUMBER_RANGE; } else { - /****************************************************************/ - /* check if key is in valid range ... */ - /****************************************************************/ - if ( lKey > NTMSTARTKEY( this ) ) + /**************************************************************/ + /* force update of header (only from time to time to avoid */ + /* too much performance degration)... */ + /**************************************************************/ + //if ( (Key & 0x020) ) { - sRc = BTREE_NUMBER_RANGE; + sRc = QDAMWriteHeader(); } /* endif */ } /* endif */ + } + else + { + /****************************************************************/ + /* check if key is in valid range ... */ + /****************************************************************/ + if ( lKey > NTMSTARTKEY( this ) ) + { + sRc = BTREE_NUMBER_RANGE; + } /* endif */ } /* endif */ /********************************************************************/ /* call QDAMDictInsert to do the dirty work of inserting entry... */ /********************************************************************/ - - sRc = QDAMDictInsertLocal( lKey, pData); + if(!sRc) + sRc = QDAMDictInsertLocal( lKey, pData); return sRc; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index 07576871..b31ce4b7 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -205,8 +205,7 @@ USHORT EqfMemory::LoadMem() USHORT usRc1 = NO_ERROR; //return value ULONG ulLen = 0; //length indicator - { - + { usRc1 = TmBtree.EQFNTMOpen((usAccessMode) ); if ( (usRc1 == NO_ERROR) || (usRc1 == BTREE_CORRUPTED) ) { @@ -364,17 +363,11 @@ USHORT EqfMemory::LoadMem() (usRc == BTREE_CORRUPTED) || (usRc == VERSION_MISMATCH) ) { - // error in memory allocations will force end of open! - USHORT usTempRc = NTMCreateLongNameTable(); - if ( usTempRc == BTREE_READ_ERROR ){ - {LOG_AND_SET_RC_W_INFO(usTempRc, T5WARNING, BTREE_CORRUPTED)<< "if ( usTempRc == BTREE_READ_ERROR )";} - } // now read any long name table from database, if there is // none (=older TM before TOP97) write our empty one to the // Translation Memory - if ( usTempRc == NO_ERROR ) { - usTempRc = NTMReadLongNameTable(); + USHORT usTempRc = NTMReadLongNameTable(); switch ( usTempRc) { @@ -399,12 +392,13 @@ USHORT EqfMemory::LoadMem() {LOG_AND_SET_RC_W_INFO(usTempRc, T5WARNING, BTREE_CORRUPTED)<<"usTempRc = " << usTempRc;} break; } /* endswitch */ + + if ( usTempRc != NO_ERROR ) + { + usRc = usTempRc; + } /* endif */ } /* endif */ - if ( usTempRc != NO_ERROR ) - { - usRc = usTempRc; - } /* endif */ } /* endif */ } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index d4b4eb22..d82e3b18 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -189,34 +189,29 @@ USHORT EqfMemory::TmtXReplace //keep id that was send }else{ // generate new id TmProposal.setSegmentId( ++stTmSign.segmentIndex); + TmProposal.fSegmentIdWasGenerated = true; } - if ( !usRc ) - { - //build tag table path - Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); - strcat( szString, "/TABLE/"); - strcat( szString, TmProposal.szMarkup ); - strcat( szString, EXT_OF_FORMAT ); - - //tokenize source segment, resulting in norm. string and tag table record - usRc = TokenizeSource( TmProposal.pInputSentence.get(), szString, - TmProposal.szSourceLanguage); - - if ( strstr( szString, "OTMUTF8" ) ) { - strcpy( TmProposal.szMarkup, "OTMUTF8" ); - TmProposal.fMarkupChanged = TRUE ; - } - } /* endif */ + //build tag table path + Properties::GetInstance()->get_value(KEY_OTM_DIR, szString, sizeof(szString)); + strcat( szString, "/TABLE/"); + strcat( szString, TmProposal.szMarkup ); + strcat( szString, EXT_OF_FORMAT ); + + //tokenize source segment, resulting in norm. string and tag table record + usRc = TokenizeSource( TmProposal.pInputSentence.get(), szString, + TmProposal.szSourceLanguage); + + if ( strstr( szString, "OTMUTF8" ) ) { + strcpy( TmProposal.szMarkup, "OTMUTF8" ); + TmProposal.fMarkupChanged = TRUE ; + } if ( !usRc ) { HashSentence( TmProposal.pInputSentence.get() ); toTmxTagtableRecord(TmProposal.pInputSentence->pTagRecord)->usTagTableId = 1; - } /* endif */ - if ( !usRc ) - { usMatchesFound = CheckCompactArea( TmProposal.pInputSentence.get(), this ); if ( usMatchesFound == TmProposal.pInputSentence->usActVote ) //all hash triples found { @@ -372,7 +367,7 @@ VOID HashSentence } /* endif */ if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - auto str = EncodingHelper::convertToUTF8(pSentence->pStrings->getOriginalStrC()); + auto str = EncodingHelper::convertToUTF8(pSentence->pStrings->getOriginalSrcStr()); T5LOG( T5DEVELOP) <<"HashSentence:: inputString =\"" << str << "\"; count = " << usCount; } @@ -1028,6 +1023,7 @@ USHORT EqfMemory::UpdateTmIndex if ( usRc == NO_ERROR ) { lKey = (*pulVotes) & START_KEY; + pIndexRecord.clear(); usRc = InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record @@ -1079,20 +1075,20 @@ USHORT EqfMemory::UpdateTmIndex usIndexEntries = (USHORT)((ulLen - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); //// check if SID is already contained in list.. - //{ - // int i = (int)usIndexEntries; - // PULONG pulIndex = (PULONG)&(toIndexRecord(pIndexRecord)->stIndexEntry); - // while ( i ) - // { - // if ( *pulIndex == ulNewSID ) - // { - // fFound = TRUE; - // break; - // } /* endif */ - // pulIndex++; - // i--; - // } /*endwhile */ - //} + { + int i = (int)usIndexEntries; + PULONG pulIndex = (PULONG)&(toIndexRecord(pIndexRecord)->stIndexEntry); + while ( i ) + { + if ( *pulIndex == ulSidKey/*ulNewSID*/ ) + { + fFound = TRUE; + break; + } /* endif */ + pulIndex++; + i--; + } /*endwhile */ + } if ( !fFound ) { @@ -1120,6 +1116,7 @@ USHORT EqfMemory::UpdateTmIndex //update index record size toIndexRecord(pIndexRecord)->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); + pIndexRecord.resize(toIndexRecord(pIndexRecord)->usRecordLen); usRc = InBtree.EQFNTMUpdate( lKey, @@ -1147,7 +1144,6 @@ USHORT EqfMemory::UpdateTmIndex else { usIndexEntries = usIndexEntries; - } /* endif */ } /* endif */ @@ -1162,7 +1158,6 @@ USHORT EqfMemory::UpdateTmIndex } /* endfor */ } /* endif */ - return( usRc ); } @@ -1268,7 +1263,8 @@ USHORT DetermineTmRecord if ( usRc == NO_ERROR ) { lKey = (*pulVotes) & START_KEY; - memset( pIndexRecord.data(), 0, pIndexRecord.size() ); + pIndexRecord.clear(); + //memset( pIndexRecord.data(), 0, pIndexRecord.size() ); usRc = pTmClb->InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record @@ -1279,6 +1275,7 @@ USHORT DetermineTmRecord usMaxEntries = (USHORT)((pIndexRecord.size() - sizeof(USHORT)) / sizeof(TMX_INDEX_ENTRY)); pIndexEntry = &toIndexRecord(pIndexRecord)->stIndexEntry; + #ifdef OLD_CODE PULONG pulSids = ulSidStart; usPos = 0; @@ -1315,7 +1312,7 @@ USHORT DetermineTmRecord } /* endif */ } /* endif */ } /* endfor */ -#else +#elif NEW_CODE_1 // End criteria: all sentence IDs in index key or only one sentence ID left in pulSids for (j = 0; (j < usMaxEntries) && (lSids.size() > 1); j++, pIndexEntry++) { if (NTMVOTES(*pIndexEntry) == static_cast(pSentence->usActVote)) { @@ -1332,6 +1329,47 @@ USHORT DetermineTmRecord } } } +#else // new code more based on old code + //PULONG pulSids = lSidsv; + //usPos = 0; + + //end criteria are all sentence ids in index key or only one + //sentence id left in pulSids + int k = 0;//sid index; + for ( j = 0; (j < usMaxEntries) && (lSids.size() - k > 1); + j++, pIndexEntry++ ) + { + if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) + { + //before adding sentence id check if already in pulsids as the + //respective tm record need only be checked once + while ( (NTMKEY(*pIndexEntry) > lSids[k]) && (lSids.size() - k> 1) ) + { + // Move on to the next position in pulSids + lSids.erase(lSids.begin() + k); + + //remove sid from pulSids and decrease sid counter + //if ( lSids.size() > usPos ) + //{ + // ulLen = (usSid - usPos) * sizeof(ULONG); + //} + //else + //{ + // ulLen = sizeof(ULONG); + //} /* endif */ + //memmove( (PBYTE) pulSids, (PBYTE)(pulSids+1), ulLen ); + //usSid--; + } /* endwhile */ + + if ( NTMKEY(*pIndexEntry) == lSids[k] ) + { + //move on one position in pulSids + k++; + //pulSids++; + //usPos++; + } /* endif */ + } /* endif */ + } /* endfor */ #endif } else @@ -1428,6 +1466,7 @@ USHORT EqfMemory::UpdateTmRecord for(LONG lKey: lSids) { iSidCount ++; + TmRecord.clear(); usRc = TmBtree.EQFNTMGet( lKey, //tm record key TmRecord); //pointer to tm record data @@ -1554,25 +1593,25 @@ USHORT EqfMemory::ComparePutData LONG& lKey // tm key ) { - BOOL fOK = true; //success indicator + BOOL fOK = true; //success indicator BOOL fStop = FALSE; //indicates whether to leave loop or not PBYTE pByte; //position ptr PBYTE pStartTarget; //position ptr PTMX_SOURCE_RECORD pTMXSourceRecord = NULL; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord = NULL; //ptr to target record PTMX_TARGET_CLB pClb = NULL; //ptr to target control block - LONG lLen = 0; //length indicator - USHORT usNormLen = 0; //length of normalized string + LONG lLen = 0; //length indicator + USHORT usNormLen = 0; //length of normalized string std::wstring pString; USHORT usRc = NO_ERROR; //returned value from function BOOL fUpdate = FALSE; // TRUE = record has been updated - USHORT usAuthorId; // ID for author string + USHORT usAuthorId = 0; // ID for author string LONG lLeftClbLen; int delTargetKey = 0, targetKey = 0; BOOL fStringEqual; // indicate string equal - USHORT usPutLang; // language id of target language - USHORT usPutFile; // file id of new entry + USHORT usPutLang = 0; // language id of target language + USHORT usPutFile = 0; // file id of new entry //get id of target language in the put structure if (NTMGetIDFromName( TmProposal.szTargetLanguage, @@ -2372,7 +2411,6 @@ USHORT NTMAdjustAddDataInTgtCLB // position at next target record // endloop //------------------------------------------------------------------------------ - USHORT TMLoopAndDelTargetClb ( BytesRef pTmRecord, diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index d71c0fe4..cc0e892b 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -295,38 +295,40 @@ USHORT EqfMemory::NTMGetIDFromNameEx( /**************************************************************/ /* ... check first against our long name table */ /**************************************************************/ - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to entry found + PTMX_LONGNAME_TABLE_ENTRY pEntry = nullptr; // ptr to entry found TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched // prepare search entry SearchEntry.pszLongName = pszLongName; - // do the actual search - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + if(LongNames.stTableEntry.size()){ + // do the actual search + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, LongNames.stTableEntry.data(), - LongNames.stTableEntry.size(), + LongNames.stTableEntry.size()-1,// last elem is always empty sizeof(TMX_LONGNAME_TABLE_ENTRY), NTMLongNameTableComp); + } // if search fails try again using case insenstive search - if (pEntry == NULL) + if (pEntry == NULL && LongNamesCaseIgnore.stTableEntry.size()) { pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, LongNamesCaseIgnore.stTableEntry.data(), - LongNamesCaseIgnore.stTableEntry.size(), + LongNamesCaseIgnore.stTableEntry.size()-1,// last elem is always empty sizeof(TMX_LONGNAME_TABLE_ENTRY), NTMLongNameTableCompCaseIgnore); } /* endif */ - if (pEntry != NULL) + if (pEntry != nullptr) { // return ID of found entry *pusID = pEntry->usId; } else if (!(lOptions & NTMGETID_NOUPDATE_OPT)) { - ULONG ulNameLen = strlen(pszLongName) + 1; - ULONG ulAddLen = ulNameLen + sizeof(USHORT); + //ULONG ulNameLen = strlen(pszLongName) + 1; + //ULONG ulAddLen = ulNameLen + sizeof(USHORT); // add a new entry for the given short name if (usRc == NO_ERROR) @@ -336,70 +338,13 @@ USHORT EqfMemory::NTMGetIDFromNameEx( // adding the short name will lead to incorrect IDs for the long name table. // In order to circumvent this we call NTMAddNameToTable for the short name table // until the returned ID is larger than or equal to pTmClb->LongNames->ulEntries - do - { - DEBUGEVENT2(NTMGETIDFROMNAME_LOC, INFO_EVENT, 1, TM_GROUP, pszName); + //do + //{ + //DEBUGEVENT2(NTMGETIDFROMNAME_LOC, INFO_EVENT, 1, TM_GROUP, pszName); usRc = NTMAddNameToTable(pszName, usTableType, pusID); - } while ((usRc == NO_ERROR) && (*pusID < LongNames.stTableEntry.size())); /* enddo */ - } /* endif */ - - // add a new entry to the long name table - if (usRc == NO_ERROR) - { - // add new entry to buffer - if (usRc == NO_ERROR) - { - TMX_LONGNAME_TABLE_ENTRY Entry; // ptr to table entry - - LongNames.pszBuffer.push_back(*pusID); - Entry.longNameStartOffset = LongNames.pszBuffer.size();//LongNames.stTableEntry.size(); - LongNames.pszBuffer.insert(LongNames.pszBuffer.end(), pszLongName, pszLongName + strlen(pszLongName)); - LongNames.pszBuffer.push_back(0);//as USHORT separator - LongNames.pszBuffer.push_back(0); - - Entry.usId = *pusID; - // add long name to buffer and table - Entry.pszLongName = pszLongName; - - // adjust entry count and buffer used size - LongNames.stTableEntry.push_back(Entry); - LongNames.restoreStrPointersAfterRealloc(); - } /* endif */ - - // sort long name array - if (usRc == NO_ERROR && (LongNames.stTableEntry.size() > 1)) - { - qsort(LongNames.stTableEntry.data(), - LongNames.stTableEntry.size()-1, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp); - } /* endif */ - - // copy to case ignore table and sort it - if (usRc == NO_ERROR) - { - LongNamesCaseIgnore.stTableEntry = LongNames.stTableEntry; - LongNamesCaseIgnore.pszBuffer = LongNames.pszBuffer; - toUpper(LongNamesCaseIgnore.pszBuffer); - LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); - - if( LongNamesCaseIgnore.stTableEntry.size() > 1){ - qsort(LongNamesCaseIgnore.stTableEntry.data(), - LongNamesCaseIgnore.stTableEntry.size()-1, - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore); - } - } /* endif */ - - // update TM record for long names - if (usRc == NO_ERROR) - { - DEBUGEVENT2(NTMGETIDFROMNAME_LOC, INFO_EVENT, 2, TM_GROUP, pszLongName); - - usRc = TmBtree.EQFNTMUpdate(LONGNAME_KEY, - LongNames.pszBuffer); - } /* endif */ + //} while ((usRc == NO_ERROR) && (*pusID < LongNames.stTableEntry.size())); /* enddo */ } /* endif */ + } /* endif */ } else @@ -411,8 +356,8 @@ USHORT EqfMemory::NTMGetIDFromNameEx( /********************************************************************/ if(pstTMTable->stTmTableEntry.size()){ pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch(pszName, - pstTMTableEntries, - pstTMTable->stTmTableEntry.size(), + pstTMTable->stTmTableEntry.data(), + pstTMTable->stTmTableEntry.size()-1,// last elem is always empty sizeof(TMX_TABLE_ENTRY), NTMCompNames); } @@ -454,17 +399,19 @@ USHORT EqfMemory::NTMGetIDFromNameEx( SearchEntry.pszLongName = pszName; // do the actual search - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + if(LongNames.stTableEntry.size()){ + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, LongNames.stTableEntry.data(), - LongNames.stTableEntry.size(), + LongNames.stTableEntry.size()-1,// last elem is always empty sizeof(TMX_LONGNAME_TABLE_ENTRY), NTMLongNameTableComp); + } - if (pEntry != NULL) + if (pEntry != NULL && LongNamesCaseIgnore.stTableEntry.size()) { pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, LongNamesCaseIgnore.stTableEntry.data(), - LongNamesCaseIgnore.stTableEntry.size(), + LongNamesCaseIgnore.stTableEntry.size()-1,// last elem is always empty sizeof(TMX_LONGNAME_TABLE_ENTRY), NTMLongNameTableCompCaseIgnore); } /* endif */ @@ -769,8 +716,9 @@ int NTMCompNames(const void *pstTMTableEntry1, // input /********************************************************************/ /* compare the names of the passed tabele entries using strcmp */ /********************************************************************/ - return (strcmp(((PTMX_TABLE_ENTRY)pstTMTableEntry1)->szName, - ((PTMX_TABLE_ENTRY)pstTMTableEntry2)->szName)); + const char* l = ((PTMX_TABLE_ENTRY)pstTMTableEntry1)->szName; + const char* r = ((PTMX_TABLE_ENTRY)pstTMTableEntry2)->szName; + return strcmp(l,r); } /* end of function NTMCompNames */ //+----------------------------------------------------------------------------+ @@ -883,17 +831,91 @@ EqfMemory::NTMGetPointersToTable( //+----------------------------------------------------------------------------+ //|Returncode type: USHORT error return code or NO_ERROR if O.K. | // ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMCreateLongNameTable() -{ - LongNames.pszBuffer.reserve(LONGNAMETABLE_ENTRIES); - LongNames.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); - LongNamesCaseIgnore.pszBuffer.reserve(LONGNAMETABLE_ENTRIES); - LongNamesCaseIgnore.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); + +USHORT EqfMemory::SortLongNameTables() +{ + // sort long name array + if( LongNames.stTableEntry.size() > 1){ + qsort(LongNames.stTableEntry.data(), + LongNames.stTableEntry.size()-1,//last element is always empty + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableComp); + } + + if( LongNamesCaseIgnore.stTableEntry.size() > 1){ + qsort(LongNamesCaseIgnore.stTableEntry.data(), + LongNamesCaseIgnore.stTableEntry.size()-1,//last element is always empty + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableCompCaseIgnore); + } return 0; -} /* end of function NTMCreateLongNameTable */ +} + +void appendLongNameEntry( + TMX_LONGNAME_TABLE& table, + USHORT id, + const std::vector& nameData, + PSZ positionInBuffer = nullptr + ) +{ + TMX_LONGNAME_TABLE_ENTRY entry; + entry.usId = id; + + //table.pszBuffer.push_back(id); + if(nullptr == positionInBuffer){//we need to add data to the buffer + table.pszBuffer.push_back(static_cast(id & 0xFF)); // low byte + table.pszBuffer.push_back(static_cast((id >> 8) & 0xFF)); // high byte + entry.longNameStartOffset = table.pszBuffer.size(); + + table.pszBuffer.insert(table.pszBuffer.end(), nameData.begin(), nameData.end()); + table.pszBuffer.push_back(0); // null-terminator + table.pszBuffer.push_back(0); + }else{// we have data in the buffer, + entry.longNameStartOffset = positionInBuffer - (PSZ)table.pszBuffer.data(); + } + + entry.pszLongName = reinterpret_cast(table.pszBuffer.data() + entry.longNameStartOffset); + table.stTableEntry.back() = entry; + + table.stTableEntry.emplace_back(); // prepare next slot + table.restoreStrPointersAfterRealloc(); +} + +USHORT EqfMemory::WriteLongNameTablesToTheTM() +{ + USHORT usRc = TmBtree.EQFNTMUpdate(LONGNAME_KEY, + LongNames.pszBuffer); + return usRc; +} + +USHORT EqfMemory::AddToLongNameTables(const char* pszLongName, PUSHORT pusID) +{ + if(LongNames.stTableEntry.empty()) + { + LongNames.pszBuffer.clear(); + LongNames.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); + } + if(LongNamesCaseIgnore.stTableEntry.empty()) + { + LongNamesCaseIgnore.stTableEntry.clear(); + LongNamesCaseIgnore.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); + } + + *pusID = LongNames.stTableEntry.size(); + std::vector longName(reinterpret_cast(pszLongName), + reinterpret_cast(pszLongName + strlen(pszLongName))); + auto longNameCaseIgnore = toUpperVector(longName); + + appendLongNameEntry(LongNames, *pusID, longName); + appendLongNameEntry(LongNamesCaseIgnore, *pusID, longNameCaseIgnore); + + SortLongNameTables(); + WriteLongNameTablesToTheTM(); + return 0; +} //+----------------------------------------------------------------------------+ //|External function | @@ -910,25 +932,34 @@ USHORT EqfMemory::NTMCreateLongNameTable() // ----------------------------------------------------------------------------+ USHORT EqfMemory::NTMReadLongNameTable() { - USHORT usRC = NO_ERROR; // function return code + if(LongNames.stTableEntry.empty()) + { + LongNames.pszBuffer.reserve(LONGNAMETABLE_ENTRIES); + LongNames.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); + LongNames.pszBuffer.clear(); + LongNames.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); + } + if(LongNamesCaseIgnore.stTableEntry.empty()) + { + LongNamesCaseIgnore.pszBuffer.reserve(LONGNAMETABLE_ENTRIES); + LongNamesCaseIgnore.stTableEntry.reserve(LONGNAMETABLE_ENTRIES); + LongNamesCaseIgnore.pszBuffer.clear(); + LongNamesCaseIgnore.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); + } // call to obtain exact length of record LONG lKey = LONGNAME_KEY; - usRC = TmBtree.EQFNTMGet(lKey, LongNames.pszBuffer); + USHORT usRC = TmBtree.EQFNTMGet(lKey, LongNames.pszBuffer); if (usRC == NO_ERROR) { - // unterse data if table data is tersed - if (usRC == NO_ERROR) - { - PTERSEHEADER pTerseHeader = (PTERSEHEADER)LongNames.pszBuffer.data(); - std::vector pNewArea; // ptr to unterse data area + PTERSEHEADER pTerseHeader = (PTERSEHEADER)LongNames.pszBuffer.data(); + std::vector pNewArea; // ptr to unterse data area - if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) - { + if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) + { T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be deleted" ; - } /* endif */ } /* endif */ // setup pointer array for long names @@ -936,44 +967,22 @@ T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be { // fill pointer array PSZ pszTemp = (PSZ)LongNames.pszBuffer.data(); - TMX_LONGNAME_TABLE_ENTRY Entry; - while (*((PUSHORT)pszTemp) != 0) + while ( (pszTemp - (PSZ)LongNames.pszBuffer.data() < LongNames.pszBuffer.size()) && *((PUSHORT)pszTemp) != 0) { - memset(&Entry, 0, sizeof(Entry)); // get ID of string - Entry.usId = *((PUSHORT)pszTemp); + USHORT usId = *((PUSHORT)pszTemp); pszTemp += sizeof(USHORT); // skip ID - Entry.longNameStartOffset = pszTemp - (PSZ)LongNames.pszBuffer.data(); - - // set pointer to long document name - Entry.pszLongName = pszTemp; + std::vector longName(reinterpret_cast(pszTemp), + reinterpret_cast(pszTemp + strlen(pszTemp))); + auto longNameCaseIgnore = toUpperVector(longName); - // continue with next entry - LongNames.stTableEntry.push_back(Entry); - pszTemp += strlen(pszTemp) + 1; // skip long name + appendLongNameEntry(LongNames, usId, longName, pszTemp); + appendLongNameEntry(LongNamesCaseIgnore, usId, longNameCaseIgnore); + pszTemp += strlen(pszTemp) + 2; // skip long name } /* endwhile */ - // sort long name array - if( LongNames.stTableEntry.size() > 1){ - qsort(LongNames.stTableEntry.data(), - LongNames.stTableEntry.size()-1,// - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp); - } - - // make copy of long name array for case ignore search - LongNamesCaseIgnore.stTableEntry = LongNames.stTableEntry; - LongNamesCaseIgnore.pszBuffer = toUpperVector(LongNames.pszBuffer); - - LongNamesCaseIgnore.restoreStrPointersAfterRealloc(); - - if( LongNamesCaseIgnore.stTableEntry.size() > 1){ - qsort(LongNamesCaseIgnore.stTableEntry.data(), - LongNamesCaseIgnore.stTableEntry.size()-1,// - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore); - } + SortLongNameTables(); } /* endif */ } /* endif */ @@ -1059,16 +1068,6 @@ USHORT EqfMemory::NTMAddNameToTable( usRc = NTMGetPointersToTable(usTableType, &pstTMTable, &pstTMTableEntries); - //bool isEmpty = true; - //for(auto& entry: pstTMTable->stTmTableEntry){ - // if(entry.usId || entry.szName[0] != '\0'){ - // isEmpty = false; - // break; - // } - //} - //if(isEmpty){ - // pstTMTable->stTmTableEntry.resize(1); - //} if (usRc == NO_ERROR) { @@ -1079,36 +1078,36 @@ USHORT EqfMemory::NTMAddNameToTable( /* If not reallocate the table and check that the table */ /* will not exceed the maximum size of a QDAM record (32K) */ /*************************************************************/ + if(usTableType == FILE_KEY || usTableType == LONGNAME_KEY){ + AddToLongNameTables(pszName, pusID); + } TMX_TABLE_ENTRY& newEntry = pstTMTable->stTmTableEntry.back(); - *pusID = newEntry.usId = pstTMTable->stTmTableEntry.size();//++pstTMTable->ulMaxEntries; + if(*pusID == 0) *pusID = pstTMTable->stTmTableEntry.size(); + newEntry.usId = *pusID; strncpy(newEntry.szName, pszName, MAX_LANG_LENGTH - 1); pstTMTable->stTmTableEntry.push_back(TMX_TABLE_ENTRY()); + /*************************************************************/ + /* insert name and id to table and sort table */ + /*************************************************************/ + if( pstTMTable->stTmTableEntry.size() > 1){ + qsort(pstTMTable->stTmTableEntry.data(), + pstTMTable->stTmTableEntry.size()-1,//keep last entry empty + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); + } - if (usRc == NO_ERROR) + // update table record in TM QDAM file + if (usTableType != LANGGROUP_KEY) { - /*************************************************************/ - /* insert name and id to table and sort table */ - /*************************************************************/ - - if( pstTMTable->stTmTableEntry.size() > 1){ - qsort(pstTMTable->stTmTableEntry.data(), - pstTMTable->stTmTableEntry.size()-1,//keep last entry empty - sizeof(TMX_TABLE_ENTRY), - NTMCompNames); - } - - // update table record in TM QDAM file - if (usTableType != LANGGROUP_KEY) - { - std::vector pOldTable; - tmxTableToBuffer(pstTMTable, pOldTable); - usRc = TmBtree.EQFNTMUpdate((LONG)usTableType,pOldTable); - //(PBYTE)table, occupSize); - } /* endif */ + std::vector pOldTable; + tmxTableToBuffer(pstTMTable, pOldTable); + usRc = TmBtree.EQFNTMUpdate((LONG)usTableType,pOldTable); + //(PBYTE)table, occupSize); } /* endif */ } /* endif */ + } else //--- pTmClb is NULL pointer or pszName is empty { diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 4e9f0d15..f9b8c5da 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -203,7 +203,7 @@ int EqfMemory::putProposal //memset( &TmPutOut, 0, sizeof(TMX_EXT_OUT_W) ); if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ std::string author = Proposal.szTargetAuthor; - std::string source = EncodingHelper::convertToUTF8(Proposal.pInputSentence->pStrings->getOriginalStrC()); + std::string source = EncodingHelper::convertToUTF8(Proposal.pInputSentence->pStrings->getOriginalSrcStr()); T5LOG( T5INFO) <<"EqfMemory::putProposal, source = " << source << "; author = " << author; } @@ -646,13 +646,10 @@ EqfMemory::EqfMemory(const std::string& tmName){ if(FLAGS_log_tm_lifetime){ T5LOG(T5TRANSACTION) << "Creating memory " << tmName; } - + readOnlyCnt = std::make_shared(0); writeCnt = std::make_shared(0); - int rc = NTMCreateLongNameTable(); - if(rc) T5LOG(T5ERROR) << "NTMCreateLongNameTable returned non-zero, but " << rc; - TmBtree.fb.fileName = FilesystemHelper::GetTmdPath(tmName); InBtree.fb.fileName = FilesystemHelper::GetTmiPath(tmName); szName = tmName; diff --git a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp index 7e67007b..e5f9ef2c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp @@ -434,7 +434,7 @@ std::wstring TagReplacer::PrintTag(TagInfo& tag){ res += '/'; res += ">"; T5LOG( T5DEVELOP) << ":: generated tag = " << res; - return EncodingHelper::convertToWChar(res.c_str()); + return EncodingHelper::convertToWChar(res); } bool is_number(std::u16string s) @@ -2885,7 +2885,7 @@ void TMXParseHandler::endElement(const XMLCh* const name ) if( len >= startTextOffset + tagStr.length() ){ if( wcsncmp(pStartText, tagStr.c_str(), tagStr.length()) == 0 ){ - T5LOG(T5INFO) << "found inclosing tag at the start \"" << EncodingHelper::convertToUTF8(tagStr.c_str()) << "\" \"" + T5LOG(T5INFO) << "found inclosing tag at the start \"" << EncodingHelper::convertToUTF8(tagStr) << "\" \"" <<"\" - > erasing it and " << (startTextOffset) << " whitespaces"; //skip begin and end tags int newLen = len- tagStr.length() - startTextOffset; @@ -2918,7 +2918,7 @@ void TMXParseHandler::endElement(const XMLCh* const name ) wchar_t* segEnd = pEndText-tagStr.size(); if( wcsncmp(segEnd, tagStr.c_str(), tagStr.length()) == 0 ){ - T5LOG(T5INFO) << "found inclosing tag at the end \"" << EncodingHelper::convertToUTF8(tagStr.c_str()) << "\" \"" + T5LOG(T5INFO) << "found inclosing tag at the end \"" << EncodingHelper::convertToUTF8(tagStr) << "\" \"" <<"\" - > erasing it and " << (whistespacesAtTheEnd) << " whitespaces"; //skip end tags segEnd[0] = 0; @@ -2978,8 +2978,8 @@ void TMXParseHandler::endElement(const XMLCh* const name ) if( wcsncmp(pStartText, begPairTag.c_str(), begPairTag.size()) == 0 && wcsncmp (segEnd, endPairTag.c_str(), endPairTag.size()) == 0 ){ T5LOG(T5INFO) << "found paired inclosing tags \"" - << EncodingHelper::convertToUTF8(begPairTag.c_str()) << "\": \"" - << EncodingHelper::convertToUTF8(endPairTag.c_str()) + << EncodingHelper::convertToUTF8(begPairTag) << "\": \"" + << EncodingHelper::convertToUTF8(endPairTag) <<"\" - > erasing them and whitespaces: " << (whitespacesAtTheStart + whistespacesAtTheEnd); int newLen = len - inclosingTagsLenSum - whitespacesAtTheStart - whistespacesAtTheEnd; //skip begin and end tags @@ -3000,7 +3000,7 @@ void TMXParseHandler::endElement(const XMLCh* const name ) result = EncodingHelper::convertToUTF8(pBuf->szData); T5LOG( T5DEVELOP) << ":: parsed end of TMX sentence, result = \'" << result << "\' -> fSkipped inclosed tag pair = " << inclosingTagsSkipped <<"; Generated tags first = \'" << - EncodingHelper::convertToUTF8(begPairTag) << "last tag =\'" << EncodingHelper::convertToUTF8(endPairTag.c_str()), "\'"; + EncodingHelper::convertToUTF8(begPairTag) << "last tag =\'" << EncodingHelper::convertToUTF8(endPairTag), "\'"; } } diff --git a/source/opentm2/core/morph/OtmMorphICU.cpp b/source/opentm2/core/morph/OtmMorphICU.cpp index 3403dfd7..6d9fe8f5 100755 --- a/source/opentm2/core/morph/OtmMorphICU.cpp +++ b/source/opentm2/core/morph/OtmMorphICU.cpp @@ -186,18 +186,8 @@ int OtmMorphICU::tokenizeByTerm( const char* pszSection, STRINGLIST& vResult ) { std::lock_guard guard(this->MorphICUMutex); - - pWordBoundary->setText(UnicodeString(pszSection)); getResultFromIterator(pWordBoundary, vResult); - -#ifdef TOKENIZELOG - T5LOG(T5DEBUG) << "results:" ; - for (auto it = begin (vResult); it != end (vResult); ++it) - { - T5LOG(T5DEBUG) << "\"" << it->c_str()<< "\""; - } -#endif } return OtmMorph::SUCCESS_RETURN; } diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index f4eef785..cb3284fa 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -1759,7 +1759,7 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); JSONFactory::JSONPARSECONTROL parseControl[] = { //searchFilterFactory @@ -1973,9 +1973,9 @@ int TagRemplacementRequestData::execute(){ json_factory.parseJSONStop( parseHandle ); } - auto result = std::make_unique(std::move(EncodingHelper::convertToWChar(strSrcData.c_str())), - std::move(EncodingHelper::convertToWChar(strTrgData.c_str())), - std::move(EncodingHelper::convertToWChar(strReqData.c_str()))); + auto result = std::make_unique(EncodingHelper::convertToWChar(strSrcData), + EncodingHelper::convertToWChar(strTrgData), + EncodingHelper::convertToWChar(strReqData)); //auto = replaceString( &_rc_); @@ -2548,7 +2548,7 @@ int addProposalToJSONString pJsonFactory->addParmToJSONW( strJSON, L"context", Data.szContext); pJsonFactory->addParmToJSONW( strJSON, L"additionalInfo", Data.szAddInfo ); Data.getInternalKey(buff, OTMPROPOSAL_MAXSEGLEN); - pJsonFactory->addParmToJSONW( strJSON, L"internalKey", EncodingHelper::convertToWChar(buff).c_str() ); + pJsonFactory->addParmToJSONW( strJSON, L"internalKey", EncodingHelper::convertToWChar(buff) ); if(fAddFuzzyInfo){ wbuff[0] = '\0'; @@ -2907,8 +2907,6 @@ int ExportRequestData::parseJSON(){ T5LOG( T5WARNING) <<"::ExportRequestData::set new threshold for logging" << loggingThreshold; T5Logger::GetInstance()->SetLogLevel(loggingThreshold); } - //std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); - //_rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); T5Logger::GetInstance()->AddToLogBuffer(", with startingKey = \"" + sKey +"\""); return 0; @@ -3089,7 +3087,7 @@ int UpdateEntryRequestData::parseJSON(){ } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); // parse input parameters ulong recordKey = 0, targetKey = 0; @@ -3250,7 +3248,7 @@ int UpdateEntryRequestData::execute(){ std::wstring outputMessageW; addProposalToJSONString(outputMessageW, Data); - outputMessage = EncodingHelper::convertToUTF8(outputMessageW.c_str()); + outputMessage = EncodingHelper::convertToUTF8(outputMessageW); return( _rc_ ); } @@ -3263,7 +3261,7 @@ int GetEntryRequestData::parseJSON(){ return buildErrorReturn( _rc_, "Missing name of memory", BAD_REQUEST); } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); // parse input parameters Data.clear(); @@ -3336,7 +3334,7 @@ int GetEntryRequestData::execute() return buildErrorReturn(BTREE_ERR_OUT_OF_BOUNDS_OF_THE_RECORD,"BTREE_ERR_OUT_OF_BOUNDS_OF_THE_RECORD: tm needs to be reorganized"); } //json_factory.terminateJSONW( outputMessageW ); - outputMessage = EncodingHelper::convertToUTF8(outputMessageW.c_str()); + outputMessage = EncodingHelper::convertToUTF8(outputMessageW); return 0; } @@ -3347,7 +3345,7 @@ int DeleteEntryRequestData::parseJSON(){ return buildErrorReturn( _rc_, "Missing name of memory", BAD_REQUEST); } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); // parse input parameters Data.clear(); @@ -3520,7 +3518,7 @@ int FuzzySearchRequestData::parseJSON(){ } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); Data.clear(); int loggingThreshold = -1; @@ -3813,7 +3811,7 @@ int ConcordanceExtendedSearchRequestData::parseJSON(){ } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); int loggingThreshold = -1; JSONFactory::JSONPARSECONTROL parseControl[] = { @@ -4138,7 +4136,7 @@ int ConcordanceSearchRequestData::parseJSON(){ } /* endif */ // parse input parameters - std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody.c_str() ); + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); int loggingThreshold = -1; JSONFactory::JSONPARSECONTROL parseControl[] = { diff --git a/source/opentm2/core/tagtable/EQFTOKEN.cpp b/source/opentm2/core/tagtable/EQFTOKEN.cpp index 75b27bf5..5f0c1380 100755 --- a/source/opentm2/core/tagtable/EQFTOKEN.cpp +++ b/source/opentm2/core/tagtable/EQFTOKEN.cpp @@ -123,12 +123,7 @@ USHORT TAProtectedPartsInQuotedText BOOL fSpecialMode ); -USHORT TAGotoNextStartStopEntry -( - BytesRef pStartStop, - PSTARTSTOP * ppCurrent, - PULONG pulTableUsed -); + static void TATagAddWSpace ( @@ -2505,60 +2500,6 @@ USHORT TAFindQuote return(usStop); } -USHORT TAGotoNextStartStopEntry -( - BytesRef pStartStop, - STARTSTOP& current, - PULONG pulTableUsed -) -{ - //PSTARTSTOP pCurrent = *ppCurrent; - //ULONG ulTableUsed = * pulTableUsed; - - //pCurrent ++; - //ulTableUsed ++; - //if ( pStartStop.size()/sizeof(STARTSTOP) <= ulTableUsed + 1){ - // pStartStop.resize(sizeof(STARTSTOP)* (ulTableUsed + 10)); - // *ppCurrent = (PSTARTSTOP)pStartStop.data() + ulTableUsed; - //} - // set return values - //*ppCurrent = pCurrent; - //*pulTableUsed = ulTableUsed; - - // for tests only is pCurrent after add as above? - //{ PSTARTSTOP pStartStop = *ppStartStop; - // pCurrent = pStartStop + ulTableUsed; - //} - #ifdef TO_BE_REMOVED - if ( ulTableAlloc <= ulTableUsed + 1) - { - LONG lOldLen; // old length - LONG lNewLen; // new length - PSTARTSTOP pNewStartStop = *ppStartStop; - - lOldLen = ulTableAlloc * sizeof(STARTSTOP); - ulTableAlloc = ulTableAlloc + 20; - lNewLen = ulTableAlloc * sizeof(STARTSTOP); - - // realloc nec - if ( !UtlAlloc( (PVOID *)&pNewStartStop, lOldLen, - lNewLen, NOMSG )) - { - LOG_AND_SET_RC(usRC, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); - } /* endif */ - else - { - // if realloc ok, set pointers - pCurrent = pNewStartStop + ulTableUsed; - *ppStartStop = pNewStartStop; - *pulTableAlloc = ulTableAlloc; - } - } - #endif - - return ( 0 ); -} - USHORT TAProtectedPartsInQuotedText ( diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 88a522ec..eaf65eec 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -531,9 +531,6 @@ static BTREEHEADRECORD header; // Static buffer for database header record /**********************************************************************/ -//#define NTMNEXTKEY( pBT ) ((PNTMVITALINFO)(&pBT->chCollate[0]))->ulNextKey -//#define NTMSTARTKEY( pBT ) ((PNTMVITALINFO)(&pBT->chCollate[0]))->ulStartKey - /**********************************************************************/ /* 'Magic word' for record containing locked terms */ /**********************************************************************/ @@ -687,50 +684,6 @@ static STENCODEBITS stEncodeBits[16] = { {3,0}, // 000 // get difference for last comparison and return collat.diff // return difference //------------------------------------------------------------------------------ -SHORT QDAMKeyCompare -( - PVOID pvBT, // pointer to tree structure - PVOID pKey1, // pointer to first key - PVOID pKey2 // pointer to second key -) -{ - SHORT sDiff; - CHAR_W c; - PBTREE pBTIda = (PBTREE)pvBT; // pointer to tree structure - PBTREE pBT = pBTIda; - PBYTE pCollate = nullptr;// (PBYTE)pBT->chCollate; // pointer to collating sequence - PSZ_W pbKey1 = (PSZ_W) pKey1; - PSZ_W pbKey2 = (PSZ_W) pKey2; - - while ( (c = *pbKey1) != 0 ) - { - /******************************************************************/ - /* ignore the following characters during matching: '/', '-', ' ' */ - /******************************************************************/ - while ( (c = *pbKey2) == ' ' || fIsPunctuation[(BYTE)c] ) - { - pbKey2++; - } /* endwhile */ - while ( (c = *pbKey1) == ' ' || fIsPunctuation[(BYTE)c] ) - { - pbKey1++; - } /* endwhile */ -// @@ TODO: check collating sequence and fIsPunctuation - - sDiff = *(pCollate+(BYTE)c) - *(pCollate + (BYTE)*pbKey2); - if ( !sDiff && *pbKey1 && *pbKey2 ) - { - pbKey1++; - pbKey2++; - } - else - { - return ( sDiff ); - } /* endif */ - } /* endwhile */ - return ( *(pCollate + (BYTE)c) - *(pCollate + (BYTE)*pbKey2)); -} - //+----------------------------------------------------------------------------+ @@ -859,35 +812,30 @@ SHORT compKeys(const LONG& lKey1, // pointer to first key // //------------------------------------------------------------------------------ SHORT BTREE::QDAMWriteHeader(){ - SHORT sRc=0; - BTREEHEADRECORD HeadRecord; // header record - DEBUGEVENT2( QDAMWRITEHEADER_LOC, FUNCENTRY_EVENT, 0, DB_GROUP, "" ); // Write the initial record (describing the index) out to disk - memset( &HeadRecord, '\0', sizeof(BTREEHEADRECORD)); - memcpy(HeadRecord.chEQF,chEQF, sizeof(chEQF)); - HeadRecord.usFirstNode = usFirstNode; - HeadRecord.usFirstLeaf = usFirstLeaf; - HeadRecord.usFreeKeyBuffer = usFreeKeyBuffer; - HeadRecord.usFreeDataBuffer = usFreeDataBuffer; - HeadRecord.usFirstDataBuffer = usFirstDataBuffer; // first data buffer - HeadRecord.fOpen = true;//fOpen; // open flag set - memcpy( HeadRecord.DataRecList, DataRecList, MAX_LIST*sizeof(HeadRecord.DataRecList[0]) ); - HeadRecord.fTerse = (EQF_BOOL) fTerse; - memcpy( HeadRecord.chCollate, &chCollate, sizeof(chCollate) ); - memcpy( HeadRecord.chCaseMap, chCaseMap, COLLATE_SIZE ); - memcpy( HeadRecord.chEntryEncode, chEntryEncode, ENTRYENCODE_LEN ); + //memset( &HeadRecord, '\0', sizeof(BTREEHEADRECORD)); + memcpy(header.chEQF,chEQF, sizeof(chEQF)); + header.usFirstNode = usFirstNode; + header.usFirstLeaf = usFirstLeaf; + header.usFreeKeyBuffer = usFreeKeyBuffer; + header.usFreeDataBuffer = usFreeDataBuffer; + header.usFirstDataBuffer = usFirstDataBuffer; // first data buffer + header.fOpen = true;//fOpen; // open flag set + memcpy( header.DataRecList, DataRecList, MAX_LIST*sizeof(header.DataRecList[0]) ); + header.fTerse = (EQF_BOOL) fTerse; + memcpy( header.chCollate, &chCollate, sizeof(chCollate) ); + memcpy( header.chEntryEncode, chEntryEncode, ENTRYENCODE_LEN ); /********************************************************************/ /* Update usNextFreeRecord field of Header and indicate that the */ /* field is valid by assigning BTREE_V1 to the bVersion field. */ /********************************************************************/ - HeadRecord.usNextFreeRecord = usNextFreeRecord; - HeadRecord.Flags.f16kRec = TRUE; + header.usNextFreeRecord = usNextFreeRecord; // check if disk is full - sRc = fb.Write((PVOID) &HeadRecord, sizeof(BTREEHEADRECORD), FILE_BEGIN); + SHORT sRc = fb.Write((PVOID) &header, sizeof(BTREEHEADRECORD), FILE_BEGIN); return sRc; } @@ -1240,12 +1188,9 @@ SHORT BTREE::QDAMReadRecord_V3 { USHORT i; SHORT sRc = 0; // return code - BOOL fMemOK = FALSE; - //PACCESSCTRTABLEENTRY pACTEntry; DEBUGEVENT2( QDAMREADRECORD_LOC, FUNCENTRY_EVENT, usNumber, DB_GROUP, "" ); INC_READ_COUNT; - if(LookupTable.count(usNumber)) { /******************************************************************/ @@ -1265,37 +1210,6 @@ SHORT BTREE::QDAMReadRecord_V3 /* Record isn't in memory -> read it from disk */ sRc = QDAMReadRecordFromDisk_V3( usNumber, pReadBuffer, fNewRec, fSuppressTmdSizeCheck ); } /* endif */ - - /****************************************************************************/ - /* If #calls of QDAMReadRecord is greater than MAX_READREC_CALLS: */ - /* write unlocked records with access counter < MAX_READREC_CALLS to disk */ - /* (if fNeedToWrite is set) and free the allocated memory */ - /* set #calls of QDAMReadRecord = 0 */ - /* else increment #calls of QDAMReadRecord */ - /****************************************************************************/ - #ifdef TO_BE_REMOVED - if ( !sRc ) - { - if ( ulReadRecCalls >= MAX_READREC_CALLS ) - { - //for ( i=0; !sRc && (i < LookupTable.size()); i++ ) - for(auto& entry: LookupTable) - { - if ( !(entry.second->fLocked) && (entry.second->usRecordNumber != usNumber))//&& (entry.second->usNumber!=usNumber)) - { - sRc = QDAMWRecordToDisk_V3(entry.second); - } /* endif */ - } /* endfor */ - ulReadRecCalls = 0L; - } - else - { - ulReadRecCalls++; - } /* endif */ - } /* endif */ - #endif //TO_BE_REMOVED - - return ( sRc ); } @@ -4845,8 +4759,7 @@ SHORT BTREE::QDAMDictOpenLocal QDAMTerseInit( chEntryEncode ); // init compression } /* endif */ - memcpy( &chCollate, header.chCollate, /*COLLATE_SIZE*/ sizeof(chCollate) ); - memcpy( chCaseMap, header.chCaseMap, COLLATE_SIZE ); + memcpy( &chCollate, header.chCollate, /*COLLATE_SIZE*/ sizeof(chCollate) ); ASDLOG(); diff --git a/source/opentm2/core/utilities/EncodingHelper.cpp b/source/opentm2/core/utilities/EncodingHelper.cpp index b5fcb908..1b14da9f 100755 --- a/source/opentm2/core/utilities/EncodingHelper.cpp +++ b/source/opentm2/core/utilities/EncodingHelper.cpp @@ -170,33 +170,27 @@ std::wstring read_with_bom(std::istream & src) } } -std::wstring EncodingHelper::convertToUTF32(std::string inputStr ){ - std::stringstream ss; - ss<> convert; std::wstring wstr = convert.from_bytes( strUTF8String ); return wstr; } -std::u16string EncodingHelper::toUtf16(std::wstring wstr){ +std::u16string EncodingHelper::toUtf16(const std::wstring& wstr){ std::string u8str = EncodingHelper::convertToUTF8(wstr); return std::wstring_convert, char16_t>{}.from_bytes(u8str); } -std::wstring EncodingHelper::toWChar(std::u16string u16str){ +std::wstring EncodingHelper::toWChar(const std::u16string& u16str){ auto u8str = std::wstring_convert, char16_t>{}.to_bytes(u16str); - return convertToWChar(u8str.c_str()); + return convertToWChar(u8str); } @@ -204,7 +198,7 @@ std::wstring EncodingHelper::toWChar(std::u16string u16str){ // std::locale(), // new std::codecvt_utf16); -std::string EncodingHelper::toChar(std::u16string u16str){ +std::string EncodingHelper::toChar(const std::u16string& u16str){ auto u8str = std::wstring_convert, char16_t>{}.to_bytes(u16str); //auto u8str = std::wstring_convert, char16_t>{}.to_bytes(u16str); return u8str; @@ -235,47 +229,6 @@ std::string EncodingHelper::convertToUTF8( const std::u16string& strUTF16String } -/*! \brief convert a UTF8 std::string to a ASCII std::string (on spot conversion) -\param strText on input string in UTF8 encoding, on output string in ASCII encoding -\returns string converted to UTF16 -*/ -void EncodingHelper::convertUTF8ToASCII( std::string& strText ) -{ - //T5LOG( T5WARNING) <<"EncodingHelper::convertUTF8ToASCII( std::string& strText ) is not implemented, strText = ", strText.c_str()); - int iUTF16Len; - int iASCIILen; - int iUTF8Len = (int)strText.length() + 1; - char *pszNewData = NULL; - iUTF16Len = MultiByteToWideChar( CP_UTF8, 0, strText.c_str(), iUTF8Len, 0, 0 ); - std::wstring strUTF16( iUTF16Len, L'\0' ); - MultiByteToWideChar( CP_UTF8, 0, strText.c_str(), iUTF8Len, &strUTF16[0], iUTF16Len ); - iASCIILen = WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iUTF16Len, 0, 0, 0, 0 ); - pszNewData = (char *)malloc( iASCIILen + 1 ); - WideCharToMultiByte( CP_OEMCP, 0, (LPWSTR)strUTF16.c_str(), iUTF16Len, pszNewData, iASCIILen, 0, 0 ); - strText = pszNewData; - free( pszNewData ); -} - -/*! \brief convert a ASCII std::string to UTF8 std::string (on spot conversion) -\param strText on input string in ASCII encoding, on output string in UTF8 encoding -*/ -void EncodingHelper::convertASCIIToUTF8( std::string& strText ) -{ - int iUTF16Len; - int iUTF8Len; - int iASCIILen = (int)strText.length() + 1; - char *pszNewData = NULL; - iUTF16Len = MultiByteToWideChar( CP_OEMCP, 0, strText.c_str(), iASCIILen, 0, 0 ); - std::wstring strUTF16( iUTF16Len, L'\0' ); - MultiByteToWideChar( CP_OEMCP, 0, strText.c_str(), iASCIILen, &strUTF16[0], iUTF16Len ); - iUTF8Len = WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iUTF16Len, 0, 0, 0, 0 ); - pszNewData = (char *)malloc( iUTF8Len + 1); - WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iUTF16Len, pszNewData, iUTF8Len, 0, 0 ); - - strText = pszNewData; - free( pszNewData ); -} - std::u16string EncodingHelper::toLower(const std::u16string& strText){ std::u16string ldata; diff --git a/source/opentm2/core/utilities/EncodingHelper.h b/source/opentm2/core/utilities/EncodingHelper.h index 8f8477c3..f4945ecd 100755 --- a/source/opentm2/core/utilities/EncodingHelper.h +++ b/source/opentm2/core/utilities/EncodingHelper.h @@ -13,12 +13,10 @@ class EncodingHelper{ \param strUTF8String string in UTF8 encoding \returns string converted to UTF16 */ - std::wstring static convertToWChar(std::string strUTF8String); - std::u16string static toUtf16(std::wstring wstr); - std::wstring static toWChar(std::u16string u16str); - std::string static toChar(std::u16string u16str); - - std::wstring static convertToUTF32(std::string str); + std::wstring static convertToWChar(const std::string& strUTF8String); + std::u16string static toUtf16(const std::wstring& wstr); + std::wstring static toWChar(const std::u16string& u16str); + std::string static toChar(const std::u16string& u16str); /*! \brief convert a UTF8 std::string to a UTF16 std::wstring \param strUTF8String string in UTF8 encoding @@ -33,16 +31,6 @@ class EncodingHelper{ std::string static convertToUTF8( const std::u16string& strUTF16String ); std::string static convertToUTF8( const wchar_t* UTF32String); - /*! \brief convert a UTF8 std::string to a ASCII std::string (on spot conversion) - \param strText on input string in UTF8 encoding, on output string in ASCII encoding - */ - void static convertUTF8ToASCII( std::string& strText ); - - /*! \brief convert a ASCII std::string to UTF8 std::string (on spot conversion) - \param strText on input string in ASCII encoding, on output string in UTF8 encoding - */ - void static convertASCIIToUTF8( std::string& strText ); - std::u16string static toLower(const std::u16string& strText); void static Base64Encode (const unsigned char* input, int inSize, std::string& output); diff --git a/source/opentm2/core/utilities/FilesystemHelper.cpp b/source/opentm2/core/utilities/FilesystemHelper.cpp index 644c4918..f275099f 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.cpp +++ b/source/opentm2/core/utilities/FilesystemHelper.cpp @@ -304,11 +304,11 @@ int FilesystemHelper::RenameFile(std::string oldPath, std::string newPath){ errno = 0; bool fOk = true; //Copy file instead - if(FileExists(newPath.c_str())){ + if(FileExists(newPath)){ T5LOG(T5ERROR) << ":: trg file \'" << newPath << "\' already exists!"; return FILE_EXISTS; } - if(FileExists(oldPath.c_str()) == false){ + if(FileExists(oldPath) == false){ T5LOG(T5ERROR) << ":: src file \'" << oldPath << "\' is missing!"; return FILE_NOT_EXISTS; } @@ -336,11 +336,11 @@ int FilesystemHelper::MoveFile(std::string oldPath, std::string newPath){ bool fOk = true; //fOk = rename(fixedOldPath.c_str(), fixedNewPath.c_str()) != -1; //Copy file instead - if(FileExists(newPath.c_str())){ + if(FileExists(newPath)){ T5LOG(T5ERROR) << ":: trg file \'" << newPath << "\' already exists!"; return FILE_EXISTS; } - if(FileExists(oldPath.c_str()) == false){ + if(FileExists(oldPath) == false){ T5LOG(T5ERROR) << ":: src file \'" << oldPath << "\' is missing!"; return FILE_NOT_EXISTS; } @@ -349,7 +349,8 @@ int FilesystemHelper::MoveFile(std::string oldPath, std::string newPath){ DeleteFile(oldPath); //remove(oldPath); - if(!fOk){ + if(!fOk) + { T5LOG(T5ERROR) << "MoveFile:: cannot move "<< fixedOldPath << " to " << fixedNewPath << ", error = " << strerror(errno); return errno; }else{ @@ -1130,25 +1131,6 @@ int FilesystemHelper::ReadFile(const std::string& path, char* buff, return __last_error_code = errcode; } -/* -int FilesystemHelper::ReadFile(FILE*& ptr, char* buff, const int buffSize, int& bytesRead){ - int errCode = FILEHELPER_NO_ERROR; - if(!ptr){ - T5LOG(T5ERROR) <<"FilesystemHelper::ReadFile():: FILEHELPER_FILE_PTR_IS_NULL"); - errCode = FILEHELPER_FILE_PTR_IS_NULL; - }else{ - bytesRead = fread(buff, buffSize, 1, ptr); - if(!bytesRead){ - T5LOG( T5WARNING) << "FilesystemHelper::ReadFile():: bytes not readed from ", toStr((long int)ptr).c_str()); - errCode = FILEHELPER_ERROR_FILE_NOT_READ; - }else{ - T5LOG( T5DEBUG) << "FilesystemHelper::ReadFile():: readed ", toStr(bytesRead).c_str(), "; data = ", buff); - } - } - return __last_error_code = errCode; -} -//*/ - int FilesystemHelper::ReadFile(FILE*& ptr, void* buff, const int buffSize, int& bytesRead){ int errCode = FILEHELPER_NO_ERROR; diff --git a/source/opentm2/core/utilities/LanguageFactory.cpp b/source/opentm2/core/utilities/LanguageFactory.cpp index c85bfdb2..6b4e9a7e 100755 --- a/source/opentm2/core/utilities/LanguageFactory.cpp +++ b/source/opentm2/core/utilities/LanguageFactory.cpp @@ -673,7 +673,7 @@ void LangParseHandler::endElement(const XMLCh* const name ) void LangParseHandler::characters(const XMLCh* const chars, const XMLSize_t length) { - auto wstr = EncodingHelper::toWChar(std::u16string((char16_t*)chars).c_str()); + auto wstr = EncodingHelper::toWChar(std::u16string((char16_t*)chars)); // add data to current data buffer int iCurLen = wcslen( szDataW ); @@ -683,7 +683,7 @@ void LangParseHandler::characters(const XMLCh* const chars, const XMLSize_t leng szDataW[iCurLen+wstr.size()] = 0; }else{ if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - auto str = EncodingHelper::convertToUTF8(wstr.c_str()); + auto str = EncodingHelper::convertToUTF8(wstr); T5LOG(T5ERROR) << ":: (iCurLen + length + 1) >= (sizeof(szData)/sizeof(szData[0])), characters = " << str; } } /* endif */ diff --git a/source/opentm2/core/utilities/Utility.h b/source/opentm2/core/utilities/Utility.h index 140c2167..88d6dc2c 100755 --- a/source/opentm2/core/utilities/Utility.h +++ b/source/opentm2/core/utilities/Utility.h @@ -5,7 +5,6 @@ //| Corporation and others. All rights reserved | //+----------------------------------------------------------------------------+ #define MAX_MEM_TABLES 8 // max. no of segment tables -#define MAX_MEM_SEGMENTS 8162 // max. no of segments for UtlAlloc #define SEG_TABLE_EMPTY (-1) // there are no segments in the table #define DATEFORMATSTRING_YMD "%4.4d%c%2.2d%c%2.2d" From 983d04daa416b283d0b4ff2e5f73300028e706d1 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 2 Jul 2025 15:14:53 +0300 Subject: [PATCH 28/62] * code refactoring --- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 7 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 118 +++++++++++------- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 12 +- source/opentm2/core/ProposalFilter.cpp | 2 - 4 files changed, 80 insertions(+), 59 deletions(-) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 045b8fb4..aef054e2 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -49,7 +49,7 @@ void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ); USHORT FillMatchTable( EqfMemory*, std::wstring&, PLONG, PTMX_TARGET_RECORD, PTMX_TARGET_CLB, - PTMX_MATCH_TABLE_W, PUSHORT, BOOL, PUSHORT, PUSHORT, PUSHORT, + PTMX_MATCH_TABLE_W, PUSHORT, BOOL, PUSHORT, PUSHORT, LONG, USHORT, PTMX_GET_W, BytesRef, USHORT, USHORT, int, int, USHORT, PBYTE); /**********************************************************************/ @@ -1124,7 +1124,7 @@ USHORT ExactTest FillMatchTable( pTmClb, genericTagsTmSeg->getNormStr(), &lSourceLen, pTMXTargetRecord, pTMXTargetClb, pstMatchTable, &usMatchLevel, TRUE, - pusMatchEntries, &pSentence->usActVote, + pusMatchEntries, &pSentence->usActVote, lKeyNum, usTgtNum, pGetIn, pSentence->pTagRecord, usTargetTranslationFlag, usContextRanking, -1, -1, pTMXSourceRecord->usLangId, endOfMemRegion ); @@ -1360,7 +1360,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct PUSHORT pusMatchLevel, //how good is match BOOL fTag, //target with/without subst. tags PUSHORT pusMatchEntries, //nr of entries in match table - PUSHORT pusMaxVotes, //nr of tm source string votes PUSHORT pusOverlaps, //nr of overlapping triples LONG lKeyNum, // key number USHORT usTgtNum, // target number @@ -2093,7 +2092,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block FillMatchTable( pTmClb, pString, &lTempSrcLen, pTMXTargetRecord, pTMXTargetClb, pstMatchTable, &usFuzzy, true, - pusMatchesFound, pusTmMaxVotes, pusOverlaps, + pusMatchesFound, pusOverlaps, lKeyNum, usTgtNum, pGetIn, pSentence->pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId, endOfMemRegion ); } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index d82e3b18..c322ad0a 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -209,7 +209,24 @@ USHORT EqfMemory::TmtXReplace if ( !usRc ) { + // #define TEST_VOTES + #ifdef TEST_VOTES + if(TmProposal.pInputSentence->pStrings->getGenericTagsString().find(L"2 x Bero 90")!= std::wstring::npos){ + int a = 0;//for breakpoint + } + #endif// TEST_VOTES HashSentence( TmProposal.pInputSentence.get() ); + #ifdef TEST_VOTES + //if(TmProposal.pInputSentence->pStrings->getGenericTagsString().find(L"2 x Bero 90")!= std::wstring::npos) + { + int a = 0; + std::string output = std::to_string(TmProposal.pInputSentence->usActVote) +"; votes = "; + for(auto vote: TmProposal.pInputSentence->pulVotes){ + output += std::to_string(vote) + ","; + } + T5LOG(T5TRANSACTION) << output; + } +#endif//TEST_VOTES toTmxTagtableRecord(TmProposal.pInputSentence->pTagRecord)->usTagTableId = 1; usMatchesFound = CheckCompactArea( TmProposal.pInputSentence.get(), this ); @@ -457,12 +474,8 @@ BuildVotes PTMX_SENTENCE pSentence // pointer to sentence structure ) { - PTMX_TERM_TOKEN pTermTokens; // pointer to active token - USHORT usIndex = 0; - PTMX_TERM_TOKEN pLastTerm = pSentence->pTermTokens.data() + pSentence->pTermTokens.size()-1; - + PTMX_TERM_TOKEN pTermTokens = pSentence->pTermTokens.data(); // pointer to active token //run through list of tokens and build tuples - pTermTokens = pSentence->pTermTokens.data(); int index = 0 , listLen = pSentence->pTermTokens.size(); while (// (indexusLength && (pSentence->usActVote < ABS_VOTES) ) @@ -471,7 +484,8 @@ BuildVotes pTermTokens++; index++; } /* endwhile */ - + + PTMX_TERM_TOKEN pLastTerm = pSentence->pTermTokens.data() + pSentence->pTermTokens.size()-2; pTermTokens = pSentence->pTermTokens.data(); index = 0; while ( //(indexusLength; + fGo = (pSentence->pTermTokens.size() > indexInList+ 2)? (pTermTokens+2)->usLength: false; } else if ((usTuple == 2) || (usTuple == 1) ) { - fGo = (pTermTokens+3)->usLength; + fGo = (pSentence->pTermTokens.size() > indexInList+ 3)?(pTermTokens+3)->usLength: false; } else { @@ -591,16 +605,18 @@ Vote { ulVote = (ulVote*131) + pTermTokens[usTLookUp[i+3*usTuple]].usHash; } /* endfor */ - - pSentence->pulVotes.push_back(ulVote); - - //pSentence->pulVotes[pSentence->usActVote] = ulVote; - pSentence->usActVote++; - - //#ifdef DEBUG - //T5LOG(T5TRANSACTION) << "pulVotes.size()=" << pSentence->pulVotes.size() << "; actVote = " << pSentence->usActVote << "; indexInList = " << indexInList; - //#endif + //if(pSentence->pulVotes.back()!= ulVote){ + pSentence->pulVotes.push_back(ulVote); + //pSentence->pulVotes[pSentence->usActVote] = ulVote; + pSentence->usActVote++; + //} } /* endif */ + + #ifdef TEST_VOTES + T5LOG(T5TRANSACTION) << "called Vote, tuple = "<< usTuple<<"; fGo="<< fGo <<"; indexInList=" + << indexInList<<"; pSentence->usActVote=" << pSentence->usActVote + << ";pSentence->pulVotes.size()=" << pSentence->pulVotes.size()<< "; ulVote ="< TmRecord; // ptr to tm record std::vector TargetClb; // ptr to target ctl block USHORT usRc = NO_ERROR; // return code - USHORT usAddDataLen = 0; - - usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - - TargetClb.resize(sizeof(TMX_TARGET_CLB)+usAddDataLen + 1); + USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); + size_t clbSize = sizeof(TMX_TARGET_CLB)+usAddDataLen + 1; + T5LOG(T5DEBUG)<<"Adding new proposal to the tm, size = " << clbSize; + TargetClb.resize(clbSize); usRc = FillClb( TargetClb, TmProposal ); if ( usRc == NO_ERROR ) @@ -1329,47 +1344,47 @@ USHORT DetermineTmRecord } } } -#else // new code more based on old code +#elif NEW_CODE_2 // new code more based on old code //PULONG pulSids = lSidsv; //usPos = 0; //end criteria are all sentence ids in index key or only one //sentence id left in pulSids int k = 0;//sid index; - for ( j = 0; (j < usMaxEntries) && (lSids.size() - k > 1); + for ( j = 0; (j < usMaxEntries) && (k < lSids.size()); j++, pIndexEntry++ ) { if ( NTMVOTES(*pIndexEntry) == (BYTE) pSentence->usActVote ) { //before adding sentence id check if already in pulsids as the //respective tm record need only be checked once - while ( (NTMKEY(*pIndexEntry) > lSids[k]) && (lSids.size() - k> 1) ) - { - // Move on to the next position in pulSids + while (k < lSids.size() && NTMKEY(*pIndexEntry) > lSids[k]) { + T5LOG(T5TRANSACTION) << "lKey=" <stIndexEntry; for(auto sid: lSids) { msg += std::to_string(sid) + "; "; @@ -1452,15 +1466,22 @@ USHORT EqfMemory::UpdateTmRecord std::vector lSids; //ptr to sentence ids int iSidCount=0; USHORT usRc = NO_ERROR; //return code - ULONG ulLen; //length indicator - LONG lKey; //tm record key std::vector TmRecord; //pointer to tm record lSids.reserve(MAX_INDEX_LEN + 5); usRc = DetermineTmRecord( this, TmProposal.pInputSentence.get(), lSids ); + if ( usRc == NO_ERROR ) - { + { + #ifdef TEST_SIDS + std::string sids; int i =0; + for(auto sid:lSids){ + sids += std::to_string(i++) + ":" + std::to_string(sid) + "; "; + } + T5LOG(T5TRANSACTION) << "0_7_sids : "<< sids; + #endif//TEST_SIDS + //get tm record(s) //while ( (*pulSids) && (usRc == NO_ERROR) && !fStop ) for(LONG lKey: lSids) @@ -1474,9 +1495,12 @@ USHORT EqfMemory::UpdateTmRecord { //compare tm record data with data passed in the get in structure usRc = ComparePutData( TmRecord, TmProposal, lKey ); + + // 5LOG(T5TRANSACTION) << "lKey = " << lKey <<"; usRc = " << usRc<<";sid n=" << iSidCount; if ( usRc == SOURCE_STRING_ERROR ) { + //T5LOG(T5TRANSACTION) << "lKey lead to source_string_error, key =" < Date: Wed, 2 Jul 2025 20:26:42 +0300 Subject: [PATCH 29/62] * updated to v0.7.5 * fixed importing the same tm multiple times lead to increase of tmd file size --- include/Filebuffer.h | 1 - source/CMakeLists.txt | 2 +- source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 1 + .../core/EqfMemoryPlugin/EqfMemoryPlugin.cpp | 14 -------------- source/opentm2/core/utilities/EQFMORPH.cpp | 2 -- source/opentm2/core/utilities/EQFUTDOS.cpp | 2 -- source/opentm2/core/utilities/EncodingHelper.h | 12 ++++++++++++ source/opentm2/core/utilities/FilesystemHelper.cpp | 6 +++--- source/opentm2/core/utilities/FilesystemHelper.h | 6 +++--- 9 files changed, 20 insertions(+), 26 deletions(-) diff --git a/include/Filebuffer.h b/include/Filebuffer.h index 83ad6391..29426c8e 100644 --- a/include/Filebuffer.h +++ b/include/Filebuffer.h @@ -32,7 +32,6 @@ struct FileBuffer{ int SetOffset(size_t newOffset, int fileAnchor); int Write(const void* buff, size_t buffSize, size_t startingPosition, bool fSuppressTmdSizeCheck = false); - int Write(const void* buff, size_t buffSize); int Read(void* buff, size_t buffSize, size_t startingPosition); int Read(void* buff, size_t buffSize); bool wasModified() const; diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 9b9a199c..47bf5583 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 4) +SET(APP_VERSION_MINOR 5) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 73e6a485..fe415ffd 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -351,6 +351,7 @@ USHORT EqfMemory::NTMGetIDFromNameEx( { if (usRc == NO_ERROR) { + pszName[MAX_LANG_LENGTH-1] = '\0'; /********************************************************************/ /* search name passed in pszName in passed table usTableType */ /********************************************************************/ diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp index 59b4f0b9..7841e7b6 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp @@ -197,20 +197,6 @@ USHORT registerPlugins() return usRC; } -/*! \brief Private data area for the memory import from data files */ -typedef struct _MEMIMPORTFROMFILEDATA -{ - EqfMemory *pInputMemory; // input memory - EqfMemory *pOutputMemory; // output memory - OtmProposal Proposal; // currently processed proposal - std::string strPropFile; // property file of the imorted memory - std::string strDataFile; // data file of the imorted memory - std::string strIndexFile; // index file of the imorted memory - char szBuffer[1014]; // general purpose buffer - -} MEMIMPORTFROMFILEDATA, *PMEMIMPORTFROMFILEDATA; - - /*! \brief Handle a return code from the memory functions and create the approbriate error message text for it \param iRC return code from memory function \param pszMemName long memory name diff --git a/source/opentm2/core/utilities/EQFMORPH.cpp b/source/opentm2/core/utilities/EQFMORPH.cpp index 3f8a1128..6a2c24b7 100755 --- a/source/opentm2/core/utilities/EQFMORPH.cpp +++ b/source/opentm2/core/utilities/EQFMORPH.cpp @@ -469,8 +469,6 @@ USHORT BuildDict { USHORT usRC = MORPH_OK; - usListType; usTermListSize; - if (ADDENDA_DICT != usDictType && ABBREV_DICT != usDictType) { return MORPH_NOT_FOUND; diff --git a/source/opentm2/core/utilities/EQFUTDOS.cpp b/source/opentm2/core/utilities/EQFUTDOS.cpp index 1401c5e9..2926709c 100755 --- a/source/opentm2/core/utilities/EQFUTDOS.cpp +++ b/source/opentm2/core/utilities/EQFUTDOS.cpp @@ -2498,8 +2498,6 @@ USHORT UtlGetFileSizeHwnd { USHORT usRC = NO_ERROR; // function return code DWORD dwSize; - - fMsg; hwndParent; DosError(0); dwSize = GetFileSize( hf ); DosError(1); diff --git a/source/opentm2/core/utilities/EncodingHelper.h b/source/opentm2/core/utilities/EncodingHelper.h index f4945ecd..03eba106 100755 --- a/source/opentm2/core/utilities/EncodingHelper.h +++ b/source/opentm2/core/utilities/EncodingHelper.h @@ -7,6 +7,18 @@ #include +template +std::string str(T t) +{ + if constexpr(std::is_constructible_v){ + return t; + }else{ + return std::to_string(t); + } +} + + + class EncodingHelper{ public: /*! \brief convert a UTF8 std::string to a UTF16 std::wstring diff --git a/source/opentm2/core/utilities/FilesystemHelper.cpp b/source/opentm2/core/utilities/FilesystemHelper.cpp index f275099f..adb7c283 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.cpp +++ b/source/opentm2/core/utilities/FilesystemHelper.cpp @@ -121,13 +121,13 @@ bool handleMissingTMI(const std::string& tmdFilename, const std::string& tmiFile ///////////////////////////////////// -std::string FilesystemHelper::parseDirectory(const std::string path){ +std::string FilesystemHelper::parseDirectory(const std::string& path){ std::size_t found = path.rfind('/'); if (found!=std::string::npos) return path.substr(0,found); return path; } -std::string FilesystemHelper::parseFilename(const std::string path){ +std::string FilesystemHelper::parseFilename(const std::string& path){ std::size_t found = path.rfind('/'); if (found!=std::string::npos) @@ -135,7 +135,7 @@ std::string FilesystemHelper::parseFilename(const std::string path){ return path; } -std::string FilesystemHelper::RemovePathAndExtention(const std::string path){ +std::string FilesystemHelper::RemovePathAndExtention(const std::string& path){ std::string filename = parseFilename(path); std::size_t found = filename.rfind('.'); if (found!=std::string::npos) diff --git a/source/opentm2/core/utilities/FilesystemHelper.h b/source/opentm2/core/utilities/FilesystemHelper.h index 1c1f963d..c29ba4c7 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.h +++ b/source/opentm2/core/utilities/FilesystemHelper.h @@ -29,9 +29,9 @@ class FilesystemHelper{ static std::mutex fsLock; static std::recursive_mutex loadingTm; static bool checkFileName(const std::string& path); - static std::string parseFilename(const std::string path); - static std::string parseDirectory(const std::string path); - static std::string RemovePathAndExtention(const std::string path); + static std::string parseFilename(const std::string& path); + static std::string parseDirectory(const std::string& path); + static std::string RemovePathAndExtention(const std::string& path); static PFileBufferMap getFileBufferInstance(); static std::string FixPath(std::string& path); From 3c1707de4021236b4c8630efc4aa7b8d2d895fc1 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 7 Jul 2025 21:29:51 +0300 Subject: [PATCH 30/62] * updated to 0.7.6 * implemented raii file locks for tmd and tmi files * refactored filesystemHelper class - code cleanup --- include/Filebuffer.h | 105 ++++++++- include/lowlevelotmdatastructs.h | 2 +- include/tm.h | 2 +- source/CMakeLists.txt | 2 +- source/RestAPI/OtmMemoryServiceWorker.cpp | 33 --- .../opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp | 4 +- .../core/EqfMemoryPlugin/EqfMemory.cpp | 27 ++- source/opentm2/core/requestdata.cpp | 6 +- source/opentm2/core/tm.cpp | 10 +- source/opentm2/core/utilities/EQFDICT.cpp | 35 +-- source/opentm2/core/utilities/EQFUTDOS.cpp | 2 +- source/opentm2/core/utilities/EQFUTFIL.cpp | 96 -------- .../core/utilities/FilesystemHelper.cpp | 218 ++++++++++++------ .../opentm2/core/utilities/FilesystemHelper.h | 1 + .../core/utilities/FilesystemWrapper.cpp | 14 -- .../OtmMarkupTablePlugin.cpp | 51 ---- source/otmd.cpp | 4 +- 17 files changed, 304 insertions(+), 308 deletions(-) diff --git a/include/Filebuffer.h b/include/Filebuffer.h index 29426c8e..287f3069 100644 --- a/include/Filebuffer.h +++ b/include/Filebuffer.h @@ -16,27 +16,116 @@ enum FilebufferType{ TMI_FILEBUFFER, }; +#include +#include +#include +#include +#include +#include + +class LockedFile { +public: + enum class Mode { Unknown, + Read, // read-only with shared lock + Write, // read-write with exclusive lock + truncate + Append, // read-write with exclusive lock + append + ReadWrite // read-write with exclusive lock, no truncate + }; + + enum class CalledFrom{ + Unknown, + Create, + Read, + Write + }; + + static constexpr std::string_view CalledFromToString(CalledFrom from) { + switch (from) { + case CalledFrom::Unknown: return "Unknown"; + case CalledFrom::Create: return "Create"; + case CalledFrom::Read: return "Read"; + case CalledFrom::Write: return "Write"; + } + return "Invalid"; // fallback for safety + } + + LockedFile() = default; + + LockedFile(const std::string& path + , CalledFrom location + //, Mode mode = Mode::ReadWrite + ) + : path_(path), + location_(location)//, mode_(mode) + { + openAndLock(); + } + + ~LockedFile() { + closeFile(); // Ensures cleanup + } + + // Delete copy + LockedFile(const LockedFile&) = delete; + LockedFile& operator=(const LockedFile&) = delete; + + // Move constructor + LockedFile(LockedFile&& other) noexcept { + *this = std::move(other); + } + + // Move assignment + LockedFile& operator=(LockedFile&& other) noexcept { + if (this != &other) { + closeFile(); // Close current file if open + location_ = other.location_; + path_ = std::move(other.path_); + std::swap(file_, other.file_); + std::swap(fd_, other.fd_); + std::swap(fl_, other.fl_ ); // Swap the flock + //mode_ = other.mode_; + } + return *this; + } + + FILE* handle() const { return file_; } + const std::string& path() const { return path_; } + size_t GetFileSize() const; + +private: + std::string path_; + FILE* file_ = nullptr; + int fd_ = -1; + struct flock fl_; // Add flock as a member + CalledFrom location_ = CalledFrom::Unknown; + //Mode mode_{};mode is disabled since we would always open file in r+b mode + + void openAndLock(); + void closeFile(); +}; + struct FileBuffer{ int status ;//FILEBUFFERSTATUS std::vector data; size_t offset = 0; - FILE* file = nullptr; - std::string fileName; + //FILE* file = nullptr; + LockedFile file; + std::string fileName; FilebufferType filebufferType; long long originalFileSize = 0; int ReadFromFile(); - //int WriteToFile(); int Flush(bool forced = false); - int atomicWriteWithBackup(); - int SetOffset(size_t newOffset, int fileAnchor); + //Methods below applied only to filebuffer(file in RAM) + int SetOffset(size_t newOffset, int fileAnchor=FILE_BEGIN); int Write(const void* buff, size_t buffSize, size_t startingPosition, bool fSuppressTmdSizeCheck = false); - int Read(void* buff, size_t buffSize, size_t startingPosition); - int Read(void* buff, size_t buffSize); - bool wasModified() const; + int Read(void* buff, size_t buffSize, ssize_t startingPosition=-1); + bool WasModified() const; +private: bool isTMDFilebuffer(); + int atomicWriteWithBackup(); } ; diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index c7e680f5..70621ea2 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -613,7 +613,7 @@ struct BTREE } - bool wasModified() const{ return fb.wasModified(); } + bool wasModified() const{ return fb.WasModified(); } int initLookupTable(); int resetLookupTable(); diff --git a/include/tm.h b/include/tm.h index a894b47b..7dcbeb40 100644 --- a/include/tm.h +++ b/include/tm.h @@ -2315,7 +2315,7 @@ class EqfMemory //: public TMX_CLB EqfMemory(const std::string &tmName); bool IsLoadedInRAM(); - int ReloadFromDisk(); + int Reload(bool lockFromDisk); int UnloadFromRAM(); int FlushFilebuffers(MutexTimeout &tmLockTimeout); diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 47bf5583..611428b2 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 5) +SET(APP_VERSION_MINOR 6) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index 58715fa1..99d61ff5 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -210,39 +210,6 @@ it is up to the caller to free this area using free() \param strError string receiving any error message text \returns 0 is sucessfull or a return code */ -int FilesystemHelper::EncodeFileInBase64( char *pszFile, char **ppStringData, std::string &strError ) -{ - int iRC = 0; - - auto file = FilesystemHelper::OpenFile(pszFile, "r", false); - size_t fSize = FilesystemHelper::GetFileSize(file); - int bytesRead = 0; - if(fSize == 0){ - - }else{ - unsigned char* pData = new unsigned char[fSize]; - FilesystemHelper::ReadFile(file, pData, fSize, bytesRead); - FilesystemHelper::CloseFile(file); - - if(bytesRead != fSize){ - - }else{ - std::string encodedData; - EncodingHelper::Base64Encode(pData, fSize, encodedData); - if(encodedData.empty()) - { - iRC = GetLastError(); - strError = "encoding of BASE64 data failed"; - T5LOG(T5ERROR) << "encodeBase64ToFile()::ecnoding of BASE64 data failed, iRC = " << iRC; - return( iRC ); - } - } - - // cleanup - delete[] pData ; - } - return( iRC ); -} /*! \brief convert a BASE64 encoded string to a binary file \param pStringData pointer to the BASE64 data diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index 025d4b70..11513e78 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -318,15 +318,13 @@ VOID EQFMemOrganizeEnd if(nullptr != pRIDA->pMemTemp){ pRIDA->pMem->TmBtree.fb.data = std::move(pRIDA->pMemTemp->TmBtree.fb.data); //get signature record and add to control block - pRIDA->pMem->TmBtree.resetLookupTable(); USHORT usRc = pRIDA->pMem->sign(); pRIDA->pMem->TmBtree.fb.Flush(true); pRIDA->pMem->InBtree.fb.data = std::move(pRIDA->pMemTemp->InBtree.fb.data); - pRIDA->pMem->InBtree.resetLookupTable(); pRIDA->pMem->InBtree.fb.Flush(true); - pRIDA->pMem->ReloadFromDisk(); + pRIDA->pMem->Reload(false); } LONG lCurTime = 0; diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index f9b8c5da..31dd910d 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -176,8 +176,8 @@ unsigned long EqfMemory::getFileSize() //if ( pTmClb != NULL ) { - unsigned long ulDataSize = FilesystemHelper::GetFileSize( TmBtree.fb.file ); - unsigned long ulIndexSize = FilesystemHelper::GetFileSize( InBtree.fb.file ); + unsigned long ulDataSize = TmBtree.fb.file.GetFileSize(); + unsigned long ulIndexSize = InBtree.fb.file.GetFileSize(); ulFileSize = ulDataSize + ulIndexSize; } @@ -655,14 +655,27 @@ EqfMemory::EqfMemory(const std::string& tmName){ szName = tmName; } -int EqfMemory::ReloadFromDisk(){ +int EqfMemory::Reload(bool fReadFromDisk){ int rc = 0; TmBtree.resetLookupTable(); InBtree.resetLookupTable(); - rc = TmBtree.fb.ReadFromFile(); - rc = InBtree.fb.ReadFromFile(); - eStatus = WAITING_FOR_LOADING_STATUS; - LoadMem(); + + //datashould be already in filebuffers in case of reorganize, otherwise - read from disk + if(fReadFromDisk){ + TmBtree.fb.file = LockedFile{};//reset fileLock + InBtree.fb.file = LockedFile{};//reset fileLock + rc = TmBtree.fb.ReadFromFile(); + if(!rc){ + rc = InBtree.fb.ReadFromFile(); + } + } + + if(rc){ + eStatus = FAILED_TO_OPEN_STATUS; + }else{ + eStatus = WAITING_FOR_LOADING_STATUS; + rc = LoadMem(); + } return rc; } diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index cb3284fa..905fdb45 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -330,8 +330,12 @@ int RequestData::requestTM(){ if(mem->isWaitingToBeLoaded() || mem->isFailedToLoad()){ _rc_ = mem->Load(); if(_rc_){ + std::string msg = "Failed to load tm"; + if(11==_rc_){ + msg += "; rc=11, file on the disk is locked;"; + } //T5LOG(T5ERROR) << "Failed to open tm, rc = " << _rc_; - return buildErrorReturn(504, "Failed to load tm", _rc_); + return buildErrorReturn(504, msg.c_str(), _rc_); } } } diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index fafad155..da68e991 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -415,9 +415,11 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem _rc_ = NewMem->InBtree.QDAMDictCreateLocal(&NewMem->stTmSign, START_KEY, keepInRamOnly); } /* endif */ - if(_rc_ == NO_ERROR){ - NewMem->TmBtree.fb.Flush(); - NewMem->InBtree.fb.Flush(); + if(_rc_ == NO_ERROR){ + if(!keepInRamOnly){ + NewMem->TmBtree.fb.Flush(); + NewMem->InBtree.fb.Flush(); + } NewMem->eStatus = OPEN_STATUS; }else { NewMem->eStatus = FAILED_TO_OPEN_STATUS; @@ -989,7 +991,7 @@ int TMManager::ReplaceMemory } T5LOG(T5INFO)<<"Mem is loaded, reloading filebuffers from disk..."; - iRC = TMManager::GetInstance()->tms[strReplace]->ReloadFromDisk(); + iRC = TMManager::GetInstance()->tms[strReplace]->Reload(true); } } // broadcast deleted memory name for replaceWith memory diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index eaf65eec..75e8135f 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -1414,22 +1414,23 @@ SHORT BTREE::QDAMDictClose() // reset open flag in header and force a write to disk // open flag will only be set if opened for r/w { - sRc = QDAMDictFlushLocal(); + //sRc = QDAMDictFlushLocal(); //fOpen = FALSE; // re-write header record - if ( sRc == NO_ERROR ) - sRc = QDAMWriteHeader(); + //if ( sRc == NO_ERROR ) + // sRc = QDAMWriteHeader(); } /* endif */ - if ( UtlClose(fb.file, FALSE) && !sRc ) - { - sRc = BTREE_CLOSE_ERROR; - } /* endif */ + //if ( UtlClose(fb.file, FALSE) && !sRc ) + //{ + // sRc = BTREE_CLOSE_ERROR; + //} /* endif */ if ( fpDummy ) { UtlClose( fpDummy, FALSE ); } /* endif */ + fb.data.resize(0); /*******************************************************************/ /* free the allocated buffers */ @@ -1788,12 +1789,12 @@ SHORT BTREE::QDAMDictCreateLocal if( false == keepInRamOnly ) { // Try to create the index file - fb.file = fopen(fb.fileName.c_str(), "w+b"); + fb.file = LockedFile(fb.fileName, LockedFile::CalledFrom::Create);// = fopen(fb.fileName.c_str(), "w+b"); - if(!fb.file){ - sRc = -1; - T5LOG(T5ERROR) << "::Can't create file " << fb.fileName; - } + //if(!fb.file){ + // sRc = -1; + // T5LOG(T5ERROR) << "::Can't create file " << fb.fileName; + //} } if ( !sRc ) @@ -1823,7 +1824,7 @@ SHORT BTREE::QDAMDictCreateLocal /******************************************************************/ chCollate = NtmVitalInfo; - sRc = initLookupTable(); + sRc = initLookupTable(); usNumberOfAllocatedBuffers = 0; } @@ -4706,7 +4707,7 @@ SHORT BTREE::QDAMDictOpenLocal usOpenFlags = usOpenFlags; // open the file - fb.ReadFromFile(); + sRc = fb.ReadFromFile(); if ( !sRc ) { @@ -5098,7 +5099,7 @@ SHORT BTREE::QDAMDictUpdSignLocal // let 2K at beginning as space writingPosition = (LONG) USERDATA_START; - sRc = fb.SetOffset( writingPosition, FILE_BEGIN); + sRc = fb.SetOffset( writingPosition); ulDataLen = BTREE_REC_SIZE_V3 - USERDATA_START - sizeof(USHORT); ulDataLen = ulLen < ulDataLen? ulLen : ulDataLen; @@ -5148,7 +5149,7 @@ SHORT BTREE::QDAMDictUpdSignLocal } /* endif */ if ( ! sRc ) { - fb.SetOffset(BTREE_REC_SIZE_V3, FILE_BEGIN); + fb.SetOffset(BTREE_REC_SIZE_V3); } /* endif */ } @@ -5250,7 +5251,7 @@ SHORT BTREE::QDAMDictSignLocal // read in data part //sRc = UtlRead( pBT->fb.file, pUserData, usLen, // &usNumBytesRead, FALSE ); - sRc = fb.Read(pUserData, usLen); + sRc = fb.Read(pUserData, usLen, USERDATA_START+sizeof(USHORT)); if ( !sRc ) { *pusLen = usLen; diff --git a/source/opentm2/core/utilities/EQFUTDOS.cpp b/source/opentm2/core/utilities/EQFUTDOS.cpp index 2926709c..76c91fe1 100755 --- a/source/opentm2/core/utilities/EQFUTDOS.cpp +++ b/source/opentm2/core/utilities/EQFUTDOS.cpp @@ -2499,7 +2499,7 @@ USHORT UtlGetFileSizeHwnd USHORT usRC = NO_ERROR; // function return code DWORD dwSize; DosError(0); - dwSize = GetFileSize( hf ); + dwSize = FilesystemHelper::GetFileSize( hf ); DosError(1); if ( dwSize == -1 ) { diff --git a/source/opentm2/core/utilities/EQFUTFIL.cpp b/source/opentm2/core/utilities/EQFUTFIL.cpp index 10b87445..0a38bd80 100755 --- a/source/opentm2/core/utilities/EQFUTFIL.cpp +++ b/source/opentm2/core/utilities/EQFUTFIL.cpp @@ -50,22 +50,6 @@ static USHORT usLastDosRc; // buffer for last DOS return code //| return to drive letter converted return code | //+----------------------------------------------------------------------------+ -BYTE UtlQCurDisk() -{ - static CHAR szCurDir[MAX_PATH+100]; - - DosError(0); -T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 65 if ( GetCurrentDirectory( sizeof(szCurDir), szCurDir ) == 0 ) { szCurDir[0] = EOS; }"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - if ( GetCurrentDirectory( sizeof(szCurDir), szCurDir ) == 0 ) - { - szCurDir[0] = EOS; - } /* endif */ -#endif //TO_BE_REPLACED_WITH_LINUX_CODE - DosError(1); - - return (BYTE)szCurDir[0]; -} //+----------------------------------------------------------------------------+ //|External Function | @@ -609,86 +593,6 @@ USHORT UtlWriteFile /*@86C*/ //| endif | //| return isEqfDrive flag | //+----------------------------------------------------------------------------+ -BOOL UtlIsEqfDrive -( - CHAR chDrive // drive letter being checked -) -{ - BOOL fIsEqfDrive = TRUE; // function return value - union - { - CHAR szDrives[MAX_DRIVELIST]; // buffer for drive letters - CHAR szPath[MAX_EQF_PATH+1]; // buffer for path names - } Buffers; - - // - // get string containing all known EQF drives - // - UtlQueryString( QST_ORGEQFDRIVES, Buffers.szDrives, - sizeof(Buffers.szDrives) ); - - // - // check if given drive is contained in EQF drive list - // - if ( !strchr( Buffers.szDrives, chDrive ) ) - { - fIsEqfDrive = FALSE; - } /* endif */ - - // - // check if given drive is a valid drive for this system - // - if ( fIsEqfDrive ) - { - UtlGetDriveList( (PBYTE)Buffers.szDrives ); // get list of attached drives - if ( !strchr( Buffers.szDrives, chDrive ) ) - { - fIsEqfDrive = FALSE; - } /* endif */ - } /* endif */ - - // - // check if required directories exist on drive - // - if ( fIsEqfDrive ) - { - UtlMakeEQFPath( Buffers.szPath, chDrive, MEM_PATH, NULL ); - if ( !UtlDirExist( Buffers.szPath ) ) - { - fIsEqfDrive = FALSE; - } - else - { - UtlMakeEQFPath( Buffers.szPath, chDrive, DIC_PATH, NULL ); - if ( !UtlDirExist( Buffers.szPath ) ) - { - fIsEqfDrive = FALSE; - } - else - { - UtlMakeEQFPath( Buffers.szPath, chDrive, TABLE_PATH, NULL ); - if ( !UtlDirExist( Buffers.szPath ) ) - { - fIsEqfDrive = FALSE; - } - else - { - UtlMakeEQFPath( Buffers.szPath, chDrive, LIST_PATH, NULL ); - if ( !UtlDirExist( Buffers.szPath ) ) - { - fIsEqfDrive = FALSE; - } - else - { - // everything seems to be ok with the given drive - } /* endif */ - } /* endif */ - } /* endif */ - } /* endif */ - } /* endif */ - - return( fIsEqfDrive ); -} /* end of UtlIsEqfDrive */ //+----------------------------------------------------------------------------+ //|External function | diff --git a/source/opentm2/core/utilities/FilesystemHelper.cpp b/source/opentm2/core/utilities/FilesystemHelper.cpp index adb7c283..a462cb84 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.cpp +++ b/source/opentm2/core/utilities/FilesystemHelper.cpp @@ -57,48 +57,48 @@ std::recursive_mutex FilesystemHelper::loadingTm; DECLARE_bool(keep_tm_backups); int FileBuffer::atomicWriteWithBackup() { try { - // Step 1: Create a temporary file std::string tempFilename = fileName + ".tmp"; + // Step 1: Write to temp file with lock { - std::ofstream tempFile(tempFilename, std::ios::out | std::ios::trunc | std::ios::binary); - if (!tempFile) { - T5LOG(T5ERROR) << "Error: Unable to open temporary file for writing"; - return 1; - } - //tempFile.write(data, data.size()); - if (!tempFile) { - T5LOG(T5ERROR) << "Error: Write operation to temporary file failed"; + LockedFile tempFile(tempFilename, LockedFile::CalledFrom::Write); //, LockedFile::Mode::Write); + FILE* f = tempFile.handle(); + + size_t written = fwrite(data.data(), 1, data.size(), f); + fflush(f); + + if (written != data.size()) { + T5LOG(T5ERROR) << "Write to temp file failed: " << written + << " of " << data.size(); return 2; } - - tempFile.write(reinterpret_cast(data.data()), data.size()); - tempFile.flush(); } - // Step 2: Rename original file to backup (if it exists) + // Step 2: Backup original file std::string backupFilename = fileName + ".old"; - if(std::filesystem::exists(backupFilename)) { + if (std::filesystem::exists(backupFilename)) { FilesystemHelper::DeleteFile(backupFilename, true); } + file = LockedFile{};//unlock our file if (std::filesystem::exists(fileName)) { std::filesystem::rename(fileName, backupFilename); } - // Step 3: Rename the temporary file to the original filename + // Step 3: Rename temp to original std::filesystem::rename(tempFilename, fileName); - // Step 4: Delete the backup file + // Step 4: Optionally delete backup if (std::filesystem::exists(backupFilename) && !FLAGS_keep_tm_backups) { FilesystemHelper::DeleteFile(backupFilename, true); - //std::filesystem::remove(backupFilename); } + file = LockedFile(fileName, LockedFile::CalledFrom::Write); + return 0; } catch (const std::exception& e) { - T5LOG(T5ERROR) << "Error: " << e.what() << ""; - return 3; + T5LOG(T5ERROR) << "atomicWriteWithBackup error: " << e.what(); + return -3; } } @@ -528,45 +528,51 @@ int FilesystemHelper::RemoveDirWithFiles(const std::string& path){ } } -int FileBuffer::ReadFromFile(){ +int FileBuffer::ReadFromFile(){ #ifdef LOG_FILE_WRITE T5LOG(T5TRANSACTION) <<"called read file " << fileName <<" from disk;";// << " , stacktrace:" << GET_STACKTRACE_EXPL; - #endif + #endif - if(fileName.empty()){ + if (fileName.empty()) { return FilesystemHelper::FILEHELPER_ERROR_WRONG_FILENAME; } - if(FilesystemHelper::FileExists(fileName) == false){ + + if (!FilesystemHelper::FileExists(fileName)) { return FilesystemHelper::FILEHELPER_ERROR_NO_FILES_FOUND; } - originalFileSize = FilesystemHelper::GetFileSize(fileName); - int readed = 0; - if( originalFileSize > 0 && originalFileSize != -1){ - data.resize(originalFileSize); - - if(VLOG_IS_ON(1)){ - T5LOG( T5INFO) << "OpenFile:: file size >0 -> Filebuffer resized to filesize(" << originalFileSize << "), fname = " << fileName; - } - file = fopen(fileName.c_str(), "rb"); - if(file == nullptr){ - T5LOG(T5ERROR) << "Can't open the file under the path=" << fileName; - }else - { - readed = fread(&data[0], originalFileSize, 1, file); - fclose(file); - file = nullptr; - } - if(VLOG_IS_ON(1)){ - T5LOG( T5INFO) << "OpenFile:: file size >0 -> Filebuffer reading to buffer, size = " << originalFileSize << - "), fname = " << fileName << "; readed = " << readed; + + try { + if(nullptr ==file.handle()){ + file = LockedFile(fileName, LockedFile::CalledFrom::Read); //, LockedFile::Mode::Read); // lock + open } - }else{ - if(VLOG_IS_ON(1)){ - T5LOG( T5INFO) << "OpenFile:: file size <=0 -> Filebuffer resized to default value, fname = " << fileName ; + + originalFileSize = file.GetFileSize(); + if (originalFileSize > 0) { + data.resize(originalFileSize); + + if (VLOG_IS_ON(1)) { + T5LOG(T5INFO) << "FileBuffer resized to " << originalFileSize + << ", fname = " << fileName; + } + + fseek(file.handle(), 0, SEEK_SET); + size_t read = fread(data.data(), 1, originalFileSize, file.handle()); + + if (VLOG_IS_ON(1)) { + T5LOG(T5INFO) << "Read " << read << " bytes from " << fileName; + } + } else { + if (VLOG_IS_ON(1)) { + T5LOG(T5INFO) << "File size <= 0, resizing buffer to default"; + } + data.resize(16384); } - data.resize(16384); + + return 0; + } catch (const std::exception& e) { + T5LOG(T5ERROR) << "ReadFromFile error: " << e.what(); + return FilesystemHelper::FILEHELPER_ERROR_CANT_OPEN_FILE; } - return 0; } int FileBuffer::SetOffset(size_t newOffset, int fileAnchor){ @@ -586,7 +592,7 @@ int FileBuffer::SetOffset(size_t newOffset, int fileAnchor){ return 0; } -bool FileBuffer::wasModified() const{ +bool FileBuffer::WasModified() const{ return status & FileBufferStatus::MODIFIED; } @@ -595,13 +601,14 @@ int FileBuffer::Flush(bool forced){ T5LOG(T5TRANSACTION) <<"called flush filebuffer " << fileName <<" to disk;";// << " , stacktrace:" << GET_STACKTRACE_EXPL; #endif - bool fileWasOpened = file != nullptr; + //bool fileWasOpened = file != nullptr; int rc = 0; size_t writenBytes = 0; - if(!FilesystemHelper::FileExists(fileName) && !forced){ - T5LOG(T5WARNING) << "File is not exists on disk so flushing would be skipped, fName = " << fileName; - }else if(forced || wasModified()){ + //if(!FilesystemHelper::FileExists(fileName) && !forced){//since we are locking files, that always should be true + // T5LOG(T5WARNING) << "File is not exists on disk so flushing would be skipped, fName = " << fileName; + //}else + if(forced || WasModified()){ if(VLOG_IS_ON(1)){ T5LOG( T5INFO) << "Flush:: writing files from buffer"; } @@ -611,7 +618,6 @@ int FileBuffer::Flush(bool forced){ oldSize = FilesystemHelper::GetFileSize(fileName); } - //writenBytes = fwrite(bufStart, size, 1, file) * size; rc = atomicWriteWithBackup(); if ( rc ){ @@ -620,9 +626,9 @@ int FileBuffer::Flush(bool forced){ } if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - T5LOG( T5DEBUG) "::Flush(" << (long int) file << ") buff = " << "void" << - ", buffsize = " << data.size() << ", path = " << fileName << ", file size = " << - FilesystemHelper::GetFileSize(file) << ", oldSize = " << oldSize; + T5LOG( T5DEBUG) "::Flush(" << (long int) file.handle() << ") buff = " << "void" << + ", buffsize = " << data.size() << ", path = " << file.path() << ", file size = " << + file.GetFileSize() << ", oldSize = " << oldSize; } @@ -634,7 +640,11 @@ int FileBuffer::Flush(bool forced){ return 0; } -int FileBuffer::Read(void* buff, size_t buffSize){ +int FileBuffer::Read(void* buff, size_t buffSize,ssize_t startingPos){ + if(startingPos>=0){ + offset = startingPos; + } + if(data.size()< offset + buffSize){ if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ T5LOG(T5ERROR) << "ReadBuffer:: Trying to read not existing bytes from buffer, fName = " << fileName; @@ -650,13 +660,6 @@ int FileBuffer::Read(void* buff, size_t buffSize){ return 0; } -int FileBuffer::Read(void* buff, size_t buffSize, size_t startingPos){ - offset = startingPos; - return Read(buff, buffSize); -} - - - bool FileBuffer::isTMDFilebuffer(){ return EncodingHelper::endsWithIgnoreCase(fileName, ".tmd" ); } @@ -789,7 +792,7 @@ int FilesystemHelper::FlushAllBuffers(std::string * modifiedFiles){ return -1; } - for(auto file : *pFileBuffers){ + for(auto& file : *pFileBuffers){ if(WriteBuffToFile(file.first) == FILEHELPER_NO_ERROR){ if(modifiedFiles){ *modifiedFiles += file.first + "; "; @@ -1390,7 +1393,6 @@ int FilesystemHelper::CreateDir(const std::string& dir, int rights) { } bool FilesystemHelper::DirExists(const std::string& path){ - bool bExists = false; DIR *pDir = opendir (path.c_str()); if (pDir != NULL) @@ -1403,3 +1405,85 @@ bool FilesystemHelper::DirExists(const std::string& path){ } return bExists; } + +size_t LockedFile::GetFileSize() const { + if (!file_) return 0; + + int fd = fileno(file_); + struct stat st{}; + if (fstat(fd, &st) == -1) { + throw std::system_error(errno, std::generic_category(), + "Failed to get file size"); + } + return static_cast(st.st_size); +} + +DECLARE_bool(log_file_locks); +void LockedFile::openAndLock() { + if(FLAGS_log_file_locks){ + T5LOG(T5TRANSACTION) << "locking \"" << path_ << "\" file, locked from " << CalledFromToString(location_) << " function"; + } + int flags = O_RDWR | O_CREAT; + //if (mode_ == Mode::Append) flags |= O_APPEND; + + fd_ = ::open(path_.c_str(), flags, 0666); + if (fd_ == -1) { + throw std::system_error(errno, std::generic_category(), + "Failed to open file"); + } + + fl_.l_type = F_WRLCK;//(mode_ == Mode::Read) ? F_RDLCK : F_WRLCK; + fl_.l_whence = SEEK_SET; + fl_.l_start = 0; + fl_.l_len = 0; + + if (fcntl(fd_, F_SETLK, &fl_) == -1) { + ::close(fd_); + throw std::system_error(errno, std::generic_category(), + "Failed to lock file"); + } + + // Only truncate in Write mode + //if (mode_ == Mode::Write) { + // if (ftruncate(fd, 0) == -1) { + // ::close(fd); + // throw std::system_error(errno, std::generic_category(), + // "Failed to truncate file"); + // } + //} + + const char* fmode = "r+b"; + //const char* fmode = "r+"; + //if (mode_ == Mode::Append) fmode = "a+"; + //else if (mode_ == Mode::Read) fmode = "rb"; + + file_ = fdopen(fd_, fmode); + if (!file_) { + ::close(fd_); + throw std::system_error(errno, std::generic_category(), + "Failed to create file stream"); + } + if(FLAGS_log_file_locks){ + T5LOG(T5TRANSACTION) << "locked \"" << path_ << "\" file locked from " << CalledFromToString(location_) << " function"; + } +} + +void LockedFile::closeFile() { + if(fd_>=1){ + fl_.l_type = F_UNLCK; + fl_.l_whence = SEEK_SET; + fl_.l_start = 0; + fl_.l_len = 0; // 0 = unlock entire file + fcntl(fd_, F_SETLK, &fl_); // Unlock the file + } + + if (file_) { + fclose(file_); // Releases lock and closes file + file_ = nullptr; + fd_ = -1; // Prevent stale fd reuse + if(FLAGS_log_file_locks){ + T5LOG(T5TRANSACTION) << "unlocked \"" << path_ << "\" file locked from " << CalledFromToString(location_) << " function"; + } + path_.clear(); + } +} \ No newline at end of file diff --git a/source/opentm2/core/utilities/FilesystemHelper.h b/source/opentm2/core/utilities/FilesystemHelper.h index c29ba4c7..f888c854 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.h +++ b/source/opentm2/core/utilities/FilesystemHelper.h @@ -131,6 +131,7 @@ class FilesystemHelper{ FILEHELPER_ERROR_NO_FILES_FOUND, FILEHELPER_ERROR_WRONG_FILENAME, + FILEHELPER_ERROR_CANT_OPEN_FILE, FILEHELPER_ERROR_CANT_OPEN_DIR, diff --git a/source/opentm2/core/utilities/FilesystemWrapper.cpp b/source/opentm2/core/utilities/FilesystemWrapper.cpp index e89ddf7d..180b1430 100755 --- a/source/opentm2/core/utilities/FilesystemWrapper.cpp +++ b/source/opentm2/core/utilities/FilesystemWrapper.cpp @@ -13,20 +13,6 @@ std::string filesystem_get_home_dir() { return FilesystemHelper::GetHomeDir(); } -int filesystem_open_file(const char* path, FILE*& ptr, const char* mode){ - T5LOG( T5WARNING) <<"TEMPORARY HARDCODED useBuffer= true in filesystem_open_file, fname = " << path; - bool useBuffer = true;//false; - if( (strcasestr(path, ".TMI") ) - || (strcasestr(path, ".TMD") ) - //|| (strcasestr(path, ".MEM") ) - ){ - T5LOG( T5INFO) << "filesystem_open_file::Openning data file(with ext. .TMI, .TMD, .MEM => forcing to use filebuffers, fName = "<< - path<< ", useFilebuffer before = " < Date: Tue, 8 Jul 2025 22:32:30 +0300 Subject: [PATCH 31/62] * updated to v0.7.7 * fixed compatibility of longname table with v0.6 * in case of forced fuzzy search, disabled removal of proposals in fuzzy, that have the same target. * fixed fuzzy crash --- include/tm.h | 4 + source/CMakeLists.txt | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 103 ++---------------- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 19 +++- 4 files changed, 32 insertions(+), 96 deletions(-) diff --git a/include/tm.h b/include/tm.h index 7dcbeb40..cda4f82f 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1803,6 +1803,10 @@ class EqfMemory //: public TMX_CLB LONG lLangIdToGroupTableUsed = 0; // size of table (bytes in use) PVOID pTagTable = nullptr; // tag table loaded for segment markup (TBLOADEDTABLE) + BYTE getMajorVersion()const{ + return stTmSign.bMajorVersion; + } + // copy of long name table sorted ignoring the case of the file names // Note: only the stTableEntry array is filled in this area, for all other // information use the entries in the pLongNames structure diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 611428b2..79a7ad3e 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 6) +SET(APP_VERSION_MINOR 7) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index aef054e2..395303ea 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -314,7 +314,7 @@ USHORT EqfMemory::TmtXGet /********************************************************************/ /* check if we have more than one equal tgt match in our proposals */ /********************************************************************/ - if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && (pTmGetIn->ulParm & GET_EXACT )) + if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && !(pTmGetIn->fForceFuzzySearch) && (pTmGetIn->ulParm & GET_EXACT )) { PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); PTMX_MATCH_TABLE_W pstActMatch; @@ -1166,91 +1166,6 @@ USHORT ExactTest return( usRc ); } -#if defined(SGMLDITA_LOGGING) || defined(INLINE_TAG_REPL_LOGGING) || defined(MATCHLIST_LOG) - void NTMMarkCRLF( PSZ_W pszSource, PSZ_W pszTarget ) - { - while ( *pszSource ) - { - if ( *pszSource == L'\n' ) - { - wcscpy( pszTarget, L"" ); - pszTarget += 4; - } - else if ( *pszSource == L'\r' ) - { - wcscpy( pszTarget, L"" ); - pszTarget += 4; - } - else - { - *pszTarget++ = *pszSource; - } /* endif */ - pszSource++; - } /*endwhile */ - *pszTarget = 0; - } - - - void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ) - { - static CHAR_W szSegBuf[4096]; - static CHAR_W szLineBuf[4096]; - int iLen; - - NTMMarkCRLF( pszSegData, szSegBuf ); - //iLen = swprintf( szLineBuf, pszForm, szSegBuf ); - //fwrite( szLineBuf, 2, iLen, hfLog ); - } - -// removed superfluos whitespace at the beggining or end of the proposal -void NTMRemoveAdditionalWhitepace -( - PSZ_W pszSource, - PSZ_W pszProposal -) -{ - int iLen; - - // only remove leading whitespace if source has none - if ( (*pszSource != L'\n') && (*pszSource != L'\r') && (*pszSource != L' ') ) - { - PSZ_W pszProp = pszProposal; - PSZ_W pszTemp = pszProposal; - - while ( (*pszTemp == L'\n') || (*pszTemp == L'\r') || (*pszTemp == L' ') ) - { - pszTemp++; - } /*endwhile */ - - if ( pszTemp != pszProp ) - { - while ( *pszTemp ) - { - *pszProp++ = *pszTemp++; - } /*endwhile */ - *pszProp = 0; - } /* endif */ - } /* endif */ - - // check trailing whitespace - iLen = wcslen( pszSource ); - if ( iLen ) - { - iLen--; - if ( (pszSource[iLen] != L'\n') && (pszSource[iLen] != L'\r') && (pszSource[iLen] != L' ') ) - { - iLen = wcslen( pszProposal ); - while ( iLen && ( (pszProposal[iLen-1] == L'\n') || (pszProposal[iLen-1] == L'\r') || (pszProposal[iLen-1] == L' ') ) ) - { - iLen--; - pszProposal[iLen] = 0; - } /*endwhile */ - } /* endif */ - } /* endif */ -} /* end of function NTMRemoveAdditionalWhitepace */ - -#endif - BOOL TMFuzzynessEx ( @@ -1393,7 +1308,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); - wcsncpy(SubstProp.szPropSource, pSourceString.c_str(), *plSourceLen); + wcsncpy(SubstProp.szPropSource, pSourceString.c_str(), pSourceString.size());// *plSourceLen); if ( usRc == NO_ERROR ) { PSZ_W pTarget; @@ -1540,9 +1455,11 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct { pByte = (PBYTE)pTMXTargetRecord; // copy normalized propsource string - if(*plSourceLen > 0 ){ - memcpy( pstMatchTable->szSource, pSourceString.c_str(), *plSourceLen * sizeof(CHAR_W)); - } + //if(*plSourceLen > 0 ){ + //T5LOG(T5TRANSACTION) << "memcpy string, len = " << *plSourceLen <<"; pSourceString.size()=" << pSourceString.size()<<"; str=\'" << EncodingHelper::convertToUTF8(pSourceString) <<"\'; position in match table = " << pstMatchTable - pstMatchTableStart; + memcpy( pstMatchTable->szSource, pSourceString.c_str(), pSourceString.size() * sizeof(CHAR_W)); + pstMatchTable->szSource[pSourceString.size()] = L'\0'; + //} if ( usRc == NO_ERROR ) { @@ -1558,7 +1475,9 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct // now ulTargetLen is # of w's //position at target tag record pByte = (PBYTE)pTMXTargetRecord; - memcpy( pstMatchTable->szTarget, pString.data(), lTargetLen * sizeof(CHAR_W)); + //memcpy( pstMatchTable->szTarget, pString.data(), lTargetLen * sizeof(CHAR_W)); + memcpy( pstMatchTable->szTarget, pString.data(), pString.size() * sizeof(CHAR_W)); + pstMatchTable->szTarget[pString.size()] = '\0'; if ( usRc == NO_ERROR ) { @@ -1908,7 +1827,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block } /* endif */ }//endif skipExactTest - if ( !fStringEqual && !usRc ) + if ( (pGetIn->fForceFuzzySearch || !fStringEqual) && !usRc ) { USHORT usTgtNum = 0; // first target number LONG lLeftTgtLen = 0; // target length record... diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index fe415ffd..761effbb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -887,8 +887,17 @@ void appendLongNameEntry( USHORT EqfMemory::WriteLongNameTablesToTheTM() { + //if(getMajorVersion() < 7)//for compatibility + { + LongNames.pszBuffer.push_back(0); + } USHORT usRc = TmBtree.EQFNTMUpdate(LONGNAME_KEY, LongNames.pszBuffer); + + //if(getMajorVersion() < 7)//for compatibility + { + LongNames.pszBuffer.pop_back(); + } return usRc; } @@ -956,7 +965,6 @@ USHORT EqfMemory::NTMReadLongNameTable() { // unterse data if table data is tersed PTERSEHEADER pTerseHeader = (PTERSEHEADER)LongNames.pszBuffer.data(); - std::vector pNewArea; // ptr to unterse data area if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) { @@ -968,8 +976,14 @@ T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be { // fill pointer array PSZ pszTemp = (PSZ)LongNames.pszBuffer.data(); + if(//getMajorVersion() < 7 && + !LongNames.pszBuffer.empty())//for compatibility + { + LongNames.pszBuffer.pop_back(); + } + ssize_t buffLen = LongNames.pszBuffer.size(); - while ( (pszTemp - (PSZ)LongNames.pszBuffer.data() < LongNames.pszBuffer.size()) && *((PUSHORT)pszTemp) != 0) + while ( (pszTemp - (PSZ)LongNames.pszBuffer.data() < buffLen) && *((PUSHORT)pszTemp) != 0) { // get ID of string USHORT usId = *((PUSHORT)pszTemp); @@ -982,7 +996,6 @@ T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be appendLongNameEntry(LongNamesCaseIgnore, usId, longNameCaseIgnore); pszTemp += strlen(pszTemp) + 2; // skip long name } /* endwhile */ - SortLongNameTables(); } /* endif */ From aaa855a69f80e7747b3bd00e10f563a15e6d2877 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 9 Jul 2025 21:07:27 +0300 Subject: [PATCH 32/62] * updated to v0.7.8 * fixed -287(saving and keeping segment with most recent timestamp from the same segments) --- source/CMakeLists.txt | 2 +- source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 12 +++--------- source/otmd.cpp | 5 ----- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 79a7ad3e..8eafa422 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 7) +SET(APP_VERSION_MINOR 8) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index c322ad0a..eb71849e 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -1558,8 +1558,6 @@ USHORT EqfMemory::UpdateTmRecord // else try next sentence key //------------------------------------------------------------------------------ - -DECLARE_bool(ignore_newer_target_exists_check); DECLARE_bool(log_memmove_in_compareputdata); //------------------------------------------------------------------------------ // External function @@ -1684,7 +1682,7 @@ USHORT EqfMemory::ComparePutData pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); pTMXTargetRecord = (PTMX_TARGET_RECORD)pStartTarget; - if ( fNewerTargetExists && !FLAGS_ignore_newer_target_exists_check) + if ( fNewerTargetExists) { fStop = TRUE; // do not continue update loop } @@ -2022,20 +2020,16 @@ USHORT EqfMemory::AddTmTarget( VOID FillTargetRecord ( PTMX_SENTENCE pSentence, //ptr to sentence structure -// PTMX_TAGTABLE_RECORD pTagRecord, //ptr to target string tag table PSZ_W pNormString, //ptr to target normalized string USHORT usNormLen, //length of target normalized string PTMX_TARGET_RECORD *ppTMXTargetRecord, //ptr to target record PTMX_TARGET_CLB pTargetClb //ptr to target control block ) { - PBYTE pTarget; //ptr to target record - PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record - - pTMXTargetRecord = *ppTMXTargetRecord; + PTMX_TARGET_RECORD pTMXTargetRecord = *ppTMXTargetRecord; //ptr to target record //position to start of target structure - pTarget = (PBYTE)pTMXTargetRecord; + PBYTE pTarget = (PBYTE)pTMXTargetRecord; //ptr to target record //set target string start offset pTMXTargetRecord->usTarget = sizeof(*pTMXTargetRecord); diff --git a/source/otmd.cpp b/source/otmd.cpp index 67fd8955..3f07e766 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -194,14 +194,9 @@ DEFINE_bool(skip_default_flags_in_init_msg, false, "if set to true, all default DEFINE_bool(logMutexes, false, "if set to true you would see mutex logs"); -//DEFINE_bool(useconfigfile, false, "Set to use values from config file that should be located under ~/.t5memory/t5memory.conf"); - DEFINE_bool(forbiddeletefiles, false, "Set to true to keep all files(including temporary and tm)"); -DEFINE_bool(ignore_newer_target_exists_check, true, "if set to true, check for newer already saved target would be skipped for saving segments" ); - - DEFINE_bool(log_memmove_in_compareputdata, false, "if set to true, when saving segment and causing memmove in compareputdata functions, just before memmove, data would be logged - use this to debug btree crashes. In that spot in come corrupted data, memmove could crash t5memory(or cause asan crash). Just to set low log level(v=2,t5logleve=0), reproduce the crash and check if the last log was one that started with \"memmove size = ...\"" ); DEFINE_bool(enable_newlines_in_logs, false, From 60cdc95249ece9af75ed9090df707044061a9b74 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Thu, 10 Jul 2025 20:50:43 +0300 Subject: [PATCH 33/62] * updated to 0.7.9 * fixed table loading crash * fixed garbage in init log * code cleanup --- include/lowlevelotmdatastructs.h | 49 ++++++------ include/tm.h | 11 --- source/CMakeLists.txt | 2 +- source/RestAPI/OtmMemoryServiceWorker.cpp | 1 - .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 77 +++---------------- 5 files changed, 35 insertions(+), 105 deletions(-) diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 70621ea2..8f7df64a 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -2435,8 +2435,6 @@ class TMCursor long _recordKey{}; ushort _targetKey{}; public: - - int SplitProposalKeyIntoRecordAndTarget(const char *pszKey, long *pulKey, ushort *pusTargetNum); int parseAndSetInternalKey(const char* pszKey){ return SplitProposalKeyIntoRecordAndTarget(pszKey, &_recordKey, &_targetKey); @@ -2480,38 +2478,37 @@ class TMCursor typedef struct _FCTDATA { std::string fileData; - ULONG lMagicWord; // FUNCDATA area identifier - LONG lCheckSum; // checksum of FUNCDATA area - SHORT sLastFunction; // last function performed + ULONG lMagicWord{}; // FUNCDATA area identifier + LONG lCheckSum{}; // checksum of FUNCDATA area + SHORT sLastFunction{}; // last function performed // area for session data - std::atomic fComplete; // current process is complete flag - CHAR szEqfResFile[MAX_EQF_PATH];// name of resource file - CHAR szSystemPropPath[MAX_EQF_PATH]; // system properties path - CHAR szMsgFile[MAX_EQF_PATH]; // message file for error messages - CHAR szError[256]; // text buffer for error messages - DDEMSGBUFFER LastMessage; // buffer for last error message - HWND hwndErrMsg; // handle for error mesages (HWND_FUNCIF) + //std::atomic + BOOL fComplete{}; // current process is complete flag + CHAR szEqfResFile[MAX_EQF_PATH]{};// name of resource file + CHAR szSystemPropPath[MAX_EQF_PATH]{}; // system properties path + CHAR szMsgFile[MAX_EQF_PATH]{}; // message file for error messages + CHAR szError[256]{}; // text buffer for error messages + DDEMSGBUFFER LastMessage{}; // buffer for last error message + HWND hwndErrMsg{}; // handle for error mesages (HWND_FUNCIF) // area used by mem import - MEM_LOAD_DLG_IDA MemLoadIda; // pointer to analysis input data - USHORT usMemLoadPhase; // current/next phase of TM import - USHORT usMemLoadRC; // return code + MEM_LOAD_DLG_IDA MemLoadIda{}; // pointer to analysis input data + USHORT usMemLoadPhase{}; // current/next phase of TM import + USHORT usMemLoadRC{}; // return code // area used by mem organize - PVOID pvMemOrganizeCommArea; // pointer to organize CommArea + PVOID pvMemOrganizeCommArea{}; // pointer to organize CommArea // area used by mem export - PVOID pvMemExportCommArea; + PVOID pvMemExportCommArea{}; // general use buffer area - OBJNAME szObjName; // buffer for object names + OBJNAME szObjName{}; // buffer for object names // current progress of some nonDDE functions //USHORT usProgress; - USHORT usExportProgress; - - + USHORT usExportProgress{}; USHORT MemFuncExportProcess(); USHORT MemFuncPrepExport ( @@ -2523,13 +2520,13 @@ typedef struct _FCTDATA std::shared_ptr mem; proxygen::ResponseHandler* responseHandler = nullptr; - TMCursor startingInternalKey; - TMCursor nextInternalKey; + TMCursor startingInternalKey{}; + TMCursor nextInternalKey{}; ulong numOfProposalsRequested = 0; - folly::IOBufQueue bufQueue; - MutexTimeout tmLockTimeout; - MutexTimeout tmListTimeout; + folly::IOBufQueue bufQueue{}; + MutexTimeout tmLockTimeout{}; + MutexTimeout tmListTimeout{}; } FCTDATA, *PFCTDATA; diff --git a/include/tm.h b/include/tm.h index cda4f82f..6c86426e 100644 --- a/include/tm.h +++ b/include/tm.h @@ -3566,18 +3566,7 @@ USHORT NTMCheckForUpdates(EqfMemory *); // magic word for recognition of tersed name tables // Note: this ULONG value has to ensure that it will never been used // as a name so it is prefixed and suffixed by a 0x00 -#define TERSEMAGICWORD 0x0019FF00L -// structure of header for tersed name tables -typedef struct _TERSEHEADER -{ - USHORT usAllocSize; // alloc size in untersed tables - USHORT usMaxEntries; // max entries in untersed tables - ULONG ulMagicWord; // magic word for tersed name - USHORT usDataSize; // size of data area when expanded - USHORT usCompression; // type of compression used for data - // (values as defined in EQFCMPR.H) -} TERSEHEADER, *PTERSEHEADER; BOOL TMDelTargetClb( BytesRef TmRecord, // ptr to TM record diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 8eafa422..1fa22042 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 8) +SET(APP_VERSION_MINOR 9) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index 99d61ff5..9831f577 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -391,7 +391,6 @@ void reorganizeMemoryProcess( void* pvData ) } // update memory status if(pRIDA && pRIDA->pMem){ - std::string strError; pRIDA->pMem->reorganizeDone(_rc_, pData->szError ); pRIDA->pMem.reset(); pRIDA->memRef.reset(); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 761effbb..dae61399 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -963,14 +963,6 @@ USHORT EqfMemory::NTMReadLongNameTable() if (usRC == NO_ERROR) { - // unterse data if table data is tersed - PTERSEHEADER pTerseHeader = (PTERSEHEADER)LongNames.pszBuffer.data(); - - if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) - { -T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be deleted" ; - } /* endif */ - // setup pointer array for long names if (usRC == NO_ERROR) { @@ -1178,62 +1170,10 @@ int NTMLongNameTableCompCaseIgnore( USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTable) { USHORT usRc = 0; - // handle tersed name tables - if (usRc == NO_ERROR) - { - PTERSEHEADER pTerseHeader = (PTERSEHEADER)oldTmxTable.data(); - std::vector pNewArea; // ptr to unterse data area - - if (pTerseHeader->ulMagicWord == TERSEMAGICWORD) - { - T5LOG(T5FATAL) << "commented out code - probably should never execute, so can be deleted" ; - #ifdef TEMPORARY_COMMENTED - // table is tersed... - pNewArea.resize(pTerseHeader->usDataSize); - // allocate buffer for untersed data - - // unterse data - if (usRc == NO_ERROR) - { - ULONG ulNewLen = 0; - - memcpy(pNewArea.data(), (PBYTE)oldTmxTable.data() + sizeof(TERSEHEADER), - *pulSize - sizeof(TERSEHEADER)); - - T5LOG(T5ERROR) << ":: TEMPORARY_COMMENTED temcom_id = 51 in NTMLoadNameTable"; -#ifdef TEMPORARY_COMMENTED - if (!fUtlHuffmanExpand((PUCHAR)pNewArea, pTerseHeader->usDataSize, - &ulNewLen)) - { - LOG_AND_SET_RC(usRc, T5WARNING, ERROR_NOT_ENOUGH_MEMORY); // expand failed most likely to - // memory shortage - } /* endif */ -#endif - } /* endif */ - - // set table data and cleanup - if (usRc == NO_ERROR) - { - // set size of name table - *pulSize = pTerseHeader->usDataSize; - - // free tersed data area - // UtlAlloc( (PVOID *)pTMTable, 0L, 0L, NOMSG ); - - // anchor untersed data area - //*pTMTable = pNewArea; - - // avoid free of untersed data area - // pNewArea = NULL; - } /* endif */ - #endif - } /* endif */ - } /* endif */ - // special handling for language name table: // correct word delimiter in "Other Languages" which is // 0xA0 under Windows (due to OemToAnsi) but should be 0xFF - if ((usRc == NO_ERROR) && (ulTableKey == LANG_KEY)) + if (ulTableKey == LANG_KEY) { LONG lRest = oldTmxTable.size();; PBYTE pbTemp = (PBYTE)oldTmxTable.data(); @@ -1250,13 +1190,18 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab if (usRc == NO_ERROR) { - //pTMTable->ulMaxEntries1 = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; pTMTable->stTmTableEntry.clear(); - int numOfEntries = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; - pTMTable->stTmTableEntry.resize(numOfEntries);// + 1)); - //if (pTMTable->stTmTableEntry.size()) + unsigned long numOfEntries = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; + unsigned long numOfEntriesCapacity = oldTmxTable.size()/sizeof(TMX_TABLE_ENTRY); + if(numOfEntries > numOfEntriesCapacity){ + T5LOG(T5ERROR) << "in the saved table num of entries"<< numOfEntries + <<" is greated than how much was saved(tableBytes.size()/sizeof(TMX_TABLE_ENTRY))=" <stTmTableEntry.resize(numOfEntries); memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, numOfEntries/*(pTMTable->stTmTableEntry.size()-1)*/ * sizeof(TMX_TABLE_ENTRY)); - } return usRc; } From 8668e1c12dc6f5506c447336dd912f8d3dd4c99c Mon Sep 17 00:00:00 2001 From: orestdrag Date: Fri, 11 Jul 2025 18:59:25 +0300 Subject: [PATCH 34/62] * updated to v0.7.10 --- include/Proposal.h | 4 + include/otm.h | 3 + include/requestdata.h | 39 ++-- source/CMakeLists.txt | 2 +- source/RestAPI/OTMMSJSONFactory.h | 2 +- source/opentm2/OTMFUNC.cpp | 1 - .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 3 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 10 +- source/opentm2/core/requestdata.cpp | 189 ++++-------------- source/opentm2/core/utilities/OtmProposal.cpp | 3 +- source/otmd.cpp | 3 +- 11 files changed, 72 insertions(+), 187 deletions(-) diff --git a/include/Proposal.h b/include/Proposal.h index c042755d..aac8e913 100644 --- a/include/Proposal.h +++ b/include/Proposal.h @@ -437,6 +437,10 @@ class OtmProposal /*! \brief Proposal data has been filled flag */ bool fFilled{}; + /*! \brief when saving request, if set to true, save segment with differen context as new(as was), otherwise if only context differs, replace previous segment */ + //bool fCheckContextWhenReplacingSegment{}; + int fNewEntryIfOnlyContextAndTimestampDifferent{}; + /*! \brief Index of memory when looking up in a list of memories */ int iMemoryIndex{}; diff --git a/include/otm.h b/include/otm.h index d8aa03bc..f64a449e 100644 --- a/include/otm.h +++ b/include/otm.h @@ -16,6 +16,7 @@ /**********************************************************************/ #define NTMREQUESTNEWKEY 0xFFFFFFFF +DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); //+----------------------------------------------------------------------------+ // Internal function @@ -79,8 +80,10 @@ struct ImportStatusDetails{ bool fReorganize{0}; // true for reorganize call, false for import size_t activeSegment{0}; InclosingTagsBehaviour inclosingTagsBehaviour{InclosingTagsBehaviour::saveAll}; + BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; void reset(){ + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; firstInvalidSegmentsSegNums.clear(); firstInvalidSegmentsSegNums.reserve(100); importMsg.str(""); diff --git a/include/requestdata.h b/include/requestdata.h index b780eb07..e188f35e 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -159,25 +159,17 @@ class ListTMRequestData:public RequestData{ //class ProxygenService::ProxygenStats; /*! \brief Data area for the processing of the importMemory function */ -class IMPORTMEMORYDATA +struct IMPORTMEMORYDATA { -public: - //HSESSION hSession; - //OtmMemoryServiceWorker *pMemoryServiceWorker; - char szMemory[260]; - char szInFile[260]; - char szError[512]; + char szMemory[260]{}; + char szInFile[260]{}; + char szError[512]{}; std::shared_ptr mem; BOOL fDeleteTmx = false; - //BOOL fImpoortFromFile = 0; - //folly::IOBuf fileData; - std::string fileData; + std::string fileData{}; InclosingTagsBehaviour inclosingTagsBehaviour = InclosingTagsBehaviour::saveAll; - MutexTimeout tmLockTimeout; - MutexTimeout tmListTimeout; - //ushort * pusImportPersent = nullptr; - //ImportStatusDetails* importDetails = nullptr; - //OtmMemoryServiceWorker::std::shared_ptr pMem = nullptr; + MutexTimeout tmLockTimeout{}; + MutexTimeout tmListTimeout{}; }; class FlushMemRequestData:public RequestData{ @@ -195,22 +187,21 @@ class ImportRequestData:public RequestData{ std::string strTmxData; bool isBase64; - - protected: int parseJSON() override ; int checkData() override ; int execute() override ; BOOL fClose = false; - //MEMORY_STATUS lastImportStatus = AVAILABLE_STATUS; // to restore in case we would break import before calling closemem - //MEMORY_STATUS lastStatus = AVAILABLE_STATUS; + BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; InclosingTagsBehaviour inclosingTagsBehaviour; IMPORTMEMORYDATA* pData = nullptr; - long timeout = 0; + int64_t timeout = 0; }; #include + +#ifdef TO_BE_REMOVED class ImportStreamRequestData:public RequestData{ public: ImportStreamRequestData(): RequestData(COMMAND::IMPORT_MEM_STREAM) { }; @@ -227,7 +218,7 @@ class ImportStreamRequestData:public RequestData{ IMPORTMEMORYDATA* pData = nullptr; long timeout = 0; }; - +#endif class ImportLocalRequestData:public RequestData{ public: ImportLocalRequestData(): RequestData(COMMAND::IMPORT_LOCAL_MEM) {}; @@ -365,7 +356,8 @@ class ReorganizeRequestData:public RequestData{ public: ReorganizeRequestData(): RequestData(COMMAND::REORGANIZE_MEM) {}; protected: - int parseJSON() override {return 0;}; + bool fNewEntryIfOnlyContextAndTimestampDifferent{}; + int parseJSON() override; int checkData() override; int execute () override; }; @@ -546,6 +538,7 @@ class DeleteEntriesReorganizeRequestData: public RequestData{ protected: SearchFilterFactory searchFilterFactory; + BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; int parseJSON() override ; int checkData() override ; @@ -560,7 +553,7 @@ class UpdateEntryRequestData: public RequestData{ int reset(); BOOL fSave2Disk = 1; - OtmProposal Data ; + OtmProposal Data{}; protected: int parseJSON() override; int checkData() override; diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 1fa22042..778989ab 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 9) +SET(APP_VERSION_MINOR 10) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/OTMMSJSONFactory.h b/source/RestAPI/OTMMSJSONFactory.h index d9bbfd1d..bebaae33 100755 --- a/source/RestAPI/OTMMSJSONFactory.h +++ b/source/RestAPI/OTMMSJSONFactory.h @@ -44,7 +44,7 @@ class JSONFactory typedef struct _JSONPARSECONTROL { - wchar_t szName[40]; // name of the parameter + wchar_t szName[240]; // name of the parameter PARMTYPE type; // type of the parameter void *pvValue; // pointer to a buffer receiving the value int iBufferLen; // size of the buffer in number of characters diff --git a/source/opentm2/OTMFUNC.cpp b/source/opentm2/OTMFUNC.cpp index cac194b4..94eb851c 100755 --- a/source/opentm2/OTMFUNC.cpp +++ b/source/opentm2/OTMFUNC.cpp @@ -127,7 +127,6 @@ USHORT EqfImportMem ( IMPORTMEMORYDATA* pData, - //std::shared_ptr mem, // Eqf session handle LONG lOptions // options for Translation Memory import ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 1093744c..94b15c1a 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -754,8 +754,6 @@ USHORT MemFuncImportProcess if ( !MemLoadStart( &(pData->MemLoadIda), HWND_FUNCIF, pData->mem->importDetails ) ) { - //TMManager *pFactory = TMManager::GetInstance(); - //pFactory->closeMemory( pLIDA->mem ); if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); pData->usMemLoadRC = UtlQueryUShort( QS_LASTERRORMSGID ); if ( pData->usMemLoadRC == 0 ) pData->usMemLoadRC = ERROR_MEMIMP_ERROR; @@ -910,6 +908,7 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT pLIDA->pProposal->setContext( pSegment->szContext ); pLIDA->pProposal->setSegmentId( pSegment->lSegNum ); pLIDA->pProposal->pInputSentence.reset(); + pLIDA->pProposal->fNewEntryIfOnlyContextAndTimestampDifferent = pLIDA->mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent; pLIDA->pProposal->pInputSentence = std::make_unique(pSegment->szSource, pSegment->szTarget); if(pLIDA->pProposal->pInputSentence->wasParsedSuccessfully()){ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index eb71849e..789b244b 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -93,14 +93,16 @@ bool isAddDataIsTheSame(OtmProposal& TmProposal,TMX_TARGET_CLB& Clb) wchar_t szBuff[MAX_SEGMENT_SIZE]{}; USHORT usAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); - if(Clb.usAddDataLen != usAddDataLen) + if(TmProposal.fNewEntryIfOnlyContextAndTimestampDifferent && Clb.usAddDataLen != usAddDataLen) return false; if (Clb.usAddDataLen) { - NtmGetAddData( &Clb, ADDDATA_CONTEXT_ID, szBuff ); - if(wcscmp(szBuff,TmProposal.szContext)) - return false; + if(TmProposal.fNewEntryIfOnlyContextAndTimestampDifferent){ + NtmGetAddData( &Clb, ADDDATA_CONTEXT_ID, szBuff ); + if(wcscmp(szBuff,TmProposal.szContext)) + return false; + } szBuff[0]='\0'; NtmGetAddData( &Clb, ADDDATA_ADDINFO_ID, szBuff ); if(wcscmp(szBuff,TmProposal.szAddInfo)) diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 905fdb45..c1bca058 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -254,6 +254,8 @@ DECLARE_int64(tmRequestLockDefaultTimeout); DECLARE_int64(tmListLockDefaultTimeout); DECLARE_int64(tmLockDefaultTimeout); +DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); + int RequestData::setMutexTimeouts(){ //set default timeouts if they were not set in request tmLockTimeout.setTimeout_ms(tmMutexTimeoutMs == -1? FLAGS_tmLockDefaultTimeout : tmMutexTimeoutMs); @@ -941,6 +943,7 @@ int ImportRequestData::parseJSON(){ std::string name; std::string value; + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; try{ while ( _rc_ == 0 ) { @@ -965,6 +968,8 @@ int ImportRequestData::parseJSON(){ } else if(strcasecmp(name.c_str(), "timeout") == 0){ timeout = std::stol(value); + }else if(strcasecmp(name.c_str(), "newEntryIfOnlyContextAndTimestampDifferent") == 0){ + fNewEntryIfOnlyContextAndTimestampDifferent = std::stol(value); }else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; @@ -1073,138 +1078,6 @@ int ImportRequestData::checkData(){ return 0; } - -int ImportStreamRequestData::parseJSON(){ - T5LOG( T5INFO) << "+POST " << strMemName << "/import"; - if ( _rc_ != 0 ) - { - return buildErrorReturn( _rc_, "parseJSON, rc is not 0 at the start of function", INTERNAL_SERVER_ERROR ); - } /* endif */ - - if ( strMemName.empty() ) - { - return buildErrorReturn( _rc_, "import::Missing name of memory", BAD_REQUEST ); - } /* endif */ - - // check if memory exists - if(int existscode = TMManager::GetInstance()->TMExistsOnDisk(strMemName)) - { - std::string msg = "import::Missing tm files on disk, code="; - msg += std::to_string(existscode); - return buildErrorReturn( existscode, msg.c_str(), NOT_FOUND); - } - - // find the memory to our memory list - // extract TMX data - int loggingThreshold = -1; //0-develop(show all logs), 1-debug+, 2-info+, 3-warnings+, 4-errors+, 5-fatals only - - void *parseHandle = json_factory.parseJSONStart( strBody, &_rc_ ); - if ( parseHandle == NULL ) - { - return buildErrorReturn( _rc_, "import::Missing or incorrect JSON data in request body", BAD_REQUEST ); - } /* end */ - - std::string name; - std::string value; - try{ - while ( _rc_ == 0 ) - { - _rc_ = json_factory.parseJSONGetNext( parseHandle, name, value ); - if ( _rc_ == 0 ) - { - if(strcasecmp(name.c_str(), "timeout") == 0){ - timeout = std::stol(value); - }else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ - loggingThreshold = std::stoi(value); - T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; - T5Logger::GetInstance()->SetLogLevel(loggingThreshold); - }else if ( strcasecmp( name.c_str(), "tmMutexTimeout" ) == 0 ) - { - tmMutexTimeoutMs = std::stoll(value.c_str()); - } - else if ( strcasecmp( name.c_str(), "tmListMutexTimeout" ) == 0 ) - { - tmListMutexTimeoutMs = std::stoll(value.c_str()); - } - else if ( strcasecmp( name.c_str(), "requestTMMutexTimeout" ) == 0 ) - { - requestTMMutexTimeoutMs = std::stoll(value.c_str()); - } - else if(strcasecmp(name.c_str(), "framingTags") == 0){ - std::string strInclosingTagsBehaviour = value; - if(strcasecmp(value.c_str(), "saveAll") == 0){ - inclosingTagsBehaviour = InclosingTagsBehaviour::saveAll; - }else if(strcasecmp(value.c_str(), "skipAll") == 0){ - inclosingTagsBehaviour = InclosingTagsBehaviour::skipAll; - }else if(strcasecmp(value.c_str(), "skipPaired") == 0){ - inclosingTagsBehaviour = InclosingTagsBehaviour::skipPaired; - } - }else{ - T5LOG( T5WARNING) << "JSON parsed unexpected name, " << name; - } - }else if(_rc_ != 2002){// _rc_ != INFO_ENDOFPARAMETERLISTREACHED - std::string msg = "failed to parse JSON, _rc_ = " + std::to_string(_rc_); - return buildErrorReturn(_rc_, msg.c_str(), BAD_REQUEST); - } - } /* endwhile */ - json_factory.parseJSONStop( parseHandle ); - } - catch(...) - { - return buildErrorReturn(444, "Json parsing failed", 400); - } - if(_rc_ == 2002) _rc_ = 0; - return _rc_; -} - -int ImportStreamRequestData::checkData() -{ - return 0; -} - -int ImportStreamRequestData::execute(){ - if ( mem == nullptr ) - { - return buildErrorReturn( 404, "mem not found or can't be opened" ); - } - // close the memory - when open - if ( false == memIsAvailableToOperate(mem.get()) ) - { - std::string msg = "mem is not available to operate, status= " + mem->getStatusString(); - return buildErrorReturn( 500, msg.c_str() ); - } - - mem->eStatus = IMPORT_RUNNING_STATUS; - mem->eImportStatus = IMPORT_RUNNING_STATUS; - - T5LOG( T5DEBUG) << "status for " << strMemName << " was changed to import"; - // start the import background process - pData = new( IMPORTMEMORYDATA ); - pData->inclosingTagsBehaviour = inclosingTagsBehaviour; - strcpy( pData->szMemory, strMemName.c_str() ); - //pData->fileData = std::move(fileData); - - if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; - } - - mem->importDetails->reset(); - - LONG lCurTime = 0; - time( &lCurTime ); - mem->importDetails->lImportStartTime = lCurTime; - mem->importDetails->lImportTimeoutSec = timeout; - pData->mem = mem; - pData->tmLockTimeout = tmLockTimeout; - - //importMemoryProcess(pData);//to do in same thread - std::thread worker_thread(importMemoryProcess, pData); - worker_thread.detach(); - - return( CREATED ); -} - - int SaveAllTMsToDiskRequestData::execute(){ if(_rc_) { outputMessage = "Error: Can't save tm files on disk"; @@ -1292,6 +1165,7 @@ int ImportRequestData::execute(){ } mem->importDetails->reset(); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; LONG lCurTime = 0; time( &lCurTime ); @@ -1441,20 +1315,6 @@ int ReorganizeRequestData::checkData(){ return 0; } -/* -int ParseSearchType(std::string& inputStr, SearchType& param){ - if(!strcasecmp(inputStr.c_str(),"concordance")) - { - param = SearchType::CONCORDANCE; - }else if(!strcasecmp(inputStr.c_str(),"EXACT")){ - param = SearchType::EXACT; - }else{ - T5LOG(T5ERROR) << ""; - return -1; - } - return 0; -}//*/ - int ProposalFilter::StrToFilterType(const char* str, FilterType &filterType, FilterOptions& options){ filterType = FilterType::UNKNOWN; if(!str){ @@ -1673,6 +1533,26 @@ USHORT MemFuncOrganizeProcess PFCTDATA pData // function I/F session data ); +int ReorganizeRequestData::parseJSON(){ + this->fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + + // parse input parameters + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); + + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + + JSONFactory::JSONPARSECONTROL parseControl[] = { + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), 0 }, + { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, + { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, + { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, + { L"", JSONFactory::ASCII_STRING_PARM_TYPE, NULL, 0 } + }; + _rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); + return _rc_; +} + + int ReorganizeRequestData::execute(){ if ( mem == nullptr ) { @@ -1682,7 +1562,7 @@ int ReorganizeRequestData::execute(){ // close the memory - when open if ( false == memIsAvailableToOperate(mem.get()) ) { - std::string msg = "mem is not available to operate, status= " + mem->getStatusString(); + std::string msg = "mem is not avaiflable to operate, status= " + mem->getStatusString(); return buildErrorReturn( 500, msg.c_str() ); } @@ -1720,6 +1600,7 @@ int ReorganizeRequestData::execute(){ } mem->importDetails->reset(); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; LONG lCurTime = 0; time( &lCurTime ); @@ -1764,6 +1645,8 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ // parse input parameters std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); + + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; JSONFactory::JSONPARSECONTROL parseControl[] = { //searchFilterFactory @@ -1773,6 +1656,7 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ { L"context", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.context ), 0 }, { L"author", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.author ), 0 }, { L"document", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.document ), 0 }, + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), 0 }, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, @@ -1857,18 +1741,15 @@ int DeleteEntriesReorganizeRequestData::execute(){ } mem->importDetails->reset(); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; LONG lCurTime = 0; time( &lCurTime ); pRIDA->pMem->importDetails->lImportStartTime = lCurTime; mem->importDetails->fReorganize = true; - //mem.reset(); - //memRef.reset(); - pRIDA->m_reorganizeFilters = searchFilterFactory.getListOfFilters(); - pRIDA->tmLockTimeout = tmLockTimeout; - + pRIDA->tmLockTimeout = tmLockTimeout; } //worker thread @@ -2216,6 +2097,7 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr json_factory.addParmToJSON( outputMessage, "tmxSegmentCount", mem->importDetails->segmentsCount ); json_factory.addParmToJSON( outputMessage, "importRuntimeSec", mem->importDetails->lImportRunTimeSec ); json_factory.addParmToJSON( outputMessage, "importTimeoutSec", mem->importDetails->lImportTimeoutSec ); + json_factory.addParmToJSON( outputMessage, "newEntryIfOnlyContextAndTimestampDifferent", mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent ); std::string invalidSegmRCs; for(auto [errCode, errCount]: mem->importDetails->invalidSegmentsRCs){ invalidSegmRCs += std::to_string(errCode) + ":" + std::to_string(errCount) +"; "; @@ -2416,7 +2298,7 @@ bool jsonAddFlagToList(std::stringstream &ss, const google::CommandLineFlagInfo return false; // flag skipped } ss << "\t{\t"; - std::string flagName = "\"" + flag.name + "\""; + 0 std::string flagName = "\"" + flag.name + "\""; std::string flagValue = "\"" + flag.current_value + "\""; ss << (flag.is_default? " " : "*");//to mark not default flags ss << (flag.filename.ends_with("otmd.cpp")? "! " : " ");//to mark t5memory own flags @@ -3115,6 +2997,7 @@ int UpdateEntryRequestData::parseJSON(){ { L"customId", JSONFactory::ASCII_STRING_PARM_TYPE, &(Data.szCustomId), sizeof( Data.szCustomId) / sizeof( Data.szCustomId[0] ) }, { L"recordKey", JSONFactory::INT_PARM_TYPE , &(recordKey), 0 }, { L"targetKey", JSONFactory::INT_PARM_TYPE , &(targetKey), 0 }, + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(Data.fNewEntryIfOnlyContextAndTimestampDifferent), 0 }, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, diff --git a/source/opentm2/core/utilities/OtmProposal.cpp b/source/opentm2/core/utilities/OtmProposal.cpp index 37049760..e9208267 100755 --- a/source/opentm2/core/utilities/OtmProposal.cpp +++ b/source/opentm2/core/utilities/OtmProposal.cpp @@ -35,12 +35,13 @@ OtmProposal::~OtmProposal() }; /* operations */ - +DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); void OtmProposal::clear()//bool clearKeys) { pInputSentence.reset(); TMCursor currentKey, nextKey; *this = OtmProposal{}; + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent;//default value } OtmProposal::eProposalType getMemProposalType( char *pszType ) diff --git a/source/otmd.cpp b/source/otmd.cpp index 3f07e766..32a3eec7 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -202,7 +202,8 @@ DEFINE_bool(log_memmove_in_compareputdata, false, "if set to true, when saving s DEFINE_bool(enable_newlines_in_logs, false, "(not working - instead of using function to remove newlines in runtime, logs was edited to have better perfomance)if set to true, would keep newline symbols in the logs, otherwise(by default) newlines would be removed and logs would be oneliners. This just calls function to remove newlines," ); - +DEFINE_bool(newEntryIfOnlyContextAndTimestampDifferent, true, "If it is set to true, the behavior should be: A new entry should be saved to the TM, if only timestamp and context are different and all other params the same;\t If it is set to false, the beahvior should be: if context is different and timestamp is different, the context and timestamp of the new to be saved segment should overwrite the context and segment of an entry, that already exists and where all other params are the same as in the new segment. This is basically the same behavior it would have, if context would not be set for both segments (the old and the new)." ); + DEFINE_bool(suppressTmVersionCheck, false, "If sets to true, allow to open tms, which was created in version, that's outside of supported versions scope. You can use this if you have version mismatch error"); DEFINE_int32(t5globversion_min_supported, 0, "sets minimal global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z"); DEFINE_int32(t5majversion_min_supported, 5, "sets minimal major version in which tm was created to be supported for open in current. it's the Y(MAJ) in vX.Y.Z"); From 0f7db63cd2e008eb6c892109af5be38b65fff2f7 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 14 Jul 2025 22:41:11 +0300 Subject: [PATCH 35/62] * updated to v0.7.11 * fixed compatibility crash when table is empty from 0.6 tm * code cleanup --- source/CMakeLists.txt | 2 +- source/RestAPI/OTMMSJSONFactory.cpp | 4 ---- source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 5 +++++ source/opentm2/core/requestdata.cpp | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 778989ab..d13e5fbb 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 10) +SET(APP_VERSION_MINOR 11) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/OTMMSJSONFactory.cpp b/source/RestAPI/OTMMSJSONFactory.cpp index bb47aff0..2332c266 100755 --- a/source/RestAPI/OTMMSJSONFactory.cpp +++ b/source/RestAPI/OTMMSJSONFactory.cpp @@ -840,10 +840,6 @@ int JSONFactory::extractString } // convert to UTF8 - //int iUTF8Len; - //iUTF8Len = WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iLen, 0, 0, 0, 0 ); - //std::string strUTF8( iUTF8Len, '\0' ); - //WideCharToMultiByte( CP_UTF8, 0, (LPWSTR)strUTF16.c_str(), iLen, &strUTF8[0], iUTF8Len, 0, 0 ); std::string strUTF8 = EncodingHelper::convertToUTF8(strUTF16); // add UTF8 character to target string for ( int i = 0; i < strUTF8.length(); i++ ) string[iTargetPos++] = strUTF8[i]; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index dae61399..3a23c5b6 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -1088,6 +1088,11 @@ USHORT EqfMemory::NTMAddNameToTable( AddToLongNameTables(pszName, pusID); } + if(pstTMTable->stTmTableEntry.empty()){// in 0.7.shouldn't be empty, but could be when reading from 0.6 + T5LOG(T5ERROR) << "tried to write to empty tm table #" << usTableType<<"; error was already catched and table was resized to 1"; + pstTMTable->stTmTableEntry.push_back(TMX_TABLE_ENTRY{}); + } + TMX_TABLE_ENTRY& newEntry = pstTMTable->stTmTableEntry.back(); if(*pusID == 0) *pusID = pstTMTable->stTmTableEntry.size(); newEntry.usId = *pusID; diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index c1bca058..5fd48b0d 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -2298,7 +2298,7 @@ bool jsonAddFlagToList(std::stringstream &ss, const google::CommandLineFlagInfo return false; // flag skipped } ss << "\t{\t"; - 0 std::string flagName = "\"" + flag.name + "\""; + std::string flagName = "\"" + flag.name + "\""; std::string flagValue = "\"" + flag.current_value + "\""; ss << (flag.is_default? " " : "*");//to mark not default flags ss << (flag.filename.ends_with("otmd.cpp")? "! " : " ");//to mark t5memory own flags From d2033633d9dbdef726cac4e2753403f732de9505 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Fri, 18 Jul 2025 19:03:55 +0300 Subject: [PATCH 36/62] * updated to v0.7.12 + added GET_TABLES endpoing + added matchTargetOnUpdate param to update segment, import tmx and reorganize, implemented functionality for it + added matchTargetOnUpdate flag for default value - code cleanup + added return id of entry from table if table entry is not found(authors, files, languages, langgroups) --- include/Proposal.h | 5 +- include/lowlevelotmdatastructs.h | 4 + include/otm.h | 8 + include/requestdata.h | 29 ++- include/tm.h | 15 +- source/RestAPI/ProxygenServer.cpp | 6 +- source/RestAPI/ProxygenStats.cpp | 20 ++ source/RestAPI/ProxygenStats.h | 18 ++ .../opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp | 1 + .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 2 + .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 12 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 27 +-- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 203 +++++++++--------- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 169 ++++----------- source/opentm2/core/requestdata.cpp | 107 ++++++++- source/opentm2/core/tm.cpp | 4 +- source/opentm2/core/utilities/OtmProposal.cpp | 5 + source/otmd.cpp | 5 +- 18 files changed, 353 insertions(+), 287 deletions(-) diff --git a/include/Proposal.h b/include/Proposal.h index aac8e913..939cc460 100644 --- a/include/Proposal.h +++ b/include/Proposal.h @@ -439,7 +439,9 @@ class OtmProposal /*! \brief when saving request, if set to true, save segment with differen context as new(as was), otherwise if only context differs, replace previous segment */ //bool fCheckContextWhenReplacingSegment{}; - int fNewEntryIfOnlyContextAndTimestampDifferent{}; + bool fNewEntryIfOnlyContextAndTimestampDifferent{}; + bool fMatchTargetOnUpdate{}; + bool fIgnoreSegIdOnUpdate{}; /*! \brief Index of memory when looking up in a list of memories */ int iMemoryIndex{}; @@ -468,6 +470,7 @@ class OtmProposal TMCursor currentInternalKey{}; TMCursor nextInternalKey{}; + int moveInternalKeyToTheNextPosition() { currentInternalKey = nextInternalKey; nextInternalKey.reset(); diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 8f7df64a..692401ee 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -92,6 +92,8 @@ typedef char* PSZ; SHUTDOWN, DELETE_MEM, EXPORT_MEM_TMX, + GET_TABLES, + ADD_TO_TABLE, EXPORT_MEM_INTERNAL_FORMAT, EXPORT_MEM_INTERNAL_FORMAT_STREAM, REORGANIZE_MEM, @@ -135,6 +137,8 @@ constexpr std::initializer_list> CommandT { EXPORT_MEM_TMX_STREAM, "EXPORT_MEM_TMX_STREAM" }, { EXPORT_MEM_INTERNAL_FORMAT_STREAM, "EXPORT_MEM_INTERNAL_FORMAT_STREAM" }, { STATUS_MEM, "STATUS_MEM" }, + { GET_TABLES, "GET_TABLES"}, + { ADD_TO_TABLE, "ADD_TO_TABLE"}, { RESOURCE_INFO, "RESOURCE_INFO" }, { FLAGS_INFO, "FLAGS_INFO" }, diff --git a/include/otm.h b/include/otm.h index f64a449e..98008306 100644 --- a/include/otm.h +++ b/include/otm.h @@ -17,6 +17,10 @@ #define NTMREQUESTNEWKEY 0xFFFFFFFF DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); +DECLARE_bool(matchTargetOnUpdate); +//DECLARE_bool(ignoreSegIdOnUpdate); + +constexpr bool FLAGS_ignoreSegIdOnUpdate = true;//not used flag //+----------------------------------------------------------------------------+ // Internal function @@ -81,9 +85,13 @@ struct ImportStatusDetails{ size_t activeSegment{0}; InclosingTagsBehaviour inclosingTagsBehaviour{InclosingTagsBehaviour::saveAll}; BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fMatchTargetOnUpdate{}; + BOOL fIgnoreSegIdOnUpdate{}; void reset(){ fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + //fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; firstInvalidSegmentsSegNums.clear(); firstInvalidSegmentsSegNums.reserve(100); importMsg.str(""); diff --git a/include/requestdata.h b/include/requestdata.h index e188f35e..63b491d4 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -141,6 +141,27 @@ class CreateMemRequestData: public RequestData{ int execute() override ; }; +class GetTablesMemRequestData: public RequestData{ +public: + GetTablesMemRequestData(): RequestData(COMMAND::GET_TABLES, "", "") {}; +protected: + int parseJSON() override{return 0;}; + int checkData() override{return 0;}; + int execute() override ; +}; + + +class AddToTableMemRequestData: public RequestData{ +public: + AddToTableMemRequestData(): RequestData(COMMAND::ADD_TO_TABLE, "", "") {}; +protected: + int tableKey{}; + int entryId{}; + char value[100]{}; + int parseJSON() override; + int checkData() override; + int execute() override ; +}; @@ -194,6 +215,8 @@ class ImportRequestData:public RequestData{ BOOL fClose = false; BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fMatchTargetOnUpdate{}; + BOOL fIgnoreSegIdOnUpdate{}; InclosingTagsBehaviour inclosingTagsBehaviour; IMPORTMEMORYDATA* pData = nullptr; int64_t timeout = 0; @@ -356,7 +379,9 @@ class ReorganizeRequestData:public RequestData{ public: ReorganizeRequestData(): RequestData(COMMAND::REORGANIZE_MEM) {}; protected: - bool fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fMatchTargetOnUpdate{}; + BOOL fIgnoreSegIdOnUpdate{}; int parseJSON() override; int checkData() override; int execute () override; @@ -539,6 +564,8 @@ class DeleteEntriesReorganizeRequestData: public RequestData{ protected: SearchFilterFactory searchFilterFactory; BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; + BOOL fMatchTargetOnUpdate{}; + BOOL fIgnoreSegIdOnUpdate{}; int parseJSON() override ; int checkData() override ; diff --git a/include/tm.h b/include/tm.h index 6c86426e..85985642 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1800,7 +1800,6 @@ class EqfMemory //: public TMX_CLB // std::vector pLongNames ; TMX_TABLE LangGroups; // table containing language group names std::vector psLangIdToGroupTable; // language ID to group ID table - LONG lLangIdToGroupTableUsed = 0; // size of table (bytes in use) PVOID pTagTable = nullptr; // tag table loaded for segment markup (TBLOADEDTABLE) BYTE getMajorVersion()const{ @@ -2021,10 +2020,7 @@ class EqfMemory //: public TMX_CLB USHORT NTMGetPointersToTable(USHORT, PTMX_TABLE *, PTMX_TABLE_ENTRY *); USHORT NTMAddNameToTable(PSZ, USHORT, PUSHORT); - PSZ NTMFindNameForID( - PUSHORT pusID, // intput - USHORT usTableType); - + USHORT FillClb(BytesRef, OtmProposal &); USHORT ComparePutData( BytesRef ppTmRecord, // ptr to ptr of tm record data buffer @@ -2101,7 +2097,7 @@ class EqfMemory //: public TMX_CLB size_t GetRAMSize() const; size_t GetExpectedRAMSize() const; - USHORT ExtractRecordV6( + USHORT ExtractRecord( BytesRef pTmRecord, PTMX_EXT_IN_W pTmExtIn, TMX_EXT_OUT_W *pTmExtOut); @@ -3041,7 +3037,7 @@ typedef struct _SEG_CTRL_DATA /*! \brief search a string in a proposal \param pProposal pointer to the proposal - \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) + \param pszSearch pointer to the search string (when fIgnoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ @@ -3052,7 +3048,7 @@ BOOL searchInProposal( /*! \brief search a string in a proposal \param pProposal pointer to the proposal - \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) + \param pszSearch pointer to the search string (when fIgnoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ @@ -3535,8 +3531,7 @@ VOID FillTmRecord(PTMX_SENTENCE, USHORT DetermineTmRecord(EqfMemory *, PTMX_SENTENCE, std::vector& pulSids ); -VOID FillTargetRecord(PTMX_SENTENCE, // PTMX_TAGTABLE_RECORD, - PSZ_W, USHORT, PTMX_TARGET_RECORD *, PTMX_TARGET_CLB); +VOID FillTargetRecord(PTMX_SENTENCE, PTMX_TARGET_RECORD, PTMX_TARGET_CLB); VOID DeleteOldestRecord(PTMX_RECORD, PULONG); diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index ae8e21d4..558c230d 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -307,7 +307,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { requestHandler->pRequest = new DeleteEntriesReorganizeRequestData(); }else if(urlCommand == "getentry"){ requestHandler->pRequest = new GetEntryRequestData(); - }else if(urlCommand == "import"){ // update + }else if(urlCommand == "import"){ requestHandler->pRequest = new ImportRequestData(); }else if(urlCommand == "importtmx"){ //requestHandler->pRequest = new ImportStreamRequestData(); @@ -318,8 +318,10 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { requestHandler->pRequest = new ImportLocalRequestData(); } }else if(methodStr == "GET"){ - if(urlCommand == "status"){ // update + if(urlCommand == "status"){ requestHandler->pRequest = new StatusMemRequestData(); + }else if(urlCommand == "tables"){ + requestHandler->pRequest = new GetTablesMemRequestData(); }else if(urlCommand == "reorganize"){ requestHandler->pRequest = new ReorganizeRequestData(); }else if(urlCommand == "download.tm"){ diff --git a/source/RestAPI/ProxygenStats.cpp b/source/RestAPI/ProxygenStats.cpp index 2353fc95..7599d90d 100644 --- a/source/RestAPI/ProxygenStats.cpp +++ b/source/RestAPI/ProxygenStats.cpp @@ -36,6 +36,16 @@ int ProxygenStats::recordRequest(int command) { statusMemReqCount_++; break; } + case COMMAND::GET_TABLES: + { + getTablesMemReqCount_++; + break; + } + case COMMAND::ADD_TO_TABLE: + { + addToTableMemReqCount_++; + break; + } case COMMAND::REORGANIZE_MEM: { reorganizeMemRequestCount_++; @@ -158,6 +168,16 @@ int ProxygenStats::addRequestTime(int command,milliseconds time) { { statusMemSumTime_ += time; break; + } + case COMMAND::GET_TABLES: + { + getTablesMemSumTime_ += time; + break; + } + case COMMAND::ADD_TO_TABLE: + { + addToTableMemSumTime_ += time; + break; } case COMMAND::REORGANIZE_MEM: { diff --git a/source/RestAPI/ProxygenStats.h b/source/RestAPI/ProxygenStats.h index ca0939f2..45e80682 100755 --- a/source/RestAPI/ProxygenStats.h +++ b/source/RestAPI/ProxygenStats.h @@ -81,6 +81,20 @@ class ProxygenStats { return statusMemSumTime_; } + virtual uint64_t getGetTablesMemRequestCount() { + return getTablesMemReqCount_; + } + virtual milliseconds getGetTablesMemSumTime() { + return getTablesMemSumTime_; + } + + virtual uint64_t getAddToTableMemRequestCount() { + return addToTableMemReqCount_; + } + virtual milliseconds getAddToTableMemSumTime() { + return addToTableMemSumTime_; + } + virtual uint64_t getFuzzyRequestCount() { return fuzzyReqCount_; } @@ -213,6 +227,8 @@ class ProxygenStats { atomic_uint64_t importMemReqCount_{0}; atomic_uint64_t importLocalReqCount_{0}; atomic_uint64_t statusMemReqCount_{0}; + atomic_uint64_t getTablesMemReqCount_{0}; + atomic_uint64_t addToTableMemReqCount_{0}; atomic_uint64_t flushMemReqCount_{0}; atomic_uint64_t fuzzyReqCount_{0}; @@ -249,6 +265,8 @@ class ProxygenStats { milliseconds multiUpdateReqSumTime_{0}; milliseconds deleteEntrySumTime_{0}; milliseconds deleteEntryReorganizeSumTime_{0}; + milliseconds getTablesMemSumTime_{0}; + milliseconds addToTableMemSumTime_{0}; milliseconds saveAllTmsSumTime_{0}; milliseconds getListOfMemoriesSumTime_{0}; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index 11513e78..e04d83c8 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -159,6 +159,7 @@ VOID EQFMemOrganizeProcess auto ll = T5Logger::GetInstance()->suppressLogging(); wchar_t seg_buff[OTMPROPOSAL_MAXSEGLEN+1]; pRIDA->pProposal->getSource(seg_buff, OTMPROPOSAL_MAXSEGLEN); + pRIDA->pProposal->fMatchTargetOnUpdate = pRIDA->pMem->importDetails->fMatchTargetOnUpdate; bool fValidXml = IsValidXml( seg_buff); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 94b15c1a..5487ab54 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -909,6 +909,8 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT pLIDA->pProposal->setSegmentId( pSegment->lSegNum ); pLIDA->pProposal->pInputSentence.reset(); pLIDA->pProposal->fNewEntryIfOnlyContextAndTimestampDifferent = pLIDA->mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent; + pLIDA->pProposal->fMatchTargetOnUpdate = pLIDA->mem->importDetails->fMatchTargetOnUpdate; + pLIDA->pProposal->fIgnoreSegIdOnUpdate = pLIDA->mem->importDetails->fIgnoreSegIdOnUpdate; pLIDA->pProposal->pInputSentence = std::make_unique(pSegment->szSource, pSegment->szTarget); if(pLIDA->pProposal->pInputSentence->wasParsedSuccessfully()){ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 2611479e..1ae7e5ab 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -84,7 +84,7 @@ USHORT EqfMemory::TmtXExtract if ( usRc == NO_ERROR ) { - usRc = ExtractRecordV6(pTmRecord, pTmExtIn, pTmExtOut ); + usRc = ExtractRecord(pTmRecord, pTmExtIn, pTmExtOut ); } /****************************************************************/ /* setup new starting point (do this even in the case we are */ @@ -124,7 +124,7 @@ USHORT EqfMemory::TmtXExtract return( usRc ); } -USHORT EqfMemory::ExtractRecordV6 +USHORT EqfMemory::ExtractRecord ( BytesRef pTmRecordData, PTMX_EXT_IN_W pTmExtIn, @@ -248,7 +248,8 @@ USHORT EqfMemory::ExtractRecordV6 lLeftTgtLen = 0; } /* endif */ - if(lLeftClbLen > 0){//need to add this check in 0.7 + //if(lLeftClbLen > 0)//need to add this check in 0.7? + { pByte += RECLEN(pTMXTargetRecord); pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); pTargetClb = (PTMX_TARGET_CLB)(pByte + pTMXTargetRecord->usClb); @@ -405,12 +406,10 @@ USHORT EqfMemory::FillExtStructure lTargetLen = pTMXTargetRecord->usClb - pTMXTargetRecord->usTarget; fOK = ( lTargetLen < MAX_SEGMENT_SIZE * sizeof(WCHAR) ); - // GQ 2008/06/18 Check if target string data is valid if ( fOK ) { - BYTE b = *pByte; - + BYTE b = *pByte; if ( (b != 0) && (b != BOCU_COMPRESS) ) { fOK = FALSE; // invalid compression ID @@ -434,7 +433,6 @@ USHORT EqfMemory::FillExtStructure //fill in the tag table name strcpy(pstExt->szTagTable, "OTMXUXLF"); - if(lTargetLen >= 0){ memcpy( pstExt->szTarget, pTargetString.c_str(), lTargetLen * sizeof(CHAR_W)); pstExt->szTarget[lTargetLen] = EOS; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 395303ea..fba39247 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -49,7 +49,7 @@ void NTMLogSegData( PSZ_W pszForm, PSZ_W pszSegData ); USHORT FillMatchTable( EqfMemory*, std::wstring&, PLONG, PTMX_TARGET_RECORD, PTMX_TARGET_CLB, - PTMX_MATCH_TABLE_W, PUSHORT, BOOL, PUSHORT, PUSHORT, + PTMX_MATCH_TABLE_W, PUSHORT, PUSHORT, PUSHORT, LONG, USHORT, PTMX_GET_W, BytesRef, USHORT, USHORT, int, int, USHORT, PBYTE); /**********************************************************************/ @@ -422,11 +422,9 @@ USHORT GetExactMatch PTMX_GET_OUT_W pTmGetOut // pointer to output struct ) { - BOOL fOK = TRUE; //success indicator std::vector lSids; //ptr to sentence ids USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator - LONG lKey; //tm record key USHORT usMatchEntries = 0; //nr of found matches BytesRef pTmRecord = pTmClb->pvTmRecord; @@ -746,7 +744,7 @@ USHORT ExactTest NULL, (USHORT)LANG_KEY, &usGetLang)) { - usGetLang = 1; + usGetLang = 0; } /* endif */ //get file name id of file name in the getin structure @@ -1123,7 +1121,7 @@ USHORT ExactTest if ( usTargetTranslationFlag == (USHORT)-1 ) usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; FillMatchTable( pTmClb, genericTagsTmSeg->getNormStr(), &lSourceLen, pTMXTargetRecord, pTMXTargetClb, - pstMatchTable, &usMatchLevel, TRUE, + pstMatchTable, &usMatchLevel, pusMatchEntries, &pSentence->usActVote, lKeyNum, usTgtNum, @@ -1273,7 +1271,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct PTMX_TARGET_CLB pTMXTargetClb, //ptr to target control block PTMX_MATCH_TABLE_W pstMatchTable, //getout match table struct PUSHORT pusMatchLevel, //how good is match - BOOL fTag, //target with/without subst. tags PUSHORT pusMatchEntries, //nr of entries in match table PUSHORT pusOverlaps, //nr of overlapping triples LONG lKeyNum, // key number @@ -1311,8 +1308,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct wcsncpy(SubstProp.szPropSource, pSourceString.c_str(), pSourceString.size());// *plSourceLen); if ( usRc == NO_ERROR ) { - PSZ_W pTarget; - //position at target string pByte = (PBYTE)pTMXTargetRecord; pByte += pTMXTargetRecord->usTarget; @@ -1336,16 +1331,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct (USHORT)LANG_KEY, SubstProp.szTargetLanguage, NULL ); } /* endif */ - - } /* endif */ - - - /********************************************************************/ - /* if src of prop is empty ( i.e. only inline tagging), add tagging */ - /********************************************************************/ - if (*plSourceLen <= 0 ) - { - fTag = TRUE; } /* endif */ if ( usRc == NO_ERROR ) @@ -1837,7 +1822,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block // we have to update the memory language table to keep the language group table up-to-date... if ( pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTargetId)) { - usTargetId = 1; // set default.. + usTargetId = 0; // set default.. } /* endif */ //strings are not equal, assume fuzzy @@ -1869,7 +1854,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block //compare target language group IDs if(pGetIn->fTargetLangIsPrefered){ - if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] ) + if (( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] )) { T5LOG( T5INFO) <<"FuzzyTest: Wrong target language!" ; fTestCLB = FALSE; @@ -2010,7 +1995,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block lTempSrcLen = ulSourceLen; FillMatchTable( pTmClb, pString, &lTempSrcLen, pTMXTargetRecord, pTMXTargetClb, - pstMatchTable, &usFuzzy, true, + pstMatchTable, &usFuzzy, pusMatchesFound, pusOverlaps, lKeyNum, usTgtNum, pGetIn, pSentence->pTagRecord, usModifiedTranslationFlag, 0, usWords, usDiffs, pTMXSourceRecord->usLangId, endOfMemRegion ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 789b244b..0b80308b 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -60,7 +60,8 @@ USHORT TMLoopAndDelTargetClb USHORT usPutFile, USHORT usPutAuthor, PBOOL fNewerTargetExists, - PINT pTargetKey + PINT pTargetKey, + PBOOL pfTargetMatch ); /**********************************************************************/ /* activate this define to get a faster memory */ @@ -1637,11 +1638,13 @@ USHORT EqfMemory::ComparePutData USHORT usPutLang = 0; // language id of target language USHORT usPutFile = 0; // file id of new entry + BOOL fTargetMatch = true;//default behaviour + //get id of target language in the put structure if (NTMGetIDFromName( TmProposal.szTargetLanguage, NULL, (USHORT)LANG_KEY, &usPutLang )) { - usPutLang = 1; + usPutLang = 0; } /* endif */ NTMGetIDFromName( TmProposal.szTargetAuthor, // get author ID @@ -1667,18 +1670,14 @@ USHORT EqfMemory::ComparePutData BOOL fNewerTargetExists = FALSE; LONG lLeftTgtLen; // remaining target length - NTASSERTLEN(RECLENB(pTmRecord), toTmxRecord(pTmRecord)->usFirstTargetRecord, 4712) - - // get length of target block to work with - //source strings equal - position at first target record - pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); + NTASSERTLEN(RECLENB(pTmRecord), toTmxRecord(pTmRecord)->usFirstTargetRecord, 4712); fStop = (usRc != 0); //RJ: 04/01/22: P018830: // loop through all target records - // delete entry if current segment has already been translated - TMLoopAndDelTargetClb(pTmRecord, TmProposal, usPutLang, usPutFile, usAuthorId, &fNewerTargetExists, &delTargetKey ); - + // delete entry if current segment has already been translated + TMLoopAndDelTargetClb(pTmRecord, TmProposal, usPutLang, usPutFile, usAuthorId, &fNewerTargetExists, &delTargetKey, &fTargetMatch); + // recalc since record may have changed during delete above! lLeftTgtLen = RECLENB(pTmRecord) - toTmxRecord(pTmRecord)->usFirstTargetRecord; pStartTarget = NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); @@ -1867,7 +1866,7 @@ USHORT EqfMemory::ComparePutData if(!usRc) { - TmProposal.currentInternalKey.setInternalKey(lKey, 1); + TmProposal.currentInternalKey.setInternalKey(lKey, ++targetKey); } } /* endif */ } @@ -1924,62 +1923,44 @@ USHORT EqfMemory::AddTmTarget( PTMX_TARGET_RECORD pTargetRecord = nullptr; // ptr to target record std::vector vTargetClb, vTargetRecord;//for managing memory USHORT usRc = NO_ERROR; // return code - BOOL fOK; // success indicator - PBYTE pByte; // position pointer - ULONG ulAddDataLen = 0; //allocate target control block record - ulAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); + ULONG ulAddDataLen = NTMComputeAddDataSize( TmProposal.szContext, TmProposal.szAddInfo ); vTargetClb.resize(sizeof(TMX_TARGET_CLB)+ulAddDataLen); vTargetRecord.resize(TMX_REC_SIZE); pTargetClb = (PTMX_TARGET_CLB)vTargetClb.data(); pTargetRecord = (PTMX_TARGET_RECORD)vTargetRecord.data(); - + usRc = FillClb( vTargetClb, TmProposal ); + if ( usRc == NO_ERROR ) { - if ( usRc == NO_ERROR ) + //fill target record + FillTargetRecord( TmProposal.pInputSentence.get(), //ptr to sentence structure + pTargetRecord, //filled tm record returned + pTargetClb ); //tm target control block + + //check space requirements + // re-alloc record buffer if too small + ULONG ulNewSize = RECLENB(pTmRecord) + sizeof(TMX_TARGET_RECORD) + + RECLEN(pTargetRecord); + if ( ulNewSize > pTmRecord.size()) { - usRc = FillClb( vTargetClb, TmProposal ); - if ( usRc == NO_ERROR ) - { - //fill target record - FillTargetRecord( TmProposal.pInputSentence.get(), //ptr to sentence structure - TmProposal.pInputSentence->pStrings->getGenericTargetStrC(), - wcslen(TmProposal.pInputSentence->pStrings->getGenericTargetStrC()), - &pTargetRecord, //filled tm record returned - pTargetClb ); //tm target control block - - //check space requirements - { - // re-alloc record buffer if too small - ULONG ulNewSize = RECLENB(pTmRecord) + sizeof(TMX_TARGET_RECORD) + - RECLEN(pTargetRecord); - if ( ulNewSize > pTmRecord.size()) - { - pTmRecord.resize(ulNewSize); - } /* endif */ - } - - if ( fOK ) - { - //postion at end on tm record - pByte = pTmRecord.data(); - pByte += RECLENB(pTmRecord); + pTmRecord.resize(ulNewSize); + } /* endif */ + + //postion at end on tm record + PBYTE pByte = pTmRecord.data(); + pByte += RECLENB(pTmRecord); - //add new target record to end - memcpy( pByte, pTargetRecord, RECLEN(pTargetRecord) ); + //add new target record to end + memcpy( pByte, pTargetRecord, RECLEN(pTargetRecord) ); - //update overall length of tm record - RECLENB(pTmRecord) += RECLEN(pTargetRecord); + //update overall length of tm record + RECLENB(pTmRecord) += RECLEN(pTargetRecord); - //add updated tm record to database - usRc = TmBtree.EQFNTMUpdate( - lKey, - pTmRecord);//.data(), - //RECLENB(pTmRecord) ); - } /* endif */ - } /* endif */ - } /* endif */ + //add updated tm record to database + usRc = TmBtree.EQFNTMUpdate(lKey, + pTmRecord); } /* endif */ return( usRc ); @@ -2022,13 +2003,12 @@ USHORT EqfMemory::AddTmTarget( VOID FillTargetRecord ( PTMX_SENTENCE pSentence, //ptr to sentence structure - PSZ_W pNormString, //ptr to target normalized string - USHORT usNormLen, //length of target normalized string - PTMX_TARGET_RECORD *ppTMXTargetRecord, //ptr to target record + PTMX_TARGET_RECORD pTMXTargetRecord, //ptr to target record PTMX_TARGET_CLB pTargetClb //ptr to target control block ) -{ - PTMX_TARGET_RECORD pTMXTargetRecord = *ppTMXTargetRecord; //ptr to target record +{ + PSZ_W pNormString = pSentence->pStrings->getGenericTargetStrC(); //ptr to target normalized string + USHORT usNormLen = pSentence->pStrings->getGenericTargetStr().size(); //length of target normalized string //position to start of target structure PBYTE pTarget = (PBYTE)pTMXTargetRecord; //ptr to target record @@ -2056,7 +2036,6 @@ VOID FillTargetRecord RECLEN(pTMXTargetRecord) = sizeof(TMX_TARGET_RECORD) + usNormLen + TARGETCLBLEN(pTargetClb); - *ppTMXTargetRecord = pTMXTargetRecord; } @@ -2153,13 +2132,10 @@ static int byteCount[6] = { 3, 2, 1, 1, 2, static long lOffset[6] = { 0x010ffff, 0x04040, 0x080, 0x080, 0x0bfc0, 0x0ef0000 }; static long lCompress[6] = { 0x10, 0x03f, 0x07f, 0x0bf, 0x0ee, 0x0ff }; - - ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG ulLenChar ) { ULONG ulLen = ulLenChar * sizeof(CHAR_W); - if ( ulLen < 20 ) // don't care about too short strings { *pTarget++ = 0; @@ -2174,18 +2150,15 @@ ULONG EQFUnicode2Compress( PBYTE pTarget, PSZ_W pInput, ULONG ulLenChar ) int oldCodePoint = 0x80; int i, bCount, bCountTemp, cp; - *pTarget++ = BOCU_COMPRESS; pTemp = pTarget; - for ( ulLen = 0; ulLen < ulLenChar; ulLen++ ) { cp = *pInput++; delta = cp - oldCodePoint; oldCodePoint = cp; - for (i=0; ; ++i) // will always break { if ( delta <= lBoundary[i]) @@ -2241,11 +2214,11 @@ LONG EQFCompress2Unicode( std::wstring& pOutput, PBYTE pTarget, ULONG ulLenComp lLen = ulLenComp / sizeof(CHAR_W); PSZ_W pWCharTarget = (PSZ_W) pTarget; pOutput.insert(pOutput.begin(), pWCharTarget, pWCharTarget + lLen); - pOutput[ lLen ] = EOS; + pOutput.push_back(EOS); // EOS should be L'\0' + //pOutput[ lLen ] = EOS; } else if (b == BOCU_COMPRESS) { - //PSZ_W pTemp = pOutput; long delta = 0; USHORT iLen = 0; int oldCodePoint = 0x80; @@ -2276,8 +2249,8 @@ LONG EQFCompress2Unicode( std::wstring& pOutput, PBYTE pTarget, ULONG ulLenComp } } } - pOutput.push_back('\0'); - lLen = pOutput.size();//(pOutput - pTemp); + pOutput.push_back(EOS); + lLen = pOutput.length();//(pOutput - pTemp); } else { @@ -2286,6 +2259,7 @@ LONG EQFCompress2Unicode( std::wstring& pOutput, PBYTE pTarget, ULONG ulLenComp } return lLen; } + #endif @@ -2439,18 +2413,21 @@ USHORT TMLoopAndDelTargetClb USHORT usPutFile, USHORT usPutAuthor, PBOOL pfNewerTargetExists, - PINT pTargetKey + PINT pTargetKey, + PBOOL pfTargetMatch ) { USHORT usRc = NO_ERROR; PTMX_TARGET_CLB pClb = NULL; //ptr to target control block - //PTMX_TAGTABLE_RECORD pTMXSrcTTable = NULL; //ptr to source tag info LONG lLeftClbLen; LONG lLeftTgtLen; BOOL fTgtRemoved = FALSE; PTMX_TARGET_RECORD pTMXTgtRec = NULL; BOOL fDel = FALSE; - + + PBYTE pByte = NULL; //position ptr + PBYTE pStartTarget = NULL; //position ptr + // preset callers flag *pfNewerTargetExists = FALSE; @@ -2461,6 +2438,7 @@ USHORT TMLoopAndDelTargetClb pTMXTgtRec = (PTMX_TARGET_RECORD)NTRecPos((PBYTE)(toTmxRecord(pTmRecord)+1), REC_FIRSTTARGET); if(pTargetKey) *pTargetKey = 0; + while ( lLeftTgtLen>0 && ( lLeftTgtLen <= RECLEN(pTMXTgtRec) ) && (RECLEN(pTMXTgtRec) != 0) && !fDel ) { NTASSERTLEN(lLeftTgtLen, RECLEN(pTMXTgtRec), 4713); @@ -2468,38 +2446,53 @@ USHORT TMLoopAndDelTargetClb pClb = (PTMX_TARGET_CLB)NTRecPos((PBYTE)pTMXTgtRec, REC_CLB); lLeftClbLen = RECLEN(pTMXTgtRec) - pTMXTgtRec->usClb; + + if(TmProposal.fMatchTargetOnUpdate){ + std::wstring pStringTrg; + // check if target string and target tag record are identical + pByte = NTRecPos((PBYTE)pTMXTgtRec, REC_TGTSTRING); + LONG lLen = pTMXTgtRec->usClb - pTMXTgtRec->usTarget; + lLen = EQFCompress2Unicode( pStringTrg, pByte, lLen ); + pStringTrg.data()[lLen] = EOS; + + *pfTargetMatch = (pStringTrg == TmProposal.pInputSentence->pStrings->getGenericTargetStr()); + } + + if(*pfTargetMatch){ + while ( ( lLeftClbLen > 0 ) && !fDel ) + { + (*pTargetKey)++; + + if ( (pClb->usLangId == usPutLang) && + (pClb->usAuthorId == usPutAuthor) && + (pClb->usFileId == usPutFile) && + isAddDataIsTheSame(TmProposal, *pClb) + ) + { // remove target CLB and target record (if only 1 CLB) + // as the segment is putted with a new value + if ( (pClb->lTime < TmProposal.lTargetTime) || !TmProposal.lTargetTime ) + { + lLeftClbLen -= TARGETCLBLEN(pClb); + // loop over all CLBs of this target record and remove + // any CLB for the current segment + fTgtRemoved = TMDelTargetClb( pTmRecord, pTMXTgtRec, pClb); + fDel = TRUE; + } + else + { // existing match is newer, continue with next one + lLeftClbLen -= TARGETCLBLEN(pClb); + *pfNewerTargetExists = TRUE; + pClb = NEXTTARGETCLB(pClb); + } /* endif */ + } + else + { // no match, try next one + lLeftClbLen -= TARGETCLBLEN(pClb); + pClb = NEXTTARGETCLB(pClb); + } /* endif */ + } /* endwhile */ + } - while ( ( lLeftClbLen > 0 ) && !fDel ) - { - (*pTargetKey)++; - if ( (pClb->usLangId == usPutLang) && - (pClb->usAuthorId == usPutAuthor) && - (pClb->usFileId == usPutFile) && - isAddDataIsTheSame(TmProposal, *pClb) - ) - { // remove target CLB and target record (if only 1 CLB) - // as the segment is putted with a new value - if ( (pClb->lTime < TmProposal.lTargetTime) || !TmProposal.lTargetTime ) - { - lLeftClbLen -= TARGETCLBLEN(pClb); - // loop over all CLBs of this target record and remove - // any CLB for the current segment - fTgtRemoved = TMDelTargetClb( pTmRecord, pTMXTgtRec, pClb); - fDel = TRUE; - } - else - { // existing match is newer, continue with next one - lLeftClbLen -= TARGETCLBLEN(pClb); - *pfNewerTargetExists = TRUE; - pClb = NEXTTARGETCLB(pClb); - } /* endif */ - } - else - { // no match, try next one - lLeftClbLen -= TARGETCLBLEN(pClb); - pClb = NEXTTARGETCLB(pClb); - } /* endif */ - } /* endwhile */ //position at next target if ( !fDel ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 3a23c5b6..d100f7eb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -589,88 +589,19 @@ USHORT EqfMemory::NTMGetNameFromID(PUSHORT pusID, // intput /* set usRc and reset pszName */ /************************************************************/ LOG_AND_SET_RC(usRc, T5INFO, ID_NOT_FOUND); - pszName[0] = EOS; + if(*pusID){ + snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", *pusID); // 'hu' = unsigned short + }else{ + pszName[0] = EOS; + } } /* endif */ } /* endif */ } /* endif */ } + return usRc; } /* end of function NTMGetNameFromID */ -PSZ EqfMemory::NTMFindNameForID( - PUSHORT pusID, // intput - USHORT usTableType) // input -{ - USHORT usRc = NO_ERROR; - BOOL fFound = FALSE; - PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry - PTMX_TABLE pstTMTable = NULL; // ptr to table structure - PSZ pszFoundName = NULL; - - //--- if input parameters OK - // if ( pTmClb ) - { - /******************************************************************/ - /* get pointer to table and table entries in dependency of the */ - /* table type */ - /******************************************************************/ - usRc = NTMGetPointersToTable(usTableType, - &pstTMTable, - &pstTMTableEntries); - /****************************************************************/ - /* search ID in passed table usTableTye and get name of ID */ - /****************************************************************/ - if (usRc == NO_ERROR) - { - if (*pusID == OVERFLOW_ID) - { - pszFoundName = OVERFLOW_NAME; - fFound = TRUE; - } - else - { - ULONG ulI; - for (ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) - { - if (*pusID == pstTMTableEntries[ulI].usId) - { - fFound = TRUE; - pszFoundName = pstTMTableEntries[ulI].szName; - } /* endif */ - } /* endfor */ - /**************************************************************/ - /* Get any long document name for this ID */ - /**************************************************************/ - if (fFound && (usTableType == FILE_KEY)) - { - BOOL fLongFound = FALSE; - for (ulI = 0; - ulI < LongNames.stTableEntry.size() && !fLongFound; - ulI++) - { - if (*pusID == LongNames.stTableEntry[ulI].usId) - { - fLongFound = TRUE; - pszFoundName = LongNames.stTableEntry[ulI].pszLongName; - } /* endif */ - } /* endfor */ - } /* endif */ - - if (!fFound) - { - /************************************************************/ - /* the ID was not found in the table */ - /* set usRc and reset pszName */ - /************************************************************/ - LOG_AND_SET_RC(usRc, T5INFO, ID_NOT_FOUND); - pszFoundName = NULL; - } /* endif */ - } /* endif */ - } /* endif */ - } - - return (pszFoundName); -} /* end of function NTMFindNameForID */ //+----------------------------------------------------------------------------+ //|Internal function | @@ -960,37 +891,33 @@ USHORT EqfMemory::NTMReadLongNameTable() // call to obtain exact length of record LONG lKey = LONGNAME_KEY; USHORT usRC = TmBtree.EQFNTMGet(lKey, LongNames.pszBuffer); - + // setup pointer array for long names if (usRC == NO_ERROR) - { - // setup pointer array for long names - if (usRC == NO_ERROR) + { + // fill pointer array + PSZ pszTemp = (PSZ)LongNames.pszBuffer.data(); + if(//getMajorVersion() < 7 && + !LongNames.pszBuffer.empty())//for compatibility { - // fill pointer array - PSZ pszTemp = (PSZ)LongNames.pszBuffer.data(); - if(//getMajorVersion() < 7 && - !LongNames.pszBuffer.empty())//for compatibility - { - LongNames.pszBuffer.pop_back(); - } - ssize_t buffLen = LongNames.pszBuffer.size(); - - while ( (pszTemp - (PSZ)LongNames.pszBuffer.data() < buffLen) && *((PUSHORT)pszTemp) != 0) - { - // get ID of string - USHORT usId = *((PUSHORT)pszTemp); - pszTemp += sizeof(USHORT); // skip ID - std::vector longName(reinterpret_cast(pszTemp), - reinterpret_cast(pszTemp + strlen(pszTemp))); - auto longNameCaseIgnore = toUpperVector(longName); - - appendLongNameEntry(LongNames, usId, longName, pszTemp); - appendLongNameEntry(LongNamesCaseIgnore, usId, longNameCaseIgnore); - pszTemp += strlen(pszTemp) + 2; // skip long name - } /* endwhile */ - SortLongNameTables(); + LongNames.pszBuffer.pop_back(); + } + ssize_t buffLen = LongNames.pszBuffer.size(); + + while ( (pszTemp - (PSZ)LongNames.pszBuffer.data() < buffLen) && *((PUSHORT)pszTemp) != 0) + { + // get ID of string + USHORT usId = *((PUSHORT)pszTemp); + pszTemp += sizeof(USHORT); // skip ID + std::vector longName(reinterpret_cast(pszTemp), + reinterpret_cast(pszTemp + strlen(pszTemp))); + auto longNameCaseIgnore = toUpperVector(longName); + + appendLongNameEntry(LongNames, usId, longName, pszTemp); + appendLongNameEntry(LongNamesCaseIgnore, usId, longNameCaseIgnore); + pszTemp += strlen(pszTemp) + 2; // skip long name + } /* endwhile */ + SortLongNameTables(); - } /* endif */ } /* endif */ // return to caller @@ -1246,7 +1173,7 @@ USHORT EqfMemory::NTMAddLangGroup( { SHORT sGroupID = 0; USHORT usRC = NO_ERROR; // function return code - CHAR szLangGroup[MAX_LANGUAGE_PROPERTIES]; // buffer for language group name + CHAR szLangGroup[MAX_LANGUAGE_PROPERTIES]{}; // buffer for language group name // get language properties (incl. group name) if (GetLanguageGroup(pszLang, szLangGroup)) @@ -1276,20 +1203,15 @@ USHORT EqfMemory::NTMAddLangGroup( // enlarge language-ID-to-group-ID table if necessary if (usRC == NO_ERROR) { - LONG lRequiredSize = (sLangID + 1) * sizeof(SHORT); - if (psLangIdToGroupTable.size() < lRequiredSize) + if (psLangIdToGroupTable.size() < sLangID + 1) { - psLangIdToGroupTable.resize(lRequiredSize + 10); - lLangIdToGroupTableUsed = lRequiredSize; + psLangIdToGroupTable.resize(sLangID + 1); } /* endif */ - } /* endif */ - - // update language ID to group ID table - if (usRC == NO_ERROR) - { + // update language ID to group ID table psLangIdToGroupTable[sLangID] = sGroupID; } /* endif */ + return (usRC); } /* end of function NTMAddLangGroup */ @@ -1299,19 +1221,16 @@ USHORT EqfMemory::NTMCreateLangGroupTable() // pTmClb->LangGroups.ulAllocSize = TMX_TABLE_SIZE; // allocate language-ID-to-group-ID-table - - LONG lSize = Languages.stTmTableEntry.size() > 100L ? Languages.stTmTableEntry.size() : 100L; + LONG lSize = Languages.stTmTableEntry.size();// > 100L ? Languages.stTmTableEntry.size() : 100L; if (lSize <= 0) lSize = 1; psLangIdToGroupTable.resize(lSize); - lLangIdToGroupTableUsed = 0L; - // get group IDs for all languages and fill map table if (usRC == NO_ERROR) { int i = 0; - while ((usRC == NO_ERROR) && (i < (int)Languages.stTmTableEntry.size()) + while ((usRC == NO_ERROR) && (i < psLangIdToGroupTable.size())//(int)Languages.stTmTableEntry.size()) ) { if(Languages.stTmTableEntry[i].szName[0]){ @@ -1508,19 +1427,7 @@ USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, wchar_t* pszBuffer, PUSHORT pDataPUShort = (PUSHORT)pData; usLength = pDataPUShort[1]; pData++; - //int offset = pData - pAddDataStart; - //int leftToTheEnd = endOfMemRegion - (PBYTE)pData; - //if(pCLB->usAddDataLen == 20){ - // int a =0; - //} - //pszBuffer.assign(pData, usLength); - //try { - //pszBuffer = std::wstring(pData); //, usLength); - wcsncpy(pszBuffer, pData, usLength); - // pszBuffer.assign(pData, usLength-1); - //} catch (const std::exception& e) { - // T5LOG(T5ERROR) << "NtmGetAddData: std::wstring assign failed: " << e.what() <<"; usLength=" << usLength <<"; usDataID=" << usDataID; - //} + wcsncpy(pszBuffer, pData, usLength); } /* endif */ } else diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 5fd48b0d..a9704f26 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -255,6 +255,8 @@ DECLARE_int64(tmListLockDefaultTimeout); DECLARE_int64(tmLockDefaultTimeout); DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); +DECLARE_bool(matchTargetOnUpdate); +//DECLARE_bool(ignoreSegIdOnUpdate); int RequestData::setMutexTimeouts(){ //set default timeouts if they were not set in request @@ -388,6 +390,7 @@ bool RequestData::isWriteRequest(){ || command == COMMAND::IMPORT_LOCAL_MEM || command == COMMAND::FLUSH_MEM || command == COMMAND::MULTIUPDATE + || command == ADD_TO_TABLE ; } @@ -401,6 +404,7 @@ bool RequestData::isReadOnlyRequest(){ || command == COMMAND::EXPORT_MEM_TMX_STREAM || command == COMMAND::EXPORT_MEM_INTERNAL_FORMAT_STREAM || command == COMMAND::GET_ENTRY + || command == COMMAND::GET_TABLES ; } @@ -944,6 +948,9 @@ int ImportRequestData::parseJSON(){ std::string name; std::string value; fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + //fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; + try{ while ( _rc_ == 0 ) { @@ -970,6 +977,10 @@ int ImportRequestData::parseJSON(){ timeout = std::stol(value); }else if(strcasecmp(name.c_str(), "newEntryIfOnlyContextAndTimestampDifferent") == 0){ fNewEntryIfOnlyContextAndTimestampDifferent = std::stol(value); + }else if(strcasecmp(name.c_str(), "ignoreSegIdOnUpdate") == 0){ + fIgnoreSegIdOnUpdate = std::stol(value); + }else if(strcasecmp(name.c_str(), "matchTargetOnUpdate") == 0){ + fMatchTargetOnUpdate = std::stol(value); }else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; @@ -1166,6 +1177,9 @@ int ImportRequestData::execute(){ mem->importDetails->reset(); mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fMatchTargetOnUpdate = fMatchTargetOnUpdate; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + LONG lCurTime = 0; time( &lCurTime ); @@ -1534,15 +1548,17 @@ USHORT MemFuncOrganizeProcess ); int ReorganizeRequestData::parseJSON(){ - this->fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; // parse input parameters std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); - - fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; JSONFactory::JSONPARSECONTROL parseControl[] = { - { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), 0 }, + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent }, + { L"matchTargetOnUpdate", JSONFactory::INT_PARM_TYPE , &(fMatchTargetOnUpdate), FLAGS_matchTargetOnUpdate }, + { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate}, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, @@ -1601,6 +1617,9 @@ int ReorganizeRequestData::execute(){ mem->importDetails->reset(); mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fMatchTargetOnUpdate = fMatchTargetOnUpdate; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + LONG lCurTime = 0; time( &lCurTime ); @@ -1647,6 +1666,8 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; + fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; JSONFactory::JSONPARSECONTROL parseControl[] = { //searchFilterFactory @@ -1656,7 +1677,9 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ { L"context", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.context ), 0 }, { L"author", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.author ), 0 }, { L"document", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.document ), 0 }, - { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), 0 }, + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent}, + { L"matchTargetOnUpdate", JSONFactory::INT_PARM_TYPE , &(fMatchTargetOnUpdate), FLAGS_matchTargetOnUpdate}, + { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate}, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, @@ -1742,6 +1765,9 @@ int DeleteEntriesReorganizeRequestData::execute(){ mem->importDetails->reset(); mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fMatchTargetOnUpdate = fMatchTargetOnUpdate; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + LONG lCurTime = 0; time( &lCurTime ); @@ -2098,6 +2124,9 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr json_factory.addParmToJSON( outputMessage, "importRuntimeSec", mem->importDetails->lImportRunTimeSec ); json_factory.addParmToJSON( outputMessage, "importTimeoutSec", mem->importDetails->lImportTimeoutSec ); json_factory.addParmToJSON( outputMessage, "newEntryIfOnlyContextAndTimestampDifferent", mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent ); + json_factory.addParmToJSON( outputMessage, "matchTargetOnUpdate", mem->importDetails->fMatchTargetOnUpdate ); + //json_factory.addParmToJSON( outputMessage, "ignoreSegIdOnUpdate", mem->importDetails->fIgnoreSegIdOnUpdate); + std::string invalidSegmRCs; for(auto [errCode, errCount]: mem->importDetails->invalidSegmentsRCs){ invalidSegmRCs += std::to_string(errCode) + ":" + std::to_string(errCount) +"; "; @@ -2162,6 +2191,70 @@ int StatusMemRequestData::execute() { return( OK ); }; +int GetTablesMemRequestData::execute() { + json_factory.startJSON( outputMessage ); + std::string longNames, longNamesCaseIgnore, langTable, langGroupTable, authorsTable, filenamesOldTable, tagTablesTable; + int i = 0; + for(const auto& entry: mem->LongNames.stTableEntry){ + longNames += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + (entry.pszLongName? entry.pszLongName:"") + "\"; "; + } + + i = 0; + for(const auto& entry: mem->LongNamesCaseIgnore.stTableEntry){ + longNamesCaseIgnore += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + (entry.pszLongName? entry.pszLongName:"") + "\"; "; + } + + i = 0; + for(const auto& entry: mem->Authors.stTmTableEntry){ + authorsTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + + i = 0; + for(const auto& entry: mem->FileNames.stTmTableEntry){ + filenamesOldTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + + i = 0; + for(const auto& entry: mem->Languages.stTmTableEntry){ + langTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + + i = 0; + for(const auto& entry: mem->LangGroups.stTmTableEntry){ + langGroupTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName[0] + "\"; "; + } + + i = 0; + for(const auto& entry: mem->TagTables.stTmTableEntry){ + tagTablesTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + + json_factory.addParmToJSON( outputMessage, "tableName(additional info)(INTERNAL_KEY_OF_THE_TABLE)", "position)idInTable: value;"); + std::string tableName = "tagTablesTable(" + toStr(TAGTABLE_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, tagTablesTable); + + tableName = "langTable(" + toStr(LANG_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, langTable); + + tableName = "langGroupTable(virtual)(" + toStr(LANGGROUP_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, langGroupTable); + + tableName = "authors(" + toStr(AUTHOR_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, authorsTable); + + tableName = "filenames(long)(" + toStr(LONGNAME_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, longNames); + + tableName = "filenames(long, caseign, virtual)(" + toStr(LONGNAME_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, longNamesCaseIgnore); + + tableName = "filenames(short)(" + toStr(FILE_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, filenamesOldTable); + + json_factory.terminateJSON( outputMessage ); + return( OK ); +}; + #include //#include // Include the flags library @@ -2997,7 +3090,9 @@ int UpdateEntryRequestData::parseJSON(){ { L"customId", JSONFactory::ASCII_STRING_PARM_TYPE, &(Data.szCustomId), sizeof( Data.szCustomId) / sizeof( Data.szCustomId[0] ) }, { L"recordKey", JSONFactory::INT_PARM_TYPE , &(recordKey), 0 }, { L"targetKey", JSONFactory::INT_PARM_TYPE , &(targetKey), 0 }, - { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(Data.fNewEntryIfOnlyContextAndTimestampDifferent), 0 }, + { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(Data.fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent }, + { L"matchTargetOnUpdate", JSONFactory::INT_PARM_TYPE , &(Data.fMatchTargetOnUpdate), FLAGS_matchTargetOnUpdate }, + { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(Data.fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate }, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, { L"requestTMMutexTimeout", JSONFactory::INT_PARM_TYPE, &requestTMMutexTimeoutMs, -1}, diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index da68e991..81d39a19 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -1068,7 +1068,7 @@ int AddMemToList( void *pvData, char *pszName,std::shared_ptr pInfo /*! \brief search a string in a proposal \param pProposal pointer to the proposal - \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) + \param pszSearch pointer to the search string (when fIgnoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ @@ -1112,7 +1112,7 @@ BOOL searchExtendedInProposal /*! \brief search a string in a proposal \param pProposal pointer to the proposal - \param pszSearch pointer to the search string (when fIngoreCase is being used, this strign has to be in uppercase) + \param pszSearch pointer to the search string (when fIgnoreCase is being used, this strign has to be in uppercase) \param lSearchOptions combination of search options \returns TRUE if the proposal contains the searched string otherwise FALSE is returned */ diff --git a/source/opentm2/core/utilities/OtmProposal.cpp b/source/opentm2/core/utilities/OtmProposal.cpp index e9208267..4a1b7637 100755 --- a/source/opentm2/core/utilities/OtmProposal.cpp +++ b/source/opentm2/core/utilities/OtmProposal.cpp @@ -36,12 +36,17 @@ OtmProposal::~OtmProposal() /* operations */ DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); +DECLARE_bool(matchTargetOnUpdate); +//DECLARE_bool(ignoreSegIdOnUpdate); + void OtmProposal::clear()//bool clearKeys) { pInputSentence.reset(); TMCursor currentKey, nextKey; *this = OtmProposal{}; fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent;//default value + fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; } OtmProposal::eProposalType getMemProposalType( char *pszType ) diff --git a/source/otmd.cpp b/source/otmd.cpp index 32a3eec7..0577072f 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -203,7 +203,10 @@ DEFINE_bool(enable_newlines_in_logs, false, "(not working - instead of using function to remove newlines in runtime, logs was edited to have better perfomance)if set to true, would keep newline symbols in the logs, otherwise(by default) newlines would be removed and logs would be oneliners. This just calls function to remove newlines," ); DEFINE_bool(newEntryIfOnlyContextAndTimestampDifferent, true, "If it is set to true, the behavior should be: A new entry should be saved to the TM, if only timestamp and context are different and all other params the same;\t If it is set to false, the beahvior should be: if context is different and timestamp is different, the context and timestamp of the new to be saved segment should overwrite the context and segment of an entry, that already exists and where all other params are the same as in the new segment. This is basically the same behavior it would have, if context would not be set for both segments (the old and the new)." ); - +DEFINE_bool(matchTargetOnUpdate, true, "If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets, if false- previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer." ); +//DEFINE_bool(ignoreSegIdOnUpdate, true, "If it is set to true, seg id check would be skipped. Could be useful for importing, because segId there are generated, so never match" ); + + DEFINE_bool(suppressTmVersionCheck, false, "If sets to true, allow to open tms, which was created in version, that's outside of supported versions scope. You can use this if you have version mismatch error"); DEFINE_int32(t5globversion_min_supported, 0, "sets minimal global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z"); DEFINE_int32(t5majversion_min_supported, 5, "sets minimal major version in which tm was created to be supported for open in current. it's the Y(MAJ) in vX.Y.Z"); From bb73625b1a3b6882fc628425f3babe30aded0fe0 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Fri, 18 Jul 2025 19:04:50 +0300 Subject: [PATCH 37/62] * updated to 0.7.12 --- source/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index d13e5fbb..285e2cb0 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 11) +SET(APP_VERSION_MINOR 12) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) From 5cf76f08498dbd97ce93538f08a14562d7677b80 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 23 Jul 2025 18:22:51 +0300 Subject: [PATCH 38/62] * updated to v0.7.13 * renamed matchTargetOnUpdate to saveDifferentTargetsForSameSource * fixed -308 and -311, fuzzy issues + added jsonparser error return for fuzzy * fuzzy results shoudn't have dublicates now and should always be sorted first is the best + improved compatibility with older tms - disabled lock for tagtable loading * optimized fuzzy search - code cleanup --- include/Proposal.h | 60 ++++++++- include/otm.h | 6 +- include/requestdata.h | 6 +- include/tm.h | 23 +++- source/CMakeLists.txt | 2 +- source/RestAPI/ProxygenHandler.cpp | 1 - .../opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp | 3 - .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 71 ++++++----- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 12 +- source/opentm2/core/document/EQFTATAG.cpp | 2 +- source/opentm2/core/requestdata.cpp | 118 +++++++++++++----- source/opentm2/core/utilities/EQFMORPH.cpp | 115 ----------------- source/opentm2/core/utilities/OtmProposal.cpp | 6 +- source/otmd.cpp | 29 ++++- 17 files changed, 247 insertions(+), 213 deletions(-) diff --git a/include/Proposal.h b/include/Proposal.h index 939cc460..427c13bc 100644 --- a/include/Proposal.h +++ b/include/Proposal.h @@ -227,7 +227,7 @@ class OtmProposal /* \brief get proposal fuzziness \returns proposal fuzziness */ - int getFuzziness(); + int getFuzziness()const; /* \brief set the proposal fuzziness \param lFuzzinessTime new fuzziness of proposal @@ -440,7 +440,7 @@ class OtmProposal /*! \brief when saving request, if set to true, save segment with differen context as new(as was), otherwise if only context differs, replace previous segment */ //bool fCheckContextWhenReplacingSegment{}; bool fNewEntryIfOnlyContextAndTimestampDifferent{}; - bool fMatchTargetOnUpdate{}; + bool fSaveDifferentTargetsForSameSource{}; bool fIgnoreSegIdOnUpdate{}; /*! \brief Index of memory when looking up in a list of memories */ @@ -494,6 +494,62 @@ class OtmProposal #include "tm.h" +#include // for std::swap, std::swap_ranges + +/* +void swap(OtmProposal& a, OtmProposal& b) noexcept { + using std::swap; + // Swap raw arrays + swap_ranges(std::begin(a.szId), std::end(a.szId), std::begin(b.szId)); + swap_ranges(std::begin(a.szSource), std::end(a.szSource), std::begin(b.szSource)); + swap_ranges(std::begin(a.szTarget), std::end(a.szTarget), std::begin(b.szTarget)); + swap_ranges(std::begin(a.szDocName), std::end(a.szDocName), std::begin(b.szDocName)); + swap_ranges(std::begin(a.szCustomId), std::end(a.szCustomId), std::begin(b.szCustomId)); + swap_ranges(std::begin(a.szSourceLanguage), std::end(a.szSourceLanguage), std::begin(b.szSourceLanguage)); + swap_ranges(std::begin(a.szOriginalSourceLanguage), std::end(a.szOriginalSourceLanguage), std::begin(b.szOriginalSourceLanguage)); + swap_ranges(std::begin(a.szTargetLanguage), std::end(a.szTargetLanguage), std::begin(b.szTargetLanguage)); + swap_ranges(std::begin(a.szTargetAuthor), std::end(a.szTargetAuthor), std::begin(b.szTargetAuthor)); + swap_ranges(std::begin(a.szMarkup), std::end(a.szMarkup), std::begin(b.szMarkup)); + swap_ranges(std::begin(a.szContext), std::end(a.szContext), std::begin(b.szContext)); + swap_ranges(std::begin(a.szAddInfo), std::end(a.szAddInfo), std::begin(b.szAddInfo)); + swap_ranges(std::begin(a.szMemory), std::end(a.szMemory), std::begin(b.szMemory)); + swap_ranges(std::begin(a.szIsoSourceLang), std::end(a.szIsoSourceLang), std::begin(b.szIsoSourceLang)); + swap_ranges(std::begin(a.szIsoTargetLang), std::end(a.szIsoTargetLang), std::begin(b.szIsoTargetLang)); + swap_ranges(std::begin(a.szError), std::end(a.szError), std::begin(b.szError)); + swap_ranges(std::begin(a.szSearchMode), std::end(a.szSearchMode), std::begin(b.szSearchMode)); + swap_ranges(std::begin(a.szSearchPos), std::end(a.szSearchPos), std::begin(b.szSearchPos)); + swap_ranges(std::begin(a.szSearchString), std::end(a.szSearchString), std::begin(b.szSearchString)); + + // Swap primitive types + swap(a.lSegmentId, b.lSegmentId); + swap(a.fSegmentIdWasGenerated, b.fSegmentIdWasGenerated); + swap(a.eType, b.eType); + swap(a.eMatch, b.eMatch); + swap(a.lTargetTime, b.lTargetTime); + swap(a.iFuzziness, b.iFuzziness); + swap(a.iDiffs, b.iDiffs); + swap(a.iWords, b.iWords); + swap(a.fFilled, b.fFilled); + swap(a.fNewEntryIfOnlyContextAndTimestampDifferent, b.fNewEntryIfOnlyContextAndTimestampDifferent); + swap(a.fSaveDifferentTargetsForSameSource, b.fSaveDifferentTargetsForSameSource); + swap(a.fIgnoreSegIdOnUpdate, b.fIgnoreSegIdOnUpdate); + swap(a.iMemoryIndex, b.iMemoryIndex); + swap(a.iContextRanking, b.iContextRanking); + swap(a.pvReplacementList, b.pvReplacementList); + swap(a.iNumOfProposals, b.iNumOfProposals); + swap(a.iSearchTime, b.iSearchTime); + swap(a.fMarkupChanged, b.fMarkupChanged); + swap(a.fIsoSourceLangIsPrefered, b.fIsoSourceLangIsPrefered); + swap(a.fIsoTargetLangIsPrefered, b.fIsoTargetLangIsPrefered); + + // Swap unique_ptr + swap(a.pInputSentence, b.pInputSentence); + + // Swap TMCursor members + swap(a.currentInternalKey, b.currentInternalKey); + swap(a.nextInternalKey, b.nextInternalKey); +}//*/ + OtmProposal::eProposalType getMemProposalType( char *pszType ); diff --git a/include/otm.h b/include/otm.h index 98008306..fbf5c062 100644 --- a/include/otm.h +++ b/include/otm.h @@ -17,7 +17,7 @@ #define NTMREQUESTNEWKEY 0xFFFFFFFF DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); -DECLARE_bool(matchTargetOnUpdate); +DECLARE_bool(saveDifferentTargetsForSameSource); //DECLARE_bool(ignoreSegIdOnUpdate); constexpr bool FLAGS_ignoreSegIdOnUpdate = true;//not used flag @@ -85,12 +85,12 @@ struct ImportStatusDetails{ size_t activeSegment{0}; InclosingTagsBehaviour inclosingTagsBehaviour{InclosingTagsBehaviour::saveAll}; BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; - BOOL fMatchTargetOnUpdate{}; + BOOL fSaveDifferentTargetsForSameSource{}; BOOL fIgnoreSegIdOnUpdate{}; void reset(){ fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; - fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; //fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; firstInvalidSegmentsSegNums.clear(); firstInvalidSegmentsSegNums.reserve(100); diff --git a/include/requestdata.h b/include/requestdata.h index 63b491d4..9c3762d1 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -215,7 +215,7 @@ class ImportRequestData:public RequestData{ BOOL fClose = false; BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; - BOOL fMatchTargetOnUpdate{}; + BOOL fSaveDifferentTargetsForSameSource{}; BOOL fIgnoreSegIdOnUpdate{}; InclosingTagsBehaviour inclosingTagsBehaviour; IMPORTMEMORYDATA* pData = nullptr; @@ -380,7 +380,7 @@ class ReorganizeRequestData:public RequestData{ ReorganizeRequestData(): RequestData(COMMAND::REORGANIZE_MEM) {}; protected: BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; - BOOL fMatchTargetOnUpdate{}; + BOOL fSaveDifferentTargetsForSameSource{}; BOOL fIgnoreSegIdOnUpdate{}; int parseJSON() override; int checkData() override; @@ -564,7 +564,7 @@ class DeleteEntriesReorganizeRequestData: public RequestData{ protected: SearchFilterFactory searchFilterFactory; BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; - BOOL fMatchTargetOnUpdate{}; + BOOL fSaveDifferentTargetsForSameSource{}; BOOL fIgnoreSegIdOnUpdate{}; int parseJSON() override ; diff --git a/include/tm.h b/include/tm.h index 85985642..a2281fc2 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1688,7 +1688,7 @@ typedef struct _TMX_GET_W //======================================================================= // structure TMX_GET_OUT -typedef struct _TMX_MATCH_TABLE_W +struct TMX_MATCH_TABLE_W { CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence with tags CHAR szFileName[MAX_FILESPEC]{}; // file name where the source comes from. @@ -1704,16 +1704,29 @@ typedef struct _TMX_MATCH_TABLE_W int iWords{}; int iDiffs{}; USHORT usOverlaps{}; // temp field - nr of overlapping triples - CHAR szTagTable[MAX_FNAME]; // tag table name of source - LONG lKey{}; // key of match + CHAR szTagTable[MAX_FNAME]{}; // tag table name of source + LONG lKey{}; // key of match USHORT usTargetNum{}; // number of target USHORT usDBIndex{}; // number of memory in current hierarchy CHAR_W szContext[MAX_SEGMENT_SIZE]{}; // segment context info CHAR_W szAddInfo[MAX_SEGMENT_SIZE]{}; // additional segment information - //std::wstring szContext, szAddInfo; USHORT usContextRanking{}; // context ranking from user exit context processing USHORT usMatchInfo{}; // for future use: special info for match -} TMX_MATCH_TABLE_W, *PTMX_MATCH_TABLE_W; + + bool sameTMLocation(LONG recordKey, USHORT targetKey)const{ + return recordKey == lKey && targetKey == usTargetNum; + } + + bool isLocationAlreadyAdded(USHORT elementsAdded, LONG recordKey, USHORT targetKey){//use this function only with the pointer to the first element + bool matchExists = std::any_of( + this, this + elementsAdded, + [&](const TMX_MATCH_TABLE_W& entry) { + return entry.sameTMLocation(recordKey, targetKey); + }); + } +}; + +using PTMX_MATCH_TABLE_W = TMX_MATCH_TABLE_W*; struct TMX_GET_OUT_W : public TMX_EXT_OUT_W { diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 285e2cb0..319833a7 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 12) +SET(APP_VERSION_MINOR 13) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/ProxygenHandler.cpp b/source/RestAPI/ProxygenHandler.cpp index e1b6f20f..fedc9185 100755 --- a/source/RestAPI/ProxygenHandler.cpp +++ b/source/RestAPI/ProxygenHandler.cpp @@ -235,7 +235,6 @@ void ProxygenHandler::onEOM() noexcept { if(!pRequest->_rest_rc_){ std::string truncatedInput = pRequest->strBody.size() > 3000 ? pRequest->strBody.substr(0, 3000) : pRequest->strBody; T5Logger::GetInstance()->SetBodyBuffer(", with body = \"" + truncatedInput +"\""); - } } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index e04d83c8..db1e3b35 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -159,7 +159,7 @@ VOID EQFMemOrganizeProcess auto ll = T5Logger::GetInstance()->suppressLogging(); wchar_t seg_buff[OTMPROPOSAL_MAXSEGLEN+1]; pRIDA->pProposal->getSource(seg_buff, OTMPROPOSAL_MAXSEGLEN); - pRIDA->pProposal->fMatchTargetOnUpdate = pRIDA->pMem->importDetails->fMatchTargetOnUpdate; + pRIDA->pProposal->fSaveDifferentTargetsForSameSource = pRIDA->pMem->importDetails->fSaveDifferentTargetsForSameSource; bool fValidXml = IsValidXml( seg_buff); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp index dcb9c77e..eac35508 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp @@ -1342,7 +1342,6 @@ USHORT MemExportWrite // prepare the function I/F TM import USHORT FCTDATA::MemFuncPrepExport ( - //PFCTDATA pData, // function I/F session data PSZ pszMemName, // name of Translation Memory PSZ pszOutFile, // fully qualified name of output file LONG lOptions, // options for Translation Memory export @@ -1555,8 +1554,6 @@ USHORT FCTDATA::MemFuncExportProcess() break; } /* endswitch */ this->usExportProgress = pCommArea->usComplete; - //this->recordKey = pIDA->pMem->ulNextKey; - //this->targetKey = pIDA->pMem->usNextTarget; return( usRC ); } /* end of function MemFuncExportProcess */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 5487ab54..5306fecb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -909,7 +909,7 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT pLIDA->pProposal->setSegmentId( pSegment->lSegNum ); pLIDA->pProposal->pInputSentence.reset(); pLIDA->pProposal->fNewEntryIfOnlyContextAndTimestampDifferent = pLIDA->mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent; - pLIDA->pProposal->fMatchTargetOnUpdate = pLIDA->mem->importDetails->fMatchTargetOnUpdate; + pLIDA->pProposal->fSaveDifferentTargetsForSameSource = pLIDA->mem->importDetails->fSaveDifferentTargetsForSameSource; pLIDA->pProposal->fIgnoreSegIdOnUpdate = pLIDA->mem->importDetails->fIgnoreSegIdOnUpdate; pLIDA->pProposal->pInputSentence = std::make_unique(pSegment->szSource, pSegment->szTarget); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index fba39247..e80db41b 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -705,13 +705,7 @@ USHORT ExactTest pContextBuffer[0] = '\0'; USHORT usTargetTranslationFlag = (USHORT)-1; // translation flag of target CLB (processed) PSZ pszDocName = NULL; // document name to check for equal document names in GET_IGNORE_PATH mode - - pString.reserve(MAX_SEGMENT_SIZE); - //allocate pString and buffer for context processing (when necessary) - if ( pGetIn->szContext[0] || (pGetIn->pvGMOptList != NULL) ) - { - //pContextBuffer.reserve(MAX_SEGMENT_SIZE); - } /* endif */ + pString.reserve(MAX_SEGMENT_SIZE); //position at beginning of source structure in tm record pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); @@ -725,9 +719,9 @@ USHORT ExactTest //copy source string for later compare function ulLen = EQFCompress2Unicode( pString, pByte, ulLen ); - auto normalizedTmStr = std::make_unique(pString.c_str()); + auto savedSrcStringVariants = std::make_unique(pString.c_str()); //compare source strings - fStringEqual = UtlCompIgnWhiteSpaceW(normalizedTmStr->getNormStrC(), pSentence->pStrings->getNormStrC(), 0) == 0L; + fStringEqual = UtlCompIgnWhiteSpaceW(savedSrcStringVariants->getNormStrC(), pSentence->pStrings->getNormStrC(), 0) == 0L; if ( fStringEqual ) { @@ -808,12 +802,10 @@ USHORT ExactTest /* (slight) differences */ /* i.e. we will create a fully qualified string and try */ /* another compare... */ - /**********************************************************/ - LONG lLenTmp = ulLen; // len of pString in # of w's - auto genericTagsTmSeg = std::make_unique(std::move(pString)); + /**********************************************************/ fStringEqual = (UtlCompIgnWhiteSpaceW( pSentence->pStrings->getNpReplStrC(), - genericTagsTmSeg->getNpReplStrC(), + savedSrcStringVariants->getNpReplStrC(), 0 ) == 0 ); if (fStringEqual) @@ -1119,7 +1111,12 @@ USHORT ExactTest // use CLB translation flag is ustargetTranslationFlag is not set if ( usTargetTranslationFlag == (USHORT)-1 ) usTargetTranslationFlag = pTMXTargetClb->bTranslationFlag; - FillMatchTable( pTmClb, genericTagsTmSeg->getNormStr(), &lSourceLen, pTMXTargetRecord, + + // probably we should do check as with + // fuzzy to skip already added elements, but since we anyway first run exact test even in fuzzy search, + // and by itself it doen't cause dublicates, next line is skipped + // bool fTestCLB = (false == pstMatchTable->isLocationAlreadyAdded(*pusMatchesFound, lKeyNum, usTrgNum)); + FillMatchTable( pTmClb, savedSrcStringVariants->getGenericTagsString(), &lSourceLen, pTMXTargetRecord, pTMXTargetClb, pstMatchTable, &usMatchLevel, pusMatchEntries, @@ -1303,6 +1300,8 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct UTF16strcpy( SubstProp.szSource, pGetIn->szSource ); pByte = (PBYTE)pTMXTargetRecord; + + MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); wcsncpy(SubstProp.szPropSource, pSourceString.c_str(), pSourceString.size());// *plSourceLen); @@ -1335,7 +1334,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct if ( usRc == NO_ERROR ) { - //locate correct position in matchtable array usCurrentEntry = 0; while ( !fFound && usCurrentEntry < MAX_MATCHES ) @@ -1838,7 +1836,6 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block T5LOG( T5INFO) << "FuzzyTest: Checking targets, lLeftTgtLen = "< 0) && ( lLeftTgtLen >= RECLEN(pTMXTargetRecord) ) && (RECLEN(pTMXTargetRecord) != 0) ) { - BOOL fTestCLB = TRUE; USHORT usModifiedTranslationFlag = 0; /**************************************************************/ @@ -1852,27 +1849,30 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block pByte += pTMXTargetRecord->usClb; pTMXTargetClb = (PTMX_TARGET_CLB)pByte; - //compare target language group IDs - if(pGetIn->fTargetLangIsPrefered){ - if (( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] )) - { - T5LOG( T5INFO) <<"FuzzyTest: Wrong target language!" ; - fTestCLB = FALSE; + bool fTestCLB = (false == pstMatchTable->isLocationAlreadyAdded(*pusMatchesFound, lKeyNum, usTgtNum)); + if(fTestCLB){ + //compare target language group IDs + if(pGetIn->fTargetLangIsPrefered){ + if (( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] )) + { + T5LOG( T5INFO) <<"FuzzyTest: Wrong target language!" ; + fTestCLB = FALSE; + } + } else { + char recordTrgLang[MAX_LANG_LENGTH]; + pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, + (USHORT)LANG_KEY, + recordTrgLang, NULL ); + if(strcasecmp(recordTrgLang, pGetIn->szTargetLanguage)){ + T5LOG( T5WARNING) <<":: target langs is different in record(" << recordTrgLang << ") and request (" << pGetIn->szTargetLanguage<< ")"; + //return SOURCE_LANG_DIFFERENT; + fTestCLB = FALSE; + } } - } else { - char recordTrgLang[MAX_LANG_LENGTH]; - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, - (USHORT)LANG_KEY, - recordTrgLang, NULL ); - if(strcasecmp(recordTrgLang, pGetIn->szTargetLanguage)){ - T5LOG( T5WARNING) <<":: target langs is different in record(" << recordTrgLang << ") and request (" << pGetIn->szTargetLanguage<< ")"; - //return SOURCE_LANG_DIFFERENT; - fTestCLB = FALSE; - } - } - // preprocessing for global memory processing - usModifiedTranslationFlag = pTMXTargetClb->bTranslationFlag; + // preprocessing for global memory processing + usModifiedTranslationFlag = pTMXTargetClb->bTranslationFlag; + } if ( fTestCLB && (pTMXTargetClb->bTranslationFlag == TRANSLFLAG_GLOBMEM) && (pGetIn->pvGMOptList != NULL) && pTMXTargetClb->usAddDataLen ) { USHORT usAddDataLen = 0; @@ -1908,7 +1908,6 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block // if the tag tables are different this is omitted as the // inline tags are suppressed in the proposal in such a case { - LONG lLenTmp = ulSourceLen; pByte = pStartTarget; pTMXTargetRecord = (PTMX_TARGET_RECORD)(pByte); } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 0b80308b..416a85ca 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -2447,7 +2447,7 @@ USHORT TMLoopAndDelTargetClb pClb = (PTMX_TARGET_CLB)NTRecPos((PBYTE)pTMXTgtRec, REC_CLB); lLeftClbLen = RECLEN(pTMXTgtRec) - pTMXTgtRec->usClb; - if(TmProposal.fMatchTargetOnUpdate){ + if(TmProposal.fSaveDifferentTargetsForSameSource){// default pfTargetMatch is true, but if flag is set, we compare saved target to new std::wstring pStringTrg; // check if target string and target tag record are identical pByte = NTRecPos((PBYTE)pTMXTgtRec, REC_TGTSTRING); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index d100f7eb..cf905bb1 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -1133,7 +1133,13 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab numOfEntries = numOfEntriesCapacity; } pTMTable->stTmTableEntry.resize(numOfEntries); - memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, numOfEntries/*(pTMTable->stTmTableEntry.size()-1)*/ * sizeof(TMX_TABLE_ENTRY)); + if(numOfEntries > 0){ + memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, numOfEntries/*(pTMTable->stTmTableEntry.size()-1)*/ * sizeof(TMX_TABLE_ENTRY)); + } + + if(pTMTable->stTmTableEntry.empty() || (pTMTable->stTmTableEntry.back().usId && pTMTable->stTmTableEntry.back().szName[0])){ + pTMTable->stTmTableEntry.push_back(TMX_TABLE_ENTRY{}); + } } return usRc; } @@ -1218,8 +1224,6 @@ USHORT EqfMemory::NTMAddLangGroup( USHORT EqfMemory::NTMCreateLangGroupTable() { USHORT usRC = NO_ERROR; - // pTmClb->LangGroups.ulAllocSize = TMX_TABLE_SIZE; - // allocate language-ID-to-group-ID-table LONG lSize = Languages.stTmTableEntry.size();// > 100L ? Languages.stTmTableEntry.size() : 100L; if (lSize <= 0) @@ -1230,7 +1234,7 @@ USHORT EqfMemory::NTMCreateLangGroupTable() if (usRC == NO_ERROR) { int i = 0; - while ((usRC == NO_ERROR) && (i < psLangIdToGroupTable.size())//(int)Languages.stTmTableEntry.size()) + while ((usRC == NO_ERROR) && (i < Languages.stTmTableEntry.size()) ) { if(Languages.stTmTableEntry[i].szName[0]){ diff --git a/source/opentm2/core/document/EQFTATAG.cpp b/source/opentm2/core/document/EQFTATAG.cpp index 5acddb62..31ba22fd 100755 --- a/source/opentm2/core/document/EQFTATAG.cpp +++ b/source/opentm2/core/document/EQFTATAG.cpp @@ -64,7 +64,7 @@ USHORT TALoadTagTableExHwnd // loads/accesses a tag table HWND hwnd // owner handle for error messages ) { - std::lock_guard l(mutexLoadTagTable); + //std::lock_guard l(mutexLoadTagTable); CHAR szTagTableFileName[MAX_EQF_PATH]; // buffer for file name szTagTableFileName[0]='\0'; PLOADEDTABLE pTable = NULL; // ptr for processing of loaded tables diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index a9704f26..6a2ddfe7 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -1,5 +1,11 @@ #include + +#include +#include +#include +#include // for std::iota + #include "requestdata.h" #include "ThreadingWrapper.h" @@ -255,7 +261,7 @@ DECLARE_int64(tmListLockDefaultTimeout); DECLARE_int64(tmLockDefaultTimeout); DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); -DECLARE_bool(matchTargetOnUpdate); +DECLARE_bool(saveDifferentTargetsForSameSource); //DECLARE_bool(ignoreSegIdOnUpdate); int RequestData::setMutexTimeouts(){ @@ -948,7 +954,7 @@ int ImportRequestData::parseJSON(){ std::string name; std::string value; fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; - fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; //fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; try{ @@ -979,8 +985,8 @@ int ImportRequestData::parseJSON(){ fNewEntryIfOnlyContextAndTimestampDifferent = std::stol(value); }else if(strcasecmp(name.c_str(), "ignoreSegIdOnUpdate") == 0){ fIgnoreSegIdOnUpdate = std::stol(value); - }else if(strcasecmp(name.c_str(), "matchTargetOnUpdate") == 0){ - fMatchTargetOnUpdate = std::stol(value); + }else if(strcasecmp(name.c_str(), "saveDifferentTargetsForSameSource") == 0){ + fSaveDifferentTargetsForSameSource = std::stol(value); }else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; @@ -1177,7 +1183,7 @@ int ImportRequestData::execute(){ mem->importDetails->reset(); mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; - mem->importDetails->fMatchTargetOnUpdate = fMatchTargetOnUpdate; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; @@ -1549,7 +1555,7 @@ USHORT MemFuncOrganizeProcess int ReorganizeRequestData::parseJSON(){ fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; - fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; // parse input parameters @@ -1557,7 +1563,7 @@ int ReorganizeRequestData::parseJSON(){ JSONFactory::JSONPARSECONTROL parseControl[] = { { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent }, - { L"matchTargetOnUpdate", JSONFactory::INT_PARM_TYPE , &(fMatchTargetOnUpdate), FLAGS_matchTargetOnUpdate }, + { L"saveDifferentTargetsForSameSource", JSONFactory::INT_PARM_TYPE , &(fSaveDifferentTargetsForSameSource), FLAGS_saveDifferentTargetsForSameSource }, { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate}, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, @@ -1617,7 +1623,7 @@ int ReorganizeRequestData::execute(){ mem->importDetails->reset(); mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; - mem->importDetails->fMatchTargetOnUpdate = fMatchTargetOnUpdate; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; LONG lCurTime = 0; @@ -1666,7 +1672,7 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; - fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; JSONFactory::JSONPARSECONTROL parseControl[] = { @@ -1678,7 +1684,7 @@ int DeleteEntriesReorganizeRequestData::parseJSON(){ { L"author", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.author ), 0 }, { L"document", JSONFactory::UTF8_STRING_CLASS_PARM_TYPE, &( searchFilterFactory.document ), 0 }, { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent}, - { L"matchTargetOnUpdate", JSONFactory::INT_PARM_TYPE , &(fMatchTargetOnUpdate), FLAGS_matchTargetOnUpdate}, + { L"saveDifferentTargetsForSameSource", JSONFactory::INT_PARM_TYPE , &(fSaveDifferentTargetsForSameSource), FLAGS_saveDifferentTargetsForSameSource}, { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate}, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, @@ -1765,7 +1771,7 @@ int DeleteEntriesReorganizeRequestData::execute(){ mem->importDetails->reset(); mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; - mem->importDetails->fMatchTargetOnUpdate = fMatchTargetOnUpdate; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; LONG lCurTime = 0; @@ -2124,7 +2130,7 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr json_factory.addParmToJSON( outputMessage, "importRuntimeSec", mem->importDetails->lImportRunTimeSec ); json_factory.addParmToJSON( outputMessage, "importTimeoutSec", mem->importDetails->lImportTimeoutSec ); json_factory.addParmToJSON( outputMessage, "newEntryIfOnlyContextAndTimestampDifferent", mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent ); - json_factory.addParmToJSON( outputMessage, "matchTargetOnUpdate", mem->importDetails->fMatchTargetOnUpdate ); + json_factory.addParmToJSON( outputMessage, "saveDifferentTargetsForSameSource", mem->importDetails->fSaveDifferentTargetsForSameSource ); //json_factory.addParmToJSON( outputMessage, "ignoreSegIdOnUpdate", mem->importDetails->fIgnoreSegIdOnUpdate); std::string invalidSegmRCs; @@ -3091,7 +3097,7 @@ int UpdateEntryRequestData::parseJSON(){ { L"recordKey", JSONFactory::INT_PARM_TYPE , &(recordKey), 0 }, { L"targetKey", JSONFactory::INT_PARM_TYPE , &(targetKey), 0 }, { L"newEntryIfOnlyContextAndTimestampDifferent", JSONFactory::INT_PARM_TYPE , &(Data.fNewEntryIfOnlyContextAndTimestampDifferent), FLAGS_newEntryIfOnlyContextAndTimestampDifferent }, - { L"matchTargetOnUpdate", JSONFactory::INT_PARM_TYPE , &(Data.fMatchTargetOnUpdate), FLAGS_matchTargetOnUpdate }, + { L"saveDifferentTargetsForSameSource", JSONFactory::INT_PARM_TYPE , &(Data.fSaveDifferentTargetsForSameSource), FLAGS_saveDifferentTargetsForSameSource }, { L"ignoreSegIdOnUpdate", JSONFactory::INT_PARM_TYPE , &(Data.fIgnoreSegIdOnUpdate), FLAGS_ignoreSegIdOnUpdate }, { L"tmMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmMutexTimeoutMs, -1}, { L"tmListMutexTimeout", JSONFactory::INT_PARM_TYPE, &tmListMutexTimeoutMs, -1}, @@ -3525,6 +3531,10 @@ int FuzzySearchRequestData::parseJSON(){ _rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); + if(_rc_){ + return buildErrorReturn(_rc_, "error in parseJson"); + } + if(loggingThreshold >=0) T5Logger::GetInstance()->SetLogLevel(loggingThreshold); return _rc_; @@ -3583,11 +3593,21 @@ int addProposalsToJSONString ) { JSONFactory *pJsonFactory = JSONFactory::getInstance(); + // Create index view [0, 1, 2, ..., n-1] + std::vector indices(iNumOfProposals); + std::iota(indices.begin(), indices.end(), 0); + + + // Sort indices by comparing the actual proposals + std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) { + return vProposals[a].getFuzziness() > vProposals[b].getFuzziness(); + }); pJsonFactory->addArrayStartToJSONW( strJSON ); - for ( int i = 0; i < iNumOfProposals; i++ ) + //for ( int i = 0; i < iNumOfProposals; i++ ) + for (size_t idx : indices)//use indices to have sorted output { - addProposalToJSONString( strJSON, vProposals[i], fAddFuzzyInfo); + addProposalToJSONString( strJSON, vProposals[idx], fAddFuzzyInfo); } /* endfor */ pJsonFactory->addArrayEndToJSONW( strJSON ); @@ -3639,38 +3659,64 @@ int FuzzySearchRequestData::execute(){ } tokens += "\n}]"; } + + std::vector addedRecords; if ( _rc_ == 0 ) { T5LOG( T5DEBUG) <<"EqfMemory::OtmProposal:: lookup complete, found " << GetOut.usNumMatchesFound << " proposals" ; + //sort GetOut.stMatchTable[j]; + std::sort( + GetOut.stMatchTable, + GetOut.stMatchTable + GetOut.usNumMatchesFound, + [](const TMX_MATCH_TABLE_W& a, const TMX_MATCH_TABLE_W& b) { + return a.usMatchLevel > b.usMatchLevel; + } + ); wchar_t szRequestedString[2049]; Data.getSource( szRequestedString, sizeof(szRequestedString) ); + - for ( int i = 0; i < (int)vProposals.size(); i++ ) + for ( int i = 0, j = 0; i < (int)vProposals.size(); j++ ) { - if ( i >= GetOut.usNumMatchesFound ) + if ( j >= GetOut.usNumMatchesFound ) { - vProposals[i].clear(); + vProposals[i++].clear(); } else { + auto& match = GetOut.stMatchTable[j]; if(T5Logger::GetInstance()->CheckLogLevel(T5DEBUG)){ - auto strSource = EncodingHelper::convertToUTF8(GetOut.stMatchTable[i].szSource ); - T5LOG( T5DEBUG) <<"EqfMemory::OtmProposal:: proposal " << i << ": match=" << GetOut.stMatchTable[i].usMatchLevel << ", source=", strSource; + auto strSource = EncodingHelper::convertToUTF8(match.szSource ); + T5LOG( T5DEBUG) <<"EqfMemory::OtmProposal:: proposal " << i << ": match=" << match.usMatchLevel << ", source=", strSource; } - //replace tags for proposal - auto result = std::make_unique(GetOut.stMatchTable[i].szSource, GetOut.stMatchTable[i].szTarget, - szRequestedString); - wcscpy(GetOut.stMatchTable[i].szSource, result->getSrcWithTagsFromRequestC()); - wcscpy(GetOut.stMatchTable[i].szTarget, result->getTrgWithTagsFromRequestC()); - - if( GetOut.stMatchTable[i].usMatchLevel >= 100){ - //correct match rate for exact match based on whitespace difference - int wsDiff = 0; - UtlCompIgnWhiteSpaceW(szRequestedString, GetOut.stMatchTable[i].szSource, 0, &wsDiff); - GetOut.stMatchTable[i].usMatchLevel -= wsDiff; + + bool proposalAlreadyAdded = std::any_of(addedRecords.begin(), addedRecords.end(), + [&](const TMCursor& cursor) { + return cursor.getRecordKey() == match.lKey && + cursor.getTargetKey() == match.usTargetNum; + }); + + if(!proposalAlreadyAdded){ + //replace tags for proposal + auto result = std::make_unique(match.szSource, match.szTarget, + szRequestedString); + wcscpy(match.szSource, result->getSrcWithTagsFromRequestC()); + wcscpy(match.szTarget, result->getTrgWithTagsFromRequestC()); + + if( match.usMatchLevel >= 100){ + //correct match rate for exact match based on whitespace difference + int wsDiff = 0; + UtlCompIgnWhiteSpaceW(szRequestedString, match.szSource, 0, &wsDiff); + match.usMatchLevel -= wsDiff; + } + + TMCursor location; + location.setInternalKey(match.lKey, match.usTargetNum); + addedRecords.push_back(location); + mem->MatchToOtmProposal( &match, &vProposals[i] ); + i++; } - mem->MatchToOtmProposal( GetOut.stMatchTable + i, &vProposals[i] ); } /* end */ } /* end */ } @@ -3700,6 +3746,14 @@ int FuzzySearchRequestData::execute(){ if ( Data.iNumOfProposals > 0 ) { + //std::sort( + // vProposals.begin(), + // vProposals.end(), + // [](const OtmProposal& a, const OtmProposal& b) { + // return a.getFuzziness() > b.getFuzziness(); + // } + //); + json_factory.addNameToJSONW( strOutputParmsW, L"results" ); addProposalsToJSONString( strOutputParmsW, vProposals, Data.iNumOfProposals, true ); } /* endif */ diff --git a/source/opentm2/core/utilities/EQFMORPH.cpp b/source/opentm2/core/utilities/EQFMORPH.cpp index 6a2c24b7..67c96398 100755 --- a/source/opentm2/core/utilities/EQFMORPH.cpp +++ b/source/opentm2/core/utilities/EQFMORPH.cpp @@ -1269,101 +1269,6 @@ BOOL MorphHashDestroy // add new element to low level hash // update recently used chain //------------------------------------------------------------------------------ -BOOL MorphHashAdd -( - PMORPHHASH pMorphHash, // pointer to our hash control strucuture - PSZ_W pszTerm, // name of term being added - ULONG ulLemmaListLen, // length of lemma list in bytes - PSZ_W pszLemmaList // lemma list for term or NULL -) -{ - HASHENTRY HashEntry; // buffer for new hash entry - PHASHENTRY pHashEntry; // pointer to found/created hash entry - ULONG ulEntry; // number (index) of new entry - - /********************************************************************/ - /* Check if term is already in cash */ - /********************************************************************/ - HashEntry.pszTerm = pszTerm; - pHashEntry = (PHASHENTRY) HashSearch( pMorphHash->pHash, &HashEntry ); - - /********************************************************************/ - /* Remove oldest entry from cache if all cache elements are in use */ - /********************************************************************/ - if ( !pHashEntry ) // if not in cache yet ... - { - if ( pMorphHash->ulElements == pMorphHash->ulMaxElements ) - { - /****************************************************************/ - /* Cache is full: remove oldest element from cache */ - /****************************************************************/ - MorphHashDelete( pMorphHash, pMorphHash->pElements + pMorphHash->ulFirstUsed ); - pHashEntry = NULL; - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* Add term and lemma list to string pool */ - /********************************************************************/ - if ( !pHashEntry ) // if not in cache yet ... - { - HashEntry.pszTerm = PoolAddStringW( pMorphHash->StringPool, pszTerm ); - HashEntry.ulListLength = ulLemmaListLen; - if ( pszLemmaList ) - { - HashEntry.pLemmaList = (PSZ_W) PoolAddData( pMorphHash->StringPool, - (ulLemmaListLen * sizeof(CHAR_W)), - pszLemmaList ); - } - else - { - HashEntry.pLemmaList = NULL; - } /* endif */ - } /* endif */ - - /********************************************************************/ - /* Add new element to low level hash */ - /********************************************************************/ - if ( !pHashEntry ) // if not in cache yet ... - { - /******************************************************************/ - /* Get entry from free chain */ - /******************************************************************/ - ulEntry = pMorphHash->ulFirstFree; - pHashEntry = pMorphHash->pElements + ulEntry; - pMorphHash->ulFirstFree = pHashEntry->ulNext; - pMorphHash->ulElements++; - - /******************************************************************/ - /* Fill-in element data */ - /******************************************************************/ - pHashEntry->pszTerm = HashEntry.pszTerm; - pHashEntry->ulListLength = HashEntry.ulListLength; - pHashEntry->pLemmaList = HashEntry.pLemmaList; - - /******************************************************************/ - /* add element to end of recently used chain */ - /******************************************************************/ - pHashEntry->ulNext = END_OF_ELEMENTS; - pHashEntry->ulPrev = pMorphHash->ulLastUsed; - if ( pMorphHash->ulLastUsed != END_OF_ELEMENTS ) - { - pMorphHash->pElements[pMorphHash->ulLastUsed].ulNext = pHashEntry->ulKey; - } /* endif */ - pMorphHash->ulLastUsed = ulEntry; - if ( pMorphHash->ulFirstUsed == END_OF_ELEMENTS ) - { - pMorphHash->ulFirstUsed = ulEntry; - } /* endif */ - - /******************************************************************/ - /* Add element to low level hash */ - /******************************************************************/ - HashAdd( pMorphHash->pHash, pHashEntry ); - } /* endif */ - - return( TRUE ); -} /* end of function MorphHashAdd */ //------------------------------------------------------------------------------ // Internal function @@ -1394,26 +1299,6 @@ BOOL MorphHashAdd //------------------------------------------------------------------------------ // Function flow: call low level hash delete function //------------------------------------------------------------------------------ -BOOL MorphHashDelete -( - PMORPHHASH pMorphHash, // pointer to our hash control strucuture - PHASHENTRY pEntry // pointer to the entry being sdeleted -) -{ - - // delete the hash entry, the callback function MorphHassFree will be called (which adjusts the recently used chain and the free entries chain) - if ( HashDelete( pMorphHash->pHash, pEntry ) ) - { - return( TRUE ); // nothing to do any more, MorphHashFree has done the house keeping - } /* endif */ - - // this part of the function is called when HashDelete failed to delete the entry, - // we will have to remove the entry from the MorphHash table ... - MorphHashFree( pEntry, (PVOID)pMorphHash ); - - return( TRUE ); -} /* end of function MorphHashDelete */ - //------------------------------------------------------------------------------ // Internal function diff --git a/source/opentm2/core/utilities/OtmProposal.cpp b/source/opentm2/core/utilities/OtmProposal.cpp index 4a1b7637..b11d79fe 100755 --- a/source/opentm2/core/utilities/OtmProposal.cpp +++ b/source/opentm2/core/utilities/OtmProposal.cpp @@ -36,7 +36,7 @@ OtmProposal::~OtmProposal() /* operations */ DECLARE_bool(newEntryIfOnlyContextAndTimestampDifferent); -DECLARE_bool(matchTargetOnUpdate); +DECLARE_bool(saveDifferentTargetsForSameSource); //DECLARE_bool(ignoreSegIdOnUpdate); void OtmProposal::clear()//bool clearKeys) @@ -45,7 +45,7 @@ void OtmProposal::clear()//bool clearKeys) TMCursor currentKey, nextKey; *this = OtmProposal{}; fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent;//default value - fMatchTargetOnUpdate = FLAGS_matchTargetOnUpdate; + fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; } @@ -338,7 +338,7 @@ void OtmProposal::setUpdateTime( long lTime ) /* \brief get proposal fuzziness \returns proposal fuzziness */ -int OtmProposal::getFuzziness() +int OtmProposal::getFuzziness()const { return( iFuzziness ); } diff --git a/source/otmd.cpp b/source/otmd.cpp index 0577072f..963d6075 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -203,7 +203,7 @@ DEFINE_bool(enable_newlines_in_logs, false, "(not working - instead of using function to remove newlines in runtime, logs was edited to have better perfomance)if set to true, would keep newline symbols in the logs, otherwise(by default) newlines would be removed and logs would be oneliners. This just calls function to remove newlines," ); DEFINE_bool(newEntryIfOnlyContextAndTimestampDifferent, true, "If it is set to true, the behavior should be: A new entry should be saved to the TM, if only timestamp and context are different and all other params the same;\t If it is set to false, the beahvior should be: if context is different and timestamp is different, the context and timestamp of the new to be saved segment should overwrite the context and segment of an entry, that already exists and where all other params are the same as in the new segment. This is basically the same behavior it would have, if context would not be set for both segments (the old and the new)." ); -DEFINE_bool(matchTargetOnUpdate, true, "If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets, if false- previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer." ); +DEFINE_bool(saveDifferentTargetsForSameSource, true, "If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets(old target = new target), if false -> previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer." ); //DEFINE_bool(ignoreSegIdOnUpdate, true, "If it is set to true, seg id check would be skipped. Could be useful for importing, because segId there are generated, so never match" ); @@ -252,6 +252,19 @@ void signal_handler(int signal) { gSignalStatus = signal; T5LOG(T5TRANSACTION) << " Received signal "<< signal<< "; Saving TM's and Shutting down :"; + + // Attempt to dump last request for debug + try { + std::string lastReq = T5Logger::GetInstance()->__getLogBuffer().str(); + std::string lastBody = T5Logger::GetInstance()->__getBodyBuffer().str(); + std::cerr << "Last request:\n"; + if (!lastReq.empty()) { + std::cerr << "=== LAST REQUEST LOG (may be incomplete) ===\n" << lastReq << "\n with body\"" << lastBody <<"\""; + } + } catch (...) { + std::cerr << "Failed to dump last request log (exception)\n"; + } + MutexTimeout mt{0}; ShutdownRequestData::CheckImportFlushTmsAndShutdown(signal, mt, FLAGS_flush_tm_at_shutdown, FLAGS_wait_for_import_and_reorganize_requests); @@ -265,6 +278,20 @@ void signal_handler(int signal) sleep(1); } + +extern "C" void __asan_on_error() { + // Called by ASan before it crashes the app + fprintf(stderr, "[ASAN-HOOK] DUMPING LAST REQUEST BUFFER:\n"); + std::string logBuff = T5Logger::GetInstance()->__getLogBuffer().str(); + std::string lastBody = T5Logger::GetInstance()->__getBodyBuffer().str(); + + if (!logBuff.empty()) { + fprintf(stderr, "%s with body %s\n", logBuff.c_str(), lastBody.c_str()); + } else { + fprintf(stderr, "(empty)\n"); + } +} + /* void CustomPrefix(std::ostream &s, const LogMessageInfo &l, void*) { l. From 49409f4b5e7519231a4a9ad6e03c17116906bc20 Mon Sep 17 00:00:00 2001 From: Leon Kyz Date: Sun, 27 Jul 2025 14:58:49 +0300 Subject: [PATCH 39/62] Update LogWrapper.h --- source/opentm2/core/utilities/LogWrapper.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/opentm2/core/utilities/LogWrapper.h b/source/opentm2/core/utilities/LogWrapper.h index 601eaa2b..e8de4ab7 100755 --- a/source/opentm2/core/utilities/LogWrapper.h +++ b/source/opentm2/core/utilities/LogWrapper.h @@ -52,7 +52,11 @@ std::basic_ostream& operator<<(std::basic_ostream Date: Mon, 28 Jul 2025 22:45:39 +0300 Subject: [PATCH 41/62] * updated to v0.7.14 + added skipStartupLogs flags(default true) * fixed non-asan builds * fixed nextInternalKey is empty log on failed export --- source/CMakeLists.txt | 2 +- source/RestAPI/ProxygenServer.cpp | 10 +++++----- source/opentm2/core/requestdata.cpp | 2 +- source/opentm2/core/utilities/LogWrapper.cpp | 6 ++++-- source/opentm2/core/utilities/LogWrapper.h | 6 +++--- source/otmd.cpp | 19 ++++++++++++++----- 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 319833a7..78bb3411 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 13) +SET(APP_VERSION_MINOR 14) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index 558c230d..f5ff5a72 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -359,7 +359,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { uint32_t receiveSessionWindowSize = 10 * (1 << 20) * 10; - T5LOG( T5TRANSACTION) << "Trying to prepare t5memory"; + T5LOG( T5INFO) << "Trying to prepare t5memory"; auto pMemService = OtmMemoryServiceWorker::getInstance(); char szServiceName[100] = "t5memory"; @@ -415,9 +415,8 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { //From here we have logging in file turned on T5Logger::GetInstance()->DesuppressLoggingInFile(); - T5Logger::GetInstance()->SetLogLevel(uiLogLevel); - char szValue[150]; + char szValue[150]{}; std::string host; @@ -438,7 +437,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { } - T5LOG(T5TRANSACTION) << ":: binding to socket, " << addr.getAddressStr() << "; port = " <SetLogLevel(uiLogLevel); if( true /*printIninMsg*/){ - T5LOG(T5TRANSACTION) << "Service details: Service name = " << szServiceName << + T5INITLOG << "Service details: Service name = " << szServiceName << "\n Additional service name: "<< additionalServiceName << "\n Address :" << host << ":" << uiPort << "\n Build date: " << buildDate << diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 6a2ddfe7..b44fcbc8 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -517,7 +517,7 @@ int RequestData::run(){ memRef.reset(); } - if(COMMAND::EXPORT_MEM_TMX == command || COMMAND::EXPORT_MEM_TMX_STREAM == command){ + if(!res && (COMMAND::EXPORT_MEM_TMX == command || COMMAND::EXPORT_MEM_TMX_STREAM == command)){ auto exportRequest = (ExportRequestData*) this; if(exportRequest->nextInternalKey.empty()){ T5LOG(T5ERROR) << "nextInternalKey is empty for" << "\t req body = \"" <strBody << "\"; " << "\t nextKey =\"" << exportRequest->nextInternalKey <<"\"; "; diff --git a/source/opentm2/core/utilities/LogWrapper.cpp b/source/opentm2/core/utilities/LogWrapper.cpp index a0565468..58d1bfa2 100755 --- a/source/opentm2/core/utilities/LogWrapper.cpp +++ b/source/opentm2/core/utilities/LogWrapper.cpp @@ -178,9 +178,11 @@ std::string T5Logger::FlushBodyBuffer(){ return str; } -int T5Logger::SetLogLevel(int level){ +int T5Logger::SetLogLevel(int level, bool fAddMessage){ if(level <= T5FATAL && level>= T5DEVELOP){ - T5LOG(T5INFO) << "SetLogLevel::Log level treshold was set to " << level; + if(fAddMessage){ + T5LOG(T5INFO) << "SetLogLevel::Log level treshold was set to " << level; + } logLevelTreshold = level; return level; }else{ diff --git a/source/opentm2/core/utilities/LogWrapper.h b/source/opentm2/core/utilities/LogWrapper.h index e8de4ab7..5196c0be 100755 --- a/source/opentm2/core/utilities/LogWrapper.h +++ b/source/opentm2/core/utilities/LogWrapper.h @@ -135,7 +135,7 @@ class T5Logger{ std::string FlushLogBuffer(); std::string FlushBodyBuffer(); - int SetLogLevel(int level); + int SetLogLevel(int level, bool fAddMessage = true); int GetLogLevel(); bool CheckLogLevel(int level); void SetLogFilter(bool filterOn); @@ -191,10 +191,10 @@ int getBuffIdForLog(int severity); : id == -2? google::LogMessageVoidify() & T5Logger::__getLogBuffer() \ : google::LogMessageVoidify() & _T5_LOG_##severity +#define T5INITLOG LOG(INFO)<< "::" << __func__ << ": " + #define T5LOG(severity) T5LOG_FROM_ID(getBuffIdForLog(severity), severity) << \ "::[" << #severity << "]::" << __func__ << ": " - - #endif //CUSTOM_LOG_MESSAGE #define LOG_AND_SET_RC_W_INFO(rcVar, logLevel, rc) int ___prev_rc = rcVar; rcVar = rc; T5LOG(logLevel) << "rc was set to " <<#rc << "(" << rcVar <<"), prev state = " <<___prev_rc <<"; " diff --git a/source/otmd.cpp b/source/otmd.cpp index 963d6075..d007ad5d 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -167,6 +167,8 @@ static bool ValidateLOGlevel(const char* flagname, int32_t value) { DEFINE_int32(t5loglevel, T5INFO, "Sets t5memory log level threshold from DEVELOP(0) to TRANSACTION(6)"); DEFINE_validator(t5loglevel, &ValidateLOGlevel); +DEFINE_bool(skipStartupLogs, true, "if set to true, values from --t5loglevel would be applied immediately, if set to false- you would see all startup logs and --v and --t5loglevel would be applied only after start of the service and it's init message. Set to false if you have issues at startup"); + DEFINE_int64(tmRequestLockDefaultTimeout, 3000, "Sets tm mutex lock timeout(in ms) for part where request is requesting tm(which is used to open and close tms, and hold list of opened tms), after which operation would be canceled and mutex would return an error, if set to 0, mutex lock would be waited without timeout"); @@ -203,7 +205,7 @@ DEFINE_bool(enable_newlines_in_logs, false, "(not working - instead of using function to remove newlines in runtime, logs was edited to have better perfomance)if set to true, would keep newline symbols in the logs, otherwise(by default) newlines would be removed and logs would be oneliners. This just calls function to remove newlines," ); DEFINE_bool(newEntryIfOnlyContextAndTimestampDifferent, true, "If it is set to true, the behavior should be: A new entry should be saved to the TM, if only timestamp and context are different and all other params the same;\t If it is set to false, the beahvior should be: if context is different and timestamp is different, the context and timestamp of the new to be saved segment should overwrite the context and segment of an entry, that already exists and where all other params are the same as in the new segment. This is basically the same behavior it would have, if context would not be set for both segments (the old and the new)." ); -DEFINE_bool(saveDifferentTargetsForSameSource, true, "If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets(old target = new target), if false -> previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer." ); +DEFINE_bool(saveDifferentTargetsForSameSource, false, "If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets(old target = new target), if false -> previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer." ); //DEFINE_bool(ignoreSegIdOnUpdate, true, "If it is set to true, seg id check would be skipped. Could be useful for importing, because segId there are generated, so never match" ); @@ -278,7 +280,7 @@ void signal_handler(int signal) sleep(1); } - +#ifdef __SANITIZE_ADDRESS__ extern "C" void __asan_on_error() { // Called by ASan before it crashes the app fprintf(stderr, "[ASAN-HOOK] DUMPING LAST REQUEST BUFFER:\n"); @@ -291,7 +293,11 @@ extern "C" void __asan_on_error() { fprintf(stderr, "(empty)\n"); } } - +#else +//void __asan_on_error() { +// // No-op fallback when ASan is disabled +//} +#endif /* void CustomPrefix(std::ostream &s, const LogMessageInfo &l, void*) { l. @@ -391,12 +397,15 @@ int main(int argc, char* argv[]) { // Install the custom LogSink //static NoNewlineLogSink sink; //google::AddLogSink(&sink); - T5LOG(T5TRANSACTION) <<"Worker thread starting, v = "< Date: Wed, 30 Jul 2025 23:06:57 +0300 Subject: [PATCH 42/62] * updated to v0.7.15 + implemented -307 * fixed outputting langs in fuzzy search * fixed -286 + added limit to prevent buffer overflow for longnames + added errCode for response message when can't load tm --- include/otm.h | 4 - include/tm.h | 5 +- source/CMakeLists.txt | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp | 30 ++--- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 10 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 104 +++++++++++------- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 5 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 32 +++--- .../core/EqfMemoryPlugin/EqfMemory.cpp | 2 +- source/opentm2/core/requestdata.cpp | 8 +- .../opentm2/core/utilities/LanguageFactory.H | 2 + .../core/utilities/LanguageFactory.cpp | 8 ++ source/opentm2/core/utilities/UtlMisc.cpp | 59 ---------- 15 files changed, 120 insertions(+), 157 deletions(-) diff --git a/include/otm.h b/include/otm.h index fbf5c062..0cc83203 100644 --- a/include/otm.h +++ b/include/otm.h @@ -71,7 +71,6 @@ struct ImportStatusDetails{ std::atomic_int invalidSegments{0}; std::atomic_int invalidSymbolErrors{0}; std::atomic_int resSegments {0}; - //std::atomic_long lActiveSegment{0}; int importRc{0}; long filteredSegments{0}; std::map invalidSegmentsRCs; @@ -82,7 +81,6 @@ struct ImportStatusDetails{ std::atomic_long lImportRunTimeSec; std::stringstream importMsg; bool fReorganize{0}; // true for reorganize call, false for import - size_t activeSegment{0}; InclosingTagsBehaviour inclosingTagsBehaviour{InclosingTagsBehaviour::saveAll}; BOOL fNewEntryIfOnlyContextAndTimestampDifferent{}; BOOL fSaveDifferentTargetsForSameSource{}; @@ -96,7 +94,6 @@ struct ImportStatusDetails{ firstInvalidSegmentsSegNums.reserve(100); importMsg.str(""); usProgress = 0; - //lActiveSegment = 0; importRc = 0; segmentsCount = 0; segmentsImported = 0; @@ -109,7 +106,6 @@ struct ImportStatusDetails{ lImportRunTimeSec = 0; importTimestamp = "not finished"; invalidSegmentsRCs.clear(); - activeSegment = 0; } std::string toString(){ diff --git a/include/tm.h b/include/tm.h index a2281fc2..70372228 100644 --- a/include/tm.h +++ b/include/tm.h @@ -2020,6 +2020,9 @@ class EqfMemory //: public TMX_CLB USHORT TokenizeSource(PTMX_SENTENCE, PSZ, PSZ); + + bool areLangsMatching(const ushort langId1, const ushort langId2, const char* langStr1);//, bool fPrefferedLang1); + USHORT NTMGetIDFromName(PSZ, PSZ, USHORT, PUSHORT); USHORT NTMGetIDFromNameEx( PSZ pszName, // input, name being looked up @@ -2029,7 +2032,7 @@ class EqfMemory //: public TMX_CLB LONG lOptions, PUSHORT pusAlternativeID // output, alternative ID ); - USHORT NTMGetNameFromID(PUSHORT, USHORT, PSZ, PSZ); + USHORT NTMGetNameFromID(const USHORT, USHORT, PSZ, PSZ, SHORT maxBuffLen = -1); USHORT NTMGetPointersToTable(USHORT, PTMX_TABLE *, PTMX_TABLE_ENTRY *); USHORT NTMAddNameToTable(PSZ, USHORT, PUSHORT); diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 78bb3411..c924b1c5 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 14) +SET(APP_VERSION_MINOR 15) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp index eac35508..60717fa2 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp @@ -390,8 +390,6 @@ USHORT EQFMemExportEnd ( PPROCESSCOMMAREA pCommArea, //--- Close MemoryDb and input file. Return codes are for testing purposes only if ( pExportIDA->pMem != NULL) { - TMManager *pFactory = TMManager::GetInstance(); - //pFactory->closeMemory( pExportIDA->pMem ); pExportIDA->pMem = NULL; } /* endif */ if ( pExportIDA->hFile ) CloseFile( &(pExportIDA->hFile) ); @@ -403,26 +401,16 @@ USHORT EQFMemExportEnd ( PPROCESSCOMMAREA pCommArea, REMOVESYMBOL( ObjName ); } - + if ( (hwndErrMsg == HWND_FUNCIF) && pExportIDA->fDataCorrupted ) { + PSZ pszParm = "ANSI/ASCII"; + T5LOG(T5ERROR) << "::ERROR_MEM_EXPORT_DATACORRUPT::" << pszParm; + LOG_AND_SET_RC(usRC, T5INFO, ERROR_MEM_EXPORT_DATACORRUPT); + } /* endif */ - if ( (hwndErrMsg == HWND_FUNCIF) && pExportIDA->fDataCorrupted ) - { - PSZ pszParm = "ANSI/ASCII"; - T5LOG(T5ERROR) << "::ERROR_MEM_EXPORT_DATACORRUPT::" << pszParm; - LOG_AND_SET_RC(usRC, T5INFO, ERROR_MEM_EXPORT_DATACORRUPT); - } /* endif */ - - if ( pExportIDA->pProposal != NULL ) delete(pExportIDA->pProposal); - UtlAlloc( (PVOID *) &pExportIDA, 0L, 0L, NOMSG ); - pCommArea->pUserIDA = NULL; - - // Dismiss the slider window if it had been created - if ( hwndErrMsg != HWND_FUNCIF ) - { - //EqfRemoveObject( TWBFORCE, hWnd ); - } /* endif */ - } + if ( pExportIDA->pProposal != NULL ) delete(pExportIDA->pProposal); + UtlAlloc( (PVOID *) &pExportIDA, 0L, 0L, NOMSG ); + pCommArea->pUserIDA = NULL; return( usRC ); } /* end of function EQFMemExportEnd */ @@ -647,14 +635,12 @@ USHORT MemExportProcess ( PMEM_EXPORT_IDA pExportIDA ) // pointer to the export { usRc = MemExportWriteFile( pExportIDA ); } /* endif */ - } else { //--- Increase the segment counter by one pExportIDA->ulSegmentCounter++; - //--- Prepare the segment and write it to the export file if ( (pExportIDA->usExpMode == MEM_FORMAT_TMX) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_NOCRLF) || (pExportIDA->usExpMode == MEM_FORMAT_TMX_UTF8_NOCRLF) ) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index 5306fecb..ad8c9b21 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -937,7 +937,7 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT { pLIDA->mem->importDetails->importRc = usRC; std::string msg = "TM is reached it's size limit, please create another one and import segments there, rc = " - + toStr(usRC) + "; aciveSegment = " + toStr(pSegment->lSegNo) + "\n\nSegment " + toStr(pSegment->lSegNo) + " not imported\r\n" + + toStr(usRC) + "; aciveSegment = " + toStr(pLIDA->mem->importDetails->segmentsCount)/*toStr(pSegment->lSegNo)*/ + "\n\ lSegNo = " + toStr(pSegment->lSegNo) + " not imported\r\n" +"\nReason = " + pSegment->szReason +"\nDocument = " + pSegment->szDocument +"\nSourceLanguage = " + pSegment->szSourceLang diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 63d1b9ca..25adcf14 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -364,7 +364,7 @@ USHORT EqfMemory::FindTargetAndDelete( pClb, TmDel.pInputSentence->pStrings->getGenericTargetStr(), &lSrcLen, pTmExtOut, endOfMemRegion ); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + NTMGetNameFromID(pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); pTmExtOut->ulRecKey = lKey; TmDel.currentInternalKey.setInternalKey(lKey, usTarget); @@ -577,7 +577,7 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( pTargetClb, pSourceString, &lSourceLen, pTmExtOut, endOfMemRegion ); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); TmDel.currentInternalKey.setInternalKey(lKey, usTarget); pTmExtOut->ulRecKey = lKey; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 1ae7e5ab..998bcc36 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -286,7 +286,7 @@ USHORT EqfMemory::ExtractRecord pTargetClb, pSourceString, &lSourceLen, pTmExtOut, endOfMemRegion); - NTMGetNameFromID( &pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage, NULL ); if ( ! usRc ) { @@ -452,8 +452,8 @@ USHORT EqfMemory::FillExtStructure } /* endif */ //fill in the target file name - NTMGetNameFromID( &pTMXTargetClb->usFileId, (USHORT)FILE_KEY, - pstExt->szFileName, pstExt->szLongName ); + NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, + pstExt->szFileName, pstExt->szLongName, sizeof(pstExt->szFileName) ); //use overflow name if no document name available if ( pstExt->szFileName[0] == EOS ) { @@ -461,11 +461,11 @@ USHORT EqfMemory::FillExtStructure } /* endif */ //fill in the target author - NTMGetNameFromID( &pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, + NTMGetNameFromID( pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, pstExt->szAuthorName, NULL ); //fill in the target language - NTMGetNameFromID( &pTMXTargetClb->usLangId, (USHORT)LANG_KEY, + NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, pstExt->szTargetLanguage, NULL ); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index e80db41b..c09c8566 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -29,6 +29,7 @@ #include "../utilities/LogWrapper.h" #include "../utilities/Property.h" #include "../utilities/EncodingHelper.h" +#include "../utilities/LanguageFactory.H" // import logging @@ -713,6 +714,17 @@ USHORT ExactTest //move pointer to corresponding position pByte = (PBYTE)(pTmRecord+1); pByte += pTMXSourceRecord->usSource; + //get id of target language in the getin structure + USHORT usSrcLangId = 0; + if (pTmClb->NTMGetIDFromName( pGetIn->szSourceLanguage, NULL, + (USHORT)LANG_KEY, &usSrcLangId)) + { + usSrcLangId = 0; + } /* endif */ + + if(!pTmClb->areLangsMatching(usSrcLangId, pTMXSourceRecord->usLangId, pGetIn->szSourceLanguage)){ + return SOURCE_LANG_DIFFERENT; + } //calculate length of source string ulLen = (RECLEN(pTMXSourceRecord) - sizeof(TMX_SOURCE_RECORD)); @@ -785,6 +797,11 @@ USHORT ExactTest pTMXTargetClb = (PTMX_TARGET_CLB)pByte; //compare target language group IDs + if(!pTmClb->areLangsMatching(usGetLang, pTMXTargetClb->usLangId, pGetIn->szTargetLanguage)){ + //T5LOG( T5WARNING) <<":: source langs is different in record(" << pTMXSourceRecord->usLangId << ") and request (" << pGetIn->szSourceLanguage << ")"; + fTestCLB = FALSE; + } + if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usGetLang] ) { fTestCLB = FALSE; @@ -1295,12 +1312,10 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /* add current tagging to source and translation of proposal */ /****************************************************************/ strcpy( SubstProp.szSourceTagTable, pGetIn->szTagTable ); - strcpy( SubstProp.szSourceLanguage, pGetIn->szSourceLanguage); - strcpy( SubstProp.szTargetLanguage, pGetIn->szTargetLanguage ); + //strcpy( SubstProp.szSourceLanguage, pGetIn->szSourceLanguage); + //strcpy( SubstProp.szTargetLanguage, pGetIn->szTargetLanguage ); UTF16strcpy( SubstProp.szSource, pGetIn->szSource ); - pByte = (PBYTE)pTMXTargetRecord; - - + pByte = (PBYTE)pTMXTargetRecord; MorphGetLanguageID( pGetIn->szSourceLanguage, &sLangID ); @@ -1326,7 +1341,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct //fill in the markup table strcpy(SubstProp.szPropTagTable, "OTMXUXLF"); //fill in the target language - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, + pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, SubstProp.szTargetLanguage, NULL ); } /* endif */ @@ -1353,10 +1368,10 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct else if ( iMatchSortKey == iNewMatchSortKey ) { CHAR szTgtFileName[ MAX_FILESPEC ]; // target file name - static CHAR szTgtLongName[ MAX_LONGFILESPEC ]; // long target file name + CHAR szTgtLongName[ MAX_LONGFILESPEC ]; // long target file name //fill in the target file name - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usFileId, (USHORT)FILE_KEY, szTgtFileName, szTgtLongName ); + pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, szTgtFileName, szTgtLongName, sizeof(szTgtLongName) ); /****************************************************************/ /* as the matching levels are the same determine the most recent*/ @@ -1465,12 +1480,12 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct if ( usRc == NO_ERROR ) { //fill in the target file name - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usFileId, + pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, pstMatchTable->szFileName, - pstMatchTable->szLongName ); + pstMatchTable->szLongName, sizeof(pstMatchTable->szLongName)); //fill in the target author - don't care about author name.... - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usAuthorId, + pTmClb->NTMGetNameFromID( pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, pstMatchTable->szTargetAuthor, NULL ); @@ -1478,11 +1493,11 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct strcpy(pstMatchTable->szTagTable, "OTMXUXLF"); //fill in the target and original src language - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, + pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, pstMatchTable->szTargetLanguage, NULL ); - pTmClb->NTMGetNameFromID( &usSrcLangId, + pTmClb->NTMGetNameFromID( usSrcLangId, (USHORT)LANG_KEY, pstMatchTable->szOriginalSrcLanguage, NULL ); //fill in the segment id @@ -1694,6 +1709,31 @@ std::wstring removeTagsFromString(std::wstring input){ return std::wstring(pOutStart); } +bool EqfMemory::areLangsMatching(const ushort langId1, const ushort langId2, + const char* langStr1 + ){ + if(!langId1 || !langId2){// if any of them has id=0 - no match + return false; + } + + //lang matching + //if they are equal + if(langId1 == langId2){ + return true; + }//or if they both are from the same langGroup and first or second are preffered(or both)// are not sublocale langs + else if (( psLangIdToGroupTable[langId1] == psLangIdToGroupTable[langId2] )){ + if(!LanguageFactory::getInstance()->hasSubLocale(langStr1)){ + return true; + } + char langStr2[MAX_LANG_LENGTH]{}; + NTMGetNameFromID( langId2, (USHORT)LANG_KEY, langStr2, NULL ); + return !LanguageFactory::getInstance()->hasSubLocale(langStr2); + } + return false; + // +} + + @@ -1757,19 +1797,20 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block T5LOG( T5INFO) << "FuzzyTest for record " <szSourceLanguage, &sLangID ); + USHORT srcLangId = 0; + // we have to update the memory language table to keep the language group table up-to-date... + if ( pTmClb->NTMGetIDFromName( pGetIn->szSourceLanguage, NULL, (USHORT)LANG_KEY, &srcLangId)) + { + srcLangId = 0; // set default.. + } /* endif */ //position at beginning of source structure in tm record pTMXSourceRecord = (PTMX_SOURCE_RECORD)(pTmRecord+1); - if(pGetIn->fSourceLangIsPrefered == false){ - char recordSrcLang[MAX_LANG_LENGTH]; - pTmClb->NTMGetNameFromID( &pTMXSourceRecord->usLangId, - (USHORT)LANG_KEY, - recordSrcLang, NULL ); - if(strcasecmp(recordSrcLang, pGetIn->szSourceLanguage)){ - T5LOG( T5WARNING) <<":: source langs is different in record(" << recordSrcLang << ") and request (" << pGetIn->szSourceLanguage << ")"; + + if(!pTmClb->areLangsMatching(srcLangId, pTMXSourceRecord->usLangId, pGetIn->szSourceLanguage)){ + //T5LOG( T5WARNING) <<":: source langs is different in record(" << pTMXSourceRecord->usLangId << ") and request (" << pGetIn->szSourceLanguage << ")"; return SOURCE_LANG_DIFFERENT; - } } - + //move pointer to corresponding position pSource = (PBYTE)(pTmRecord+1); pSource += pTMXSourceRecord->usSource; @@ -1852,24 +1893,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block bool fTestCLB = (false == pstMatchTable->isLocationAlreadyAdded(*pusMatchesFound, lKeyNum, usTgtNum)); if(fTestCLB){ //compare target language group IDs - if(pGetIn->fTargetLangIsPrefered){ - if (( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usTargetId] )) - { - T5LOG( T5INFO) <<"FuzzyTest: Wrong target language!" ; - fTestCLB = FALSE; - } - } else { - char recordTrgLang[MAX_LANG_LENGTH]; - pTmClb->NTMGetNameFromID( &pTMXTargetClb->usLangId, - (USHORT)LANG_KEY, - recordTrgLang, NULL ); - if(strcasecmp(recordTrgLang, pGetIn->szTargetLanguage)){ - T5LOG( T5WARNING) <<":: target langs is different in record(" << recordTrgLang << ") and request (" << pGetIn->szTargetLanguage<< ")"; - //return SOURCE_LANG_DIFFERENT; - fTestCLB = FALSE; - } - } - + fTestCLB = pTmClb->areLangsMatching(usTargetId, pTMXTargetClb->usLangId ,pGetIn->szTargetLanguage); // preprocessing for global memory processing usModifiedTranslationFlag = pTMXTargetClb->bTranslationFlag; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 416a85ca..150124b0 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -214,15 +214,12 @@ USHORT EqfMemory::TmtXReplace { // #define TEST_VOTES #ifdef TEST_VOTES - if(TmProposal.pInputSentence->pStrings->getGenericTagsString().find(L"2 x Bero 90")!= std::wstring::npos){ - int a = 0;//for breakpoint - } + #endif// TEST_VOTES HashSentence( TmProposal.pInputSentence.get() ); #ifdef TEST_VOTES //if(TmProposal.pInputSentence->pStrings->getGenericTagsString().find(L"2 x Bero 90")!= std::wstring::npos) { - int a = 0; std::string output = std::to_string(TmProposal.pInputSentence->usActVote) +"; votes = "; for(auto vote: TmProposal.pInputSentence->pulVotes){ output += std::to_string(vote) + ","; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index cf905bb1..fe77dc5c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -510,10 +510,11 @@ USHORT EqfMemory::NTMGetIDFromNameEx( //| | //| return usRc; | // ----------------------------------------------------------------------------+ -USHORT EqfMemory::NTMGetNameFromID(PUSHORT pusID, // intput +USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput USHORT usTableType, // input PSZ pszName, // output - PSZ pszLongName) // output, long name (only for FILE_KEY) + PSZ pszLongName, + SHORT maxBuffLen) // output, long name (only for FILE_KEY) { USHORT usRc = NO_ERROR; BOOL fFound = FALSE; @@ -540,17 +541,16 @@ USHORT EqfMemory::NTMGetNameFromID(PUSHORT pusID, // intput /****************************************************************/ if (usRc == NO_ERROR) { - if (*pusID == OVERFLOW_ID) + if ( usID == OVERFLOW_ID) { strcpy(pszName, OVERFLOW_NAME); fFound = TRUE; } else { - ULONG ulI; - for (ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) + for (ULONG ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) { - if (*pusID == pstTMTableEntries[ulI].usId) + if ( usID == pstTMTableEntries[ulI].usId) { fFound = TRUE; constexpr int sizeOfField = MAX_LANG_LENGTH ; @@ -564,15 +564,21 @@ USHORT EqfMemory::NTMGetNameFromID(PUSHORT pusID, // intput if (fFound && (usTableType == FILE_KEY) && (pszLongName != NULL)) { BOOL fLongFound = FALSE; - for (ulI = 0; + for (ULONG ulI = 0; ulI < LongNames.stTableEntry.size() && !fLongFound; ulI++) { - if (*pusID == LongNames.stTableEntry[ulI].usId) + if (usID == LongNames.stTableEntry[ulI].usId) { fLongFound = TRUE; - strcpy(pszLongName, - LongNames.stTableEntry[ulI].pszLongName); + + if(maxBuffLen>0){ + strncpy(pszName, pstTMTableEntries[ulI].szName, maxBuffLen-1); + pszName[maxBuffLen-1] = '\0'; + }else{ + strcpy(pszLongName, + LongNames.stTableEntry[ulI].pszLongName); + } } /* endif */ } /* endfor */ @@ -588,9 +594,9 @@ USHORT EqfMemory::NTMGetNameFromID(PUSHORT pusID, // intput /* the ID was not found in the table */ /* set usRc and reset pszName */ /************************************************************/ - LOG_AND_SET_RC(usRc, T5INFO, ID_NOT_FOUND); - if(*pusID){ - snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", *pusID); // 'hu' = unsigned short + LOG_AND_SET_RC_W_INFO(usRc, T5ERROR, ID_NOT_FOUND) << "id = " << usID; + if(usID){ + snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", usID); // 'hu' = unsigned short }else{ pszName[0] = EOS; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 31dd910d..f0df89b2 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -514,7 +514,7 @@ int EqfMemory::MatchToOtmProposal pProposal->setAuthor( pMatch->szTargetAuthor ); pProposal->setMarkup( pMatch->szTagTable ); pProposal->setTargetLanguage( pMatch->szTargetLanguage ); - pProposal->setSourceLanguage( stTmSign.szSourceLanguage); + pProposal->setSourceLanguage( pMatch->szOriginalSrcLanguage); pProposal->setAddInfo( pMatch->szAddInfo ); pProposal->setContext( pMatch->szContext ); pProposal->setDocName( pMatch->szLongName ); diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index b44fcbc8..f39e0150 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -340,9 +340,10 @@ int RequestData::requestTM(){ if(mem->isWaitingToBeLoaded() || mem->isFailedToLoad()){ _rc_ = mem->Load(); if(_rc_){ - std::string msg = "Failed to load tm"; + std::string msg = "Failed to load tm, rc = "; + msg += _rc_; if(11==_rc_){ - msg += "; rc=11, file on the disk is locked;"; + msg += "; file on the disk is locked;"; } //T5LOG(T5ERROR) << "Failed to open tm, rc = " << _rc_; return buildErrorReturn(504, msg.c_str(), _rc_); @@ -4165,8 +4166,7 @@ int ConcordanceExtendedSearchRequestData::execute() int ConcordanceSearchRequestData::parseJSON(){ - - if ( strMemName.empty() ) + if ( strMemName.empty()) { return buildErrorReturn( _rc_, "::concordanceSearch::Missing name of memory", BAD_REQUEST); } /* endif */ diff --git a/source/opentm2/core/utilities/LanguageFactory.H b/source/opentm2/core/utilities/LanguageFactory.H index e9b4bd95..281a8d72 100755 --- a/source/opentm2/core/utilities/LanguageFactory.H +++ b/source/opentm2/core/utilities/LanguageFactory.H @@ -92,6 +92,8 @@ int getISOName bool isPreferedLang(const char* lang); +bool hasSubLocale(const char* lang); + bool isMatchingLangs( const char* strMemLang,//can be prefered const char* strSegLang diff --git a/source/opentm2/core/utilities/LanguageFactory.cpp b/source/opentm2/core/utilities/LanguageFactory.cpp index 6b4e9a7e..6e6ad45c 100755 --- a/source/opentm2/core/utilities/LanguageFactory.cpp +++ b/source/opentm2/core/utilities/LanguageFactory.cpp @@ -224,6 +224,14 @@ bool LanguageFactory::isPreferedLang(const char* lang){ return (LanguageFactory::getInstance()->findIfPreferedLanguage( lang ) != -1); } +bool hasDash(const char* langCode) { + return std::strchr(langCode, '-') != nullptr; +} + +bool LanguageFactory::hasSubLocale(const char* lang){ + return hasDash(lang); +} + bool LanguageFactory::isMatchingLangs( const char* strMemLang,//can be prefered const char* strSegLang diff --git a/source/opentm2/core/utilities/UtlMisc.cpp b/source/opentm2/core/utilities/UtlMisc.cpp index 8bca0f6f..6bf839ed 100755 --- a/source/opentm2/core/utilities/UtlMisc.cpp +++ b/source/opentm2/core/utilities/UtlMisc.cpp @@ -408,65 +408,6 @@ USHORT UtlCompFDates // function searching first character causing a conversion error and showing an // approbriate error message const int MB_ERR_INVALID_CHARS = 1234;//just to compile -USHORT UtlFindAndShowConversionError -( - PSZ pszBuffer, - ULONG ulLen, - ULONG ulCP -) -{ - CHAR_W chBufOut[10]; - USHORT usRC = NO_ERROR; - - ULONG ulOutPut; - - // always use 932 when 943 is specified - if ( ulCP == 943 ) ulCP = 932; - - while ( !usRC && ulLen ) - { - ulOutPut = MultiByteToWideChar( ulCP, MB_ERR_INVALID_CHARS, pszBuffer, 1, chBufOut, 1 ); - - // try conversion together with next byte, might be a DBCS character - if ( !ulOutPut && ulLen ) - { - ulOutPut = MultiByteToWideChar( ulCP, MB_ERR_INVALID_CHARS, pszBuffer, 2, chBufOut, 2 ); - if ( ulOutPut ) - { - pszBuffer++; - ulLen--; - } /* endif */ - } /* endif */ - - if ( !ulOutPut ) - { - PSZ pszParms[3]; - CHAR szRC[10], szCP[10], szCharacter[20]; - int iCh = (UCHAR)*pszBuffer; -T5LOG(T5ERROR) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 74 usRC = (USHORT)GetLastError();"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - usRC = (USHORT)GetLastError(); -#endif //TO_BE_REPLACED_WITH_LINUX_CODE - - sprintf( szCharacter, "%d (hex %2.2X)", iCh, iCh ); - sprintf( szCP, "%lu", ulCP ); - sprintf( szRC, "%u", usRC ); - - pszParms[0] = szCharacter; - pszParms[1] = szCP; - pszParms[2] = szRC; - - UtlError( ERROR_DATA_CONVERSION_EXINFO, MB_CANCEL, 3, pszParms, EQF_ERROR ); - } - else - { - pszBuffer++; - ulLen--; - } /* endif */ - } /* endwhile */ - return( usRC ); -} /* end of function UtlFindAndShowConversionError */ - /*! UtlInitUtils Initialization of utilities Description: General utilities initialization routine From c20171c571abeb08ac200e838540e7bd772d1b9c Mon Sep 17 00:00:00 2001 From: orestdrag Date: Thu, 31 Jul 2025 22:33:13 +0300 Subject: [PATCH 43/62] * updated to v0.7.16 + added init message about optimizations * fixed -308 * fixed langs in fuzzy output - from DE-DE to de-DE * fixed few function that suppose to return some value, havent returned any - big code cleanup * optimized statusRequest and import-reorganize requests regarding importDetails sync --- include/otm.h | 30 +- include/requestdata.h | 1 + include/tm.h | 1 + source/CMakeLists.txt | 24 +- source/RestAPI/OtmMemoryServiceWorker.cpp | 6 +- .../opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp | 89 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 127 ++- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 63 +- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 8 - .../core/EqfMemoryPlugin/EqfMemory.cpp | 23 +- .../EqfMemoryPlugin/GenericTagReplace.cpp | 98 -- .../opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp | 18 +- .../core/EqfMemoryPlugin/TMPluginWrapper.cpp | 24 - source/opentm2/core/document/EQFMAIN.cpp | 9 +- source/opentm2/core/requestdata.cpp | 80 +- source/opentm2/core/utilities/EQFUTLNG.cpp | 8 +- source/opentm2/core/utilities/Stopwatch.hpp | 1 + source/opentm2/core/utilities/UtlMisc.cpp | 29 - .../OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c | 907 ------------------ .../OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.h | 1 - .../OtmMarkupTablePlugin/OTMMSOFC/PARSE.CPP | 414 -------- .../OtmMarkupTablePlugin/OTMXML/PARSE.CPP | 217 ----- .../OtmMarkupTablePlugin/OTMXML/parse.h | 2 - .../OtmMarkupTablePlugin/OTMXMODC/parse.cpp | 703 -------------- .../OtmMarkupTablePlugin/OTMXMODC/parse.h | 2 - source/otmd.cpp | 6 +- 26 files changed, 287 insertions(+), 2604 deletions(-) diff --git a/include/otm.h b/include/otm.h index 0cc83203..9ef240b3 100644 --- a/include/otm.h +++ b/include/otm.h @@ -65,20 +65,29 @@ class JSONFactory; #include "lowlevelotmdatastructs.h" struct ImportStatusDetails{ - std::atomic_short usProgress{0}; - std::atomic_int segmentsCount{0}; - std::atomic_int segmentsImported{0}; - std::atomic_int invalidSegments{0}; - std::atomic_int invalidSymbolErrors{0}; - std::atomic_int resSegments {0}; + std::mutex importDetailsMutex; + short usProgress{0}; + int segmentsCount{0}; + int segmentsImported{0}; + int invalidSegments{0}; + int invalidSymbolErrors{0}; + int resSegments {0}; int importRc{0}; long filteredSegments{0}; + + //std::mutex invalidSegmentsRCsMutex; std::map invalidSegmentsRCs; + + //std::mutex importTimestampMutex; std::string importTimestamp; + + //std::mutex firstInvalidSegmentsSegNumsMutex; std::vector> firstInvalidSegmentsSegNums; long lImportStartTime; long lImportTimeoutSec = 0; - std::atomic_long lImportRunTimeSec; + long lImportRunTimeSec; + + //std::mutex importMsgMutex; std::stringstream importMsg; bool fReorganize{0}; // true for reorganize call, false for import InclosingTagsBehaviour inclosingTagsBehaviour{InclosingTagsBehaviour::saveAll}; @@ -87,9 +96,11 @@ struct ImportStatusDetails{ BOOL fIgnoreSegIdOnUpdate{}; void reset(){ + std::scoped_lock l(importDetailsMutex); fNewEntryIfOnlyContextAndTimestampDifferent = FLAGS_newEntryIfOnlyContextAndTimestampDifferent; fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; //fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; + //std::lock_guard l(firstInvalidSegmentsSegNumsMutex); firstInvalidSegmentsSegNums.clear(); firstInvalidSegmentsSegNums.reserve(100); importMsg.str(""); @@ -104,11 +115,14 @@ struct ImportStatusDetails{ lImportStartTime = 0; lImportTimeoutSec = 0; lImportRunTimeSec = 0; - importTimestamp = "not finished"; + //std::lock_guard lk2(importTimestampMutex); + importTimestamp = "not finished"; + //std::lock_guard lk(invalidSegmentsRCsMutex); invalidSegmentsRCs.clear(); } std::string toString(){ + std::scoped_lock l(importDetailsMutex); std::string res = "progress = " + std::to_string(usProgress); if(fReorganize){ res += "; segmentsReorganized = "; diff --git a/include/requestdata.h b/include/requestdata.h index 9c3762d1..e6b0cec9 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _REQUEST_DATA_H_INCLUDED_ #define _REQUEST_DATA_H_INCLUDED_ diff --git a/include/tm.h b/include/tm.h index 70372228..80da335e 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1723,6 +1723,7 @@ struct TMX_MATCH_TABLE_W [&](const TMX_MATCH_TABLE_W& entry) { return entry.sameTMLocation(recordKey, targetKey); }); + return matchExists; } }; diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index c924b1c5..75c1ec2e 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,17 +1,26 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 15) +SET(APP_VERSION_MINOR 16) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) + +#set(CMAKE_C_COMPILER "clang") +#set(CMAKE_CXX_COMPILER "clang++") + SET(APP_VERSION "${APP_VERSION_GLOBAL}.${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}") project(T5MEMORY VERSION ${APP_VERSION}) #add_compile_options(-Wall -Wextra -Wpedantic) add_compile_options( +# -O3 +# -W +# -Wunused-function +# -Wno-attributes +# -Wno-write-strings # -Wall # -Wundef # -Wuninitialized @@ -52,6 +61,19 @@ if(ENABLE_UBSAN) add_definitions(-DUBSAN_ENABLED) endif() + +if(CMAKE_CXX_FLAGS MATCHES "-O0") + add_definitions(-DOPT_LEVEL=0) +elseif(CMAKE_CXX_FLAGS MATCHES "-O1") + add_definitions(-DOPT_LEVEL=1) +elseif(CMAKE_CXX_FLAGS MATCHES "-O2") + add_definitions(-DOPT_LEVEL=2) +elseif(CMAKE_CXX_FLAGS MATCHES "-O3") + add_definitions(-DOPT_LEVEL=3) +else() + add_definitions(-DOPT_LEVEL=-1) # unknown or other +endif() + # Apply the chosen sanitizer flags if ENABLE_ASAN is ON if(ENABLE_ASAN) message("AddressSanitizer is enabled") diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index 9831f577..8369c170 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -293,7 +293,11 @@ void importMemoryProcess( void* pvData ) if(!pData->mem->importDetails){ pData->mem->importDetails = new ImportStatusDetails; } - pData->mem->importDetails->inclosingTagsBehaviour = pData->inclosingTagsBehaviour; + + { + std::scoped_lock l(pData->mem->importDetails->importDetailsMutex); + pData->mem->importDetails->inclosingTagsBehaviour = pData->inclosingTagsBehaviour; + } int iRC =(int)EqfImportMem( pData, TMX_OPT | COMPLETE_IN_ONE_CALL_OPT); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index db1e3b35..1f8f93eb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -151,7 +151,11 @@ VOID EQFMemOrganizeProcess if ( iRC == NO_ERROR ) { pCommArea->usComplete = (USHORT)iProgress; - pRIDA->pMem->importDetails->usProgress = (USHORT) iProgress; + { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + pRIDA->pMem->importDetails->usProgress = (USHORT) iProgress; + pRIDA->pProposal->fSaveDifferentTargetsForSameSource = pRIDA->pMem->importDetails->fSaveDifferentTargetsForSameSource; + } // do some consistency checking pRIDA->pProposal->getMarkup( pRIDA->szTagTable, sizeof(pRIDA->szTagTable) ); @@ -159,7 +163,6 @@ VOID EQFMemOrganizeProcess auto ll = T5Logger::GetInstance()->suppressLogging(); wchar_t seg_buff[OTMPROPOSAL_MAXSEGLEN+1]; pRIDA->pProposal->getSource(seg_buff, OTMPROPOSAL_MAXSEGLEN); - pRIDA->pProposal->fSaveDifferentTargetsForSameSource = pRIDA->pMem->importDetails->fSaveDifferentTargetsForSameSource; bool fValidXml = IsValidXml( seg_buff); @@ -180,9 +183,12 @@ VOID EQFMemOrganizeProcess if (!fValidXml || (sourceLen == 0) || (sourceLen >= OTMPROPOSAL_MAXSEGLEN) || (targetLen == 0) || (targetLen>=OTMPROPOSAL_MAXSEGLEN) || (pRIDA->szTargetLanguage[0] == EOS) || (pRIDA->szTagTable[0] == EOS) ) { + + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); // ignore invalid proposal pRIDA->pMem->importDetails->invalidSegments++; if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + //std::lock_guard l(pRIDA->pMem->importDetails->firstInvalidSegmentsSegNumsMutex); pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), -8)); } } @@ -200,34 +206,42 @@ VOID EQFMemOrganizeProcess break; } } - - if( fSkipSegment ){ - pRIDA->pMem->importDetails->filteredSegments++; - }else{ - pRIDA->pProposal->pInputSentence = std::make_unique(pRIDA->pProposal->szSource, pRIDA->pProposal->szTarget); - iRC = pRIDA->pMemTemp->putProposal( *(pRIDA->pProposal) ); - if ( iRC != 0 ) - { - T5LOG(T5ERROR) << "segment in reorganize was skipped! segment: "<< *pRIDA->pProposal; - pRIDA->pMem->importDetails->invalidSegments++; - if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ - pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), iRC)); + { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + if( fSkipSegment ){ + pRIDA->pMem->importDetails->filteredSegments++; + }else{ + pRIDA->pProposal->pInputSentence = std::make_unique(pRIDA->pProposal->szSource, pRIDA->pProposal->szTarget); + iRC = pRIDA->pMemTemp->putProposal( *(pRIDA->pProposal) ); + if ( iRC != 0 ) + { + T5LOG(T5ERROR) << "segment in reorganize was skipped! segment: "<< *pRIDA->pProposal; + pRIDA->pMem->importDetails->invalidSegments++; + if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + //std::lock_guard l(pRIDA->pMem->importDetails->firstInvalidSegmentsSegNumsMutex); + pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), iRC)); + } + { + //std::lock_guard lk(pRIDA->pMem->importDetails->invalidSegmentsRCsMutex); + pRIDA->pMem->importDetails->invalidSegmentsRCs[iRC] ++; + } } - pRIDA->pMem->importDetails->invalidSegmentsRCs[iRC] ++; + else + { + pRIDA->pMem->importDetails->segmentsImported++; + } /* endif */ } - else - { - pRIDA->pMem->importDetails->segmentsImported++; - } /* endif */ } } /* endif */ } else if ( iRC == EqfMemory::ERROR_ENTRYISCORRUPTED// || iRC == BTREE_BUFFER_SMALL ) { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); pCommArea->usComplete = (USHORT)iProgress; pRIDA->pMem->importDetails->invalidSegments++; - if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + if(pRIDA->pMem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + //std::lock_guard l(pRIDA->pMem->importDetails->firstInvalidSegmentsSegNumsMutex); pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), -9)); } T5LOG(T5ERROR) << "Skipping proposal iRC == EqfMemory::ERROR_ENTRYISCORRUPTED ; proposal :" << *pRIDA->pProposal; @@ -235,7 +249,10 @@ VOID EQFMemOrganizeProcess else if ( iRC == EqfMemory::INFO_ENDREACHED ) { pRIDA->pMemTemp->NTMOrganizeIndexFile(); - T5LOG(T5WARNING) << "reorganize is done, segCount = " << pRIDA->pMem->importDetails->segmentsImported << "; invSegCount = " << pRIDA->pMem->importDetails->invalidSegments; + { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + T5LOG(T5WARNING) << "reorganize is done, segCount = " << pRIDA->pMem->importDetails->segmentsImported << "; invSegCount = " << pRIDA->pMem->importDetails->invalidSegments; + } int use_count = pRIDA->pMemTemp.use_count(); if(use_count != 1){ T5LOG(T5WARNING) << ":: use_count for temporary tm is not 1, but " << use_count; @@ -280,7 +297,10 @@ VOID EQFMemOrganizeProcess } /* endif */ } /* endif */ - pRIDA->pMem->importDetails->usProgress = pCommArea->usComplete + 1; + { + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + pRIDA->pMem->importDetails->usProgress = pCommArea->usComplete + 1; + } // ----------------------------------------------------- // Issue message WM_EQF_MEMORGANIZE_END pRIDA->NextTask = MEM_END_ORGANIZE; @@ -330,19 +350,22 @@ VOID EQFMemOrganizeEnd LONG lCurTime = 0; time( &lCurTime ); - - if ( pRIDA->pMem->importDetails->lImportStartTime ) { - LONG lDiff = lCurTime - pRIDA->pMem->importDetails->lImportStartTime; - char buff[256]; - sprintf( buff, "Overall reorganize time is : %ld:%2.2ld:%2.2ld\n", lDiff / 3600, - (lDiff - (lDiff / 3600 * 3600)) / 60, - (lDiff - (lDiff / 3600 * 3600)) % 60 ); - - pRIDA->pMem->importDetails->importTimestamp = buff; - } + std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); + if ( pRIDA->pMem->importDetails->lImportStartTime ) + { + LONG lDiff = lCurTime - pRIDA->pMem->importDetails->lImportStartTime; + char buff[256]; + sprintf( buff, "Overall reorganize time is : %ld:%2.2ld:%2.2ld\n", lDiff / 3600, + (lDiff - (lDiff / 3600 * 3600)) / 60, + (lDiff - (lDiff / 3600 * 3600)) % 60 ); + + //std::lock_guard l(pRIDA->pMem->importDetails->importTimestampMutex); + pRIDA->pMem->importDetails->importTimestamp = buff; + } - pRIDA->pMem->importDetails->usProgress = 100; + pRIDA->pMem->importDetails->usProgress = 100; + } } /* end of function EQFMemOrganizeEnd */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index ad8c9b21..fa215aae 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -86,7 +86,7 @@ //--- declaration of internal functions static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pIda, HWND, ImportStatusDetails* pImportData); -static USHORT MemLoadProcess( MEM_LOAD_DLG_IDA* pLIDA ); + USHORT /*APIENTRY*/ MEMINSERTSEGMENT @@ -384,6 +384,8 @@ static USHORT MemLoadStart( MEM_LOAD_DLG_IDA* pLIDA, if ( pLIDA->pProposal != NULL ) delete(pLIDA->pProposal); } /* endif */ }else{ + + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); time( &pLIDA->mem->importDetails->lImportStartTime ); T5LOG(T5DEBUG) << "************ Memory Import Log ********************* Memory import started at : "<< asctime( localtime( &pLIDA->mem->importDetails->lImportStartTime ) )<< "\tMemory name : " << pLIDA->szMemName ; @@ -746,23 +748,27 @@ USHORT MemFuncImportProcess { case MEM_START_IMPORT: T5LOG( T5INFO) << "::MEM_START_IMPORT"; - pData->mem->importDetails->usProgress = 0; - - if(!pData->fileData.empty()){ - pLIDA->fileData = std::move(pData->fileData); + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + pData->mem->importDetails->usProgress = 0; } - if ( !MemLoadStart( &(pData->MemLoadIda), HWND_FUNCIF, pData->mem->importDetails ) ) - { - if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); - pData->usMemLoadRC = UtlQueryUShort( QS_LASTERRORMSGID ); - if ( pData->usMemLoadRC == 0 ) pData->usMemLoadRC = ERROR_MEMIMP_ERROR; - usPhase = 0; + if(!pData->fileData.empty()){ + pLIDA->fileData = std::move(pData->fileData); } - else { - usPhase = MEM_IMPORT_TASK; - } /* endif */ + if ( !MemLoadStart( &(pData->MemLoadIda), HWND_FUNCIF, pData->mem->importDetails ) ) + { + if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); + pData->usMemLoadRC = UtlQueryUShort( QS_LASTERRORMSGID ); + if ( pData->usMemLoadRC == 0 ) pData->usMemLoadRC = ERROR_MEMIMP_ERROR; + usPhase = 0; + } + else + { + usPhase = MEM_IMPORT_TASK; + } /* endif */ + } break; case MEM_IMPORT_TASK: @@ -776,9 +782,12 @@ USHORT MemFuncImportProcess usPhase = MEM_IMPORT_TASK; break; case MEM_PROCESS_END: + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); if(!pData->mem->importDetails->importRc){ pData->mem->importDetails->usProgress = 100; } + } if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); if ( pLIDA->mem ) { @@ -821,10 +830,11 @@ USHORT MemFuncImportProcess { LONG lCurTime = 0; - time( &lCurTime ); - + time( &lCurTime ); + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); if ( pData->mem->importDetails->lImportStartTime ) { + //std::scoped_lock l(pData->mem->importDetails->importTimestampMutex); LONG lDiff = lCurTime - pData->mem->importDetails->lImportStartTime; char buff[256]; sprintf( buff, "Overall import time is : %ld:%2.2ld:%2.2ld\n", lDiff / 3600, @@ -835,6 +845,7 @@ USHORT MemFuncImportProcess } if(VLOG_IS_ON(1)){ + //std::scoped_lock l(pData->mem->importDetails->importTimestampMutex); T5LOG( T5TRANSACTION) << "::Memory import for \""<< pData->mem->szName << "\" is finished at : " << asctime( localtime( &lCurTime )) << "\tNumber of segments imported : " << pData->mem->importDetails->segmentsImported << "\tNumber of invalid segments : " << pData->mem->importDetails->invalidSegments << @@ -846,9 +857,12 @@ USHORT MemFuncImportProcess if ( pLIDA->pProposal != NULL ) delete(pLIDA->pProposal); } /* endif */ usPhase = 0;; - if(!pData->mem->importDetails->importRc){ + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + if(!pData->mem->importDetails->importRc){ pData->mem->importDetails->usProgress = 100; } + } break; } /* endswitch */ @@ -882,8 +896,10 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT MEM_LOAD_DLG_IDA* pLIDA = (MEM_LOAD_DLG_IDA*)lMemHandle; - pLIDA->mem->importDetails->segmentsCount++; // Increase segment sequence number in any case - + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + pLIDA->mem->importDetails->segmentsCount++; // Increase segment sequence number in any case + } // prepare segment data if ( pSegment->fValid ) { @@ -908,9 +924,12 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT pLIDA->pProposal->setContext( pSegment->szContext ); pLIDA->pProposal->setSegmentId( pSegment->lSegNum ); pLIDA->pProposal->pInputSentence.reset(); - pLIDA->pProposal->fNewEntryIfOnlyContextAndTimestampDifferent = pLIDA->mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent; - pLIDA->pProposal->fSaveDifferentTargetsForSameSource = pLIDA->mem->importDetails->fSaveDifferentTargetsForSameSource; - pLIDA->pProposal->fIgnoreSegIdOnUpdate = pLIDA->mem->importDetails->fIgnoreSegIdOnUpdate; + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + pLIDA->pProposal->fNewEntryIfOnlyContextAndTimestampDifferent = pLIDA->mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent; + pLIDA->pProposal->fSaveDifferentTargetsForSameSource = pLIDA->mem->importDetails->fSaveDifferentTargetsForSameSource; + pLIDA->pProposal->fIgnoreSegIdOnUpdate = pLIDA->mem->importDetails->fIgnoreSegIdOnUpdate; + } pLIDA->pProposal->pInputSentence = std::make_unique(pSegment->szSource, pSegment->szTarget); if(pLIDA->pProposal->pInputSentence->wasParsedSuccessfully()){ @@ -922,37 +941,48 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT usRC = -2; } - if ( !usRC ) - { - pLIDA->mem->importDetails->segmentsImported++; // Increase the segment counter - } - else { - pLIDA->mem->importDetails->invalidSegmentsRCs[usRC] ++; - pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter - if(pLIDA->mem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ - pLIDA->mem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pSegment->lSegNo, usRC) ); - } - if(usRC == BTREE_LOOKUPTABLE_CORRUPTED || usRC == BTREE_LOOKUPTABLE_TOO_SMALL || usRC == TMD_SIZE_IS_BIGGER_THAN_ALLOWED) - { - pLIDA->mem->importDetails->importRc = usRC; - std::string msg = "TM is reached it's size limit, please create another one and import segments there, rc = " - + toStr(usRC) + "; aciveSegment = " + toStr(pLIDA->mem->importDetails->segmentsCount)/*toStr(pSegment->lSegNo)*/ + "\n\ lSegNo = " + toStr(pSegment->lSegNo) + " not imported\r\n" - +"\nReason = " + pSegment->szReason - +"\nDocument = " + pSegment->szDocument - +"\nSourceLanguage = " + pSegment->szSourceLang - +"\nTargetLanguage = " + pSegment->szTargetLang - +"\nMarkup = " + pSegment->szFormat - +"\nSource = " + EncodingHelper::convertToUTF8(pSegment->szSource) - +"\nTarget = " + EncodingHelper::convertToUTF8(pSegment->szTarget) ; - throw xercesc::SAXException(msg.c_str()); + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + if ( !usRC ) + { + pLIDA->mem->importDetails->segmentsImported++; // Increase the segment counter } - } /* endif */ + else + { + { + //std::lock_guard lk(pLIDA->mem->importDetails->invalidSegmentsRCsMutex); + pLIDA->mem->importDetails->invalidSegmentsRCs[usRC] ++; + } + pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter + if(pLIDA->mem->importDetails->invalidSegments < FLAGS_maxBadSegmentsIdsSaved){ + //std::lock_guard l(pLIDA->mem->importDetails->firstInvalidSegmentsSegNumsMutex); + pLIDA->mem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pSegment->lSegNo, usRC) ); + } + if(usRC == BTREE_LOOKUPTABLE_CORRUPTED || usRC == BTREE_LOOKUPTABLE_TOO_SMALL || usRC == TMD_SIZE_IS_BIGGER_THAN_ALLOWED) + { + pLIDA->mem->importDetails->importRc = usRC; + std::string msg = "TM is reached it's size limit, please create another one and import segments there, rc = " + + toStr(usRC) + "; aciveSegment = " + toStr(pLIDA->mem->importDetails->segmentsCount)/*toStr(pSegment->lSegNo)*/ + "\n\ lSegNo = " + toStr(pSegment->lSegNo) + " not imported\r\n" + +"\nReason = " + pSegment->szReason + +"\nDocument = " + pSegment->szDocument + +"\nSourceLanguage = " + pSegment->szSourceLang + +"\nTargetLanguage = " + pSegment->szTargetLang + +"\nMarkup = " + pSegment->szFormat + +"\nSource = " + EncodingHelper::convertToUTF8(pSegment->szSource) + +"\nTarget = " + EncodingHelper::convertToUTF8(pSegment->szTarget) ; + throw xercesc::SAXException(msg.c_str()); + } + } /* endif */ + }//std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); } else { - pLIDA->mem->importDetails->invalidSegmentsRCs[-1]++; - pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter + { + std::scoped_lock l(pLIDA->mem->importDetails->importDetailsMutex); + //std::lock_guard lk(pLIDA->mem->importDetails->invalidSegmentsRCsMutex); + pLIDA->mem->importDetails->invalidSegmentsRCs[-1]++; + pLIDA->mem->importDetails->invalidSegments++; // increase invalid segment counter + } #ifdef IMPORT_LOGGING // write segment to log file @@ -967,7 +997,6 @@ USHORT /*APIENTRY*/ MEMINSERTSEGMENT } /* endif */ #endif } /* endif */ - return( usRC ); } /* end of function MEMINSERTSEGMENT */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index c09c8566..0aac69c9 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -35,11 +35,6 @@ static LARGE_INTEGER liLast = { 0 }; static LARGE_INTEGER liFrequency = { 0 }; -static int QueryPerformanceFrequency(LARGE_INTEGER* li){ - //int res = clock_gettime(CLOCK_REALTIME, li->QuadPart); - return 0; -} - //#endif #define ACTIVATE_NTMGenericDelete @@ -58,21 +53,9 @@ USHORT FillMatchTable( EqfMemory*, std::wstring&, PLONG, PTMX_TARGET_RECORD, /**********************************************************************/ //For R012645 begin static std::map Name2FamilyMap; -static std::string& GetAndCacheFamilyName -( - char* pszMarkupName -); -//For R012645 end -static VOID -MakeHashValueW -( - PULONG pulRandom, // array of random numbers for hashing - USHORT usMaxRandom, // maximum random numbers - PSZ_W pData, // ptr to data to be hashed - PULONG pulHashVal // resulting hash value -); +//For R012645 end USHORT NTMCompareContext @@ -312,36 +295,6 @@ USHORT EqfMemory::TmtXGet } /* endif */ } /* endif */ - /********************************************************************/ - /* check if we have more than one equal tgt match in our proposals */ - /********************************************************************/ - if ( !usRc && (pTmGetOut->usNumMatchesFound > 1 ) && !(pTmGetIn->fForceFuzzySearch) && (pTmGetIn->ulParm & GET_EXACT )) - { - PTMX_MATCH_TABLE_W pstMatchTable = &(pTmGetOut->stMatchTable[0]); - PTMX_MATCH_TABLE_W pstActMatch; - USHORT i, j; // index values - INT wsDiff; - - for ( j = 0 ; j < pTmGetOut->usNumMatchesFound - 1 ;j++ ) - { - pstActMatch = &(pTmGetOut->stMatchTable[j]); - for ( i=j+1; i< pTmGetOut->usNumMatchesFound ; i++) - { - //if (UtlCompIgnWhiteSpaceW(pstActMatch->szTarget, (pstMatchTable+i)->szTarget, 0, &wsDiff) == 0L && wsDiff == 0) - if(wcscmp(pstActMatch->szTarget, (pstMatchTable+i)->szTarget) == 0) - { - /*************************************************************/ - /* delete this match... */ - /*************************************************************/ - ULONG ulLen = ( MAX_MATCHES - i - 1 ) * sizeof(TMX_MATCH_TABLE_W); - memmove( pstMatchTable+i, pstMatchTable+1+i, ulLen ); - pTmGetOut->usNumMatchesFound--; - i--; // reset i because we deleted the i-th one - } /* endif */ - } /* endfor */ - } /* endfor */ - } /* endif */ - /********************************************************************/ /* restrict number of matches found to the maximum number requested */ /* by user */ @@ -2590,17 +2543,3 @@ static char szLastTargetLang[MAX_LANG_LENGTH] = ""; static ULONG ulLastSrcOemCP = 0; static ULONG ulLastTgtOemCP = 0; -// If existed, return it, otherwise it will determine markup family name then cache it -std::string& GetAndCacheFamilyName(char* pszMarkupName) -{ - if( Name2FamilyMap.find(pszMarkupName) == Name2FamilyMap.end() ) - { - OtmMarkup *pMarkup = GetMarkupObject( pszMarkupName, NULL); - if( pMarkup != NULL ) - Name2FamilyMap[pszMarkupName] = pMarkup->getFamilyName(); - else - Name2FamilyMap[pszMarkupName] = ""; - } - return Name2FamilyMap[pszMarkupName]; -} - diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 150124b0..10340627 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -112,14 +112,6 @@ bool isAddDataIsTheSame(OtmProposal& TmProposal,TMX_TARGET_CLB& Clb) return true; } -static -BOOL CheckForAlloc -( - PTMX_SENTENCE pSentence, - PTMX_TERM_TOKEN * ppTermTokens, - USHORT usEntries -); - //------------------------------------------------------------------------------ diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index f0df89b2..ddecb2f3 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -495,6 +495,25 @@ int EqfMemory::ExtOutToOtmProposal return( iRC ); } + +#include // for std::tolower, std::toupper +#include // for std::strlen + +void convertLangCode(const char* src, char* trg) { + bool afterDash = false; + for (size_t i = 0; src[i] != '\0'; ++i) { + if (src[i] == '-') { + trg[i] = '-'; + afterDash = true; + } else if (afterDash) { + trg[i] = std::toupper(static_cast(src[i])); + } else { + trg[i] = std::tolower(static_cast(src[i])); + } + } + trg[std::strlen(src)] = '\0'; // null-terminate +} + /*! \brief Fill OtmProposal from TMX_MATCH_TABLE_W structure \param pMatch pointer to the TMX_MATCH_TABLE_W structure \param Proposal reference to the OtmProposal being filled @@ -513,8 +532,8 @@ int EqfMemory::MatchToOtmProposal pProposal->setTarget( pMatch->szTarget ); pProposal->setAuthor( pMatch->szTargetAuthor ); pProposal->setMarkup( pMatch->szTagTable ); - pProposal->setTargetLanguage( pMatch->szTargetLanguage ); - pProposal->setSourceLanguage( pMatch->szOriginalSrcLanguage); + convertLangCode(pMatch->szOriginalSrcLanguage, pProposal->szSourceLanguage); + convertLangCode(pMatch->szTargetLanguage, pProposal->szTargetLanguage); pProposal->setAddInfo( pMatch->szAddInfo ); pProposal->setContext( pMatch->szContext ); pProposal->setDocName( pMatch->szLongName ); diff --git a/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp b/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp index 568073c8..a890f4a8 100755 --- a/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/GenericTagReplace.cpp @@ -26,14 +26,6 @@ #include // event logging #include "../utilities/LogWrapper.h" -static VOID -MakeHashValueW -( - PULONG pulRandom, // array of random numbers for hashing - USHORT usMaxRandom, // maximum random numbers - PSZ_W pData, // ptr to data to be hashed - PULONG pulHashVal // resulting hash value -); // helper function for token logging #ifdef INLINE_TAG_REPL_LOGGING @@ -88,76 +80,6 @@ void NTMMarkCRLF( PSZ_W pszSource, PSZ_W pszTarget ); #define TAG_TOKEN -7 -static -BOOL isWhiteSpaceW( PSZ_W pData ) -{ - PSZ_W pTemp = pData; - CHAR_W c; - //while ( ((c = *pTemp)!= NULC) && iswspace( c ) ) - while ( ((c = *pTemp)!= NULC) && isspace( c ) ) - { - pTemp++; - } /* endwhile */ - return (*pTemp == EOS); -} - - -//+----------------------------------------------------------------------------+ -//|Internal function | -//+----------------------------------------------------------------------------+ -//|Function name: MakeHashValues | -//+----------------------------------------------------------------------------+ -//|Function call: MakeHashValues( pulRandom, usMaxNum, pszString, pulHash);| -//+----------------------------------------------------------------------------+ -//|Description: build a quasi hash value of the passed string | -//+----------------------------------------------------------------------------+ -//|Parameters: PULONG pulRandom, array of random numbers | -//| USHORT usMaxRandom, maximum random numbers | -//| PSZ pData, ptr to data to be hashed | -//| PULONG pulHashVal resulting hash value | -//+----------------------------------------------------------------------------+ -//|Returncode type: VOID | -//+----------------------------------------------------------------------------+ -//|Function flow: go through the passed string and build hashvalue | -//| of all characters up to the specified max value | -//| Characters not in the alphanumeric range are ignored.. | -//| (Check in case of DBCS might be necessary...) | -//+----------------------------------------------------------------------------+ -static VOID -MakeHashValueW -( - PULONG pulRandom, // array of random numbers for hashing - USHORT usMaxRandom, // maximum random numbers - PSZ_W pData, // ptr to data to be hashed - PULONG pulHashVal // resulting hash value -) -{ - USHORT usRandomIndex = 0; - ULONG ulHashVal = 0; - CHAR_W c; - UCHAR cSingle; - - while ( ((c = *pData++)!= NULC) && (usRandomIndex < usMaxRandom)) - { - cSingle = (UCHAR) c; - if ( !isspace( cSingle ) ) - { - ulHashVal += pulRandom[usRandomIndex++] * c; - } /* endif */ - } /* endwhile */ - /********************************************************************/ - /* normalize hashvalue, just in case it is 0 and we may look at it */ - /* as the end of our further processing.... */ - /********************************************************************/ - if ( ulHashVal == 0L ) - { - ulHashVal = 1L; - } /* endif */ - - *pulHashVal = ulHashVal; - - return; -} /* end of function MakeHashValue */ //+----------------------------------------------------------------------------+ //|Internal function | @@ -191,26 +113,6 @@ MakeHashValueW -//reset replace tag pair used flags -static void NTMResetUsedFlag( PTMX_REPLTAGPAIR pCurTagPair ) -{ - while ( pCurTagPair ) - { - if ( pCurTagPair->pPropTok ) - { - pCurTagPair->fUsed = FALSE; - pCurTagPair++; - } - else - { - pCurTagPair = NULL; - } /* endif */ - } /*endwhile */ -} /* end of function NTMResetUsedFlag */ - - - - typedef struct _NTMGETMATCHLEVELDATA { CHAR szSegmentTagTable[MAX_EQF_PATH]; // tag table name for segment markup diff --git a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp index e5f9ef2c..04354d51 100755 --- a/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/OTMTMXIE.cpp @@ -1321,6 +1321,7 @@ USHORT CTMXExportImport::ImportNext errorCount = m_parser->getErrorCount() - m_handler->getInvalidCharacterErrorCount(); if(pImportData != nullptr){ + std::scoped_lock l(pImportData->importDetailsMutex); pImportData->invalidSymbolErrors = m_handler->getInvalidCharacterErrorCount(); if(pImportData->importRc) errorCount++; @@ -1369,6 +1370,7 @@ USHORT CTMXExportImport::ImportNext //m_handler->GetErrorText( m_pMemInfo->szError, sizeof(m_pMemInfo->szError) ); m_pMemInfo->fError = TRUE; + std::scoped_lock l(pImportData->importDetailsMutex); pImportData->importMsg << buff <<";;; "; }else if(pImportData->importRc){ //std::string errormsg = "Error occured with rc = " + std::to_string(m_handler->iStopImportWRc); @@ -3276,10 +3278,12 @@ void TMXParseHandler::fatalError(const SAXParseException& exception) T5LOG(T5ERROR) << "Fatal Error: " << message <<" at column " << col <<" in line "<< line<<""; sprintf( this->pBuf->szErrorMessage, "Fatal Error: %s at column %ld in line %ld", message, col, line ); if(pImportDetails){ + std::scoped_lock l(pImportDetails->importDetailsMutex); pImportDetails->importMsg << "IMPORT: " << this->pBuf->szErrorMessage << ";"; } }else{ if(pImportDetails){ + std::scoped_lock l(pImportDetails->importDetailsMutex); pImportDetails->importMsg << "IMPORT: INVCHAR: " << message <<" at column " << col <<" in line "<< line<<";"; } @@ -3313,10 +3317,12 @@ void TMXParseHandler::fatalInternalError(const SAXException& exception) col = m_locator->getColumnNumber(); } this->fError = TRUE; - sprintf( this->pBuf->szErrorMessage, " Fatal internal Error at column %ld in line %ld, import stopped at progress = %i%, errorMsg: %s ", + { + std::scoped_lock l(pImportDetails->importDetailsMutex); + sprintf( this->pBuf->szErrorMessage, " Fatal internal Error at column %ld in line %ld, import stopped at progress = %i%, errorMsg: %s ", col, line, (int)pImportDetails->usProgress, msg.c_str() ); - msg = std::string(pBuf->szErrorMessage); - if(pImportDetails){ + msg = std::string(pBuf->szErrorMessage); + pImportDetails->importMsg << msg; } @@ -3331,7 +3337,8 @@ void TMXParseHandler::error(const SAXParseException& exception) long col = (long)exception.getColumnNumber(); this->fError = TRUE; sprintf( this->pBuf->szErrorMessage, "Error: %s at column %ld in line %ld", message, col, line ); - if(pImportDetails){ + if(pImportDetails){ + std::scoped_lock l(pImportDetails->importDetailsMutex); pImportDetails->importMsg << "IMPORT: " << this->pBuf->szErrorMessage << ";"; } T5LOG(T5ERROR) << ": "<< this->pBuf->szErrorMessage; @@ -3344,7 +3351,8 @@ void TMXParseHandler::warning(const SAXParseException& exception) long line = (long)exception.getLineNumber(); long col = (long)exception.getColumnNumber(); T5LOG(T5WARNING) << "Warning: "<< message << " at column "<< col << " in line " << line << ""; - if(pImportDetails){ + if(pImportDetails){ + std::scoped_lock l(pImportDetails->importDetailsMutex); pImportDetails->importMsg << "Warning: "<< message << " at column "<< col << " in line " << line << ";"; } XMLString::release( &message ); diff --git a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp index 93bc2bf9..4b81f21c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp @@ -127,30 +127,6 @@ BOOL NTMDocMatch return( fMatch ); } /* end of function NTMDocMatch */ - -static -BOOL CheckForAlloc -( - PTMX_SENTENCE pSentence, - PTMX_TERM_TOKEN * ppTermTokens, - USHORT usEntries -) -{ - BOOL fOK = TRUE; - PTMX_TERM_TOKEN pTermTokens; - - auto lFilled = pTermTokens - pSentence->pTermTokens.data(); - - if ( (usEntries + lFilled/*pSentence->pTermTokens.size()*/) > pSentence->pTermTokens.size()) - { - //remember offset of pTagEntry - pSentence->pTermTokens.resize( pSentence->pTermTokens.size() + usEntries + 1); - } /* endif */ - *ppTermTokens= &pSentence->pTermTokens[lFilled]; - return(fOK); -} - - //------------------------------------------------------------------------------ // External function //------------------------------------------------------------------------------ diff --git a/source/opentm2/core/document/EQFMAIN.cpp b/source/opentm2/core/document/EQFMAIN.cpp index ce26c536..5f747b24 100755 --- a/source/opentm2/core/document/EQFMAIN.cpp +++ b/source/opentm2/core/document/EQFMAIN.cpp @@ -105,11 +105,11 @@ BOOL TwbGetCheckProfileData( PSZ pEqfSystemMsgFile, PSZ pEqfSystemPropPath, { T5LOG( T5DEVELOP) << "TwbGetCheckProfileData( pEqfSystemMsgFile= "<< pEqfSystemMsgFile<< ", pEqfSystemPropPath= "<< pEqfSystemPropPath << ", pEqfResFile = " << pEqfResFile << ")" ; - CHAR szDrive[MAX_DRIVE]; // buffer for drive list - CHAR szLanDrive[MAX_DRIVE]; // buffer for lan drive list - CHAR szSysPath[MAX_EQF_PATH]; // buffer for system path + CHAR szDrive[MAX_DRIVE]{}; // buffer for drive list + CHAR szLanDrive[MAX_DRIVE]{}; // buffer for lan drive list + CHAR szSysPath[MAX_EQF_PATH]{}; // buffer for system path BOOL fOK = TRUE; // internal OK flag - CHAR szOTMPath[MAX_EQF_PATH]; + CHAR szOTMPath[MAX_EQF_PATH]{}; // Get system drive and system path Properties::GetInstance()->get_value(KEY_OTM_DIR, szOTMPath, MAX_EQF_PATH); @@ -118,7 +118,6 @@ BOOL TwbGetCheckProfileData( PSZ pEqfSystemMsgFile, PSZ pEqfSystemPropPath, fOK = UtlQuerySysLangFile( szEqfSysLanguage, szEqfResFile, EqfSystemHlpFile, EqfSystemMsgFile ); - /******************************************************************/ /* return strings to caller if requested... */ /******************************************************************/ diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index f39e0150..9bf26b06 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -1183,15 +1183,18 @@ int ImportRequestData::execute(){ } mem->importDetails->reset(); - mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; - mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; - mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; - LONG lCurTime = 0; - time( &lCurTime ); - mem->importDetails->lImportStartTime = lCurTime; - mem->importDetails->lImportTimeoutSec = timeout; + LONG lCurTime = 0; + time( &lCurTime ); + mem->importDetails->lImportStartTime = lCurTime; + mem->importDetails->lImportTimeoutSec = timeout; + } pData->mem = mem; //importMemoryProcess(pData);//to do in same thread @@ -1313,8 +1316,11 @@ int ImportLocalRequestData::execute(){ mem->importDetails->reset(); LONG lCurTime = 0; - time( &lCurTime ); - mem->importDetails->lImportStartTime = lCurTime; + time( &lCurTime ); + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->lImportStartTime = lCurTime; + } pData->mem = mem; pData->tmLockTimeout = tmLockTimeout; @@ -1559,6 +1565,9 @@ int ReorganizeRequestData::parseJSON(){ fSaveDifferentTargetsForSameSource = FLAGS_saveDifferentTargetsForSameSource; fIgnoreSegIdOnUpdate = FLAGS_ignoreSegIdOnUpdate; + if(strBody.empty()){ + return 0; + } // parse input parameters std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); @@ -1623,15 +1632,20 @@ int ReorganizeRequestData::execute(){ } mem->importDetails->reset(); - mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; - mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; - mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + } LONG lCurTime = 0; time( &lCurTime ); - - pRIDA->pMem->importDetails->lImportStartTime = lCurTime; - mem->importDetails->fReorganize = true; + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->lImportStartTime = lCurTime; + mem->importDetails->fReorganize = true; + } pRIDA->tmLockTimeout = tmLockTimeout; //mem.reset(); @@ -1771,15 +1785,18 @@ int DeleteEntriesReorganizeRequestData::execute(){ } mem->importDetails->reset(); - mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; - mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; - mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; LONG lCurTime = 0; time( &lCurTime ); - - pRIDA->pMem->importDetails->lImportStartTime = lCurTime; - mem->importDetails->fReorganize = true; + { + std::scoped_lock l(mem->importDetails->importDetailsMutex); + mem->importDetails->fNewEntryIfOnlyContextAndTimestampDifferent = fNewEntryIfOnlyContextAndTimestampDifferent; + mem->importDetails->fSaveDifferentTargetsForSameSource = fSaveDifferentTargetsForSameSource; + mem->importDetails->fIgnoreSegIdOnUpdate = fIgnoreSegIdOnUpdate; + + pRIDA->pMem->importDetails->lImportStartTime = lCurTime; + mem->importDetails->fReorganize = true; + } pRIDA->m_reorganizeFilters = searchFilterFactory.getListOfFilters(); pRIDA->tmLockTimeout = tmLockTimeout; @@ -2120,10 +2137,13 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr json_factory.addParmToJSON( outputMessage, "activeRequest", mem->getActiveRequestString()); if(mem->importDetails != nullptr){ - + std::scoped_lock l(mem->importDetails->importDetailsMutex); json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "reorganizeStatus":"tmxImportStatus", pszStatus ); json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "reorganizeProgress":"importProgress", mem->importDetails->usProgress ); - json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "reorganizeTime":"importTime", mem->importDetails->importTimestamp ); + { + //std::scoped_lock l(mem->importDetails->importTimestampMutex); + json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "reorganizeTime":"importTime", mem->importDetails->importTimestamp ); + } json_factory.addParmToJSON( outputMessage, mem->importDetails->fReorganize? "segmentsReorganized":"segmentsImported", mem->importDetails->segmentsImported ); if(mem->importDetails->fReorganize) json_factory.addParmToJSON( outputMessage, "segmentsFilteredOut", mem->importDetails->filteredSegments ); json_factory.addParmToJSON( outputMessage, "invalidSegments", mem->importDetails->invalidSegments ); @@ -2135,15 +2155,21 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr //json_factory.addParmToJSON( outputMessage, "ignoreSegIdOnUpdate", mem->importDetails->fIgnoreSegIdOnUpdate); std::string invalidSegmRCs; - for(auto [errCode, errCount]: mem->importDetails->invalidSegmentsRCs){ - invalidSegmRCs += std::to_string(errCode) + ":" + std::to_string(errCount) +"; "; + { + //std::lock_guard lk(mem->importDetails->invalidSegmentsRCsMutex); + for(auto [errCode, errCount]: mem->importDetails->invalidSegmentsRCs){ + invalidSegmRCs += std::to_string(errCode) + ":" + std::to_string(errCount) +"; "; + } } std::string firstInvalidSegments; int i=0; - for(auto [segNum, invSegErrCode] : mem->importDetails->firstInvalidSegmentsSegNums){ - firstInvalidSegments += std::to_string(++i) + ":" + std::to_string(segNum) + ":" + std::to_string(invSegErrCode) + "; "; + { + //std::lock_guard l(mem->importDetails->firstInvalidSegmentsSegNumsMutex); + for(auto [segNum, invSegErrCode] : mem->importDetails->firstInvalidSegmentsSegNums){ + firstInvalidSegments += std::to_string(++i) + ":" + std::to_string(segNum) + ":" + std::to_string(invSegErrCode) + "; "; + } } json_factory.addParmToJSON( outputMessage, "invalidSegmentsRCs", invalidSegmRCs); json_factory.addParmToJSON( outputMessage, "firstInvalidSegments(num:segId:errCode)", firstInvalidSegments); diff --git a/source/opentm2/core/utilities/EQFUTLNG.cpp b/source/opentm2/core/utilities/EQFUTLNG.cpp index 827a8fe3..36bdc94d 100755 --- a/source/opentm2/core/utilities/EQFUTLNG.cpp +++ b/source/opentm2/core/utilities/EQFUTLNG.cpp @@ -46,16 +46,16 @@ BOOL UtlQuerySysLangFile PSZ pszMsgFile // buffer for name of message file ) { - CHAR szFileName[MAX_EQF_PATH]; // buffer for file name + CHAR szFileName[MAX_EQF_PATH]{}; // buffer for file name BOOL fOK = TRUE; // function return code std::vector pszLangFileBuffer; // buffer for in-memory copy of language file PSZ pszCurLine; // ptr to current line in file BOOL fFound = FALSE; ULONG ulLength; // length of language file USHORT usI; // loop index - CHAR szDrive[MAX_DRIVE]; // buffer for drive list - CHAR szSysPath[MAX_EQF_PATH]; // global system path - CHAR EqfSystemPath[MAX_EQF_PATH];// global system path + CHAR szDrive[MAX_DRIVE]{}; // buffer for drive list + CHAR szSysPath[MAX_EQF_PATH]{}; // global system path + CHAR EqfSystemPath[MAX_EQF_PATH]{};// global system path /********************************************************************/ /* initialize result fields */ diff --git a/source/opentm2/core/utilities/Stopwatch.hpp b/source/opentm2/core/utilities/Stopwatch.hpp index 6b32f9be..a32e5b08 100644 --- a/source/opentm2/core/utilities/Stopwatch.hpp +++ b/source/opentm2/core/utilities/Stopwatch.hpp @@ -49,6 +49,7 @@ namespace stopwatch{ const std::uint64_t ns_count = std::chrono::duration_cast(duration).count(); std::uint64_t up = ((ns_count/100000)%10 >= 5) ? 1 : 0; const auto ms_count = (ns_count /1000000) + up; + return ms_count; } std::string getString(){ diff --git a/source/opentm2/core/utilities/UtlMisc.cpp b/source/opentm2/core/utilities/UtlMisc.cpp index 6bf839ed..b39c8eb7 100755 --- a/source/opentm2/core/utilities/UtlMisc.cpp +++ b/source/opentm2/core/utilities/UtlMisc.cpp @@ -510,32 +510,3 @@ BOOL UtlInitUtils( HAB hab ) #define TAISWHITESPACE( c ) \ ( (c == LF) || (c == CR) || (c == SPACE) || (c == 0x09) ) - -static BOOL TACheckForWS -( - CHAR_W c, - PSZ_W pWSList -) -{ - BOOL fIsWS = FALSE; - - fIsWS = TAISWHITESPACE( c ) ; - - if (pWSList && !fIsWS) - { - while (*pWSList && !fIsWS) - { - if (c == *pWSList) - { - fIsWS = TRUE; - } - else - { - pWSList++; - } - } /* endwhile */ - } /* endif */ - - - return(fIsWS); -} diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c index aa5e1b9e..8f6c4ef2 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c @@ -122,726 +122,6 @@ extern short sTPVersion ; /* From USRCALLS.C */ #define TITLE_MSOFC_PARAGRAPH_MISMATCH "Paragraph Mismatch" #define TITLE_MSOFC_TEXT_UNITS_MISMATCH "Text Units Mismatch" #define TITLE_MSOFC_FILE_CONVERSION "File Conversion Error" - - - -/******************************************************************************* -* -* function: EQFPRESEG/EQFPRESEG2 -* -* ----------------------------------------------------------------------------- -* Description: -* called before text segmentation is invoked. Prepares the source -* file for segmentation and stores the prepared file temporarily. -* Text Analysis will remove the temporary file. -* EQFPRESEG2 will pass a slider handle which could be used -* together with the EQFSETSLIDER call to set the slider in cases -* of user exits -* -* Arguments: PSZ ... pointer to markup table name (full path) -* -* PSZ ... pointer to editor name -* -* PSZ ... pointer to program path -* -* PSZ ... pointer to source file name (full path) -* -* PSZ ... pointer to the buffer for the name of the -* temp output file, the buffer is allocated by Text -* Analysis, with a size of 144 byte. -* -* BOOL * ... output flag which indicates whether text -* segmentation shall be performed, -* TRUE: do not perform Text Analysis, -* i.e. no segmentation of the text -* provided temp. file will be copied -* as segmented source and segmented target -* FALSE: do perform text analysis -* -* HWND ... handle of the slider window -* -* PBOOL ... ptr to 'kill' flag -* if this flag changes to TRUE the userexit -* should return ASAP -* -* Return: BOOL ... TRUE: processing was OK -* FALSE: an error occured during processing, -* e.g. a file could not be written, so -* that no further processing should be done -* -*******************************************************************************/ - - -/******************************************************************************* -* -* function: EQFPOSTSEGW -* -* ----------------------------------------------------------------------------- -* Description: -* called after text segmentation is invoked, to change the -* segmented source and target file before translation takes place. -* EQFPOSTSEG2 will pass a slider handle which could be used -* together with the EQFSETSLIDER call to set the slider in cases -* of user exits -* -* Arguments: PSZ ... pointer to markup table name -* -* PSZ ... pointer to editor name -* -* PSZ ... pointer to program path -* -* PSZ ... pointer to segmented source file name -* -* PSZ ... pointer to the segmented target file name -* -* PTATAG ... pointer to tags inserted by text segmentation, -* ( see layout of tag structure ) -* -* HWND ... handle of the slider window -* -* PBOOL ... ptr to 'kill' flag -* if this flag changes to TRUE the userexit -* should return ASAP -* -* Return: BOOL ... TRUE: processing was OK -* FALSE: an error occured during processing -* -* Note: It is vital that the name of the segmented source and target file -* is not changed! -*******************************************************************************/ - - -EQF_BOOL EQFPOSTSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pSegSource, - PSZ pSegTarget, - PTATAG_W pTATag, - HWND hSlider, - PEQF_BOOL pfKill ) -{ - BOOL bReturn = TRUE; - - - PrepDocLanguageInfo( pSegSource ) ; /* Set language unqiue processing, like DBCS */ - strcpy( szProgPath, pProgPath ) ; - - bReturn = PostParse( pSegSource, pSegTarget, hSlider ) ; - if ( bReturn ) { - CopyFileA( pSegTarget, pSegSource, FALSE ) ; - } - - return(bReturn); -} /* EQFPOSTSEG2 */ - - -/******************************************************************************* -* -* function: EQFPREUNSEG / EQFPREUNSEG2 -* -* ----------------------------------------------------------------------------- -* Description: -* called before unsegmentation is invoked, to change the -* segmented target file and store the prepared file temporarily -* before unsegmentation takes place. -* -* -* Arguments: PSZ ... pointer to markup table name (full path) -* -* PSZ ... pointer to editor name -* -* PSZ ... pointer to program path -* -* PSZ ... pointer to the segmented target file name (full path) -* -* PSZ ... pointer to the buffer for the name of the -* temp output file, the buffer is allocated by Text -* Analysis, with a size of 144 bytes. -* -* PTATAG ... pointer to tags inserted by text segmentation -* -* BOOL * ... output flag whether unsegmentation is -* required -* TRUE: unsegmentation not required -* FALSE: unsegmentation required -* If unsegmentation is not required, only a file -* copy from the temp file to the target path -* will be done. -* -* PBOOL ... ptr to 'kill' flag -* if this flag changes to TRUE the userexit -* should return ASAP -* -* Return: BOOL ... TRUE: processing was OK -* FALSE: an error occured during processing -* -*******************************************************************************/ - - -EQF_BOOL EQFPREUNSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pTarget, - PSZ pTemp, - PTATAG_W pTATag, - PEQF_BOOL pfNoUnseg, - PEQF_BOOL pfKill ) -{ - - return TRUE; -} /* EQFPREUNSEG2 */ - - -/******************************************************************************* -* -* function: EQFPOSTUNSEGW -* -* ----------------------------------------------------------------------------- -* Description: -* called after unsegmentation is invoked, to change the -* target file to its final form. -* -* -* Arguments: PSZ ... pointer to markup table name (full path) -* -* PSZ ... pointer to editor name -* -* PSZ ... pointer to program path (full path) -* -* PSZ ... pointer to target file name (full path) -* -* PTATAG ... pointer to tags inserted by text segmentation -* -* PBOOL ... ptr to 'kill' flag -* if this flag changes to TRUE the userexit -* should return ASAP -* -* -* Return: BOOL ... TRUE: processing was OK -* FALSE: an error occured during processing -* -*******************************************************************************/ - - -EQF_BOOL EQFPOSTUNSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pTarget, - PTATAG pTATag, - PEQF_BOOL pfKill ) -{ - struct _stat findbuf ; - BOOL bReturn = TRUE; - USHORT usFileFormat ; - char szMarkup[80] ; - char *szAltTempExt3 = ".$3$"; - char *szAltTempExt4 = ".$4$"; - char SourceFile[512]; - char TempFile[512], TempFile2[512], TempFile3[512] ; - char *ptrChar ; - int rc; - - - PrepDocLanguageInfo( pTarget ) ; /* Set language unqiue processing, like DBCS */ - strcpy( szProgPath, pProgPath ) ; - - CreateTempFileName2( TempFile2, pTarget, szAltTempExt3, TEMPNAME_TARGET ) ; - CreateTempFileName2( TempFile3, pTarget, szAltTempExt4, TEMPNAME_TARGET ) ; - strcpy( TempFile, TempFile2 ) ; - strcat( TempFile, ".DOC" ) ; - - CopyFileA(pTarget, TempFile, FALSE); - strcpy( SourceFile, pTarget ) ; - ptrChar = strstr( SourceFile, "\\TARGET\\" ) ; - if ( ptrChar ) { - memmove( ptrChar+6, ptrChar+8, strlen(ptrChar+8)+1 ) ; - strncpy( ptrChar, "\\MISC\\", 6 ) ; - if ( _stat( SourceFile, &findbuf ) ) { - MessageBoxA(HWND_DESKTOP, MSG_MSOFC_MISSING_TARGET_FILE, TITLE_MSOFC_MISSING_FILE, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - } - } else { - MessageBoxA(HWND_DESKTOP, MSG_MSOFC_MISSING_TARGET_FILE, TITLE_MSOFC_MISSING_FILE, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - } - - if ( bReturn ) { - bReturn = PostExport( TempFile, pTarget, SourceFile, &usFileFormat ) ; - rc=remove(TempFile); - } - - if ( bReturn ) { - rc = ConvertExport(pTarget, EQF_UTF162UTF8); - if ( !rc ) { - MessageBoxA(HWND_DESKTOP, MSG_MSOFC_FILE_CONVERSION, TITLE_MSOFC_FILE_CONVERSION, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - } - } - - - if ( bReturn ) { - if ( ! ConvertXmlToWord( pTarget, TempFile, szErrMsg, usFileFormat ) ) { - MessageBoxA(HWND_DESKTOP, szErrMsg, TITLE_MSOFC_XML_WORD_ERROR, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - return FALSE; - } - } - - remove( TempFile ) ; - remove( TempFile2 ) ; - remove( TempFile3 ); /* Original temp file */ - - return(bReturn); - -} /* EQFPOSTUNSEG2 */ - - - - - - -USHORT EQFQUERYEXITINFO( - PSZ pszTagTable, // name of the markup table, e.g. "OTMHTM32" - USHORT usMode, // type of information being queried - PSZ pszBuffer, // buffer area receiving the information returned by the exit - USHORT usBufLen // length of buffer area -) -{ - - if( usMode == QUERYEXIT_ADDFILES) { - QueryExportFiles(pszTagTable, pszBuffer, usBufLen, FALSE); - } - - return 0; -} - - -/******************************************************************************* -* -* function: EQFCHECKSEGEXW -* -* ----------------------------------------------------------------------------- -* Description: -* Allow to validate of the current segment. -* -* Arguments: PSZ ... pointer to previous source segment -* PSZ ... pointer to current source segment -* PSZ ... pointer to current target segment -* PBOOL ... ptr to whether segement was changed - LONG ... handle to use with EQFGETPREVSEG - ULONG ... current segment number -* BOOL ... whether message display is requested -* Return: - * TRUE - Text successfully checked and no validation problem. -* FALSE - Text failed the check, so text should not be saved. -*******************************************************************************/ - - - - - -/*****************************************************************************/ -/* ConvertXmlToWord */ -/* */ -/* Function called by EQFPOSTUNSEGW. */ -/* */ -/* Return: TRUE - File successfully processed. */ -/* FALSE - File could not be processed. */ -/*****************************************************************************/ - -BOOL ConvertXmlToWord( char *in, char *temp, char *ErrText, USHORT usFileFormat ) -{ - WCHAR swTemp[512]; - char szTemp[512]; - BOOL bReturn = TRUE; - BOOL bReturn2 = TRUE; - BOOL bWordActive = FALSE ; - - - HRESULT hr; - CLSID clsid; - IDispatch *pDisp; - IUnknown *pUnk = NULL; - IDispatch *pDispRoot = NULL; - IDispatch *pDispApp = NULL; - IDispatch *pDispDocs = NULL; - IDispatch *pDispDoc = NULL; - IDispatch *pDispProperties = NULL; - - HANDLE hProcess = NULL; - ULONG ulStartWordIDs[50] ; - ULONG ulEndWordIDs[50] ; - USHORT i, j ; - - DISPID dispID; - - - ErrText[0] = 0 ; - - /**********************************************************************/ - /* XML file needs no conversion. */ - /**********************************************************************/ - if ( usFileFormat == FILE_FORMAT_XML ) { - - } else - - /**********************************************************************/ - /* Convert XML to MS Word 2003 document. */ - /**********************************************************************/ - if ( usFileFormat == FILE_FORMAT_WORD ) { - - OleInitialize(NULL); - - - // Get CLSID for our server... - hr=CLSIDFromProgID(L"Word.Application",&clsid); - if ( FAILED(hr) ) { - bReturn = FALSE ; - strcat( ErrText, "\nGetting CLSID for MS Word failed." ) ; - } - - - GetWordProcessList( &ulStartWordIDs[0] ) ; - - if ( bReturn ) { - // Start the server... - hr=CoCreateInstance(clsid,NULL,CLSCTX_LOCAL_SERVER, IID_IUnknown, (void**) &pUnk); - if ( FAILED(hr) ) { - bReturn = FALSE; - strcat( ErrText, "\nStarting MS Word server failed." ) ; - } - } - - GetWordProcessList( &ulEndWordIDs[0] ) ; - - if ( bReturn ) { - // Query for IDispatch - hr = pUnk->QueryInterface(IID_IDispatch, (void**)&pDispRoot); - if ( FAILED(hr) ) { - pUnk->Release(); - strcat( ErrText, "\nQuery for IDispatch failed." ) ; - bReturn = FALSE; - } - } - - if ( bReturn ) { - // GET App IDispatch - VARIANT Result; - VariantInit(&Result); - bReturn = AutomationWrapper(DISPATCH_PROPERTYGET,&Result,pDispRoot,"Application",ErrText,0); - pDispApp = Result.pdispVal; - if ( ! bReturn ) { - strcat( ErrText, "\nGet Application IDispatch." ) ; - } else { - bWordActive = TRUE ; - } - } - - if ( bReturn ) { - // Get Documents collection... [WordApp~Documents] - VARIANT result; - VariantInit(&result); - bReturn = AutomationWrapper(DISPATCH_PROPERTYGET, &result, pDispApp, "Documents", ErrText, 0); - pDispDocs = result.pdispVal; - if ( ! bReturn ) { - strcat( ErrText, "\nGet document collection failed." ) ; - } - } - - - if ( bReturn ) { - // Call Documents::Add()... [WordApp~Add] - VARIANT result; - VariantInit(&result); - VARIANT parm ; - parm.vt = VT_BSTR; - mbstowcs( swTemp, in, 256 ) ; - parm.bstrVal = SysAllocString(swTemp) ; - bReturn = AutomationWrapper(DISPATCH_METHOD|DISPATCH_PROPERTYGET, &result, pDispDocs, "Add", ErrText, 1,parm); - pDispDoc = result.pdispVal; - if ( ! bReturn ) { - strcat( ErrText, "\nInternal translated XML file could not be opened in MS Word." ) ; - } - } - - if ( bReturn ) { - // Call Documents::Add()... [WordApp~Selection] - VARIANT result; - VariantInit(&result); - bReturn = AutomationWrapper(DISPATCH_METHOD|DISPATCH_PROPERTYGET, &result, pDispApp, "Selection", ErrText, 0); - if ( ! bReturn ) { - strcat( ErrText, "\nOpening MS Word \"File\" menu item for \"SaveAs\" operation failed." ) ; - } - } - - if ( bReturn ) { - // SaveAs Word document [WordApp~SaveAs] - int num = 0 ; /* SaveAs Word document format */ - VARIANT result; - VariantInit(&result); - VARIANT parm; - parm.vt = VT_BSTR; - mbstowcs( swTemp, temp, 256 ) ; - parm.bstrVal = SysAllocString(swTemp) ; - VARIANT parm1; - parm1.vt = VT_I4; - parm1.lVal = num; - ErrText[0] = NULL ; - bReturn = AutomationWrapper(DISPATCH_METHOD, &result, pDispDoc, "SaveAs", ErrText, 2, parm1, parm); - SysFreeString(parm.bstrVal); - if ( ! bReturn ) { - strcat( ErrText, "\nInternal translated XML file could not be saved as an MS Word document." ) ; - - parm1.lVal = 0 ; /* Close Word doc without saving to avoid pop-up: do you want to save changes? */ - bReturn2 = AutomationWrapper(DISPATCH_METHOD, &result, pDispDoc, "Close", szTemp, 1, parm1); - } - - } - - - if ( bWordActive ) { - // Call Documents::Add()... [WordApp~Selection] - VARIANT result; - VariantInit(&result); - bReturn2 = AutomationWrapper(DISPATCH_METHOD|DISPATCH_PROPERTYGET, &result, pDispApp, "Selection", ErrText, 0); - if ( ! bReturn2 ) { - bReturn = FALSE ; - strcat( ErrText, "\nOpening MS Word \"File\" menu item for \"Quit\" operation failed." ) ; - } - } - - - if ( bWordActive ) { - //Quit [WordApp~Quit] - VARIANT result; - VariantInit(&result); - bReturn2 = AutomationWrapper(DISPATCH_METHOD, 0 , pDispApp, "Quit", ErrText, 0); - if ( ! bReturn2 ) { - bReturn = FALSE ; - strcat( ErrText, "\nMS Word \"Quit\" operation failed." ) ; - } - - CoDisconnectObject( pUnk, NULL ) ; - } - - - - // Clean up... - if ( pDispDoc ) pDispDoc->Release(); - if ( pDispDocs ) pDispDocs->Release(); - if ( pDispApp ) pDispApp->Release(); - if ( pDispRoot ) pDispRoot->Release(); - if ( pUnk ) pUnk->Release(); - - // Uninitialize OLE Libraries... - OleUninitialize(); - - for( i=0 ; i<50 && ulEndWordIDs[i] ; ++i ) { - for( j=0 ; j<50 && ulStartWordIDs[j] ; ++j ) { - if ( ulEndWordIDs[i] == ulStartWordIDs[j] ) - break; - } - if ( ulStartWordIDs[j] == 0 ) { - hProcess = OpenProcess( PROCESS_ALL_ACCESS,FALSE,ulEndWordIDs[i] ) ; - if ( hProcess != NULL ) { - TerminateProcess(hProcess, 0xffffffff); - FilesystemHelper::CloseFile( hProcess); - } - } - } - } else - - /**********************************************************************/ - /* Convert MS Office 2007 document to XML. */ - /**********************************************************************/ - if ( usFileFormat == FILE_FORMAT_ZIP ) { - if ( ! ReplaceXmlInZip( in, temp, szErrMsg ) ) { - bReturn = FALSE ; - } - } - - - return(bReturn); -} - - - -/*****************************************************************************/ -/* GetWordProcessList */ -/* */ -/* Get a list of the process IDs which are running MS Word. */ -/* */ -/* Return: TRUE - File successfully processed. */ -/* FALSE - File could not be processed. */ -/*****************************************************************************/ - -BOOL GetWordProcessList( ULONG *ulList ) -{ - HANDLE hSnapshot = NULL; - PROCESSENTRY32 pe32 = {0}; - USHORT i = 0 ; - - // Take a snapshot of all processes currently in the system. - hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); - if (hSnapshot == (HANDLE)-1) - return (FALSE); - - // Fill in the size of the structure before using it. - pe32.dwSize = sizeof(PROCESSENTRY32); - - // Walk the snapshot of the processes. - if (Process32First(hSnapshot, &pe32)) { - do { - if ( ! wcscmp( pe32.szExeFile, L"WINWORD.EXE" ) ) { - ulList[i++] = pe32.th32ProcessID ; - } - } - while (Process32Next(hSnapshot, &pe32)); - } - FilesystemHelper::CloseFile(hSnapshot); - ulList[i] = 0 ; - - return( TRUE ) ; -} - - - - -//============================================================================== -// AutomationWrapper | -//=============================================================================| -// Wrapper function for making generic 16-bit Automation calls... | -// | -// from HOWTO: Do 16-bit Automation in C++ Using VC 1.52 | -// Article ID: Q194656 | -// | -// | -// Prereqs: | -// None. | -//=============================================================================| -// SideEffects: | -// None. | -//============================================================================== - -BOOL AutomationWrapper ( - int autoType, - VARIANT *pvResult, - IDispatch *pDisp, - char *ptName, - char *ErrText, - int cArgs... - ) -{ - va_list marker; - BOOL fOk = TRUE; - // Variables used... - DISPPARAMS dp = { NULL, NULL, 0, 0}; - DISPID dispidNamed = DISPID_PROPERTYPUT; - DISPID dispID; - HRESULT hr; - - CHAR szTemp[512]; - WCHAR swTemp[512]; - PSZ pszMsgTable[2]; - - // Allocate memory for arguments... - VARIANT *pArgs = new VARIANT[cArgs+1]; - - va_start(marker, cArgs); - - if ( !pDisp ) { - if ( ErrText[0] ) - strcat( ErrText, "\n\n" ) ; - strcat( ErrText, "No Dispatch Interface passed. " ) ; - fOk = FALSE; - } - - if ( fOk ) { - - LPOLESTR lpUniBuffer; - mbstowcs( swTemp, ptName, 256 ) ; - lpUniBuffer = swTemp ; - - // Get DISPID for name passed... - hr = pDisp->GetIDsOfNames(IID_NULL, - &lpUniBuffer, - 1, - LOCALE_USER_DEFAULT, - &dispID); - if ( hr != S_OK ) { - if ( ErrText[0] ) - strcat( ErrText, "\n\n" ) ; - sprintf( szTemp, "Get Name IDs failed: %lx. ",hr ) ; - strcat( ErrText, szTemp ) ; - fOk = FALSE; - } - - } - - if ( fOk ) { - // Extract arguments... - for ( int i=0; iInvoke(dispID, IID_NULL, LOCALE_SYSTEM_DEFAULT, - autoType, &dp, pvResult, NULL, NULL); -// if ( FAILED(hr) ) { - if ( hr != S_OK ) { - if ( ErrText[0] ) - strcat( ErrText, "\n\n" ) ; - sprintf( szTemp, "Invoke failed: %lx. ",hr ) ; - strcat( ErrText, szTemp ) ; - fOk = FALSE; - - } - - } - - // End variable-argument section... - va_end(marker); - - delete [] pArgs; - - - - return fOk; - -} - - - - - -/*****************************************************************************/ -/* ExtractXmlFromZip */ -/* */ -/* Extract the XML files from the ZIP file so that the translatable text */ -/* can be identified. All of the XML files will be concatenated together */ -/* into one output XML file. */ -/* */ -/* Input: ZipFile - Input ZIP file to extract from. */ -/* XmlFile - Output combined XML file. */ -/* TempFile - Temporary work file to use. */ -/* ErrText - Any error text to be shown to user. */ -/* */ -/* Return: TRUE - File successfully processed. */ -/* FALSE - File could not be processed. */ -/*****************************************************************************/ - /*****************************************************************************/ /* ReplaceXmlInZip */ @@ -956,46 +236,6 @@ BOOL ReplaceXmlInZip( char *TargetFile, char *TempFile, char *ErrText ) return( bReturn ) ; } - - -/*****************************************************************************/ -/* ConcatFiles */ -/* */ -/* Concatenate the contents of one file to the end of another file. */ -/* */ -/* Input: BaseFile - File to append to. */ -/* FromFile - File to copy from. */ -/* FromFileName - File name contained in ZIP file. */ -/* */ -/* Return: TRUE - Action was successful. */ -/* FALSE - Action failed. */ -/*****************************************************************************/ - -BOOL ConcatFiles( char *BaseFile, char *FromFile, char *FromFileName ) -{ - FILE *fConcat, *fFrom ; - CHAR szIn[MAX_RCD_LENGTH*2] ; - BOOL bReturn = FALSE ; - - fConcat = fopen( BaseFile, "a" ) ; - fFrom = fopen( FromFile, "r" ) ; - if ( ( fConcat ) && - ( fFrom ) ) { - fprintf( fConcat, ZIP_FILE_SEPARATOR, FromFileName ) ; - while( fgets( szIn, MAX_RCD_LENGTH, fFrom ) != NULL ) { - fputs( szIn, fConcat ) ; - } - bReturn = TRUE ; - } - - fclose( fConcat ) ; - fclose( fFrom ) ; - - return( bReturn ) ; -} - - - /*****************************************************************************/ /* SplitFiles */ /* */ @@ -1112,153 +352,6 @@ BOOL SplitFiles( char *XmlFile, char *OutputDir, char *ErrText ) } - -/*****************************************************************************/ -/* ReorderFiles */ -/* */ -/* Reorder the PPTX files so that all of the slide information is */ -/* together in the concatenated file. This it done so that the */ -/* speaker notes for a slide come immediately after that slide. */ -/* */ -/* Input: ListFile - List of XML files in this PPTX file. */ -/* */ -/* Return: None. 3-6-15 */ -/*****************************************************************************/ - -VOID ReorderFiles( char *ListFile ) -{ - -typedef struct -{ - char szFile[1024] ; // Complete file name - USHORT usSlide ; // Slide number - USHORT usType ; // Slide type - VOID *ptrNext ; // Pointer to next item in linked list -} SLIDE_INFO ; - -#define SLIDE_TYPE_NONE 0 -#define SLIDE_TYPE_SLIDE 1 -#define SLIDE_TYPE_NOTES 2 -#define SLIDE_TYPE_LAYOUT 3 -#define SLIDE_TYPE_OTHER 4 - - SLIDE_INFO *ptrSlideHead ; - SLIDE_INFO *ptrSlideCur ; - SLIDE_INFO *ptrSlidePrev ; - SLIDE_INFO *ptrSlideNew ; - - FILE *fList ; - CHAR szIn[MAX_RCD_LENGTH] ; - CHAR szNum[8] ; - CHAR szTemp[4096] ; - CHAR *ptrChar, *ptrChar2 ; - USHORT usSlide ; - USHORT usType ; - BOOL bPPT = FALSE ; - - - fList = fopen( ListFile, "r" ) ; - if ( fList ) { - - /*-----------------------------------------------------------*/ - /* See if this is a PowerPoint file or not. */ - /*-----------------------------------------------------------*/ - while( fgets( szIn, MAX_RCD_LENGTH, fList ) != NULL ) { - if ( strstr( szIn, "notesSlide" ) ) { - bPPT = TRUE ; - break; - } - } - - /*-----------------------------------------------------------*/ - /* Identify which slide each file belongs to. */ - /*-----------------------------------------------------------*/ - if ( bPPT ) { - fseek( fList, 0, SEEK_SET ) ; /* Reset to start of file */ - - ptrSlideHead = 0 ; - while( fgets( szIn, MAX_RCD_LENGTH, fList ) != NULL ) { - ptrSlideNew = (SLIDE_INFO*)calloc( sizeof(SLIDE_INFO), 1 ) ; - strcpy( ptrSlideNew->szFile, szIn ) ; - usSlide = 0 ; - usType = 0 ; - - ptrChar = strrchr( szIn, '\\' ) ; - if ( ptrChar ) { - ++ptrChar ; - if ( ! strncmp( ptrChar, "notesSlide", 10 ) ) { - usType = SLIDE_TYPE_NOTES ; - ptrChar += 10 ; - } else - if ( ! strncmp( ptrChar, "slideLayout", 11 ) ) { - usType = SLIDE_TYPE_LAYOUT ; - ptrChar += 11 ; - } else - if ( ( ! strncmp( ptrChar, "slide", 5 ) ) && - ( isdigit( *(ptrChar+5) ) ) ) { - usType = SLIDE_TYPE_SLIDE ; - ptrChar += 5 ; - } else { - usType = SLIDE_TYPE_NONE ; - } - ptrSlideNew->usType = usType ; - - if ( usType != SLIDE_TYPE_NONE ) { - for( ptrChar2=ptrChar ; *ptrChar2 && isdigit(*ptrChar2) ; ++ptrChar2 ) ; - *ptrChar2 = 0 ; - usSlide = atoi( ptrChar ) ; - ptrSlideNew->usSlide = usSlide ; - } - } - - /*-----------------------------------------------------------*/ - /* Add this node to the list in the right location. */ - /*-----------------------------------------------------------*/ - ptrSlidePrev = 0 ; - for( ptrSlideCur=ptrSlideHead ; - ( ( ptrSlideCur ) && - ( ( usSlide > ptrSlideCur->usSlide ) || - ( ( usSlide == ptrSlideCur->usSlide ) && - ( usType >= ptrSlideCur->usType ) ) ) ) ; - ptrSlidePrev=ptrSlideCur, - ptrSlideCur=(SLIDE_INFO*)ptrSlideCur->ptrNext ) ; - if ( ptrSlideHead ) { - if ( ptrSlideCur == ptrSlideHead ) { - ptrSlideNew->ptrNext = ptrSlideHead ; - ptrSlideHead = ptrSlideNew ; - } else { - ptrSlideNew->ptrNext = ptrSlideCur ; - ptrSlidePrev->ptrNext = ptrSlideNew ; - } - } else { - ptrSlideHead = ptrSlideNew ; - } - } - - /*-----------------------------------------------------------*/ - /* Rewrite the files in the proper order. */ - /*-----------------------------------------------------------*/ - if ( ptrSlideHead ) { - fclose( fList ) ; - fList = fopen( ListFile, "w" ) ; - if ( fList ) { - for( ptrSlideCur=ptrSlideHead ; ptrSlideCur ; ptrSlideCur=ptrSlidePrev ) { - fputs( ptrSlideCur->szFile, fList ) ; - ptrSlidePrev = (SLIDE_INFO*)ptrSlideCur->ptrNext ; - free( ptrSlideCur ) ; - } - } - } - } - } - if ( fList ) - fclose( fList ) ; - - return ; -} - - - /*****************************************************************************/ /* ExecuteCommand */ /* */ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.h b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.h index 748dce81..94925bb1 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.h +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.h @@ -238,7 +238,6 @@ typedef struct // Structure to define spreadsheet // Routines defined in PARSE.CPP ---------------------------------------------- - BOOL PreParse(PSZ, PSZ, PSZ, HWND, USHORT); BOOL PostParse(PSZ, PSZ, HWND); USHORT fnCreateInputList( P_INFO **, SSNT_INFO **, ULONG, HWND, USHORT *, USHORT * ) ; diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/PARSE.CPP b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/PARSE.CPP index 3a067e49..11f06e8f 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/PARSE.CPP +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/PARSE.CPP @@ -304,420 +304,6 @@ static NEUTRALTAG NEUTRAL_TAGS[ NUM_NEUTRAL_TAGS ] = { /*****************************************************************************/ -/* PreParse */ -/* */ -/* Function called by EQFPRESEG2. */ -/* */ -/* Return: TRUE - File successfully processed. */ -/* FALSE - File could not be analyzed. */ -/*****************************************************************************/ - -BOOL PreParse(PSZ in, PSZ out, PSZ markup, HWND hSlider, USHORT usFileFormat) -{ - wofstream OutFile ( out ) ; - - InputFile = new wifstream( in ) ; - - - - P_INFO *ptrParaHead ; - P_INFO *ptrParaCur ; - P_INFO *ptrParaTemp ; - - R_INFO *ptrRunHead ; - R_INFO *ptrRunCur ; - - T_INFO *ptrTextHead ; - T_INFO *ptrTextCur ; - - SSNT_INFO *ptrSSNTHead ; - SSNT_INFO *ptrSSNTCur ; - - WCHAR szErrTitle[128] ; - WCHAR szErrText[512] ; - WCHAR szIn[MAX_XML_RCD_LENGTH2] ; - WCHAR szPreTag[2] ; - WCHAR szPostTag[2] ; - ULONG FileSize ; - ULONG ulFilePos = 0 ; - ULONG ulSeqNum ; - ULONG ulTextBlocks ; - USHORT usFixLevel = 1 ; - USHORT i, j ; - USHORT rc ; - BOOL bConcatAllText = FALSE ; - BOOL bReturn = TRUE; - - - // int FileSize ; - int TotalByteRead = 0 ; - EQFSETSLIDER(hSlider, 0) ; - bFileExport = FALSE ; - bExportSSNTContent = FALSE ; - - - /* first find file size*/ - (*InputFile).fseekt(0, std::ios::end) ; - FileSize = (*InputFile).ftellt() ; - (*InputFile).fseekt(0, std::ios::beg) ; - - if (FileSize == 0) { - MessageBoxA(HWND_DESKTOP, MSG_MSOFC_EMPTY_SOURCE, TITLE_MSOFC_EMPTY_SOURCE, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE; - } - FileSize = (FileSize+99)/100 ; - - if ( ( ! stricmp( szDocTargetLanguage, "ARABIC" ) ) || - ( ! stricmp( szDocTargetLanguage, "HEBREW" ) ) || - ( ! stricmp( szDocSourceLanguage, "ARABIC" ) ) || - ( ! stricmp( szDocSourceLanguage, "HEBREW" ) ) ) - bBidi = TRUE ; - else - bBidi = FALSE ; - - - bUTF16 = (*InputFile).IsUTF16(); - bReadSource = TRUE ; - bInGfxdata = FALSE ; - bInLongAttr = FALSE ; - bSetCommonProperty = TRUE ; - - - /*************************************************************************/ - /* Create a linked list which contains all of the required */ - /* information necessary to segment this file. */ - /*************************************************************************/ - rc = fnCreateInputList( &ptrParaHead, &ptrSSNTHead, FileSize, hSlider, NULL, &usFixLevel ) ; - if ( rc > 0 ) { - MessageBoxA(HWND_DESKTOP, MSG_MSOFC_PARSING_ERROR, TITLE_MSOFC_PARSING_ERROR, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - } - - - /*************************************************************************/ - /* Create the output file for the translatable text. */ - /*************************************************************************/ - if ( bReturn ) { - (*InputFile).fseekt(0, std::ios::beg) ; - szPrevXMLInputText[0] = NULL ; - OutFile.InsertBOM(); - EQFSETSLIDER(hSlider, 0) ; - - if ( usFileFormat == FILE_FORMAT_XML ) { - OutFile << L"" ; - } else - if ( usFileFormat == FILE_FORMAT_ZIP ) { - OutFile << L"" ; - } - OutFile << L"" ; - OutFile << L"" ; - OutFile << L"\n" ; - - /**********************************************************************/ - /* Process each paragraph node. */ - /**********************************************************************/ - ulTextBlocks = 0 ; - for( ptrParaCur=ptrParaHead ; - ptrParaCur ; - ptrParaCur=(P_INFO*)ptrParaCur->ptrNext ) { - - EQFSETSLIDER(hSlider, ptrParaCur->StartPos/FileSize) ; - ulTextBlocks += ptrParaCur->NumTextTags ; - - /*******************************************************************/ - /* If paragraph has text associated with it, then write it out. */ - /*******************************************************************/ - if ( ( ptrParaCur->NumTextTags > 0 ) && /* Paragraph contains text */ - ( ptrParaCur->NodeType != NODE_TYPE_FILE_NAME ) && - ( ptrParaCur->NodeType != NODE_TYPE_SHEET_REF ) && - ( ptrParaCur->NodeType != NODE_TYPE_TEXT_REF ) ) { - - - /*************************************************************/ - /* Special case where paragraph consists of only text */ - /* which is concatenated because of same properties. */ - /* No need for leading/trailing inline tags. */ - /*************************************************************/ - bConcatAllText = FALSE ; - if ( ( ptrParaCur->ptrRunList ) && - ( ptrParaCur->NumTextTags > 1 ) && - ( ptrParaCur->NumNeutralTags == 0 ) ) { - for( ptrRunCur=(R_INFO*)ptrParaCur->ptrRunList ; - ptrRunCur ; - ptrRunCur=(R_INFO*)ptrRunCur->ptrNext ) { - if ( ( ptrRunCur->ptrTextList ) || - ( ptrRunCur->TagId != TAG_ID_TEXT ) || - ( ( ptrRunCur != (R_INFO*)ptrParaCur->ptrRunList ) && - ( ptrRunCur->Concat != CONCAT_TEXT_YES ) ) ) - break ; - } - if ( ! ptrRunCur ) - bConcatAllText = TRUE ; - } - - /*************************************************************/ - /* Write out paragraph tag. */ - /*************************************************************/ - if ( ! wcscmp( ptrParaCur->Tag, L"sheet" ) ) { - if ( ( ptrParaCur->CommonProperty ) && - ( ! bConcatAllText ) ) - swprintf( szErrText, L"", - ptrParaCur->SeqNum,ptrParaCur->Tag,ptrParaCur->CommonProperty ) ; - else - swprintf( szErrText, L"", - ptrParaCur->SeqNum,ptrParaCur->Tag ) ; - } else { - if ( ( ptrParaCur->CommonProperty ) && - ( ! bConcatAllText ) ) - swprintf( szErrText, L"", - ptrParaCur->SeqNum,ptrParaCur->Tag,ptrParaCur->CommonProperty ) ; - else - swprintf( szErrText, L"", - ptrParaCur->SeqNum,ptrParaCur->Tag ) ; - } - OutFile << szErrText ; - if ( ptrParaCur->bDoNotTranslate ) - OutFile << L"" ; - - /****************************************************************/ - /* If no run nodes, then this paragraph contains only 1 text */ - /* block. Write out the translatable text for this */ - /* paragraph. */ - /****************************************************************/ - if ( ptrParaCur->ptrRunList == 0 ) { - fnWriteBlock( WRITE_SOURCE, - ptrParaCur->BlockStartPos, - ptrParaCur->BlockEndPos, - &OutFile, FALSE ) ; - szPrevXMLInputText[0] = NULL ; - } else { - - /*************************************************************/ - /* Process each run block in this paragraph. */ - /*************************************************************/ - for( ptrRunCur=(R_INFO*)ptrParaCur->ptrRunList ; - ptrRunCur ; - ptrRunCur=(R_INFO*)ptrRunCur->ptrNext ) { - - /**********************************************************/ - /* If no text nodes, then this run node consists of */ - /* 1 control, either text or neutral tags. */ - /**********************************************************/ - if ( ptrRunCur->ptrTextList == 0 ) { - - /*******************************************************/ - /* If this is a text node, write out the */ - /* translatable text for this node. */ - /*******************************************************/ - if ( ptrRunCur->TagId == TAG_ID_TEXT ) { - - if ( ptrRunCur->BreakBeforeText ) { - swprintf( szErrText, L"
" ) ; - OutFile << L"
"; - } - if ( ptrParaCur->NumTextTags > 1 ) { /* If > 1 text */ - if ( ( ptrRunCur->Concat != CONCAT_TEXT_YES ) && - ( ! bConcatAllText ) && - ( ( ptrRunCur->CommonProperty == 0 ) || - ( ptrRunCur->CommonProperty == COMMONPROPERTY_FORCE_OFF ) ) ) { - swprintf( szErrText, L"<%s%ld>", STYLE_TAG_LONG, ptrRunCur->SeqNum ) ; - OutFile << szErrText ; - } - } - - fnWriteBlock( WRITE_SOURCE, - ptrRunCur->BlockStartPos, - ptrRunCur->BlockEndPos, - &OutFile, FALSE ) ; - szPrevXMLInputText[0] = NULL ; - - if ( ptrParaCur->NumTextTags > 1 ) { /* If > 1 text */ - if ( ( ( ptrRunCur->ptrNext == NULL ) || - ( ((R_INFO*)(ptrRunCur->ptrNext))->Concat != CONCAT_TEXT_YES ) ) && - ( ! bConcatAllText ) && - ( ( ptrRunCur->CommonProperty == 0 ) || - ( ptrRunCur->CommonProperty == COMMONPROPERTY_FORCE_OFF ) ) ) { - swprintf( szErrText, L"", STYLE_TAG_LONG, ptrRunCur->SeqNum ) ; - OutFile << szErrText ; - } - } - } else { - - /****************************************************/ - /* This is a neutral tag node. Write out the */ - /* placeholder for this information. */ - /****************************************************/ - for( j=0 ; - jTagId!=NEUTRAL_TAGS[j].TagId ; - ++j ) ; - - if ( ( ptrRunCur->TagType == TAG_TYPE_END ) && - ( ptrRunCur->BeginSeqNum > 0 ) ) - ulSeqNum = ptrRunCur->BeginSeqNum ; - else - ulSeqNum = ptrRunCur->SeqNum ; - - szPreTag[0] = NULL ; - szPostTag[0] = NULL ; - if ( ptrRunCur->TagType == TAG_TYPE_END ) - wcscpy( szPreTag, L"/" ) ; - else - if ( ptrRunCur->TagType == TAG_TYPE_EMPTY ) - wcscpy( szPostTag, L"/" ) ; - - if ( j < NUM_NEUTRAL_TAGS ) { - swprintf( szErrText, L"<%s%s%ld_%s%s>", szPreTag, NEUTRAL_TAG_LONG, - ulSeqNum, NEUTRAL_TAGS[j].PseudoTag, szPostTag ) ; - } else { - swprintf( szErrText, L"<%s%s%ld_X%s>", szPreTag, NEUTRAL_TAG_LONG, - ulSeqNum, szPostTag ) ; - } - - OutFile << szErrText ; - } - } else { - - /*******************************************************/ - /* If multiple text nodes, then must surround this */ - /* run block with neutral tag information. */ - /*******************************************************/ - if ( ( ptrRunCur->NumTextTags > 1 ) && - ( ((R_INFO*)(ptrParaCur->ptrRunList))->ptrNext != NULL ) ) { - swprintf( szErrText, L"<%s%ld_X>", STYLE_TAG_LONG, ptrRunCur->SeqNum ) ; - OutFile << szErrText ; - } - - - /*******************************************************/ - /* Process each text node in this run block. */ - /*******************************************************/ - for( ptrTextCur=(T_INFO*)ptrRunCur->ptrTextList ; - ptrTextCur ; - ptrTextCur=(T_INFO*)ptrTextCur->ptrNext ) { - - /****************************************************/ - /* If this is a text node, write out the */ - /* translatable text for this node. */ - /****************************************************/ - if ( ptrTextCur->TagId == TAG_ID_TEXT ) { - - if ( ptrParaCur->NumTextTags > 1 ) { /* If > 1 text */ - swprintf( szErrText, L"<%s%ld>", STYLE_TAG_LONG, ptrTextCur->SeqNum ) ; - OutFile << szErrText ; - } - - fnWriteBlock( WRITE_SOURCE, - ptrTextCur->BlockStartPos, - ptrTextCur->BlockEndPos, - &OutFile, FALSE ) ; - - if ( ptrParaCur->NumTextTags > 1 ) { /* If > 1 text */ - swprintf( szErrText, L"", STYLE_TAG_LONG, ptrTextCur->SeqNum ) ; - OutFile << szErrText ; - } - } else { - - /*************************************************/ - /* This is a neutral tag node. Write out the */ - /* placeholder for this information. */ - /*************************************************/ - for( j=0 ; - jTagId!=NEUTRAL_TAGS[j].TagId ; - ++j ) ; - - if ( ( ptrTextCur->TagType == TAG_TYPE_END ) && - ( ptrTextCur->BeginSeqNum > 0 ) ) - ulSeqNum = ptrTextCur->BeginSeqNum ; - else - ulSeqNum = ptrTextCur->SeqNum ; - - szPreTag[0] = NULL ; - szPostTag[0] = NULL ; - if ( ptrTextCur->TagType == TAG_TYPE_END ) - wcscpy( szPreTag, L"/" ) ; - else - if ( ptrTextCur->TagType == TAG_TYPE_EMPTY ) - wcscpy( szPostTag, L"/" ) ; - - if ( j < NUM_NEUTRAL_TAGS ) { - swprintf( szErrText, L"<%s%s%ld_%s%s>", szPreTag, NEUTRAL_TAG_LONG, - ulSeqNum, NEUTRAL_TAGS[j].PseudoTag, szPostTag ) ; - } else { - swprintf( szErrText, L"<%s%s%ld_X%s>", szPreTag, NEUTRAL_TAG_LONG, - ulSeqNum, szPostTag ) ; - } - - OutFile << szErrText ; - } - - } - - /*******************************************************/ - /* If multiple text nodes, then must surround this */ - /* run block with neutral tag information. */ - /*******************************************************/ - if ( ( ptrRunCur->NumTextTags > 1 ) && - ( ((R_INFO*)(ptrParaCur->ptrRunList))->ptrNext != NULL ) ) { - swprintf( szErrText, L"", STYLE_TAG_LONG, ptrRunCur->SeqNum ) ; - OutFile << szErrText ; - } - - } - } - } - - - /****************************************************************/ - /* End this paragraph block. */ - /****************************************************************/ - if ( ptrParaCur->bDoNotTranslate ) - OutFile << L"" ; - OutFile << L"" << ENDL ; - } else { - - if ( ( ptrParaCur->NodeType == NODE_TYPE_FILE_NAME ) && - ( ptrParaCur->ptrNext ) ) { - for( ptrParaTemp=(P_INFO*)ptrParaCur->ptrNext ; - ptrParaTemp && ptrParaTemp->NodeType!=NODE_TYPE_FILE_NAME ; - ptrParaTemp=(P_INFO*)ptrParaTemp->ptrNext ) { - if ( ( ptrParaTemp->NodeType != NODE_TYPE_SHEET_REF ) && - ( ptrParaTemp->NodeType != NODE_TYPE_TEXT_REF ) ) - break ; - } - if ( ( ptrParaTemp ) && - ( ptrParaTemp->NodeType != NODE_TYPE_FILE_NAME ) ) { - OutFile << L"" << ENDL ; - } - } - } - } - - /**********************************************************************/ - /* If paragraph has NO text associated with it, write out dummy */ - /* paragraph so that the segmented file is not empty. */ - /**********************************************************************/ - if ( ulTextBlocks == 0 ) { - swprintf( szErrText, L"" ) ; - OutFile << szErrText ; - } - } - - - /*************************************************************************/ - /* Free linked list space. */ - /*************************************************************************/ - fnFreeParaList( &ptrParaHead ) ; - fnFreeSSNTList( &ptrSSNTHead ) ; - - (*InputFile).freefile() ; - EQFSETSLIDER(hSlider, 100) ; - - return(bReturn); - -} /* PreParse */ - - - /*****************************************************************************/ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/PARSE.CPP b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/PARSE.CPP index cafb1559..e9aa51cc 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/PARSE.CPP +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/PARSE.CPP @@ -198,223 +198,6 @@ BOOL ShowError( wchar_t *Title, wchar_t *Message, BOOL bOKCancel ) -/*****************************************************************************/ -/* PreParse */ -/* */ -/* Function called by EQFPRESEG2. */ -/* */ -/* Return: TRUE - File successfully processed. */ -/* FALSE - File could not be analyzed. */ -/*****************************************************************************/ - -BOOL PreParse(PSZ in, PSZ out, PSZ markup, PSZ sourcefile, PSZ tempfile, HWND hSlider) -{ - BOOL bReturn = TRUE; - Token XTok ; - XTbl.initializeStates(); - XTok.initializeToken(); - wofstream OutFile ( out ) ; - - InputFile = new wifstream( in ) ; - - IString s ; - IString t ; - wchar_t szErrTitle[128] ; - wchar_t szEOLType[10] ; - wchar_t *szErrText ; - wchar_t *ptrChar, *ptrChar2 ; - wchar_t szChar1[10] ; - int FileSize = 1; - int ONE = 1; - int ZERO = 0; - int iTemp ; - long save_pos; - long lWrite ; - long lIndex ; - BOOL b1stRead = TRUE ; - BOOL bTranslate ; - - struct stat statBuf; - - - // int FileSize ; - int TotalByteRead = 0 ; - EQFSETSLIDER(hSlider, 0) ; - - - szErrText = (wchar_t*)malloc( MAX_BUFFER_SIZE*sizeof(wchar_t)*3 ) ; - -#ifdef _UNICODE - mbstowcs(szErrText, markup, strlen(markup)+1); -#else - strcpy( szErrText, markup ) ; /* Find file type from markup as */ -#endif - - bEscapeDquote = FALSE ; - bXLIFF = FALSE ; - bSDLXLF= FALSE ; - bXHTML = FALSE ; - sXHTML_FontState = XHTML_FONT_NONE ; - sXHTML_TagState = XHTML_TAG_NONE ; - sXHTML_PrevTagState = XHTML_TAG_NONE ; - bXHTML_InCommentCheck = FALSE ; - bXHTMLCondAttr = FALSE ; - bTransAttrHasNoCondAttr = FALSE; - bCDATA_XHTMLTag = FALSE ; - bEmptyEntityValue = FALSE ; - if ( ( ! stricmp( szDocTargetLanguage, "ARABIC" ) ) || - ( ! stricmp( szDocTargetLanguage, "HEBREW" ) ) || - ( ! stricmp( szDocSourceLanguage, "ARABIC" ) ) || - ( ! stricmp( szDocSourceLanguage, "HEBREW" ) ) ) - bBidi = TRUE ; - else - bBidi = FALSE ; - - bPreserveEOL = FALSE ; - bInsertEOL = FALSE ; - wcscat( szErrText, L".DTD\"" ) ; /* default if no !DOCTYPE tag. */ - s=IString(szErrText) ; - XTok.SetDTDInfo( s, TRUE ) ; - - bUTF16 = (*InputFile).IsUTF16(); - - if ( bPreserveEOL ) { - (*InputFile).freefile() ; - (*InputFile).~wifstream() ; - if ( ConvertEOL( in, tempfile, szEOLType ) ) { - InputFile = new wifstream( tempfile ) ; - } else { - InputFile = new wifstream( in ) ; - } - } - - /* first find file size*/ - (*InputFile).fseekt(0, std::ios::end) ; - FileSize = (*InputFile).ftellt() ; - (*InputFile).fseekt(0, std::ios::beg) ; - - - sCopyTextState = COPYTEXT_STATE_OFF ; - sIgnoreCommentState = IGNORECOMMENT_STATE_OFF ; - sLinesState = LINES_STATE_OFF ; - sPrevScriptState = SCRIPT_STATE_NONE ; - sPrevTransState = TRANS_STATE_NONE ; - sScriptState = SCRIPT_STATE_NONE ; - bFirstTag = TRUE ; - bInFirstTag = FALSE ; - sSCRIPT_ParenCount=0 ; - bNeutralTag = FALSE ; - bLongSegTrans = TRUE ; - sLongSegLen = 0 ; - sMaxLength = 0 ; - - OutFile.InsertBOM(); - - - if ( bPreserveEOL ) { - OutFile << L"" << ENDL ; - } - - - /* look into the file before we do a read */ - while ( (*InputFile).getline(szErrText, MAX_BUFFER_SIZE, bUTF16) != NULL ) { - - if ( ( wcslen(szErrText) == MAX_BUFFER_SIZE-1 ) || - ( ( b1stRead ) && - ( bUTF16 ) && - ( wcslen(szErrText) == MAX_BUFFER_SIZE-2 ) ) ) { - bTruncatedRead = TRUE ; - ptrChar = wcsrchr( szErrText, L'<' ) ; - ptrChar2 = wcsrchr( szErrText, L'>' ) ; - if ( ( ptrChar ) && - ( ( ptrChar2 == NULL ) || - ( ptrChar2 < ptrChar ) ) ) { - lIndex = wcslen(szErrText) ; - while( (*InputFile).getline(szChar1,(int)2,bUTF16)!=NULL ) { - szErrText[lIndex++] = szChar1[0] ; - szErrText[lIndex] = NULL ; - if ( ( szChar1[0] == L'>' ) || - ( lIndex > MAX_BUFFER_SIZE*2 ) ) - break; - } - } - } else { - bTruncatedRead = FALSE ; - } - s = szErrText; - b1stRead = FALSE ; - - - XTok.SetString(s) ; - TotalByteRead += s.length() + 1 ; - EQFSETSLIDER(hSlider, (TotalByteRead*2*100)/FileSize) ; - - if ( sLinesState == LINES_STATE_ON ) { - if ( ( XTok.GetTransState() == TRANS_STATE_OFF ) || - ( XTok.GetTransStateAttr() == TRANS_STATE_OFF ) || - ( XTbl.off(COMENT) ) ) { - } else { - XTok.AddTagTransOff() ; - XTok.AddTagTransOn() ; - } - } - - if( wcsstr(szErrText,L" ") || (wcslen(szErrText) != MAX_BUFFER_SIZE-1 ) || wcsstr(szErrText,L"<")) { //if not binary data - ParseTags(s, XTok) ; - } else { - s = XTok.GetString() ; /* Force 2K split for non-trans block */ - } - - - for( lIndex=1, lWrite=s.length() ; lWrite>0 ; lWrite-=MAX_BUFFER_SIZE,lIndex+=MAX_BUFFER_SIZE ) { - OutFile << s.subString(lIndex,MAX_BUFFER_SIZE) ; - } - if ( ! bTruncatedRead ) { //end of line - OutFile << ENDL ; - } - } - - if (FileSize == ZERO) { - wcscpy( szErrText, L"File does not contain any information, 0 bytes in size. Analysis terminated." ) ; - wcscpy( szErrTitle, L"Invalid Source" ) ; - ShowError( szErrTitle, szErrText, FALSE ) ; - bReturn = FALSE; - } - - if ( bReturn ) { - if ( bCodePageANSI ) { - OutFile << L"" << ENDL ; - } else - if ( bCodePageHTML ) { - OutFile << L"" << ENDL ; - } else - if ( bCodePageUTF8 ) { - if ( bCodePageUTF8_BOM ) - OutFile << L"" << ENDL ; - else - OutFile << L"" << ENDL ; - } else - if ( bCodePageUTF16 ) { - OutFile << L"" << ENDL ; - } else { - OutFile << L"" << ENDL ; - } - } - - EQFSETSLIDER(hSlider, 100) ; - XTok.FreeDTDInfo() ; - - (*InputFile).freefile() ; - (*InputFile).~wifstream() ; - - if ( szErrText ) free(szErrText) ; - - return(bReturn); - -} /* PreParse */ - - - /*****************************************************************************/ /* PostParse */ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/parse.h b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/parse.h index 71296f6a..4aef26c9 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/parse.h +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/parse.h @@ -12,8 +12,6 @@ * files. * */ - /* This is called by EQFPRESEG2 */ - BOOL PreParse(PSZ in, PSZ out, PSZ markup, PSZ sourcefile, PSZ tempfile, HWND hSlider); /* This is called by EQFPOSTSEG2 */ BOOL PostParse(PSZ in, PSZ out, HWND hSlider); diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.cpp b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.cpp index 3dbd28d7..edd52969 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.cpp +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.cpp @@ -67,709 +67,6 @@ static TAGLIST REF_TAGS[ NUM_REF_TAGS ] = } ; - -/****************************************************************************/ -/* */ -/* PreParse1 */ -/* */ -/* Function is called by EQFPRESEG2 -- before -- OTMXML processing. */ -/* */ -/* Input: InFile - Input file name. */ -/* OutFile - Output file name. */ -/* StyleFile - Output file for saved style information. */ -/* hwndSlider - Handle of status slider. */ -/* Output: TRUE - Porcessing successful. */ -/* FALSE - Processing failed. */ -/****************************************************************************/ -BOOL PreParse1(PSZ InFile, PSZ OutFile, PSZ StyleFile, HWND hwndSlider) -{ - FILE *fIn, *fOut, *fStyle ; - CHAR szIn[MAX_RCD_LENGTH*3] ; - CHAR *ptrChar, *ptrChar2 ; - CHAR *ptrTrackedChangesStart ; - LONG lFileSize, lFileBytes ; - USHORT i, j ; - USHORT usTrackedChangesState = 0 ; - USHORT usStyleState = ODC_STYLE_STATE_BEFORE ; - BOOL bParagraphFound = FALSE ; - BOOL bReturn = TRUE; - - - fIn = fopen( InFile, "r" ) ; - fOut = fopen( OutFile, "w" ) ; - if ( StyleFile[0] ) - fStyle = fopen( StyleFile, "w" ) ; - else - usStyleState = ODC_STYLE_STATE_AFTER ; - if ( !fIn || !fOut || ( !fStyle && usStyleState==ODC_STYLE_STATE_BEFORE ) ) { - if ( fIn ) - fclose( fIn ) ; - if ( fOut ) - fclose( fOut ) ; - if ( fStyle ) - fclose( fStyle ) ; - return( FALSE ) ; - } - - /*------------------------------------------------------------------------*/ - /* Get number of characters in entire file. Used to determine percent */ - /* completed for updating slider information. */ - /*------------------------------------------------------------------------*/ - fseek( fIn, 0, SEEK_END ) ; /* Seek to the end of the file */ - lFileSize = ftell( fIn ) ; /* Get size of file */ - fseek( fIn, 0, SEEK_SET ) ; /* Seek to the beginning of file */ - lFileBytes = 0 ; - - if ( hwndSlider ) - EQFSETSLIDER( hwndSlider, 0 ) ; /* Initialize slider to 0. */ - /*------------------------------------------------------------------------*/ - /* Process each line. */ - /*------------------------------------------------------------------------*/ - while ( GetRcd( szIn, MAX_RCD_LENGTH, fIn, TRUE ) != NULL ) { - - lFileBytes += strlen( szIn ) ; - if ( hwndSlider ) - EQFSETSLIDER( hwndSlider, (USHORT)((lFileBytes * 100)/lFileSize) ) ; - - - /*---------------------------------------------------------------------*/ - /* Find each tag and make sure it starts on a new line. */ - /*---------------------------------------------------------------------*/ - for( i=0 ; szIn[i] && isspace(szIn[i]) ; ++i ) ; - if ( szIn[i] ) { - for( ptrChar=strstr(&szIn[++i],szODC_P) ; - ptrChar ; - ptrChar=strstr(ptrChar+1,szODC_P) ) { - if ( ( isspace(*(ptrChar+strlen(szODC_P)) ) ) || - ( strchr( "/>", *(ptrChar+strlen(szODC_P)) ) ) ) { - bParagraphFound = TRUE ; - if ( *(ptrChar+strlen(szODC_P)) != '/' ) { - memmove( ptrChar+9, ptrChar, strlen(ptrChar)+1 ) ; - strncpy( ptrChar, szODC_ODCNL_NL, 9 ) ; - ptrChar += 10 ; - } - } - } - } - - /*---------------------------------------------------------------------*/ - /* Remove all tracked-changes tags and content. */ - /*---------------------------------------------------------------------*/ - if ( usTrackedChangesState == 0 ) { - ptrChar = strstr( szIn, szODC_TRACKEDCHANGES_START ) ; - if ( ptrChar ) { - ptrChar2 = strchr( ptrChar, '>' ) ; - if ( ptrChar2 ) { - if ( *(ptrChar2-1) == '/' ) { - usTrackedChangesState = 0 ; /* No tracks to be removed */ - } else { - ptrTrackedChangesStart = ptrChar ; - usTrackedChangesState = 1 ; - } - } - } - } - if ( usTrackedChangesState == 1 ) { - ptrChar = strstr( szIn, szODC_TRACKEDCHANGES_END ) ; - if ( ptrChar ) { - ptrChar2 = strchr( ptrChar, '>' ) ; - if ( ptrChar2 ) { - memmove( ptrTrackedChangesStart, ptrChar2+1, strlen(ptrChar2+1)+1 ) ; - usTrackedChangesState = 0 ; - } - } else { - *ptrTrackedChangesStart = 0 ; - ptrTrackedChangesStart = szIn ; - } - } - - /*---------------------------------------------------------------------*/ - /* Remove all change flagging tags from the translatable text. */ - /*---------------------------------------------------------------------*/ - if ( usTrackedChangesState != 1 ) { - for( ptrChar=strstr(szIn,szODC_TEXT_CHANGE) ; - ptrChar ; - ptrChar=strstr(++ptrChar,szODC_TEXT_CHANGE) ) { - if ( ptrChar > szIn ) { - ptrChar2 = strchr( ptrChar, '>' ) ; - if ( ( ptrChar2 ) && - ( ( ! strncmp( ptrChar-1, szODC_CHANGE, 12 ) ) || - ( ! strncmp( ptrChar-1, szODC_CHANGE_END, 16 ) ) || - ( ! strncmp( ptrChar-1, szODC_CHANGE_START, 18 ) ) ) ) { - memmove( --ptrChar, ++ptrChar2, strlen(ptrChar2)+1 ) ; - if ( ptrChar > szIn ) - --ptrChar ; - } - } - } - } - - /*---------------------------------------------------------------------*/ - /* Save all style information to a separate file in order to reduce */ - /* the size of the segmented source files, and to reduce the time */ - /* to open the file in the editors. */ - /*---------------------------------------------------------------------*/ - if ( usStyleState != ODC_STYLE_STATE_AFTER ) { - if ( usStyleState == ODC_STYLE_STATE_BEFORE ) { - ptrChar = strstr( szIn, szODC_DOCUMENTCONTENT ) ; - if ( ptrChar ) { - usStyleState = ODC_STYLE_STATE_IN ; - *ptrChar = 0 ; - fputs( szIn, fOut ) ; - fputs( szODC_TWBSTYLE, fOut ) ; - *ptrChar = '<' ; - memmove( szIn, ptrChar, strlen(ptrChar)+1 ) ; - } else { - if ( bParagraphFound ) - usStyleState = ODC_STYLE_STATE_AFTER ; - } - } - if ( usStyleState == ODC_STYLE_STATE_IN ) { - ptrChar = strstr( szIn, szODC_BODY ) ; - if ( ptrChar ) { - usStyleState = ODC_STYLE_STATE_AFTER ; - *ptrChar = 0 ; - fputs( szIn, fStyle ) ; - *ptrChar = '<' ; - memmove( szIn, ptrChar, strlen(ptrChar)+1 ) ; - } else { - if ( bParagraphFound ) { - usStyleState = ODC_STYLE_STATE_AFTER ; - for( ptrChar=strstr(szIn,szODC_P) ; - ptrChar ; - ptrChar=strstr(ptrChar+1,szODC_P) ) { - if ( ( isspace(*(ptrChar+strlen(szODC_P)) ) ) || - ( strchr( "/>", *(ptrChar+strlen(szODC_P)) ) ) ) { - break ; - } - } - if ( ptrChar ) - *ptrChar = 0 ; - fputs( szIn, fStyle ) ; - *ptrChar = szODC_P[0] ; - memmove( szIn, ptrChar, strlen(ptrChar)+1 ) ; - } else { - fputs( szIn, fStyle ) ; - continue ; - } - } - } - } - - - fputs( szIn, fOut ) ; - } - fclose( fIn ) ; - fclose( fOut ) ; - fclose( fStyle ) ; - if ( usStyleState == ODC_STYLE_STATE_BEFORE ) /* No style info saved */ - remove( StyleFile ) ; - - EQFSETSLIDER(hwndSlider, 100) ; - return(bReturn); - -} /* PreParse1 */ - - -/****************************************************************************/ -/* */ -/* PreParse2 */ -/* */ -/* Function is called by EQFPRESEG2 -- after -- OTMXML processing. */ -/* */ -/* Input: InFile - Input file name. */ -/* OutFile - Output file name. */ -/* hwndSlider - Handle of status slider. */ -/* Output: TRUE - Porcessing successful. */ -/* FALSE - Processing failed. */ -/****************************************************************************/ -BOOL PreParse2(PSZ in, PSZ out, HWND hwndSlider) -{ - FILE *fIn, *fOut ; - WCHAR szIn[MAX_RCD_LENGTH*2] ; - WCHAR szSave[MAX_RCD_LENGTH*2] ; - WCHAR szTag[ODC_TAG_SIZE] ; - WCHAR szTag1[ODC_TAG_SIZE] ; - WCHAR szTag2[ODC_TAG_SIZE] ; - WCHAR szAttribute1[512] ; - WCHAR szAttribute2[512] ; - WCHAR szSurroundTags[20][ODC_TAG_SIZE] ; - WCHAR *ptrSurround1[20] ; - WCHAR *ptrSurround2[20] ; - WCHAR *ptrTwbEnd, *ptrTwbStart ; - WCHAR *ptrChar, *ptrChar2, *ptrChar3 ; - WCHAR *ptrChar1s, *ptrChar1e, *ptrChar2s, *ptrChar2e ; - LONG lFileSize, lFileBytes ; - USHORT usTagIndex ; - USHORT i, j ; - BOOL bTagsRemoved ; - BOOL bMoveTags ; - BOOL bBeginEndTags ; - BOOL bReturn = TRUE; - - fIn = fopen( in, "rb" ) ; - fOut = fopen( out, "wb" ) ; - if ( !fIn || !fOut ) { - if ( fIn ) - fclose( fIn ) ; - if ( fOut ) - fclose( fOut ) ; - return( FALSE ) ; - } - - /*------------------------------------------------------------------------*/ - /* Get number of characters in entire file. Used to determine percent */ - /* completed for updating slider information. */ - /*------------------------------------------------------------------------*/ - fseek( fIn, 0, SEEK_END ) ; /* Seek to the end of the file */ - lFileSize = ftell( fIn ) ; /* Get size of file */ - fseek( fIn, 0, SEEK_SET ) ; /* Seek to the beginning of file */ - lFileBytes = 0 ; - - if ( hwndSlider ) - EQFSETSLIDER( hwndSlider, 0 ) ; /* Initialize slider to 0. */ - - /*------------------------------------------------------------------------*/ - /* Process each line. */ - /*------------------------------------------------------------------------*/ - while ( fgetws( szIn, MAX_RCD_LENGTH, fIn ) != NULL ) { - lFileBytes += wcslen(szIn) * 2 ; - if ( hwndSlider ) - EQFSETSLIDER( hwndSlider, (USHORT)((lFileBytes * 100)/lFileSize) ) ; - - ptrTwbEnd = wcsstr( szIn, szODC_TWB_END ) ; - if ( ptrTwbEnd ) - ptrTwbStart = wcsstr( ptrTwbEnd, szODC_TWB_START ) ; - else - ptrTwbStart = 0 ; - - /*------------------------------------------------------------------------*/ - /* Repeat checking for conditions where inline tags can be moved */ - /* from the translatable section of the text to the non-translatable */ - /* section. */ - /*------------------------------------------------------------------------*/ - bTagsRemoved = TRUE ; - while( bTagsRemoved ) { - bTagsRemoved = FALSE ; - - /*---------------------------------------------------------------------*/ - /* Handle begin/end tag pairs before the start of the translatable */ - /* text. */ - /* */ - /* --twb>text */ - /* --twb>text */ - /* Becomes: */ - /* --twb>text */ - /* --twb>text */ - /*---------------------------------------------------------------------*/ - if ( ( ptrTwbEnd ) && - ( *(ptrTwbEnd+wcslen(szODC_TWB_END)) == L'<' ) ) { - ptrChar1s = ptrTwbEnd + wcslen(szODC_TWB_END) ; - for( i=0 ; i tag before text. */ - /*---------------------------------------------------------------*/ - if ( iswspace( *ptrChar ) ) { - ptrChar1e = wcschr( ptrChar, L'>' ) ; - ptrChar2 = wcsstr( ptrChar, szODC_STRING_VALUE ) ; - if ( ( ptrChar1e ) && - ( *(ptrChar1e-1) == L'/' ) && - ( ( ! ptrChar2 ) || - ( ptrChar2 > ptrChar1e ) ) ) { - bMoveTags = TRUE ; - } - } else - /*---------------------------------------------------------------*/ - /* Handle tag before text. */ - /*---------------------------------------------------------------*/ - if ( ! wcsncmp( ptrChar, szODC_TAG_END, wcslen(szODC_TAG_END) ) ) { - ptrChar1e = wcschr( ptrChar, L'>' ) ; - if ( ( ptrChar1e ) && - ( *(ptrChar1e-1) == L'/' ) ) { - bMoveTags = TRUE ; - } - } else - /*---------------------------------------------------------------*/ - /* Handle tag before text. */ - /*---------------------------------------------------------------*/ - if ( ! wcsncmp( ptrChar, szODC_TAG_START, wcslen(szODC_TAG_START) ) ) { - ptrChar1e = wcschr( ptrChar, L'>' ) ; - if ( ( ptrChar1e ) && - ( *(ptrChar1e-1) == L'/' ) ) { - bBeginEndTags = FALSE ; - /*---------------------------------------------------------*/ - /* Handle */ - /* tags before text which have the same text:name */ - /* attribute values. */ - /*---------------------------------------------------------*/ - if ( ! wcsncmp( (ptrChar1e+2), szTag, wcslen(szTag) ) ) { - ptrChar2s = ptrChar1e + 1 ; - ptrChar = ptrChar2s + wcslen(szTag) + 1 ; - if ( ! wcsncmp( ptrChar, szODC_TAG_END, wcslen(szODC_TAG_END) ) ) { - ptrChar2e = wcschr( (ptrChar2s), L'>' ) ; - if ( ( ptrChar2e ) && - ( *(ptrChar2e-1) == L'/' ) ) { - if ( ( GetAttributeValue( ptrChar1s, L"text:name", szAttribute1 ) ) && - ( GetAttributeValue( ptrChar2s, L"text:name", szAttribute2 ) ) && - ( ! wcscmp( szAttribute1, szAttribute2 ) ) ) { - bMoveTags = TRUE ; - bBeginEndTags = TRUE ; - ptrChar1e = ptrChar2e ; - } else - if ( ( GetAttributeValue( ptrChar1s, L"text:id", szAttribute1 ) ) && - ( GetAttributeValue( ptrChar2s, L"text:id", szAttribute2 ) ) && - ( ! wcscmp( szAttribute1, szAttribute2 ) ) ) { - bMoveTags = TRUE ; - bBeginEndTags = TRUE ; - ptrChar1e = ptrChar2e ; - } - } - } - } - /*---------------------------------------------------------*/ - /* Handle tag before text and */ - /* no other "text:bookmark" tag after that. */ - /*---------------------------------------------------------*/ - if ( ( ! bBeginEndTags ) && - ( ! wcsstr( ptrChar1e, szTag ) ) ) { - bMoveTags = TRUE ; - } - } - } - if ( bMoveTags ) { - bTagsRemoved = TRUE ; - wmemmove( ptrTwbEnd, ptrChar1s, ptrChar1e-ptrChar1s+1 ) ; - ptrTwbEnd = ptrChar1e - wcslen(szODC_TWB_END) + 1 ; - wcsncpy( ptrTwbEnd, szODC_TWB_END, wcslen(szODC_TWB_END) ) ; - } - } - } - - - - /*------------------------------------------------------------------------*/ - /* Handle begin/end tag pairs after the end of the translatable text. */ - /* */ - /* --twb>text--twb>text--twb>text */ - /* --twb>text */ - /*------------------------------------------------------------------------*/ - if ( ( ptrTwbEnd ) && - ( ptrTwbStart ) && - ( *(ptrTwbStart-1) == L'>' ) ) { - ptrChar1e = ptrTwbStart - 1 ; - for( ptrChar1s=ptrChar1e-1 ; - ptrChar1s>ptrTwbEnd && *ptrChar1s!=L'<' ; - --ptrChar1s ) ; - if ( *ptrChar1s == L'<' ) { - for( i=0 ; i tag after text. */ - /*---------------------------------------------------------------*/ - if ( iswspace( *ptrChar ) ) { - ptrChar2 = wcsstr( ptrChar, szODC_STRING_VALUE ) ; - if ( ( *(ptrChar1e-1) == L'/' ) && - ( ( ! ptrChar2 ) || - ( ptrChar2 > ptrChar1e ) ) ) { - bMoveTags = TRUE ; - } - } else - /*---------------------------------------------------------------*/ - /* Handle tag after text. */ - /*---------------------------------------------------------------*/ - if ( ! wcsncmp( ptrChar, szODC_TAG_START, wcslen(szODC_TAG_START) ) ) { - bMoveTags = TRUE ; - } else - /*---------------------------------------------------------------*/ - /* Handle tag after text. */ - /*---------------------------------------------------------------*/ - if ( ! wcsncmp( ptrChar, szODC_TAG_END, wcslen(szODC_TAG_END) ) ) { - if ( *(ptrChar1e-1) == L'/' ) { - - /*---------------------------------------------------------*/ - /* Handle */ - /* tags after text which have the same text:name */ - /* attribute values. */ - /*---------------------------------------------------------*/ - bBeginEndTags = FALSE ; - if ( *(ptrChar1s-1) == L'>') { - ptrChar2e = ptrChar1s - 1 ; - for( ptrChar2s=ptrChar2e-1 ; - ptrChar2s>ptrTwbEnd && *ptrChar2s!=L'<' ; - --ptrChar2s ) ; - if ( ( *ptrChar2s == L'<' ) && - ( ! wcsncmp( (ptrChar1s+1), szTag, wcslen(szTag) ) ) ) { - ptrChar = ptrChar2s + wcslen(szTag) + 1 ; - if ( ! wcsncmp( ptrChar, szODC_TAG_START, wcslen(szODC_TAG_START) ) ) { - if ( *(ptrChar2e-1) == L'/' ) { - if ( ( GetAttributeValue( ptrChar1s, L"text:name", szAttribute1 ) ) && - ( GetAttributeValue( ptrChar2s, L"text:name", szAttribute2 ) ) && - ( ! wcscmp( szAttribute1, szAttribute2 ) ) ) { - bMoveTags = TRUE ; - ptrChar1s = ptrChar2s ; - } else - if ( ( GetAttributeValue( ptrChar1s, L"text:id", szAttribute1 ) ) && - ( GetAttributeValue( ptrChar2s, L"text:id", szAttribute2 ) ) && - ( ! wcscmp( szAttribute1, szAttribute2 ) ) ) { - bMoveTags = TRUE ; - ptrChar1s = ptrChar2s ; - } - } - } - } - } - /*---------------------------------------------------------*/ - /* Handle tag after text and */ - /* no other "text:bookmark" tag before that. */ - /*---------------------------------------------------------*/ - ptrChar = wcsstr( ptrTwbEnd, szTag ) ; - if ( ( ! bBeginEndTags ) && - ( ptrChar ) && - ( ptrChar > ptrChar1s ) ) { - bMoveTags = TRUE ; - } - } - } - if ( bMoveTags ) { - bTagsRemoved = TRUE ; - wmemmove( ptrChar1s+wcslen(szODC_TWB_START), ptrChar1s, ptrChar1e-ptrChar1s+1 ) ; - ptrTwbStart = ptrChar1s ; - wcsncpy( ptrTwbStart, szODC_TWB_START, wcslen(szODC_TWB_START) ) ; - } - } - } - } - - - - /*---------------------------------------------------------------------*/ - /* Handle matching begin/end tags which surround the entire piece of */ - /* translatable text with no imbedded tags of the same type between */ - /* them. */ - /* */ - /* --twb>text--twb>text */ - /*---------------------------------------------------------------------*/ - if ( ( ptrTwbEnd ) && - ( ptrTwbEnd < ptrTwbStart ) && - ( *(ptrTwbEnd+wcslen(szODC_TWB_END)) == L'<' ) && - ( *(ptrTwbStart-1)== L'>' ) ) { - wcscpy( szSave, szIn ) ; - usTagIndex = 0 ; - while( ptrTwbEnd < ptrTwbStart ) { - /*---------------------------------------------------------------*/ - /* See if "--twb>" is followed by a tag and "' ) ) { - /*------------------------------------------------------------*/ - /* See if the leading tag is a begin tag and if the trailing */ - /* tag is an end tag. */ - /*------------------------------------------------------------*/ - ptrChar1e = wcschr( ptrChar1s, L'>' ) ; - for( ptrChar2s=ptrChar2e ; - ptrChar2s>ptrChar1s && *ptrChar2s!=L'<' ; - --ptrChar2s ) ; - if ( ( ptrChar1e ) && - ( ptrChar2s > ptrChar1e ) ) { - if ( ( ( *(ptrChar1e-1) != L'/' ) && /* Not empty tags */ - ( *(ptrChar2s+1) == L'/' ) ) || - ( ( *(ptrChar1e-1) == L'/' ) && /* Both are empty tags */ - ( *(ptrChar2e-1) == L'/' ) ) ) { - wcsncpy( szTag1, ptrChar1s+1, ODC_TAG_SIZE ) ; - szTag1[ODC_TAG_SIZE-1] = NULL ; - wcstok( szTag1, L" />\n\t\r" ) ; - wcsncpy( szTag2, ptrChar2s+2, ODC_TAG_SIZE ) ; - szTag1[ODC_TAG_SIZE-1] = NULL ; - wcstok( szTag2, L" />\n\t\r" ) ; - /*---------------------------------------------------------*/ - /* See if the name of the leading and trailing tags are */ - /* the same. */ - /*---------------------------------------------------------*/ - if ( ( *(ptrChar1e-1) != L'/' ) && - ( ! wcscmp( szTag1, szTag2 ) ) ) { - /*---------------------------------------------------*/ - /* Matched leading and trailing tag. Make them */ - /* part of the non-translatable text surrounding */ - /* this translatable text. */ - /*---------------------------------------------------*/ - if ( usTagIndex < 19 ) { - wcscpy( szSurroundTags[usTagIndex], szTag1 ) ; - ptrSurround1[usTagIndex] = ptrTwbEnd ; - ptrSurround2[usTagIndex] = ptrTwbStart + wcslen(szODC_TWB_START) - 1 ; - ++usTagIndex ; - } - bTagsRemoved = TRUE ; - wmemmove( ptrTwbEnd, ptrChar1s, ptrChar1e-ptrChar1s+2 ) ; - ptrTwbEnd = ptrChar1e - wcslen(szODC_TWB_END) + 1 ; - wcsncpy( ptrTwbEnd, szODC_TWB_END, wcslen(szODC_TWB_END) ) ; - wmemmove( ptrChar2s+wcslen(szODC_TWB_START), ptrChar2s, ptrChar2e-ptrChar2s+2 ) ; - ptrTwbStart = ptrChar2s ; - wcsncpy( ptrTwbStart, szODC_TWB_START, wcslen(szODC_TWB_START) ) ; - continue ; - - } else - /*---------------------------------------------------------*/ - /* See if the name of the leading and trailing tags are */ - /* matching begin/end tags. For example: */ - /* bookmark-start and bookmark-end */ - /*---------------------------------------------------------*/ - if ( *(ptrChar1e-1) == L'/' ) { - for( i=0 ; i 0 ) { - for( i=0 ; i\n\t\r", *(ptrChar+wcslen(szSurroundTags[i])) ) ) ) { - bTagsRemoved = FALSE ; - ptrChar1s = ptrSurround1[i] ; - wmemmove( ptrChar1s+wcslen(szODC_TWB_END), ptrChar1s, ptrTwbEnd-ptrChar1s ) ; - ptrTwbEnd = ptrChar1s ; - wcsncpy( ptrTwbEnd, szODC_TWB_END, wcslen(szODC_TWB_END) ) ; - ptrChar2e = ptrSurround2[i] ; - ptrChar2s = ptrTwbStart + wcslen(szODC_TWB_START) ; - wmemmove( ptrTwbStart, ptrChar2s, ptrChar2e-ptrChar2s+1 ) ; - ptrTwbStart = ptrChar2e - wcslen(szODC_TWB_START ) + 1 ; - wcsncpy( ptrTwbStart, szODC_TWB_START, wcslen(szODC_TWB_START) ) ; - break ; - } - } - } - } - - - - /*---------------------------------------------------------------------*/ - /* Handle consecutive elements which do not change the */ - /* style. */ - /* */ - /* text */ - /* text */ - /* Becomes: */ - /* text text */ - /*---------------------------------------------------------------------*/ - if ( ( ptrTwbEnd ) && - ( ( ptrTwbEnd < ptrTwbStart ) || - ( ! ptrTwbStart ) ) && - ( wcsstr( ptrTwbEnd, L"' ) ; - if ( ptrChar1e ) { - for( ptrChar2s=wcschr(ptrChar1e,L'<') ; - ( ptrChar2s < ptrChar3 ) && - ( ( ! wcsncmp( ptrChar2s, L"", 9 ) ) || - ( ! wcsncmp( ptrChar2s, L"", 11 ) ) || - ( ! wcsncmp( ptrChar2s, L"", 18 ) ) || - ( ! wcsncmp( ptrChar2s, L"", 23 ) ) ) ; - ptrChar2s=wcschr(ptrChar2s+1,L'<') ) ; - if ( ( ptrChar2s ) && - ( ptrChar2s < ptrChar3 ) && - ( ! wcsncmp( ptrChar2s, L"", 12 ) ) ) { - i = ptrChar1e - ptrChar1s + 1 ; - ptrChar2 = ptrChar2s + 12 ; - if ( ! wcsncmp( ptrChar2, ptrChar1s, i ) ) { - wmemmove( ptrChar2s, ptrChar2+i, wcslen(ptrChar2+i)+1 ) ; - if ( ptrTwbStart ) - ptrTwbStart -= i + 12 ; - ptrChar3 -= i + 12 ; - --ptrChar ; - bTagsRemoved = TRUE ; - } - } - } - } - } - } - - - fputws( szIn, fOut ) ; - } - fclose( fIn ) ; - fclose( fOut ) ; - - - EQFSETSLIDER(hwndSlider, 100) ; - return(bReturn); - -} /* PreParse2 */ - - - /* Function called by EQFPOSTSEGW */ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.h b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.h index ef57ebd1..d559888c 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.h +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.h @@ -17,8 +17,6 @@ */ /* this should be called by EQFPRESEG2 */ - BOOL PreParse1(PSZ in, PSZ out, PSZ style, HWND hSlider) ; - BOOL PreParse2(PSZ in, PSZ out, HWND hSlider) ; BOOL PostParse(PSZ in, PSZ out, HWND hSlider) ; BOOL GetRcd(char *Rcd, long MaxLen, FILE *file, BOOL FullTag) ; diff --git a/source/otmd.cpp b/source/otmd.cpp index d007ad5d..05deac84 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -207,7 +207,6 @@ DEFINE_bool(enable_newlines_in_logs, false, DEFINE_bool(newEntryIfOnlyContextAndTimestampDifferent, true, "If it is set to true, the behavior should be: A new entry should be saved to the TM, if only timestamp and context are different and all other params the same;\t If it is set to false, the beahvior should be: if context is different and timestamp is different, the context and timestamp of the new to be saved segment should overwrite the context and segment of an entry, that already exists and where all other params are the same as in the new segment. This is basically the same behavior it would have, if context would not be set for both segments (the old and the new)." ); DEFINE_bool(saveDifferentTargetsForSameSource, false, "If it is set to true, when saving new segment, and found matching source and attributes, to replace previous need to have matching targets(old target = new target), if false -> previous target would be replaced with new one. In other words, true would make t5memory save multiple translation to exactly same source and attributes(document, authors etc), false would still save multiple pairs of attributes, but translation could be replaced with newer." ); //DEFINE_bool(ignoreSegIdOnUpdate, true, "If it is set to true, seg id check would be skipped. Could be useful for importing, because segId there are generated, so never match" ); - DEFINE_bool(suppressTmVersionCheck, false, "If sets to true, allow to open tms, which was created in version, that's outside of supported versions scope. You can use this if you have version mismatch error"); DEFINE_int32(t5globversion_min_supported, 0, "sets minimal global version in which tm was created to be supported for open in current. it's the X(GLOB) in vX.Y.Z"); @@ -397,7 +396,10 @@ int main(int argc, char* argv[]) { // Install the custom LogSink //static NoNewlineLogSink sink; //google::AddLogSink(&sink); - T5LOG(T5TRANSACTION) <<"Worker thread starting, v = "< Date: Fri, 1 Aug 2025 16:11:17 +0300 Subject: [PATCH 45/62] * updated to 0.7.18 - O3 build * fixed tables crash --- source/CMakeLists.txt | 2 +- source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index a71c36b9..463c2d76 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 17) +SET(APP_VERSION_MINOR 18) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index fe77dc5c..35fa22d2 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -573,7 +573,7 @@ USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput fLongFound = TRUE; if(maxBuffLen>0){ - strncpy(pszName, pstTMTableEntries[ulI].szName, maxBuffLen-1); + strncpy(pszName, LongNames.stTableEntry[ulI].pszLongName, maxBuffLen-1); pszName[maxBuffLen-1] = '\0'; }else{ strcpy(pszLongName, From 7c71997225095f9f34a7a6262874629c746f84d7 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Fri, 1 Aug 2025 22:23:38 +0300 Subject: [PATCH 46/62] * updated to v0.7.19- debug * implemented reorganize only mode for tm that throw BTREE_CORRUPTED on loading * disabled short filename * fixed langgroup in tables request - code cleanup --- include/lowlevelotmdatastructs.h | 1 - include/tm.h | 2 - source/CMakeLists.txt | 10 ++-- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 58 ++++++++----------- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 42 +++++++------- .../core/EqfMemoryPlugin/EqfMemory.cpp | 2 +- source/opentm2/core/requestdata.cpp | 10 +++- source/opentm2/core/utilities/EQFDICT.cpp | 28 --------- 8 files changed, 59 insertions(+), 94 deletions(-) diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 692401ee..938b27b6 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -589,7 +589,6 @@ struct BTREE char chEQF[7]{}; // The type of file char bVersion=0; // version flag ushort usOpenFlags=0; // settings used for open - HFILE fpDummy=nullptr; // dummy/lock semaphore file handle ushort usNumberOfAllocatedBuffers=0; // Number of allocated buffers unsigned long ulReadRecCalls=0; // Number of calls to QDAMReadRecord char bRecSizeVersion=0; // record size version flag diff --git a/include/tm.h b/include/tm.h index 80da335e..7c9c555e 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1794,8 +1794,6 @@ class EqfMemory //: public TMX_CLB { - USHORT LoadMem(); - public: std::recursive_timed_mutex tmMutex; // bool fOpen = false; diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 463c2d76..295f2a82 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 18) +SET(APP_VERSION_MINOR 19) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) @@ -14,8 +14,8 @@ SET(APP_VERSION "${APP_VERSION_GLOBAL}.${APP_VERSION_MAJOR}.${APP_VERSION_MINOR project(T5MEMORY VERSION ${APP_VERSION}) -set(CMAKE_BUILD_TYPE RelWithDebInfo) -#set(CMAKE_BUILD_TYPE Debug) +#set(CMAKE_BUILD_TYPE RelWithDebInfo) +set(CMAKE_BUILD_TYPE Debug) #add_compile_options(-Wall -Wextra -Wpedantic) add_compile_options( @@ -53,8 +53,8 @@ option(ENABLE_ASAN "Enable AddressSanitizer" OFF) option(ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF) # Sanitizer options -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") +#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3") +#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") # Optionally enable ASan diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index b31ce4b7..a641db9c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -156,15 +156,7 @@ std::string EqfMemory::getActiveRequestString()const //| return open out structure | // ----------------------------------------------------------------------------+ DECLARE_bool(allowLoadingMultipleTmsSimultaneously); -USHORT EqfMemory::Load(){ - USHORT usRC = LoadMem(); - // create memory object if create function completed successfully - if ( (usRC != 0) && (usRC != BTREE_CORRUPTED) /*&& (usAccessMode == FOR_ORGANIZE)*/){ - T5LOG(T5ERROR) << "EqfMemoryPlugin::openMemory:: TmOpen fails, fName = "<< szName<< "; error = "<< this->strLastError<<"; iLastError = "<< - this->iLastError << "; rc = " << usRC; - } - return usRC; -} + DECLARE_int32(triplesthreshold); @@ -176,7 +168,7 @@ DECLARE_int32(t5globversion_max_supported); DECLARE_int32(t5majversion_max_supported); DECLARE_int32(t5minversion_max_supported); -USHORT EqfMemory::LoadMem() +USHORT EqfMemory::Load() { //TimedMutexGuard l(tmMutex); if(isLoaded()) @@ -253,20 +245,21 @@ USHORT EqfMemory::LoadMem() usRc = NO_ERROR; } /* endif */ - if ( (usRc == NO_ERROR) || (usRc == VERSION_MISMATCH) ) + if(usRc == NO_ERROR){ + usRc = usRc1; + } + + if ( (usRc == NO_ERROR) || (usRc == BTREE_CORRUPTED) ) { - if ( (usRc == NO_ERROR) || - (usRc == BTREE_CORRUPTED) || - (usRc == VERSION_MISMATCH) ) { //get compact area and add to control block LONG lKey = COMPACT_KEY; USHORT usTempRc = TmBtree.EQFNTMGet( lKey, bCompact ); // in organize mode allow continue if compact area is corrupted - if ( (usTempRc != NO_ERROR) && (usTempRc != VERSION_MISMATCH) ) + if ( usTempRc != NO_ERROR)//&& (usTempRc != VERSION_MISMATCH) ) { - {LOG_AND_SET_RC_W_INFO(usTempRc, T5WARNING, BTREE_CORRUPTED) << "if ( (usTempRc != NO_ERROR) && (usTempRc != VERSION_MISMATCH) ); usTempRc= " <stTmTableEntry.size() && !fFound; ulI++) - { - if ( usID == pstTMTableEntries[ulI].usId) + if(usTableType != FILE_KEY){ + for (ULONG ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) { - fFound = TRUE; - constexpr int sizeOfField = MAX_LANG_LENGTH ; - strncpy(pszName, pstTMTableEntries[ulI].szName, sizeOfField-1); - pszName[sizeOfField-1] = '\0'; - } /* endif */ - } /* endfor */ - /**************************************************************/ - /* Get any long document name for this ID */ - /**************************************************************/ - if (fFound && (usTableType == FILE_KEY) && (pszLongName != NULL)) + if (usID && usID == pstTMTableEntries[ulI].usId) + { + fFound = TRUE; + constexpr int sizeOfField = MAX_LANG_LENGTH ; + strncpy(pszName, pstTMTableEntries[ulI].szName, sizeOfField-1); + pszName[sizeOfField-1] = '\0'; + } /* endif */ + } /* endfor */ + }else{ + /**************************************************************/ + /* Get any long document name for this ID */ + /**************************************************************/ + if ( (pszLongName != NULL)) { - BOOL fLongFound = FALSE; + //BOOL fLongFound = FALSE; for (ULONG ulI = 0; - ulI < LongNames.stTableEntry.size() && !fLongFound; + ulI < LongNames.stTableEntry.size() && !fFound; ulI++) { - if (usID == LongNames.stTableEntry[ulI].usId) + if (usID && usID == LongNames.stTableEntry[ulI].usId) { - fLongFound = TRUE; + fFound = TRUE; if(maxBuffLen>0){ strncpy(pszName, LongNames.stTableEntry[ulI].pszLongName, maxBuffLen-1); @@ -582,20 +584,20 @@ USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput } /* endif */ } /* endfor */ - if (!fLongFound) + if (!fFound) { strcpy(pszLongName, pszName); } } /* endif */ - + } if (!fFound) { /************************************************************/ /* the ID was not found in the table */ /* set usRc and reset pszName */ /************************************************************/ - LOG_AND_SET_RC_W_INFO(usRc, T5ERROR, ID_NOT_FOUND) << "id = " << usID; if(usID){ + LOG_AND_SET_RC_W_INFO(usRc, T5ERROR, ID_NOT_FOUND) << "id = " << usID; snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", usID); // 'hu' = unsigned short }else{ pszName[0] = EOS; diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index ddecb2f3..62053a36 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -693,7 +693,7 @@ int EqfMemory::Reload(bool fReadFromDisk){ eStatus = FAILED_TO_OPEN_STATUS; }else{ eStatus = WAITING_FOR_LOADING_STATUS; - rc = LoadMem(); + rc = Load(); } return rc; } diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 9bf26b06..b24fc6ff 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -338,10 +338,16 @@ int RequestData::requestTM(){ } if(mem->isWaitingToBeLoaded() || mem->isFailedToLoad()){ + USHORT accessModeBeforeOpenning = mem->usAccessMode; + if(REORGANIZE_MEM == command || DELETE_ENTRIES_REORGANIZE == command){ + mem->usAccessMode |= ASD_ORGANIZE; + } _rc_ = mem->Load(); + mem->usAccessMode=accessModeBeforeOpenning; + if(_rc_){ std::string msg = "Failed to load tm, rc = "; - msg += _rc_; + msg += toStr(_rc_); if(11==_rc_){ msg += "; file on the disk is locked;"; } @@ -2254,7 +2260,7 @@ int GetTablesMemRequestData::execute() { i = 0; for(const auto& entry: mem->LangGroups.stTmTableEntry){ - langGroupTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName[0] + "\"; "; + langGroupTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; } i = 0; diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 75e8135f..6b70ca61 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -1410,32 +1410,7 @@ SHORT BTREE::QDAMDictFlushLocal() SHORT BTREE::QDAMDictClose() { SHORT sRc = 0; // error return - - // reset open flag in header and force a write to disk - // open flag will only be set if opened for r/w - { - //sRc = QDAMDictFlushLocal(); - //fOpen = FALSE; - // re-write header record - //if ( sRc == NO_ERROR ) - // sRc = QDAMWriteHeader(); - } /* endif */ - - //if ( UtlClose(fb.file, FALSE) && !sRc ) - //{ - // sRc = BTREE_CLOSE_ERROR; - //} /* endif */ - - if ( fpDummy ) - { - UtlClose( fpDummy, FALSE ); - } /* endif */ fb.data.resize(0); - - /*******************************************************************/ - /* free the allocated buffers */ - /*******************************************************************/ - freeLookupTable(); return sRc; @@ -1808,8 +1783,6 @@ SHORT BTREE::QDAMDictCreateLocal usFreeKeyBuffer = 0; usFreeDataBuffer = 0; usFirstDataBuffer = 0; // first data buffer - //fOpen = TRUE; // open flag set - fpDummy = NULLHANDLE; /******************************************************************/ /* do settings depending if we are dealing with a dict or a tm.. */ @@ -4741,7 +4714,6 @@ SHORT BTREE::QDAMDictOpenLocal usFreeKeyBuffer = header.usFreeKeyBuffer; usFreeDataBuffer = header.usFreeDataBuffer; usFirstDataBuffer = header.usFirstDataBuffer; // data buffer - fpDummy = NULLHANDLE; // load usNextFreeRecord either from header record of from file info { From 372f9799acffe45ed7fdc91ee53c4dd3e7780559 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 5 Aug 2025 21:26:17 +0300 Subject: [PATCH 47/62] * updated to v0.7.20 - removed-disabled short filename functionality + added fcorrupted flag * implemented tmx export for BTREE_CORRUPTED on openning * code cleanup and refactoring --- include/EQF.H | 29 -- include/EQFTPI.H | 14 - include/tm.h | 27 +- source/CMakeLists.txt | 3 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 53 +-- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 6 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 18 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 47 +-- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 1 + .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 15 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 369 ++++++------------ .../core/EqfMemoryPlugin/EqfMemory.cpp | 10 +- .../core/EqfMemoryPlugin/TMPluginWrapper.cpp | 7 - source/opentm2/core/requestdata.cpp | 20 +- source/opentm2/core/tm.cpp | 4 +- source/opentm2/core/utilities/EQFHASH.cpp | 33 +- source/opentm2/core/utilities/EQFTADIT.cpp | 0 source/opentm2/core/utilities/EQFUTDOS.cpp | 44 --- .../core/utilities/FilesystemWrapper.h | 25 -- .../core/utilities/LanguageFactory.cpp | 4 - 20 files changed, 221 insertions(+), 508 deletions(-) delete mode 100755 source/opentm2/core/utilities/EQFTADIT.cpp diff --git a/include/EQF.H b/include/EQF.H index 6af728f9..7d23aa69 100755 --- a/include/EQF.H +++ b/include/EQF.H @@ -555,22 +555,6 @@ } -#ifdef TEMPORARY_COMMENTED -#define CBQUERYSELECTEDITEMTEXT( sItem, hwnd, id, szBuffer ) \ - { \ - sItem = (SHORT)SendDlgItemMessage( hwnd, id, CB_GETCURSEL, 0, 0L ); \ - if( sItem != CB_ERR ) \ - { \ - SendDlgItemMessage( hwnd, id, CB_GETLBTEXT, MP1FROMSHORT( sItem ), \ - MP2FROMP( szBuffer ) ); \ - } \ - else \ - { \ - szBuffer [0] = EOS; \ - } /* endif */ \ - } -#endif - #define CBQUERYSELECTEDITEMTEXTHWND( sItem, hwnd, szBuffer ) \ { \ sItem = (SHORT)CPLUSPLUS SendMessage( hwnd, CB_GETCURSEL, 0, 0L ); \ @@ -5639,19 +5623,6 @@ BOOL DicGetCheckPassword( PSZ pszPassword, PULONG pulPassword, BOOL fCheckPasswo #define XLATEOS2_MARKUP "XLATEOS2" #define XLATEWIN_MARKUP "XLATEWIN" -// support additional pages in document import/export -typedef struct _EXTRAIDA -{ - CHAR szFileName[ MAX_PATH144 ]; // filename to be used - HWND hwndExpListbox; // handle of listbox containing selected docs - OBJNAME szFolderObjName; // folder object name - PVOID pIda; // product specifics - USHORT usExtraPage; // number of extra page - HWND hwnd; // handle of parent dialog - OBJNAME szParentObjName; // parent (sub)folder object name -} EXTRAIDA, *PEXTRAIDA; - - // defines for SGML Formats #define SGMLFORMAT_ANSI 1 diff --git a/include/EQFTPI.H b/include/EQFTPI.H index a7d90ced..ac0ed078 100755 --- a/include/EQFTPI.H +++ b/include/EQFTPI.H @@ -1482,20 +1482,6 @@ typedef struct _USERSET } USERSET, *PUSERSET; -// Profile ONE Dialog IDA (initial values) -typedef struct _PROFONEIDA -{ - USERSET LBItemArray [MAXLBITEMS]; // text of listbox - CHAR szBuffer [60]; // get choice from listbox - PTBDOCUMENT pDoc; - HBITMAP hbm; // menucheck bitmap - CHAR szStyleArray [MAXSTYLES][40]; // text for prop.wnd.styles - CHAR szMarginBuffer[RMARGIN_LENGTH + 1]; // text for rmargin cbs - HWND hwndPages[10]; // support up to 10 prop pages -#ifdef R004422_BACKSAVE - CHAR szMinute[5]; // text for minutes CBS -#endif -} PROFONEIDA, *PPROFONEIDA; // TOC goto diealog IDA typedef struct _TOCGOTOIDA diff --git a/include/tm.h b/include/tm.h index 7c9c555e..f387650e 100644 --- a/include/tm.h +++ b/include/tm.h @@ -72,7 +72,6 @@ struct TMX_EXT_OUT_W CHAR szTargetLanguage[MAX_LANG_LENGTH]{}; // language name of target CHAR szAuthorName[MAX_FILESPEC]{}; // author name of target USHORT usTranslationFlag = 0; /* type of translation, 0 = human, 1 = machine, 2 = Global Memory */ - CHAR szFileName[MAX_FILESPEC]{}; // where source comes from name+ext LONG_FN szLongName{}; // name of source file (long name or EOS) ULONG ulSourceSegmentId = 0; // seg. num. of source sentence from analysis TIME_L lTargetTime = 0; // time stamp of target @@ -1667,7 +1666,6 @@ typedef struct _TMX_GET_W CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence CHAR szTagTable[MAX_FNAME]{}; // tag table name of source CHAR szSourceLanguage[MAX_LANG_LENGTH]{}; // language of source - CHAR szFileName[MAX_FILESPEC]{}; // file name the source comes from LONG_FN szLongName{}; // name of source file (long name or EOS) ULONG ulSegmentId = 0; // segment number from analysis CHAR szAuthorName[MAX_LANG_LENGTH]{}; // author name @@ -1691,7 +1689,6 @@ typedef struct _TMX_GET_W struct TMX_MATCH_TABLE_W { CHAR_W szSource[MAX_SEGMENT_SIZE]{}; // source sentence with tags - CHAR szFileName[MAX_FILESPEC]{}; // file name where the source comes from. LONG_FN szLongName{}; // name of source file (long name or EOS) ULONG ulSegmentId{}; // segment number from analysis CHAR_W szTarget[MAX_SEGMENT_SIZE]{}; // target sentence with tags @@ -1793,7 +1790,7 @@ class EqfMemory //: public TMX_CLB */ { - + bool fCorrupted = false;; public: std::recursive_timed_mutex tmMutex; // bool fOpen = false; @@ -1908,6 +1905,10 @@ class EqfMemory //: public TMX_CLB return FAILED_TO_OPEN_STATUS == eStatus; } + bool isCorrupted()const{ + return fCorrupted; + } + bool isReorganizeRunning() const { return REORGANIZE_RUNNING_STATUS == eStatus; @@ -1928,6 +1929,7 @@ class EqfMemory //: public TMX_CLB { unsigned int mask = ~ASD_ORGANIZE; usAccessMode &= mask; + fCorrupted = false; } void updateLastUseTime() @@ -2022,16 +2024,11 @@ class EqfMemory //: public TMX_CLB bool areLangsMatching(const ushort langId1, const ushort langId2, const char* langStr1);//, bool fPrefferedLang1); - USHORT NTMGetIDFromName(PSZ, PSZ, USHORT, PUSHORT); - USHORT NTMGetIDFromNameEx( - PSZ pszName, // input, name being looked up - PSZ pszLongName, // input, long name (only for FILE_KEY) - USHORT usTableType, // input, type of table to use - PUSHORT pusID, // output, ID for name being looked up - LONG lOptions, - PUSHORT pusAlternativeID // output, alternative ID - ); - USHORT NTMGetNameFromID(const USHORT, USHORT, PSZ, PSZ, SHORT maxBuffLen = -1); + USHORT NTMGetIDFromName(PSZ, USHORT, PUSHORT, + LONG options = 0, + PUSHORT pusAlternativeID = nullptr); + + USHORT NTMGetNameFromID(const USHORT, USHORT, PSZ, SHORT maxBuffLen = -1); USHORT NTMGetPointersToTable(USHORT, PTMX_TABLE *, PTMX_TABLE_ENTRY *); USHORT NTMAddNameToTable(PSZ, USHORT, PUSHORT); @@ -2611,7 +2608,7 @@ USHORT MorphAddTermToList2W( ULONG ulFlags // flags for the ter ); -BOOL NTMDocMatch(PSZ pszShort1, PSZ pszLong1, PSZ pszShort2, PSZ pszLong2); +BOOL NTMDocMatch(PSZ pszLong1, PSZ pszLong2); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /*! tm.h Internal header file for Translation Memory functions diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 295f2a82..4e8212de 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 19) +SET(APP_VERSION_MINOR 20) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) @@ -172,7 +172,6 @@ set(T5MEMORY_Source_Files "${CORE_DIR}/utilities/EqfPluginWrapper.cpp" "${CORE_DIR}/utilities/EQFPRO00.cpp" "${CORE_DIR}/utilities/EQFSETUP.cpp" - "${CORE_DIR}/utilities/EQFTADIT.cpp" "${CORE_DIR}/utilities/EQFUTDOS.cpp" "${CORE_DIR}/utilities/EQFUTERR.cpp" "${CORE_DIR}/utilities/EQFUTFIL.cpp" diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index fa215aae..d2609692 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -570,17 +570,13 @@ USHORT MemFuncPrepImport /* it will be freed here. */ /*******************************************************************/ memset(pLoadIDA, 0, sizeof(*pLoadIDA)); - if ( fOK ) - { // use def.target lang. from system preferences of TM - pLoadIDA->ulOemCP = 1; - pLoadIDA->ulAnsiCP = 1; - }else if(!fOkFailHandled){ - fOkFailHandled = true; - T5LOG(T5ERROR) << "MemFuncPrepImport:: memory allocation fails"; - } - + //if ( fOK ) + { // use def.target lang. from system preferences of TM + pLoadIDA->ulOemCP = 1; + pLoadIDA->ulAnsiCP = 1; + } + // set options for import - if ( fOK ) { pLoadIDA->fSkipInvalidSegments = ( lOptions & IGNORE_OPT ); pLoadIDA->fAscii = TRUE; // TM in external format @@ -630,39 +626,21 @@ USHORT MemFuncPrepImport } /* endif */ // prepare segment match ID prefix - if ( fOK ){ - pLoadIDA->szMatchIDPrefix[0] = '\0'; - }else if(!fOkFailHandled){ - fOkFailHandled = true; - T5LOG(T5ERROR) << "MemFuncPrepImport::set options for import fails"; - } /* endif */ - + pLoadIDA->szMatchIDPrefix[0] = '\0'; + /*******************************************************************/ /* Check if there is a TM with the given name */ /*******************************************************************/ - if ( fOK ) - { - strcpy( pLoadIDA->szMemName, pData->mem.get()->szName.c_str() ); - }else if(!fOkFailHandled){ - fOkFailHandled = true; - T5LOG(T5ERROR) << "MemFuncPrepImport::prepare segment match ID prefix fails"; - } /* endif */ - + strcpy( pLoadIDA->szMemName, pData->mem->szName.c_str() ); /*******************************************************************/ /* Check if input file exists */ /*******************************************************************/ - if ( fOK ) - { - if ( pData->fileData.empty() && !UtlFileExist( pszInFile ) ) - { - pszParm = pszInFile; - T5LOG(T5ERROR) << "::FILE_NOT_EXISTS::" << pszParm; - fOK = FALSE; - } /* endif */ - }else if(!fOkFailHandled){ - fOkFailHandled = true; - T5LOG(T5ERROR) << "MemFuncPrepImport::Open the TM fails"; + if ( pData->fileData.empty() && !UtlFileExist( pszInFile ) ) + { + pszParm = pszInFile; + T5LOG(T5ERROR) << "::FILE_NOT_EXISTS::" << pszParm; + fOK = FALSE; } /* endif */ /*******************************************************************/ @@ -670,7 +648,6 @@ USHORT MemFuncPrepImport /*******************************************************************/ if ( fOK ) { - //USHORT usDosRc; // Return code from Dos functions USHORT usAction; // Action for DosOpen if(pData->fileData.empty()){ strcpy( pLoadIDA->szFilePath, pszInFile ); @@ -707,7 +684,7 @@ USHORT MemFuncPrepImport T5LOG(T5ERROR) << "MemFuncPrepImport::fOk in false, making Cleanup"; if ( pLoadIDA ) { - if ( pLoadIDA->hFile && pLoadIDA->hFile ) + if ( pLoadIDA->hFile ) { UtlClose( pLoadIDA->hFile, FALSE ); } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 25adcf14..c84f6898 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -299,7 +299,6 @@ USHORT EqfMemory::FindTargetAndDelete( //get id of target language in the put structure usRc = NTMGetIDFromName( TmDel.szTargetLanguage, - NULL, (USHORT)LANG_KEY, &usId ); //compare target language ids if ( (pClb->usLangId == usId) && !usRc ) @@ -336,7 +335,6 @@ USHORT EqfMemory::FindTargetAndDelete( //get id of filename in the put structure usRc = NTMGetIDFromName( TmDel.szDocName, - TmDel.szDocName, (USHORT)FILE_KEY, &usId ); if ( !usRc ) { @@ -365,7 +363,7 @@ USHORT EqfMemory::FindTargetAndDelete( TmDel.pInputSentence->pStrings->getGenericTargetStr(), &lSrcLen, pTmExtOut, endOfMemRegion ); NTMGetNameFromID(pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); + pTmExtOut->szOriginalSourceLanguage ); pTmExtOut->ulRecKey = lKey; TmDel.currentInternalKey.setInternalKey(lKey, usTarget); pTmExtOut->usTargetKey = usTarget; @@ -578,7 +576,7 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( pSourceString, &lSourceLen, pTmExtOut, endOfMemRegion ); NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); + pTmExtOut->szOriginalSourceLanguage ); TmDel.currentInternalKey.setInternalKey(lKey, usTarget); pTmExtOut->ulRecKey = lKey; pTmExtOut->usTargetKey = usTarget; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 998bcc36..4bed559e 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -287,7 +287,7 @@ USHORT EqfMemory::ExtractRecord pSourceString, &lSourceLen, pTmExtOut, endOfMemRegion); NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, - pTmExtOut->szOriginalSourceLanguage, NULL ); + pTmExtOut->szOriginalSourceLanguage ); if ( ! usRc ) { //check for another target @@ -446,27 +446,27 @@ USHORT EqfMemory::FillExtStructure // values as here the corresponding tables may be corrupted if ( usAccessMode & ASD_ORGANIZE ) { - strcpy( pstExt->szFileName, OVERFLOW_NAME ); + strcpy( pstExt->szLongName, OVERFLOW_NAME ); pstExt->szLongName[0] = EOS; strcpy( pstExt->szAuthorName, OVERFLOW_NAME ); } /* endif */ //fill in the target file name NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, - pstExt->szFileName, pstExt->szLongName, sizeof(pstExt->szFileName) ); + pstExt->szLongName, sizeof(pstExt->szLongName) ); //use overflow name if no document name available - if ( pstExt->szFileName[0] == EOS ) - { - strcpy( pstExt->szFileName, OVERFLOW_NAME ); - } /* endif */ + //if ( pstExt->szLongName[0] == EOS ) + //{ + // strcpy( pstExt->szLongName, OVERFLOW_NAME ); + //} /* endif */ //fill in the target author NTMGetNameFromID( pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, - pstExt->szAuthorName, NULL ); + pstExt->szAuthorName ); //fill in the target language NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, - pstExt->szTargetLanguage, NULL ); + pstExt->szTargetLanguage ); //fill in the segment id diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 0aac69c9..c85ae1fe 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -323,12 +323,8 @@ USHORT EqfMemory::TmtXGet { PTMX_MATCH_TABLE_W pMatch = &(pTmGetOut->stMatchTable[i]); T5LOG( T5DEVELOP) << "TmtXGet:: Match "<< i << " , MatchLevel = "<< pMatch->usMatchLevel<< - " , MFlag = " << pMatch->usTranslationFlag << " , segment = " << pMatch->ulSegmentId; - if(pMatch->szLongName[0] != '0') { - T5LOG( T5DEVELOP)<<"TmtXGet:: origin = "<szFileName; - }else{ - T5LOG( T5DEVELOP)<<"TmtXGet:: origin = "<szLongName; - } + " , MFlag = " << pMatch->usTranslationFlag << " , segment = " << pMatch->ulSegmentId<< "; origin = "<szLongName; + auto strSource = EncodingHelper::convertToUTF8(pMatch->szSource); auto strTarget = EncodingHelper::convertToUTF8(pMatch->szTarget); @@ -669,7 +665,7 @@ USHORT ExactTest pByte += pTMXSourceRecord->usSource; //get id of target language in the getin structure USHORT usSrcLangId = 0; - if (pTmClb->NTMGetIDFromName( pGetIn->szSourceLanguage, NULL, + if (pTmClb->NTMGetIDFromName( pGetIn->szSourceLanguage, (USHORT)LANG_KEY, &usSrcLangId)) { usSrcLangId = 0; @@ -700,20 +696,18 @@ USHORT ExactTest //get id of target language in the getin structure if (pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, - NULL, (USHORT)LANG_KEY, &usGetLang)) { usGetLang = 0; } /* endif */ //get file name id of file name in the getin structure - if ( pTmClb->NTMGetIDFromNameEx( pGetIn->szFileName, - pGetIn->szLongName, + if ( pTmClb->NTMGetIDFromName( pGetIn->szLongName, (USHORT)FILE_KEY, &usGetFile, - NTMGETID_NOUPDATE_OPT,//0, + NTMGETID_NOUPDATE_OPT, &usAlternateGetFile )) { - usGetFile = 1; + usGetFile = 0; } /* endif */ /****************************************************************/ @@ -1248,7 +1242,6 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct USHORT usContextRanking, int iWords, int iDiffs, USHORT usSrcLangId, PBYTE endOfMemRegion) { - PSZ pSrcFileName = pGetIn->szFileName; // source file name PSZ pSrcLongName = pGetIn->szLongName; // long source file name PBYTE pByte; //position pointer USHORT usRc = NO_ERROR; //return code @@ -1296,7 +1289,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct //fill in the target language pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, - SubstProp.szTargetLanguage, NULL ); + SubstProp.szTargetLanguage ); } /* endif */ } /* endif */ @@ -1320,11 +1313,10 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct } else if ( iMatchSortKey == iNewMatchSortKey ) { - CHAR szTgtFileName[ MAX_FILESPEC ]; // target file name CHAR szTgtLongName[ MAX_LONGFILESPEC ]; // long target file name //fill in the target file name - pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, szTgtFileName, szTgtLongName, sizeof(szTgtLongName) ); + pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, szTgtLongName, sizeof(szTgtLongName) ); /****************************************************************/ /* as the matching levels are the same determine the most recent*/ @@ -1332,15 +1324,14 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /* higher priority */ /****************************************************************/ - if ( NTMDocMatch( szTgtFileName, szTgtLongName, pSrcFileName, pSrcLongName ) ) + if ( NTMDocMatch( szTgtLongName, pSrcLongName ) ) { /**************************************************************/ /* if compared match is from same file, check the update time */ /* because now we deal with two matches from same source doc. */ /**************************************************************/ if ( (pstMatchTable->lTargetTime > pTMXTargetClb->lTime) && - NTMDocMatch( pstMatchTable->szFileName, pstMatchTable->szLongName, - pSrcFileName, pSrcLongName ) ) + NTMDocMatch( pstMatchTable->szLongName, pSrcLongName ) ) { pstMatchTable++; usCurrentEntry++; @@ -1368,7 +1359,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct /* otherwise put in our match, it's coming from same source */ /* document */ /**************************************************************/ - if ( NTMDocMatch( pstMatchTable->szFileName, pstMatchTable->szLongName, pSrcFileName, pSrcLongName ) ) + if ( NTMDocMatch( pstMatchTable->szLongName, pSrcLongName ) ) { // match from match table is from same document, so skip to next position pstMatchTable++; @@ -1435,12 +1426,11 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct //fill in the target file name pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, - pstMatchTable->szFileName, pstMatchTable->szLongName, sizeof(pstMatchTable->szLongName)); //fill in the target author - don't care about author name.... pTmClb->NTMGetNameFromID( pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, - pstMatchTable->szTargetAuthor, NULL ); + pstMatchTable->szTargetAuthor ); //fill in the markup table strcpy(pstMatchTable->szTagTable, "OTMXUXLF"); @@ -1448,11 +1438,11 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct //fill in the target and original src language pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, - pstMatchTable->szTargetLanguage, NULL ); + pstMatchTable->szTargetLanguage ); pTmClb->NTMGetNameFromID( usSrcLangId, (USHORT)LANG_KEY, - pstMatchTable->szOriginalSrcLanguage, NULL ); + pstMatchTable->szOriginalSrcLanguage ); //fill in the segment id pstMatchTable->ulSegmentId = pTMXTargetClb->ulSegmId; //state whether machine translation @@ -1496,8 +1486,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct if ((iMatchSortKeyTemp < iMatchSortKey) || ((iMatchSortKeyTemp == iMatchSortKey) && (pstMatchTableTemp->lTargetTime < pstMatchTable->lTargetTime) && - NTMDocMatch( pstMatchTable->szFileName, pstMatchTable->szLongName, - pstMatchTableTemp->szFileName, pstMatchTableTemp->szLongName)) ) + NTMDocMatch( pstMatchTable->szLongName, pstMatchTableTemp->szLongName)) ) { // interchange entries pstMatchTable moved on position pstMatchTableTemp TMX_MATCH_TABLE_W stMatchEntry; @@ -1679,7 +1668,7 @@ bool EqfMemory::areLangsMatching(const ushort langId1, const ushort langId2, return true; } char langStr2[MAX_LANG_LENGTH]{}; - NTMGetNameFromID( langId2, (USHORT)LANG_KEY, langStr2, NULL ); + NTMGetNameFromID( langId2, (USHORT)LANG_KEY, langStr2 ); return !LanguageFactory::getInstance()->hasSubLocale(langStr2); } return false; @@ -1752,7 +1741,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block USHORT srcLangId = 0; // we have to update the memory language table to keep the language group table up-to-date... - if ( pTmClb->NTMGetIDFromName( pGetIn->szSourceLanguage, NULL, (USHORT)LANG_KEY, &srcLangId)) + if ( pTmClb->NTMGetIDFromName( pGetIn->szSourceLanguage, (USHORT)LANG_KEY, &srcLangId)) { srcLangId = 0; // set default.. } /* endif */ @@ -1812,7 +1801,7 @@ USHORT FuzzyTest ( EqfMemory* pTmClb, //ptr to control block LONG lTempSrcLen; // temp. source length // we have to update the memory language table to keep the language group table up-to-date... - if ( pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTargetId)) + if ( pTmClb->NTMGetIDFromName( pGetIn->szTargetLanguage, (USHORT)LANG_KEY, &usTargetId)) { usTargetId = 0; // set default.. } /* endif */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index a641db9c..863ae629 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -431,6 +431,7 @@ USHORT EqfMemory::Load() /* database files have been opened */ /********************************************************************/ if((usRc == BTREE_CORRUPTED) && isOpenedOnlyForReorganize()){ + fCorrupted = true;//could be opened only for exportTMX or reorganize, or mass deletion(also reorganize) usRc = NO_ERROR; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 10340627..256d0c7d 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -754,7 +754,7 @@ USHORT EqfMemory::AddToTm if ( usRc == NO_ERROR ) { USHORT usSrcLang = 0; - usRc = NTMGetIDFromName( TmProposal.szSourceLanguage, NULL, (USHORT)LANG_KEY, &usSrcLang ); + usRc = NTMGetIDFromName( TmProposal.szSourceLanguage, (USHORT)LANG_KEY, &usSrcLang ); TmRecord.resize(TMX_REC_SIZE); //fill tm record to add to database FillTmRecord ( TmProposal.pInputSentence.get(), // ptr to sentence struct for source info @@ -924,18 +924,18 @@ USHORT EqfMemory::FillClb USHORT usRc = NO_ERROR; // returned value //get id of target language, call - usRc = NTMGetIDFromName( TmProposal.szTargetLanguage, NULL, (USHORT)LANG_KEY, &usTrgLang ); + usRc = NTMGetIDFromName( TmProposal.szTargetLanguage, (USHORT)LANG_KEY, &usTrgLang ); //get id of file name, call if ( !usRc ) { - usRc = NTMGetIDFromName( TmProposal.szDocName, TmProposal.szDocName, (USHORT)FILE_KEY, &usFile ); + usRc = NTMGetIDFromName( TmProposal.szDocName,(USHORT)FILE_KEY, &usFile ); } /* endif */ //get id of target author if ( !usRc ) { - usRc = NTMGetIDFromName( TmProposal.szTargetAuthor, NULL, (USHORT)AUTHOR_KEY, &usAuthor ); + usRc = NTMGetIDFromName( TmProposal.szTargetAuthor,(USHORT)AUTHOR_KEY, &usAuthor ); } /* endif */ if ( !usRc ) @@ -1630,17 +1630,16 @@ USHORT EqfMemory::ComparePutData BOOL fTargetMatch = true;//default behaviour //get id of target language in the put structure - if (NTMGetIDFromName( TmProposal.szTargetLanguage, - NULL, (USHORT)LANG_KEY, &usPutLang )) + if (NTMGetIDFromName( TmProposal.szTargetLanguage,(USHORT)LANG_KEY, &usPutLang )) { usPutLang = 0; } /* endif */ NTMGetIDFromName( TmProposal.szTargetAuthor, // get author ID - NULL, (USHORT)AUTHOR_KEY, &usAuthorId ); + (USHORT)AUTHOR_KEY, &usAuthorId ); usRc = NTMGetIDFromName( TmProposal.szDocName, // get file id - TmProposal.szDocName, (USHORT)FILE_KEY, &usPutFile ); + (USHORT)FILE_KEY, &usPutFile ); if ( !usRc ) { //position at beginning of source structure in tm record diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index d6845584..def4ac58 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -207,28 +207,17 @@ int NTMLongNameTableCompCaseIgnore( const void *, const void * ); //| | //| return usRc; | // ----------------------------------------------------------------------------+ -USHORT -EqfMemory::NTMGetIDFromName( - PSZ pszName, // input - PSZ pszLongName, // input, long name (only for FILE_KEY) - USHORT usTableType, // input - PUSHORT pusID) // output -{ - return (NTMGetIDFromNameEx(pszName, pszLongName, usTableType, - pusID, 0, NULL)); -} /* end of function NTMGetIDFromName */ -// NTMGetIDFromNameEx -// this is an enhanced version of NTMGetIDFromName + +// NTMGetIDFromName // using the new option NTMGETID_NOUPDATE_OPT the update of the name table can // suppressed for new names (important for get operations which should // not change the database) // the pusAlternativeID is filled with the ID of the name in the long name // table if no pszLongName has been specified and the pszName is found in // the short name table as well -USHORT EqfMemory::NTMGetIDFromNameEx( +USHORT EqfMemory::NTMGetIDFromName( PSZ pszName, // input, name being looked up - PSZ pszLongName, // input, long name (only for FILE_KEY) USHORT usTableType, // input, type of table to use PUSHORT pusID, // output, ID for name being looked up LONG lOptions, @@ -239,7 +228,6 @@ USHORT EqfMemory::NTMGetIDFromNameEx( PTMX_TABLE_ENTRY pstTMTableEntry = NULL; // ptr to a table entry for bsearch PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry PTMX_TABLE pstTMTable = NULL; // ptr to table structure - BOOL fLongName = FALSE; // initialize ID if (lOptions & NTMGETID_NOUPDATE_OPT) @@ -254,192 +242,110 @@ USHORT EqfMemory::NTMGetIDFromNameEx( if (pusAlternativeID) *pusAlternativeID = NTMGETID_NOTFOUND_ID; + //--- if input parameters OK if (pszName[0] != EOS) { - //--- if input parameters OK - if (pszName[0] != EOS) + if (usTableType == FILE_KEY) { - //--- capitalize input string - if (usTableType != FILE_KEY) - { - strupr(pszName); - } + /**************************************************************/ + /* ... check first against our long name table */ + /**************************************************************/ + PTMX_LONGNAME_TABLE_ENTRY pEntry = nullptr; // ptr to entry found + TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched - /******************************************************************/ - /* get pointer to table and table entries in dependency of the */ - /* table type */ - /******************************************************************/ - usRc = NTMGetPointersToTable(usTableType, - &pstTMTable, - &pstTMTableEntries); - /****************************************************************/ - /* If we are looking for documents and a long document name is */ - /* available ... */ - /****************************************************************/ - - // skip any leading backslash (added by bug in ITM code ...) - if (pszLongName != NULL) - { - if (*pszLongName == BACKSLASH) - pszLongName++; - } /* endif */ + // prepare search entry + SearchEntry.pszLongName = pszName; - fLongName = (usTableType == FILE_KEY) && - (pszLongName != NULL) && - (pszLongName[0] != EOS) - //&& (strcmp( pszLongName, pszName ) != 0) - ; + if(LongNames.stTableEntry.size()){ + // do the actual search + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + LongNames.stTableEntry.data(), + LongNames.stTableEntry.size()-1,// last elem is always empty + sizeof(TMX_LONGNAME_TABLE_ENTRY), + NTMLongNameTableComp); + } - if ((usRc == NO_ERROR) && fLongName) + // if search fails try again using case insenstive search + if (pEntry == NULL && LongNamesCaseIgnore.stTableEntry.size()) { - /**************************************************************/ - /* ... check first against our long name table */ - /**************************************************************/ - PTMX_LONGNAME_TABLE_ENTRY pEntry = nullptr; // ptr to entry found - TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched - - // prepare search entry - SearchEntry.pszLongName = pszLongName; - - if(LongNames.stTableEntry.size()){ - // do the actual search - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, - LongNames.stTableEntry.data(), - LongNames.stTableEntry.size()-1,// last elem is always empty + pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, + LongNamesCaseIgnore.stTableEntry.data(), + LongNamesCaseIgnore.stTableEntry.size()-1,// last elem is always empty sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp); - } - - // if search fails try again using case insenstive search - if (pEntry == NULL && LongNamesCaseIgnore.stTableEntry.size()) - { - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, - LongNamesCaseIgnore.stTableEntry.data(), - LongNamesCaseIgnore.stTableEntry.size()-1,// last elem is always empty - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore); - } /* endif */ - - if (pEntry != nullptr) - { - // return ID of found entry - *pusID = pEntry->usId; - } - else if (!(lOptions & NTMGETID_NOUPDATE_OPT)) - { - //ULONG ulNameLen = strlen(pszLongName) + 1; - //ULONG ulAddLen = ulNameLen + sizeof(USHORT); + NTMLongNameTableCompCaseIgnore); + } /* endif */ - // add a new entry for the given short name - if (usRc == NO_ERROR) - { - // for some (yet unknown) reasons, there are memories which have fewer entries in - // the short name table when in the long name table, using the ID generated when - // adding the short name will lead to incorrect IDs for the long name table. - // In order to circumvent this we call NTMAddNameToTable for the short name table - // until the returned ID is larger than or equal to pTmClb->LongNames->ulEntries - //do - //{ - //DEBUGEVENT2(NTMGETIDFROMNAME_LOC, INFO_EVENT, 1, TM_GROUP, pszName); - usRc = NTMAddNameToTable(pszName, usTableType, pusID); - //} while ((usRc == NO_ERROR) && (*pusID < LongNames.stTableEntry.size())); /* enddo */ - } /* endif */ - - } /* endif */ + if (pEntry != nullptr) + { + // return ID of found entry + *pusID = pEntry->usId; } - else + else if (!(lOptions & NTMGETID_NOUPDATE_OPT)) { + // add a new entry for the given short name if (usRc == NO_ERROR) { - pszName[MAX_LANG_LENGTH-1] = '\0'; - /********************************************************************/ - /* search name passed in pszName in passed table usTableType */ - /********************************************************************/ - if(pstTMTable->stTmTableEntry.size()){ - pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch(pszName, - pstTMTable->stTmTableEntry.data(), - pstTMTable->stTmTableEntry.size()-1,// last elem is always empty - sizeof(TMX_TABLE_ENTRY), - NTMCompNames); - } - //--- if passed name found in table - if (pstTMTableEntry) - { - //-- get ID of entry and set output parameter - *pusID = pstTMTableEntry->usId; - } - else - { - // name is not contained in name table - if (!(lOptions & NTMGETID_NOUPDATE_OPT)) - { - usRc = NTMAddNameToTable(pszName, usTableType, - pusID); - - // for new languages update our language group table - if ((usRc == NO_ERROR) && - (usTableType == LANG_KEY)) - { - usRc = NTMAddLangGroup(pszName, *pusID); - } /* endif */ - } /* endif */ - } /* endif */ + usRc = NTMAddNameToTable(pszName, usTableType, pusID); } /* endif */ } /* endif */ + }else{// (usTableType != FILE_KEY) + + //--- capitalize input string + strupr(pszName); - // file name table only: - // look for given short name in long name table if not found yet or - // an alternative file ID is requested - if ((usRc == NO_ERROR) && (usTableType == FILE_KEY) && !fLongName && - ((*pusID == NTMGETID_NOTFOUND_ID) || (pusAlternativeID != NULL))) + /******************************************************************/ + /* get pointer to table and table entries in dependency of the */ + /* table type */ + /******************************************************************/ + usRc = NTMGetPointersToTable(usTableType, + &pstTMTable, + &pstTMTableEntries); + if (usRc == NO_ERROR) { - PTMX_LONGNAME_TABLE_ENTRY pEntry; // ptr to entry found - TMX_LONGNAME_TABLE_ENTRY SearchEntry; // entry being searched - - // prepare search entry - SearchEntry.pszLongName = pszName; - - // do the actual search - if(LongNames.stTableEntry.size()){ - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, - LongNames.stTableEntry.data(), - LongNames.stTableEntry.size()-1,// last elem is always empty - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableComp); + pszName[MAX_LANG_LENGTH-1] = '\0'; + /********************************************************************/ + /* search name passed in pszName in passed table usTableType */ + /********************************************************************/ + if(pstTMTable->stTmTableEntry.size()){ + pstTMTableEntry = (PTMX_TABLE_ENTRY)bsearch(pszName, + pstTMTable->stTmTableEntry.data(), + pstTMTable->stTmTableEntry.size()-1,// last elem is always empty + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); } - - if (pEntry != NULL && LongNamesCaseIgnore.stTableEntry.size()) + //--- if passed name found in table + if (pstTMTableEntry) { - pEntry = (PTMX_LONGNAME_TABLE_ENTRY)bsearch(&SearchEntry, - LongNamesCaseIgnore.stTableEntry.data(), - LongNamesCaseIgnore.stTableEntry.size()-1,// last elem is always empty - sizeof(TMX_LONGNAME_TABLE_ENTRY), - NTMLongNameTableCompCaseIgnore); - } /* endif */ - - if (pEntry != NULL) + //-- get ID of entry and set output parameter + *pusID = pstTMTableEntry->usId; + } + else { - // return ID of found entry - if (*pusID == NTMGETID_NOTFOUND_ID) - { - *pusID = pEntry->usId; - } - else if (pusAlternativeID != NULL) + // name is not contained in name table + if (!(lOptions & NTMGETID_NOUPDATE_OPT)) { - *pusAlternativeID = pEntry->usId; + usRc = NTMAddNameToTable(pszName, usTableType, + pusID); + + // for new languages update our language group table + if ((usRc == NO_ERROR) && + (usTableType == LANG_KEY)) + { + usRc = NTMAddLangGroup(pszName, *pusID); + } /* endif */ } /* endif */ - } + } /* endif */ } /* endif */ - } - else //--- pTmClb is NULL pointer or pszName is empty - { - //--- wrong function paramters - LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); - } /* endif */ + } /* endif */ + } + else //--- pTmClb is NULL pointer or pszName is empty + { + //--- wrong function paramters + LOG_AND_SET_RC(usRc, T5INFO, ERROR_INTERNAL); } /* endif */ return usRc; -} /* end of function NTMGetIDFromNameEx */ +} /* end of function NTMGetIDFromName */ //+----------------------------------------------------------------------------+ //|External function | @@ -513,22 +419,37 @@ USHORT EqfMemory::NTMGetIDFromNameEx( USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput USHORT usTableType, // input PSZ pszName, // output - PSZ pszLongName, SHORT maxBuffLen) // output, long name (only for FILE_KEY) { USHORT usRc = NO_ERROR; BOOL fFound = FALSE; PTMX_TABLE_ENTRY pstTMTableEntries = NULL; // ptr to first table entry PTMX_TABLE pstTMTable = NULL; // ptr to table structure - - if (pszLongName != NULL) + if (pszName != NULL) { - pszLongName[0] = EOS; + pszName[0] = EOS; } /* endif */ - - //--- if input parameters OK - // if ( pTmClb ) - { + + if(usTableType == FILE_KEY){ + /**************************************************************/ + /* Get any long document name for this ID */ + /**************************************************************/ + for (ULONG ulI = 0; + ulI < LongNames.stTableEntry.size() && !fFound; + ulI++){ + if (usID && usID == LongNames.stTableEntry[ulI].usId) + { + fFound = TRUE; + if(maxBuffLen>0){ + strncpy(pszName, LongNames.stTableEntry[ulI].pszLongName, maxBuffLen-1); + pszName[maxBuffLen-1] = '\0'; + }else{ + strcpy(pszName, + LongNames.stTableEntry[ulI].pszLongName); + } + } /* endif */ + } /* endfor */ + } else{//not file_key /******************************************************************/ /* get pointer to table and table entries in dependency of the */ /* table type */ @@ -548,65 +469,35 @@ USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput } else { - if(usTableType != FILE_KEY){ - for (ULONG ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) - { - if (usID && usID == pstTMTableEntries[ulI].usId) - { - fFound = TRUE; - constexpr int sizeOfField = MAX_LANG_LENGTH ; - strncpy(pszName, pstTMTableEntries[ulI].szName, sizeOfField-1); - pszName[sizeOfField-1] = '\0'; - } /* endif */ - } /* endfor */ - }else{ - /**************************************************************/ - /* Get any long document name for this ID */ - /**************************************************************/ - if ( (pszLongName != NULL)) + for (ULONG ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) { - //BOOL fLongFound = FALSE; - for (ULONG ulI = 0; - ulI < LongNames.stTableEntry.size() && !fFound; - ulI++) + if (usID && usID == pstTMTableEntries[ulI].usId) { - if (usID && usID == LongNames.stTableEntry[ulI].usId) - { - fFound = TRUE; - - if(maxBuffLen>0){ - strncpy(pszName, LongNames.stTableEntry[ulI].pszLongName, maxBuffLen-1); - pszName[maxBuffLen-1] = '\0'; - }else{ - strcpy(pszLongName, - LongNames.stTableEntry[ulI].pszLongName); - } - } /* endif */ - } /* endfor */ - - if (!fFound) - { - strcpy(pszLongName, pszName); - } - } /* endif */ - } - if (!fFound) - { - /************************************************************/ - /* the ID was not found in the table */ - /* set usRc and reset pszName */ - /************************************************************/ - if(usID){ - LOG_AND_SET_RC_W_INFO(usRc, T5ERROR, ID_NOT_FOUND) << "id = " << usID; - snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", usID); // 'hu' = unsigned short - }else{ - pszName[0] = EOS; - } - } /* endif */ - } /* endif */ - } /* endif */ + fFound = TRUE; + constexpr int sizeOfField = MAX_LANG_LENGTH ; + strncpy(pszName, pstTMTableEntries[ulI].szName, sizeOfField-1); + pszName[sizeOfField-1] = '\0'; + } /* endif */ + } /* endfor */ + } + } } + if (!fFound) + { + /************************************************************/ + /* the ID was not found in the table */ + /* set usRc and reset pszName */ + /************************************************************/ + if(usID){ + LOG_AND_SET_RC_W_INFO(usRc, T5ERROR, ID_NOT_FOUND) << "id = " << usID << "; table = " << usTableType; + snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", usID); // 'hu' = unsigned short + }else{ + pszName[0] = EOS; + } + } /* endif */ + + return usRc; } /* end of function NTMGetNameFromID */ @@ -1211,7 +1102,7 @@ USHORT EqfMemory::NTMAddLangGroup( // get ID of group name if ((usRC == NO_ERROR) && (sGroupID == 0)) { - usRC = NTMGetIDFromName(szLangGroup, NULL, LANGGROUP_KEY, (PUSHORT)&sGroupID); + usRC = NTMGetIDFromName(szLangGroup, LANGGROUP_KEY, (PUSHORT)&sGroupID); } /* endif */ // enlarge language-ID-to-group-ID table if necessary diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 62053a36..a8976d3d 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -611,15 +611,10 @@ int EqfMemory::OtmProposalToGetIn Proposal.getAddInfo( pGetIn->szAddInfo, sizeof(pGetIn->szAddInfo) ); Proposal.getContext( pGetIn->szContext, sizeof(pGetIn->szContext) ); Proposal.getDocName( pGetIn->szLongName, sizeof(pGetIn->szLongName) ); - Proposal.getDocName( pGetIn->szFileName, sizeof(pGetIn->szFileName) ); - //pGetIn->usMatchThreshold = TM_DEFAULT_THRESHOLD; - int threshold = FLAGS_triplesthreshold; - //Properties::GetInstance()->get_value_or_default(KEY_TRIPLES_THRESHOLD, threshold, threshold); - pGetIn->usMatchThreshold = threshold; + pGetIn->usMatchThreshold = FLAGS_triplesthreshold; pGetIn->ulSegmentId = Proposal.getSegmentId(); pGetIn->pvReplacementList = (PVOID)Proposal.getReplacementList(); - //pGetIn->fSourceLangIsPrefered = Proposal.isSourceLangIsPrefered(); pGetIn->fSourceLangIsPrefered = LanguageFactory::getInstance()->isPreferedLang( (pGetIn->szSourceLanguage) ); pGetIn->fTargetLangIsPrefered = LanguageFactory::getInstance()->isPreferedLang( (pGetIn->szTargetLanguage) ); return( iRC ); @@ -699,6 +694,9 @@ int EqfMemory::Reload(bool fReadFromDisk){ } int EqfMemory::FlushFilebuffers(MutexTimeout& tmLockTimeout){ + if(fCorrupted){ + return 0; + } TimedMutexGuard l(tmMutex, tmLockTimeout, "tmMutex", __func__, __LINE__); int rc = TmBtree.fb.Flush(); rc = rc? rc : InBtree.fb.Flush(); diff --git a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp index 4b81f21c..76e54574 100755 --- a/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/TMPluginWrapper.cpp @@ -104,9 +104,7 @@ PSZ_W MADSkipWhitespace( PSZ_W pszData ); //+----------------------------------------------------------------------------+ BOOL NTMDocMatch ( - PSZ pszDoc1ShortName, // ptr to short name of 1st doc PSZ pszDoc1LongName, // ptr to long name of 1st doc - PSZ pszDoc2ShortName, // ptr to short name of 2nd doc PSZ pszDoc2LongName // ptr to long name of 2nd doc ) { @@ -119,11 +117,6 @@ BOOL NTMDocMatch // base the match on the long names fMatch = (strcmp( pszDoc1LongName, pszDoc2LongName ) == 0 ); } - else - { - // use the short names for the match - fMatch = (strcmp( pszDoc1ShortName, pszDoc2ShortName ) == 0 ); - } /* endif */ return( fMatch ); } /* end of function NTMDocMatch */ diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index b24fc6ff..14184001 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -339,11 +339,17 @@ int RequestData::requestTM(){ if(mem->isWaitingToBeLoaded() || mem->isFailedToLoad()){ USHORT accessModeBeforeOpenning = mem->usAccessMode; - if(REORGANIZE_MEM == command || DELETE_ENTRIES_REORGANIZE == command){ + + if(REORGANIZE_MEM == command || DELETE_ENTRIES_REORGANIZE == command + || EXPORT_MEM_TMX == command || EXPORT_MEM_TMX_STREAM == command){ mem->usAccessMode |= ASD_ORGANIZE; } + _rc_ = mem->Load(); - mem->usAccessMode=accessModeBeforeOpenning; + + if(!mem->isCorrupted()){ + mem->usAccessMode=accessModeBeforeOpenning; + } if(_rc_){ std::string msg = "Failed to load tm, rc = "; @@ -351,7 +357,6 @@ int RequestData::requestTM(){ if(11==_rc_){ msg += "; file on the disk is locked;"; } - //T5LOG(T5ERROR) << "Failed to open tm, rc = " << _rc_; return buildErrorReturn(504, msg.c_str(), _rc_); } } @@ -360,6 +365,14 @@ int RequestData::requestTM(){ if(mem->isLoaded()) { fValid = true; + if(mem->isCorrupted() && + ( EXPORT_MEM_TMX != command + && EXPORT_MEM_TMX_STREAM != command + && REORGANIZE_MEM != command) + ) + { + return buildErrorReturn(504, "Memory is corrupted, so it can only be tmx-exported or reorganized", _rc_); + } if( command == EXPORT_MEM_TMX || command == EXPORT_MEM_INTERNAL_FORMAT @@ -2138,6 +2151,7 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr // return the status of the memory json_factory.startJSON( outputMessage ); json_factory.addParmToJSON( outputMessage, "status", mem->getStatusString() ); + json_factory.addParmToJSON( outputMessage, "isCorrupted", mem->isCorrupted()); //json_factory.addParmToJSON( outputMessage, "status", "open" ); json_factory.addParmToJSON( outputMessage, "sizeInRAM", mem->GetRAMSize()); json_factory.addParmToJSON( outputMessage, "activeRequest", mem->getActiveRequestString()); diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index 81d39a19..a3c486db 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -224,17 +224,19 @@ void EqfMemory::reorganizeDone(int iRC, char *pszError ) { eStatus = OPEN_STATUS; - resetOpenedOnlyForReorganize(); + //resetOpenedOnlyForReorganize(); if ( iRC == 0 ) { eImportStatus = AVAILABLE_STATUS; + resetOpenedOnlyForReorganize(); T5LOG( T5INFO) <<":: success, memName = " << szName; } else { eImportStatus = REORGANIZE_FAILED_STATUS; strError = pszError; + fCorrupted = true; T5LOG(T5ERROR) << ":: memName = " << szName <<", reorganize failed: " << pszError << " import details = " << importDetails->toString() ; } diff --git a/source/opentm2/core/utilities/EQFHASH.cpp b/source/opentm2/core/utilities/EQFHASH.cpp index 106cf148..6ba7ceb3 100755 --- a/source/opentm2/core/utilities/EQFHASH.cpp +++ b/source/opentm2/core/utilities/EQFHASH.cpp @@ -565,26 +565,7 @@ PPOOL PoolCreate //| add string to pool using PoolAddData | //| return pointer to added string | //+----------------------------------------------------------------------------+ -PSZ_W PoolAddStringW -( - PPOOL pPool, // pointer to a pool created using - // PoolCreate - PSZ_W pszString // string which should be stored in the - // pool -) -{ - LONG lStringLen; // length of string - PSZ_W pszStringInPool; // ptr to location of string in pool - - lStringLen = UTF16strlenCHAR( pszString) + 1;// get length of string + delimiter - - pszStringInPool = (PSZ_W)PoolAddData( pPool, - (lStringLen * sizeof(CHAR_W)), - pszString ); - return( pszStringInPool ); - -} /* end of function PoolAddStringW */ //+----------------------------------------------------------------------------+ //|External function | @@ -731,18 +712,14 @@ HATOMTBL APIENTRY WinCreateAtomTable USHORT cBuckets ) { - BOOL fOK = TRUE; // internal O.K. flag HATOMTBL pAtomTable = NULL; // ptr to new atom table - cbInitial; /********************************************************************/ /* Allocate atom table control area */ /********************************************************************/ - if ( fOK ) - { - fOK = UtlAlloc( (PVOID *)&pAtomTable, 0L, + + BOOL fOK = UtlAlloc( (PVOID *)&pAtomTable, 0L, get_max( MIN_ALLOC, (LONG)sizeof(ATOMTBL)), ERROR_STORAGE ); - } /* endif */ /********************************************************************/ /* Create low level hash */ @@ -857,8 +834,6 @@ ULONG AtomHashKeyValue ULONG hash1 = 0; ULONG hash2 = 0; - ulSize; // avoid compiler warning - while ( *pbTerm ) { hash1 = random1[hash1 ^ *pbTerm ]; @@ -901,8 +876,6 @@ LONG AtomHashCompare PVOID pUserPtr ) { - pUserPtr; // avoid compiler warning - return strcmp( pEntry1->szAtomName, pEntry2->szAtomName ); } /* end of function AtomHashCompare */ @@ -932,9 +905,7 @@ LONG AtomHashCompare //+----------------------------------------------------------------------------+ SHORT AtomHashFree( PATOMENTRY pEntry, PVOID pUserPtr ) { - pUserPtr; UtlAlloc( (PVOID *) &pEntry, 0L, 0L, NOMSG ); - return( TRUE ); } /* end of function AtomHashFree */ diff --git a/source/opentm2/core/utilities/EQFTADIT.cpp b/source/opentm2/core/utilities/EQFTADIT.cpp deleted file mode 100755 index e69de29b..00000000 diff --git a/source/opentm2/core/utilities/EQFUTDOS.cpp b/source/opentm2/core/utilities/EQFUTDOS.cpp index 76c91fe1..8c0949a7 100755 --- a/source/opentm2/core/utilities/EQFUTDOS.cpp +++ b/source/opentm2/core/utilities/EQFUTDOS.cpp @@ -740,50 +740,6 @@ USHORT UtlCloseHwnd { FilesystemHelper::CloseFile( hf ) ; return 0; - #ifdef TEMPORARY_COMMENTED - USHORT usRetCode = 0; // function return code - USHORT usMBCode = 0; // message box/UtlError return code - - /*******************************************************************/ - /* close only if file has been opened */ - /*******************************************************************/ - if ( hf != NULL ) - { - // remove handle from our handle/drive array - int i = 0; - while ( i < MAX_OPEN_FILES ) - { - if ( DriveHandles[i].hf == hf ) - { - // mark as empty and leave loop - DriveHandles[i].chDrive = BLANK; - DriveHandles[i].hf = NULLHANDLE; - break; - } - else - { - i++; - } /* endif */ - } /* endwhile */ - - do { - DosError(0); - - if ( FilesystemHelper::CloseFile( hf ) == 0 ) - { - //usRetCode = (USHORT)GetLastError(); - } /* endif */ - - DosError(1); - if ( fMsg && usRetCode ) - { - usMBCode = MB_CANCEL; - T5LOG(T5ERROR) << ":: rc = " << usRetCode; - } /* endif */ - } while ( fMsg && usRetCode && (usMBCode == MBID_RETRY) ); /* enddo */ - } /* endif */ - return( usRetCode ); - #endif } //+----------------------------------------------------------------------------+ diff --git a/source/opentm2/core/utilities/FilesystemWrapper.h b/source/opentm2/core/utilities/FilesystemWrapper.h index d15170b7..2dd2048c 100755 --- a/source/opentm2/core/utilities/FilesystemWrapper.h +++ b/source/opentm2/core/utilities/FilesystemWrapper.h @@ -104,21 +104,6 @@ DWORD GetLastError(); int GetFileSize(HFILE file); -/* -typedef struct _WIN32_FIND_DATA { - DWORD dwFileAttributes; - FILETIME ftCreationTime; - FILETIME ftLastAccessTime; - FILETIME ftLastWriteTime; - DWORD nFileSizeHigh; - DWORD nFileSizeLow; - DWORD dwReserved0; - DWORD dwReserved1; - TCHAR cFileName[ MAX_PATH ]; - TCHAR cAlternateFileName[ 14 ]; - } WIN32_FIND_DATA, *PWIN32_FIND_DATA; -//*/ - HFILE FindFirstFile( LPCTSTR lpFileName, LPWIN32_FIND_DATA lpFindFileData @@ -129,16 +114,6 @@ BOOL FindNextFile( LPWIN32_FIND_DATA lpFindFileData ); - - -/* -BOOL SetFilePointerEx( - HANDLE hFile, - LARGE_INTEGER liDistanceToMove, - PLARGE_INTEGER lpNewFilePointer, - DWORD dwMoveMethod - );//*/ - BOOL SetFilePointerEx( HFILE hFile, LARGE_INTEGER liDistanceToMove, diff --git a/source/opentm2/core/utilities/LanguageFactory.cpp b/source/opentm2/core/utilities/LanguageFactory.cpp index 6e6ad45c..40546940 100755 --- a/source/opentm2/core/utilities/LanguageFactory.cpp +++ b/source/opentm2/core/utilities/LanguageFactory.cpp @@ -390,7 +390,6 @@ int LanguageFactory::findIfPreferedLanguage const char *pszLanguage ) { - int i = 0; int mid, st = 0, end = vpPreferedLanguageList.size() -1; while ( st <= end) //binary search since languages was sorted { @@ -411,7 +410,6 @@ int LanguageFactory::findPreferedLangForThisLang const char *pszLanguage ) { - int i = 0; int mid, st = 0, end = vpPreferedLanguageList.size() -1; while ( st <= end) //binary search since languages was sorted { @@ -436,7 +434,6 @@ int LanguageFactory::findLanguage const char *pszLanguage ) { - int i = 0; int mid, st = 0, end = vLanguageList.size() -1; while ( st <= end) //binary search since languages was sorted { @@ -625,7 +622,6 @@ void LangParseHandler::startElement(const XMLCh* const name, AttributeList& attr case UNKNOWN_ELEMENT: default: if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - std::string sName = EncodingHelper::convertToUTF8(u16name); T5LOG( T5WARNING) << "LangParseHandler::startElement::unknown_element found: "<< EncodingHelper::convertToUTF8(u16name) << "; id = " << CurID << "; skipping..."; } From 60c6a0f3c084c949a5dfa4cf0153e962b06cc18e Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 6 Aug 2025 22:06:01 +0300 Subject: [PATCH 48/62] * updated to v0.7.21 * fixed -314 - added check for vitalInfo on loading tm + added btree info to status info * fixed export tmx in case of corrupted vitalinfo - code cleanup --- include/EQFQDAMI.H | 10 +- include/Filebuffer.h | 2 +- include/lowlevelotmdatastructs.h | 48 +------- include/tm.h | 5 +- source/CMakeLists.txt | 2 +- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 10 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 19 ++- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 108 ++--------------- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 14 +-- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 6 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 110 +++++++++--------- .../core/EqfMemoryPlugin/EqfMemory.cpp | 8 +- source/opentm2/core/requestdata.cpp | 25 +++- source/opentm2/core/utilities/EQFDICT.cpp | 83 ++++++------- .../core/utilities/FilesystemHelper.cpp | 2 +- 15 files changed, 156 insertions(+), 296 deletions(-) diff --git a/include/EQFQDAMI.H b/include/EQFQDAMI.H index 021f72e2..8e6de758 100755 --- a/include/EQFQDAMI.H +++ b/include/EQFQDAMI.H @@ -921,16 +921,14 @@ typedef struct _DICTTIME SHORT QDAMValidateIndex_V3( PBTREE, std::shared_ptr& ); LONG QDAMGetszKey_V3 ( std::shared_ptr&, USHORT ); - RECPARAM QDAMGetrecData_V3 ( std::shared_ptr&, SHORT, USHORT ); - VOID QDAMSetrecData_V3 ( std::shared_ptr&, SHORT, RECPARAM, USHORT ); + RECPARAM QDAMGetrecData_V3 ( std::shared_ptr&, SHORT ); + VOID QDAMSetrecData_V3 ( std::shared_ptr&, SHORT, RECPARAM ); RECPARAM QDAMGetrecKey_V3 ( std::shared_ptr&, SHORT ); VOID QDAMReArrangeKRec_V3 ( PBTREE, std::shared_ptr& ); - VOID QDAMCopyKeyTo_V3( std::shared_ptr&, SHORT, std::shared_ptr&, SHORT, USHORT ); - VOID QDAMCopyDataTo_V3( std::shared_ptr&, SHORT, std::shared_ptr&, SHORT, USHORT ); + VOID QDAMCopyKeyTo_V3( std::shared_ptr&, SHORT, std::shared_ptr&, SHORT ); + VOID QDAMCopyDataTo_V3( std::shared_ptr&, SHORT, std::shared_ptr&, SHORT ); SHORT QDAMLastEntry_V3( PBTREE, std::shared_ptr& ); SHORT QDAMFirstEntry_V3( PBTREE, std::shared_ptr& ); - BOOL QDAMFetchFromIndexList_V3( PBTREE, std::shared_ptr&, USHORT ); - VOID QDAMAddToIndexList_V3( PBTREE, std::shared_ptr& ); diff --git a/include/Filebuffer.h b/include/Filebuffer.h index 287f3069..69e12907 100644 --- a/include/Filebuffer.h +++ b/include/Filebuffer.h @@ -122,9 +122,9 @@ struct FileBuffer{ int Write(const void* buff, size_t buffSize, size_t startingPosition, bool fSuppressTmdSizeCheck = false); int Read(void* buff, size_t buffSize, ssize_t startingPosition=-1); bool WasModified() const; + bool isTMDFilebuffer()const; private: - bool isTMDFilebuffer(); int atomicWriteWithBackup(); } ; diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 938b27b6..4dc160ad 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -320,6 +320,7 @@ typedef struct _TMVITALINFO LONG lNextKey = 0; // currently active key } NTMVITALINFO, * PNTMVITALINFO; +const long LASTALLOWEDKEY = 0xFFFFFF; #define MAX_MEM_DESCRIPTION 41 // length of a memory description field @@ -585,7 +586,6 @@ struct BTREE ushort usFirstDataBuffer=0; // first data buffer std::shared_ptr BTreeTempBuffer_V3; // temporary V3 buffer long lTime=0; // time of last update/open - ushort usVersion=0; // version identification... char chEQF[7]{}; // The type of file char bVersion=0; // version flag ushort usOpenFlags=0; // settings used for open @@ -789,34 +789,6 @@ struct BTREE //); - - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMGetMaxNumber - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMGetNextNumber( pBTIda, &ulKey, &ulNextFree ); - //+----------------------------------------------------------------------------+ - // Description: get the start key and the next free key ... - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pBTIda, pointer to btree struct - // PULONG pulStartKey first key - // PULONG pulNextKey next key to be assigned - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: 0 always - //+----------------------------------------------------------------------------+ - // Function flow: access data from internal structure - //+----------------------------------------------------------------------------+ - SHORT - EQFNTMGetNextNumber - ( - PLONG plStartKey, // return start key number - PLONG plNextKey // return next key data - ); - - //+----------------------------------------------------------------------------+ // External function //+----------------------------------------------------------------------------+ @@ -1036,12 +1008,10 @@ struct BTREE bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed ); SHORT QDAMAllocKeyRecords( USHORT usNum); - SHORT QDAMDictOpenLocal( - SHORT sNumberOfBuffers, // number of buffers - USHORT usOpenFlags // Read Only or Read/Write + SHORT EQFNTMOpen( + USHORT usOpenFlags, // Read Only or Read/Write + SHORT sNumberOfBuffers=20 // number of buffers ); - - SHORT EQFNTMOpen( USHORT usOpenFlags); // Read Only or Read/Write SHORT QDAMCheckDict( PSZ pName); // name of dictionary @@ -1135,15 +1105,6 @@ BOOL QDAMTerseData typedef BTREE * PBTREE; typedef BTREE ** PPBTREE; - - -#define NTMNEXTKEY( pBT ) pBT->chCollate.lNextKey -#define NTMSTARTKEY( pBT ) pBT->chCollate.lStartKey - - - - - //typedef TMX_CLB* PTMX_CLB; //prefix for each output structures typedef struct _TMX_PREFIX_OUT @@ -1153,7 +1114,6 @@ typedef struct _TMX_PREFIX_OUT } TMX_PREFIX_OUT, * PTMX_PREFIX_OUT, XOUT, * PXOUT; - #define MEM_EXPORT_OUT_BUFFER 8200 // Output buffer for export #define MAX_HANDLER_NAME 40 diff --git a/include/tm.h b/include/tm.h index f387650e..6cc3cddd 100644 --- a/include/tm.h +++ b/include/tm.h @@ -81,7 +81,7 @@ struct TMX_EXT_OUT_W LONG lTmKey = 0; // tm record key USHORT usNextTarget = 0; // which target record to address next - ULONG ulMaxEntries = 0; // number of entries in tm data file + LONG lMaxEntries = 0; // number of entries in tm data file TMX_EXT_OUT_W(){}; }; using PTMX_EXT_OUT_W = TMX_EXT_OUT_W *; @@ -2121,8 +2121,7 @@ class EqfMemory //: public TMX_CLB USHORT NTMLoadNameTable( LONG lTableKey, // key of table record - PTMX_TABLE pTMTable, // ptr to table data pointer - PULONG pulSize // ptr to buffer for size of table data + PTMX_TABLE pTMTable // ptr to table data pointer ); /*! \brief Get a list of memory proposals matching the given search key diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 4e8212de..1e11be98 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 20) +SET(APP_VERSION_MINOR 21) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index c84f6898..0e7aad71 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -59,18 +59,15 @@ USHORT EqfMemory::TmtXDelSegm TMX_EXT_OUT_W* pTmDelOut //ptr to output struct ) { - BOOL fOK = true; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound = 0; // compact area hits - ULONG ulLen = 0; // length indication - CHAR szString[MAX_EQF_PATH]; // character string + CHAR szString[MAX_EQF_PATH]{}; // character string std::vector plSids; // ptr to sentence ids PULONG pulSidStart = NULL; // ptr to sentence ids std::vector pTmRecord; // space for user data - pTmRecord.resize(TMX_REC_SIZE); + //pTmRecord.resize(TMX_REC_SIZE); - if ( !usRc ) { //build tag table path @@ -105,7 +102,6 @@ USHORT EqfMemory::TmtXDelSegm usRc = DetermineTmRecord( this, TmDelIn.pInputSentence.get(), plSids ); if ( !usRc ) { - ulLen = TMX_REC_SIZE; for (LONG lKey : plSids ) { usRc = TmBtree.EQFNTMGet(lKey, //tm record key @@ -153,7 +149,6 @@ USHORT EqfMemory::TmtXDelSegmByKey CHAR szString[MAX_EQF_PATH]; // character strin ULONG ulRecBufSize = TMX_REC_SIZE; // current size of record buffer std::vector pTmRecord; // space for user data - //pTmRecord.resize(TMX_REC_SIZE); // update TM databse if ( !usRc ) @@ -163,7 +158,6 @@ USHORT EqfMemory::TmtXDelSegmByKey if ( usRc == NO_ERROR ) { - //find target record and delete, if the target record was the //only target in the tm record, delete the entire record usRc = FindTargetByKeyAndDelete( pTmRecord, diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 4bed559e..02380962 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -67,14 +67,14 @@ USHORT EqfMemory::TmtXExtract USHORT usRc = BTREE_NOT_FOUND; //return code PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record - LONG lStartKey; //start tm key - LONG lNextKey; //last tm key + LONG lStartKey = TmBtree.chCollate.lStartKey; //start tm key + LONG lNextKey = TmBtree.chCollate.lNextKey; //last tm key std::vector pTmRecord; // space for user data - //pTmRecord.resize(2*TMX_REC_SIZE); - - TmBtree.EQFNTMGetNextNumber( &lStartKey, &lNextKey); - pTmExtOut->ulMaxEntries = (lNextKey - lStartKey); + pTmExtOut->lMaxEntries = (lNextKey - lStartKey); + if(fCorrupted && lStartKey != FIRST_KEY){ + lNextKey = std::max(lNextKey, LASTALLOWEDKEY) ; //some big value to go + } /******************************************************************/ /* return one matching entry (if any available) */ /******************************************************************/ @@ -106,7 +106,7 @@ USHORT EqfMemory::TmtXExtract /******************************************************************/ /* set end of file condition ... */ /******************************************************************/ - if ( (usRc == BTREE_NOT_FOUND) && (pTmExtIn->lTmKey == lNextKey) ) + if ((usRc == BTREE_NOT_FOUND) && (pTmExtIn->lTmKey == lNextKey) ) { //arrived at last tm record LOG_AND_SET_RC(usRc, T5INFO, BTREE_EOF_REACHED); @@ -454,11 +454,6 @@ USHORT EqfMemory::FillExtStructure //fill in the target file name NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, pstExt->szLongName, sizeof(pstExt->szLongName) ); - //use overflow name if no document name available - //if ( pstExt->szLongName[0] == EOS ) - //{ - // strcpy( pstExt->szLongName, OVERFLOW_NAME ); - //} /* endif */ //fill in the target author NTMGetNameFromID( pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index f3793c61..ca988f33 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -258,24 +258,6 @@ QDAMCheckDict //| return Rc | // ----------------------------------------------------------------------------+ -SHORT BTREE::EQFNTMOpen -( - USHORT usOpenFlags // Read Only or Read/Write -) -{ - SHORT sRc = 0; // return code - - SHORT RetryCount = MAX_RETRY_COUNT; // retry counter for in-use condition - RetryCount = MAX_RETRY_COUNT; - do - { - sRc = QDAMDictOpenLocal( 20, usOpenFlags ); - - } while ( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && - (RetryCount > 0) ); /* enddo */ - - return ( sRc ); -} //+----------------------------------------------------------------------------+ //|Internal function | @@ -467,6 +449,7 @@ BTREE::EQFNTMInsert ULONG Key = lKey = (chCollate.lNextKey)++; if ( Key > 0xFFFFFF ) { + (chCollate.lNextKey)--; sRc = BTREE_NUMBER_RANGE; } else @@ -486,7 +469,7 @@ BTREE::EQFNTMInsert /****************************************************************/ /* check if key is in valid range ... */ /****************************************************************/ - if ( lKey > NTMSTARTKEY( this ) ) + if ( lKey > chCollate.lStartKey ) { sRc = BTREE_NUMBER_RANGE; } /* endif */ @@ -867,8 +850,7 @@ VOID QDAMCopyDataTo_V3 std::shared_ptr& pRecord, SHORT i, std::shared_ptr& pNew, - SHORT j, - USHORT usVersion // version of BTREE + SHORT j ) { //static int nCalled = 0; @@ -1083,7 +1065,7 @@ SHORT BTREE::QDAMDeleteDataFromBuffer_V3 T5LOG(T5DEVELOP) << "while(isUsed = " << pRecord->sUsed; while ( i < (SHORT) OCCUPIED( pRecord )) { - QDAMCopyDataTo_V3( pRecord, i, pNew, i, usVersion ); + QDAMCopyDataTo_V3( pRecord, i, pNew, i ); i++; } /* endwhile */ // copy record data back to avoid misplacement in file @@ -1140,8 +1122,7 @@ VOID QDAMSetrecData_V3 ( std::shared_ptr& pRecord, SHORT sMid, // key number - RECPARAM recData, // data pointer - USHORT usVersion // version of database + RECPARAM recData // data pointer ) { PCHAR pData = NULL; @@ -1336,7 +1317,7 @@ SHORT BTREE::EQFNTMUpdate usCurrentRecord = RECORDNUM( pRecord ); // get data value associated with key recOldKey = QDAMGetrecKey_V3( pRecord,i ); - recOldData = QDAMGetrecData_V3( pRecord, i, usVersion ); + recOldData = QDAMGetrecData_V3( pRecord, i ); // set new data value if ( recOldKey.usNum && recOldData.usNum ) { @@ -1347,7 +1328,7 @@ SHORT BTREE::EQFNTMUpdate T5LOG(T5ERROR) << ":: tried to set bigget ulLen than rec size, ulLen = " << pData.size(); } recData.ulLen = pData.size(); - QDAMSetrecData_V3( pRecord, i, recData, usVersion ); + QDAMSetrecData_V3( pRecord, i, recData ); sRc = QDAMWriteRecord_V3( pRecord ); } /* endif */ /****************************************************************/ @@ -1519,7 +1500,6 @@ SHORT BTREE::EQFNTMGet ( LONG& lKey, // key to be searched for BytesRef pchBuffer // space for user data - //USHORT usSearchSubType=FEXACT // special hyphenation lookup flag ) { SHORT i; @@ -1530,9 +1510,6 @@ SHORT BTREE::EQFNTMGet pchBuffer.clear(); { std::shared_ptr pRecord; - SHORT sRetries = MAX_RETRY_COUNT; - do - { /*******************************************************************/ /* For shared databases: discard all in-memory pages if database */ /* has been changed since last access */ @@ -1554,7 +1531,7 @@ SHORT BTREE::EQFNTMGet sCurrentIndex = i; usCurrentRecord = RECORDNUM( pRecord ); - recData = QDAMGetrecData_V3( pRecord, i, usVersion ); + recData = QDAMGetrecData_V3( pRecord, i ); sRc = QDAMGetszData_V3( recData, pchBuffer ); } @@ -1564,83 +1541,16 @@ SHORT BTREE::EQFNTMGet // set new current position sCurrentIndex = sNearKey; usCurrentRecord = RECORDNUM( pRecord ); - //*pulLength = 0; // init returned length } /* endif */ } /* endif */ - } /* endif */ - - if ( (sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED) ) - { - UtlWait( MAX_WAIT_TIME ); - sRetries--; - } /* endif */ - } - while( ((sRc == BTREE_IN_USE) || (sRc == BTREE_INVALIDATED)) && (sRetries > 0)); + } /* endif */ } /* endif */ - - if ( sRc && (sRc != BTREE_NOT_FOUND)) - { - ERREVENT2( QDAMDICTEXACTLOCAL_LOC, INTFUNCFAILED_EVENT, sRc, DB_GROUP, "" ); - } /* endif */ - return ( sRc ); } -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: EQFNTMGet | -//+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMGet( pBTIda, lKey, chData, &usLen ); | -//+----------------------------------------------------------------------------+ -//|Description: get the data string for the passed key | -//+----------------------------------------------------------------------------+ -//|Parameters: PBTREE pBTIda, pointer to btree struct | -//| ULONG lKey, key to be searched for | -//| PCHAR pchBuffer, space for user data | -//| PUSHORT pusLength in/out length of returned user data | -//+----------------------------------------------------------------------------+ -//|Returncode type: SHORT | -//+----------------------------------------------------------------------------+ -//|Returncodes: same as for QDAMDictExactLocal... | -//+----------------------------------------------------------------------------+ -//|Function flow: call QDAMDictExactLocal to retrieve the data | -// ----------------------------------------------------------------------------+ - -//+----------------------------------------------------------------------------+ -//|External function | -//+----------------------------------------------------------------------------+ -//|Function name: EQFNTMGetMaxNumber | -//+----------------------------------------------------------------------------+ -//|Function call: sRc = EQFNTMGetNextNumber( pBTIda, &lKey, &ulNextFree );| -//+----------------------------------------------------------------------------+ -//|Description: get the start key and the next free key ... | -//+----------------------------------------------------------------------------+ -//|Parameters: PBTREE pBTIda, pointer to btree struct | -//| PULONG pulStartKey first key | -//| PULONG pulNextKey next key to be assigned | -//+----------------------------------------------------------------------------+ -//|Returncode type: SHORT | -//+----------------------------------------------------------------------------+ -//|Returncodes: 0 always | -//+----------------------------------------------------------------------------+ -//|Function flow: access data from internal structure | -// ----------------------------------------------------------------------------+ -SHORT -BTREE::EQFNTMGetNextNumber -( - PLONG plStartKey, // return start key number - PLONG plNextKey // return next key data -) -{ - *plStartKey = chCollate.lStartKey; - *plNextKey = chCollate.lNextKey; - - return 0; -} /* end of function EQFNTMGetNextNumber */ //+----------------------------------------------------------------------------+ //|External function | diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index 863ae629..ccf58e7d 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -195,7 +195,6 @@ USHORT EqfMemory::Load() BOOL fOK; //success indicator USHORT usRc = NO_ERROR; //return value USHORT usRc1 = NO_ERROR; //return value - ULONG ulLen = 0; //length indicator { usRc1 = TmBtree.EQFNTMOpen((usAccessMode) ); @@ -278,12 +277,8 @@ USHORT EqfMemory::Load() if ( (usRc == NO_ERROR) || (usRc == BTREE_CORRUPTED)) { - DEBUGEVENT( TMTXOPEN_LOC, STATE_EVENT, 2 ); - - //call to obtain exact length of record - ulLen = 0; USHORT usTempRc = NTMLoadNameTable( LANG_KEY, - &Languages, &ulLen ); + &Languages ); if ( usTempRc == BTREE_READ_ERROR ) { {LOG_AND_SET_RC_W_INFO(usTempRc, T5ERROR, BTREE_CORRUPTED) << "if ( usTempRc == BTREE_READ_ERROR )";} @@ -301,8 +296,7 @@ USHORT EqfMemory::Load() (usRc == BTREE_CORRUPTED)) { USHORT usTempRc = NTMLoadNameTable( FILE_KEY, - &FileNames, - &ulLen ); + &FileNames); // in organize mode allow continue if file name area is corrupted if ( usTempRc == BTREE_READ_ERROR ){ @@ -319,7 +313,7 @@ USHORT EqfMemory::Load() (usRc == BTREE_CORRUPTED)) { USHORT usTempRc = NTMLoadNameTable( AUTHOR_KEY, - &Authors, &ulLen ); + &Authors ); // in organize mode allow continue if author name area is corrupted if ( usTempRc == BTREE_READ_ERROR ){ @@ -336,7 +330,7 @@ USHORT EqfMemory::Load() (usRc == BTREE_CORRUPTED) ) { USHORT usTempRc = NTMLoadNameTable( TAGTABLE_KEY, - &TagTables, &ulLen ); + &TagTables ); if ( usTempRc == BTREE_READ_ERROR ){ {LOG_AND_SET_RC_W_INFO(usTempRc, T5ERROR, BTREE_CORRUPTED)<<"if ( usTempRc == BTREE_READ_ERROR )";} diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index 256d0c7d..d74af19d 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -172,9 +172,8 @@ USHORT EqfMemory::TmtXReplace BOOL fOK = TRUE; // success indicator USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound; // compact area hits - CHAR szString[MAX_EQF_PATH]; // character string + CHAR szString[MAX_EQF_PATH]{}; // character string szString[0] = 0; - BOOL fLocked = FALSE; // TM-database-has-been-locked flag BOOL fUpdateOfIndexFailed = FALSE; // TRUE = update of index failed if(TmProposal.lSegmentId > 0) { @@ -767,8 +766,7 @@ USHORT EqfMemory::AddToTm if(size > 0){ TmRecord.resize(size); } - usRc = TmBtree.EQFNTMInsert(//ptr to tm structure - lNewKey, //to be allocated in funct + usRc = TmBtree.EQFNTMInsert(lNewKey, //to be allocated in funct TmRecord); //pointer to tm record if(!usRc){ TmProposal.currentInternalKey.setInternalKey(lNewKey, 1); diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index def4ac58..1aac29d1 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -430,55 +430,57 @@ USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput pszName[0] = EOS; } /* endif */ - if(usTableType == FILE_KEY){ - /**************************************************************/ - /* Get any long document name for this ID */ - /**************************************************************/ - for (ULONG ulI = 0; - ulI < LongNames.stTableEntry.size() && !fFound; - ulI++){ - if (usID && usID == LongNames.stTableEntry[ulI].usId) + if(usID){ + if(usTableType == FILE_KEY){ + /**************************************************************/ + /* Get any long document name for this ID */ + /**************************************************************/ + for (ULONG ulI = 0; + ulI < LongNames.stTableEntry.size() && !fFound; + ulI++){ + if (usID == LongNames.stTableEntry[ulI].usId) + { + fFound = TRUE; + if(maxBuffLen>0){ + strncpy(pszName, LongNames.stTableEntry[ulI].pszLongName, maxBuffLen-1); + pszName[maxBuffLen-1] = '\0'; + }else{ + strcpy(pszName, + LongNames.stTableEntry[ulI].pszLongName); + } + } /* endif */ + } /* endfor */ + } else{//not file_key + /******************************************************************/ + /* get pointer to table and table entries in dependency of the */ + /* table type */ + /******************************************************************/ + usRc = NTMGetPointersToTable(usTableType, + &pstTMTable, + &pstTMTableEntries); + /****************************************************************/ + /* search ID in passed table usTableTye and get name of ID */ + /****************************************************************/ + if (usRc == NO_ERROR) { - fFound = TRUE; - if(maxBuffLen>0){ - strncpy(pszName, LongNames.stTableEntry[ulI].pszLongName, maxBuffLen-1); - pszName[maxBuffLen-1] = '\0'; - }else{ - strcpy(pszName, - LongNames.stTableEntry[ulI].pszLongName); + if ( usID == OVERFLOW_ID) + { + strcpy(pszName, OVERFLOW_NAME); + fFound = TRUE; } - } /* endif */ - } /* endfor */ - } else{//not file_key - /******************************************************************/ - /* get pointer to table and table entries in dependency of the */ - /* table type */ - /******************************************************************/ - usRc = NTMGetPointersToTable(usTableType, - &pstTMTable, - &pstTMTableEntries); - /****************************************************************/ - /* search ID in passed table usTableTye and get name of ID */ - /****************************************************************/ - if (usRc == NO_ERROR) - { - if ( usID == OVERFLOW_ID) - { - strcpy(pszName, OVERFLOW_NAME); - fFound = TRUE; - } - else - { - for (ULONG ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) + else { - if (usID && usID == pstTMTableEntries[ulI].usId) + for (ULONG ulI = 0; ulI < pstTMTable->stTmTableEntry.size() && !fFound; ulI++) { - fFound = TRUE; - constexpr int sizeOfField = MAX_LANG_LENGTH ; - strncpy(pszName, pstTMTableEntries[ulI].szName, sizeOfField-1); - pszName[sizeOfField-1] = '\0'; - } /* endif */ - } /* endfor */ + if (usID == pstTMTableEntries[ulI].usId) + { + fFound = TRUE; + constexpr int sizeOfField = MAX_LANG_LENGTH ; + strncpy(pszName, pstTMTableEntries[ulI].szName, sizeOfField-1); + pszName[sizeOfField-1] = '\0'; + } /* endif */ + } /* endfor */ + } } } } @@ -991,7 +993,7 @@ int NTMLongNameTableCompCaseIgnore( PSZ lname1 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName; PSZ lname2 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName; return strcasecmp(lname1,lname2);//string should be in lower case here - //return strcasecmp(lname1,lname2); + } /* end of function NTMCompNames */ //+----------------------------------------------------------------------------+ @@ -1053,21 +1055,14 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab //| If the table is tersed it is uncompressed. | // ----------------------------------------------------------------------------+ USHORT EqfMemory::NTMLoadNameTable( - LONG lTableKey, // key of table record - PTMX_TABLE pTMTable, // ptr to table data pointer - PULONG pulSize // ptr to buffer for size of table data + LONG lTableKey, // key of table record + PTMX_TABLE pTMTable // ptr to table data pointer ) { - USHORT usRc = NO_ERROR; // function return code std::vector pOldTable; - - // call to obtain exact length of record - *pulSize = 0; - usRc = TmBtree.EQFNTMGet(lTableKey, pOldTable); + USHORT usRc = TmBtree.EQFNTMGet(lTableKey, pOldTable); if(!usRc) usRc = bytesToOldTable(lTableKey, pOldTable, pTMTable); - - // return to caller return (usRc); } /* end of function NTMLoadNameTable */ @@ -1345,7 +1340,6 @@ USHORT NtmGetAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, wchar_t* pszBuffer, // find a string in a specific data area BOOL NtmFindInAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch) { - USHORT usLength = 0; BOOL fFound = FALSE; if (pCLB->usAddDataLen != 0) // do we have additional data ? @@ -1359,7 +1353,7 @@ BOOL NtmFindInAddData(PTMX_TARGET_CLB pCLB, USHORT usDataID, PSZ_W pszSearch) if (pData != NULL) { PUSHORT pDataUShort = (PUSHORT)pData; - usLength = pDataUShort[1]; + USHORT usLength = pDataUShort[1]; if (usLength > 0) { if (wcsstr((pData + 1), pszSearch) != NULL) diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index a8976d3d..70b0c725 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -267,17 +267,17 @@ int EqfMemory::getNextProposal TMX_EXT_IN_W TmExtIn{}; TMX_EXT_OUT_W TmExtOut{}; - TmExtIn.lTmKey = Proposal.nextInternalKey.getRecordKey();//this->ulNextKey; - TmExtIn.usNextTarget = Proposal.nextInternalKey.getTargetKey();//this->usNextTarget; + TmExtIn.lTmKey = Proposal.nextInternalKey.getRecordKey(); + TmExtIn.usNextTarget = Proposal.nextInternalKey.getTargetKey(); Proposal.clear(); iRC = (int)TmtXExtract( &TmExtIn, &TmExtOut); if ( (iRC == 0) || (iRC == BTREE_CORRUPTED) ) { - if ( (piProgress != NULL) && (TmExtOut.ulMaxEntries != 0) ) + if ( (piProgress != NULL) && (TmExtOut.lMaxEntries > 0) ) { - *piProgress = (int)(( TmExtIn.lTmKey - FIRST_KEY) * 100) / (int)TmExtOut.ulMaxEntries; + *piProgress = (int)(( TmExtIn.lTmKey - FIRST_KEY) * 100) / (int)TmExtOut.lMaxEntries; } /* endif */ } /* endif */ diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 14184001..925aad1d 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -347,9 +347,9 @@ int RequestData::requestTM(){ _rc_ = mem->Load(); - if(!mem->isCorrupted()){ + //if(!mem->isCorrupted()){ mem->usAccessMode=accessModeBeforeOpenning; - } + //} if(_rc_){ std::string msg = "Failed to load tm, rc = "; @@ -368,7 +368,8 @@ int RequestData::requestTM(){ if(mem->isCorrupted() && ( EXPORT_MEM_TMX != command && EXPORT_MEM_TMX_STREAM != command - && REORGANIZE_MEM != command) + && REORGANIZE_MEM != command + && GET_TABLES != command) ) { return buildErrorReturn(504, "Memory is corrupted, so it can only be tmx-exported or reorganized", _rc_); @@ -390,7 +391,7 @@ int RequestData::requestTM(){ } } - if(command != STATUS_MEM) + if(command != STATUS_MEM && command != GET_TABLES) { mem->updateLastUseTime(); } @@ -2207,7 +2208,23 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr json_factory.addParmToJSON( outputMessage, "sourceLang", mem->stTmSign.szSourceLanguage ); json_factory.addParmToJSON( outputMessage, "internalDescription", mem->stTmSign.szDescription ); json_factory.addParmToJSON( outputMessage, "tmd fbuffer was modified", mem->TmBtree.wasModified()); + json_factory.addParmToJSON( outputMessage, "tmd lStartKey", mem->TmBtree.chCollate.lStartKey); + json_factory.addParmToJSON( outputMessage, "tmd lNextKey", mem->TmBtree.chCollate.lNextKey); + json_factory.addParmToJSON( outputMessage, "tmd usNextFreeRecord", mem->TmBtree.usNextFreeRecord); + json_factory.addParmToJSON( outputMessage, "tmd usFirstLeaf", mem->TmBtree.usFirstLeaf); + json_factory.addParmToJSON( outputMessage, "tmd usFirstNode", mem->TmBtree.usFirstNode); + json_factory.addParmToJSON( outputMessage, "tmd usFirstDataBuffer", mem->TmBtree.usFirstDataBuffer); + json_factory.addParmToJSON( outputMessage, "tmd usCurrentRecord", mem->TmBtree.usCurrentRecord); + json_factory.addParmToJSON( outputMessage, "tmd NumOfSavedRecords", mem->TmBtree.GetNumOfSavedRecords()); json_factory.addParmToJSON( outputMessage, "tmi fbuffer was modified", mem->InBtree.wasModified()); + json_factory.addParmToJSON( outputMessage, "tmi lStartKey", mem->InBtree.chCollate.lStartKey); + json_factory.addParmToJSON( outputMessage, "tmi lNextKey", mem->InBtree.chCollate.lNextKey); + json_factory.addParmToJSON( outputMessage, "tmi usNextFreeRecord", mem->InBtree.usNextFreeRecord); + json_factory.addParmToJSON( outputMessage, "tmi usFirstLeaf", mem->InBtree.usFirstLeaf); + json_factory.addParmToJSON( outputMessage, "tmi usFirstNode", mem->InBtree.usFirstNode); + json_factory.addParmToJSON( outputMessage, "tmi usFirstDataBuffer", mem->InBtree.usFirstDataBuffer); + json_factory.addParmToJSON( outputMessage, "tmi usCurrentRecord", mem->InBtree.usCurrentRecord); + json_factory.addParmToJSON( outputMessage, "tmi NumOfSavedRecords", mem->InBtree.GetNumOfSavedRecords()); //json_factory.addParmToJSON( outputMessage, "internalName", mem->stTmSign.szName ); if ( ( (mem->eImportStatus == IMPORT_FAILED_STATUS) || (mem->eImportStatus == REORGANIZE_FAILED_STATUS)) && ( !mem->strError.empty() ) ) diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 6b70ca61..9e23620a 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -1764,12 +1764,7 @@ SHORT BTREE::QDAMDictCreateLocal if( false == keepInRamOnly ) { // Try to create the index file - fb.file = LockedFile(fb.fileName, LockedFile::CalledFrom::Create);// = fopen(fb.fileName.c_str(), "w+b"); - - //if(!fb.file){ - // sRc = -1; - // T5LOG(T5ERROR) << "::Can't create file " << fb.fileName; - //} + fb.file = LockedFile(fb.fileName, LockedFile::CalledFrom::Create); } if ( !sRc ) @@ -1787,7 +1782,6 @@ SHORT BTREE::QDAMDictCreateLocal /******************************************************************/ /* do settings depending if we are dealing with a dict or a tm.. */ /******************************************************************/ - usVersion = NTM_VERSION3;// changed from NTM_VERSION2 strcpy(chEQF,BTREE_HEADER_VALUE_TM3); fTerse = FALSE; /******************************************************************/ @@ -2065,16 +2059,13 @@ LONG QDAMGetszKey_V3 RECPARAM QDAMGetrecData_V3 ( std::shared_ptr& pRecord, - SHORT sMid, // key number - USHORT usVersion // version of database + SHORT sMid // key number ) { RECPARAM recData; // data description structure - PUSHORT pusOffset; - // use record number of passed entry , read in record and pass // back pointer - pusOffset = (PUSHORT) pRecord->contents.uchData; + PUSHORT pusOffset = (PUSHORT) pRecord->contents.uchData; pusOffset += sMid; // point to key PCHAR pData = (PCHAR)(pRecord->contents.uchData + *pusOffset); pData += sizeof(USHORT ); // get pointer to datarec @@ -2200,7 +2191,7 @@ SHORT BTREE::QDAMFindRecord_V3 { sHigh = 0; } /* endif */ - recData = QDAMGetrecData_V3( pRecord, sHigh, usVersion ); + recData = QDAMGetrecData_V3( pRecord, sHigh ); } /* endif */ BTREEUNLOCKRECORD( pRecord ); // unlock previous record. @@ -3036,7 +3027,7 @@ SHORT QDAMFirstEntry_V3 { while (!sRc && !IS_LEAF(pRecord)) { - recData = QDAMGetrecData_V3( pRecord, 0, pBT->usVersion); + recData = QDAMGetrecData_V3( pRecord, 0); sRc = pBTIda->QDAMReadRecord_V3( recData.usNum, pRecord , FALSE ); } /* endwhile */ if ( !sRc ) @@ -3189,7 +3180,7 @@ SHORT QDAMFindParent_V3 { sHigh = 0; } /* endif */ - recData = QDAMGetrecData_V3( pTempRec, sHigh, pBT->usVersion ); + recData = QDAMGetrecData_V3( pTempRec, sHigh ); } /* endif */ BTREEUNLOCKRECORD( pTempRec ); // unlock previous record. @@ -3233,8 +3224,7 @@ VOID QDAMCopyKeyTo_V3 std::shared_ptr& pRecord, SHORT i, std::shared_ptr& pNew, - SHORT j, - USHORT usVersion // version of database + SHORT j ) { PUSHORT pusOldOffset; // offset of data @@ -3324,7 +3314,7 @@ VOID QDAMReArrangeKRec_V3 { if ( *(pusOffset+i) ) { - QDAMCopyKeyTo_V3( pRecord, i, pNew, j, pBT->usVersion ); + QDAMCopyKeyTo_V3( pRecord, i, pNew, j ); i++; j++; } @@ -3469,7 +3459,6 @@ SHORT QDAMSplitNode_V3 recData.usNum = RECORDNUM(record); recData.usOffset = 0; - //lParentKey = QDAMGetszKey_V3( *record, 0, pBT->usVersion ); lParentKey = QDAMGetszKey_V3( record, 0 ); //if ( lParentKey ) @@ -3526,7 +3515,6 @@ SHORT QDAMSplitNode_V3 TYPE(newRecord) = (CHAR)(TYPE(record) & ~ROOT_NODE); // don't copy Root bit // Decide where to split the record - //lParentKey = QDAMGetszKey_V3( *record, (SHORT)(OCCUPIED(*record)/2), pBT->usVersion ); lParentKey = QDAMGetszKey_V3( record, (SHORT)(OCCUPIED(record)/2) ); if ( lParentKey != -1 ) @@ -3579,7 +3567,7 @@ SHORT QDAMSplitNode_V3 pusOffset = (PUSHORT) (record)->contents.uchData; while ( i < (SHORT) OCCUPIED( record )) { - QDAMCopyKeyTo_V3( record, i, newRecord, j, pBT->usVersion ); + QDAMCopyKeyTo_V3( record, i, newRecord, j ); *(pusOffset+i) = 0 ; // mark it as deleted j++; i++; @@ -4153,7 +4141,7 @@ SHORT QDAMFindChild_V3 { sHigh = 0; } /* endif */ - recData = QDAMGetrecData_V3( pRecord, sHigh, pBT->usVersion ); + recData = QDAMGetrecData_V3( pRecord, sHigh ); } /* endif */ BTREEUNLOCKRECORD( pRecord ); // unlock previous record. @@ -4600,9 +4588,9 @@ USHORT UtlSetFHandStateHwnd( HFILE hf, USHORT fsState, BOOL fMsg, HWND hwnd ) //------------------------------------------------------------------------------ // Internal function //------------------------------------------------------------------------------ -// Function name: QDAMDictOpenLocal Open local Dictionary +// Function name: EQFNTMOpen Open local Dictionary //------------------------------------------------------------------------------ -// Function call: QDAMDictOpenLocal( PSZ, SHORT, BOOL, PPBTREE ); +// Function call: EQFNTMOpen( PSZ, SHORT, BOOL, PPBTREE ); // //------------------------------------------------------------------------------ // Description: Open a file locally for processing @@ -4655,14 +4643,13 @@ USHORT UtlSetFHandStateHwnd( HFILE hf, USHORT fsState, BOOL fMsg, HWND hwnd ) // endif // return Rc //------------------------------------------------------------------------------ -SHORT BTREE::QDAMDictOpenLocal +SHORT BTREE::EQFNTMOpen ( - SHORT sNumberOfBuffers, // number of buffers - USHORT usOpenFlags // Read Only or Read/Write + USHORT usOpenFlags, // Read Only or Read/Write + SHORT sNumberOfBuffers // number of buffers ) { SHORT i; - //BTREEHEADRECORD header; SHORT sRc = 0; // return code USHORT usFlags; // set the open flags USHORT usAction; // return code from UtlOpen @@ -4703,8 +4690,6 @@ SHORT BTREE::QDAMDictOpenLocal /* (Check only first 3 characters of BTREE identifier to */ /* ignore the version number) */ /************************************************************/ - usVersion = (USHORT) header.chEQF[3]; - //compare = NTMKeyCompare; UtlTime( &(lTime) ); // set open time sCurrentIndex = 0; @@ -4721,6 +4706,31 @@ SHORT BTREE::QDAMDictOpenLocal } /* endif */ ASDLOG(); + memcpy( &chCollate, header.chCollate, /*COLLATE_SIZE*/ sizeof(chCollate) ); + //fix chCollate if neede + const long numOfRecordsInTheFile = fb.data.size()/BTREE_REC_SIZE_V3; + if(fb.isTMDFilebuffer()){ + if(chCollate.lStartKey != FIRST_KEY){ + T5LOG(T5ERROR) + << "info in the header of the tmd is corrupted, and chCollate.lStartKey=" + << chCollate.lStartKey <<"; (expected " << FIRST_KEY <<"); isCorrupted for this memory is set to true;"; + //chCollate.lStartKey = FIRST_KEY; + sRc = BTREE_CORRUPTED; + } + + if(chCollate.lNextKey >= LASTALLOWEDKEY){ + T5LOG(T5ERROR) << "info in the header of the tm is corrupted, and chCollate.lNextKey=" + << chCollate.lNextKey <<"; (expected <" << LASTALLOWEDKEY <<"); isCorrupted for this memory is set to true;"; + sRc = BTREE_CORRUPTED; + } + }else{ + if(chCollate.lStartKey != START_KEY){ + T5LOG(T5ERROR) << "info in the header of the tmi is corrupted, and chCollate.lStartKey=" + << chCollate.lStartKey <<"; (expected " << START_KEY <<"); isCorrupted for this memory is set to true;"; + sRc = BTREE_CORRUPTED; + } + } + if ( !sRc ) { memcpy(DataRecList, header.DataRecList, MAX_LIST*sizeof(DataRecList[0])); @@ -4731,8 +4741,6 @@ SHORT BTREE::QDAMDictOpenLocal { QDAMTerseInit( chEntryEncode ); // init compression } /* endif */ - - memcpy( &chCollate, header.chCollate, /*COLLATE_SIZE*/ sizeof(chCollate) ); ASDLOG(); @@ -4759,19 +4767,12 @@ SHORT BTREE::QDAMDictOpenLocal } /* endif */ ASDLOG(); // close in case of error - if ( sRc ) + if ( sRc && (sRc != BTREE_CORRUPTED)) { QDAMDictClose(); } /* endif */ ASDLOG(); } - - if ( sRc ) - { - T5LOG(T5ERROR) << ":: sRc" << sRc; - ERREVENT2( QDAMDICTOPENLOCAL_LOC, INTFUNCFAILED_EVENT, sRc, DB_GROUP, "" ); - } /* endif */ - //fOpen = true; return ( sRc ); } @@ -5665,7 +5666,7 @@ SHORT QDAMFirst_V3 precBTree->usOffset = pBTIda->sCurrentIndex; precBTree->usNum = pBTIda->usCurrentRecord; *precKey = QDAMGetrecKey_V3( pRecord, 0 ); - *precData = QDAMGetrecData_V3( pRecord, 0, pBT->usVersion ); + *precData = QDAMGetrecData_V3( pRecord, 0 ); } } } diff --git a/source/opentm2/core/utilities/FilesystemHelper.cpp b/source/opentm2/core/utilities/FilesystemHelper.cpp index a462cb84..c6bf9c95 100755 --- a/source/opentm2/core/utilities/FilesystemHelper.cpp +++ b/source/opentm2/core/utilities/FilesystemHelper.cpp @@ -660,7 +660,7 @@ int FileBuffer::Read(void* buff, size_t buffSize,ssize_t startingPos){ return 0; } -bool FileBuffer::isTMDFilebuffer(){ +bool FileBuffer::isTMDFilebuffer()const{ return EncodingHelper::endsWithIgnoreCase(fileName, ".tmd" ); } From 57930397dc04b771a23a478122c29b4444edd19b Mon Sep 17 00:00:00 2001 From: orestdrag Date: Fri, 8 Aug 2025 18:02:36 +0300 Subject: [PATCH 49/62] * updated to v0.7.22 + added check for reorganize to have at least 3 langs in the table and errcode 6011 --- source/CMakeLists.txt | 2 +- source/opentm2/core/requestdata.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 1e11be98..0a86da1b 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 21) +SET(APP_VERSION_MINOR 22) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 925aad1d..024b2d0d 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -1618,6 +1618,13 @@ int ReorganizeRequestData::execute(){ return buildErrorReturn( 500, msg.c_str() ); } + const auto numOfLangs = mem->Languages.stTmTableEntry.size(); + if (numOfLangs <3)// if there are less than 3 langs-reorganize shouldn'b be allowed(we neew srcLang, trgLang and empty lang) + { + std::string msg = "mem can't be reorganized without losing all the data, because there are " + std::to_string(numOfLangs) + + " language entries in the table(min is 3 - srcLang, trgLang and emptyLang)"; + return buildErrorReturn( TMERR_LANG_TOO_SMALL_FOR_REORGANIZE, msg.c_str(), 500 ); + } mem->eStatus = REORGANIZE_RUNNING_STATUS; mem->eImportStatus = REORGANIZE_RUNNING_STATUS; From 9409e34a248fa379fbc3f030c4a6d2eae3e79ae8 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Fri, 8 Aug 2025 18:05:07 +0300 Subject: [PATCH 50/62] * fixed compilation issue --- include/lowlevelotmdatastructs.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 4dc160ad..70aa5367 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -1644,8 +1644,9 @@ typedef enum _PROCWINSTYLE /**********************************************************************/ /* Return Codes */ /**********************************************************************/ -#define TMERR_BASE ERR_MEM_BASE /* Base for errors */ +#define TMERR_BASE ERR_MEM_BASE /* Base for errors 6000 */ +#define TMERR_LANG_TOO_SMALL_FOR_REORGANIZE TMERR_BASE + 11 #define TMERR_EOF TMERR_BASE + 12 #define DISK_FULL TMERR_BASE + 13 // for Add /Replace / Create From ee7c507e5945845f4f68f26ae48d0bc4480a5cfc Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 12 Aug 2025 23:05:31 +0300 Subject: [PATCH 51/62] * updated to v0.7.23 * fixed longname table deserialization * separated memory state to other class to - disabled and deleted shortname table + added psLangIdToGroupTable to tables request * fixed import-reorganize btree crash * optimized reoganizeDone - code cleanup --- include/EQFOSWIN.H | 7 +- include/Filebuffer.h | 3 +- include/lowlevelotmdatastructs.h | 372 ++++-------------- include/tm.h | 203 +++------- source/CMakeLists.txt | 2 +- source/RestAPI/OtmMemoryServiceWorker.cpp | 19 +- source/RestAPI/OtmMemoryServiceWorker.h | 10 - source/RestAPI/ProxygenHandler.cpp | 2 - source/RestAPI/ProxygenHandler.h | 3 +- source/RestAPI/ProxygenServer.cpp | 6 +- .../opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp | 24 +- .../opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 8 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 14 +- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 38 +- .../opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp | 55 ++- .../opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp | 84 ++-- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 282 ++++++------- .../core/EqfMemoryPlugin/EqfMemory.cpp | 108 +++-- .../core/EqfMemoryPlugin/EqfMemoryPlugin.cpp | 2 +- source/opentm2/core/requestdata.cpp | 130 +++--- source/opentm2/core/tm.cpp | 77 ++-- source/opentm2/core/utilities/EQFDICT.cpp | 8 +- source/opentm2/core/utilities/EQFUTDOS.cpp | 53 +-- .../core/utilities/FilesystemHelper.cpp | 42 +- .../opentm2/core/utilities/LanguageFactory.H | 2 - .../core/utilities/LanguageFactory.cpp | 7 +- source/opentm2/core/utilities/LogWrapper.h | 14 +- source/opentm2/core/utilities/OSWrapper.cpp | 45 +-- source/opentm2/core/utilities/OSWrapper.h | 9 - source/opentm2/core/utilities/UtlMisc.cpp | 42 -- source/opentm2/core/utilities/UtlString.cpp | 26 +- .../OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c | 233 ----------- .../OtmMarkupTablePlugin.cpp | 59 --- .../UserMarkupTablePlugin.cpp | 108 ----- 36 files changed, 572 insertions(+), 1533 deletions(-) diff --git a/include/EQFOSWIN.H b/include/EQFOSWIN.H index 4270ad0e..3f2cf910 100755 --- a/include/EQFOSWIN.H +++ b/include/EQFOSWIN.H @@ -1047,12 +1047,7 @@ USHORT DosGetDateTime( PDATETIME pDateTime ); /**********************************************************************/ SHORT GetNextSelection( HWND hwndLB, SHORT sPos ); -/**********************************************************************/ -/* force use of GetOEMCP- needed for 32-bit */ -/* GetKBCodePage returns 437 for TChin/SChin/Korean under Win95! */ -/* which causes that no DBCS chars are detected!! */ -/**********************************************************************/ -#define GetKBCodePage() GetOEMCP() + /**********************************************************************/ /* allow to fill the rectangle in the selected color */ diff --git a/include/Filebuffer.h b/include/Filebuffer.h index 69e12907..56dd03fb 100644 --- a/include/Filebuffer.h +++ b/include/Filebuffer.h @@ -91,6 +91,7 @@ class LockedFile { FILE* handle() const { return file_; } const std::string& path() const { return path_; } size_t GetFileSize() const; + std::string fileName; private: std::string path_; @@ -108,9 +109,7 @@ struct FileBuffer{ int status ;//FILEBUFFERSTATUS std::vector data; size_t offset = 0; - //FILE* file = nullptr; LockedFile file; - std::string fileName; FilebufferType filebufferType; long long originalFileSize = 0; diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index 70aa5367..b09ecd10 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -337,9 +337,9 @@ typedef struct _TMX_SIGN BYTE bGlobVersion=0; BYTE bMajorVersion=0; BYTE bMinorVersion=0; - CHAR szName[_TMX_SIGN_SZ_NAME]; - CHAR szSourceLanguage[MAX_LANG_LENGTH]; - CHAR szDescription[MAX_MEM_DESCRIPTION]; + CHAR szName[_TMX_SIGN_SZ_NAME]{}; + CHAR szSourceLanguage[MAX_LANG_LENGTH]{}; + CHAR szDescription[MAX_MEM_DESCRIPTION]{}; LONG segmentIndex = 0;//highest generated segment id, used for generating new id's } TMX_SIGN, * PTMX_SIGN; @@ -492,16 +492,16 @@ constexpr int NUM_OF_TMX_TABLE_ENTRIES = (BTREE_REC_SIZE_V3 - sizeof(ULONG)) / s struct TMX_TABLE_OLD{ ULONG ulMaxEntries = 0; - TMX_TABLE_ENTRY stTmTableEntry[NUM_OF_TMX_TABLE_ENTRIES]; + TMX_TABLE_ENTRY table[NUM_OF_TMX_TABLE_ENTRIES]; }; typedef TMX_TABLE_OLD * PTMX_TABLE_OLD; struct TMX_TABLE { - std::vector stTmTableEntry; + std::vector table; TMX_TABLE(){ - stTmTableEntry.push_back(TMX_TABLE_ENTRY{});//push_empty_entry + table.push_back(TMX_TABLE_ENTRY{});//push_empty_entry } }; @@ -604,18 +604,17 @@ struct BTREE std::vector TempRecord; std::map> LookupTable; // Pointer to lookup-table + //methods BTREE(){ TempRecord = std::vector(MAXDATASIZE, 0); // Ensure proper initialization for arrays and complex types BTreeTempBuffer_V3 = std::make_shared(); } - void freeMem(){ freeLookupTable(); } - bool wasModified() const{ return fb.WasModified(); } int initLookupTable(); @@ -624,136 +623,18 @@ struct BTREE int allocateNewLookupTableBuffer(int number,std::shared_ptr& pBuffer); int deallocateLookupTableBuffer(int number); - //+----------------------------------------------------------------------------+ - // Internal function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMUpdSign Write User Data - //+----------------------------------------------------------------------------+ - // Function call: QDAMDictUpdSignLocal( PBTREE, PCHAR, USHORT ); - // - //+----------------------------------------------------------------------------+ - // Description: Writes the second part of the first record (user data) - // This is done using the original QDAMDictUpdSignLocal - // function - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pointer to btree structure - // PCHAR pointer to user data - // USHORT length of user data - // - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: 0 no error happened - // BTREE_DISK_FULL disk full condition encountered - // BTREE_WRITE_ERROR write error to disk - // BTREE_INVALID pointer invalid - // BTREE_USERDATA user data too long - // BTREE_CORRUPTED dictionary is corrupted - //+----------------------------------------------------------------------------+ - // NOTE: This function could be implemented as MACRO too, but - // for consistency reasons, the little overhead was used... - //+----------------------------------------------------------------------------+ - ULONG GetNumOfSavedRecords()const; - SHORT QDAMDictFlushLocal(); - - //+----------------------------------------------------------------------------+ - // Internal function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMSign Read signature record - //+----------------------------------------------------------------------------+ - // Function call: QDAMDictSignLocal( PBTREE, PCHAR, PUSHORT ); - //+----------------------------------------------------------------------------+ - // Description: Gets the second part of the first record ( user data ) - // This is done using the original QDAMDictSignLocal func. - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pointer to btree structure - // PCHAR pointer to user data - // PUSHORT length of user data area (input) - // filled length (output) - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: 0 no error happened - // BTREE_INVALID pointer invalid - // BTREE_USERDATA user data too long - // BTREE_NO_BUFFER no buffer free - // BTREE_READ_ERROR read error from disk - // - //+----------------------------------------------------------------------------+ - // Side effects: return signature record even if dictionary is corrupted - //+----------------------------------------------------------------------------+ - // NOTE: This function could be implemented as MACRO too, but - // for consistency reasons, the little overhead was used... - //+----------------------------------------------------------------------------+ - SHORT EQFNTMSign(TMX_SIGN& ); - - - VOID QDAMTerseInit (unsigned char* ); - - //SHORT - //EQFNTMInsert - //( - // PULONG pulKey, // pointer to key - // PBYTE pData, // pointer to user data - // ULONG ulLen // length of user data - //); - - - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMInsert - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMInsert( pBTIda, &ulKey, pData, usLen ); - //+----------------------------------------------------------------------------+ - // Description: insert a new key (ULONG) with data - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pBTIda, pointer to binary tree struct - // PULONG pulKey, pointer to key - // PBYTE pData, pointer to user data - // ULONG ulLen length of user data - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: BTREE_NUMBER_RANGE requested key not in allowed range - // BTREE_READONLY file is opened read only - no write - // BTREE_CORRUPTED file is corrupted - // errors returned by QDAMDictInsertLocal - // 0 success indicator - //+----------------------------------------------------------------------------+ + VOID QDAMTerseInit (unsigned char* ); SHORT EQFNTMInsert ( - LONG& lKey, // pointer to key - BytesRef pData // pointer to user data + LONG& lKey, // pointer to key + BytesRef pData // pointer to user data ); - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMUpdate - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMUpdate( pBTIda, ulKey, pData, usLen ); - //+----------------------------------------------------------------------------+ - // Description: update the data of an already inserted key - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pBTIda, pointer to binary tree struct - // ULONG ulKey, key value - // PBYTE pData, pointer to user data - // ULONG ulLen length of user data - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: BTREE_NUMBER_RANGE requested key not in allowed range - // BTREE_READONLY file is opened read only - no write - // BTREE_CORRUPTED file is corrupted - // errors returned by QDAMDictInsertLocal - // 0 success indicator - //+----------------------------------------------------------------------------+ SHORT EQFNTMUpdate ( @@ -762,102 +643,11 @@ struct BTREE bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed ); - - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMGet - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMGet( pBTIda, ulKey, chData, &usLen ); - //+----------------------------------------------------------------------------+ - // Description: get the data string for the passed key - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pBTIda, pointer to btree struct - // ULONG ulKey, key to be searched for - // PCHAR pchBuffer, space for user data - // PULONG pulLength in/out length of returned user data - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: same as for QDAMDictExactLocal... - //+----------------------------------------------------------------------------+ - //SHORT - //EQFNTMGet - //( - //LONG lKey, // key to be searched for - //BytesRef uchBuff // space for user data - //); - - - //+----------------------------------------------------------------------------+ - // External function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMPhysLock - //+----------------------------------------------------------------------------+ - // Function call: sRc = EQFNTMPhysLock( pBTIda ); - //+----------------------------------------------------------------------------+ - // Description: Physicall lock or unlock database. - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE The database to be locked - // BOOL TRUE = LOCK, FALSE = Unlock - // PBOOL ptr to locked flag (set to TRUE if - // locking was successful - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ SHORT EQFNTMPhysLock ( - BOOL fLock, - PBOOL pfLocked + BOOL fLock, + PBOOL pfLocked ); - - //+----------------------------------------------------------------------------+ - // Internal function - //+----------------------------------------------------------------------------+ - // Function name: QDAMIncrUpdCounter Inrement database update counter - //+----------------------------------------------------------------------------+ - // Function call: QDAMIncrUpdCounter( PBTREE, SHORT sIndex ) - // - //+----------------------------------------------------------------------------+ - // Description: Update one of the update counter field in the dummy - // /locked terms file - // - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pointer to btree structure - // SHORT index of counter field - // PLONG ptr to buffer for new counte value - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ - // Returncodes: 0 no error happened - // BTREE_DISK_FULL disk full condition encountered - // BTREE_WRITE_ERROR write error to disk - // - //+----------------------------------------------------------------------------+ - // Function flow: read update counter from dummy file - // increment update counter - // position ptr to begin of file - // write update counter to disk - //+----------------------------------------------------------------------------+ - - - //+----------------------------------------------------------------------------+ - // Internal function - //+----------------------------------------------------------------------------+ - // Function name: EQFNTMGetUpdCounter Get database update counter - //+----------------------------------------------------------------------------+ - // Function call: EQFNTMGetUpdCounter( PBTREE, PLONG, SHORT, SHORT ); - //+----------------------------------------------------------------------------+ - // Description: Get one or more of the the database update counters - // from the dummy/locked terms file - //+----------------------------------------------------------------------------+ - // Parameters: PBTREE pointer to btree structure - // PLONG ptr to buffer for update counter - // SHORT index of requested update counter - // SHORT number of counters requested - //+----------------------------------------------------------------------------+ - // Returncode type: SHORT - //+----------------------------------------------------------------------------+ VOID QDAMUpdateList_V3( std::shared_ptr& ); @@ -870,7 +660,6 @@ struct BTREE SHORT EQFNTMGet ( LONG&, BytesRef ); - SHORT QDAMDictCreateLocal ( TMX_SIGN*, LONG, bool keepInRamOnly = false); SHORT QDAMDictInsertLocal ( LONG&, BytesRef pData ); @@ -878,7 +667,6 @@ struct BTREE VOID QDAMDictUpdStatus (); SHORT QDAMFindRecord_V3( LONG&, std::shared_ptr&); - //------------------------------------------------------------------------------ // Internal function //------------------------------------------------------------------------------ @@ -984,7 +772,7 @@ struct BTREE SHORT QDAMReadRecord_V3 ( - USHORT usNumber, + USHORT usNumber, std::shared_ptr& pReadBuffer, BOOL fNewRec, bool fSuppressTmdSizeCheck = false //for deletion there no need to check if size is bigger than allowed @@ -996,7 +784,7 @@ struct BTREE SHORT QDAMDictUpdSignLocal ( - PTMX_SIGN pSign // pointer to user data + PTMX_SIGN pSign // pointer to user data ); SHORT QDAMReadRecordFromDisk_V3( USHORT, std::shared_ptr&, BOOL, bool ); @@ -1014,92 +802,64 @@ struct BTREE ); SHORT QDAMCheckDict( PSZ pName); // name of dictionary + SHORT QDAMDictClose(); - //SHORT QDAMDictCloseLocal (); - //+----------------------------------------------------------------------------+ -// External function -//+----------------------------------------------------------------------------+ -// Function name: QDAMDictClose close the dictionary -//+----------------------------------------------------------------------------+ -// Function call: QDAMDictClose( PPBTREE ); -// -//+----------------------------------------------------------------------------+ -// Description: Close the file -// -//+----------------------------------------------------------------------------+ -// Parameters: PPBTREE pointer to btree structure -//+----------------------------------------------------------------------------+ -// Returncode type: SHORT -//+----------------------------------------------------------------------------+ -// Returncodes: 0 no error happened -// BTREE_INVALID incorrect pointer -// BTREE_DISK_FULL disk full condition encountered -// BTREE_WRITE_ERROR write error to disk -// BTREE_CORRUPTED dictionary is corrupted -// BTREE_CLOSE_ERROR error closing dictionary -//+----------------------------------------------------------------------------+ -SHORT QDAMDictClose(); - - -SHORT QDAMDictSignLocal -( - PCHAR pUserData, // pointer to user data - PUSHORT pusLen // length of user data -); - -//+----------------------------------------------------------------------------+ -// Internal function -//+----------------------------------------------------------------------------+ -// Function name: EQFNTMClose close the TM file -//+----------------------------------------------------------------------------+ -// Function call: EQFNTMClose( PPBTREE ); -//+----------------------------------------------------------------------------+ -// Description: Close the file -//+----------------------------------------------------------------------------+ -// Parameters: PPBTREE pointer to btree structure -//+----------------------------------------------------------------------------+ -// Returncode type: SHORT -//+----------------------------------------------------------------------------+ -// Returncodes: 0 no error happened -// BTREE_INVALID incorrect pointer -// BTREE_DISK_FULL disk full condition encountered -// BTREE_WRITE_ERROR write error to disk -// BTREE_CORRUPTED dictionary is corrupted -// BTREE_CLOSE_ERROR error closing dictionary -//+----------------------------------------------------------------------------+ - -SHORT EQFNTMClose(); - -SHORT QDAMNewRecord_V3 -( - std::shared_ptr& pRecord, - RECTYPE recType, // data or key record - bool fSuppressTmdSizeCheck = false -); - -SHORT QDAMInsertKey_V3 -( - std::shared_ptr& pRecord, // record where key is to be inserted - LONG& ulKey, - RECPARAM recKey, // position/offset for key - RECPARAM recData // position/offset for data -); - -size_t GetFileSize()const; + SHORT QDAMDictSignLocal + ( + PCHAR pUserData, // pointer to user data + PUSHORT pusLen // length of user data + ); + //+----------------------------------------------------------------------------+ + // Internal function + //+----------------------------------------------------------------------------+ + // Function name: EQFNTMClose close the TM file + //+----------------------------------------------------------------------------+ + // Function call: EQFNTMClose( PPBTREE ); + //+----------------------------------------------------------------------------+ + // Description: Close the file + //+----------------------------------------------------------------------------+ + // Parameters: PPBTREE pointer to btree structure + //+----------------------------------------------------------------------------+ + // Returncode type: SHORT + //+----------------------------------------------------------------------------+ + // Returncodes: 0 no error happened + // BTREE_INVALID incorrect pointer + // BTREE_DISK_FULL disk full condition encountered + // BTREE_WRITE_ERROR write error to disk + // BTREE_CORRUPTED dictionary is corrupted + // BTREE_CLOSE_ERROR error closing dictionary + //+----------------------------------------------------------------------------+ -BOOL QDAMTerseData -( - BytesRef , - PULONG pulLen // length of the string -); + SHORT EQFNTMClose(); + SHORT QDAMNewRecord_V3 + ( + std::shared_ptr& pRecord, + RECTYPE recType, // data or key record + bool fSuppressTmdSizeCheck = false + ); - SHORT QDAMLocateKey_V3( - std::shared_ptr&, LONG&, PSHORT, SEARCHTYPE, PSHORT); - - SHORT QDAMGetszData_V3 ( RECPARAM, BytesRef ); + SHORT QDAMInsertKey_V3 + ( + std::shared_ptr& pRecord, // record where key is to be inserted + LONG& ulKey, + RECPARAM recKey, // position/offset for key + RECPARAM recData // position/offset for data + ); + + size_t GetFileSize()const; + + BOOL QDAMTerseData + ( + BytesRef , + PULONG pulLen // length of the string + ); + SHORT QDAMLocateKey_V3( + std::shared_ptr&, LONG&, PSHORT, SEARCHTYPE, PSHORT); + + SHORT QDAMGetszData_V3 ( RECPARAM, BytesRef ); };//BTREE typedef BTREE * PBTREE; diff --git a/include/tm.h b/include/tm.h index 6cc3cddd..529898f3 100644 --- a/include/tm.h +++ b/include/tm.h @@ -924,7 +924,7 @@ typedef struct _TM_HEADER // { // USHORT usAllocSize; // USHORT usMaxEntries; -// TMX_TABLE_ENTRY stTmTableEntry; +// TMX_TABLE_ENTRY table; // } TMX_VER1_TABLE, * PTMX_VER1_TABLE; /**********************************************************************/ @@ -1785,36 +1785,31 @@ class RequestData; std::string StatusToString(int eStatus); #include -class EqfMemory //: public TMX_CLB -/*! \brief This class implements the standard translation memory (EQF) for OpenTM2. - */ -{ - bool fCorrupted = false;; -public: - std::recursive_timed_mutex tmMutex; - // bool fOpen = false; +namespace EqfMemoryError { + constexpr int ERROR_INTERNALKEY_MISSING = 1001; + constexpr int ERROR_MEMORYOBJECTISNULL = 8003; + constexpr int INFO_ENDREACHED = 8008; + constexpr int ERROR_ENTRYISCORRUPTED = 8009; +} + +struct EqfMemoryState{ BTREE TmBtree; BTREE InBtree; TMX_TABLE Languages; - TMX_TABLE FileNames; TMX_TABLE Authors; TMX_TABLE TagTables; + TMX_TABLE LangGroups; // table containing language group names + std::vector psLangIdToGroupTable; // language ID to group ID table + + size_t expectedSizeInRam = 0; + bool fCorrupted = false; USHORT usAccessMode = 0; - USHORT usThreshold = 0; TMX_SIGN stTmSign; std::vector bCompact; BYTE bCompactChanged = 0; - LONG alUpdCounter[MAX_UPD_COUNTERS]; - // std::vector pLongNames ; - TMX_TABLE LangGroups; // table containing language group names - std::vector psLangIdToGroupTable; // language ID to group ID table PVOID pTagTable = nullptr; // tag table loaded for segment markup (TBLOADEDTABLE) - BYTE getMajorVersion()const{ - return stTmSign.bMajorVersion; - } - // copy of long name table sorted ignoring the case of the file names // Note: only the stTableEntry array is filled in this area, for all other // information use the entries in the pLongNames structure @@ -1824,62 +1819,67 @@ class EqfMemory //: public TMX_CLB // fields for work area pointers of various subfunctions which are allocated // only once for performance reasons std::vector pvTempMatchList; // matchlist of FillMatchEntry function - std::vector pvIndexRecord; // index record area of FillMatchEntry function std::vector pvTmRecord; // buffer for memory record used by GetFuzzy and GetExact - ULONG ulRecBufSize = 0; // current size of pvTMRecord; - - // fields for time measurements and logging - BOOL fTimeLogging = 0; // TRUE = Time logging is active - LONG64 lAllocTime = 0; // time for memory allocation - LONG64 lTokenizeTime = 0; // time for tokenization - LONG64 lGetExactTime = 0; // time for GetExact - LONG64 lOtherTime = 0; // time for other activities - LONG64 lGetFuzzyTime = 0; // time for GetFuzzy - LONG64 lFuzzyOtherTime = 0; // other time spent in GetFuzzy - LONG64 lFuzzyGetTime = 0; // NTMGet time spent in GetFuzzy - LONG64 lFuzzyFillMatchEntry = 0; // FillMatchEntry time spent in GetFuzzy - LONG64 lFillMatchAllocTime = 0; // FillMatchEntry: allocation time - LONG64 lFillMatchOtherTime = 0; // FillMatchEntry: other times - LONG64 lFillMatchReadTime = 0; // FillMatchEntry: read index DB time - LONG64 lFillMatchFillTime = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchCleanupTime = 0; // FillMatchEntry: cleanup match list time - LONG64 lFillMatchFill1Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill2Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill3Time = 0; // FillMatchEntry: fill match list time - LONG64 lFillMatchFill4Time = 0; // FillMatchEntry: fill match list time - - // std::shared_ptr readOnlyPtr; - // std::shared_ptr writePtr; + +}; + +class EqfMemory //: public TMX_CLB +/*! \brief This class implements the standard translation memory (EQF) for OpenTM2. + */ + +{ +public: + std::recursive_timed_mutex tmMutex; + + std::unique_ptr state; + std::shared_ptr readOnlyCnt; std::shared_ptr writeCnt; - - // std::weak_ptr pActiveRequest; RequestData *pActiveRequest; - int lastErrorCode = 0; + COMMAND activeCommand = UNKNOWN_COMMAND; + time_t tLastAccessTime = 0; // last time memory has been used + MEMORY_STATUS eStatus; // status of the memory + MEMORY_STATUS eImportStatus; // status of the current/last memory import + ImportStatusDetails *importDetails = nullptr; - size_t expectedSizeInRam = 0; + std::string strError; // pointer to an error message (only when eStatus is IMPORT_FAILED_STATUS) + + std::string szName; // name of the memory + std::string szDescription; // description of the memory + std::string szSourceLanguage; // memory source language + + +//methods + + /*! \brief Constructors + */ + + /*! \brief Destructor + */ + ~EqfMemory(); - int getErrorCode() const; - int setErrorCode(int rc); - void reserErrorCode(); + USHORT Load(); + + int sign(){ + return state->TmBtree.EQFNTMSign(state->stTmSign); + } COMMAND getActiveRequestCommand(); void setActiveRequest(std::weak_ptr request); void setActiveRequest(RequestData *request); void resetActiveRequest(); - COMMAND activeCommand = UNKNOWN_COMMAND; void setActiveRequestCommand(COMMAND command); void resetActiveRequestCommand(); void setExpecedSizeInRAM(size_t newSize) { - expectedSizeInRam = newSize; + state->expectedSizeInRam = newSize; } size_t getExpectedSizeInRAM() const { - return expectedSizeInRam; + return state->expectedSizeInRam; } std::string getStatusString() const; @@ -1906,7 +1906,7 @@ class EqfMemory //: public TMX_CLB } bool isCorrupted()const{ - return fCorrupted; + return state->fCorrupted; } bool isReorganizeRunning() const @@ -1922,14 +1922,14 @@ class EqfMemory //: public TMX_CLB // bool fReorganizeOnly = false; bool isOpenedOnlyForReorganize() const { - return usAccessMode & ASD_ORGANIZE; + return state->usAccessMode & ASD_ORGANIZE; } void resetOpenedOnlyForReorganize() { unsigned int mask = ~ASD_ORGANIZE; - usAccessMode &= mask; - fCorrupted = false; + state->usAccessMode &= mask; + state->fCorrupted = false; } void updateLastUseTime() @@ -1937,65 +1937,10 @@ class EqfMemory //: public TMX_CLB tLastAccessTime = time(0); } - /*! \brief OtmMemory related return codes - - */ - static const int ERROR_INTERNALKEY_MISSING = 1001; - - /*! \brief Constructors - */ - - /*! \brief Destructor - */ - ~EqfMemory(); - - USHORT Load(); - - int sign(){ - return TmBtree.EQFNTMSign(stTmSign); + BYTE getMajorVersion()const{ + return state->stTmSign.bMajorVersion; } - /*! \brief Info structure for an open translation memory - */ - // typedef struct _OPENEDMEMORY - //{ - // char szName[260]; // name of the memory - time_t tLastAccessTime = 0; // last time memory has been used - long lHandle = 0; // memory handle - MEMORY_STATUS eStatus; // status of the memory - MEMORY_STATUS eImportStatus; // status of the current/last memory import - // std::atomic dImportProcess; - // ushort * pusImportPersent = nullptr; - ImportStatusDetails *importDetails = nullptr; - std::string strError; // pointer to an error message (only when eStatus is IMPORT_FAILED_STATUS) - - PPROP_NTM memoryProperties; - /*! \brief structure for memory information */ - // typedef struct _MEMORYINFO - // { - - // } MEMORYINFO, *PMEMORYINFO; - //} OPENEDMEMORY ; - - /*! \brief Error code definition - */ - static const int ERROR_MEMORYOBJECTISNULL = 8003; - static const int ERROR_BUFFERTOOSMALL = 8004; - static const int ERROR_INVALIDOBJNAME = 8005; - static const int ERROR_MEMORYEXISTS = 8006; - static const int INFO_ENDREACHED = 8008; - static const int ERROR_ENTRYISCORRUPTED = 8009; - - /*! \brief Flags for the update of proposals */ - - static const int UPDATE_MARKUP = 0x01; // update markup/tag table - static const int UPDATE_MTFLAG = 0x02; // update machine translation flag - static const int UPDATE_TARGLANG = 0x04; // update target language - static const int UPDATE_DATE = 0x08; // update proposal update time - - // void setInternalCursor(ULONG recordKey, USHORT targetKey); - // void resetInternalCursor(); - ushort TmtXDelSegm( OtmProposal &pTmDelIn, // ptr to input struct TMX_EXT_OUT_W *pTmDelOut // ptr to output struct @@ -2146,8 +2091,6 @@ class EqfMemory //: public TMX_CLB USHORT UpdateTmRecord(OtmProposal &); - - int getNumOfMarkupNames(); /*! \brief Get markup name at position n [n = 0.. GetNumOfMarkupNames()-1] @@ -2238,30 +2181,6 @@ class EqfMemory //: public TMX_CLB char *pszError, int iBufSize); - /*! \brief Set or clear the pointer to a loaded global memory option file - - This method sets a pointer to a loaded global memory option file. - When set the option file will be used to decide how global memory proposals will be processed. - - \param pvGlobalMemoryOptions pointer to a loaded global memory option file or NULL to clear the current option file pointer - - \returns 0 or error code in case of errors - */ - - std::string strLastError; - int iLastError = 0; - -public: - std::string szName; // name of the memory - std::string szDescription; // description of the memory - std::string szFullPath; // full path to memory file(s) (if applicable only) - std::string szFullDataFilePath; // full path to memory file(s) (if applicable only) - std::string szFullIndexFilePath; // full path to memory file(s) (if applicable only) - std::string szSourceLanguage; // memory source language - std::string szOwner; // ID of the memory owner - std::string szDescrMemoryType; // descriptive name of the memory type - unsigned long ulSize = 0; // size of the memory - BOOL fEnabled = 0; // memory-is-enabled flag std::string getName() const { return szName; } // functions dealing with long document tables @@ -2322,7 +2241,6 @@ class EqfMemory //: public TMX_CLB PTMX_PUT_W pPutIn); /// newTM code -public: EqfMemory(const std::string &tmName); bool IsLoadedInRAM(); @@ -2330,7 +2248,6 @@ class EqfMemory //: public TMX_CLB int UnloadFromRAM(); int FlushFilebuffers(MutexTimeout &tmLockTimeout); -private: }; OtmProposal::eProposalType FlagToProposalType(USHORT usTranslationFlag); diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 0a86da1b..6d71d9f6 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 22) +SET(APP_VERSION_MINOR 23) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index 8369c170..5cc15297 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -81,23 +81,6 @@ OtmMemoryServiceWorker::OtmMemoryServiceWorker(){ } } -/*! \brief Data area for the processing of the openMemory function -*/ -typedef struct _OPENMEMORYDATA -{ - char szMemory[260]; - wchar_t szError[512]; -} OPENMEMORYDATA, *POPENMEMORYDATA; - -/*! \brief Data area for the processing of the createMemory function -*/ -typedef struct _CLOSEMEMORYDATA -{ - char szMemory[260]; - wchar_t szError[512]; -} CLOSEMEMORYDATA, *PCLOSEMEMORYDATA; - - /*! \brief convert a long time value to a date time string in the form YYYY-MM-DD HH:MM:SS \param lTime long time value @@ -364,7 +347,7 @@ void reorganizeMemoryProcess( void* pvData ) pRIDA->pMem->setActiveRequestCommand(COMMAND::REORGANIZE_MEM); int _rc_ = 0; - pRIDA->pMem->usAccessMode |= ASD_ORGANIZE; + pRIDA->pMem->state->usAccessMode |= ASD_ORGANIZE; while((_rc_ == NO_ERROR || _rc_ == CONTINUE_RC) && !pData->fComplete ) { // continue current organize process diff --git a/source/RestAPI/OtmMemoryServiceWorker.h b/source/RestAPI/OtmMemoryServiceWorker.h index efa2371f..811b7460 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.h +++ b/source/RestAPI/OtmMemoryServiceWorker.h @@ -16,9 +16,7 @@ #include #include #include "ProxygenStats.h" -//#include "EQFFUNCI.H" #include "tm.h" -#include "requestdata.h" @@ -38,12 +36,6 @@ class OtmMemoryServiceWorker */ static OtmMemoryServiceWorker* getInstance(); - - /* change the status of a memory in our memory list at the end of an import - \param pszMemory name of the memory - \param - */ - void importDone( char *pszMemory, int iRC, char *pszError ); /*! \brief OpenTM2 API session handle*/ HSESSION hSession = 0; @@ -52,8 +44,6 @@ class OtmMemoryServiceWorker /*! \brief Last return code*/ int iLastRC; -/*! \brief Buffer for last error message*/ - wchar_t szLastError[4000]; }; diff --git a/source/RestAPI/ProxygenHandler.cpp b/source/RestAPI/ProxygenHandler.cpp index fedc9185..f4525486 100755 --- a/source/RestAPI/ProxygenHandler.cpp +++ b/source/RestAPI/ProxygenHandler.cpp @@ -14,7 +14,6 @@ #include "../opentm2/core/utilities/Property.h" #include "ProxygenStats.h" -#include "OtmMemoryServiceWorker.h" #include #include "LogWrapper.h" #include "lowlevelotmdatastructs.h" @@ -55,7 +54,6 @@ void ProxygenHandler::onRequest(std::unique_ptr req) noexcept { auto path = req->getPath () ; auto queryString = req->getQueryString () ; auto headers = req->getHeaders(); - pMemService = OtmMemoryServiceWorker::getInstance(); auto command = pRequest->command; headers_ = std::move(req); diff --git a/source/RestAPI/ProxygenHandler.h b/source/RestAPI/ProxygenHandler.h index c5ec29cb..dde668c9 100755 --- a/source/RestAPI/ProxygenHandler.h +++ b/source/RestAPI/ProxygenHandler.h @@ -16,9 +16,9 @@ #include #include #include -#include "OtmMemoryServiceWorker.h" #include "ProxygenStats.h" #include +#include "requestdata.h" using namespace std::chrono; @@ -69,7 +69,6 @@ class ProxygenHandler : public proxygen::RequestHandler { std::string responseText = "WRONG_REQUEST"; proxygen::ResponseBuilder* builder = nullptr; - OtmMemoryServiceWorker* pMemService = nullptr; std::ofstream fileStream; private: diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index f5ff5a72..8e9eb8a0 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -23,7 +23,6 @@ #include "ProxygenStats.h" #include "EQF.H" -#include "OtmMemoryServiceWorker.h" #include "../opentm2/core/utilities/LogWrapper.h" #include "../opentm2/core/utilities/Property.h" #include "../opentm2/core/utilities/FilesystemWrapper.h" @@ -31,8 +30,8 @@ #include "ProxygenHTTPAcceptor.h" #include "../cmake/git_version.h" #include "config.h" -#include "OtmMemoryServiceWorker.h" #include "ProxygenHTTPAcceptor.h" +#include "OtmMemoryServiceWorker.h" DECLARE_string(servicename); DECLARE_string(t5_ip); @@ -360,7 +359,6 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { T5LOG( T5INFO) << "Trying to prepare t5memory"; - auto pMemService = OtmMemoryServiceWorker::getInstance(); char szServiceName[100] = "t5memory"; @@ -459,7 +457,6 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { // Start HTTPServer mainloop in a separate thread std::thread t([&]() { server.start(); }); - OtmMemoryServiceWorker::getInstance();//to init instance std::string runDate = getTimeStr(); Properties::GetInstance()->set_anyway(KEY_RUN_DATE, runDate); @@ -502,6 +499,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { }; int proxygen_server_init(){ + OtmMemoryServiceWorker::getInstance(); return ProxygenHandlerFactory::startService(); } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp index 1f8f93eb..6cab1feb 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEM00.cpp @@ -145,7 +145,7 @@ VOID EQFMemOrganizeProcess }catch(...) { pRIDA->pProposal->nextInternalKey.moveToNextRecord(); - iRC = EqfMemory::ERROR_ENTRYISCORRUPTED; + iRC = EqfMemoryError::ERROR_ENTRYISCORRUPTED; } if ( iRC == NO_ERROR ) @@ -234,7 +234,7 @@ VOID EQFMemOrganizeProcess } } /* endif */ } - else if ( iRC == EqfMemory::ERROR_ENTRYISCORRUPTED// || iRC == BTREE_BUFFER_SMALL + else if ( iRC == EqfMemoryError::ERROR_ENTRYISCORRUPTED// || iRC == BTREE_BUFFER_SMALL ) { std::scoped_lock l(pRIDA->pMem->importDetails->importDetailsMutex); @@ -244,9 +244,9 @@ VOID EQFMemOrganizeProcess //std::lock_guard l(pRIDA->pMem->importDetails->firstInvalidSegmentsSegNumsMutex); pRIDA->pMem->importDetails->firstInvalidSegmentsSegNums.push_back(std::make_tuple(pRIDA->pProposal->getSegmentId(), -9)); } - T5LOG(T5ERROR) << "Skipping proposal iRC == EqfMemory::ERROR_ENTRYISCORRUPTED ; proposal :" << *pRIDA->pProposal; + T5LOG(T5ERROR) << "Skipping proposal iRC == EqfMemoryError::ERROR_ENTRYISCORRUPTED ; proposal :" << *pRIDA->pProposal; } - else if ( iRC == EqfMemory::INFO_ENDREACHED ) + else if ( iRC == EqfMemoryError::INFO_ENDREACHED ) { pRIDA->pMemTemp->NTMOrganizeIndexFile(); { @@ -337,15 +337,17 @@ VOID EQFMemOrganizeEnd PMEM_ORGANIZE_IDA pRIDA = (PMEM_ORGANIZE_IDA)pCommArea->pUserIDA; if(nullptr != pRIDA->pMemTemp){ - pRIDA->pMem->TmBtree.fb.data = std::move(pRIDA->pMemTemp->TmBtree.fb.data); + auto TMDFileLock = std::move(pRIDA->pMem->state->TmBtree.fb.file); + auto TMIFileLock = std::move(pRIDA->pMem->state->InBtree.fb.file); + pRIDA->pMem->state = std::move(pRIDA->pMemTemp->state); + pRIDA->pMem->state->TmBtree.fb.file = std::move(TMDFileLock); + pRIDA->pMem->state->InBtree.fb.file = std::move(TMIFileLock); + + //get signature record and add to control block USHORT usRc = pRIDA->pMem->sign(); - pRIDA->pMem->TmBtree.fb.Flush(true); - - - pRIDA->pMem->InBtree.fb.data = std::move(pRIDA->pMemTemp->InBtree.fb.data); - pRIDA->pMem->InBtree.fb.Flush(true); - pRIDA->pMem->Reload(false); + pRIDA->pMem->state->TmBtree.fb.Flush(true); + pRIDA->pMem->state->InBtree.fb.Flush(true); } LONG lCurTime = 0; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp index 60717fa2..abd03721 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMEP.cpp @@ -619,9 +619,9 @@ USHORT MemExportProcess ( PMEM_EXPORT_IDA pExportIDA ) // pointer to the export } int iTmRC = pExportIDA->pMem->getNextProposal( *(pExportIDA->pProposal), &(pExportIDA->iComplete) ); - if ( (iTmRC == NO_ERROR) || iTmRC == EqfMemory::INFO_ENDREACHED ) + if ( (iTmRC == NO_ERROR) || iTmRC == EqfMemoryError::INFO_ENDREACHED ) { - if ( iTmRC == EqfMemory::INFO_ENDREACHED ) + if ( iTmRC == EqfMemoryError::INFO_ENDREACHED ) { //--- Stop Address has been reached before END_OF_TM pExportIDA->fEOF = TRUE; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index 0e7aad71..a694751c 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -104,7 +104,7 @@ USHORT EqfMemory::TmtXDelSegm { for (LONG lKey : plSids ) { - usRc = TmBtree.EQFNTMGet(lKey, //tm record key + usRc = state->TmBtree.EQFNTMGet(lKey, //tm record key pTmRecord ); //length if ( usRc == NO_ERROR ) @@ -153,7 +153,7 @@ USHORT EqfMemory::TmtXDelSegmByKey // update TM databse if ( !usRc ) { - usRc = TmBtree.EQFNTMGet( lKey, //tm record key + usRc = state->TmBtree.EQFNTMGet( lKey, //tm record key pTmRecord ); //pointer to tm record data if ( usRc == NO_ERROR ) @@ -372,7 +372,7 @@ USHORT EqfMemory::FindTargetAndDelete( /* by a new translation, we will not remove */ /* the key (only get rid of any target data) */ /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( lKey, TmRecord, true); + usRc = state->TmBtree.EQFNTMUpdate( lKey, TmRecord, true); } /* endif */ //leave while loop @@ -589,7 +589,7 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( /* by a new translation, we will not remove */ /* the key (only get rid of any target data) */ /**********************************************/ - usRc = TmBtree.EQFNTMUpdate( lKey, TmRecord, true); + usRc = state->TmBtree.EQFNTMUpdate( lKey, TmRecord, true); } /* endif */ if ( ! usRc ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 02380962..108b21a1 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -67,12 +67,12 @@ USHORT EqfMemory::TmtXExtract USHORT usRc = BTREE_NOT_FOUND; //return code PTMX_SOURCE_RECORD pTMXSourceRecord; //ptr to source record PTMX_TARGET_RECORD pTMXTargetRecord; //ptr to target record - LONG lStartKey = TmBtree.chCollate.lStartKey; //start tm key - LONG lNextKey = TmBtree.chCollate.lNextKey; //last tm key + LONG lStartKey = state->TmBtree.chCollate.lStartKey; //start tm key + LONG lNextKey = state->TmBtree.chCollate.lNextKey; //last tm key std::vector pTmRecord; // space for user data pTmExtOut->lMaxEntries = (lNextKey - lStartKey); - if(fCorrupted && lStartKey != FIRST_KEY){ + if(state->fCorrupted && lStartKey != FIRST_KEY){ lNextKey = std::max(lNextKey, LASTALLOWEDKEY) ; //some big value to go } /******************************************************************/ @@ -80,7 +80,7 @@ USHORT EqfMemory::TmtXExtract /******************************************************************/ while ( (pTmExtIn->lTmKey < lNextKey) && (usRc == BTREE_NOT_FOUND) ) { - usRc = TmBtree.EQFNTMGet(pTmExtIn->lTmKey, pTmRecord ); + usRc = state->TmBtree.EQFNTMGet(pTmExtIn->lTmKey, pTmRecord ); if ( usRc == NO_ERROR ) { @@ -92,7 +92,7 @@ USHORT EqfMemory::TmtXExtract /****************************************************************/ if ( usRc != NO_ERROR ) { - if ( (usRc == BTREE_NOT_FOUND) || ( usRc == BTREE_CORRUPTED ) || (usRc == BTREE_BUFFER_SMALL) || ((usAccessMode & ASD_ORGANIZE) != 0) ) + if ( (usRc == BTREE_NOT_FOUND) || ( usRc == BTREE_CORRUPTED ) || (usRc == BTREE_BUFFER_SMALL) || ((state->usAccessMode & ASD_ORGANIZE) != 0) ) { pTmExtIn->lTmKey ++; pTmExtIn->usNextTarget = 1; @@ -116,7 +116,7 @@ USHORT EqfMemory::TmtXExtract pTmExtOut->stPrefixOut.usTmtXRc = usRc; // in organize mode only: change any error code (except BTREE_EOF_REACHED) to BTREE_CORRUPTED - if ( (usRc != NO_ERROR ) && (usRc != BTREE_EOF_REACHED) && ((usAccessMode & ASD_ORGANIZE) != 0) ) + if ( (usRc != NO_ERROR ) && (usRc != BTREE_EOF_REACHED) && ((state->usAccessMode & ASD_ORGANIZE) != 0) ) { {LOG_AND_SET_RC_W_INFO(usRc, T5WARNING, BTREE_CORRUPTED) << "if ( (usRc != NO_ERROR ) && (usRc != BTREE_EOF_REACHED) && ((usAccessMode & ASD_ORGANIZE) != 0) ) usRc =" << usRc;} } /* endif */ @@ -444,7 +444,7 @@ USHORT EqfMemory::FillExtStructure // in organize mode preset author and file name field with default // values as here the corresponding tables may be corrupted - if ( usAccessMode & ASD_ORGANIZE ) + if ( state->usAccessMode & ASD_ORGANIZE ) { strcpy( pstExt->szLongName, OVERFLOW_NAME ); pstExt->szLongName[0] = EOS; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index c85ae1fe..4ff30ee4 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -166,7 +166,7 @@ USHORT EqfMemory::TmtXGet if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ auto str_source = EncodingHelper::convertToUTF8(pTmGetIn->szSource ); - T5LOG(T5DEBUG) << "== Lookup in memory ==" << stTmSign.szName <<" == ;\tLookupSource = >>>" <stTmSign.szName <<" == ;\tLookupSource = >>>" <szSource); T5LOG(T5INFO) << "-------------- Looking up:" << str << "" ; } @@ -309,10 +309,10 @@ USHORT EqfMemory::TmtXGet // release access to any tag table loaded for segment markup // (may have been loaded for context handling) - if ( pTagTable ) + if ( state->pTagTable ) { - TAFreeTagTable( (PLOADEDTABLE)(pTagTable) ); - pTagTable = NULL; + TAFreeTagTable( (PLOADEDTABLE)(state->pTagTable) ); + state->pTagTable = NULL; } /* endif */ T5LOG( T5INFO) <<"TmtXGet::Lookup complete, found "<usNumMatchesFound<<" matches" ; @@ -376,7 +376,7 @@ USHORT GetExactMatch USHORT usRc = NO_ERROR; //return code ULONG ulLen; //length indicator USHORT usMatchEntries = 0; //nr of found matches - BytesRef pTmRecord = pTmClb->pvTmRecord; + BytesRef pTmRecord = pTmClb->state->pvTmRecord; lSids.reserve(MAX_INDEX_LEN+5); @@ -387,7 +387,7 @@ USHORT GetExactMatch for(LONG lKey: lSids) { T5LOG( T5INFO) << "GetExactMatch: EQFNTMGET of record #" <TmBtree.EQFNTMGet( lKey, pTmRecord ); + usRc = pTmClb->state->TmBtree.EQFNTMGet( lKey, pTmRecord ); if ( usRc == NO_ERROR ) { @@ -749,7 +749,7 @@ USHORT ExactTest fTestCLB = FALSE; } - if ( pTmClb->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->psLangIdToGroupTable[usGetLang] ) + if ( pTmClb->state->psLangIdToGroupTable[pTMXTargetClb->usLangId] != pTmClb->state->psLangIdToGroupTable[usGetLang] ) { fTestCLB = FALSE; } @@ -1557,7 +1557,7 @@ USHORT GetFuzzyMatch USHORT usMatchEntries = 0; //nr of matches found USHORT usOverlaps = 0; //nr of overlapping triples - BytesRef pTmRecord = pTmClb->pvTmRecord; + BytesRef pTmRecord = pTmClb->state->pvTmRecord; //fill pMatchEntry with the tm keys with the highest frequency usRc = FillMatchEntry( pTmClb, pSentence, pMatchEntries, &pGetIn->usMatchThreshold ); @@ -1570,7 +1570,7 @@ USHORT GetFuzzyMatch //(*pusMatchesFound <= pGetIn->usRequestedMatches) ) { T5LOG( T5INFO) << "GetFuzzyMatch: EQFNTMGET of record " << pMatchEntry->lKey; - usRc = pTmClb->TmBtree.EQFNTMGet( pMatchEntry->lKey, pTmRecord ); + usRc = pTmClb->state->TmBtree.EQFNTMGet( pMatchEntry->lKey, pTmRecord ); if ( usRc == NO_ERROR ) { @@ -1663,7 +1663,7 @@ bool EqfMemory::areLangsMatching(const ushort langId1, const ushort langId2, if(langId1 == langId2){ return true; }//or if they both are from the same langGroup and first or second are preffered(or both)// are not sublocale langs - else if (( psLangIdToGroupTable[langId1] == psLangIdToGroupTable[langId2] )){ + else if (( state->psLangIdToGroupTable[langId1] == state->psLangIdToGroupTable[langId2] )){ if(!LanguageFactory::getInstance()->hasSubLocale(langStr1)){ return true; } @@ -2028,15 +2028,13 @@ USHORT FillMatchEntry USHORT usMaxSentences = MAX_INDEX_LEN * 4; LONG lMatchListSize = (LONG)((usMaxSentences+1) * sizeof(TMX_MATCHENTRY)); - pTmClb->pvIndexRecord.reserve(TMX_REC_SIZE); - // allocate match list array if not done yet - if ( pTmClb->pvTempMatchList.empty() ) + if ( pTmClb->state->pvTempMatchList.empty() ) { - pTmClb->pvTempMatchList.resize(lMatchListSize/sizeof(pTmClb->pvTempMatchList[0])); + pTmClb->state->pvTempMatchList.resize(lMatchListSize/sizeof(pTmClb->state->pvTempMatchList[0])); } - pTempStart = pTempMatch = pTmClb->pvTempMatchList.data(); + pTempStart = pTempMatch = pTmClb->state->pvTempMatchList.data(); memset( pTempMatch , 0, lMatchListSize ); { @@ -2047,7 +2045,7 @@ USHORT FillMatchEntry { lKey = (*pulVotes) & START_KEY; - usRc = pTmClb->InBtree.EQFNTMGet( lKey, pIndexRecord ); + usRc = pTmClb->state->InBtree.EQFNTMGet( lKey, pIndexRecord ); if ( usRc == NO_ERROR ) { @@ -2475,18 +2473,18 @@ USHORT NTMCompareContext pszContext2 && *pszContext2 ) { // load tag table if not done yet - if ( pTmClb->pTagTable == NULL ) + if ( pTmClb->state->pTagTable == NULL ) { - (TALoadTagTableExHwnd( pszMarkup, (PLOADEDTABLE *)&(pTmClb->pTagTable), + (TALoadTagTableExHwnd( pszMarkup, (PLOADEDTABLE *)&(pTmClb->state->pTagTable), FALSE, TALOADUSEREXIT | TALOADCOMPCONTEXTFUNC, FALSE, NULLHANDLE ) == NO_ERROR); } /* endif */ // call compare function of user exit if available - if ( pTmClb->pTagTable != NULL ) + if ( pTmClb->state->pTagTable != NULL ) { - PLOADEDTABLE pTable = (PLOADEDTABLE)pTmClb->pTagTable; + PLOADEDTABLE pTable = (PLOADEDTABLE)pTmClb->state->pTagTable; PFNCOMPCONTEXT pfnCompContext = (PFNCOMPCONTEXT)pTable->pfnCompareContext; if ( pfnCompContext ) diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp index ca988f33..82495adf 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTMDB.cpp @@ -478,9 +478,9 @@ BTREE::EQFNTMInsert /********************************************************************/ /* call QDAMDictInsert to do the dirty work of inserting entry... */ /********************************************************************/ - if(!sRc) + if(!sRc){ sRc = QDAMDictInsertLocal( lKey, pData); - + } return sRc; } /* end of function EQFNTMInsert */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp index ccf58e7d..d477dc93 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTOP.cpp @@ -24,17 +24,6 @@ #include "requestdata.h" -int EqfMemory::getErrorCode()const{ - return lastErrorCode; -} - -int EqfMemory::setErrorCode(int rc){ - return lastErrorCode = rc; -} - -void EqfMemory::reserErrorCode(){ - lastErrorCode = 0; -} std::string StatusToString(int eStatus) { @@ -197,7 +186,7 @@ USHORT EqfMemory::Load() USHORT usRc1 = NO_ERROR; //return value { - usRc1 = TmBtree.EQFNTMOpen((usAccessMode) ); + usRc1 = state->TmBtree.EQFNTMOpen((state->usAccessMode) ); if ( (usRc1 == NO_ERROR) || (usRc1 == BTREE_CORRUPTED) ) { //get signature record and add to control block @@ -208,20 +197,20 @@ USHORT EqfMemory::Load() if ( usRc == NO_ERROR ) { std::stringstream msg; - if ( stTmSign.bGlobVersion > FLAGS_t5globversion_max_supported - || stTmSign.bMajorVersion > FLAGS_t5majversion_max_supported - //|| (stTmSign.bMajorVersion == T5MAJVERSION_MAX_SUPPORTED && stTmSign.bMinorVersion > T5MINVERSION_MAX_SUPPORTED ) + if ( state->stTmSign.bGlobVersion > FLAGS_t5globversion_max_supported + || state->stTmSign.bMajorVersion > FLAGS_t5majversion_max_supported + //|| (state->stTmSign.bMajorVersion == T5MAJVERSION_MAX_SUPPORTED && state->stTmSign.bMinorVersion > T5MINVERSION_MAX_SUPPORTED ) ) { - msg << "TM was created in newer vertions of t5memory, v" << toStr(stTmSign.bGlobVersion) << "." << toStr(stTmSign.bMajorVersion) << "." << toStr(stTmSign.bMinorVersion); + msg << "TM was created in newer vertions of t5memory, v" << toStr(state->stTmSign.bGlobVersion) << "." << toStr(state->stTmSign.bMajorVersion) << "." << toStr(state->stTmSign.bMinorVersion); usRc = VERSION_MISMATCH; } - else if (stTmSign.bGlobVersion < FLAGS_t5globversion_min_supported - || stTmSign.bMajorVersion < FLAGS_t5majversion_min_supported - || (stTmSign.bMajorVersion == FLAGS_t5majversion_min_supported && stTmSign.bMinorVersion < FLAGS_t5minversion_min_supported ) ) + else if (state->stTmSign.bGlobVersion < FLAGS_t5globversion_min_supported + || state->stTmSign.bMajorVersion < FLAGS_t5majversion_min_supported + || (state->stTmSign.bMajorVersion == FLAGS_t5majversion_min_supported && state->stTmSign.bMinorVersion < FLAGS_t5minversion_min_supported ) ) { - msg << "TM was created in older vertions of t5memory, v"<< toStr(stTmSign.bGlobVersion)<<"."<< toStr(stTmSign.bMajorVersion)<<"."<< toStr(stTmSign.bMinorVersion); + msg << "TM was created in older vertions of t5memory, v"<< toStr(state->stTmSign.bGlobVersion)<<"."<< toStr(state->stTmSign.bMajorVersion)<<"."<< toStr(state->stTmSign.bMinorVersion); usRc = VERSION_MISMATCH; } /* endif */ @@ -253,7 +242,7 @@ USHORT EqfMemory::Load() { //get compact area and add to control block LONG lKey = COMPACT_KEY; - USHORT usTempRc = TmBtree.EQFNTMGet( lKey, bCompact ); + USHORT usTempRc = state->TmBtree.EQFNTMGet( lKey, state->bCompact ); // in organize mode allow continue if compact area is corrupted if ( usTempRc != NO_ERROR)//&& (usTempRc != VERSION_MISMATCH) ) @@ -263,8 +252,8 @@ USHORT EqfMemory::Load() if ( usTempRc == BTREE_CORRUPTED ) { - bCompact.resize(MAX_COMPACT_SIZE); - memset( bCompact.data(), 0, bCompact.size() ); + state->bCompact.resize(MAX_COMPACT_SIZE); + memset( state->bCompact.data(), 0, state->bCompact.size() ); } /* endif */ if ( usTempRc != NO_ERROR ) { @@ -278,7 +267,7 @@ USHORT EqfMemory::Load() (usRc == BTREE_CORRUPTED)) { USHORT usTempRc = NTMLoadNameTable( LANG_KEY, - &Languages ); + &state->Languages ); if ( usTempRc == BTREE_READ_ERROR ) { {LOG_AND_SET_RC_W_INFO(usTempRc, T5ERROR, BTREE_CORRUPTED) << "if ( usTempRc == BTREE_READ_ERROR )";} @@ -292,6 +281,7 @@ USHORT EqfMemory::Load() } /* endif */ //get file names and add to control block + /* if ( (usRc == NO_ERROR) || (usRc == BTREE_CORRUPTED)) { @@ -305,7 +295,7 @@ USHORT EqfMemory::Load() if ( usTempRc != NO_ERROR ) { usRc = usTempRc; - } /* endif */ + } } /* endif */ //get authors and add to control block @@ -313,7 +303,7 @@ USHORT EqfMemory::Load() (usRc == BTREE_CORRUPTED)) { USHORT usTempRc = NTMLoadNameTable( AUTHOR_KEY, - &Authors ); + &state->Authors ); // in organize mode allow continue if author name area is corrupted if ( usTempRc == BTREE_READ_ERROR ){ @@ -330,7 +320,7 @@ USHORT EqfMemory::Load() (usRc == BTREE_CORRUPTED) ) { USHORT usTempRc = NTMLoadNameTable( TAGTABLE_KEY, - &TagTables ); + &state->TagTables ); if ( usTempRc == BTREE_READ_ERROR ){ {LOG_AND_SET_RC_W_INFO(usTempRc, T5ERROR, BTREE_CORRUPTED)<<"if ( usTempRc == BTREE_READ_ERROR )";} @@ -385,14 +375,14 @@ USHORT EqfMemory::Load() } /* endif */ //add threshold to control block - usThreshold = FLAGS_triplesthreshold;//pTmOpenIn->stTmOpen.usThreshold; + //usThreshold = FLAGS_triplesthreshold;//pTmOpenIn->stTmOpen.usThreshold; if ( (usRc == NO_ERROR) || (usRc == BTREE_CORRUPTED) ) { //call open function for index file - USHORT usIndexRc = InBtree.EQFNTMOpen(usAccessMode ); + USHORT usIndexRc = state->InBtree.EQFNTMOpen(state->usAccessMode ); if ( usIndexRc != NO_ERROR && !isOpenedOnlyForReorganize()) { usRc = usIndexRc; @@ -425,17 +415,16 @@ USHORT EqfMemory::Load() /* database files have been opened */ /********************************************************************/ if((usRc == BTREE_CORRUPTED) && isOpenedOnlyForReorganize()){ - fCorrupted = true;//could be opened only for exportTMX or reorganize, or mass deletion(also reorganize) + state->fCorrupted = true;//could be opened only for exportTMX or reorganize, or mass deletion(also reorganize) usRc = NO_ERROR; } if (usRc != NO_ERROR) { - TmBtree.QDAMDictClose(); - InBtree.QDAMDictClose(); + state->TmBtree.QDAMDictClose(); + state->InBtree.QDAMDictClose(); T5LOG(T5ERROR) << "Failed to open tm \""<< this->szName << "\" with rc = " << usRc; - setErrorCode(usRc); eStatus = FAILED_TO_OPEN_STATUS; }else{ eStatus = OPEN_STATUS; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp index d74af19d..aaa51aa8 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTPUT.cpp @@ -173,16 +173,15 @@ USHORT EqfMemory::TmtXReplace USHORT usRc = NO_ERROR; // return code USHORT usMatchesFound; // compact area hits CHAR szString[MAX_EQF_PATH]{}; // character string - szString[0] = 0; BOOL fUpdateOfIndexFailed = FALSE; // TRUE = update of index failed if(TmProposal.lSegmentId > 0) { - if(TmProposal.lSegmentId >= stTmSign.segmentIndex - 1){ - stTmSign.segmentIndex = TmProposal.lSegmentId; + if(TmProposal.lSegmentId >= state->stTmSign.segmentIndex - 1){ + state->stTmSign.segmentIndex = TmProposal.lSegmentId; } //keep id that was send }else{ // generate new id - TmProposal.setSegmentId( ++stTmSign.segmentIndex); + TmProposal.setSegmentId( ++state->stTmSign.segmentIndex); TmProposal.fSegmentIdWasGenerated = true; } @@ -203,21 +202,7 @@ USHORT EqfMemory::TmtXReplace if ( !usRc ) { - // #define TEST_VOTES - #ifdef TEST_VOTES - - #endif// TEST_VOTES HashSentence( TmProposal.pInputSentence.get() ); - #ifdef TEST_VOTES - //if(TmProposal.pInputSentence->pStrings->getGenericTagsString().find(L"2 x Bero 90")!= std::wstring::npos) - { - std::string output = std::to_string(TmProposal.pInputSentence->usActVote) +"; votes = "; - for(auto vote: TmProposal.pInputSentence->pulVotes){ - output += std::to_string(vote) + ","; - } - T5LOG(T5TRANSACTION) << output; - } -#endif//TEST_VOTES toTmxTagtableRecord(TmProposal.pInputSentence->pTagRecord)->usTagTableId = 1; usMatchesFound = CheckCompactArea( TmProposal.pInputSentence.get(), this ); @@ -264,7 +249,7 @@ USHORT EqfMemory::TmtXReplace pTmPutOut->usTargetKey = TmProposal.currentInternalKey.getTargetKey(); RewriteCompactTable(); - usRc = TmBtree.QDAMDictUpdSignLocal(&stTmSign); + usRc = state->TmBtree.QDAMDictUpdSignLocal(&state->stTmSign); } pTmPutOut->stPrefixOut.usLengthOutput = sizeof( TMX_EXT_OUT_W ); @@ -642,7 +627,7 @@ USHORT CheckCompactArea BYTE bTuple = (BYTE) ulVote; // active byte BYTE bRest = bTuple & 0x7; // relevant bit of byte - PBYTE pByte = pTmClb->bCompact.data() + (ulVote >> 3); // byte pointer + PBYTE pByte = pTmClb->state->bCompact.data() + (ulVote >> 3); // byte pointer if ( *pByte & (1 << bRest) ) { usMatch++; @@ -766,7 +751,7 @@ USHORT EqfMemory::AddToTm if(size > 0){ TmRecord.resize(size); } - usRc = TmBtree.EQFNTMInsert(lNewKey, //to be allocated in funct + usRc = state->TmBtree.EQFNTMInsert(lNewKey, //to be allocated in funct TmRecord); //pointer to tm record if(!usRc){ TmProposal.currentInternalKey.setInternalKey(lNewKey, 1); @@ -1029,7 +1014,7 @@ USHORT EqfMemory::UpdateTmIndex { lKey = (*pulVotes) & START_KEY; pIndexRecord.clear(); - usRc = InBtree.EQFNTMGet( lKey, //index key + usRc = state->InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record if ( usRc == BTREE_NOT_FOUND ) @@ -1040,17 +1025,17 @@ USHORT EqfMemory::UpdateTmIndex toIndexRecord(pIndexRecord)->stIndexEntry = NTMINDEX(pSentence->pulVotes.size(),ulSidKey); - usRc = InBtree.EQFNTMInsert( lKey, pIndexRecord); //pointer to index + usRc = state->InBtree.EQFNTMInsert( lKey, pIndexRecord); //pointer to index //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL))// && (usAccessMode & ASD_LOCKED) ) { - usRc = EQFNTMOrganizeIndex( &(InBtree), usAccessMode, START_KEY ); + usRc = EQFNTMOrganizeIndex( &(state->InBtree), state->usAccessMode, START_KEY ); if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMInsert(lKey, pIndexRecord);//, toIndexRecord(pIndexRecord)->usRecordLen ); + usRc = state->InBtree.EQFNTMInsert(lKey, pIndexRecord);//, toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ @@ -1058,10 +1043,10 @@ USHORT EqfMemory::UpdateTmIndex { //add the match(tuple) to compact area lKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); - *(bCompact.data() + (lKey >> 3)) |= + *(state->bCompact.data() + (lKey >> 3)) |= 1 << ((BYTE)lKey & 0x07); // 1 << (USHORT)(lKey & 0x07); @01M - bCompactChanged = TRUE; + state->bCompactChanged = TRUE; } /* endif */ } else @@ -1123,27 +1108,27 @@ USHORT EqfMemory::UpdateTmIndex toIndexRecord(pIndexRecord)->usRecordLen = (USHORT)(ulLen + sizeof( TMX_INDEX_ENTRY )); pIndexRecord.resize(toIndexRecord(pIndexRecord)->usRecordLen); - usRc = InBtree.EQFNTMUpdate( + usRc = state->InBtree.EQFNTMUpdate( lKey, pIndexRecord);//.data(), //pointer to index //toIndexRecord(pIndexRecord)->usRecordLen ); //length // if index DB is full and memory is in exclusive access we try to compact the index file if ( (usRc == BTREE_LOOKUPTABLE_TOO_SMALL))// && (usAccessMode & ASD_LOCKED) ) { - usRc = EQFNTMOrganizeIndex( &(InBtree), usAccessMode, START_KEY ); + usRc = EQFNTMOrganizeIndex( &(state->InBtree), state->usAccessMode, START_KEY ); if ( usRc == NO_ERROR ) { - usRc = InBtree.EQFNTMUpdate(lKey, pIndexRecord);//.data(), toIndexRecord(pIndexRecord)->usRecordLen ); + usRc = state->InBtree.EQFNTMUpdate(lKey, pIndexRecord);//.data(), toIndexRecord(pIndexRecord)->usRecordLen ); } /* endif */ } /* endif */ if ( !usRc ) { //add the match(tuple) to compact area lKey = *pulVotes % ((LONG)(MAX_COMPACT_SIZE-1) * 8); - *((PBYTE)bCompact.data() + (lKey >> 3)) |= + *((PBYTE)state->bCompact.data() + (lKey >> 3)) |= 1 << ((BYTE)lKey & 0x07); - bCompactChanged = TRUE; + state->bCompactChanged = TRUE; } /* endif */ } else @@ -1221,7 +1206,7 @@ USHORT DetermineTmRecord pulVotes = pSentence->pulVotes.data(); lKey = (*pulVotes) & START_KEY; - usRc = pTmClb->InBtree.EQFNTMGet( lKey, //index key + usRc = pTmClb->state->InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record if ( usRc == NO_ERROR ) { @@ -1270,7 +1255,7 @@ USHORT DetermineTmRecord lKey = (*pulVotes) & START_KEY; pIndexRecord.clear(); //memset( pIndexRecord.data(), 0, pIndexRecord.size() ); - usRc = pTmClb->InBtree.EQFNTMGet( lKey, //index key + usRc = pTmClb->state->InBtree.EQFNTMGet( lKey, //index key pIndexRecord); //pointer to index record if ( usRc == NO_ERROR ) @@ -1478,7 +1463,7 @@ USHORT EqfMemory::UpdateTmRecord { iSidCount ++; TmRecord.clear(); - usRc = TmBtree.EQFNTMGet( lKey, //tm record key + usRc = state->TmBtree.EQFNTMGet( lKey, //tm record key TmRecord); //pointer to tm record data if ( usRc == NO_ERROR ) @@ -1647,7 +1632,6 @@ USHORT EqfMemory::ComparePutData //calculate length of source string lLen = (RECLEN(pTMXSourceRecord) - sizeof( TMX_SOURCE_RECORD )); //copy and compare source string - //memset( pString.data(), 0, MAX_SEGMENT_SIZE * sizeof(CHAR_W)); lLen = EQFCompress2Unicode( pString, pByte, lLen ); fStringEqual = ! wcscmp( pString.c_str(), TmProposal.pInputSentence->pStrings->getGenericTagStrC() ); @@ -1784,15 +1768,35 @@ USHORT EqfMemory::ComparePutData usRc = BTREE_NODE_IS_FULL; fStop = true; fOK = false; - }else{ + }else{ + auto pClbOffset = (PBYTE)pClb - pTmRecord.data(); + auto newSize = pClbOffset+ lNewClbLen + size;//RECLENB(pTmRecord) + lNewClbLen; if(FLAGS_log_memmove_in_compareputdata) { T5LOG(T5TRANSACTION) << "memmove size = "<< size << "; lNewClbLen = " << lNewClbLen << "; RECLEN(pTmRecord) = " << RECLENB(pTmRecord) << "; RECLEN(pTMXTargetRecord) = " << RECLEN(pTMXTargetRecord) <<"; occupiedSize = " << occupiedSize - << "; ulKey = " << lKey << "; TmRecord.size() = " << pTmRecord.size();// <<"; size before = " << size_before; + << "; ulKey = " << lKey << "; TmRecord.size() = " << pTmRecord.size() << "; newSize = " << newSize << "; pClbOffset = " << pClbOffset;// <<"; size before = " << size_before; + } + if(newSize > pTmRecord.size()){ + T5LOG(T5INFO) << "resizing pTmRecord, old size = "<< pTmRecord.size() <<"; new size = " <table.push_back(TMX_TABLE_ENTRY{}); + } - if (usRc == NO_ERROR) - { -/*************************************************************/ -/* Check if space is available in table so that new entry */ -/* will fit into the table. */ -/* Check if the allocated size can hold the new entry. */ -/* If not reallocate the table and check that the table */ -/* will not exceed the maximum size of a QDAM record (32K) */ -/*************************************************************/ - if(usTableType == FILE_KEY || usTableType == LONGNAME_KEY){ - AddToLongNameTables(pszName, pusID); - } - - if(pstTMTable->stTmTableEntry.empty()){// in 0.7.shouldn't be empty, but could be when reading from 0.6 - T5LOG(T5ERROR) << "tried to write to empty tm table #" << usTableType<<"; error was already catched and table was resized to 1"; - pstTMTable->stTmTableEntry.push_back(TMX_TABLE_ENTRY{}); - } - - TMX_TABLE_ENTRY& newEntry = pstTMTable->stTmTableEntry.back(); - if(*pusID == 0) *pusID = pstTMTable->stTmTableEntry.size(); - newEntry.usId = *pusID; - strncpy(newEntry.szName, pszName, MAX_LANG_LENGTH - 1); - pstTMTable->stTmTableEntry.push_back(TMX_TABLE_ENTRY()); - - /*************************************************************/ - /* insert name and id to table and sort table */ - /*************************************************************/ - if( pstTMTable->stTmTableEntry.size() > 1){ - qsort(pstTMTable->stTmTableEntry.data(), - pstTMTable->stTmTableEntry.size()-1,//keep last entry empty - sizeof(TMX_TABLE_ENTRY), - NTMCompNames); - } + TMX_TABLE_ENTRY& newEntry = pstTMTable->table.back(); + if(*pusID == 0) *pusID = pstTMTable->table.size(); + newEntry.usId = *pusID; + strncpy(newEntry.szName, pszName, MAX_LANG_LENGTH - 1); + pstTMTable->table.push_back(TMX_TABLE_ENTRY()); + + /*************************************************************/ + /* insert name and id to table and sort table */ + /*************************************************************/ + if( pstTMTable->table.size() > 1){ + qsort(pstTMTable->table.data(), + pstTMTable->table.size()-1,//keep last entry empty + sizeof(TMX_TABLE_ENTRY), + NTMCompNames); + } - // update table record in TM QDAM file - if (usTableType != LANGGROUP_KEY) - { - std::vector pOldTable; - tmxTableToBuffer(pstTMTable, pOldTable); - usRc = TmBtree.EQFNTMUpdate((LONG)usTableType,pOldTable); - //(PBYTE)table, occupSize); + // update table record in TM QDAM file + if (usTableType != LANGGROUP_KEY) + { + std::vector pOldTable; + tmxTableToBuffer(pstTMTable, pOldTable); + usRc = state->TmBtree.EQFNTMUpdate((LONG)usTableType,pOldTable); + //(PBYTE)table, occupSize); + } /* endif */ } /* endif */ - } /* endif */ - + } } else //--- pTmClb is NULL pointer or pszName is empty { @@ -980,9 +989,9 @@ int NTMLongNameTableComp( /********************************************************************/ /* compare the long names of the passed table entries using strcmp */ /********************************************************************/ - PSZ lname1 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName; - PSZ lname2 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName; - return (strcmp(lname1, lname2)); + auto entry1 = (PTMX_LONGNAME_TABLE_ENTRY)pEntry1; + auto entry2 = (PTMX_LONGNAME_TABLE_ENTRY)pEntry2; + return (strcmp(entry1->pszLongName, entry2->pszLongName)); } /* end of function NTMCompNames */ int NTMLongNameTableCompCaseIgnore( @@ -990,10 +999,9 @@ int NTMLongNameTableCompCaseIgnore( const void *pEntry2 // input ) { - PSZ lname1 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry1)->pszLongName; - PSZ lname2 = ((PTMX_LONGNAME_TABLE_ENTRY)pEntry2)->pszLongName; - return strcasecmp(lname1,lname2);//string should be in lower case here - + auto entry1 = (PTMX_LONGNAME_TABLE_ENTRY)pEntry1; + auto entry2 = (PTMX_LONGNAME_TABLE_ENTRY)pEntry2; + return strcasecmp(entry1->pszLongName,entry2->pszLongName);//string should be in lower case here } /* end of function NTMCompNames */ //+----------------------------------------------------------------------------+ @@ -1023,7 +1031,7 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab if (usRc == NO_ERROR) { - pTMTable->stTmTableEntry.clear(); + pTMTable->table.clear(); unsigned long numOfEntries = ((PTMX_TABLE_OLD)oldTmxTable.data())->ulMaxEntries; unsigned long numOfEntriesCapacity = oldTmxTable.size()/sizeof(TMX_TABLE_ENTRY); if(numOfEntries > numOfEntriesCapacity){ @@ -1033,13 +1041,13 @@ USHORT bytesToOldTable(ULONG ulTableKey, BytesRef oldTmxTable, PTMX_TABLE pTMTab numOfEntries = numOfEntriesCapacity; } - pTMTable->stTmTableEntry.resize(numOfEntries); + pTMTable->table.resize(numOfEntries); if(numOfEntries > 0){ - memcpy(&pTMTable->stTmTableEntry[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->stTmTableEntry, numOfEntries/*(pTMTable->stTmTableEntry.size()-1)*/ * sizeof(TMX_TABLE_ENTRY)); + memcpy(&pTMTable->table[0], &((PTMX_TABLE_OLD)oldTmxTable.data())->table, numOfEntries/*(pTMTable->table.size()-1)*/ * sizeof(TMX_TABLE_ENTRY)); } - if(pTMTable->stTmTableEntry.empty() || (pTMTable->stTmTableEntry.back().usId && pTMTable->stTmTableEntry.back().szName[0])){ - pTMTable->stTmTableEntry.push_back(TMX_TABLE_ENTRY{}); + if(pTMTable->table.empty() || (pTMTable->table.back().usId && pTMTable->table.back().szName[0])){ + pTMTable->table.push_back(TMX_TABLE_ENTRY{}); } } return usRc; @@ -1060,7 +1068,7 @@ USHORT EqfMemory::NTMLoadNameTable( ) { std::vector pOldTable; - USHORT usRc = TmBtree.EQFNTMGet(lTableKey, pOldTable); + USHORT usRc = state->TmBtree.EQFNTMGet(lTableKey, pOldTable); if(!usRc) usRc = bytesToOldTable(lTableKey, pOldTable, pTMTable); return (usRc); @@ -1103,12 +1111,12 @@ USHORT EqfMemory::NTMAddLangGroup( // enlarge language-ID-to-group-ID table if necessary if (usRC == NO_ERROR) { - if (psLangIdToGroupTable.size() < sLangID + 1) + if (state->psLangIdToGroupTable.size() < sLangID + 1) { - psLangIdToGroupTable.resize(sLangID + 1); + state->psLangIdToGroupTable.resize(sLangID + 1); } /* endif */ // update language ID to group ID table - psLangIdToGroupTable[sLangID] = sGroupID; + state->psLangIdToGroupTable[sLangID] = sGroupID; } /* endif */ @@ -1119,21 +1127,21 @@ USHORT EqfMemory::NTMCreateLangGroupTable() { USHORT usRC = NO_ERROR; // allocate language-ID-to-group-ID-table - LONG lSize = Languages.stTmTableEntry.size();// > 100L ? Languages.stTmTableEntry.size() : 100L; + LONG lSize = state->Languages.table.size();// > 100L ? Languages.table.size() : 100L; if (lSize <= 0) lSize = 1; - psLangIdToGroupTable.resize(lSize); + state->psLangIdToGroupTable.resize(lSize); // get group IDs for all languages and fill map table if (usRC == NO_ERROR) { int i = 0; - while ((usRC == NO_ERROR) && (i < Languages.stTmTableEntry.size()) + while ((usRC == NO_ERROR) && (i < state->Languages.table.size()) ) { - if(Languages.stTmTableEntry[i].szName[0]){ - usRC = NTMAddLangGroup(Languages.stTmTableEntry[i].szName, - Languages.stTmTableEntry[i].usId); + if(state->Languages.table[i].szName[0]){ + usRC = NTMAddLangGroup(state->Languages.table[i].szName, + state->Languages.table[i].usId); } i++; } /* endwhile */ @@ -1150,7 +1158,7 @@ USHORT EqfMemory::NTMCreateLangGroupTable() USHORT EqfMemory::NTMOrganizeIndexFile() { USHORT usRC = NO_ERROR; - usRC = EQFNTMOrganizeIndex(&(InBtree), usAccessMode, START_KEY); + usRC = EQFNTMOrganizeIndex(&(state->InBtree), state->usAccessMode, START_KEY); return (usRC); } /* end of function NTMOrganizeIndexFile */ diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp index 70b0c725..a9c7a2de 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemory.cpp @@ -43,7 +43,7 @@ EqfMemory::~EqfMemory() */ int EqfMemory::getNumOfMarkupNames() { - return( (int)(TagTables.stTmTableEntry.size()) ); + return( (int)(state->TagTables.table.size()) ); } /*! \brief Get markup name at position n [n = 0.. GetNumOfMarkupNames()-1] @@ -59,9 +59,9 @@ int EqfMemory::getMarkupName int iSize ) { - if ( (iPos >= 0) && (iPos < (int)(TagTables.stTmTableEntry.size())) ) + if ( (iPos >= 0) && (iPos < (int)(state->TagTables.table.size())) ) { - return( CopyToBuffer( TagTables.stTmTableEntry[iPos].szName, pszBuffer, iSize ) ); + return( CopyToBuffer( state->TagTables.table[iPos].szName, pszBuffer, iSize ) ); } else { @@ -80,7 +80,7 @@ int EqfMemory::getSourceLanguage int iSize ) { - return( CopyToBuffer( stTmSign.szSourceLanguage, pszBuffer, iSize ) ); + return( CopyToBuffer( state->stTmSign.szSourceLanguage, pszBuffer, iSize ) ); } /*! \brief Get description of the memory @@ -94,7 +94,7 @@ int EqfMemory::getDescription int iSize ) { - return( CopyToBuffer(stTmSign.szDescription, pszBuffer, iSize ) ); + return( CopyToBuffer(state->stTmSign.szDescription, pszBuffer, iSize ) ); } /*! \brief Set the description of the memory @@ -118,15 +118,15 @@ void EqfMemory::setDescription // update description field if ( fOK ) { - strncpy( stTmSign.szDescription, pszBuffer, sizeof(stTmSign.szDescription)-1 ); - stTmSign.szDescription[sizeof(stTmSign.szDescription)-1] = EOS; + strncpy( state->stTmSign.szDescription, pszBuffer, sizeof(state->stTmSign.szDescription)-1 ); + state->stTmSign.szDescription[sizeof(state->stTmSign.szDescription)-1] = EOS; } /* endif */ // re-write signature record if ( fOK ) { - usRc = TmBtree.QDAMDictUpdSignLocal(&stTmSign); + usRc = state->TmBtree.QDAMDictUpdSignLocal(&state->stTmSign); fOK = (usRc == NO_ERROR); } /* endif */ @@ -176,8 +176,8 @@ unsigned long EqfMemory::getFileSize() //if ( pTmClb != NULL ) { - unsigned long ulDataSize = TmBtree.fb.file.GetFileSize(); - unsigned long ulIndexSize = InBtree.fb.file.GetFileSize(); + unsigned long ulDataSize = state->TmBtree.fb.file.GetFileSize(); + unsigned long ulIndexSize = state->InBtree.fb.file.GetFileSize(); ulFileSize = ulDataSize + ulIndexSize; } @@ -200,7 +200,6 @@ int EqfMemory::putProposal { int iRC = 0; TMX_EXT_OUT_W TmPutOut{}; - //memset( &TmPutOut, 0, sizeof(TMX_EXT_OUT_W) ); if(T5Logger::GetInstance()->CheckLogLevel(T5INFO)){ std::string author = Proposal.szTargetAuthor; std::string source = EncodingHelper::convertToUTF8(Proposal.pInputSentence->pStrings->getOriginalSrcStr()); @@ -224,12 +223,12 @@ int EqfMemory::putProposal int EqfMemory::RewriteCompactTable(){ USHORT usRc = 0; - if ( bCompactChanged ) + if ( state->bCompactChanged ) { - usRc = TmBtree.EQFNTMUpdate( COMPACT_KEY, bCompact ); + usRc = state->TmBtree.EQFNTMUpdate( COMPACT_KEY, state->bCompact ); if ( !usRc ) { - bCompactChanged = FALSE; + state->bCompactChanged = FALSE; } /* endif */ } /* endif */ @@ -239,15 +238,15 @@ int EqfMemory::RewriteCompactTable(){ size_t EqfMemory::GetRAMSize()const{ size_t size = sizeof(*this); - size += TmBtree.fb.data.capacity(); - size += InBtree.fb.data.capacity(); + size += state->TmBtree.fb.data.capacity(); + size += state->InBtree.fb.data.capacity(); return size; }; size_t EqfMemory::GetExpectedRAMSize()const{ size_t size = sizeof(*this); - size += TmBtree.GetFileSize(); - size += InBtree.GetFileSize(); + size += state->TmBtree.GetFileSize(); + size += state->InBtree.GetFileSize(); return size; }; @@ -291,7 +290,7 @@ int EqfMemory::getNextProposal Proposal.currentInternalKey.reset(); } /* endif */ - if ( (iRC == 0) || (iRC == BTREE_CORRUPTED) || iRC == BTREE_BUFFER_SMALL || (usAccessMode & ASD_ORGANIZE) ) + if ( (iRC == 0) || (iRC == BTREE_CORRUPTED) || iRC == BTREE_BUFFER_SMALL || (state->usAccessMode & ASD_ORGANIZE) ) { Proposal.nextInternalKey.setInternalKey(TmExtOut.lTmKey, TmExtOut.usNextTarget); } /* endif */ @@ -302,12 +301,12 @@ int EqfMemory::getNextProposal } else if ( iRC == BTREE_EOF_REACHED ) { - iRC = INFO_ENDREACHED; + iRC = EqfMemoryError::INFO_ENDREACHED; } else { T5LOG_RC(iRC, this->szName); - if ( iRC == BTREE_CORRUPTED ) iRC = ERROR_ENTRYISCORRUPTED; + if ( iRC == BTREE_CORRUPTED ) iRC = EqfMemoryError::ERROR_ENTRYISCORRUPTED; } return( iRC ); @@ -351,37 +350,6 @@ void *EqfMemory::getPlugin() } - -/*! \brief Get the error message for the last error occured - - \param strError reference to a string receiving the error mesage text - \returns last error code -*/ -int EqfMemory::getLastError -( - std::string &strError -) -{ - strError = this->strLastError; - return( this->iLastError ); -} - - /*! \brief Get the error message for the last error occured - - \param pszError pointer to a buffer for the error text - \param iBufSize size of error text buffer in number of characters - \returns last error code -*/ -int EqfMemory::getLastError -( - char *pszError, - int iBufSize -) -{ - strError = strLastError.c_str(); - return( this->iLastError ); -} - /*! \brief Get language at position n [n = 0.. GetNumOfLanguages()-1] \param iPos position of language \param pszBuffer pointer to a buffer for the document name @@ -395,9 +363,9 @@ int EqfMemory::getLanguage int iSize ) { - if ( (iPos >= 0) && (iPos < (int)(Languages.stTmTableEntry.size())) ) + if ( (iPos >= 0) && (iPos < (int)(state->Languages.table.size())) ) { - return( CopyToBuffer( Languages.stTmTableEntry[iPos].szName, pszBuffer, iSize ) ); + return( CopyToBuffer( state->Languages.table[iPos].szName, pszBuffer, iSize ) ); } else { @@ -453,7 +421,7 @@ int TMCursor::SplitProposalKeyIntoRecordAndTarget } else { - iRC = EqfMemory::ERROR_INTERNALKEY_MISSING; + iRC = EqfMemoryError::ERROR_INTERNALKEY_MISSING; } /* end */ return( iRC ); @@ -481,7 +449,7 @@ int EqfMemory::ExtOutToOtmProposal Proposal.setAuthor( pExtOut->szAuthorName ); Proposal.setMarkup( pExtOut->szTagTable ); Proposal.setTargetLanguage( pExtOut->szTargetLanguage ); - Proposal.setSourceLanguage( stTmSign.szSourceLanguage); + Proposal.setSourceLanguage( state->stTmSign.szSourceLanguage); Proposal.setAddInfo( pExtOut->szAddInfo.c_str() ); Proposal.setContext( pExtOut->szContext.c_str() ); Proposal.setDocName( pExtOut->szLongName ); @@ -663,21 +631,33 @@ EqfMemory::EqfMemory(const std::string& tmName){ readOnlyCnt = std::make_shared(0); writeCnt = std::make_shared(0); + state = std::make_unique(); - TmBtree.fb.fileName = FilesystemHelper::GetTmdPath(tmName); - InBtree.fb.fileName = FilesystemHelper::GetTmiPath(tmName); + state->TmBtree.fb.file.fileName = FilesystemHelper::GetTmdPath(tmName); + state->InBtree.fb.file.fileName = FilesystemHelper::GetTmiPath(tmName); szName = tmName; } int EqfMemory::Reload(bool fReadFromDisk){ int rc = 0; + /* TmBtree.resetLookupTable(); InBtree.resetLookupTable(); + LongNames.pszBuffer.clear(); + LongNames.stTableEntry.clear(); + LongNamesCaseIgnore.pszBuffer.clear(); + LongNamesCaseIgnore.stTableEntry.clear(); + Authors.table.clear(); + LangGroups.table.clear(); + Languages.table.clear(); + psLangIdToGroupTable.clear(); + + //datashould be already in filebuffers in case of reorganize, otherwise - read from disk if(fReadFromDisk){ - TmBtree.fb.file = LockedFile{};//reset fileLock - InBtree.fb.file = LockedFile{};//reset fileLock + state->TmBtree.fb.file = LockedFile{};//reset fileLock + state->InBtree.fb.file = LockedFile{};//reset fileLock rc = TmBtree.fb.ReadFromFile(); if(!rc){ rc = InBtree.fb.ReadFromFile(); @@ -689,17 +669,17 @@ int EqfMemory::Reload(bool fReadFromDisk){ }else{ eStatus = WAITING_FOR_LOADING_STATUS; rc = Load(); - } + }//*/ return rc; } int EqfMemory::FlushFilebuffers(MutexTimeout& tmLockTimeout){ - if(fCorrupted){ + if(state->fCorrupted){ return 0; } TimedMutexGuard l(tmMutex, tmLockTimeout, "tmMutex", __func__, __LINE__); - int rc = TmBtree.fb.Flush(); - rc = rc? rc : InBtree.fb.Flush(); + int rc = state->TmBtree.fb.Flush(); + rc = rc? rc : state->InBtree.fb.Flush(); return rc; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp index 7841e7b6..21155761 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp @@ -99,7 +99,7 @@ EqfMemory* EqfMemoryPlugin::initTM( pMemory->updateLastUseTime(); pMemory->eStatus = WAITING_FOR_LOADING_STATUS; if(fReorganizeOnly){ - pMemory->usAccessMode |= ASD_ORGANIZE; + pMemory->state->usAccessMode |= ASD_ORGANIZE; } //pMemory->fReorganizeOnly = fReorganizeOnly; }else{ diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 024b2d0d..09f82bd1 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -320,16 +320,14 @@ int RequestData::requestTM(){ return 0; } + bool isServiceReq = isServiceRequest(); if(mem.get()== nullptr){ - fValid = isServiceRequest(); - return 0; + fValid = isServiceReq; }else if(mem->isReorganizeRunning()){ return buildErrorReturn(505, "Reorganize is running for requested tm", _rc_); }else if(mem->isImportRunning()){ return buildErrorReturn(505, "Import is running for requested tm", _rc_); - } - - { + }else{ TimedMutexGuard l(mem->tmMutex, tmLockTimeout, "tmMutex", __func__, __LINE__); if(tmLockTimeout.failed()){ @@ -338,17 +336,17 @@ int RequestData::requestTM(){ } if(mem->isWaitingToBeLoaded() || mem->isFailedToLoad()){ - USHORT accessModeBeforeOpenning = mem->usAccessMode; + USHORT accessModeBeforeOpenning = mem->state->usAccessMode; if(REORGANIZE_MEM == command || DELETE_ENTRIES_REORGANIZE == command || EXPORT_MEM_TMX == command || EXPORT_MEM_TMX_STREAM == command){ - mem->usAccessMode |= ASD_ORGANIZE; + mem->state->usAccessMode |= ASD_ORGANIZE; } _rc_ = mem->Load(); //if(!mem->isCorrupted()){ - mem->usAccessMode=accessModeBeforeOpenning; + mem->state->usAccessMode=accessModeBeforeOpenning; //} if(_rc_){ @@ -362,7 +360,7 @@ int RequestData::requestTM(){ } } - if(mem->isLoaded()) + if(mem && mem->isLoaded()) { fValid = true; if(mem->isCorrupted() && @@ -858,13 +856,13 @@ int CreateMemRequestData::execute(){ if(_rc_ == ERROR_MEM_NAME_EXISTS){ outputMessage = "{\"" + strMemName + "\": \"ERROR_MEM_NAME_EXISTS\" }" ; - T5LOG(T5ERROR) << "OtmMemoryServiceWorker::createMemo()::usRC = " << _rc_ << " _rc_ = ERROR_MEM_NAME_EXISTS"; + T5LOG(T5ERROR) << "::createMemo()::usRC = " << _rc_ << " _rc_ = ERROR_MEM_NAME_EXISTS"; return _rc_; }else if ( _rc_ != 0 ) { char szError[PATH_MAX]; unsigned short usRC = 0; - EqfGetLastError( NULL /*OtmMemoryServiceWorker::getInstance()->hSession*/, &usRC, szError, sizeof( szError ) ); + EqfGetLastError( NULL /*::getInstance()->hSession*/, &usRC, szError, sizeof( szError ) ); //buildErrorReturn( _rc_, szError, outputMessage ); outputMessage = "{\"" + strMemName + "\": \"" ; @@ -1010,7 +1008,7 @@ int ImportRequestData::parseJSON(){ fSaveDifferentTargetsForSameSource = std::stol(value); }else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); - T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; + T5LOG( T5WARNING) <<"::import::set new threshold for logging" << loggingThreshold; T5Logger::GetInstance()->SetLogLevel(loggingThreshold); }else if(strcasecmp(name.c_str(), "framingTags") == 0){ std::string strInclosingTagsBehaviour = value; @@ -1279,7 +1277,7 @@ int ImportLocalRequestData::parseJSON(){ } else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); - T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; + T5LOG( T5WARNING) <<"::import::set new threshold for logging" << loggingThreshold; T5Logger::GetInstance()->SetLogLevel(loggingThreshold); }else{ T5LOG( T5WARNING) << "JSON parsed unexpected name, " << name; @@ -1454,7 +1452,7 @@ int SearchFilterFactory::parseJSONNameAndValue(const std::string& name, const st else if(strcasecmp(name.c_str(), "loggingThreshold") == 0){ loggingThreshold = std::stoi(value); - T5LOG( T5WARNING) <<"OtmMemoryServiceWorker::import::set new threshold for logging" << loggingThreshold; + T5LOG( T5WARNING) <<"::import::set new threshold for logging" << loggingThreshold; T5Logger::GetInstance()->SetLogLevel(loggingThreshold); }else { @@ -1618,7 +1616,7 @@ int ReorganizeRequestData::execute(){ return buildErrorReturn( 500, msg.c_str() ); } - const auto numOfLangs = mem->Languages.stTmTableEntry.size(); + const auto numOfLangs = mem->state->Languages.table.size(); if (numOfLangs <3)// if there are less than 3 langs-reorganize shouldn'b be allowed(we neew srcLang, trgLang and empty lang) { std::string msg = "mem can't be reorganized without losing all the data, because there are " + std::to_string(numOfLangs) @@ -1634,7 +1632,7 @@ int ReorganizeRequestData::execute(){ PMEM_ORGANIZE_IDA pRIDA = new MEM_ORGANIZE_IDA; // pointer to instance area // validate session handle - //_rc_ = FctValidateSession( OtmMemoryServiceWorker::getInstance()->hSession, &pData ); + //_rc_ = FctValidateSession( ::getInstance()->hSession, &pData ); // reorganize the memory if ( !_rc_ ) @@ -1674,9 +1672,6 @@ int ReorganizeRequestData::execute(){ mem->importDetails->fReorganize = true; } pRIDA->tmLockTimeout = tmLockTimeout; - - //mem.reset(); - //memRef.reset(); } //worker thread @@ -2206,33 +2201,33 @@ std::string StatusMemRequestData::prepareStatusString(std::shared_ptr json_factory.addParmToJSON( outputMessage, "rc", mem->importDetails->importRc); } json_factory.addParmToJSON( outputMessage, "lastAccessTime", printTime(mem->tLastAccessTime) ); - json_factory.addParmToJSON( outputMessage, "creationTime", printTime(mem->stTmSign.creationTime) ); - std::string creationT5MVersion = std::to_string(mem->stTmSign.bGlobVersion) - + ":"+ std::to_string(mem->stTmSign.bMajorVersion) - + ":" + std::to_string(mem->stTmSign.bMinorVersion); + json_factory.addParmToJSON( outputMessage, "creationTime", printTime(mem->state->stTmSign.creationTime) ); + std::string creationT5MVersion = std::to_string(mem->state->stTmSign.bGlobVersion) + + ":"+ std::to_string(mem->state->stTmSign.bMajorVersion) + + ":" + std::to_string(mem->state->stTmSign.bMinorVersion); json_factory.addParmToJSON( outputMessage, "tmCreatedInT5M_version", creationT5MVersion.c_str() ); - json_factory.addParmToJSON( outputMessage, "segmentIndex", mem->stTmSign.segmentIndex ); - json_factory.addParmToJSON( outputMessage, "sourceLang", mem->stTmSign.szSourceLanguage ); - json_factory.addParmToJSON( outputMessage, "internalDescription", mem->stTmSign.szDescription ); - json_factory.addParmToJSON( outputMessage, "tmd fbuffer was modified", mem->TmBtree.wasModified()); - json_factory.addParmToJSON( outputMessage, "tmd lStartKey", mem->TmBtree.chCollate.lStartKey); - json_factory.addParmToJSON( outputMessage, "tmd lNextKey", mem->TmBtree.chCollate.lNextKey); - json_factory.addParmToJSON( outputMessage, "tmd usNextFreeRecord", mem->TmBtree.usNextFreeRecord); - json_factory.addParmToJSON( outputMessage, "tmd usFirstLeaf", mem->TmBtree.usFirstLeaf); - json_factory.addParmToJSON( outputMessage, "tmd usFirstNode", mem->TmBtree.usFirstNode); - json_factory.addParmToJSON( outputMessage, "tmd usFirstDataBuffer", mem->TmBtree.usFirstDataBuffer); - json_factory.addParmToJSON( outputMessage, "tmd usCurrentRecord", mem->TmBtree.usCurrentRecord); - json_factory.addParmToJSON( outputMessage, "tmd NumOfSavedRecords", mem->TmBtree.GetNumOfSavedRecords()); - json_factory.addParmToJSON( outputMessage, "tmi fbuffer was modified", mem->InBtree.wasModified()); - json_factory.addParmToJSON( outputMessage, "tmi lStartKey", mem->InBtree.chCollate.lStartKey); - json_factory.addParmToJSON( outputMessage, "tmi lNextKey", mem->InBtree.chCollate.lNextKey); - json_factory.addParmToJSON( outputMessage, "tmi usNextFreeRecord", mem->InBtree.usNextFreeRecord); - json_factory.addParmToJSON( outputMessage, "tmi usFirstLeaf", mem->InBtree.usFirstLeaf); - json_factory.addParmToJSON( outputMessage, "tmi usFirstNode", mem->InBtree.usFirstNode); - json_factory.addParmToJSON( outputMessage, "tmi usFirstDataBuffer", mem->InBtree.usFirstDataBuffer); - json_factory.addParmToJSON( outputMessage, "tmi usCurrentRecord", mem->InBtree.usCurrentRecord); - json_factory.addParmToJSON( outputMessage, "tmi NumOfSavedRecords", mem->InBtree.GetNumOfSavedRecords()); - //json_factory.addParmToJSON( outputMessage, "internalName", mem->stTmSign.szName ); + json_factory.addParmToJSON( outputMessage, "segmentIndex", mem->state->stTmSign.segmentIndex ); + json_factory.addParmToJSON( outputMessage, "sourceLang", mem->state->stTmSign.szSourceLanguage ); + json_factory.addParmToJSON( outputMessage, "internalDescription", mem->state->stTmSign.szDescription ); + json_factory.addParmToJSON( outputMessage, "tmd fbuffer was modified", mem->state->TmBtree.wasModified()); + json_factory.addParmToJSON( outputMessage, "tmd lStartKey", mem->state->TmBtree.chCollate.lStartKey); + json_factory.addParmToJSON( outputMessage, "tmd lNextKey", mem->state->TmBtree.chCollate.lNextKey); + json_factory.addParmToJSON( outputMessage, "tmd usNextFreeRecord", mem->state->TmBtree.usNextFreeRecord); + json_factory.addParmToJSON( outputMessage, "tmd usFirstLeaf", mem->state->TmBtree.usFirstLeaf); + json_factory.addParmToJSON( outputMessage, "tmd usFirstNode", mem->state->TmBtree.usFirstNode); + json_factory.addParmToJSON( outputMessage, "tmd usFirstDataBuffer", mem->state->TmBtree.usFirstDataBuffer); + json_factory.addParmToJSON( outputMessage, "tmd usCurrentRecord", mem->state->TmBtree.usCurrentRecord); + json_factory.addParmToJSON( outputMessage, "tmd NumOfSavedRecords", mem->state->TmBtree.GetNumOfSavedRecords()); + json_factory.addParmToJSON( outputMessage, "tmi fbuffer was modified", mem->state->InBtree.wasModified()); + json_factory.addParmToJSON( outputMessage, "tmi lStartKey", mem->state->InBtree.chCollate.lStartKey); + json_factory.addParmToJSON( outputMessage, "tmi lNextKey", mem->state->InBtree.chCollate.lNextKey); + json_factory.addParmToJSON( outputMessage, "tmi usNextFreeRecord", mem->state->InBtree.usNextFreeRecord); + json_factory.addParmToJSON( outputMessage, "tmi usFirstLeaf", mem->state->InBtree.usFirstLeaf); + json_factory.addParmToJSON( outputMessage, "tmi usFirstNode", mem->state->InBtree.usFirstNode); + json_factory.addParmToJSON( outputMessage, "tmi usFirstDataBuffer", mem->state->InBtree.usFirstDataBuffer); + json_factory.addParmToJSON( outputMessage, "tmi usCurrentRecord", mem->state->InBtree.usCurrentRecord); + json_factory.addParmToJSON( outputMessage, "tmi NumOfSavedRecords", mem->state->InBtree.GetNumOfSavedRecords()); + //json_factory.addParmToJSON( outputMessage, "internalName", mem->state->stTmSign.szName ); if ( ( (mem->eImportStatus == IMPORT_FAILED_STATUS) || (mem->eImportStatus == REORGANIZE_FAILED_STATUS)) && ( !mem->strError.empty() ) ) { @@ -2270,40 +2265,39 @@ int StatusMemRequestData::execute() { int GetTablesMemRequestData::execute() { json_factory.startJSON( outputMessage ); - std::string longNames, longNamesCaseIgnore, langTable, langGroupTable, authorsTable, filenamesOldTable, tagTablesTable; + std::string longNames, longNamesCaseIgnore, langTable, langGroupTable, authorsTable, filenamesOldTable, tagTablesTable, langIdToGroupTable; int i = 0; - for(const auto& entry: mem->LongNames.stTableEntry){ + for(const auto& entry: mem->state->LongNames.stTableEntry){ longNames += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + (entry.pszLongName? entry.pszLongName:"") + "\"; "; } i = 0; - for(const auto& entry: mem->LongNamesCaseIgnore.stTableEntry){ + for(const auto& entry: mem->state->LongNamesCaseIgnore.stTableEntry){ longNamesCaseIgnore += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + (entry.pszLongName? entry.pszLongName:"") + "\"; "; } i = 0; - for(const auto& entry: mem->Authors.stTmTableEntry){ + for(const auto& entry: mem->state->Authors.table){ authorsTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; } i = 0; - for(const auto& entry: mem->FileNames.stTmTableEntry){ - filenamesOldTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; - } - - i = 0; - for(const auto& entry: mem->Languages.stTmTableEntry){ + for(const auto& entry: mem->state->Languages.table){ langTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; } i = 0; - for(const auto& entry: mem->LangGroups.stTmTableEntry){ + for(const auto& entry: mem->state->LangGroups.table){ langGroupTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; } i = 0; - for(const auto& entry: mem->TagTables.stTmTableEntry){ + for(const auto& entry: mem->state->TagTables.table){ tagTablesTable += toStr(i++) + ")"+ toStr(entry.usId) + ": \"" + entry.szName + "\"; "; + } + i = 0; + for(const auto& entry: mem->state->psLangIdToGroupTable){ + langIdToGroupTable += toStr(i++) + ")"+ toStr(entry); } json_factory.addParmToJSON( outputMessage, "tableName(additional info)(INTERNAL_KEY_OF_THE_TABLE)", "position)idInTable: value;"); @@ -2325,8 +2319,8 @@ int GetTablesMemRequestData::execute() { tableName = "filenames(long, caseign, virtual)(" + toStr(LONGNAME_KEY) + ")"; json_factory.addParmToJSON( outputMessage, tableName, longNamesCaseIgnore); - tableName = "filenames(short)(" + toStr(FILE_KEY) + ")"; - json_factory.addParmToJSON( outputMessage, tableName, filenamesOldTable); + tableName = "langIdToGroupTable"; + json_factory.addParmToJSON( outputMessage, tableName, longNamesCaseIgnore); json_factory.terminateJSON( outputMessage ); return( OK ); @@ -2526,7 +2520,7 @@ void ShutdownRequestData::CheckImportFlushTmsAndShutdown(int signal, MutexTimeou return ; } if( ++j % 15 == 0){ - T5LOG(T5WARNING) << "SHUTDOWN:: memory still in import..waiting 15 sec more... shutdown request was = "<< j* 15; + T5LOG(T5WARNING) << "SHUTDOWN:: memory still in import..waiting 15 sec more... shutdown request was = "<< j* 15 <<" seconds ago"; } T5LOG(T5DEBUG) << "SHUTDOWN:: memory still in import..waiting 1 sec more... mem in import = "<< i; @@ -2703,7 +2697,6 @@ int ResourceInfoRequestData::execute(){ std::string memInfo = StatusMemRequestData::prepareStatusString(tm.second); ssOutput << "\"StatusInfo\": " << memInfo << "\n"; //AddToJson(ssOutput, "StatusInfo", memInfo, false ); - ssOutput << " }"; total += fSize; @@ -2830,11 +2823,6 @@ int ResourceInfoRequestData::execute(){ sumTime = ProxygenStats::getInstance()->getDeleteEntrySumTime(); AddRequestDataToJson(ssOutput, "DeleteEntry", sumTime, requestCount); ssOutput <<","; - - //requestCount = ProxygenStats::getInstance()->getReorganizeRequestCount(); - //sumTime = ProxygenStats::getInstance()->getReorganizeExecutionTime(); - //AddRequestDataToJson(ssOutput, "Reorganize", sumTime, requestCount); - //ssOutput <<","; requestCount = ProxygenStats::getInstance()->getSaveAllTmsRequestCount(); sumTime = ProxygenStats::getInstance()->getSaveAllTmsSumTime(); @@ -3368,10 +3356,10 @@ int GetEntryRequestData::execute() "Error: memptr is null", 500); } - auto accMode=mem->usAccessMode;//temp. reset access mode - mem->usAccessMode=0; + auto accMode=mem->state->usAccessMode;//temp. reset access mode + mem->state->usAccessMode=0; _rc_ = mem->getNextProposal(Data); - mem->usAccessMode = accMode; + mem->state->usAccessMode = accMode; if((BTREE_ERR_OUT_OF_BOUNDS_OF_THE_RECORD!=_rc_) && (Data.currentInternalKey.getTargetKey() != targetKey || Data.currentInternalKey.getRecordKey() != recordKey)){ if(recordKey == 7 && targetKey == 1 //requested first record @@ -4141,7 +4129,7 @@ int ConcordanceExtendedSearchRequestData::execute() { strncpy(Data.szSearchPos, OProposal.nextInternalKey.getStr().c_str(), 20); } /* endif */ - else if ( _rc_ == EqfMemory::INFO_ENDREACHED ) + else if ( _rc_ == EqfMemoryError::INFO_ENDREACHED ) { _rc_ = ENDREACHED_RC; } @@ -4457,7 +4445,7 @@ int ConcordanceSearchRequestData::execute(){ { strncpy(Data.szSearchPos, OProposal.nextInternalKey.getStr().c_str(), 20); } /* endif */ - else if ( _rc_ == EqfMemory::INFO_ENDREACHED ) + else if ( _rc_ == EqfMemoryError::INFO_ENDREACHED ) { _rc_ = ENDREACHED_RC; } diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index a3c486db..c100504a 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -198,19 +198,19 @@ size_t TMManager::CleanupMemoryList(size_t memoryRequested, MutexTimeout& tmList void EqfMemory::importDone(int iRC, char *pszError ) { eStatus = OPEN_STATUS; - TmBtree.fb.Flush(); - InBtree.fb.Flush(); + state->TmBtree.fb.Flush(); + state->InBtree.fb.Flush(); if ( iRC == 0 ) { eImportStatus = OPEN_STATUS; - T5LOG( T5INFO) <<"OtmMemoryServiceWorker::importDone:: success, memName = " << szName; + T5LOG( T5INFO) <<"::importDone:: success, memName = " << szName; } else { eImportStatus = IMPORT_FAILED_STATUS; strError = pszError; - T5LOG(T5ERROR) << "OtmMemoryServiceWorker::importDone:: memName = " << szName <<", import failed: " << strError << " import details = " << importDetails->toString() ; + T5LOG(T5ERROR) << "::importDone:: memName = " << szName <<", import failed: " << strError << " import details = " << importDetails->toString() ; } resetActiveRequestCommand(); } @@ -236,7 +236,7 @@ void EqfMemory::reorganizeDone(int iRC, char *pszError ) { eImportStatus = REORGANIZE_FAILED_STATUS; strError = pszError; - fCorrupted = true; + state->fCorrupted = true; T5LOG(T5ERROR) << ":: memName = " << szName <<", reorganize failed: " << pszError << " import details = " << importDetails->toString() ; } @@ -328,25 +328,25 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem if(_rc_ == NO_ERROR) { - NewMem->stTmSign.bGlobVersion = T5GLOBVERSION; - NewMem->stTmSign.bMajorVersion = T5MAJVERSION; - NewMem->stTmSign.bMinorVersion = T5MINVERSION; + NewMem->state->stTmSign.bGlobVersion = T5GLOBVERSION; + NewMem->state->stTmSign.bMajorVersion = T5MAJVERSION; + NewMem->state->stTmSign.bMinorVersion = T5MINVERSION; //build name and extension of tm data file //fill signature record structure - strcpy( NewMem->stTmSign.szName, NewMem->TmBtree.fb.fileName.c_str() ); - UtlTime( &(NewMem->stTmSign.lTime) ); - UtlTime( &(NewMem->stTmSign.creationTime) ); - strcpy( NewMem->stTmSign.szSourceLanguage, + strcpy( NewMem->state->stTmSign.szName, NewMem->state->TmBtree.fb.file.fileName.c_str() ); + UtlTime( &(NewMem->state->stTmSign.lTime) ); + UtlTime( &(NewMem->state->stTmSign.creationTime) ); + strcpy( NewMem->state->stTmSign.szSourceLanguage, strSrcLang.c_str() ); - strcpy( NewMem->stTmSign.szDescription, + strcpy( NewMem->state->stTmSign.szDescription, strMemDescription.c_str() ); //call create function for data file - NewMem->usAccessMode = 1;//ASD_LOCKED; // new TMs are always in exclusive access... + NewMem->state->usAccessMode = 1;//ASD_LOCKED; // new TMs are always in exclusive access... - _rc_ = NewMem->TmBtree.QDAMDictCreateLocal( &(NewMem->stTmSign),FIRST_KEY, keepInRamOnly); + _rc_ = NewMem->state->TmBtree.QDAMDictCreateLocal( &(NewMem->state->stTmSign),FIRST_KEY, keepInRamOnly); } std::vector buffer; @@ -356,28 +356,28 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem { //insert initialized record to tm data file lKey = AUTHOR_KEY; - tmxTableToBuffer(&NewMem->Authors,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); + tmxTableToBuffer(&NewMem->state->Authors,buffer); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey,buffer); } if ( _rc_ == NO_ERROR ) { lKey = FILE_KEY; - tmxTableToBuffer(&NewMem->FileNames,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); + //tmxTableToBuffer(&NewMem->FileNames,buffer); + // _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { lKey = TAGTABLE_KEY; - tmxTableToBuffer(&NewMem->TagTables,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); + tmxTableToBuffer(&NewMem->state->TagTables,buffer); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) { lKey = LANG_KEY; - tmxTableToBuffer(&NewMem->Languages,buffer); - _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey,buffer); + tmxTableToBuffer(&NewMem->state->Languages,buffer); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey,buffer); } /* endif */ if ( _rc_ == NO_ERROR ) @@ -385,11 +385,11 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem int size = MAX_COMPACT_SIZE-1 ; //initialize and insert compact area record //memset( NewMem->bCompact, 0, size ); - NewMem->bCompact.resize(size); + NewMem->state->bCompact.resize(size); lKey = COMPACT_KEY; - _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey, - NewMem->bCompact); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey, + NewMem->state->bCompact); } /* endif */ @@ -397,10 +397,10 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem if ( _rc_ == NO_ERROR ) { lKey = LONGNAME_KEY; - NewMem->LongNames.pszBuffer.resize(sizeof(TMX_RECORD));//to have some size + NewMem->state->LongNames.pszBuffer.resize(sizeof(TMX_RECORD));//to have some size // write long document name buffer area to the database - _rc_ = NewMem->TmBtree.EQFNTMInsert(lKey, - NewMem->LongNames.pszBuffer ); + _rc_ = NewMem->state->TmBtree.EQFNTMInsert(lKey, + NewMem->state->LongNames.pszBuffer ); } /* endif */ @@ -413,21 +413,21 @@ std::shared_ptr TMManager::CreateNewEmptyTM(const std::string& strMem if ( _rc_ == NO_ERROR ) { //fill signature record structure - strcpy( NewMem->stTmSign.szName, NewMem->InBtree.fb.fileName.c_str() ); - _rc_ = NewMem->InBtree.QDAMDictCreateLocal(&NewMem->stTmSign, START_KEY, keepInRamOnly); + strcpy( NewMem->state->stTmSign.szName, NewMem->state->InBtree.fb.file.fileName.c_str() ); + _rc_ = NewMem->state->InBtree.QDAMDictCreateLocal(&NewMem->state->stTmSign, START_KEY, keepInRamOnly); } /* endif */ if(_rc_ == NO_ERROR){ if(!keepInRamOnly){ - NewMem->TmBtree.fb.Flush(); - NewMem->InBtree.fb.Flush(); + NewMem->state->TmBtree.fb.Flush(); + NewMem->state->InBtree.fb.Flush(); } NewMem->eStatus = OPEN_STATUS; }else { NewMem->eStatus = FAILED_TO_OPEN_STATUS; //something went wrong during create or insert so delete data file - UtlDelete( (PSZ)NewMem->InBtree.fb.fileName.c_str(), 0L, FALSE ); - UtlDelete((PSZ) NewMem->TmBtree.fb.fileName.c_str(), 0L, FALSE); + UtlDelete( (PSZ)NewMem->state->InBtree.fb.file.fileName.c_str(), 0L, FALSE ); + UtlDelete((PSZ) NewMem->state->TmBtree.fb.file.fileName.c_str(), 0L, FALSE); } /* endif */ return NewMem; } @@ -579,11 +579,6 @@ int TMManager::RenameTM( std::string& TMManager::getLastError( EqfMemory *pMemory, int& iLastError, std::string& strError) { - if ( pMemory != NULL ) - this->iLastError = pMemory->getLastError( this->strLastError ); - - iLastError = this->iLastError; - strError = this->strLastError; return strError; } @@ -1408,7 +1403,7 @@ int TMManager::OpenTM(const std::string& strMemName, MutexTimeout& tmListTimeout //if ( pMem == nullptr ) if(memLeftAfterOpening <= 0){ - //buildErrorReturn( _rc_, "OtmMemoryServiceWorker::import::Error: too many open translation memory databases" ); + //buildErrorReturn( _rc_, "::import::Error: too many open translation memory databases" ); return( INTERNAL_SERVER_ERROR ); } /* endif */ std::shared_ptr pMem( EqfMemoryPlugin::GetInstance()->initTM(strMemName, requiredMemory, 0, fReorganizeOnly)); diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 9e23620a..9133511e 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -1764,7 +1764,7 @@ SHORT BTREE::QDAMDictCreateLocal if( false == keepInRamOnly ) { // Try to create the index file - fb.file = LockedFile(fb.fileName, LockedFile::CalledFrom::Create); + fb.file = LockedFile(fb.file.fileName, LockedFile::CalledFrom::Create); } if ( !sRc ) @@ -4222,7 +4222,7 @@ SHORT QDAMChangeKey_V3 size_t BTREE::GetFileSize()const{ - return FilesystemHelper::GetFileSize(fb.fileName); + return FilesystemHelper::GetFileSize(fb.file.fileName); } @@ -4756,7 +4756,7 @@ SHORT BTREE::EQFNTMOpen } else { - T5LOG(T5ERROR) <<"Can't understand file " << fb.fileName << ". File has illegal structure!"; + T5LOG(T5ERROR) <<"Can't understand file " << fb.file.fileName << ". File has illegal structure!"; //fOpen = FALSE; sRc = BTREE_ILLEGAL_FILE; } /* endif */ @@ -5200,7 +5200,7 @@ SHORT BTREE::QDAMDictSignLocal //sRc = SkipBytesFromBeginningInFile(pBT->fb.file, USERDATA_START); sRc = fb.Read((PVOID) &usLen, sizeof(USHORT), USERDATA_START); if(sRc){ - T5LOG(T5ERROR) << "Can't read from file \'"<CheckLogLevel(T5DEBUG)){ - oldSize = FilesystemHelper::GetFileSize(fileName); + oldSize = FilesystemHelper::GetFileSize(file.fileName); } //writenBytes = fwrite(bufStart, size, 1, file) * size; @@ -634,7 +635,7 @@ int FileBuffer::Flush(bool forced){ }else{ if(VLOG_IS_ON(1)){ - T5LOG( T5INFO) <<"Flush:: buffer not modified, so no need to overwrite file, fName = " << fileName; + T5LOG( T5INFO) <<"Flush:: buffer not modified, so no need to overwrite file, fName = " << file.fileName; } } return 0; @@ -647,12 +648,12 @@ int FileBuffer::Read(void* buff, size_t buffSize,ssize_t startingPos){ if(data.size()< offset + buffSize){ if(T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - T5LOG(T5ERROR) << "ReadBuffer:: Trying to read not existing bytes from buffer, fName = " << fileName; + T5LOG(T5ERROR) << "ReadBuffer:: Trying to read not existing bytes from buffer, fName = " << file.fileName; } return FilesystemHelper::FILEHELPER_WARNING_FILE_IS_SMALLER_THAN_REQUESTED; } if(VLOG_IS_ON(1) && T5Logger::GetInstance()->CheckLogLevel(T5DEVELOP)){ - T5LOG( T5DEVELOP) << ":: fName = " << fileName << "; buff size = " << buffSize << "; data.size = " << data.size() << "; offset = " << offset << ";"; + T5LOG( T5DEVELOP) << ":: fName = " << file.fileName << "; buff size = " << buffSize << "; data.size = " << data.size() << "; offset = " << offset << ";"; } memcpy(buff, &(data[offset]), buffSize); @@ -661,7 +662,7 @@ int FileBuffer::Read(void* buff, size_t buffSize,ssize_t startingPos){ } bool FileBuffer::isTMDFilebuffer()const{ - return EncodingHelper::endsWithIgnoreCase(fileName, ".tmd" ); + return EncodingHelper::endsWithIgnoreCase(file.fileName, ".tmd" ); } DECLARE_int64(allowedtmdsize); @@ -670,14 +671,14 @@ int FileBuffer::Write(const void* buff, size_t buffSize, size_t startingPosition status |= MODIFIED; if(startingPosition + buffSize > data.size()){ if(VLOG_IS_ON(1)){ - T5LOG( T5DEBUG) << "::Resizing filebuffer "< allowedSize){ - T5LOG(T5INFO) << "filebuffer vector for " << fileName + T5LOG(T5INFO) << "filebuffer vector for " << file.fileName << " resizing in unsuccessfull, because it's bigger than allowed size " << allowedSize <<" bytes" ; //return FilesystemHelper::FILEHELPER_ERROR_DATA_SIZE_BIGGER_THAN_ALLOWED_FILEBUFFER_SIZE; return TMD_SIZE_IS_BIGGER_THAN_ALLOWED; @@ -686,7 +687,7 @@ int FileBuffer::Write(const void* buff, size_t buffSize, size_t startingPosition data.resize(offset + buffSize); if( offset + buffSize > data.size() ){ - T5LOG(T5ERROR) << "filebuffer vector resizing in unsuccessfull for " << fileName ; + T5LOG(T5ERROR) << "filebuffer vector resizing in unsuccessfull for " << file.fileName ; return FilesystemHelper::FILEHELPER_ERROR_DATA_SIZE_BIGGER_THAN_RECORD_SIZE; } } @@ -782,7 +783,6 @@ int FilesystemHelper::CreateFilebuffer(std::string name){ if(pfb->find(name) != pfb->end()) return FILEHELPER_WARNING_FILEBUFFER_EXISTS; (*pfb)[name].offset=0; - return FILEHELPER_NO_ERROR; } diff --git a/source/opentm2/core/utilities/LanguageFactory.H b/source/opentm2/core/utilities/LanguageFactory.H index 281a8d72..310ac2df 100755 --- a/source/opentm2/core/utilities/LanguageFactory.H +++ b/source/opentm2/core/utilities/LanguageFactory.H @@ -120,8 +120,6 @@ int getOpenTM2NameFromISO ); - - /* \brief Checks if the given language is a valid target language \param pszLanguage OpenTM2 language name \returns true when the specified language is a valid source language diff --git a/source/opentm2/core/utilities/LanguageFactory.cpp b/source/opentm2/core/utilities/LanguageFactory.cpp index 40546940..8b2e9dc3 100755 --- a/source/opentm2/core/utilities/LanguageFactory.cpp +++ b/source/opentm2/core/utilities/LanguageFactory.cpp @@ -527,14 +527,11 @@ unsigned short LanguageFactory::loadLanguageList( const char *pszLangList ) unsigned short usRC = 0; // parse the XML file containing the language properties - if ( !usRC ) - { SAXParser parser; // Create a SAX parser object // create an instance of our handler LangParseHandler *handler = new LangParseHandler(); - // install our SAX handler as the document and error handler. parser.setDocumentHandler( handler ); parser.setErrorHandler( handler ); @@ -576,9 +573,7 @@ unsigned short LanguageFactory::loadLanguageList( const char *pszLangList ) } } } - } /* endif */ - - + return( usRC ); } diff --git a/source/opentm2/core/utilities/LogWrapper.h b/source/opentm2/core/utilities/LogWrapper.h index 5196c0be..a6ae7f1c 100755 --- a/source/opentm2/core/utilities/LogWrapper.h +++ b/source/opentm2/core/utilities/LogWrapper.h @@ -52,11 +52,21 @@ std::basic_ostream& operator<<(std::basic_ostream +//std::string toStr(T i) { +// if constexpr (std::is_pointer::value) { +// std::ostringstream oss; +// oss << static_cast(i); // print pointer address +// return oss.str(); +// } else { +// return std::to_string(i); +// } +//} enum LOGLEVEL{ T5DEVELOP=0, diff --git a/source/opentm2/core/utilities/OSWrapper.cpp b/source/opentm2/core/utilities/OSWrapper.cpp index 06322c35..37a0156f 100755 --- a/source/opentm2/core/utilities/OSWrapper.cpp +++ b/source/opentm2/core/utilities/OSWrapper.cpp @@ -9,21 +9,6 @@ #include #include "EncodingHelper.h" -int GetLocaleInfo( - LCID Locale, - LCTYPE LCType, - LPSTR lpLCData, - int cchData - ){ - LOG_UNIMPLEMENTED_FUNCTION; - return 0; - } - -UINT GetOEMCP(){ - return 0; - -} - unsigned long int _ttol(const char* source){ return atoi(source); @@ -34,7 +19,6 @@ unsigned long int _ttol(const char* source){ int MultiByteToWideChar( UINT CodePage, DWORD dwFlags, - //_In_NLS_string_(cbMultiByte)LPCCH lpMultiByteStr, LPCSTR lpMultiByteStr, int cbMultiByte, LPWSTR lpWideCharStr, @@ -43,7 +27,6 @@ int MultiByteToWideChar( mbstate_t state; int cnt = 0; memset (&state, '\0', sizeof (state)); - //return mbsnrtowcs(lpMultiByteStr, lpWideCharStr, cchWideChar, cbMultiByte, &state ); if(cbMultiByte<=0){ cbMultiByte = strlen(lpMultiByteStr); } @@ -51,34 +34,24 @@ int MultiByteToWideChar( lpWideCharStr[cbMultiByte] = 0; return mbsnrtowcs(lpWideCharStr, (const char**)&lpMultiByteStr, cbMultiByte, cchWideChar, &state ); - //EncodingHelper::convertToWChar(lpMultiByteStr); - //return mbtowc(lpWideCharStr, lpMultiByteStr, cchWideChar); } int WideCharToMultiByte( UINT CodePage, DWORD dwFlags, - //_In_NLS_string_(cchWideChar)LPCWCH lpWideCharStr, LPWSTR lpWideCharStr, int cchWideChar, LPSTR lpMultiByteStr, int cbMultiByte, - //LPCCH char* lpDefaultChar, LPBOOL lpUsedDefaultChar ){ - //mbstate_t state; - //int cnt = 0; - //memset (&state, '\0', sizeof (state)); - if(cchWideChar<=0){ cchWideChar = wcslen(lpWideCharStr); } if(cchWideChar > cbMultiByte-1) cchWideChar = cbMultiByte-1; lpMultiByteStr[cchWideChar] = 0; - return wcstombs(lpMultiByteStr, (const wchar_t*) &lpWideCharStr, cchWideChar); - //return wcsnrtombs(lpMultiByteStr, (const wchar_t**)&lpWideCharStr, cchWideChar, cbMultiByte, &state ); - + return wcstombs(lpMultiByteStr, (const wchar_t*) &lpWideCharStr, cchWideChar); } @@ -90,22 +63,6 @@ DWORD WaitForSingleObject( return 0; } -HANDLE CreateMutex( - LPSECURITY_ATTRIBUTES lpMutexAttributes, - BOOL bInitialOwner, - LPCSTR lpName - ){ - LOG_UNIMPLEMENTED_FUNCTION; - return NULL; - } - -BOOL ReleaseMutex( - HANDLE hMutex -){ - LOG_UNIMPLEMENTED_FUNCTION; - return true; -} - BOOL FindClose( HANDLE hFindFile ){ diff --git a/source/opentm2/core/utilities/OSWrapper.h b/source/opentm2/core/utilities/OSWrapper.h index d47fa87c..b194a5dc 100755 --- a/source/opentm2/core/utilities/OSWrapper.h +++ b/source/opentm2/core/utilities/OSWrapper.h @@ -148,15 +148,6 @@ enum LCTYPES }; -int GetLocaleInfo( - LCID Locale, - LCTYPE LCType, - LPSTR lpLCData, - int cchData -); - -UINT GetOEMCP(); - unsigned long int _ttol(const char* source); int _strcmp(const char* a, const char* b); diff --git a/source/opentm2/core/utilities/UtlMisc.cpp b/source/opentm2/core/utilities/UtlMisc.cpp index b39c8eb7..47a0dc6b 100755 --- a/source/opentm2/core/utilities/UtlMisc.cpp +++ b/source/opentm2/core/utilities/UtlMisc.cpp @@ -362,48 +362,6 @@ LONG UtlTime( PLONG plTime ) // Returncodes: 0 function completed successfully // other DOS error return codes //+----------------------------------------------------------------------------+ -USHORT UtlCompFDates -( - PSZ pszFile1, // fully qualified path first file - PSZ pszFile2, // fully qualified path of second file - PSHORT psResult, // address of result buffer - BOOL fMsg // error handling flag -) -{ - USHORT usRC = NO_ERROR; // function return code - FILESTATUS stStatus1; // File status information of 1st file - FILESTATUS stStatus2; // File status information of 2nd file - LONG lResult = 0L; // buffer for result value - - /********************************************************************/ - /* Get date and time of files */ - /********************************************************************/ - usRC = UtlQPathInfo( pszFile1, 1, (PBYTE)&stStatus1, sizeof(stStatus1), - 0L, fMsg ); - if ( usRC == NO_ERROR ) - { - usRC = UtlQPathInfo( pszFile2, 1, (PBYTE)&stStatus2, sizeof(stStatus2), - 0L, fMsg ); - } /* endif */ - - /********************************************************************/ - /* Compare file dates */ - /********************************************************************/ - if ( usRC == NO_ERROR ) - { - lResult = UtlCompareDate( &(stStatus1.fdateLastWrite), - &(stStatus2.fdateLastWrite) ); - if ( lResult == 0L ) - { - lResult = UtlCompareTime( &(stStatus1.ftimeLastWrite), - &(stStatus2.ftimeLastWrite) ); - } /* endif */ - *psResult = (SHORT)lResult; - } /* endif */ - - return usRC; -} /* end of function UtlCompFDates */ - // function searching first character causing a conversion error and showing an // approbriate error message diff --git a/source/opentm2/core/utilities/UtlString.cpp b/source/opentm2/core/utilities/UtlString.cpp index 0eab3ca5..a55f7c20 100755 --- a/source/opentm2/core/utilities/UtlString.cpp +++ b/source/opentm2/core/utilities/UtlString.cpp @@ -775,17 +775,7 @@ USHORT UtlQueryCharTableEx case 813 : pTable = chAnsiToPC813; break; case 737 : T5LOG( T5DEBUG) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 78 if ( (GetOEMCP() == 869) && (GetKBCodePage() == 869)) { // fix for sev1 Greek: Win NT problem (01/09/23)"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - if ( (GetOEMCP() == 869) && (GetKBCodePage() == 869)) - { // fix for sev1 Greek: Win NT problem (01/09/23) - usCP = 869; - pTable = chAnsiToPC869; - } - else - { - pTable = chAnsiToPC737; - } /* endif */ -#endif //TO_BE_REPLACED_WITH_LINUX_CODE + break; case 775 : pTable = chAnsiToPC775; break; case 864 : pTable = chAnsiToPC864; break; // Arabic OS/2 @@ -840,19 +830,7 @@ T5LOG( T5DEBUG) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 78 if ( (GetOEMCP() = case 813 : pTable = chAnsiToPC813; pInvTable = chPC813ToAnsi; break; case 737 : T5LOG( T5DEBUG) << ":: TO_BE_REPLACED_WITH_LINUX_CODE id = 79 if ( (GetOEMCP() == 869) && (GetKBCodePage() == 869) ) { // fix for sev1 Greek: Win NT problem (01/09/23)"; -#ifdef TO_BE_REPLACED_WITH_LINUX_CODE - if ( (GetOEMCP() == 869) && (GetKBCodePage() == 869) ) - { // fix for sev1 Greek: Win NT problem (01/09/23) - usCP = 869; - pTable = chAnsiToPC869; - pInvTable = chPC869ToAnsi; - } - else - { - pTable = chAnsiToPC737; - pInvTable = chPC737ToAnsi; - } /* endif */ -#endif //TO_BE_REPLACED_WITH_LINUX_CODE + break; case 775 : pTable = chAnsiToPC775; pInvTable = chPC775ToAnsi; break; case 864 : pTable = chAnsiToPC864; pInvTable = chPC864ToAnsi; break; // Arabic OS/2 diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c index 68b4b5b1..809f1ac4 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c @@ -85,7 +85,6 @@ extern char szDocSourceLanguage[80]; /* From USRCALLS.C */ extern short sTPVersion ; /* From USRCALLS.C */ - BOOL ReplaceXmlInZip(char*, char*, char* ); BOOL ExecuteCommand( char*, char*, char* ); BOOL ConcatFiles( char*, char*, char* ); BOOL SplitFiles_1( char*, char*, char*, char*, char* ); @@ -350,81 +349,6 @@ EQF_BOOL /*APIENTRY*/ EQFPREUNSEGW( * *******************************************************************************/ -EQF_BOOL /*APIENTRY*/ EQFPOSTUNSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pTarget, - PTATAG pTATag, - PEQF_BOOL pfKill ) -{ -// ============================================================ - BOOL bReturn = TRUE; - char szStyleFile[512]; - char szTempFile[512]; /* Required to avoid abend */ - char szTempFile2[512]; - char szDLLFile[512]; - char *ptrChar ; - - char *szAltTempExt1 = ".$6$"; - - PSZ ModuleName = "OTMXML" ; - PSZ ProcName = "EQFPOSTUNSEGW"; - HINSTANCE hInst_DLL; - USHORT rc ; - - typedef EQF_BOOL (* /*APIENTRY*/ XMLPOSTUNSEG_DCL)( PSZ, PSZ, PSZ, PSZ, PTATAG, PEQF_BOOL ) ; - XMLPOSTUNSEG_DCL XMLPOSTUNSEG_addr; -// ============================================================ - - - PrepDocLanguageInfo( pTarget ) ; /* Set language unqiue processing, like DBCS */ - - CreateTempFileName2( szTempFile2, pTarget, szAltTempExt1, TEMPNAME_TARGET ) ; - - - /* Explicitly load OTMXML.DLL */ - rc = 0 ; - GetOTMDllPath( pProgPath, szDLLFile ) ; - strcat( szDLLFile, ModuleName ) ; - hInst_DLL = LoadLibraryA( szDLLFile ); - if ( hInst_DLL != NULL ) { - XMLPOSTUNSEG_addr = (XMLPOSTUNSEG_DCL)GetProcAddress((HMODULE)hInst_DLL, ProcName); - if (XMLPOSTUNSEG_addr != NULL ) { - rc = (*XMLPOSTUNSEG_addr)(pTagTable,pEdit,pProgPath,pTarget,pTATag,pfKill); - } - FreeLibrary( hInst_DLL ) ; - } - if ( rc == 0 ) - bReturn = FALSE ; - - - if ( bReturn ) { - strcpy( szStyleFile, pTarget ) ; - ptrChar = strstr( szStyleFile, "\\TARGET\\" ) ; - if ( ptrChar ) { - strcpy( ptrChar, "\\RTF\\" ) ; - ptrChar = strrchr( pTarget, '\\' ) ; - strcat( szStyleFile, ++ptrChar ) ; - } else { - szStyleFile[0] = 0 ; - } - bReturn = PostUnseg( pTarget, szStyleFile ); - } - if ( bReturn ) { - if ( ! ReplaceXmlInZip( pTarget, szTempFile2, szErrMsg ) ) { - MessageBoxA(HWND_DESKTOP, szErrMsg, ODC_REPLACE_ERROR_TITLE, MB_OK); - bReturn = FALSE ; - } - } - - remove( szTempFile2 ) ; - - return(bReturn); -} /* EQFPOSTUNSEGW */ - - - /*****************************************************************************/ @@ -441,163 +365,6 @@ EQF_BOOL /*APIENTRY*/ EQFPOSTUNSEGW( /* FALSE - File could not be processed. */ /*****************************************************************************/ -BOOL ReplaceXmlInZip( char *TargetFile, char *TempFile, char *ErrText ) -{ - STARTUPINFOA StartupInfo ; - PROCESS_INFORMATION piProcessInfo ; - FILE *fTemp ; - struct stat FileInfo ; - char szSourceFile[256] ; - char szCommand[1024] ; - char szTempDir1[256] ; - char szTempDir2[256] ; - char szTempFile[256] ; - char szTempZip[256] ; - char szSaveDir[256] ; - char *ptrChar ; - USHORT rc ; - DWORD dwCode; - BOOL bFormatXml = FALSE ; - BOOL bReturn = TRUE; - - - /*-----------------------------------------------------------------------*/ - /* Determine input file format, whether ZIP file or CONTENT.XML */ - /*-----------------------------------------------------------------------*/ - strcpy( szSourceFile, TargetFile ) ; - ptrChar = strstr( szSourceFile, "\\TARGET\\" ) ; - if ( ptrChar ) { - strncpy( ptrChar, "\\SOURCE\\", 8 ) ; - if ( FindFileType( szSourceFile, &bFormatXml ) ) { - if ( ! bFormatXml ) { - GetOTMDllPath( TargetFile, szZipExe) ; - strcat( szZipExe, "ZIP.EXE" ) ; - rc = stat( szZipExe, &FileInfo ) ; - if ( rc != 0 ) { - strcpy( ErrText, ODC_REPLACE_MISSING_ZIP ) ; - bReturn = FALSE ; - } - GetOTMDllPath( TargetFile, szUnzipExe) ; - strcat( szUnzipExe, "UNZIP.EXE" ) ; - rc = stat( szUnzipExe, &FileInfo ) ; - if ( rc != 0 ) { - strcpy( ErrText, ODC_EXTRACT_MISSING_UNZIP ) ; - bReturn = FALSE ; - } - } - } else { - strcpy( ErrText, ODC_BAD_FORMAT ) ; - bReturn = FALSE ; - } - } else { - sprintf( ErrText, ODC_REPLACE_TARGET_PATH, TargetFile ) ; - bReturn = FALSE ; - } - - if ( bReturn ) { - /*-----------------------------------------------------------------------*/ - /* File is already in XML format. No action necessary. */ - /*-----------------------------------------------------------------------*/ - if ( bFormatXml ) { - } else { - /*--------------------------------------------------------------------*/ - /* XML files must be replaced into the original document ZIP file. */ - /* 1. Create temporary directories. */ - /* 2. Copy target file to CONTENT.XML in temporary directory. */ - /* 3. Copy target ZIP to same name in temporary directory. */ - /* 4. Zip temp dir CONTENT.XML into temp dir ZIP file. */ - /* 5. Copy temp dir ZIP file to output file. */ - /* 6. Delete temporary directories. */ - /*--------------------------------------------------------------------*/ - strcpy( szTempDir1, TargetFile ) ; - ptrChar = strstr( szTempDir1, "\\TARGET\\" ) ; - if ( ptrChar ) { - /*-----------------------------------------------------------------*/ - /* Create temporary directories. */ - /*-----------------------------------------------------------------*/ - strcpy( ptrChar, "\\MISC\\" ) ; - _mkdir( szTempDir1 ) ; /* Create \EQF\...\MISC\ */ - strcpy( szTempDir2, szTempDir1 ) ; - ptrChar = strrchr( TargetFile, '\\' ) ; - strcat( szTempDir2, ++ptrChar ) ; - strcat( szTempDir2, "\\" ) ; - _mkdir( szTempDir2 ) ; /* Create \EQF\...\MISC\...\ */ - - strcpy( szTempFile, szTempDir2 ) ; - strcat( szTempFile, ODC_TEMP_XML ) ; - DosCopy( TargetFile, szTempFile, DCPY_EXISTING ) ; - strcpy( szTempZip, szTempDir2 ) ; - strcat( szTempZip, ptrChar ) ; - DosCopy( szSourceFile, szTempZip, DCPY_EXISTING ) ; - - /*-----------------------------------------------------------------*/ - /* Split the one XML file into its individual files. */ - /*-----------------------------------------------------------------*/ - if ( FindSTargetType( TargetFile ) ) { - bReturn = SplitFiles_1( TargetFile, szTempDir2, szSourceFile, TempFile, ErrText ) ; - if ( bReturn ) { - /*-----------------------------------------------------------*/ - /* Update the files in the ZIP file. */ - /*-----------------------------------------------------------*/ - sprintf( szCommand, "\"%s\" -j %s %s*.xml > %s", - szZipExe, szTempZip, szTempDir2, TempFile ) ; - sprintf( ErrText, ODC_REPLACE_FAILED, ODC_CONTENT_XML ) ; - bReturn = ExecuteCommand( szCommand, TempFile, ErrText ) ; - if ( bReturn ) { - DosCopy( szTempZip, TargetFile, DCPY_EXISTING ) ; - } - } - } else { - bReturn = SplitFiles_2( TargetFile, szTempDir2, ErrText ) ; - if ( bReturn ) { - /*-----------------------------------------------------------*/ - /* Update the files in the ZIP file. */ - /*-----------------------------------------------------------*/ - _getcwd( szSaveDir, sizeof(szSaveDir) ) ; - chdir( szTempDir2 ) ; - sprintf( szCommand, "\"%s\" -r -f %s *.xml > %s", - szZipExe, szTempZip, TempFile ) ; - strcpy( ErrText, ODC_REPLACE_FAILED ) ; - bReturn = ExecuteCommand( szCommand, TempFile, ErrText ) ; - if ( bReturn ) { - DosCopy( szTempZip, TargetFile, DCPY_EXISTING ) ; - } - chdir( szSaveDir ) ; - } - } - - /*-----------------------------------------------------------------*/ - /* Clean-up. */ - /*-----------------------------------------------------------------*/ - // remove( szTempFile ) ; - // remove( szTempZip ) ; - // strcpy( szTempFile, szTempDir2 ) ; - // strcat( szTempFile, ODC_CONTENT_XML ) ; - // remove( szTempFile ) ; - // strcpy( szTempFile, szTempDir2 ) ; - // strcat( szTempFile, ODC_META_XML ) ; - // remove( szTempFile ) ; - // strcpy( szTempFile, szTempDir2 ) ; - // strcat( szTempFile, ODC_STYLES_XML ) ; - // remove( szTempFile ) ; - // - // _rmdir( szTempDir2 ) ; - // _rmdir( szTempDir1 ) ; - - remove( szTempZip ) ; - sprintf( szCommand, "cmd /C rd /s /q %s > %s", szTempDir2, TempFile ) ; - ExecuteCommand( szCommand, TempFile, NULL ) ; - } else { - sprintf( ErrText, ODC_REPLACE_TARGET_PATH, szTempDir1 ) ; - bReturn = FALSE ; - } - } - } - - return( bReturn ) ; -} - - /*****************************************************************************/ /* ExecuteCommand */ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp index 213ea511..3ba1c461 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp @@ -803,65 +803,6 @@ BOOL CheckFilesExist( char *pszFile, char *pszBasePath, char *pszErrMsg ) 3 = Unsuccessful copy */ -int CopyMarkupFile( char *pszNewFile, char *pszMarkupFile, - BOOL bSaveInUse ) -{ - char szTemp[MAX_LONGFILESPEC]; - char *ptrChar ; - short sResult ; - int iReturn = COPY_MARKUP_NOCHANGE ; - - - if ( UtlFileExist( pszNewFile ) ) { - if ( UtlFileExist( pszMarkupFile ) ) { - if ( ( UtlCompFDates( pszNewFile, pszMarkupFile, &sResult, FALSE ) == 0 ) && - ( sResult > 0 ) ) { - sResult = UtlCopy( pszNewFile, pszMarkupFile, 1, 0L, FALSE ); - if ( ( bSaveInUse ) && - ( sResult == ERROR_SHARING_VIOLATION ) ) { - iReturn = COPY_MARKUP_INUSE ; - - // copy markup table file to "copy pending user exit" directory - strcpy( szTemp, pszMarkupFile ) ; - ptrChar = strrchr( szTemp, BACKSLASH ) ; - if ( ptrChar ) { - int iLen = strlen(PENDINGEXITS_DIR) ; - memmove( ptrChar+iLen+1, ptrChar, strlen(ptrChar)+1 ) ; - *ptrChar = BACKSLASH ; - strncpy( ptrChar+1, PENDINGEXITS_DIR, iLen ) ; - ptrChar += iLen + 1 ; - *ptrChar = 0 ; - UtlMkDir( szTemp, 0, FALSE ); - *ptrChar = BACKSLASH ; - iLen = CopyMarkupFile( pszNewFile, szTemp, FALSE ) ; - if ( iLen == COPY_MARKUP_ERROR ) { - iReturn = iLen ; - } - } else { - iReturn = COPY_MARKUP_ERROR ; - } - } - else - if ( ! sResult ) { - iReturn = COPY_MARKUP_COPIED ; - } else { - iReturn = COPY_MARKUP_ERROR ; - } - } - } else { - sResult = UtlCopy( pszNewFile, pszMarkupFile, 1, 0L, TRUE ); - if ( ! sResult ) - iReturn = COPY_MARKUP_COPIED ; - else - iReturn = COPY_MARKUP_ERROR ; - } - } else { - iReturn = COPY_MARKUP_ERROR ; - } - return( iReturn ) ; -} - - // perform any pending updates on startup void PerformPendingUpdates( char *pszBasePath ) { diff --git a/source/opentm2/plugins/markup/UserMarkupTablePlugin/UserMarkupTablePlugin.cpp b/source/opentm2/plugins/markup/UserMarkupTablePlugin/UserMarkupTablePlugin.cpp index 9244321c..54c102c7 100755 --- a/source/opentm2/plugins/markup/UserMarkupTablePlugin/UserMarkupTablePlugin.cpp +++ b/source/opentm2/plugins/markup/UserMarkupTablePlugin/UserMarkupTablePlugin.cpp @@ -982,56 +982,6 @@ BOOL CheckFilesExist( char *pszFile, char *pszBasePath, char *pszErrMsg ) 3 = Unsuccessful copy */ -int CopyMarkupFiles( char *pszNewFile, char *pszOldFile, - char *pszBasePath, char *pszTableDirectory, - char *pszUserExitDirectory, BOOL bSaveInUse ) -{ - char szTemp[8096]; - char szTemp2[8096]; - char *ptrNewFile, *ptrOldFile ; - char *ptrChar, *ptrChar2 ; - int iRC ; - int iReturn = COPY_MARKUP_NOCHANGE ; - - - strcpy( szTemp2, pszNewFile ) ; - ptrNewFile = strtok( szTemp2, " ," ) ; - while( ( ptrNewFile ) && - ( iReturn != COPY_MARKUP_ERROR ) ) { - ptrChar = strrchr( ptrNewFile, '\\' ) ; - if ( ptrChar ) { - strcpy( szTemp, pszOldFile ) ; - ptrChar2 = strstr( szTemp, ptrChar ) ; - if ( ptrChar2 ) { - *(ptrChar2+strlen(ptrChar)) = '\0' ; - ptrOldFile = strrchr( szTemp, ',' ) ; - if ( ! ptrOldFile ) - ptrOldFile = szTemp ; - else - ++ptrOldFile ; - } else { - ptrChar2 = strrchr( pszNewFile, '.' ) ; - if ( ( ptrChar2 ) && - ( ( ! strcmp( ptrChar2+1, "DLL" ) ) || - ( ! strcmp( ptrChar2+1, "EXE" ) ) ) ) - sprintf( szTemp, "%s\\%s\\%s", pszBasePath, pszUserExitDirectory, ptrChar+1 ) ; - else - sprintf( szTemp, "%s\\%s\\%s", pszBasePath, pszTableDirectory, ptrChar+1 ) ; - ptrOldFile = szTemp ; - } - - iRC = CopyMarkupFile( ptrNewFile, ptrOldFile, bSaveInUse ) ; - if ( ( iRC == COPY_MARKUP_INUSE ) || - ( iRC == COPY_MARKUP_ERROR ) || - ( ( iRC == COPY_MARKUP_COPIED ) && - ( iReturn != COPY_MARKUP_INUSE ) ) ) { - iReturn = iRC ; - } - } - ptrNewFile = strtok( NULL, " ," ) ; - } - return( iReturn ) ; -} /* -------------------------------------------------------------- */ /* CopyMarkupFile */ @@ -1054,64 +1004,6 @@ int CopyMarkupFiles( char *pszNewFile, char *pszOldFile, 3 = Unsuccessful copy */ -int CopyMarkupFile( char *pszNewFile, char *pszMarkupFile, - BOOL bSaveInUse ) -{ - char szTemp[MAX_LONGFILESPEC]; - char *ptrChar ; - short sResult ; - int iReturn = COPY_MARKUP_NOCHANGE ; - - - if ( UtlFileExist( pszNewFile ) ) { - if ( UtlFileExist( pszMarkupFile ) ) { - if ( ( UtlCompFDates( pszNewFile, pszMarkupFile, &sResult, FALSE ) == 0 ) && - ( sResult > 0 ) ) { - sResult = UtlCopy( pszNewFile, pszMarkupFile, 1, 0L, FALSE ); - if ( ( bSaveInUse ) && - ( sResult == ERROR_SHARING_VIOLATION ) ) { - iReturn = COPY_MARKUP_INUSE ; - - // copy markup table file to "copy pending user exit" directory - strcpy( szTemp, pszMarkupFile ) ; - ptrChar = strrchr( szTemp, BACKSLASH ) ; - if ( ptrChar ) { - int iLen = strlen(PENDINGEXITS_DIR) ; - memmove( ptrChar+iLen+1, ptrChar, strlen(ptrChar)+1 ) ; - *ptrChar = BACKSLASH ; - strncpy( ptrChar+1, PENDINGEXITS_DIR, iLen ) ; - ptrChar += iLen + 1 ; - *ptrChar = 0 ; - UtlMkDir( szTemp, 0, FALSE ); - *ptrChar = BACKSLASH ; - iLen = CopyMarkupFile( pszNewFile, szTemp, FALSE ) ; - if ( iLen == COPY_MARKUP_ERROR ) { - iReturn = iLen ; - } - } else { - iReturn = COPY_MARKUP_ERROR ; - } - } - else - if ( ! sResult ) { - iReturn = COPY_MARKUP_COPIED ; - } else { - iReturn = COPY_MARKUP_ERROR ; - } - } - } else { - sResult = UtlCopy( pszNewFile, pszMarkupFile, 1, 0L, TRUE ); - if ( ! sResult ) - iReturn = COPY_MARKUP_COPIED ; - else - iReturn = COPY_MARKUP_ERROR ; - } - } else { - iReturn = COPY_MARKUP_ERROR ; - } - return( iReturn ) ; -} - /* -------------------------------------------------------------- */ /* UpdateXMLControlFile */ From f8d0119fd4e19b7b0c019e313abc8049edb42a90 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Wed, 13 Aug 2025 21:08:46 +0300 Subject: [PATCH 52/62] * updated to 0.7.24 + added skip_0s_at_the_table_start * fixed tagreplacement request * fixed json for flags and tagreplacement requests - code cleanup --- include/tm.h | 10 +- source/CMakeLists.txt | 2 +- source/RestAPI/OtmMemoryServiceWorker.cpp | 1 - .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 13 +- source/opentm2/core/requestdata.cpp | 66 +- .../OtmMarkupTablePlugin/OTMMSOFC/EXPORT.CPP | 1787 ----------------- .../OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c | 285 --- .../OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c | 331 --- source/otmd.cpp | 3 + 9 files changed, 51 insertions(+), 2447 deletions(-) delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/EXPORT.CPP diff --git a/include/tm.h b/include/tm.h index 529898f3..b1c2d5b2 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1525,11 +1525,11 @@ class StringTagVariants : public StringVariants std::wstring &getGenericTargetStr() { return genericTarget; } std::wstring &getNormTargetStr() { return normTarget; } - wchar_t *getNormStrC() { return &norm[0]; } - wchar_t *getNpReplStrC() { return &npReplaced[0]; } - wchar_t *getGenericTagStrC() { return &genericTags[0]; } - wchar_t *getGenericTargetStrC() { return &genericTarget[0]; } - wchar_t *getNormalizedTargetStrC() { return &normTarget[0]; } + wchar_t *getNormStrC() { return norm.data(); } + wchar_t *getNpReplStrC() { return npReplaced.data(); } + wchar_t *getGenericTagStrC() { return genericTags.data(); } + wchar_t *getGenericTargetStrC() { return genericTarget.data(); } + wchar_t *getNormalizedTargetStrC() { return normTarget.data(); } }; class RequestTagReplacer : public StringVariants diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 6d71d9f6..012fc33f 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 23) +SET(APP_VERSION_MINOR 24) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index 5cc15297..d85a555c 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -205,7 +205,6 @@ int FilesystemHelper::DecodeBase64ToFile( const char *pStringData, const char *p int iRC = 0; // get decoded length of data - DWORD dwDecodedLength = 0; std::string sData(pStringData); unsigned char* pData = NULL; int pDataSize = 0; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 6e835940..fb619e5f 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -774,10 +774,11 @@ bool endsWithTripleNull(const std::vector& buffer) { //+----------------------------------------------------------------------------+ //|Description: Reads the data of a long name table from the database. | //+----------------------------------------------------------------------------+ -//|Parameters: EqfMemory* pTmClb pointer to control block | +//|Parameters: EqfMemory* pTmClb pointer to control block | //+----------------------------------------------------------------------------+ //|Returncode type: USHORT error return code or NO_ERROR if O.K. | // ----------------------------------------------------------------------------+ +DECLARE_bool(skip_0s_at_the_table_start); USHORT EqfMemory::NTMReadLongNameTable() { if(state->LongNames.stTableEntry.empty()) @@ -810,7 +811,13 @@ USHORT EqfMemory::NTMReadLongNameTable() } ssize_t buffLen = state->LongNames.pszBuffer.size(); - while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen) && *((PUSHORT)pszTemp) != 0) + if(FLAGS_skip_0s_at_the_table_start){ + while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen-1) && *(pszTemp) == 0){//skip 0s at the start of the table; + pszTemp++; + } + } + + while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen - sizeof(USHORT)) && *((PUSHORT)pszTemp) != 0) { // get ID of string USHORT usId = *((PUSHORT)pszTemp); @@ -829,9 +836,7 @@ USHORT EqfMemory::NTMReadLongNameTable() } } /* endwhile */ SortLongNameTables(); - } /* endif */ - // return to caller return (usRC); } /* end of function NTMReadLongNameTable */ diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 09f82bd1..49082344 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -1890,6 +1890,22 @@ int DeleteMemRequestData::execute(){ } +std::wstring json_escape_quotes(const std::wstring& input) { + std::wstring output; + output.reserve(input.size()); // reserve space to avoid many reallocations + + for (wchar_t ch : input) { + if (ch == L'"') { + output.push_back(L'\\'); // add backslash + output.push_back(L'"'); // add quote + } else { + output.push_back(ch); + } + } + + return output; +} + int TagRemplacementRequestData::execute(){ _rc_ = 0; std::string strSrcData, strTrgData, strReqData; @@ -1930,20 +1946,24 @@ int TagRemplacementRequestData::execute(){ json_factory.parseJSONStop( parseHandle ); } - auto result = std::make_unique(EncodingHelper::convertToWChar(strSrcData), + wstr = L"{\n "; + + if(strReqData.size()){//request + auto result = std::make_unique(EncodingHelper::convertToWChar(strSrcData), EncodingHelper::convertToWChar(strTrgData), EncodingHelper::convertToWChar(strReqData)); - - //auto = replaceString( &_rc_); - - wstr = L"{\n "; - std::wstring segmentLocations[] = {L"source", L"target", L"request"}; + wstr += L"\"source\" :\"" + json_escape_quotes(result->getSrcWithTagsFromRequest()) + L"\",\n "; + wstr += L"\"target\" :\"" + json_escape_quotes(result->getTrgWithTagsFromRequest()) + L"\",\n "; + wstr += L"\"request\" :\"" + json_escape_quotes(result->getReqGenericTagStr()) + L"\"\n "; + }else{ + auto result = std::make_unique(EncodingHelper::convertToWChar(strSrcData), + EncodingHelper::convertToWChar(strTrgData)); - wstr += L"\'source\' :\'" + result->getSrcWithTagsFromRequest() + L"\',\n "; - wstr += L"\'target\' :\'" + result->getTrgWithTagsFromRequest() + L"\',\n "; - wstr += L"\'request\' :\'" + result->getReqGenericTagStr() + L"\',\n "; + wstr += L"\"source\" :\"" + json_escape_quotes(result->getGenericTagsString()) + L"\",\n "; + wstr += L"\"target\" :\"" + json_escape_quotes(result->getGenericTargetStr()) + L"\"\n "; + } wstr += L"\n}"; outputMessage = EncodingHelper::convertToUTF8(wstr); return _rest_rc_ = 200; @@ -2014,12 +2034,7 @@ int CloneTMRequestData::checkData(){ // check mem if is not in import state if(!_rc_){ - if(mem == nullptr){ - // tm is probably not opened, buf files presence was checked before, so it should be "AVAILABLE" status - // outputMessage = "\'newName\' " + strMemName +" was not found in memory list"; - // T5LOG(T5ERROR) << outputMessage << "; for request for mem "<< strMemName <<"; with body = ", strBody ; - // _rc_ = 500; - }else{ + if(mem){ // close the memory - if open if(mem->eImportStatus == IMPORT_RUNNING_STATUS){ std::string msg = "src tm \'" + strMemName +"\' is in import running status. Repeat request later; for request for mem " @@ -2076,24 +2091,9 @@ int CloneTMRequestData::execute(){ if(FilesystemHelper::FileExists(dstTmdPath.c_str())) FilesystemHelper::DeleteFile(dstTmdPath.c_str()); _rc_ = 500; } - if(!_rc_){ - - //EqfMemoryPlugin::GetInstance()->addMemoryToList(newName.c_str()); - }else{ + if(_rc_){ return buildErrorReturn(_rc_, msg.c_str()); } - if( mem != nullptr ) - { - //fClose = true; - //lHandle = pMem->lHandle; - //lastStatus = pMem->eStatus; - //lastImportStatus = pMem->eImportStatus; - - //pMem->lHandle = 0; - //pMem->eStatus = AVAILABLE_STATUS; - //pMem->eImportStatus = AVAILABLE_STATUS; //IMPORT_RUNNING_STATUS; - //pMem->dImportProcess = 0; - } if(_rc_ == 0 ){ outputMessage = newName + " was cloned successfully"; @@ -2449,8 +2449,8 @@ bool jsonAddFlagToList(std::stringstream &ss, const google::CommandLineFlagInfo if (fMoreInfo) { ss << "{\t\"name\": \"" << flag.name << "\",\t"; - ss << "\"value\": \"" << flag.current_value << "\"\t"; - ss << "\",\tfrom\": \"" << flag.filename << "\",\t "; + ss << "\"value\": \"" << flag.current_value << "\",\t"; + ss << "\"from\": \"" << flag.filename << "\",\t "; ss << "\"description\": \"" << flag.description << "\",\t"; ss << "\"type\": \"" << flag.type << "\",\t"; ss << "\"default\": \"" << flag.default_value << "\"\t}"; diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/EXPORT.CPP b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/EXPORT.CPP deleted file mode 100755 index cff8b495..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/EXPORT.CPP +++ /dev/null @@ -1,1787 +0,0 @@ -/* -* -* Copyright (C) 1998-2016, International Business Machines -* Corporation and others. All rights reserved -* -*/ -/****************************************************************************/ -/* */ -/* EXPORT.CPP */ -/* */ -/*FUNCTIONS: */ -/* PostExport */ -/* */ -/*==========================================================================*/ -/* Date : Who : Description */ -/*==========================================================================*/ -/* 2/26/2016: IBM : Original Source */ -/*==========================================================================*/ - -#include "unicode.h" -#include "otmmsofc.h" - - - -extern wifstream *InputFile ; -extern wifstream *InputFile2 ; - -extern WCHAR szPrevXMLInputText[MAX_XML_RCD_LENGTH2-MAX_XML_RCD_LENGTH+1] ; -extern BOOL bBidi ; -extern BOOL bDBCS ; -extern BOOL bUTF16 ; -extern BOOL bReadSource ; -extern BOOL bFileExport ; -extern BOOL bSetCommonProperty ; -extern BOOL bExportSSNTContent ; - - -extern short sTPVersion ; /* From USRCALLS.C */ -extern char szDocTargetLanguage[80]; /* From USRCALLS.C */ -extern char szDocSourceLanguage[80]; /* From USRCALLS.C */ - - - -#define TITLE_MSOFC_PARSING_ERROR "Parsing Error" -#define TITLE_MSOFC_PARAGRAPH_MISMATCH "Paragraph Mismatch" -#define TITLE_MSOFC_TEXT_UNITS_MISMATCH "Text Units Mismatch" - -#define MSG_MSOFC_PARSING_ERROR "File could not be parsed.\n\nProcessing is terminated." -#define MSG_MSOFC_PARAGRAPH_MISMATCH "Target paragraph ID=%ld does not match source paragraph ID=%ld.\n\nProcessing is terminated." -#define MSG_MSOFC_TEXT_UNITS_MISMATCH "Target paragraph %ld contains %ld text units while source contains %ld.\n\nProcessing is terminated." - - -/*****************************************************************************/ -/* */ -/* ===> PROBLEM WITH EXCEL 2007 <======= */ -/* */ -/* When the text is changed, any attribute which contains that same */ -/* English text must also be changed. The actual text is also used as */ -/* a reference to that cell, column, row. */ -/* */ -/* If the text is never referenced by an attribute, then it can just */ -/* be translated. */ -/* */ -/* */ -/* For example (inventory.xlsx): */ -/* Reorder Level */ -/* */ -/* */ -/* If the translatable text is changed: */ -/* Trans Reorder Level */ -/* */ -/* Then the reference also has to be changed: */ -/* */ -/* */ -/* */ -/* Sales.xlsx same problem for element. */ -/* workbook.xml */ -/* */ -/* app.xml */ -/* */ -/* */ -/* Sales By Region */ -/* pivotCacheDefinition1.xml */ -/* */ -/* table1.xml */ -/* */ -/* pivotTable2.xml */ -/* */ -/* */ -/* Statistics.xlsx ( is formula) */ -/* workbook.xml */ -/* */ -/* chart.xml */ -/* Skewness!$D$13:$D$21 */ -/* 'Normal Distribution'!$B$2:$B$802 */ -/* Sheet1!xxxx */ -/* Frequency */ -/* */ -/* Expenses.xlsx */ -/* sheet1.xml */ -/* "The expense total for this quarter in 2007 is " & F11 */ -/* */ -/* */ -/* Others: */ -/* */ -/* */ -/* */ -/* */ -/* */ -/* */ -/* */ -/* text (cell) */ -/* text (new cell) */ -/* */ -/* &CCenter &"-,Bold"Bold */ -/* */ -/* */ -/* */ -/* */ -/* Formula: */ -/* String must be within double quotes ("text""more") */ -/* */ -/* Sheet name: */ -/* sheet_name! */ -/* sheet_name:sheet_name! */ -/* [workbook_name]sheet_name:sheet_name! */ -/* 'sheet_name'! */ -/* 'sheet_name:sheet_name'! */ -/* '[workbook_name]sheet_name:sheet_name'! */ -/* */ -/*****************************************************************************/ - - - -/*****************************************************************************/ -/* PostExport */ -/* */ -/* Function called by EQFPOSTUNSEG2. */ -/* */ -/* Return: TRUE - File successfully processed. */ -/* FALSE - File could not be processed. */ -/*****************************************************************************/ - -BOOL PostExport( PSZ in, PSZ out, PSZ source, USHORT *usFileFormat ) -{ - - wofstream OutFile ( out ) ; - InputFile = new wifstream( source ) ; - InputFile2 = new wifstream( in ) ; - - P_INFO *ptrSrcParaHead ; - P_INFO *ptrTgtParaHead ; - P_INFO *ptrSrcParaCur ; - P_INFO *ptrTgtParaCur ; - - R_INFO *ptrSrcRunHead ; - R_INFO *ptrTgtRunHead ; - R_INFO *ptrSrcRunCur ; - R_INFO *ptrTgtRunCur ; - R_INFO *ptrRunTemp ; - - T_INFO *ptrSrcTextCur ; - R_INFO *ptrTextTemp ; - - SSNT_INFO *ptrSrcSSNTHead ; - SSNT_INFO *ptrTgtSSNTHead ; - SSNT_INFO *ptrSSNTCur ; - - DBCS_INFO *ptrDbcsHead ; - DBCS_INFO *ptrDbcsCur ; - - WCHAR szTemp[MAX_XML_RCD_LENGTH2*2] ; - WCHAR szValue[MAX_XML_RCD_LENGTH] ; - WCHAR szValue2[MAX_XML_RCD_LENGTH] ; - WCHAR szValue3[MAX_XML_RCD_LENGTH] ; - WCHAR szErrTitle[128] ; - WCHAR szErrText[512] ; - char szTempC1[512] ; - char szTempC2[512] ; - ULONG ulFromSrcFilePos ; - ULONG ulToSrcFilePos ; - ULONG ulFromTgtFilePos ; - ULONG ulToTgtFilePos ; - ULONG ulSeqNum ; - USHORT sRunConcat ; - USHORT sRunSrcNodes ; - USHORT sRunTgtNodes ; - USHORT usFixLevel = 0 ; - USHORT rc ; - BOOL bConcat ; - BOOL bPreserveBlanks ; - BOOL bReturn = TRUE; - - - bFileExport = TRUE ; - bSetCommonProperty = FALSE ; - bExportSSNTContent = FALSE ; - - - if ( ( ! stricmp( szDocTargetLanguage, "ARABIC" ) ) || - ( ! stricmp( szDocTargetLanguage, "HEBREW" ) ) || - ( ! stricmp( szDocSourceLanguage, "ARABIC" ) ) || - ( ! stricmp( szDocSourceLanguage, "HEBREW" ) ) ) - bBidi = TRUE ; - else - bBidi = FALSE ; - - if ( ( ! stricmp( szDocTargetLanguage, "CHINESE(SIMPL.)" ) ) || - ( ! stricmp( szDocTargetLanguage, "CHINESE(TRAD.)" ) ) || - ( ! stricmp( szDocTargetLanguage, "JAPANESE" ) ) || - ( ! stricmp( szDocTargetLanguage, "KOREAN" ) ) ) { - bDBCS = TRUE ; - } - - /*************************************************************************/ - /* Create a linked list which contains all of the required */ - /* information necessary for the translated file. */ - /*************************************************************************/ - if ( bReturn ) { - bReadSource = FALSE ; - bUTF16 = (*InputFile2).IsUTF16(); - rc = fnCreateInputList( &ptrTgtParaHead, &ptrTgtSSNTHead, 0, NULL, usFileFormat, &usFixLevel ) ; - if ( rc > 0 ) { - MessageBoxA(HWND_DESKTOP, MSG_MSOFC_PARSING_ERROR, TITLE_MSOFC_PARSING_ERROR, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - } - } - /*************************************************************************/ - /* Create a linked list which contains all of the required */ - /* information necessary for the source file. */ - /*************************************************************************/ - if ( bReturn ) { - bReadSource = TRUE ; - bUTF16 = (*InputFile).IsUTF16(); - rc = fnCreateInputList( &ptrSrcParaHead, &ptrSrcSSNTHead, 0, NULL, NULL, &usFixLevel ) ; - if ( rc > 0 ) { - MessageBoxA(HWND_DESKTOP, MSG_MSOFC_PARSING_ERROR, TITLE_MSOFC_PARSING_ERROR, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - } - } - - - /*************************************************************************/ - /* Create the exported file. */ - /*************************************************************************/ - if ( bReturn ) { - ulFromSrcFilePos = 0 ; - - /**********************************************************************/ - /* If file contains no translatable text, then simply copy entire */ - /* source file to export file. */ - /**********************************************************************/ - if ( ( ptrTgtParaHead ) && - ( ptrTgtParaHead->SeqNum == 0 ) && - ( ptrTgtParaHead->ptrNext == 0 ) ) { - ulFromSrcFilePos = 0 ; - ulToSrcFilePos = 0 ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - } - else { - - /**********************************************************************/ - /* */ - /* Process each paragraph node. */ - /* */ - /**********************************************************************/ - for( ptrSrcParaCur=ptrSrcParaHead, ptrTgtParaCur=ptrTgtParaHead ; - ptrSrcParaCur || ptrTgtParaCur ; - ptrSrcParaCur=(P_INFO*)ptrSrcParaCur->ptrNext, - ptrTgtParaCur=(P_INFO*)ptrTgtParaCur->ptrNext ) { - - /*******************************************************************/ - /* Handle reference nodes by finding the references source text */ - /* and writing out the associated target text. */ - /*******************************************************************/ - if ( ( ptrSrcParaCur->NodeType == NODE_TYPE_SHEET_REF ) || - ( ptrSrcParaCur->NodeType == NODE_TYPE_TEXT_REF ) ) { - for( ; ptrSrcParaCur && ( ptrSrcParaCur->NodeType==NODE_TYPE_SHEET_REF || - ptrSrcParaCur->NodeType==NODE_TYPE_TEXT_REF ) ; - ptrSrcParaCur=(P_INFO*)ptrSrcParaCur->ptrNext ) { - /*******************************************************************/ - /* Copy end of previous paragraph block to the start of this */ - /* paragraph block. (SOURCE COPY) */ - /*******************************************************************/ - ulToSrcFilePos = (ptrSrcParaCur->BlockStartPos) - 1*sizeof(WCHAR) ; - if ( ptrTgtParaCur ) - bPreserveBlanks = ptrTgtParaCur->PreserveBlanks ; - else - bPreserveBlanks = FALSE ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, - bPreserveBlanks ) ; - ulFromSrcFilePos = (ptrSrcParaCur->BlockEndPos) + 1*sizeof(WCHAR) ; - fnWriteReferenceText( ptrSrcParaCur, ptrSrcParaHead, ptrTgtParaHead, &OutFile ) ; - } - } - if ( ( ! ptrSrcParaCur ) || - ( ! ptrTgtParaCur ) ) { - break ; - } - - if ( ptrSrcParaCur->SeqNum != ptrTgtParaCur->SeqNum ) { - strcpy( szTempC1, MSG_MSOFC_PARAGRAPH_MISMATCH ) ; - sprintf( szTempC2, szTempC1, ptrTgtParaCur->SeqNum, ptrSrcParaCur->SeqNum ); - MessageBoxA(HWND_DESKTOP, szTempC2, TITLE_MSOFC_PARAGRAPH_MISMATCH, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - break ; - } - if ( ptrSrcParaCur->NumTextTags != ptrTgtParaCur->NumTextTags ) { - if ( bSetCommonProperty ) { - sRunSrcNodes = 0 ; - if ( ptrSrcParaCur->CommonProperty ) - ++sRunSrcNodes ; - for( ptrSrcRunCur=(R_INFO*)ptrSrcParaCur->ptrRunList ; - ptrSrcRunCur ; - ptrSrcRunCur=(R_INFO*)ptrSrcRunCur->ptrNext ) { - if ( ( ptrSrcRunCur->Concat != CONCAT_TEXT_YES ) && - ( ( ptrSrcRunCur->CommonProperty == 0 ) || - ( ptrSrcRunCur->CommonProperty == COMMONPROPERTY_FORCE_OFF ) ) ) { - if ( ( ptrSrcRunCur->NumTextTags == 0 ) && - ( ptrSrcRunCur->NumNeutralTags > 1 ) ) - sRunSrcNodes += ptrSrcRunCur->NumNeutralTags-1 ; - else - ++sRunSrcNodes ; - } - } - if ( ( ptrTgtParaCur->CommonProperty ) || - ( ptrTgtParaCur->ptrRunList == 0 ) ) - sRunTgtNodes = 1 ; - else - sRunTgtNodes = 0 ; - for( ptrTgtRunCur=(R_INFO*)ptrTgtParaCur->ptrRunList ; - ptrTgtRunCur ; - ptrTgtRunCur=(R_INFO*)ptrTgtRunCur->ptrNext ) { - if ( ( ptrTgtRunCur->Concat != CONCAT_TEXT_YES ) && - ( ptrTgtRunCur->CommonProperty == 0 ) ) - ++sRunTgtNodes ; - } - if ( sRunSrcNodes != sRunTgtNodes ) { - strcpy( szTempC1, MSG_MSOFC_TEXT_UNITS_MISMATCH ) ; - sprintf( szTempC2, szTempC1,ptrTgtParaCur->SeqNum, sRunTgtNodes, sRunSrcNodes ); - MessageBoxA(HWND_DESKTOP, szTempC2, TITLE_MSOFC_TEXT_UNITS_MISMATCH, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - break ; - } - } else { - for( ptrSrcRunCur=(R_INFO*)ptrSrcParaCur->ptrRunList, sRunConcat=0 ; - ptrSrcRunCur ; - ptrSrcRunCur=(R_INFO*)ptrSrcRunCur->ptrNext ) { - if ( ptrSrcRunCur->Concat == CONCAT_TEXT_YES ) - ++sRunConcat ; - } - if ( ptrSrcParaCur->NumTextTags - sRunConcat != ptrTgtParaCur->NumTextTags ) { - strcpy( szTempC1, MSG_MSOFC_TEXT_UNITS_MISMATCH ) ; - sprintf( szTempC2, szTempC1,ptrTgtParaCur->SeqNum, ptrTgtParaCur->NumTextTags, ptrSrcParaCur->NumTextTags ); - MessageBoxA(HWND_DESKTOP, szTempC2, TITLE_MSOFC_TEXT_UNITS_MISMATCH, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - bReturn = FALSE ; - break ; - } - } - } - - /*******************************************************************/ - /* If paragraph contains no translatable text, then skip it. */ - /*******************************************************************/ - if ( ptrSrcParaCur->NumTextTags == 0 ) - continue ; - - - /*******************************************************************/ - /* Copy end of previous paragraph block to the start of this */ - /* paragraph block. (SOURCE COPY) */ - /*******************************************************************/ - ulToSrcFilePos = (ptrSrcParaCur->BlockStartPos) - 1*sizeof(WCHAR) ; - if ( ptrTgtParaCur ) - bPreserveBlanks = ptrTgtParaCur->PreserveBlanks ; - else - bPreserveBlanks = FALSE ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, - bPreserveBlanks ) ; - - - /*******************************************************************/ - /* If this paragraph has no run nodes, then the text block is */ - /* defined in this paragraph node. (TARGET COPY) */ - /*******************************************************************/ - if ( ptrSrcParaCur->ptrRunList == 0 ) { - ulFromTgtFilePos = ptrTgtParaCur->BlockStartPos ; - ulToTgtFilePos = ptrTgtParaCur->BlockEndPos ; - fnWriteBlock( WRITE_TARGET, - ulFromTgtFilePos, - ulToTgtFilePos, - &OutFile, FALSE ) ; - ulFromSrcFilePos = ptrSrcParaCur->BlockEndPos + 1*sizeof(WCHAR) ; - - - continue ; - } - - - /*****************************************************************/ - /* Handle special case where paragraph is only text nodes */ - /* which are all concatenated together. */ - /*****************************************************************/ - if ( ( ptrSrcParaCur->NumTextTags > 1 ) && - ( ptrSrcParaCur->NumNeutralTags == 0 ) && - ( ptrTgtParaCur->NumTextTags == 1 ) && - ( ! ptrTgtParaCur->ptrRunList ) ) { - for( ptrSrcRunCur=(R_INFO*)ptrSrcParaCur->ptrRunList ; - ptrSrcRunCur ; - ptrSrcRunCur=(R_INFO*)ptrSrcRunCur->ptrNext ) { - if ( ( ptrSrcRunCur->ptrTextList ) || - ( ptrSrcRunCur->TagId != TAG_ID_TEXT ) || - ( ( ptrSrcRunCur != (R_INFO*)ptrSrcParaCur->ptrRunList ) && - ( ptrSrcRunCur->Concat != CONCAT_TEXT_YES ) ) ) - break ; - } - if ( ! ptrSrcRunCur ) { - ptrSrcRunCur=(R_INFO*)ptrSrcParaCur->ptrRunList ; - ulFromSrcFilePos = ptrSrcRunCur->StartPos ; - ulToSrcFilePos = ptrSrcRunCur->BlockStartPos - 1*sizeof(WCHAR) ; - if ( ptrTgtParaCur ) - bPreserveBlanks = ptrTgtParaCur->PreserveBlanks ; - else - bPreserveBlanks = FALSE ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, - bPreserveBlanks ) ; - - ulFromTgtFilePos = ptrTgtParaCur->BlockStartPos ; - ulToTgtFilePos = ptrTgtParaCur->BlockEndPos ; - fnWriteBlock( WRITE_TARGET, - ulFromTgtFilePos, - ulToTgtFilePos, - &OutFile, FALSE ) ; - for( ; ptrSrcRunCur ; - ptrRunTemp=ptrSrcRunCur, ptrSrcRunCur=(R_INFO*)ptrSrcRunCur->ptrNext ) ; - ulFromSrcFilePos = ptrRunTemp->BlockEndPos + 1*sizeof(WCHAR) ; - ulToSrcFilePos = ptrRunTemp->EndPos ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - - ulFromSrcFilePos = ptrSrcParaCur->BlockEndPos + 1*sizeof(WCHAR) ; - continue ; - } - } - - - /*****************************************************************/ - /* */ - /* Process each run node within this paragraph. */ - /* */ - /* For each target run node, find the corresponding source run */ - /* or text node. */ - /*****************************************************************/ - for( ptrTgtRunCur=(R_INFO*)ptrTgtParaCur->ptrRunList ; - ptrTgtRunCur ; - ptrTgtRunCur=(R_INFO*)ptrTgtRunCur->ptrNext ) { - - /**************************************************************/ - /* Loop through each source run node to find a match for */ - /* the current target run node. */ - /**************************************************************/ - for( ptrSrcRunCur=(R_INFO*)ptrSrcParaCur->ptrRunList ; - ptrSrcRunCur ; - ptrSrcRunCur=(R_INFO*)ptrSrcRunCur->ptrNext ) { - - if ( ( ptrTgtRunCur->TagType == TAG_TYPE_END ) && - ( ptrSrcRunCur->BeginSeqNum > 0 ) ) - ulSeqNum = ptrSrcRunCur->BeginSeqNum ; - else - ulSeqNum = ptrSrcRunCur->SeqNum ; - - - /***********************************************************/ - /* If this source run node has no text nodes, then see if */ - /* this node matches target node. */ - /***********************************************************/ - if ( ptrSrcRunCur->ptrTextList == 0 ) { - - if ( ulSeqNum != ptrTgtRunCur->SeqNum ) - continue; - - /********************************************************/ - /* If run node defines text, then write the pre-source */ - /* text, target text, post-source text. */ - /********************************************************/ - if ( ptrSrcRunCur->TagId == TAG_ID_TEXT ) { - ulFromSrcFilePos = ptrSrcRunCur->StartPos ; - ulToSrcFilePos = ptrSrcRunCur->BlockStartPos - 1*sizeof(WCHAR) ; - if ( ptrTgtRunCur ) - bPreserveBlanks = ptrTgtRunCur->PreserveBlanks ; - else - bPreserveBlanks = FALSE ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, - bPreserveBlanks ) ; - - ulFromTgtFilePos = ptrTgtRunCur->BlockStartPos ; - ulToTgtFilePos = ptrTgtRunCur->BlockEndPos ; - fnWriteBlock( WRITE_TARGET, - ulFromTgtFilePos, - ulToTgtFilePos, - &OutFile, FALSE ) ; - - if ( sRunConcat ) { - for( ptrRunTemp=(R_INFO*)ptrSrcRunCur->ptrNext,bConcat=FALSE ; - ptrRunTemp && ptrRunTemp->Concat==CONCAT_TEXT_YES; - ptrSrcRunCur=ptrRunTemp, ptrRunTemp=(R_INFO*)ptrRunTemp->ptrNext, bConcat=TRUE ) ; - - /* Concat text may end with empty tag. Adjust output */ - if ( ( bConcat ) && - ( ptrSrcRunCur->BlockEndPos < ptrSrcRunCur->BlockStartPos ) ) { - ulFromSrcFilePos = ptrSrcRunCur->BlockStartPos ; - ulToSrcFilePos = ulFromSrcFilePos + 1*sizeof(WCHAR); - fnReadBlock( WRITE_SOURCE, - ulFromSrcFilePos, ulToSrcFilePos, - szValue, sizeof(szValue) ) ; - if ( ! wcscmp( szValue, L"/>" ) ) { - ulFromSrcFilePos -= 4*sizeof(WCHAR) ; - fnReadBlock( WRITE_SOURCE, - ulFromSrcFilePos, ulToSrcFilePos, - szValue, sizeof(szValue) ) ; - if ( szValue[0] == L'<' ) { - ptrSrcRunCur->BlockEndPos += 2*sizeof(WCHAR) ; - wmemmove( &szValue[4], &szValue[5], wcslen(&szValue[5])+1 ) ; - wmemmove( &szValue[2], &szValue[1], wcslen(&szValue[1])+1 ) ; - szValue[1] = L'/' ; - OutFile << szValue ; - } - } - } - } - ulFromSrcFilePos = ptrSrcRunCur->BlockEndPos + 1*sizeof(WCHAR) ; - ulToSrcFilePos = ptrSrcRunCur->EndPos ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - } else { - - /*****************************************************/ - /* If run node defines neutral tag, then write the */ - /* source text. */ - /*****************************************************/ - ulFromSrcFilePos = ptrSrcRunCur->StartPos ; - ulToSrcFilePos = ptrSrcRunCur->EndPos ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - ulFromSrcFilePos = ptrSrcRunCur->EndPos + 1*sizeof(WCHAR) ; - - - } - - break ; - } /* End no source text nodes */ - - - if ( ulSeqNum == ptrTgtRunCur->SeqNum ) { - if ( ptrTgtRunCur->TagType == TAG_TYPE_END ) { - ulFromSrcFilePos = ptrSrcRunCur->BlockEndPos + 1*sizeof(WCHAR) ; - ulToSrcFilePos = ptrSrcRunCur->EndPos ; - } else { - ulFromSrcFilePos = ptrSrcRunCur->StartPos ; - ulToSrcFilePos = ptrSrcRunCur->BlockStartPos - 1*sizeof(WCHAR) ; - } - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - break ; - - } - - for( ptrSrcTextCur=(T_INFO*)ptrSrcRunCur->ptrTextList ; - ptrSrcTextCur ; - ptrSrcTextCur=(T_INFO*)ptrSrcTextCur->ptrNext ) { - - if ( ( ptrTgtRunCur->TagType == TAG_TYPE_END ) && - ( ptrSrcTextCur->BeginSeqNum > 0 ) ) - ulSeqNum = ptrSrcTextCur->BeginSeqNum ; - else - ulSeqNum = ptrSrcTextCur->SeqNum ; - - - - if ( ulSeqNum == ptrTgtRunCur->SeqNum ) { - - /*********************************************************/ - /* If text node defines text, then write the pre-source */ - /* text, target text, post-source text. */ - /*********************************************************/ - if ( ptrSrcTextCur->TagId == TAG_ID_TEXT ) { - ulFromSrcFilePos = ptrSrcTextCur->StartPos ; - ulToSrcFilePos = ptrSrcTextCur->BlockStartPos - 1*sizeof(WCHAR) ; - if ( ptrTgtRunCur ) - bPreserveBlanks = ptrTgtRunCur->PreserveBlanks ; - else - bPreserveBlanks = FALSE ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, - bPreserveBlanks ) ; - - ulFromTgtFilePos = ptrTgtRunCur->BlockStartPos ; - ulToTgtFilePos = ptrTgtRunCur->BlockEndPos ; - fnWriteBlock( WRITE_TARGET, - ulFromTgtFilePos, - ulToTgtFilePos, - &OutFile, FALSE ) ; - - ulFromSrcFilePos = ptrSrcTextCur->BlockEndPos + 1*sizeof(WCHAR) ; - ulToSrcFilePos = ptrSrcTextCur->EndPos ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - - } else { - - if ( ( ptrSrcRunCur->ptrTextList == ptrSrcTextCur ) && /* 9-22-08 */ - ( ptrSrcRunCur->StartPos > 0 ) && - ( ptrSrcRunCur->BlockStartPos == 0 ) && - ( ptrSrcRunCur->StartPos < ptrSrcTextCur->StartPos ) ) { - ulFromSrcFilePos = ptrSrcRunCur->StartPos ; - ulToSrcFilePos = ptrSrcTextCur->StartPos-1*sizeof(WCHAR) ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - } - - ulFromSrcFilePos = ptrSrcTextCur->StartPos ; - ulToSrcFilePos = ptrSrcTextCur->EndPos ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - - if ( ( ptrSrcTextCur->ptrNext == NULL ) && - ( ptrSrcRunCur->EndPos > 0 ) && - ( ptrSrcRunCur->BlockEndPos == 0 ) && - ( ptrSrcRunCur->EndPos > ptrSrcTextCur->EndPos ) ) { - ulFromSrcFilePos = ptrSrcTextCur->EndPos + 1*sizeof(WCHAR) ; - ulToSrcFilePos = ptrSrcRunCur->EndPos ; - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - ulFromSrcFilePos = ptrSrcRunCur->EndPos + 1*sizeof(WCHAR) ; - } else { - ulFromSrcFilePos = ptrSrcTextCur->EndPos + 1*sizeof(WCHAR) ; - } - - } - break ; - } - } /* End source text node loop */ - } /* End run node loop */ - - if ( ( ! ptrSrcRunCur ) && - ( ptrTgtRunCur->Tag ) && - ( ! wcscmp( ptrTgtRunCur->Tag, L"INSERT" ) ) ) { - OutFile << L"" ; - ulFromTgtFilePos = ptrTgtRunCur->StartPos ; - ulToTgtFilePos = ptrTgtRunCur->EndPos ; - fnWriteBlock( WRITE_TARGET, - ulFromTgtFilePos, - ulToTgtFilePos, - &OutFile, FALSE ) ; - OutFile << L"" ; - - } - } - ulFromSrcFilePos = ptrSrcParaCur->BlockEndPos + 1*sizeof(WCHAR) ; - } - - /*****************************************************************/ - /* Copy from end of last paragraph block to the end of the */ - /* file. (SOURCE COPY) */ - /*****************************************************************/ - ulToSrcFilePos = 0 ; /* Indicate to end of file */ - fnWriteBlock( WRITE_SOURCE, - ulFromSrcFilePos, - ulToSrcFilePos, - &OutFile, FALSE ) ; - } - } - - - /***********************************************************************/ - /* Free linked list space. */ - /***********************************************************************/ - fnFreeParaList( &ptrSrcParaHead ) ; - fnFreeParaList( &ptrTgtParaHead ) ; - - (*InputFile).freefile() ; - (*InputFile2).freefile() ; - OutFile.close() ; - - - /***********************************************************************/ - /* SpreadSheets. Determine if there are shared strings which */ - /* are defined as both translatable and non-translatable. If so, */ - /* then modify non-translatable instances to be hard-coded text */ - /* rather than using the shared string. */ - /***********************************************************************/ - if ( ptrSrcSSNTHead ) { - fnUpdateSpreadsheetNonTrans( out, in, ptrSrcSSNTHead ) ; - } - fnFreeSSNTList( &ptrSrcSSNTHead ) ; - fnFreeSSNTList( &ptrTgtSSNTHead ) ; - - return(bReturn); - -}/* PostExport */ - - - - - - - - - -/****************************************************************************/ -/* */ -/* fnCreateDbcsList */ -/* */ -/* Create a linked list containing the necessary information for exporting */ -/* this DBCS file. */ -/* */ -/* Input: None. */ -/* Output: ptrList - Pointer to start of list. */ -/* Return: 0 - List created successfully. */ -/* 1 - Failed to create the list. */ -/* */ -/****************************************************************************/ - -USHORT fnCreateDbcsList( DBCS_INFO** ptrList ) -{ - - DBCS_INFO *ptrDbcsHead = 0 ; - DBCS_INFO *ptrDbcsTail = 0 ; - - WCHAR szIn[MAX_XML_RCD_LENGTH2*2] ; - WCHAR szTag[XML_TAG_LEN] ; - WCHAR szTagText[MAX_XML_RCD_LENGTH2*2] ; - WCHAR *ptrChar ; - - ULONG ulFilePos = 0 ; - ULONG ulTagStartPos = 0 ; - ULONG ulTagEndPos = 0 ; - ULONG i, j ; - USHORT rc ; - USHORT usReturn = 0 ; - USHORT usDbcsState = DBCS_STATE_NONE ; - - BOOL bForever = TRUE ; - BOOL bNewNode = FALSE ; - BOOL bPartialTag = FALSE ; - - - szIn[0] = NULL ; - szTag[0] = NULL ; - i = 0 ; - - - /***********************************************************************/ - /* Read through the file to determine where the text is defined. */ - /***********************************************************************/ - while ( bForever ) { - - - /***********************************************************************/ - /* Get next tag from this line. */ - /***********************************************************************/ - rc = fnGetXMLTag( szIn, &i, &ulFilePos, szTag, szTagText, - &ulTagStartPos, &ulTagEndPos, &bPartialTag ) ; - if ( rc == 1 ) { - MessageBoxA(HWND_DESKTOP, MSG_MSOFC_PARSING_ERROR, TITLE_MSOFC_PARSING_ERROR, MB_OK | MB_DEFBUTTON1 | MB_ICONWARNING | MB_SYSTEMMODAL); - usReturn = 1 ; - break ; - } - if ( rc == 2 ) { /* End of file */ - break ; - } - - /*====================================================================*/ - /* */ - /* Handle start of DBCS-related sections. */ - /* */ - /*====================================================================*/ - if ( usDbcsState == DBCS_STATE_NONE ) { - - /*******************************************************************/ - /* Handle start of document fonts */ - /*******************************************************************/ - if ( ! wcscmp( szTag, L"w:fonts" ) ) { - - /* ---- Allocate a DBCS node. ------------------------------- */ - fnAllocateDbcsNode( &ptrDbcsHead, &ptrDbcsTail, szTag ) ; - - /* ---- Initialize the DBCS node. --------------------------- */ - usDbcsState = DBCS_STATE_FONTS ; /* Set DBCS state */ - ptrDbcsTail->NodeType = DBCS_STATE_FONTS ; - ptrDbcsTail->StartPos = ulTagEndPos + 1*sizeof(WCHAR); - - continue ; - } - - /*******************************************************************/ - /* Handle start of run block fonts */ - /*******************************************************************/ - if ( ! wcscmp( szTag, L"w:rFonts" ) ) { - - /* ---- Allocate a DBCS node. ------------------------------- */ - fnAllocateDbcsNode( &ptrDbcsHead, &ptrDbcsTail, szTag ) ; - - /* ---- Initialize the DBCS node. --------------------------- */ - ptrDbcsTail->NodeType = DBCS_STATE_RFONTS ; - ptrDbcsTail->StartPos = ulTagStartPos ; - - if ( wcschr( szTagText, L'/' ) ) { - usDbcsState = DBCS_STATE_NONE ; /* In empty tag */ - ptrDbcsTail->EndPos = ulTagEndPos ; - } else { - usDbcsState = DBCS_STATE_RFONTS ; /* Set DBCS state */ - } - - continue ; - } - - /*******************************************************************/ - /* Handle start of run block fonts */ - /*******************************************************************/ - if ( ! wcscmp( szTag, L"wx:font" ) ) { - - /* ---- Allocate a DBCS node. ------------------------------- */ - fnAllocateDbcsNode( &ptrDbcsHead, &ptrDbcsTail, szTag ) ; - - /* ---- Initialize the DBCS node. --------------------------- */ - ptrDbcsTail->NodeType = DBCS_STATE_XFONT ; - ptrDbcsTail->StartPos = ulTagStartPos ; - - if ( wcschr( szTagText, L'/' ) ) { - usDbcsState = DBCS_STATE_NONE ; /* In empty tag */ - ptrDbcsTail->EndPos = ulTagEndPos ; - } else { - usDbcsState = DBCS_STATE_XFONT ; /* Set DBCS state */ - } - - continue ; - } - - continue ; - } - - /**********************************************************************/ - /* Handle end of document fonts */ - /**********************************************************************/ - if ( usDbcsState == DBCS_STATE_FONTS ) { - - if ( ! wcscmp( szTag, L"/w:fonts" ) ) { - - /* ---- Initialize the DBCS node. --------------------------- */ - usDbcsState = DBCS_STATE_NONE ; /* Reset DBCS state */ - ptrDbcsTail->EndPos = ulTagStartPos - 1*sizeof(WCHAR) ; - } - - continue ; - } - - /**********************************************************************/ - /* Handle end of run fonts */ - /**********************************************************************/ - if ( usDbcsState == DBCS_STATE_RFONTS ) { - - if ( ! wcscmp( szTag, L"/w:rFonts" ) ) { - - /* ---- Initialize the DBCS node. --------------------------- */ - usDbcsState = DBCS_STATE_NONE ; /* Reset DBCS state */ - ptrDbcsTail->EndPos = ulTagEndPos ; - - } - - continue ; - } - - /**********************************************************************/ - /* Handle end of run fonts */ - /**********************************************************************/ - if ( usDbcsState == DBCS_STATE_XFONT ) { - - if ( ! wcscmp( szTag, L"/w:rFonts" ) ) { - - /* ---- Initialize the DBCS node. --------------------------- */ - usDbcsState = DBCS_STATE_NONE ; /* Reset DBCS state */ - ptrDbcsTail->EndPos = ulTagEndPos ; - - } - - continue ; - } - - - } /* End reading file for parsing */ - - *ptrList = ptrDbcsHead ; - - return( usReturn ) ; -} - - - - - -/****************************************************************************/ -/* */ -/* fnAllocateDbcsNode */ -/* */ -/* Allocate storage for a new DBCS node. */ -/* */ -/* Input: Tag - Tag defined by this node. */ -/* In/Out: ptrHead - The first node in the linked list. */ -/* ptrTail - The last node in the linked list. */ -/* Return: TRUE - Node allocated. */ -/* FALSE - Allocation failure */ -/* */ -/****************************************************************************/ - -BOOL fnAllocateDbcsNode( DBCS_INFO** ptrHead, DBCS_INFO** ptrTail, - WCHAR *Tag ) -{ - DBCS_INFO *ptrNew ; - BOOL bReturn = FALSE ; - - ptrNew = (DBCS_INFO*)calloc( sizeof(DBCS_INFO), 1 ) ; - if ( ptrNew ) { - bReturn = TRUE ; - if ( *ptrHead ) { - (*ptrTail)->ptrNext = ptrNew ; - ptrNew->ptrPrev = *ptrTail ; - } else { - *ptrHead = ptrNew ; - } - *ptrTail = ptrNew ; - } - - if ( Tag[0] != NULL ) { - (*ptrTail)->Tag = (WCHAR*)malloc( (wcslen(Tag)+1) * sizeof(WCHAR) ) ; - wcscpy( (*ptrTail)->Tag, Tag ) ; - } - - return( bReturn ) ; -} - - - - -/****************************************************************************/ -/* */ -/* fnFreeDbcsList */ -/* */ -/* Free all of the storage used in the DBCS list. */ -/* */ -/****************************************************************************/ - -VOID fnFreeDbcsList( DBCS_INFO** ptrList ) -{ - DBCS_INFO *ptrDbcsHead ; - DBCS_INFO *ptrDbcsCur ; - - - - /***********************************************************************/ - /* Free linked list space. */ - /***********************************************************************/ - ptrDbcsHead = *ptrList ; - - while ( ptrDbcsHead ) { - - if ( ptrDbcsHead->Tag ) - free( ptrDbcsHead->Tag ) ; - - ptrDbcsCur = (DBCS_INFO*)ptrDbcsHead->ptrNext ; - free( ptrDbcsHead ) ; - ptrDbcsHead = ptrDbcsCur ; - } - - *ptrList = 0; -} - - - - -/****************************************************************************/ -/* */ -/* fnGetDbcsBlock */ -/* */ -/* Get a blockof DBCS text to process. */ -/* */ -/* Input: StartPos - Starting position to read from. */ -/* EndPos - Ending position to read from. */ -/* Output: Block - DBCS text. */ -/* Return: 0 - Block read OK. */ -/* 1 - Failure. */ -/* */ -/****************************************************************************/ - -USHORT fnGetDbcsBlock( ULONG StartPos, ULONG EndPos, WCHAR *szBlock ) -{ - WCHAR szIn[MAX_XML_RCD_LENGTH2*2] ; - ULONG ulBlockLen ; - ULONG ulReadLen ; - ULONG ulFilePos ; - ULONG i ; - USHORT usReturn = 0 ; - BOOL bReadEOF = FALSE ; - - szBlock[0] = NULL ; - if ( EndPos == 0 ) { - bReadEOF = TRUE ; - } else { - if ( StartPos > EndPos ) { - usReturn = 1 ; - ulBlockLen = 0 ; - } else { - ulBlockLen = ( EndPos - StartPos ) / sizeof(WCHAR) + 1 ; - } - } - (*InputFile).fseekt(StartPos, std::ios::beg) ; - - ulReadLen = ulBlockLen ; - szPrevXMLInputText[0] = NULL ; - - while( ( usReturn == 0 ) && - ( ( ulReadLen > 0 ) || - ( bReadEOF ) ) ) { - if ( fnGetXMLRcd( szIn, &ulFilePos ) ) { - wcscat( szBlock, szIn ) ; - i = wcslen( szBlock ) ; - if ( ( i > ulReadLen ) && - ( ! bReadEOF ) ) { - i = ulReadLen ; - szBlock[i] = NULL ; - } - ulReadLen -= i ; - } else { - usReturn = 1 ; - } - } - - return( usReturn ) ; -} - - - - -/****************************************************************************/ -/* */ -/* fnAddTagAttr */ -/* */ -/* Add/replace a tag attribute. */ -/* */ -/* Input: szTag - Tag text to be changed. */ -/* szAttr - Attribute value to be added or changed. */ -/* szValue - Attribute value to be set. */ -/* usCond - 0= Add attr or change existing value. */ -/* 1= Only change existing value. */ -/* 2= Only add attr if it does not already exist. */ -/* Output: szTag - Updated tag text. */ -/* Return: n/a */ -/* */ -/****************************************************************************/ - -VOID fnAddTagAttr( WCHAR *szTag, WCHAR *szAttr, WCHAR *szValue, USHORT usCond ) -{ - WCHAR szScanAttr[80] ; - WCHAR AttrQuote ; - WCHAR *ptrAttr ; - WCHAR *ptr, *ptr2 ; - USHORT usValueLen ; - BOOL bDone = 0 ; - - szScanAttr[0] = L' ' ; - wcscpy( &szScanAttr[1], szAttr ) ; - usValueLen = wcslen( szValue ) ; - - /***********************************************************************/ - /* Update attribute value if it already exists. */ - /***********************************************************************/ - for( ptrAttr=wcsstr(szTag,szScanAttr ) ; - ptrAttr ; - ptrAttr=wcsstr(ptrAttr+1,szScanAttr ) ) { - ptr = ptrAttr ; - for( ptr+=wcslen(szScanAttr) ; *ptr && iswspace(*ptr) ; ++ptr ) ; - if ( *ptr == L'=' ) { - ++ptr ; - for( ptr2=0 ; *ptr && iswspace(*ptr) ; ++ptr ) ; - if ( ( *ptr == L'\'' ) || - ( *ptr == L'\"' ) ) { - AttrQuote = *ptr ; - ptr2 = ptr + 1; - for( ; *ptr2 && (*ptr2!=AttrQuote) ; ++ptr2 ) ; - } - if ( ( ptr2 ) && - ( ( usCond == COND_ADDATTR_ALWAYS ) || - ( usCond == COND_ADDATTR_CHGONLY ) ) ) { - if ( ( ptr2-ptr-1) != usValueLen ) - wmemmove( ptr+usValueLen+1, ptr2 , wcslen(ptr2)+1 ) ; - wcsncpy( ptr+1, szValue, usValueLen ) ; - bDone = TRUE ; - } - } - } - - /***********************************************************************/ - /* Add attribute when it does not already exist. */ - /***********************************************************************/ - if ( ( ! bDone ) && - ( ( usCond == COND_ADDATTR_ALWAYS ) || - ( usCond == COND_ADDATTR_NEWONLY ) ) ) { - ptr = szTag + wcslen(szTag)- 1 ; - if ( *(ptr-1) == L'/' ) - --ptr ; - wmemmove( ptr+wcslen(szAttr)+usValueLen+4, ptr, wcslen(ptr)+1 ) ; - *(ptr++) = L' ' ; - wcsncpy( ptr, szAttr, wcslen(szAttr) ) ; - ptr += wcslen(szAttr) ; - *(ptr++) = L'=' ; - *(ptr++) = L'\"' ; - wcsncpy( ptr, szValue, usValueLen ) ; - ptr += wcslen(szValue) ; - *(ptr) = L'\"' ; - } - return ; -} - - - - -/****************************************************************************/ -/* */ -/* fnRemoveTagAttr */ -/* */ -/* Remove a tag attribute. */ -/* */ -/* Input: szTag - Tag text to be changed. */ -/* szAttr - Attribute value to be added or changed. */ -/* Output: szTag - Updated tag text. */ -/* Return: n/a */ -/* */ -/****************************************************************************/ - -VOID fnRemoveTagAttr( WCHAR *szTag, WCHAR *szAttr ) -{ - WCHAR szScanAttr[80] ; - WCHAR AttrQuote ; - WCHAR *ptrAttr ; - WCHAR *ptr, *ptr2 ; - - szScanAttr[0] = L' ' ; - wcscpy( &szScanAttr[1], szAttr ) ; - - /***********************************************************************/ - /* Remove attribute, if it already exists. */ - /***********************************************************************/ - for( ptrAttr=wcsstr(szTag,szScanAttr ) ; - ptrAttr ; - ptrAttr=wcsstr(ptrAttr+1,szScanAttr ) ) { - ptr = ptrAttr ; - for( ptr+=wcslen(szScanAttr) ; *ptr && iswspace(*ptr) ; ++ptr ) ; - if ( *ptr == L'=' ) { - ++ptr ; - for( ptr2=0 ; *ptr && iswspace(*ptr) ; ++ptr ) ; - if ( ( *ptr == L'\'' ) || - ( *ptr == L'\"' ) ) { - AttrQuote = *ptr ; - ptr2 = ptr + 1; - for( ; *ptr2 && (*ptr2!=AttrQuote) ; ++ptr2 ) ; - } - if ( ptr2 ) { - for( ptr=ptrAttr-1 ; ptr>szTag && iswspace(*ptr) ; --ptr ) ; - wmemmove( ptr+1, ptr2+1, wcslen(ptr2+1)+1 ) ; - } - } - } - return ; -} - -/*****************************************************************************/ -/* */ -/* fnWriteReferenceText */ -/* */ -/* Find and write translated reference text. */ -/* */ -/* Input: ptrRefCur - Ptr to current source reference node. */ -/* ptrSrcHead - Head of source paragraph linked list. */ -/* ptrTgtHead - Head of target paragraph linked list. */ -/* Output: - Write target reference text. */ -/* Return: n/a */ -/*****************************************************************************/ - -BOOL fnWriteReferenceText( P_INFO *ptrRefCur, P_INFO *ptrSrcHead, - P_INFO *ptrTgtHead, wofstream *OutFile ) -{ - - P_INFO *ptrSrcCur ; - P_INFO *ptrTgtCur ; - - WCHAR szRefValue[MAX_XML_RCD_LENGTH] ; - WCHAR szValue[MAX_XML_RCD_LENGTH] ; - ULONG ulFromFilePos ; - ULONG ulToFilePos ; - ULONG ulRefLen ; - ULONG ulRefBytes ; - ULONG ulRefId ; - USHORT rc ; - BOOL bSheetReference = FALSE ; - BOOL bMatch = FALSE ; - BOOL bQuoted = FALSE ; - BOOL bReturn = TRUE; - - - ulFromFilePos = ptrRefCur->BlockStartPos ; - ulToFilePos = ptrRefCur->BlockEndPos ; - fnReadBlock( WRITE_SOURCE, - ulFromFilePos, ulToFilePos+1*sizeof(WCHAR), - szRefValue, sizeof(szRefValue) ) ; - ulRefLen = wcslen(szRefValue) ; - if ( ulRefLen ) { - if ( szRefValue[ulRefLen-1] == L'!' ) - bSheetReference = TRUE ; - szRefValue[ulRefLen-1] = NULL ; - ulRefLen = wcslen(szRefValue) ; - } - if ( ( ulRefLen > 2 ) && - ( szRefValue[0] == L'\'' ) && - ( szRefValue[ulRefLen-1] == L'\'' ) ) { - bQuoted = TRUE ; - szRefValue[ulRefLen-1] = NULL ; - wmemmove( szRefValue, &szRefValue[1], ulRefLen ) ; - } - ulRefLen = wcslen(szRefValue) ; - ulRefBytes = ulRefLen * sizeof(WCHAR) ; - szValue[0] = NULL ; - - /**********************************************************************/ - /* */ - /* Process each paragraph node to find matching text. */ - /* */ - /**********************************************************************/ - for( ptrSrcCur=ptrSrcHead ; - ptrSrcCur ; - ptrSrcCur=(P_INFO*)ptrSrcCur->ptrNext ) { - if ( ( ptrSrcCur->NodeType == NODE_TYPE_FILE_NAME ) || - ( ptrSrcCur->NodeType == NODE_TYPE_SHEET_REF ) || - ( ptrSrcCur->NodeType == NODE_TYPE_TEXT_REF ) ) - continue ; - if ( ( ptrRefCur->NodeType == NODE_TYPE_SHEET_REF ) && - ( wcscmp( ptrSrcCur->Tag, L"sheet" ) ) ) - continue ; - if ( ptrSrcCur->BlockEndPos-ptrSrcCur->BlockStartPos+1*sizeof(WCHAR) == ulRefBytes ) { - ulFromFilePos = ptrSrcCur->BlockStartPos ; - ulToFilePos = ptrSrcCur->BlockEndPos ; - fnReadBlock( WRITE_SOURCE, - ulFromFilePos, ulToFilePos, - szValue, sizeof(szValue) ) ; - if ( ! wcscmp( szValue, szRefValue ) ) { - bMatch = TRUE ; - break ; - } - szValue[0] = NULL ; - } - } - if ( ptrSrcCur ) { - ulRefId = ptrSrcCur->SeqNum ; - for( ptrTgtCur=ptrTgtHead ; - ptrTgtCur && ptrTgtCur->SeqNum != ulRefId ; - ptrTgtCur=(P_INFO*)ptrTgtCur->ptrNext ) ; - if ( ptrTgtCur ) { - ulFromFilePos = ptrTgtCur->BlockStartPos ; - ulToFilePos = ptrTgtCur->BlockEndPos ; - fnReadBlock( WRITE_TARGET, - ulFromFilePos, ulToFilePos, - szValue, sizeof(szValue) ) ; - } - } - - if ( wcslen(szValue) >= 1 ) - wcscpy( szRefValue, szValue ) ; - - if ( ( ( !wcscmp( ptrRefCur->Tag, L"c:f" ) ) || - ( !wcscmp( ptrRefCur->Tag, L"f" ) ) || - ( !wcscmp( ptrRefCur->Tag, L"definedName" ) ) ) && - ( bSheetReference ) && - ( bMatch || bQuoted ) ) { - wmemmove( &szRefValue[1], szRefValue, wcslen(szRefValue)+1 ) ; - szRefValue[0] = L'\'' ; - wcscat( szRefValue, L"\'" ) ; - } - - *OutFile << szRefValue ; - - return(bReturn); - -} - - - - - -/****************************************************************************/ -/* */ -/* fnUpdateSpreadsheetNonTrans */ -/* */ -/* If there is a shared string which is defined both as translatable and */ -/* non-translatable, then change the non-translatable references so that */ -/* the text is defined directly in the worksheet rather than using the */ -/* shared string value. */ -/* */ -/* For example, if the following cell references the shared string "option" */ -/* and it can be both translatable and non-translatable: */ -/* */ -/* 1261 */ -/* */ -/* Then if this is translatable cell, then change nothing. */ -/* Then if this is a non-translatable cell, then change this to: */ -/* */ -/* option */ -/* */ -/* */ -/****************************************************************************/ - -BOOL fnUpdateSpreadsheetNonTrans( PSZ In, PSZ Work, SSNT_INFO* ptrCells ) -{ -#define SSNT_STATE_NONE 0 -#define SSNT_STATE_WORKSHEET 1 -#define SSNT_STATE_COLS 2 -#define SSNT_STATE_ROW 3 -#define SSNT_STATE_CELL 4 -#define MAX_COLS 200 - - SSNT_INFO *ptrStylesHead = 0 ; - SSNT_INFO *ptrStylesCur = 0 ; - - SSNT_INFO *ptrStyleXfsHead = 0 ; - SSNT_INFO *ptrStyleXfsCur = 0 ; - - SSNT_INFO *ptrNTCellHead = 0 ; - SSNT_INFO *ptrNTCellPrev = 0 ; - SSNT_INFO *ptrTRCellHead = 0 ; - SSNT_INFO *ptrTRCellPrev = 0 ; - SSNT_INFO *ptrCellHead = 0 ; - SSNT_INFO *ptrCellPrev = 0 ; - SSNT_INFO *ptrCellCur = 0 ; - SSNT_INFO *ptrCellNew = 0 ; - - wofstream TempFile ( Work ) ; -// wofstream *TempFile ; - - WCHAR szIn[MAX_XML_RCD_LENGTH2*2] ; - WCHAR szTag[XML_TAG_LEN] ; - WCHAR szTagText[MAX_XML_RCD_LENGTH2*2] ; - WCHAR szValue[MAX_XML_RCD_LENGTH] ; - WCHAR *ptrChar, *ptrChar2 ; - ULONG ulTagStartPos = 0 ; - ULONG ulTagEndPos = 0 ; - ULONG ulFilePos = 0 ; - ULONG ulColMin, ulColMax ; - ULONG ulTemp ; - ULONG i, k ; - LONG lStyleCount ; - USHORT usColsNonTrans[MAX_COLS] = { 0 } ; - USHORT usState ; - USHORT usCol ; - USHORT rc ; - BOOL bPartialTag = FALSE ; - BOOL bInStyles = FALSE ; - BOOL bInXfs = FALSE ; - BOOL bInWorksheet = FALSE ; - BOOL bInCols = FALSE ; - BOOL bInRow = FALSE ; - BOOL bInCell = FALSE ; - BOOL bTransCell = FALSE ; - BOOL bSkipSSRef = FALSE ; - BOOL bForever = TRUE ; - BOOL bReturn = TRUE ; - - ptrNTCellHead = ptrCells ; - - /***********************************************************************/ - /* Determine if there are any strings defined as both translatable */ - /* and non-translatable. If not, do nothing. */ - /***********************************************************************/ - for( ptrCellCur=ptrNTCellHead ; ptrCellCur ; ptrCellCur=(SSNT_INFO*)ptrCellCur->ptrNext ) { - if ( ( ptrCellCur->bTrans ) && - ( ptrCellCur->szString ) ) - break; - } - - if ( ! ptrCellCur ) { - return( 0 ) ; - } - - InputFile = new wifstream( In ) ; - - - /***********************************************************************/ - /* Find the element(s) which define "CN INTERNAL" styles. */ - /* */ - /* */ - /* */ - /* */ - /* */ - /***********************************************************************/ - bInStyles = FALSE ; - szIn[0] = NULL ; - szTag[0] = NULL ; - szPrevXMLInputText[0] = NULL ; - ulFilePos = 0 ; - i = 0 ; - while ( bReturn ) { - rc = fnGetXMLTag( szIn, &i, &ulFilePos, szTag, szTagText, - &ulTagStartPos, &ulTagEndPos, &bPartialTag ) ; - if ( rc == 1 ) { - bReturn = FALSE ; - } - if ( rc == 2 ) { /* End of file */ - break; - } - - if ( ! wcscmp( szTag, L"cellStyles" ) ) { - bInStyles = TRUE ; - } - - if ( bInStyles ) { - if ( ( ! wcscmp( szTag, L"cellStyle" ) ) && - ( ( wcsstr( szTagText, L"CNInternal" ) ) || - ( wcsstr( szTagText, L"CN Internal" ) ) || - ( wcsstr( szTagText, L"DNT" ) ) || - ( wcsstr( szTagText, L"dnt" ) ) ) ) { - if ( ( fnGetAttributeValue( szTagText, L"xfId", szValue ) ) && - ( iswdigit( szValue[0] ) ) ) { - ptrStylesCur = (SSNT_INFO*)calloc( sizeof(SSNT_INFO), 1 ) ; - ptrStylesCur->ptrNext = ptrStylesHead ; - ptrStylesCur->ulData = _wtoi( szValue ) ; - ptrStylesHead = ptrStylesCur ; - } - } - if ( ! wcscmp( szTag, L"/cellStyles" ) ) { - bInStyles = FALSE ; - if ( ! ptrStylesHead ) - bReturn = FALSE ; - break ; - } - } - } - - (*InputFile).fseekt(0, std::ios::beg) ; - - - /***********************************************************************/ - /* Using the "xfId"s from the previously found elements, */ - /* find the element(s) which have the same "xfID"s. */ - /* The zero-based index into is the style number which is */ - /* referenced in the "s" attribute of the element. */ - /* */ - /* */ - /* */ - /* */ - /* */ - /***********************************************************************/ - bInXfs = FALSE ; - bPartialTag = FALSE ; - szIn[0] = NULL ; - szTag[0] = NULL ; - szPrevXMLInputText[0] = NULL ; - ulFilePos = 0 ; - lStyleCount = 0 ; - i = 0 ; - while ( bReturn ) { - rc = fnGetXMLTag( szIn, &i, &ulFilePos, szTag, szTagText, - &ulTagStartPos, &ulTagEndPos, &bPartialTag ) ; - if ( rc == 1 ) { - bReturn = FALSE ; - } - if ( rc == 2 ) { /* End of file */ - break; - } - - if ( ! wcscmp( szTag, L"cellXfs" ) ) { - bInXfs = TRUE ; - lStyleCount = -1 ; /* Zero-based index */ - } - - if ( bInXfs ) { - if ( ! wcscmp( szTag, L"xf" ) ) { - ++lStyleCount ; - if ( ( fnGetAttributeValue( szTagText, L"xfId", szValue ) ) && - ( iswdigit( szValue[0] ) ) ) { - ulTemp = _wtoi( szValue ); - for( ptrStylesCur=ptrStylesHead ; - ptrStylesCur && ( ulTemp != ptrStylesCur->ulData ); - ptrStylesCur=(SSNT_INFO*)ptrStylesCur->ptrNext ) { - } - if ( ptrStylesCur ) { - ptrStyleXfsCur = (SSNT_INFO*)calloc( sizeof(SSNT_INFO), 1 ) ; - ptrStyleXfsCur->ptrNext = ptrStyleXfsHead ; - ptrStyleXfsCur->ulData = lStyleCount ; - ptrStyleXfsHead = ptrStyleXfsCur ; - } - } - } - if ( ! wcscmp( szTag, L"/cellXfs" ) ) { - bInXfs = FALSE ; - if ( ! ptrStyleXfsHead ) - bReturn = FALSE ; - break ; - } - } - } - - (*InputFile).fseekt(0, std::ios::beg) ; - - - /***********************************************************************/ - /* Find individual cells which use the non-translatable styles. */ - /* Save a list of SharedString IDs for later processing. */ - /* */ - /* */ - /* */ - /* */ - /* */ - /* */ - /* 27 */ - /* */ - /* */ - /***********************************************************************/ - - usState = SSNT_STATE_NONE ; - bExportSSNTContent = TRUE ; - bSkipSSRef = FALSE ; - bPartialTag = FALSE ; - szIn[0] = NULL ; - szTag[0] = NULL ; - szPrevXMLInputText[0] = NULL ; - ulFilePos = 0 ; - i = 0 ; - while ( bReturn ) { - if ( i ) { - wcsncpy( szTagText, szIn, i ) ; - szTagText[i] = NULL ; - TempFile << szTagText ; - wmemmove( szIn, &szIn[i], wcslen(&szIn[i])+1 ) ; - i = 0 ; - } - rc = fnGetXMLTag( szIn, &i, &ulFilePos, szTag, szTagText, - &ulTagStartPos, &ulTagEndPos, &bPartialTag ) ; - if ( rc == 1 ) { - bReturn = FALSE ; - } - if ( rc == 2 ) { /* End of file */ - if ( i ) { - wcsncpy( szTagText, szIn, i ) ; - szTagText[i] = NULL ; - TempFile << szTagText ; - } - break; - } - if ( bSkipSSRef ) { - wmemmove( szIn, &szIn[i], wcslen(&szIn[i])+1 ) ; - i = 0 ; - if ( ! wcscmp( szTag, L"/v" ) ) - bSkipSSRef = FALSE ; - continue ; - - } - - if ( ! wcscmp( szTag, L"worksheet" ) ) { - usState = SSNT_STATE_WORKSHEET ; - usCol = 0 ; - memset( usColsNonTrans, 0, sizeof(usColsNonTrans) ) ; - } - - if ( usState == SSNT_STATE_WORKSHEET ) { - if ( ! wcscmp( szTag, L"/worksheet" ) ) { - usState = SSNT_STATE_NONE ; - } - - if ( ( ! wcscmp( szTag, L"cols" ) ) && - ( ! fnIsEmptyTag( szTagText ) ) ) { /* Not an empty tag */ - usState = SSNT_STATE_COLS ; - } - - if ( ( ! wcscmp( szTag, L"row" ) ) && - ( ! fnIsEmptyTag( szTagText ) ) ) { /* Not an empty tag */ - usState = SSNT_STATE_ROW ; - usCol = 0 ; - } - } - - if ( usState == SSNT_STATE_COLS ) { - if ( ! wcscmp( szTag, L"/cols" ) ) { - usState = SSNT_STATE_WORKSHEET ; - } - - if ( ! wcscmp( szTag, L"col" ) ) { - ++usCol ; - usColsNonTrans[usCol] = 0 ; - ulColMin = 0 ; - ulColMax = 0 ; - if ( ( fnGetAttributeValue( szTagText, L"min", szValue ) ) && - ( iswdigit( szValue[0] ) ) ) { - ulColMin = _wtoi( szValue ); - if ( ( fnGetAttributeValue( szTagText, L"max", szValue ) ) && - ( iswdigit( szValue[0] ) ) ) { - ulColMax = _wtoi( szValue ); - if ( ulColMax >= MAX_COLS ) - ulColMax = MAX_COLS - 1 ; - } - if ( ( ulColMin > 0 ) && - ( ulColMin < ulColMax ) ) { - for( k=ulColMax-ulColMin ; k>0 ; usColsNonTrans[usCol+k]=0, --k ) ; - } - } - if ( ( fnGetAttributeValue( szTagText, L"style", szValue ) ) && - ( iswdigit( szValue[0] ) ) ) { - ulTemp = _wtoi( szValue ); - for( ptrStyleXfsCur=ptrStyleXfsHead ; - ptrStyleXfsCur && ( ulTemp != ptrStyleXfsCur->ulData ); - ptrStyleXfsCur=(SSNT_INFO*)ptrStyleXfsCur->ptrNext ) { - } - if ( ptrStyleXfsCur ) { - usColsNonTrans[usCol] = 1 ; - if ( ( ulColMin > 0 ) && - ( ulColMin < ulColMax ) ) { - for( k=ulColMax-ulColMin ; k>0 ; usColsNonTrans[usCol+k]=1, --k ) ; - usCol = ulColMax ; - } - } - } - if ( ulColMax > usCol ) - usCol = ulColMax ; - } - } - - if ( usState == SSNT_STATE_ROW ) { - if ( ! wcscmp( szTag, L"/row" ) ) { - usState = SSNT_STATE_WORKSHEET ; - } - - if ( ( ! wcscmp( szTag, L"c" ) ) && - ( ! fnIsEmptyTag( szTagText ) ) ) { /* Not an empty tag */ - usState = SSNT_STATE_CELL ; - bTransCell = TRUE ; - ++usCol ; -// Column setting is not used as default for its cells. -// Each cell has its own style. If no "s" attribute, then default s="0". - if ( ( fnGetAttributeValue( szTagText, L"t", szValue ) ) && - ( ! wcscmp( szValue,L"s" ) ) ) { - if ( ( fnGetAttributeValue( szTagText, L"s", szValue ) ) && - ( iswdigit( szValue[0] ) ) ) - ulTemp = _wtoi( szValue ); - else - ulTemp = 0 ; - for( ptrStyleXfsCur=ptrStyleXfsHead ; - ptrStyleXfsCur && ( ulTemp != ptrStyleXfsCur->ulData ); - ptrStyleXfsCur=(SSNT_INFO*)ptrStyleXfsCur->ptrNext ) { - } - if ( ptrStyleXfsCur ) { - bTransCell = FALSE ; /* Non-trans cell */ - } - } else { - if ( usColsNonTrans[usCol] ) { /* Column is non-trans */ - bTransCell = FALSE ; /* Non-trans cell */ - } - } - if ( ! bTransCell ) { /* Look ahead for value */ - fnGetCompleteString( &szIn[i], L"/", szValue, sizeof(szValue)/sizeof(WCHAR) ) ; - ptrChar = wcsstr( szValue, L"" ) ; - if ( ptrChar ) { - ptrChar += 3 ; - ptrChar2 = wcsstr( ptrChar, L"<" ) ; - if ( ptrChar2 ) { - *ptrChar2 = NULL ; - for( ptrChar2=ptrChar ; *ptrChar2 && iswdigit(*ptrChar2) ; ++ptrChar2 ) ; - *ptrChar2 = NULL ; - ulTemp = _wtoi( ptrChar ); - ptrCellHead = ptrNTCellHead ; - for( ptrCellCur=ptrCellHead, ptrCellPrev=0 ; - ptrCellCur && ( ulTemp > ptrCellCur->ulData ); - ptrCellPrev=ptrCellCur, ptrCellCur=(SSNT_INFO*)ptrCellCur->ptrNext ) { - } - if ( ( ptrCellCur ) && - ( ulTemp == ptrCellCur->ulData ) && - ( ptrCellCur->szString ) ) { - for( ptrChar=&szIn[i-1] ; ptrChar>szIn && wcsncmp( ptrChar, L" t=", 3 ) ; --ptrChar ) ; - if ( ! wcsncmp( ptrChar, L" t=\"", 4 ) ) { - bSkipSSRef = TRUE ; - ptrChar += 4 ; - for( ptrChar2=ptrChar ; *ptrChar2 && *ptrChar2!=L'\"' ; ptrChar2++ ); - wmemmove( ptrChar+9, ptrChar2, wcslen(ptrChar2)+1 ) ; - wcsncpy( ptrChar, L"inlineStr", 9 ) ; - i += 9 - (ptrChar2 - ptrChar) ; - wcsncpy( szTagText, szIn, i ) ; - szTagText[i] = NULL ; - TempFile << szTagText ; - TempFile << L"" ; - TempFile << ptrCellCur->szString ; - TempFile << L"" ; - wmemmove( szIn, &szIn[i], wcslen(&szIn[i])+1 ) ; - i = 0 ; - } - - } - } - } - } - } - } - - if ( usState == SSNT_STATE_CELL ) { - if ( ! wcscmp( szTag, L"/c" ) ) { - usState = SSNT_STATE_ROW ; - } - } - } - - - bExportSSNTContent = FALSE ; - - (*InputFile).fseekt(0, std::ios::beg) ; - - - /***********************************************************************/ - /* Determine which strings are defined as both translatable and */ - /* non-translatable. */ - /***********************************************************************/ - ptrCellCur = ptrTRCellHead ; - while ( ptrCellCur ) { - for( ptrCellPrev = ptrNTCellHead ; - ptrCellPrev && (ptrCellPrev->ulData < ptrCellCur->ulData) ; - ptrCellPrev = (SSNT_INFO*)ptrCellPrev->ptrNext ) ; - if ( ( ptrCellPrev ) && - ( ptrCellPrev->ulData == ptrCellCur->ulData ) ) - ptrCellPrev->bTrans = TRUE ; - ptrCellCur = (SSNT_INFO*)ptrCellCur->ptrNext ; - } - - - /***********************************************************************/ - /* Free allocated storage. */ - /***********************************************************************/ - - fnFreeSSNTList( &ptrStylesHead ) ; - fnFreeSSNTList( &ptrStyleXfsHead ) ; - fnFreeSSNTList( &ptrTRCellHead ) ; - - - - (*InputFile).freefile() ; - TempFile.close() ; - CopyFileA(Work, In, FALSE); - - return( bReturn ) ; -} diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c index 8f6c4ef2..506f1f28 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c @@ -123,288 +123,3 @@ extern short sTPVersion ; /* From USRCALLS.C */ #define TITLE_MSOFC_TEXT_UNITS_MISMATCH "Text Units Mismatch" #define TITLE_MSOFC_FILE_CONVERSION "File Conversion Error" -/*****************************************************************************/ -/* ReplaceXmlInZip */ -/* */ -/* Replace the XML files into the ZIP file after translation. */ -/* */ -/* Input: TargetFile - Input CONTENT.XML. Output updated ZIP file. */ -/* TempFile - Temporary work file to use. */ -/* ErrText - Any error text to be shown to user. */ -/* */ -/* */ -/* Return: TRUE - File successfully processed. */ -/* FALSE - File could not be processed. */ -/*****************************************************************************/ - -BOOL ReplaceXmlInZip( char *TargetFile, char *TempFile, char *ErrText ) -{ - STARTUPINFOA StartupInfo ; - PROCESS_INFORMATION piProcessInfo ; - FILE *fTemp ; - char szSourceFile[256] ; - char szZipExe[256] ; - char szCommand[1024] ; - char szTempDir1[256] ; - char szTempDir2[256] ; - char szTempZip[256] ; - char szSaveDir[256] ; - char szTemp[4096] ; - char *ptrChar ; - USHORT rc ; - DWORD dwCode; - BOOL bReturn = TRUE; - - - /*-----------------------------------------------------------------------*/ - /* Determine input file format, whether ZIP file or CONTENT.XML */ - /*-----------------------------------------------------------------------*/ - strcpy( szSourceFile, TargetFile ) ; - ptrChar = strstr( szSourceFile, "\\TARGET\\" ) ; - if ( ptrChar ) { - strncpy( ptrChar, "\\SOURCE\\", 8 ) ; - GetOTMDllPath( szProgPath, szZipExe ) ; - strcat( szZipExe, "ZIP.EXE" ) ; - if ( ! szZipExe[0] ) { - strcpy( ErrText, MSG_MSOFC_XML_ZIP_MISSING_ZIP ) ; - bReturn = FALSE ; - } - } else { - strcpy( szTemp, MSG_MSOFC_XML_ZIP_TARGET_PATH ) ; - sprintf( ErrText, szTemp, TargetFile ) ; - bReturn = FALSE ; - } - - if ( bReturn ) { - /*--------------------------------------------------------------------*/ - /* XML files must be replaced into the original document ZIP file. */ - /* 1. Create temporary directories. */ - /* 2. Split XML file into individual files in temporary directory.*/ - /* 3. Copy source ZIP to same name in temporary directory. */ - /* 4. Zip updated files in temp dir into temp ZIP file. */ - /* 5. Copy temp ZIP file to output file. */ - /* 6. Delete temporary directories. */ - /*--------------------------------------------------------------------*/ - - /*-----------------------------------------------------------------*/ - /* Create temporary directories. */ - /*-----------------------------------------------------------------*/ - strcpy( szTempDir1, TargetFile ) ; - ptrChar = strstr( szTempDir1, "\\TARGET\\" ) ; - strcpy( ptrChar, "\\MISC\\" ) ; - _mkdir( szTempDir1 ) ; /* Create \EQF\...\MISC\ */ - strcpy( szTempDir2, szTempDir1 ) ; - ptrChar = strrchr( TargetFile, '\\' ) ; - strcat( szTempDir2, ++ptrChar ) ; - strcat( szTempDir2, "$\\" ) ; - _mkdir( szTempDir2 ) ; /* Create \EQF\...\MISC\...\ */ - - strcpy( szTempZip, szTempDir1 ) ; - strcat( szTempZip, ptrChar ) ; - strcat( szTempZip, ".ZIP" ) ; - DosCopy( szSourceFile, szTempZip, DCPY_EXISTING ) ; - - /*-----------------------------------------------------------------*/ - /* Split the one XML file into its individual files. */ - /*-----------------------------------------------------------------*/ - bReturn = SplitFiles( TargetFile, szTempDir2, ErrText ) ; - - /*-----------------------------------------------------------------*/ - /* Update the files in the ZIP file. */ - /*-----------------------------------------------------------------*/ - if ( bReturn ) { - _getcwd( szSaveDir, sizeof(szSaveDir) ) ; - chdir( szTempDir2 ) ; - sprintf( szCommand, "\"%s\" -r -f %s *.xml > %s", - szZipExe, szTempZip, TempFile ) ; - strcpy( ErrText, MSG_MSOFC_XML_ZIP_REPLACE_FAILED ) ; - bReturn = ExecuteCommand( szCommand, TempFile, ErrText ) ; - if ( bReturn ) { - DosCopy( szTempZip, TargetFile, DCPY_EXISTING ) ; - } - chdir( szSaveDir ) ; - } - - /*-----------------------------------------------------------------*/ - /* Clean-up. */ - /*-----------------------------------------------------------------*/ - remove( szTempZip ) ; - sprintf( szCommand, "cmd /C rd /s /q %s > %s", szTempDir2, TempFile ) ; - ExecuteCommand( szCommand, TempFile, NULL ) ; - } - - return( bReturn ) ; -} - -/*****************************************************************************/ -/* SplitFiles */ -/* */ -/* Split the combined XML file into its individual files. */ -/* */ -/* Input: XmlFile - XML file to split into its components. */ -/* OutputDir - Output directory to save split files into. */ -/* Output: ErrText - Error message if failure. */ -/* */ -/* Return: TRUE - Action was successful. */ -/* FALSE - Action failed. */ -/*****************************************************************************/ - -BOOL SplitFiles( char *XmlFile, char *OutputDir, char *ErrText ) -{ - FILE *fSource, *fOut ; - CHAR szIn[MAX_RCD_LENGTH*2] ; - CHAR szOutputFile[256] ; - CHAR szOutputName[256] ; - CHAR szTemp[4096] ; - CHAR *ptrChar, *ptrChar2 ; - CHAR *ptrText, *ptrFile ; - USHORT rc ; - BOOL bReturn = TRUE ; - - - fSource = fopen( XmlFile, "r" ) ; - if ( ( fSource ) && - ( fgets( szIn, MAX_RCD_LENGTH, fSource ) != NULL ) && - ( ! strncmp( szIn, ZIP_FILE_SEPARATOR_START, strlen(ZIP_FILE_SEPARATOR_START) ) ) ) { - while( ( bReturn ) && - ( ! strncmp( szIn, ZIP_FILE_SEPARATOR_START, strlen(ZIP_FILE_SEPARATOR_START) ) ) ) { - - /*-----------------------------------------------------------------*/ - /* Split the next records into a separate XML file. */ - /*-----------------------------------------------------------------*/ - ptrChar = szIn + strlen( ZIP_FILE_SEPARATOR_START ) ; - ptrChar2 = strstr( ptrChar, ZIP_FILE_SEPARATOR_END ) ; - if ( ! ptrChar2 ) { /* If only part of separator line in buffer 11-6-14 */ - fgets( &szIn[strlen(szIn)], MAX_RCD_LENGTH, fSource ) ; - ptrChar2 = strstr( ptrChar, ZIP_FILE_SEPARATOR_END ) ; - } - if ( ptrChar2 ) { - *ptrChar2 = 0 ; - ptrText = ptrChar2 + strlen( ZIP_FILE_SEPARATOR_END ) ; - strcpy( szOutputFile, ptrChar ) ; - /*--------------------------------------------------------------*/ - /* Create all of the directories needed for this file. */ - /*--------------------------------------------------------------*/ - for( ptrChar=strchr(szOutputFile,'\\') ; - ptrChar ; - ptrChar=strchr(ptrChar+1,'\\') ) { - *ptrChar = 0 ; - strcpy( szOutputName, OutputDir ) ; - strcat( szOutputName, szOutputFile ) ; - _mkdir( szOutputName ) ; - *ptrChar = '\\' ; - } - strcpy( szOutputName, OutputDir ) ; - strcat( szOutputName, szOutputFile ) ; - fOut = fopen( szOutputName, "w" ) ; - if ( fOut ) { - /*-----------------------------------------------------------*/ - /* Copy all records until the start of the next file. */ - /*-----------------------------------------------------------*/ - while ( bReturn ) { - ptrFile = strstr( ptrText, ZIP_FILE_SEPARATOR_COMMENT ) ; /* 1-5-11 */ - if ( ( ptrFile ) && - ( ! strstr( ptrFile, ZIP_FILE_SEPARATOR_COMMENT_END ) ) ) { - *ptrFile = 0 ; - fputs( ptrText, fOut ) ; - *ptrFile = '<' ; - memmove( szIn, ptrFile, strlen(ptrFile)+1 ) ; - ptrText = szIn ; - fgets( &szIn[strlen(szIn)], MAX_RCD_LENGTH, fSource ) ; - } - ptrFile = strstr( ptrText, ZIP_FILE_SEPARATOR_START ) ; - if ( ptrFile ) { - *ptrFile = 0 ; - fputs( ptrText, fOut ) ; - *ptrFile = '<' ; - memmove( szIn, ptrFile, strlen(ptrFile)+1 ) ; - break ; - } else { - fputs( ptrText, fOut ) ; - if ( fgets( szIn, MAX_RCD_LENGTH, fSource ) == NULL ) { - szIn[0] = 0 ; - break ; - } - ptrText = szIn ; - } - } - fclose( fOut ) ; - } else { - strcpy( szTemp, MSG_MSOFC_XML_ZIP_CREATE_XML_FILE ) ; - sprintf( ErrText, szTemp, szOutputFile ) ; - bReturn = FALSE ; - } - - } else { - strcpy( ErrText, MSG_MSOFC_XML_ZIP_CREATE_XML_FORMAT ) ; - bReturn = FALSE ; - } - } - } else { - strcpy( ErrText, MSG_MSOFC_XML_ZIP_CREATE_XML_FORMAT ) ; - bReturn = FALSE ; - } - - fclose( fSource ) ; - fclose( fOut ) ; - - return( bReturn ) ; -} - - -/*****************************************************************************/ -/* ExecuteCommand */ -/* */ -/* Execute the ZIP or UNZIP command. */ -/* */ -/* Input: Command - Command string to execute. */ -/* ErrFile - Temporary file to capture error messages. */ -/* Output: ErrText - Error message if failure. */ -/* */ -/* Return: TRUE - Action was successful. */ -/* FALSE - Action failed. */ -/*****************************************************************************/ - -BOOL ExecuteCommand( char *Command, char *ErrFile, char *ErrText ) -{ - STARTUPINFOA StartupInfo ; - PROCESS_INFORMATION piProcessInfo ; - FILE *fTemp ; - CHAR szText[256] ; - USHORT rc ; - DWORD dwCode; - BOOL bReturn = FALSE ; - - /*-----------------------------------------------------------------------*/ - /* Execute command so that DOS window does not pop up. */ - /*-----------------------------------------------------------------------*/ - GetStartupInfoA( &StartupInfo ) ; - StartupInfo.dwFlags = STARTF_USESHOWWINDOW ; - StartupInfo.wShowWindow = SW_HIDE ; - bReturn = CreateProcessA( NULL, Command, NULL, NULL, FALSE, (DWORD)0, NULL, NULL, - &StartupInfo, &piProcessInfo ) ; - WaitForSingleObject( piProcessInfo.hProcess, INFINITE ) ; - rc = GetExitCodeProcess(piProcessInfo.hProcess, &dwCode); - - if ( dwCode == 0 ) { - bReturn = TRUE ; - } else { - bReturn = FALSE ; - if ( ErrText ) { - strcat( ErrText, Command ) ; - strcat( ErrText, "\n" ) ; - sprintf( szText, "RC=%d %ld\n", rc, dwCode); - strcat( ErrText, szText ) ; - fTemp = fopen( ErrFile, "r" ) ; - if ( fTemp ) { - while( fgets( szText, sizeof(szText), fTemp ) != NULL ) { - strcat( ErrText, szText ) ; - } - fclose( fTemp ) ; - } - } - } - - return( bReturn ) ; -} - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c index 809f1ac4..e2cb93c2 100755 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c +++ b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c @@ -491,337 +491,6 @@ BOOL ConcatFiles( char *BaseFile, char *FromFile, char *FromFileName ) } - -/*****************************************************************************/ -/* SplitFiles_1 */ -/* */ -/* Split the CONTENT.XML file into these parts: */ -/* CONTENT.XML */ -/* META.XML */ -/* STYLES.XML */ -/* */ -/* Input: ContentFile - File to split into its components. */ -/* OutputDir - Output directory to save split files into. */ -/* SourceZip - Source ZIP file to get start of split files. */ -/* Output: ErrText - Error message if failure. */ -/* */ -/* Return: TRUE - Action was successful. */ -/* FALSE - Action failed. */ -/*****************************************************************************/ - -BOOL SplitFiles_1( char *ContentFile, char *OutputDir, char *SourceZip, - char *TempFile, char *ErrText ) -{ - FILE *fSplit, *fContent, *fIn, *fOut ; - CHAR szIn[MAX_RCD_LENGTH*2] ; - CHAR szIn2[MAX_RCD_LENGTH*2] ; - CHAR szOutputFile[256] ; - CHAR szTempFile[256] ; - char szCommand[1024] ; - CHAR *ptrChar ; - USHORT usFileType ; - USHORT rc ; - BOOL bReturn = TRUE ; - - - fSplit = fopen( ContentFile, "r" ) ; - strcpy( szOutputFile, OutputDir ) ; - strcat( szOutputFile, ODC_CONTENT_XML ) ; - fOut = fopen( szOutputFile, "w" ) ; - if ( ( ! fSplit ) || - ( ! fOut ) ) { - if ( fSplit ) - fclose( fSplit ) ; - if ( fOut ) - fclose( fOut ) ; - sprintf( ErrText, ODC_REPLACE_OPEN, ODC_CONTENT_XML ) ; - return( FALSE ) ; - } - usFileType = 1 ; - - /*-----------------------------------------------------------------*/ - /* Copy the records for the CONTENT.XML file. */ - /*-----------------------------------------------------------------*/ - while( GetRcd( szIn, MAX_RCD_LENGTH, fSplit, TRUE ) != NULL ) { - ptrChar = strstr( szIn, ODC_SEPARATOR1_META_XML ) ; - if ( ptrChar ) { - *ptrChar = 0 ; - fputs( szIn, fOut ) ; - ptrChar += strlen(ODC_SEPARATOR1_META_XML) ; - if ( *ptrChar ) - memmove( szIn, ptrChar, strlen(ptrChar)+1 ) ; - else - szIn[0] = 0 ; - usFileType = 2 ; - break ; - } - ptrChar = strstr( szIn, ODC_SEPARATOR1_STYLES_XML ) ; - if ( ptrChar ) { - *ptrChar = 0 ; - fputs( szIn, fOut ) ; - ptrChar += strlen(ODC_SEPARATOR1_STYLES_XML) ; - if ( *ptrChar ) - memmove( szIn, ptrChar, strlen(ptrChar)+1 ) ; - else - szIn[0] = 0 ; - usFileType = 3 ; - break; - } - fputs( szIn, fOut ) ; - } - fclose( fOut ) ; - - /*-----------------------------------------------------------------*/ - /* Copy the records for the META.XML file. */ - /*-----------------------------------------------------------------*/ - if ( usFileType == 2 ) { - strcpy( szOutputFile, OutputDir ) ; - strcat( szOutputFile, ODC_META_XML ) ; - - /*--------------------------------------------------------------*/ - /* Unzip source META.XML. */ - /*--------------------------------------------------------------*/ - sprintf( szCommand, "\"%s\" -q -o %s %s -d %s > %s", - szUnzipExe, SourceZip, ODC_META_XML, - OutputDir, TempFile ) ; - sprintf( ErrText, ODC_EXTRACT_FAILED, ODC_META_XML ) ; - bReturn = ExecuteCommand( szCommand, TempFile, ErrText ) ; - if ( bReturn ) { - DosCopy( szOutputFile, TempFile, DCPY_EXISTING ) ; - - /*-----------------------------------------------------------*/ - /* Copy 1st unchanged part of META.XML file. */ - /*-----------------------------------------------------------*/ - fOut = fopen( szOutputFile, "w" ) ; - fIn = fopen( TempFile, "r" ) ; - if ( ( fIn ) && - ( fOut ) ) { - while( GetRcd( szIn2, MAX_RCD_LENGTH, fIn, TRUE ) != NULL ) { - ptrChar = strstr( szIn2, ODC_STARTTAG_META_XML ) ; - if ( ! ptrChar ) /* 2-4-11 */ - ptrChar = strstr( szIn2, ODC_STARTTAG_META_XML2 ) ; - if ( ptrChar ) { - *ptrChar = 0 ; - fputs( szIn2, fOut ) ; - break ; - } - fputs( szIn2, fOut ) ; - } - fclose( fIn ) ; - } else { - sprintf( ErrText, ODC_REPLACE_OPEN, ODC_META_XML ) ; - bReturn = FALSE ; - } - remove( TempFile ) ; - - /*-----------------------------------------------------------*/ - /* Copy 2nd translated part of META.XML from CONTENT.XML. */ - /*-----------------------------------------------------------*/ - if ( bReturn ) { - fputs( szIn, fOut ) ; - while( GetRcd( szIn, MAX_RCD_LENGTH, fSplit, TRUE ) != NULL ) { - ptrChar = strstr( szIn, ODC_SEPARATOR1_STYLES_XML ) ; - if ( ptrChar ) { - *ptrChar = 0 ; - fputs( szIn, fOut ) ; - ptrChar += strlen(ODC_SEPARATOR1_STYLES_XML) ; - if ( *ptrChar ) - memmove( szIn, ptrChar, strlen(ptrChar)+1 ) ; - else - szIn[0] = 0 ; - usFileType = 3 ; - break; - } - fputs( szIn, fOut ) ; - } - } - fclose( fOut ) ; - } else { - sprintf( ErrText, ODC_REPLACE_OPEN, ODC_META_XML ) ; - bReturn = FALSE ; - } - } - - - /*-----------------------------------------------------------------*/ - /* Copy the records for the STYLES.XML file. */ - /*-----------------------------------------------------------------*/ - if ( ( usFileType == 3 ) && - ( bReturn ) ) { - strcpy( szOutputFile, OutputDir ) ; - strcat( szOutputFile, ODC_STYLES_XML ) ; - - /*--------------------------------------------------------------*/ - /* Unzip source STYLES.XML. */ - /*--------------------------------------------------------------*/ - sprintf( szCommand, "\"%s\" -q -o %s %s -d %s 2> %s", - szUnzipExe, SourceZip, ODC_STYLES_XML, - OutputDir, TempFile ) ; - sprintf( ErrText, ODC_EXTRACT_FAILED, ODC_STYLES_XML ) ; - bReturn = ExecuteCommand( szCommand, TempFile, ErrText ) ; - if ( bReturn ) { - strcpy( szTempFile, OutputDir ) ; - strcat( szTempFile, ODC_STYLES_XML ) ; - DosCopy( szTempFile, TempFile, DCPY_EXISTING ) ; - - /*-----------------------------------------------------------*/ - /* Copy 1st unchanged part of STYLES.XML file. */ - /*-----------------------------------------------------------*/ - fOut = fopen( szOutputFile, "w" ) ; - fIn = fopen( TempFile, "r" ) ; - if ( fIn ) { - while( GetRcd( szIn2, MAX_RCD_LENGTH, fIn, TRUE ) != NULL ) { - ptrChar = strstr( szIn2, ODC_STARTTAG_STYLES_XML ) ; - if ( ptrChar ) { - *ptrChar = 0 ; - fputs( szIn2, fOut ) ; - break ; - } - fputs( szIn2, fOut ) ; - } - fclose( fIn ) ; - } else { - sprintf( ErrText, ODC_REPLACE_OPEN, ODC_STYLES_XML ) ; - bReturn = FALSE ; - } - remove( TempFile ) ; - - /*-----------------------------------------------------------*/ - /* Copy 2nd translated part of STYLES.XML from CONTENT.XML. */ - /*-----------------------------------------------------------*/ - if ( bReturn ) { - fputs( szIn, fOut ) ; - while( GetRcd( szIn, MAX_RCD_LENGTH, fSplit, TRUE ) != NULL ) { - fputs( szIn, fOut ) ; - } - } - fclose( fOut ) ; - } else { - sprintf( ErrText, ODC_REPLACE_OPEN, ODC_STYLES_XML ) ; - bReturn = FALSE ; - } - } - - fclose( fSplit ) ; - - return( bReturn ) ; -} - - - -/*****************************************************************************/ -/* SplitFiles_2 */ -/* */ -/* Split the CONTENT.XML file into these parts, including subdirectories: */ -/* CONTENT.XML */ -/* META.XML */ -/* STYLES.XML */ -/* */ -/* Input: XmlFile - File to split into its components. */ -/* OutputDir - Output directory to save split files into. */ -/* Output: ErrText - Error message if failure. */ -/* */ -/* Return: TRUE - Action was successful. */ -/* FALSE - Action failed. */ -/*****************************************************************************/ -BOOL SplitFiles_2( char *XmlFile, char *OutputDir, char *ErrText ) -{ - FILE *fSource, *fOut ; - CHAR szIn[MAX_RCD_LENGTH*2] ; - CHAR szOutputFile[256] ; - CHAR szOutputName[256] ; - CHAR *ptrChar, *ptrChar2 ; - CHAR *ptrText, *ptrFile ; - USHORT rc ; - BOOL bReturn = TRUE ; - - - fSource = fopen( XmlFile, "r" ) ; - if ( ( fSource ) && - ( fgets( szIn, MAX_RCD_LENGTH, fSource ) != NULL ) && - ( ! strncmp( szIn, ODC_SEPARATOR_START, strlen(ODC_SEPARATOR_START) ) ) ) { - while( ( bReturn ) && - ( ! strncmp( szIn, ODC_SEPARATOR_START, strlen(ODC_SEPARATOR_START) ) ) ) { - - /*-----------------------------------------------------------------*/ - /* Split the next records into a separate XML file. */ - /*-----------------------------------------------------------------*/ - ptrChar = szIn + strlen( ODC_SEPARATOR_START ) ; - ptrChar2 = strstr( ptrChar, ODC_SEPARATOR_END ) ; - if ( ptrChar2 ) { - *ptrChar2 = 0 ; - ptrText = ptrChar2 + strlen( ODC_SEPARATOR_END ) ; - strcpy( szOutputFile, ptrChar ) ; - /*--------------------------------------------------------------*/ - /* Create all of the directories needed for this file. */ - /*--------------------------------------------------------------*/ - for( ptrChar=strchr(szOutputFile,'\\') ; - ptrChar ; - ptrChar=strchr(ptrChar+1,'\\') ) { - *ptrChar = 0 ; - strcpy( szOutputName, OutputDir ) ; - strcat( szOutputName, szOutputFile ) ; - _mkdir( szOutputName ) ; - *ptrChar = '\\' ; - } - strcpy( szOutputName, OutputDir ) ; - strcat( szOutputName, szOutputFile ) ; - fOut = fopen( szOutputName, "w" ) ; - if ( fOut ) { - /*-----------------------------------------------------------*/ - /* Copy all records until the start of the next file. */ - /*-----------------------------------------------------------*/ - while ( bReturn ) { - ptrFile = strstr( ptrText, ODC_SEPARATOR_COMMENT ) ; /* 1-5-11 */ - if ( ( ptrFile ) && - ( ! strstr( ptrFile, ODC_SEPARATOR_COMMENT_END ) ) ) { - *ptrFile = 0 ; - fputs( ptrText, fOut ) ; - *ptrFile = '<' ; - memmove( szIn, ptrFile, strlen(ptrFile)+1 ) ; - ptrText = szIn ; - fgets( &szIn[strlen(szIn)], MAX_RCD_LENGTH, fSource ) ; - } - ptrFile = strstr( ptrText, ODC_SEPARATOR_START ) ; - if ( ptrFile ) { - *ptrFile = 0 ; - fputs( ptrText, fOut ) ; - *ptrFile = '<' ; - memmove( szIn, ptrFile, strlen(ptrFile)+1 ) ; - break ; - } else { - fputs( ptrText, fOut ) ; - if ( fgets( szIn, MAX_RCD_LENGTH, fSource ) == NULL ) { - szIn[0] = 0 ; - break ; - } - ptrText = szIn ; - } - } - fclose( fOut ) ; - } else { - sprintf( ErrText, ODC_REPLACE_XML_FILE, szOutputFile ) ; - bReturn = FALSE ; - } - - } else { - sprintf( ErrText, ODC_REPLACE_SPLIT ) ; - bReturn = FALSE ; - } - } - } else { - sprintf( ErrText, ODC_REPLACE_SPLIT ) ; - bReturn = FALSE ; - } - - fclose( fSource ) ; - fclose( fOut ) ; - - return( bReturn ) ; -} - - - /*****************************************************************************/ /* FindFileType */ /* */ diff --git a/source/otmd.cpp b/source/otmd.cpp index 05deac84..80f66cdb 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -190,6 +190,9 @@ DEFINE_bool(allowLoadingMultipleTmsSimultaneously, false, "If set to true, multiple tms could be loaded from the disk at the same time. Loading multiple TMs at the same time was disabled because of I/0 disk operation limit at the server. Enable this if you don't care about num of IO operations and want some perfomance boost"); +DEFINE_bool(skip_0s_at_the_table_start, false, + "If LongName table was corrupted or sorted in a way that empty entry was first, so in binaries it starts with some number of 0s, this helps to skipt that and restore that tm via reorganize"); + DEFINE_bool(keep_tm_backups, false, "if set to true, when saving tmd and tmi files, old copies would be saved with .old suffix"); DEFINE_bool(skip_default_flags_in_init_msg, false, "if set to true, all default flags would be skipped in init msg"); From 9370fde03231248990e8c7213524896c7338e2c9 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Thu, 14 Aug 2025 21:47:49 +0300 Subject: [PATCH 53/62] * updated to v0.7.25 + added error ERR_WRITE_REQUESTS_NOT_ALLOWED(3001) and ERR_LOADING_TM_ARE_NOT_ALLOWED(3002) errors * fixed langIdToGroup table in tables response + added fix for longnames table from early 0.7.x versions --- include/lowlevelotmdatastructs.h | 4 + include/requestdata.h | 2 +- include/tm.h | 16 +++- source/CMakeLists.txt | 2 +- source/RestAPI/ProxygenHandler.cpp | 5 +- source/RestAPI/ProxygenServer.cpp | 6 ++ .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 7 ++ source/opentm2/core/requestdata.cpp | 90 ++++++++++++------- source/opentm2/core/tm.cpp | 24 +---- source/otmd.cpp | 2 +- 10 files changed, 98 insertions(+), 60 deletions(-) diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index b09ecd10..da6c962c 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -1294,6 +1294,7 @@ typedef enum _PROCWINSTYLE #define Err_Prop 2000 // property handler rc #define Err_ObjM 2100 // object manager rc #define Err_Nfol 2200 // new folder messages +#define ERR_SERVICE_BASE 3000 #define ERR_BTREE_BASE 5000 #define ERR_BTREE_END 5999 #define ERR_MEM_BASE 6000 @@ -1354,6 +1355,9 @@ typedef enum _PROCWINSTYLE #define ErrProp_InvalidHandle (Err_Prop+23) // invalid handle to properties +#define ERR_WRITE_REQUESTS_NOT_ALLOWED ERR_SERVICE_BASE + 1 //3000 +#define ERR_LOADING_TM_ARE_NOT_ALLOWED ERR_SERVICE_BASE + 2 //3000 + #define BTREE_BASE ERR_BTREE_BASE //5000 #define BTREE_NO_ROOM BTREE_BASE+1 // Insufficent memory #define BTREE_ILLEGAL_FILE BTREE_BASE+2 // Not an index file diff --git a/include/requestdata.h b/include/requestdata.h index e6b0cec9..a56d5cac 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -427,7 +427,7 @@ class StatusMemRequestData: public RequestData{ int checkData() override ; int execute() override ; - static std::string prepareStatusString(std::shared_ptr mem); + static std::string prepareStatusString(std::shared_ptr& mem); }; class FlagsRequestData: public RequestData{ diff --git a/include/tm.h b/include/tm.h index b1c2d5b2..c825570d 100644 --- a/include/tm.h +++ b/include/tm.h @@ -3706,7 +3706,6 @@ class EqfMemoryPlugin : public OtmPlugin int iLastError; std::string strLastError; - std::vector> m_MemInfoVector; private: std::shared_ptr findMemory(const char *pszName); @@ -3727,12 +3726,23 @@ class EqfMemoryPlugin : public OtmPlugin // TIMEOUT = -2 // }; +enum TMManagarState{ + INIT=0, + READY=1, + SHUTDOWN_CALLED, + WAITING_FOR_IMPORTS, + OFF +}; + + class TMManager { public: - std::atomic_bool fWriteRequestsAllowed{0}; - std::atomic_bool fServiceIsRunning{1}; + inline static std::mutex mutex_tmManager; + inline static bool fWriteRequestsAllowed{0}; + inline static bool fServiceIsRunning{1}; + inline static bool fLoadingTmIsAllowed{0}; /*! \brief Pointer to the list of opened memories */ // std::vector vMemoryList; diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 012fc33f..98f67eba 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 24) +SET(APP_VERSION_MINOR 25) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/ProxygenHandler.cpp b/source/RestAPI/ProxygenHandler.cpp index f4525486..087b42ec 100755 --- a/source/RestAPI/ProxygenHandler.cpp +++ b/source/RestAPI/ProxygenHandler.cpp @@ -32,7 +32,6 @@ DEFINE_bool(request_number, namespace ProxygenService { ProxygenHandler::ProxygenHandler() { - TMManager::GetInstance()->fWriteRequestsAllowed = true; } @@ -294,7 +293,9 @@ void ProxygenHandler::sendResponse()noexcept{ case 423: { //responseText - pRequest->outputMessage = "{\n\"msg\": \"WRITE REQUESTS ARE NOT ALLOWED\",\n\"rc\" = 423\n}\n" ; + if(pRequest->outputMessage.empty()){ + pRequest->outputMessage = "{\n\"msg\": \"WRITE REQUESTS ARE NOT ALLOWED\",\n\"rc\" = 423\n}\n" ; + } responseText = "Locked"; break; } diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index 8e9eb8a0..4804e39a 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -467,6 +467,12 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { serviceName = szServiceName; additionalServiceName = serviceName+"_service"; T5Logger::GetInstance()->SetLogLevel(uiLogLevel); + + { + std::scoped_lock(TMManager::mutex_tmManager); + TMManager::fWriteRequestsAllowed = true; + TMManager::fLoadingTmIsAllowed = true; + } if( true /*printIninMsg*/){ T5INITLOG << "Service details: Service name = " << szServiceName << diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index fb619e5f..7b1fab0b 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -815,6 +815,13 @@ USHORT EqfMemory::NTMReadLongNameTable() while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen-1) && *(pszTemp) == 0){//skip 0s at the start of the table; pszTemp++; } + auto lenOf0s = pszTemp - (PSZ)state->LongNames.pszBuffer.data(); + if (lenOf0s > 0) {//erase 0s at the start + state->LongNames.pszBuffer.erase( + state->LongNames.pszBuffer.begin(), + state->LongNames.pszBuffer.begin() + lenOf0s + ); + } } while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen - sizeof(USHORT)) && *((PUSHORT)pszTemp) != 0) diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 49082344..afc38024 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -283,38 +283,59 @@ int RequestData::requestTM(){ fValid = true; return 0; } + + if(isReadOnlyRequest() || isWriteRequest()){//shortcut when app is closing state + bool fWriteRequestsAllowed = false, fLoadingTmIsAllowed = false; + { + std::scoped_lock(TMManager::mutex_tmManager); + fWriteRequestsAllowed = TMManager::fWriteRequestsAllowed; + fLoadingTmIsAllowed = TMManager::fLoadingTmIsAllowed; + } + + if(!fLoadingTmIsAllowed){ + if(!TMManager::GetInstance()->IsMemoryInList(strMemName, tmListTimeout)){ + return buildErrorReturn(ERR_LOADING_TM_ARE_NOT_ALLOWED, "Memory is not loaded and loading tm are not allowed, because shutdown of the service is scheduled or service is not initialized", 423); + } + //if(TMManager::GetInstance()->IsMemoryLoading()) + } + if(isWriteRequest() && !fWriteRequestsAllowed){ + return buildErrorReturn(ERR_WRITE_REQUESTS_NOT_ALLOWED, "Write requests are not allowed, because shutdown of the service is scheduled or service is not initialized", 423); + } + } + //check if memory is loaded to tmmanager if(isReadOnlyRequest()) { mem = TMManager::GetInstance()->requestReadOnlyTMPointer(strMemName, memRef, requestTMTimeout, tmListTimeout, command); if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg("Failed to lock tm list:", __func__, __LINE__); - return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), 506); } if(requestTMTimeout.failed()){ requestTMTimeout.addToErrMsg("Failed to requestTm:", __func__, __LINE__); - return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), 506); } }else if(isWriteRequest()) { + // mem = TMManager::GetInstance()->requestWriteTMPointer(strMemName, memRef, requestTMTimeout, tmListTimeout, command); if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg("Failed to lock tm list:", __func__, __LINE__); - return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), 506); } if(requestTMTimeout.failed()){ requestTMTimeout.addToErrMsg("Failed to lock requestTm:", __func__, __LINE__); - return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), 506); } }else if(STATUS_MEM == command){ mem = TMManager::GetInstance()->requestServicePointer(strMemName, requestTMTimeout, tmListTimeout, command); if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg("Failed to lock tm list:", __func__, __LINE__); - return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), 506); } if(requestTMTimeout.failed()){ requestTMTimeout.addToErrMsg("Failed to lock tm:", __func__, __LINE__); - return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, requestTMTimeout.getErrMsg().c_str(), 506); } fValid = true;//for status we don't care about tm pointer here return 0; @@ -324,15 +345,15 @@ int RequestData::requestTM(){ if(mem.get()== nullptr){ fValid = isServiceReq; }else if(mem->isReorganizeRunning()){ - return buildErrorReturn(505, "Reorganize is running for requested tm", _rc_); + return buildErrorReturn(505, "Reorganize is running for requested tm", 505); }else if(mem->isImportRunning()){ - return buildErrorReturn(505, "Import is running for requested tm", _rc_); + return buildErrorReturn(505, "Import is running for requested tm", 505); }else{ TimedMutexGuard l(mem->tmMutex, tmLockTimeout, "tmMutex", __func__, __LINE__); if(tmLockTimeout.failed()){ tmLockTimeout.addToErrMsg("Failed to lock tm:", __func__, __LINE__); - return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), 506); } if(mem->isWaitingToBeLoaded() || mem->isFailedToLoad()){ @@ -355,7 +376,7 @@ int RequestData::requestTM(){ if(11==_rc_){ msg += "; file on the disk is locked;"; } - return buildErrorReturn(504, msg.c_str(), _rc_); + return buildErrorReturn(512, msg.c_str(), 512); } } } @@ -370,7 +391,7 @@ int RequestData::requestTM(){ && GET_TABLES != command) ) { - return buildErrorReturn(504, "Memory is corrupted, so it can only be tmx-exported or reorganized", _rc_); + return buildErrorReturn(504, "Memory is corrupted, so it can only be tmx-exported or reorganized", 504); } if( command == EXPORT_MEM_TMX @@ -385,7 +406,7 @@ int RequestData::requestTM(){ mem->FlushFilebuffers(tmLockTimeout); if(tmLockTimeout.failed()){ tmLockTimeout.addToErrMsg("Failed to lock tm:", __func__, __LINE__); - return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), 506); } } @@ -515,7 +536,7 @@ int RequestData::run(){ TimedMutexGuard l(mem->tmMutex, tmLockTimeout, "tmMutex", __func__, __LINE__) ; if(tmLockTimeout.failed()){ tmLockTimeout.addToErrMsg("Failed to lock tm:", __func__, __LINE__); - return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), 506); } mem->setActiveRequestCommand(command); @@ -665,10 +686,10 @@ int CreateMemRequestData::importInInternalFomat(){ if(binTmData.size() > 0){ auto filesize = FilesystemHelper::WriteToFile(strTempFile, &binTmData[0], binTmData.size()); if(0 == filesize){ - return buildErrorReturn( _rc_, "cant write tm data to temp file", 500); + return buildErrorReturn( 500, "cant write tm data to temp file", 500); } }else{ - return buildErrorReturn( _rc_, "binary tm file data not found", 400); + return buildErrorReturn( 400, "binary tm file data not found", 400); } } } @@ -1130,7 +1151,7 @@ int SaveAllTMsToDiskRequestData::execute(){ if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg(".Failed to lock tm list:", __func__, __LINE__); - return buildErrorReturn(506, tmListTimeout.getErrMsg().c_str(), TM_LIST_MUTEX_TIMEOUT_FAILED); + return buildErrorReturn(TM_LIST_MUTEX_TIMEOUT_FAILED, tmListTimeout.getErrMsg().c_str(), 506); } for(const auto& tm: TMManager::GetInstance()->tms){ @@ -1142,7 +1163,7 @@ int SaveAllTMsToDiskRequestData::execute(){ errMsg += tm.second->getName() + ":" + tmLockTimeout.getErrMsg() + "; "; tmLockTimeout.reset(); tmLockTimeout.setTimeout_ms(tmLockTimeoutms); - //return buildErrorReturn(506, tmLockTimeout.getErrMsg().c_str(), _rc_); + //return buildErrorReturn(_rc_, tmLockTimeout.getErrMsg().c_str(), 506); } tms += tm.first; } @@ -1169,13 +1190,13 @@ std::string RequestData::reserveFilename(){ int ImportRequestData::execute(){ if ( mem == nullptr ) { - return buildErrorReturn( 404, "mem not found or can't be opened" ); + return buildErrorReturn( 404, "mem not found or can't be opened",404 ); } // close the memory - when open if ( false == memIsAvailableToOperate(mem.get()) ) { std::string msg = "mem is not available to operate, status= " + mem->getStatusString(); - return buildErrorReturn( 500, msg.c_str() ); + return buildErrorReturn( 500, msg.c_str(),500 ); } if(strTempFile.empty()){ @@ -2138,7 +2159,7 @@ void AddRequestDataToJson(std::stringstream& ss, std::string reqType, millisecon -std::string StatusMemRequestData::prepareStatusString(std::shared_ptr mem) +std::string StatusMemRequestData::prepareStatusString(std::shared_ptr& mem) { std::string outputMessage; // set status value @@ -2297,7 +2318,7 @@ int GetTablesMemRequestData::execute() { } i = 0; for(const auto& entry: mem->state->psLangIdToGroupTable){ - langIdToGroupTable += toStr(i++) + ")"+ toStr(entry); + langIdToGroupTable += toStr(i++) + ")"+ toStr(entry) + "; "; } json_factory.addParmToJSON( outputMessage, "tableName(additional info)(INTERNAL_KEY_OF_THE_TABLE)", "position)idInTable: value;"); @@ -2320,7 +2341,7 @@ int GetTablesMemRequestData::execute() { json_factory.addParmToJSON( outputMessage, tableName, longNamesCaseIgnore); tableName = "langIdToGroupTable"; - json_factory.addParmToJSON( outputMessage, tableName, longNamesCaseIgnore); + json_factory.addParmToJSON( outputMessage, tableName, langIdToGroupTable); json_factory.terminateJSON( outputMessage ); return( OK ); @@ -2507,24 +2528,28 @@ void ShutdownRequestData::CheckImportFlushTmsAndShutdown2(int signal, size_t tmL void ShutdownRequestData::CheckImportFlushTmsAndShutdown(int signal, MutexTimeout& tmListTimeout, bool flushTmsToDisk, bool waitForImportAndReorganizeProcesses) { - TMManager::GetInstance()->fWriteRequestsAllowed = false; - TMManager::GetInstance()->fServiceIsRunning = true; + { + std::scoped_lock(TMManager::mutex_tmManager); + TMManager::fWriteRequestsAllowed = false; + TMManager::fServiceIsRunning = true; + TMManager::fLoadingTmIsAllowed = false; + } //pMemService->closeAll(); T5Logger::GetInstance()->LogStop(); - if(waitForImportAndReorganizeProcesses){ int j= 3; - while(int i = TMManager::GetInstance()->GetMemImportInProcessCount(tmListTimeout)){ + while( int i = TMManager::GetInstance()->GetMemImportInProcessCount(tmListTimeout)){ if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg(".Failed to lock tm list:", __func__, __LINE__); - return ; + //return ; + tmListTimeout.reset(); } if( ++j % 15 == 0){ - T5LOG(T5WARNING) << "SHUTDOWN:: memory still in import..waiting 15 sec more... shutdown request was = "<< j* 15 <<" seconds ago"; + T5LOG(T5WARNING) << "SHUTDOWN:: tms still in import..waiting 15 sec more... shutdown request was = "<< j <<" seconds ago"; } - T5LOG(T5DEBUG) << "SHUTDOWN:: memory still in import..waiting 1 sec more... mem in import = "<< i; + //T5LOG(T5DEBUG) << "SHUTDOWN:: tms still in import..waiting 1 sec more... # of tm in import = "<< i; - //sleep(1); + sleep(1); } } @@ -2541,7 +2566,10 @@ void ShutdownRequestData::CheckImportFlushTmsAndShutdown(int signal, MutexTimeou } } if(signal != SHUTDOWN_CALLED_FROM_MAIN){ - TMManager::GetInstance()->fServiceIsRunning = false; + { + std::scoped_lock(TMManager::mutex_tmManager); + TMManager::fServiceIsRunning = false; + } //sleep(1); exit(signal); } diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index c100504a..92471d17 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -43,19 +43,7 @@ int TMManager::GetMemImportInProcessCount(MutexTimeout& tmListTimeout){ size_t TMManager::CalculateOccupiedRAM(MutexTimeout& tmListTimeout){ char memFolder[260]; size_t UsedMemory = 0; - #ifdef CALCULATE_ONLY_MEM_FILES - Properties::GetInstance()->get_value(KEY_MEM_DIR, memFolder, 260); - std::string path; - for(int i = 0; i < EqfMemoryPlugin::GetInstance()->m_MemInfoVector.size() ;i++){ - if(EqfMemoryPlugin::GetInstance()->m_MemInfoVector[i].szName != 0){ - path = memFolder; - path += EqfMemoryPlugin::GetInstance()->m_MemInfoVector[i].szName; - UsedMemory += FilesystemHelper::GetFilebufferSize( std::string(path + ".TMI")); - UsedMemory += FilesystemHelper::GetFilebufferSize( std::string(path + ".TMD")); - UsedMemory += FilesystemHelper::GetFilebufferSize( std::string(path + ".MEM")); - } - } - #else + { TimedMutexGuard l{mutex_access_tms, tmListTimeout, "tmListMutex", __func__, __LINE__};// lock tms list if(tmListTimeout.failed()){ @@ -73,7 +61,7 @@ size_t TMManager::CalculateOccupiedRAM(MutexTimeout& tmListTimeout){ } //UsedMemory += FilesystemHelper::GetTotalFilebuffersSize(); UsedMemory += MEMORY_RESERVED_FOR_SERVICE; - #endif + T5LOG( T5INFO) << ":: calculated occupied ram = " << UsedMemory/1000000 << " MB"; return UsedMemory; @@ -248,13 +236,7 @@ void EqfMemory::reorganizeDone(int iRC, char *pszError ) */ int TMManager::closeAll() { - for ( int i = 0; i < ( int )EqfMemoryPlugin::GetInstance()->m_MemInfoVector.size(); i++ ) - { - if ( EqfMemoryPlugin::GetInstance()->m_MemInfoVector[i]->szName[0] != 0 ) - { - //removeFromMemoryList( i ); - } - } /* endfor */ + return( 0 ); } diff --git a/source/otmd.cpp b/source/otmd.cpp index 80f66cdb..17d9527d 100755 --- a/source/otmd.cpp +++ b/source/otmd.cpp @@ -190,7 +190,7 @@ DEFINE_bool(allowLoadingMultipleTmsSimultaneously, false, "If set to true, multiple tms could be loaded from the disk at the same time. Loading multiple TMs at the same time was disabled because of I/0 disk operation limit at the server. Enable this if you don't care about num of IO operations and want some perfomance boost"); -DEFINE_bool(skip_0s_at_the_table_start, false, +DEFINE_bool(skip_0s_at_the_table_start, true, "If LongName table was corrupted or sorted in a way that empty entry was first, so in binaries it starts with some number of 0s, this helps to skipt that and restore that tm via reorganize"); DEFINE_bool(keep_tm_backups, false, "if set to true, when saving tmd and tmi files, old copies would be saved with .old suffix"); From 65d1e4feb7868c23c33de7c2112d2ac52e27e164 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Fri, 15 Aug 2025 18:05:38 +0300 Subject: [PATCH 54/62] * updated to 0.7.26 * improved memory management * fixed longname issue handling for empty tm + added closing app msg - code cleanup --- include/tm.h | 15 +- source/CMakeLists.txt | 2 +- source/RestAPI/OtmMemoryServiceWorker.cpp | 2 +- source/RestAPI/ProxygenHandler.cpp | 25 +-- source/RestAPI/ProxygenHandler.h | 4 +- source/RestAPI/ProxygenServer.cpp | 69 ++++---- .../opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 7 + .../core/EqfMemoryPlugin/EqfMemoryPlugin.cpp | 158 ------------------ source/opentm2/core/requestdata.cpp | 21 ++- source/opentm2/core/tm.cpp | 4 +- 11 files changed, 72 insertions(+), 239 deletions(-) diff --git a/include/tm.h b/include/tm.h index c825570d..c68cf3df 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1840,7 +1840,7 @@ class EqfMemory //: public TMX_CLB time_t tLastAccessTime = 0; // last time memory has been used MEMORY_STATUS eStatus; // status of the memory MEMORY_STATUS eImportStatus; // status of the current/last memory import - ImportStatusDetails *importDetails = nullptr; + std::unique_ptr importDetails; std::string strError; // pointer to an error message (only when eStatus is IMPORT_FAILED_STATUS) @@ -3693,23 +3693,12 @@ class EqfMemoryPlugin : public OtmPlugin \returns TRUE when successful */ bool stopPlugin(bool fForce = false); - /*! \brief Handle a return code from the memory functions and create the approbriate error message text for it - \param iRC return code from memory function - \param pszMemName long memory name - \param pszMarkup markup table name or NULL if not available - \param pszMemPath fully qualified memory path name or NULL if not available - \param strLastError reference to string object receiving the message text - \param iLastError reference to a integer variable receiving the error code - \returns original or modified error return code - */ - static int handleError(int iRC, char *pszMemName, char *pszMarkup, char *pszMemPath, std::string &strLastError, int &iLastError); + int iLastError; std::string strLastError; private: - std::shared_ptr findMemory(const char *pszName); - int findMemoryIndex(const char *pszName); std::string name; std::string shortDesc; diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 98f67eba..6a8c9d88 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 25) +SET(APP_VERSION_MINOR 26) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/OtmMemoryServiceWorker.cpp b/source/RestAPI/OtmMemoryServiceWorker.cpp index d85a555c..6342f312 100755 --- a/source/RestAPI/OtmMemoryServiceWorker.cpp +++ b/source/RestAPI/OtmMemoryServiceWorker.cpp @@ -273,7 +273,7 @@ void importMemoryProcess( void* pvData ) pData->szError[0] = 0; if(!pData->mem->importDetails){ - pData->mem->importDetails = new ImportStatusDetails; + pData->mem->importDetails = std::make_unique(); } { diff --git a/source/RestAPI/ProxygenHandler.cpp b/source/RestAPI/ProxygenHandler.cpp index 087b42ec..207efe59 100755 --- a/source/RestAPI/ProxygenHandler.cpp +++ b/source/RestAPI/ProxygenHandler.cpp @@ -43,11 +43,11 @@ void ProxygenHandler::onRequest(std::unique_ptr req) noexcept { //time(&startingTime); #endif - builder = new ResponseBuilder(downstream_); + builder = std::make_unique(downstream_); auto methodStr = req->getMethodString (); auto method = req->getMethod (); if(!pRequest){ - pRequest = new UnknownRequestData(); + pRequest = std::make_unique(); } pRequest->strUrl = req->getURL () ; auto path = req->getPath () ; @@ -246,23 +246,13 @@ void ProxygenHandler::onUpgrade(UpgradeProtocol /*protocol*/) noexcept { } void ProxygenHandler::requestComplete() noexcept { - //ResetLogBuffer(); - if(pRequest){ - //TimedMutexGuard l(pRequest->request_mutex); - //while(pRequest->fRunning){}; - delete pRequest; - pRequest = nullptr; - } + pRequest.reset(); delete this; } void ProxygenHandler::onError(ProxygenError /*err*/) noexcept { - if(pRequest){ - //while(pRequest->fRunning){}; - //TimedMutexGuard l(pRequest->request_mutex); - delete pRequest; - pRequest = nullptr; - } + + pRequest.reset(); delete this; } @@ -346,7 +336,7 @@ void ProxygenHandler::sendResponse()noexcept{ } if(COMMAND::EXPORT_MEM_TMX_STREAM == pRequest->command && pRequest->isSuccessful()){ // Add headers to the response std::stringstream contDisposition; - ExportRequestData* exp_request = (ExportRequestData*) pRequest; + ExportRequestData* exp_request = (ExportRequestData*) pRequest.get(); builder->header("Content-Type", "application/octet-stream"); contDisposition << "attachment; filename=\"" << pRequest->strMemName << ".tmx\""; @@ -373,7 +363,7 @@ void ProxygenHandler::sendResponse()noexcept{ ) ){ // Respond with OK status - pRequest->sendStreamFile(builder); + pRequest->sendStreamFile(builder.get()); }else if(pRequest->outputMessage.size()){ builder->body(pRequest->outputMessage); builder->sendWithEOM(); @@ -383,6 +373,7 @@ void ProxygenHandler::sendResponse()noexcept{ } } pRequest->fRunning = false; + pRequest.reset(); T5Logger::GetInstance()->ResetLogBuffer(); if(FLAGS_limit_num_of_active_requests) diff --git a/source/RestAPI/ProxygenHandler.h b/source/RestAPI/ProxygenHandler.h index dde668c9..4970cf50 100755 --- a/source/RestAPI/ProxygenHandler.h +++ b/source/RestAPI/ProxygenHandler.h @@ -43,7 +43,7 @@ std::string iobufToString(const std::unique_ptr& buf); #define TIME_MEASURES class ProxygenHandler : public proxygen::RequestHandler { public: - RequestData *pRequest = nullptr; + std::unique_ptr pRequest; size_t bodyPartsReceived = 0; std::string bodyPart; @@ -68,7 +68,7 @@ class ProxygenHandler : public proxygen::RequestHandler { std::string errorStr; std::string responseText = "WRONG_REQUEST"; - proxygen::ResponseBuilder* builder = nullptr; + std::unique_ptr builder; std::ofstream fileStream; private: diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index 4804e39a..b75e159b 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -148,7 +148,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { if (FLAGS_limit_num_of_active_requests && (++ProxygenStats::getInstance()->activeRequestsCounter >= FLAGS_servicethreads)) { - requestHandler->pRequest = new ErrorRequestData; + requestHandler->pRequest = std::make_unique(); requestHandler->pRequest->buildErrorReturn(503, "Server overloaded. Try again later.", 503); //ProxygenStats::getInstance()->activeRequestsCounter.fetch_sub(1); return requestHandler; @@ -206,12 +206,12 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { if(urlService != serviceName && urlService != additionalServiceName ){ T5LOG(T5ERROR) <<":: Wrong url \'" << urlService << "\', should be \'"<< serviceName << "\' or \'" << additionalServiceName <<"\'"; - requestHandler->pRequest = new UnknownRequestData; + requestHandler->pRequest = std::make_unique(); //ProxygenStats::getInstance()->activeRequestsCounter.fetch_sub(1) return requestHandler; } - requestHandler->pRequest = nullptr; + requestHandler->pRequest.reset(); url = url.size() > urlSeparator ? url.substr(urlSeparator + 1) : ""; @@ -249,21 +249,21 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { if( urlCommand.empty() ){ if( urlMemName.empty()){ if( methodStr == "GET"){ - requestHandler->pRequest = new ListTMRequestData(); + requestHandler->pRequest = std::make_unique(); }else if( methodStr == "POST" ){ std::string contentTypeHeader = headers.getSingleOrEmpty("Content-Type"); bool isMultipart = contentTypeHeader.find("multipart/form-data") != std::string::npos; - requestHandler->pRequest = new CreateMemRequestData(!isMultipart); + requestHandler->pRequest = std::make_unique(!isMultipart); } }else{// mem name is not empty, but command is empty if(methodStr == "POST"){ - //requestHandler->pRequest = new ImportRequestData(); + //requestHandler->pRequest = std::make_uniquepRequest = new ExportRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(methodStr == "DELETE"){ - requestHandler->pRequest = new DeleteMemRequestData(); + requestHandler->pRequest = std::make_unique(); } } }else{ @@ -272,63 +272,62 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { if ( urlService == additionalServiceName ){//for calls %serviceName%_service if( methodStr == "GET"){ if(urlCommand == "shutdown"){ - requestHandler->pRequest = new ShutdownRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "resources"){ - auto request = new ResourceInfoRequestData(); - requestHandler->pRequest = request; + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "flags"){ - requestHandler->pRequest = new FlagsRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "savetms"){ - requestHandler->pRequest = new SaveAllTMsToDiskRequestData(); + requestHandler->pRequest = std::make_unique(); } }else if( methodStr == "POST"){ if(urlCommand == "tagreplacement"){ - requestHandler->pRequest = new TagRemplacementRequestData(); + requestHandler->pRequest = std::make_unique(); } } } if(!urlMemName.empty()){//for command memName should always exists if(methodStr == "POST"){ if(urlCommand == "multifuzzysearch"){ - requestHandler->pRequest = new MultiFuzzySearchRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "multiupdate"){ - requestHandler->pRequest = new MultiUpdateRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "fuzzysearch"){ - requestHandler->pRequest = new FuzzySearchRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "concordancesearch"){ - requestHandler->pRequest = new ConcordanceSearchRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "search"){ - requestHandler->pRequest = new ConcordanceExtendedSearchRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "entry"){ // update - requestHandler->pRequest = new UpdateEntryRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "entrydelete"){ - requestHandler->pRequest = new DeleteEntryRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "entriesdelete"){ - requestHandler->pRequest = new DeleteEntriesReorganizeRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "getentry"){ - requestHandler->pRequest = new GetEntryRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "import"){ - requestHandler->pRequest = new ImportRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "importtmx"){ - //requestHandler->pRequest = new ImportStreamRequestData(); - requestHandler->pRequest = new ImportRequestData(false); + //requestHandler->pRequest = std::make_unique(); + requestHandler->pRequest = std::make_unique(false); }else if(urlCommand == "clone"){ - requestHandler->pRequest = new CloneTMRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "importlocal"){ - requestHandler->pRequest = new ImportLocalRequestData(); + requestHandler->pRequest = std::make_unique(); } }else if(methodStr == "GET"){ if(urlCommand == "status"){ - requestHandler->pRequest = new StatusMemRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "tables"){ - requestHandler->pRequest = new GetTablesMemRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "reorganize"){ - requestHandler->pRequest = new ReorganizeRequestData(); + requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "download.tm"){ - requestHandler->pRequest = new ExportRequestData(EXPORT_MEM_INTERNAL_FORMAT_STREAM); + requestHandler->pRequest = std::make_unique(EXPORT_MEM_INTERNAL_FORMAT_STREAM); }else if(urlCommand == "download.tmx"){ - requestHandler->pRequest = new ExportRequestData(EXPORT_MEM_TMX_STREAM); + requestHandler->pRequest = std::make_unique(EXPORT_MEM_TMX_STREAM); }else if(urlCommand == "flush"){ - requestHandler->pRequest = new FlushMemRequestData(); + requestHandler->pRequest = std::make_unique(); } } } @@ -339,7 +338,7 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { } if(!requestHandler->pRequest){ - requestHandler->pRequest = new UnknownRequestData; + requestHandler->pRequest = std::make_unique(); requestHandler->pRequest->strUrl = url; } diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp index d2609692..32a5fc11 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFMEMLP.cpp @@ -734,7 +734,7 @@ USHORT MemFuncImportProcess pLIDA->fileData = std::move(pData->fileData); } { - if ( !MemLoadStart( &(pData->MemLoadIda), HWND_FUNCIF, pData->mem->importDetails ) ) + if ( !MemLoadStart( &(pData->MemLoadIda), HWND_FUNCIF, pData->mem->importDetails.get() ) ) { if ( pLIDA->hFile ) CloseFile( &(pLIDA->hFile)); pData->usMemLoadRC = UtlQueryUShort( QS_LASTERRORMSGID ); @@ -752,7 +752,7 @@ USHORT MemFuncImportProcess { T5LOG( T5DEVELOP) << "::MEM_IMPORT_TASK, progress = " << pData->mem->importDetails->usProgress; - USHORT usRc = MemLoadProcess( pLIDA, pData->mem->importDetails ); + USHORT usRc = MemLoadProcess( pLIDA, pData->mem->importDetails.get() ); switch ( usRc ) { case MEM_PROCESS_OK: diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 7b1fab0b..1d6919a5 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -694,6 +694,10 @@ void appendLongNameEntry( TMX_LONGNAME_TABLE_ENTRY entry; entry.usId = id; + if(table.pszBuffer.size() < 4){//if in buffer we have only bytes for one id and one delimiter- then erase buff before adding new data + table.pszBuffer.clear(); + } + //table.pszBuffer.push_back(id); if(nullptr == positionInBuffer){//we need to add data to the buffer table.pszBuffer.push_back(static_cast(id & 0xFF)); // low byte @@ -799,6 +803,7 @@ USHORT EqfMemory::NTMReadLongNameTable() // call to obtain exact length of record LONG lKey = LONGNAME_KEY; USHORT usRC = state->TmBtree.EQFNTMGet(lKey, state->LongNames.pszBuffer); + // setup pointer array for long names if (usRC == NO_ERROR) { @@ -824,6 +829,8 @@ USHORT EqfMemory::NTMReadLongNameTable() } } + buffLen = state->LongNames.pszBuffer.size(); + pszTemp = (PSZ)state->LongNames.pszBuffer.data(); while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen - sizeof(USHORT)) && *((PUSHORT)pszTemp) != 0) { // get ID of string diff --git a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp index 21155761..d70438c3 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EqfMemoryPlugin.cpp @@ -197,164 +197,6 @@ USHORT registerPlugins() return usRC; } -/*! \brief Handle a return code from the memory functions and create the approbriate error message text for it - \param iRC return code from memory function - \param pszMemName long memory name - \param pszMarkup markup table name or NULL if not available - \param pszMemPath fully qualified memory path name or NULL if not available - \param strLastError reference to string object receiving the message text - \param iLastError reference to a integer variable receiving the error code - \returns original or modified error return code -*/ -int EqfMemoryPlugin::handleError( int iRC, char *pszMemName, char *pszMarkup, char *pszMemPath, std::string &strLastError, int &iLastError ) -{ - char* pReplAddr[2]; - - if ( iRC == 0 ) return( iRC ); - - char *pszErrorTextBuffer = (char *)malloc(8096); - if ( pszErrorTextBuffer == NULL ) return( iRC ); - pszErrorTextBuffer[0] = '\0'; - - pReplAddr[0] = pszMemName; - - switch ( iRC ) - { - case ERROR_TA_ACC_TAGTABLE: - if (pszMarkup != NULL) - pReplAddr[0] = pszMarkup; - else - strcpy(pReplAddr[0], "n/a"); - UtlGetMsgTxt( ERROR_TA_ACC_TAGTABLE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case ERROR_SHARING_VIOLATION: - case BTREE_DICT_LOCKED: - case BTREE_ENTRY_LOCKED: - case BTREE_ACCESS_ERROR: - UtlGetMsgTxt( ERROR_MEM_NOT_ACCESSIBLE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_NO_MORE_MEMORY_AVAILABLE: - case ERROR_NOT_ENOUGH_MEMORY: - case BTREE_NO_ROOM: - case BTREE_NO_BUFFER: - UtlGetMsgTxt( ERROR_STORAGE, pszErrorTextBuffer, 0, NULL ); - break; - - case VERSION_MISMATCH: - case CORRUPT_VERSION_MISMATCH: - case ERROR_VERSION_NOT_SUPPORTED: - UtlGetMsgTxt( ERROR_MEM_VERSION_NOT_SUPPORTED, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case FILE_MIGHT_BE_CORRUPTED: - case BTREE_CORRUPTED: - case BTREE_USERDATA: - case ERROR_OLD_PROPERTY_FILE: - UtlGetMsgTxt( ITM_TM_NEEDS_ORGANIZE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case BTREE_DISK_FULL: - case BTREE_WRITE_ERROR : - case DISK_FULL: - { - char szDrive[2]; - if ( pszMemPath != NULL ) - { - szDrive[0] = *pszMemPath; - szDrive[1] = '\0'; - } - if (pszMemPath != NULL) - pReplAddr[0] = szDrive; - else - strcpy(pReplAddr[0], "n/a"); - UtlGetMsgTxt( ERROR_DISK_FULL_MSG, pszErrorTextBuffer, 1, pReplAddr ); - } - break; - case DB_FULL: - case BTREE_LOOKUPTABLE_TOO_SMALL: - UtlGetMsgTxt( ERROR_MEM_DB_FULL, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case FILE_ALREADY_EXISTS: - UtlGetMsgTxt( ERROR_MEM_NAME_INVALID, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case NOT_REPLACED_OLD_SEGMENT: - UtlGetMsgTxt( ERROR_MEM_NOT_REPLACED, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TM_FILE_SCREWED_UP: - case NOT_A_MEMORY_DATABASE: - case BTREE_ILLEGAL_FILE: - UtlGetMsgTxt( ERROR_MEM_DESTROYED, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TM_FILE_NOT_FOUND: - case BTREE_FILE_NOTFOUND: - UtlGetMsgTxt( ERROR_TM_FILE_NOT_FOUND, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_TM_OPENED_EXCLUSIVELY: - UtlGetMsgTxt( ERROR_TM_OPENED_EXCLUSIVELY, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_PROP_EXIST: - UtlGetMsgTxt( ERROR_PROP_EXIST, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_PROP_WRITE_ERROR: - UtlGetMsgTxt( ERROR_PROP_WRITE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TM_PROPERTIES_NOT_OPENED: - UtlGetMsgTxt( ERROR_OPEN_TM_PROPERTIES, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case ERROR_NETWORK_ACCESS_DENIED : - case BTREE_NETWORK_ACCESS_DENIED: - UtlGetMsgTxt( ERROR_ACCESS_DENIED_MSG, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case SEGMENT_BUFFER_FULL: - case BTREE_BUFFER_SMALL: - UtlGetMsgTxt( ERROR_MEM_SEGMENT_TOO_LARGE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case ERROR_INTERNAL : - UtlGetMsgTxt( ERROR_INTERNAL, pszErrorTextBuffer, 0, NULL ); - break; - - case ERROR_INTERNAL_PARM: - UtlGetMsgTxt( ERROR_INTERNAL_PARM, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case BTREE_IN_USE: - UtlGetMsgTxt( ERROR_MEM_NOT_ACCESSIBLE, pszErrorTextBuffer, 1, pReplAddr ); - break; - - case TMERR_TM_OPENED_SHARED: - case ERROR_PATH_NOT_FOUND: - case ERROR_INVALID_DRIVE: - case BTREE_INVALID_DRIVE : - default: - { - char szError[20]; - sprintf( szError, "%ld", iRC ); - pReplAddr[1] = szError; - T5LOG(T5ERROR) << ":: error " << iRC; - UtlGetMsgTxt( ERROR_MEM_UNDEFINED, pszErrorTextBuffer, 2, pReplAddr ); - } - } - - strLastError = pszErrorTextBuffer; - iLastError = iRC; - free( pszErrorTextBuffer ); - - return( iRC ); -} - unsigned short getPluginInfo( POTMPLUGININFO pPluginInfo ) { strcpy( pPluginInfo->szName, pszPluginName ); diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index afc38024..508a4576 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -193,6 +193,9 @@ int convertUTCTimeToLong( char *pszDateTime, PLONG plTime ) #include +DECLARE_int32(t5loglevel); + + /*! \brief build return JSON string in case of errors \param _rc_ error return code \param pszErrorMessage error message text @@ -1218,7 +1221,7 @@ int ImportRequestData::execute(){ strcpy( pData->szMemory, strMemName.c_str() ); if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; + mem->importDetails = std::make_unique(); } mem->importDetails->reset(); @@ -1350,7 +1353,7 @@ int ImportLocalRequestData::execute(){ strcpy( pData->szMemory, strMemName.c_str() ); if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; + mem->importDetails = std::make_unique(); } mem->importDetails->reset(); @@ -1674,7 +1677,7 @@ int ReorganizeRequestData::execute(){ pData->sLastFunction = FCT_EQFORGANIZEMEM; pData->pvMemOrganizeCommArea = pCommArea; if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; + mem->importDetails = std::make_unique(); } mem->importDetails->reset(); @@ -1824,7 +1827,7 @@ int DeleteEntriesReorganizeRequestData::execute(){ pData->sLastFunction = FCT_EQFORGANIZEMEM; pData->pvMemOrganizeCommArea = pCommArea; if(mem->importDetails == nullptr){ - mem->importDetails = new ImportStatusDetails; + mem->importDetails = std::make_unique(); } mem->importDetails->reset(); @@ -2534,11 +2537,10 @@ void ShutdownRequestData::CheckImportFlushTmsAndShutdown(int signal, MutexTimeou TMManager::fServiceIsRunning = true; TMManager::fLoadingTmIsAllowed = false; } - //pMemService->closeAll(); - T5Logger::GetInstance()->LogStop(); + if(waitForImportAndReorganizeProcesses){ int j= 3; - while( int i = TMManager::GetInstance()->GetMemImportInProcessCount(tmListTimeout)){ + while(int i = TMManager::GetInstance()->GetMemImportInProcessCount(tmListTimeout)) { if(tmListTimeout.failed()){ tmListTimeout.addToErrMsg(".Failed to lock tm list:", __func__, __LINE__); //return ; @@ -2565,6 +2567,10 @@ void ShutdownRequestData::CheckImportFlushTmsAndShutdown(int signal, MutexTimeou T5LOG(T5ERROR) << "saveTm returned rest code not 200, but " << saveTmRD._rest_rc_ << "; rc = " << saveTmRD._rc_; } } + + FLAGS_v = 2; FLAGS_t5loglevel=0; + T5LOG(T5TRANSACTION) << "Closing t5memory"; + T5Logger::GetInstance()->LogStop(); if(signal != SHUTDOWN_CALLED_FROM_MAIN){ { std::scoped_lock(TMManager::mutex_tmManager); @@ -3006,7 +3012,6 @@ int ExportRequestData::checkData(){ return _rc_; } - DECLARE_int32(t5loglevel); int ExportRequestData::execute(){ if(command != EXPORT_MEM_TMX_STREAM && command != EXPORT_MEM_TMX){ // get a temporary file name for the memory package file or TMX file diff --git a/source/opentm2/core/tm.cpp b/source/opentm2/core/tm.cpp index 92471d17..1d8099b0 100644 --- a/source/opentm2/core/tm.cpp +++ b/source/opentm2/core/tm.cpp @@ -25,9 +25,9 @@ int TMManager::GetMemImportInProcessCount(MutexTimeout& tmListTimeout){ return TM_LIST_MUTEX_TIMEOUT_FAILED; } - for(auto& tm: tms) + for(const auto& tm: tms) { - ImportStatusDetails* pImportDetails = tm.second->importDetails; + ImportStatusDetails* pImportDetails = tm.second->importDetails.get(); if( (tm.second->eImportStatus == IMPORT_RUNNING_STATUS) || (tm.second->eImportStatus == REORGANIZE_RUNNING_STATUS) ) From 44619a34a79567f8ef59b55b16ff3cde93ec6330 Mon Sep 17 00:00:00 2001 From: orestdrag Date: Mon, 18 Aug 2025 21:11:18 +0300 Subject: [PATCH 55/62] * updated to 0.7.27 * fixed 328 - added endpoint to add new entry to the table * sorted tables output by key --- include/lowlevelotmdatastructs.h | 2 +- include/requestdata.h | 7 +- include/tm.h | 24 +-- source/CMakeLists.txt | 2 +- source/RestAPI/ProxygenServer.cpp | 2 + .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 8 +- .../opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp | 54 +++--- source/opentm2/core/requestdata.cpp | 156 ++++++++++++++++-- source/opentm2/core/utilities/EQFDICT.cpp | 1 - 9 files changed, 191 insertions(+), 65 deletions(-) diff --git a/include/lowlevelotmdatastructs.h b/include/lowlevelotmdatastructs.h index da6c962c..3c2f6c91 100644 --- a/include/lowlevelotmdatastructs.h +++ b/include/lowlevelotmdatastructs.h @@ -93,7 +93,6 @@ typedef char* PSZ; DELETE_MEM, EXPORT_MEM_TMX, GET_TABLES, - ADD_TO_TABLE, EXPORT_MEM_INTERNAL_FORMAT, EXPORT_MEM_INTERNAL_FORMAT_STREAM, REORGANIZE_MEM, @@ -109,6 +108,7 @@ typedef char* PSZ; MULTIFUZZY, MULTIUPDATE, CONCORDANCE, + ADD_TO_TABLE, DELETE_ENTRY, GET_ENTRY, DELETE_ENTRIES_REORGANIZE, diff --git a/include/requestdata.h b/include/requestdata.h index a56d5cac..025b9ed5 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -28,6 +28,7 @@ class RequestData{ int requestType = 0; std::string strTempFile; + bool fSuppressBuildErrorLog{}; //RequestData(); // json was parsed in sub class public: //std::recursive_timed_mutex request_mutex; @@ -142,6 +143,9 @@ class CreateMemRequestData: public RequestData{ int execute() override ; }; + + + class GetTablesMemRequestData: public RequestData{ public: GetTablesMemRequestData(): RequestData(COMMAND::GET_TABLES, "", "") {}; @@ -158,7 +162,8 @@ class AddToTableMemRequestData: public RequestData{ protected: int tableKey{}; int entryId{}; - char value[100]{}; + BOOL fOverwrite = false; + char value[256]{}; int parseJSON() override; int checkData() override; int execute() override ; diff --git a/include/tm.h b/include/tm.h index c68cf3df..60be0444 100644 --- a/include/tm.h +++ b/include/tm.h @@ -1970,13 +1970,12 @@ class EqfMemory //: public TMX_CLB bool areLangsMatching(const ushort langId1, const ushort langId2, const char* langStr1);//, bool fPrefferedLang1); USHORT NTMGetIDFromName(PSZ, USHORT, PUSHORT, - LONG options = 0, - PUSHORT pusAlternativeID = nullptr); + LONG options = 0); - USHORT NTMGetNameFromID(const USHORT, USHORT, PSZ, SHORT maxBuffLen = -1); + USHORT NTMGetNameFromID(const USHORT, USHORT, PSZ, SHORT maxBuffLen = -1, bool fSuppressLogsAndResetValueIfNotFound = false); USHORT NTMGetPointersToTable(USHORT, PTMX_TABLE *, PTMX_TABLE_ENTRY *); - USHORT NTMAddNameToTable(PSZ, USHORT, PUSHORT); + USHORT NTMAddNameToTable(PSZ, USHORT, PUSHORT, bool fUseProvidedId = false); USHORT FillClb(BytesRef, OtmProposal &); USHORT ComparePutData( @@ -2189,7 +2188,7 @@ class EqfMemory //: public TMX_CLB USHORT NTMWriteLongNameTable(); USHORT NTMDestroyLongNameTable(); - USHORT AddToLongNameTables(const char* pszLongName,PUSHORT pusID); + USHORT AddToLongNameTables(const char* pszLongName,PUSHORT pusID, bool fGenerateId =true, PSZ positionInReadBuffer = nullptr); USHORT WriteLongNameTablesToTheTM(); USHORT SortLongNameTables(); @@ -2451,21 +2450,6 @@ USHORT TmUpdSegHwndW( HWND hwnd //(in) handle for error messages ); -// structure for MT_TMMERGE pointer -typedef struct _MT_TMMERGE -{ - CHAR chMemory[MAX_EQF_PATH]; // TM name to be merged into - CHAR chSGMLFile[MAX_EQF_PATH]; // external memory - OBJNAME szObjName; // document object name - HWND hwndNotify; // parent to be notified - CHAR szTargetLang[MAX_LANG_LENGTH]; - OBJNAME szFolObjName; // object name of folder -} MT_TMMERGE, *PMT_TMMERGE; - -BOOL MTTMMergeStart( - PLISTCOMMAREA pCommArea, - PMT_TMMERGE pTMMerge); - // return codes of function MemConvertMem // memory converted successfully diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 6a8c9d88..df72de00 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 26) +SET(APP_VERSION_MINOR 27) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/ProxygenServer.cpp b/source/RestAPI/ProxygenServer.cpp index b75e159b..772757bc 100755 --- a/source/RestAPI/ProxygenServer.cpp +++ b/source/RestAPI/ProxygenServer.cpp @@ -314,6 +314,8 @@ class ProxygenHandlerFactory : public RequestHandlerFactory { requestHandler->pRequest = std::make_unique(); }else if(urlCommand == "importlocal"){ requestHandler->pRequest = std::make_unique(); + }else if(urlCommand == "addtotable"){ + requestHandler->pRequest = std::make_unique(); } }else if(methodStr == "GET"){ if(urlCommand == "status"){ diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 4ff30ee4..0d007df3 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -690,7 +690,6 @@ USHORT ExactTest LONG lLeftTgtLen; // remaining target length USHORT usGetLang; // id of target language USHORT usGetFile; // id of target file - USHORT usAlternateGetFile; // alternate ID of target file USHORT usOldMatches = *pusMatchEntries; @@ -704,8 +703,7 @@ USHORT ExactTest //get file name id of file name in the getin structure if ( pTmClb->NTMGetIDFromName( pGetIn->szLongName, (USHORT)FILE_KEY, &usGetFile, - NTMGETID_NOUPDATE_OPT, - &usAlternateGetFile )) + NTMGETID_NOUPDATE_OPT )) { usGetFile = 0; } /* endif */ @@ -855,7 +853,7 @@ USHORT ExactTest // check for matching document names // we can compare the document name IDs - fMatchingDocName = ((pClb->usFileId == usGetFile) || (pClb->usFileId == usAlternateGetFile)); + fMatchingDocName = (pClb->usFileId == usGetFile); if ( fIgnoreProposal ) { @@ -899,7 +897,7 @@ USHORT ExactTest { // GQ 2015-04-10 New approach: If we have an exact-exact match use this one, otherwise use timestamp for the comparism BOOL fExactExactNewCLB = fMatchingDocName && (pClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); - BOOL fExactExactExistingCLB = ((pTMXTargetClb->usFileId == usGetFile) || (pTMXTargetClb->usFileId == usAlternateGetFile)) && + BOOL fExactExactExistingCLB = (pTMXTargetClb->usFileId == usGetFile) && (pTMXTargetClb->ulSegmId >= (pGetIn->ulSegmentId - 1)) && (pTMXTargetClb->ulSegmId <= (pGetIn->ulSegmentId + 1)); if ( fExactExactNewCLB && !fExactExactExistingCLB ) { diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp index 1d6919a5..f9db7ff1 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTUTL.cpp @@ -220,8 +220,7 @@ USHORT EqfMemory::NTMGetIDFromName( PSZ pszName, // input, name being looked up USHORT usTableType, // input, type of table to use PUSHORT pusID, // output, ID for name being looked up - LONG lOptions, - PUSHORT pusAlternativeID // output, alternative ID + LONG lOptions ) { USHORT usRc = NO_ERROR; // function return coed @@ -239,9 +238,6 @@ USHORT EqfMemory::NTMGetIDFromName( *pusID = 0; } /* endif */ - if (pusAlternativeID) - *pusAlternativeID = NTMGETID_NOTFOUND_ID; - //--- if input parameters OK if (pszName[0] != EOS) { @@ -419,7 +415,7 @@ USHORT EqfMemory::NTMGetIDFromName( USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput USHORT usTableType, // input PSZ pszName, // output - SHORT maxBuffLen) // output, long name (only for FILE_KEY) + SHORT maxBuffLen, bool fSuppressLogsAndResetValueIfNotFound) // output, long name (only for FILE_KEY) { USHORT usRc = NO_ERROR; BOOL fFound = FALSE; @@ -492,8 +488,13 @@ USHORT EqfMemory::NTMGetNameFromID(const ushort usID, // intput /* set usRc and reset pszName */ /************************************************************/ if(usID){ - LOG_AND_SET_RC_W_INFO(usRc, T5ERROR, ID_NOT_FOUND) << "id = " << usID << "; table = " << usTableType; - snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", usID); // 'hu' = unsigned short + if(fSuppressLogsAndResetValueIfNotFound){ + usRc = ID_NOT_FOUND; + pszName[0] = EOS; + }else{ + LOG_AND_SET_RC_W_INFO(usRc, T5ERROR, ID_NOT_FOUND) << "id = " << usID << "; table = " << usTableType; + snprintf(pszName, sizeof(TMX_TABLE_ENTRY::szName), "%hu", usID); // 'hu' = unsigned short + } }else{ pszName[0] = EOS; } @@ -736,7 +737,7 @@ USHORT EqfMemory::WriteLongNameTablesToTheTM() return usRc; } -USHORT EqfMemory::AddToLongNameTables(const char* pszLongName, PUSHORT pusID) +USHORT EqfMemory::AddToLongNameTables(const char* pszLongName, PUSHORT pusID, bool fGenerateId, PSZ positionInReadBuffer) { if(state->LongNames.stTableEntry.empty()) { @@ -748,13 +749,14 @@ USHORT EqfMemory::AddToLongNameTables(const char* pszLongName, PUSHORT pusID) state->LongNamesCaseIgnore.pszBuffer.clear(); state->LongNamesCaseIgnore.stTableEntry.emplace_back(TMX_LONGNAME_TABLE_ENTRY{}); } - - *pusID = state->LongNames.stTableEntry.size(); + if(fGenerateId){ + *pusID = state->LongNames.stTableEntry.size(); + } std::vector longName(reinterpret_cast(pszLongName), reinterpret_cast(pszLongName + strlen(pszLongName))); auto longNameCaseIgnore = toUpperVector(longName); - appendLongNameEntry(state->LongNames, *pusID, longName); + appendLongNameEntry(state->LongNames, *pusID, longName, positionInReadBuffer); appendLongNameEntry(state->LongNamesCaseIgnore, *pusID, longNameCaseIgnore); SortLongNameTables(); @@ -829,23 +831,20 @@ USHORT EqfMemory::NTMReadLongNameTable() } } - buffLen = state->LongNames.pszBuffer.size(); + buffLen = state->LongNames.pszBuffer.size() - sizeof(USHORT); //we need to have more than 3 bytes just to parse ID and endDelimiter pszTemp = (PSZ)state->LongNames.pszBuffer.data(); - while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen - sizeof(USHORT)) && *((PUSHORT)pszTemp) != 0) + auto startOfBuff = (PSZ)state->LongNames.pszBuffer.data(); + while ( (pszTemp - (PSZ)state->LongNames.pszBuffer.data() < buffLen) && *((PUSHORT)pszTemp) != 0) { // get ID of string USHORT usId = *((PUSHORT)pszTemp); pszTemp += sizeof(USHORT); // skip ID - std::vector longName(reinterpret_cast(pszTemp), - reinterpret_cast(pszTemp + strlen(pszTemp))); - auto longNameCaseIgnore = toUpperVector(longName); - - appendLongNameEntry(state->LongNames, usId, longName, pszTemp); - appendLongNameEntry(state->LongNamesCaseIgnore, usId, longNameCaseIgnore); auto len = strlen(pszTemp); + AddToLongNameTables(pszTemp, &usId, false, pszTemp); + pszTemp += len; pszTemp += 1; // skip delimiter - if(!*pszTemp){// skip delimiter in tms from0.7.0-0.7.22 + if(!*pszTemp){// skip delimiter in tms from 0.7.0-0.7.22 pszTemp++; } } /* endwhile */ @@ -908,7 +907,8 @@ int tmxTableToBuffer(PTMX_TABLE pTable, BytesRef buff){ USHORT EqfMemory::NTMAddNameToTable( PSZ pszName, // input USHORT usTableType, // input - PUSHORT pusID // output + PUSHORT pusID, // output + bool fUseProvidedId ) { USHORT usRc = NO_ERROR; // function return coed @@ -920,7 +920,9 @@ USHORT EqfMemory::NTMAddNameToTable( /********************************************************************/ /* initialize ID, that in error case a 0 - ID is returned */ /********************************************************************/ - *pusID = 0; + if(!fUseProvidedId){ + *pusID = 0; + } //--- if input parameters OK if (pszName[0] != EOS) @@ -930,7 +932,7 @@ USHORT EqfMemory::NTMAddNameToTable( /* table type */ /******************************************************************/ if(usTableType == FILE_KEY || usTableType == LONGNAME_KEY){ - AddToLongNameTables(pszName, pusID); + AddToLongNameTables(pszName, pusID, !fUseProvidedId);// !fUseProvidedId=> generateNewId }else{ usRc = NTMGetPointersToTable(usTableType, &pstTMTable, @@ -950,7 +952,9 @@ USHORT EqfMemory::NTMAddNameToTable( } TMX_TABLE_ENTRY& newEntry = pstTMTable->table.back(); - if(*pusID == 0) *pusID = pstTMTable->table.size(); + if(*pusID == 0){ + *pusID = pstTMTable->table.size(); + } newEntry.usId = *pusID; strncpy(newEntry.szName, pszName, MAX_LANG_LENGTH - 1); pstTMTable->table.push_back(TMX_TABLE_ENTRY()); diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 508a4576..0236e48f 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -228,7 +228,9 @@ int RequestData::buildErrorReturn json_factory.addParmToJSON( outputMessage, "ErrorMsg", pszErrorMsg ); json_factory.terminateJSON( outputMessage ); _rest_rc_ = rest_rc; - T5LOG_RC(_rc_, outputMessage); + if(!fSuppressBuildErrorLog){ + T5LOG_RC(_rc_, outputMessage); + } return( rc ); } @@ -2325,23 +2327,23 @@ int GetTablesMemRequestData::execute() { } json_factory.addParmToJSON( outputMessage, "tableName(additional info)(INTERNAL_KEY_OF_THE_TABLE)", "position)idInTable: value;"); - std::string tableName = "tagTablesTable(" + toStr(TAGTABLE_KEY) + ")"; - json_factory.addParmToJSON( outputMessage, tableName, tagTablesTable); - - tableName = "langTable(" + toStr(LANG_KEY) + ")"; + std::string tableName = "langTable(" + toStr(LANG_KEY) + ")"; json_factory.addParmToJSON( outputMessage, tableName, langTable); - tableName = "langGroupTable(virtual)(" + toStr(LANGGROUP_KEY) + ")"; - json_factory.addParmToJSON( outputMessage, tableName, langGroupTable); + tableName = "filenames(long)(" + toStr(LONGNAME_KEY) +"-physical, (" + toStr(FILE_KEY)+ "-use this for commands))"; + json_factory.addParmToJSON( outputMessage, tableName, longNames); + + tableName = "filenames(long, caseign, virtual)(" + toStr(LONGNAME_KEY)+", (" + toStr(FILE_KEY) + "))"; + json_factory.addParmToJSON( outputMessage, tableName, longNamesCaseIgnore); tableName = "authors(" + toStr(AUTHOR_KEY) + ")"; json_factory.addParmToJSON( outputMessage, tableName, authorsTable); - tableName = "filenames(long)(" + toStr(LONGNAME_KEY) + ")"; - json_factory.addParmToJSON( outputMessage, tableName, longNames); + tableName = "tagTablesTable(" + toStr(TAGTABLE_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, tagTablesTable); - tableName = "filenames(long, caseign, virtual)(" + toStr(LONGNAME_KEY) + ")"; - json_factory.addParmToJSON( outputMessage, tableName, longNamesCaseIgnore); + tableName = "langGroupTable(virtual)(" + toStr(LANGGROUP_KEY) + ")"; + json_factory.addParmToJSON( outputMessage, tableName, langGroupTable); tableName = "langIdToGroupTable"; json_factory.addParmToJSON( outputMessage, tableName, langIdToGroupTable); @@ -2351,6 +2353,138 @@ int GetTablesMemRequestData::execute() { }; +int AddToTableMemRequestData::parseJSON(){ + std::wstring strInputParmsW = EncodingHelper::convertToWChar( strBody ); + // parse input parameters + auto loggingThreshold = -1; + + JSONFactory::JSONPARSECONTROL parseControl[] = { + { L"loggingThreshold", JSONFactory::INT_PARM_TYPE , &(loggingThreshold), -1}, + { L"tableKey", JSONFactory::INT_PARM_TYPE , &(tableKey), 0 }, + { L"entryId", JSONFactory::INT_PARM_TYPE , &(entryId), 0 }, + { L"value", JSONFactory::ASCII_STRING_PARM_TYPE, &(value), sizeof(value) }, + { L"", JSONFactory::ASCII_STRING_PARM_TYPE, NULL, 0 } }; + _rc_ = json_factory.parseJSON( strInputParmsW, parseControl ); + + if(_rc_){ + return buildErrorReturn( _rc_, "::json parsing failed", BAD_REQUEST); + } + + return _rc_; +} + +int AddToTableMemRequestData::checkData(){ + //check tableKey + switch(tableKey){ + case LANG_KEY: + case AUTHOR_KEY: + case FILE_KEY: + //case TAGTABLE_KEY: + //case LONGNAME_KEY: + { + //if(tableKey == LONGNAME_KEY){ + // tableKey = FILE_KEY; + //} + //okay, go to the next check + break; + }; + + default:{ + std::string errMsg = "Invalid table key(" + std::to_string(tableKey) + "); available keys = LANG_KEY(" + std::to_string(LANG_KEY) + + "), FILE_KEY(" + std::to_string(FILE_KEY) + "), AUTHOR_KEY(" + std::to_string(AUTHOR_KEY) + ")";///, TAGTABLE_KEY(" + std::to_string(TAGTABLE_KEY) + ")"; + return buildErrorReturn(BAD_REQUEST, errMsg.c_str(), BAD_REQUEST); + }; + } + + if(entryId<=0 || entryId >=NTMGETID_NOTFOUND_ID){ + std::string errMsg = "Value for the entryId should be between " + std::to_string(1) + " and " + std::to_string(NTMGETID_NOTFOUND_ID-1); + return buildErrorReturn(BAD_REQUEST, errMsg.c_str(), BAD_REQUEST); + + } + + //if(!value[0]){ + // return buildErrorReturn(BAD_REQUEST, "Value field is empty", BAD_REQUEST); + //}else + { + const int len = strlen(value); + const int maxSize = tableKey == FILE_KEY? 255 : (sizeof(TMX_TABLE_ENTRY::szName) - 1); + if( len > maxSize){ + std::string errMsg = "Value for the entry is too long(" + std::to_string(len) + ") for selected table, max supported len = " + std::to_string(maxSize); + return buildErrorReturn(BAD_REQUEST, errMsg.c_str(), BAD_REQUEST); + } + } + + return 0; +} + +int AddToTableMemRequestData::execute(){ + //if tm is loaded + if(!mem || !mem->isLoaded()){ + return buildErrorReturn(BAD_REQUEST, "Tm is not loaded", BAD_REQUEST); + } + //if table have that id already + //bool fEntryExists = false; + USHORT usRc = 0; + fSuppressBuildErrorLog = true; + + { + char buffer[256]{}; + usRc = mem->NTMGetNameFromID( entryId, (USHORT) tableKey, + buffer, sizeof(buffer), true ); + if(usRc && usRc != ID_NOT_FOUND){ + std::string msg = "GetNameFromId returned error: " + std::to_string(usRc); + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + }else if(buffer[0]){ + std::string msg = "Id match in Table " + std::to_string(tableKey) +" already have entry with matching id = " + + std::to_string(entryId) + " and value = \""s + buffer +"\";"s; + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + } + } + + + //check value + if(false /*fCheckValueMatch*/){ + USHORT idFromTable = 0; + usRc = mem->NTMGetIDFromName( value, + (USHORT)tableKey, &idFromTable, + NTMGETID_NOUPDATE_OPT); + if (usRc && usRc != ID_NOT_FOUND){ + std::string msg = "GetNameFromId returned error: " + std::to_string(usRc); + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + }else if(idFromTable){ + char buffer[256]{}; + usRc = mem->NTMGetNameFromID( idFromTable, (USHORT) tableKey, + buffer, sizeof(buffer) ); + if(usRc && usRc != ID_NOT_FOUND){ + std::string msg = "GetNameFromId returned error: " + std::to_string(usRc); + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + }else{ + std::string msg = "Value match Table " + std::to_string(tableKey) +" already have entry with id = " + + std::to_string(idFromTable) + " and value = \""s + buffer +"\";"s; + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + } + } + } + + if(!value[0]){ + return buildErrorReturn(BAD_REQUEST, "Value field is empty", BAD_REQUEST); + } + + //if present-> check overwrite flag + USHORT id = entryId; + //if(!fEntryExists || fOverwrite){ + usRc = mem->NTMAddNameToTable(value, tableKey, &id, true); + //} + if(usRc){ + std::string msg = "AddNameToTable returned error: " + std::to_string(usRc); + return buildErrorReturn(BAD_REQUEST, msg.c_str(), BAD_REQUEST); + } + std::string msg = "Added entry to the table"+ std::to_string(tableKey) + ", id = " + std::to_string(id) + "; value = \\\""s + value + "\\\""; + outputMessage = "{\n\t\"msg\": \"" + msg + "\"\n}"; + return _rc_; +} + + #include //#include // Include the flags library #ifdef TEMPORARY_COMMENTED // code to set flags diff --git a/source/opentm2/core/utilities/EQFDICT.cpp b/source/opentm2/core/utilities/EQFDICT.cpp index 9133511e..7f83f7ec 100755 --- a/source/opentm2/core/utilities/EQFDICT.cpp +++ b/source/opentm2/core/utilities/EQFDICT.cpp @@ -5544,7 +5544,6 @@ SHORT QDAMGetszKeyParam_V3 PCHAR pData = NULL; std::shared_ptr pRecord; // active record SHORT sRc = 0; // return code - PBTREE pBT = pBTIda; USHORT usLen; sRc = pBTIda->QDAMReadRecord_V3( recKey.usNum, pRecord, FALSE ); From d7eac760a062176afdc90d487d15cf03c6f4b0fc Mon Sep 17 00:00:00 2001 From: orestdrag Date: Tue, 19 Aug 2025 17:30:37 +0300 Subject: [PATCH 56/62] * updated to 0.7.28 + added needToReorganize handling for missing id's in the tables + added callers ip and port for unknownRequests --- include/requestdata.h | 4 +- source/CMakeLists.txt | 2 +- source/RestAPI/ProxygenHandler.cpp | 15 +++++- .../opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp | 4 +- .../opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp | 33 +++++++----- .../opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp | 53 +++++++++++-------- source/opentm2/core/requestdata.cpp | 2 +- 7 files changed, 72 insertions(+), 41 deletions(-) diff --git a/include/requestdata.h b/include/requestdata.h index 025b9ed5..7cbd9600 100644 --- a/include/requestdata.h +++ b/include/requestdata.h @@ -398,12 +398,14 @@ class UnknownRequestData: public RequestData{ public: UnknownRequestData(): RequestData(UNKNOWN_COMMAND){}; + std::string methodStr,srcAddress; + protected: int parseJSON() override {return 0;}; int checkData() override {return 0;}; int execute() override { _rest_rc_ = 404; - std::string msg = "Url \"" + strUrl + "\" was not parsed correctly"; + std::string msg = "Called from " + srcAddress + "; Method str:\"" + methodStr +"\"; Url:\"" + strUrl + "\" was not parsed correctly"; return buildErrorReturn(404, msg.c_str(), 404); }; }; diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index df72de00..2afaad53 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 27) +SET(APP_VERSION_MINOR 28) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/RestAPI/ProxygenHandler.cpp b/source/RestAPI/ProxygenHandler.cpp index 207efe59..e5f55c79 100755 --- a/source/RestAPI/ProxygenHandler.cpp +++ b/source/RestAPI/ProxygenHandler.cpp @@ -45,10 +45,23 @@ void ProxygenHandler::onRequest(std::unique_ptr req) noexcept { builder = std::make_unique(downstream_); auto methodStr = req->getMethodString (); - auto method = req->getMethod (); + auto method = req->getMethod (); + + // Client address + auto& addr = req->getClientAddress(); + auto ipStr = addr.getAddressStr(); // returns std::string with IP + auto port = addr.getPort(); // client port (uint16_t) + + if(!pRequest){ pRequest = std::make_unique(); } + if(COMMAND::UNKNOWN_COMMAND == pRequest->command){ + UnknownRequestData* unknownRequest = (UnknownRequestData*)pRequest.get(); + unknownRequest->methodStr = methodStr; + unknownRequest->srcAddress = ipStr + ":"s + toStr(port); + } + pRequest->strUrl = req->getURL () ; auto path = req->getPath () ; auto queryString = req->getQueryString () ; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp index a694751c..9bfe35ee 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTDEL.cpp @@ -356,7 +356,7 @@ USHORT EqfMemory::FindTargetAndDelete( pClb, TmDel.pInputSentence->pStrings->getGenericTargetStr(), &lSrcLen, pTmExtOut, endOfMemRegion ); - NTMGetNameFromID(pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + auto langKeyRc = NTMGetNameFromID(pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage ); pTmExtOut->ulRecKey = lKey; TmDel.currentInternalKey.setInternalKey(lKey, usTarget); @@ -569,7 +569,7 @@ USHORT EqfMemory::FindTargetByKeyAndDelete( pTargetClb, pSourceString, &lSourceLen, pTmExtOut, endOfMemRegion ); - NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + auto langKeyRc = NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage ); TmDel.currentInternalKey.setInternalKey(lKey, usTarget); pTmExtOut->ulRecKey = lKey; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp index 108b21a1..d3c10237 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTEXT.cpp @@ -286,7 +286,7 @@ USHORT EqfMemory::ExtractRecord pTargetClb, pSourceString, &lSourceLen, pTmExtOut, endOfMemRegion); - NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, + auto srcLangKeyRc = NTMGetNameFromID( pTMXSourceRecord->usLangId, (USHORT)LANG_KEY, pTmExtOut->szOriginalSourceLanguage ); if ( ! usRc ) { @@ -451,18 +451,25 @@ USHORT EqfMemory::FillExtStructure strcpy( pstExt->szAuthorName, OVERFLOW_NAME ); } /* endif */ - //fill in the target file name - NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, - pstExt->szLongName, sizeof(pstExt->szLongName) ); - - //fill in the target author - NTMGetNameFromID( pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, - pstExt->szAuthorName ); - - //fill in the target language - NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, - pstExt->szTargetLanguage ); - + { + //fill in the target file name + auto fileKeyRc = NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, + pstExt->szLongName, sizeof(pstExt->szLongName) ); + + //fill in the target author + auto authorKeyRc = NTMGetNameFromID( pTMXTargetClb->usAuthorId, (USHORT)AUTHOR_KEY, + pstExt->szAuthorName ); + + //fill in the target language + auto langKeyRc = NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, + pstExt->szTargetLanguage ); + if( ID_NOT_FOUND == fileKeyRc + || ID_NOT_FOUND == authorKeyRc + //|| ID_NOT_FOUND == langKeyRc //skip lang check because reorganize would delete that records + ){ + state->fCorrupted = true;//reorganize requested + } + } //fill in the segment id pstExt->ulSourceSegmentId = pTMXTargetClb->ulSegmId; diff --git a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp index 0d007df3..a5a82cd9 100755 --- a/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp +++ b/source/opentm2/core/EqfMemoryPlugin/EQFNTGET.cpp @@ -1285,7 +1285,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct //fill in the markup table strcpy(SubstProp.szPropTagTable, "OTMXUXLF"); //fill in the target language - pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, + auto targetKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, (USHORT)LANG_KEY, SubstProp.szTargetLanguage ); } /* endif */ @@ -1314,7 +1314,7 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct CHAR szTgtLongName[ MAX_LONGFILESPEC ]; // long target file name //fill in the target file name - pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, szTgtLongName, sizeof(szTgtLongName) ); + auto targetKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, (USHORT)FILE_KEY, szTgtLongName, sizeof(szTgtLongName) ); /****************************************************************/ /* as the matching levels are the same determine the most recent*/ @@ -1419,28 +1419,37 @@ USHORT FillMatchTable( EqfMemory* pTmClb, //ptr to ctl block struct memcpy( pstMatchTable->szTarget, pString.data(), pString.size() * sizeof(CHAR_W)); pstMatchTable->szTarget[pString.size()] = '\0'; + //fill in the markup table + strcpy(pstMatchTable->szTagTable, "OTMXUXLF"); + if ( usRc == NO_ERROR ) { - //fill in the target file name - pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, - (USHORT)FILE_KEY, - pstMatchTable->szLongName, sizeof(pstMatchTable->szLongName)); - //fill in the target author - don't care about author name.... - pTmClb->NTMGetNameFromID( pTMXTargetClb->usAuthorId, - (USHORT)AUTHOR_KEY, - pstMatchTable->szTargetAuthor ); - - //fill in the markup table - strcpy(pstMatchTable->szTagTable, "OTMXUXLF"); - - //fill in the target and original src language - pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, - (USHORT)LANG_KEY, - pstMatchTable->szTargetLanguage ); - - pTmClb->NTMGetNameFromID( usSrcLangId, - (USHORT)LANG_KEY, - pstMatchTable->szOriginalSrcLanguage ); + { + //fill in the target file name + auto fileKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usFileId, + (USHORT)FILE_KEY, + pstMatchTable->szLongName, sizeof(pstMatchTable->szLongName)); + //fill in the target author - don't care about author name.... + auto authorKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usAuthorId, + (USHORT)AUTHOR_KEY, + pstMatchTable->szTargetAuthor ); + + //fill in the target and original src language + auto targetLangKeyRc = pTmClb->NTMGetNameFromID( pTMXTargetClb->usLangId, + (USHORT)LANG_KEY, + pstMatchTable->szTargetLanguage ); + + auto srcLangKeyRc = pTmClb->NTMGetNameFromID( usSrcLangId, + (USHORT)LANG_KEY, + pstMatchTable->szOriginalSrcLanguage ); + if( ID_NOT_FOUND == fileKeyRc + || ID_NOT_FOUND == authorKeyRc + //|| ID_NOT_FOUND == langKeyRc //skip lang check because reorganize would delete that records + ){ + pTmClb->state->fCorrupted = true;//reorganize requested + } + + } //fill in the segment id pstMatchTable->ulSegmentId = pTMXTargetClb->ulSegmId; //state whether machine translation diff --git a/source/opentm2/core/requestdata.cpp b/source/opentm2/core/requestdata.cpp index 0236e48f..076ca178 100644 --- a/source/opentm2/core/requestdata.cpp +++ b/source/opentm2/core/requestdata.cpp @@ -396,7 +396,7 @@ int RequestData::requestTM(){ && GET_TABLES != command) ) { - return buildErrorReturn(504, "Memory is corrupted, so it can only be tmx-exported or reorganized", 504); + return buildErrorReturn(504, "Memory is corrupted, so it can only be tmx-exported or reorganized; BTREE_CORRUPTED, rc = 5008; ", 504); // 5008 = BTREE_CORRUPTED- call reorganize } if( command == EXPORT_MEM_TMX From 44e3059c0120bd6c4da8fc22d54bbebbff35259e Mon Sep 17 00:00:00 2001 From: orestdrag Date: Thu, 21 Aug 2025 16:02:05 +0300 Subject: [PATCH 57/62] * updated to 0.7.29 - removed 136 files from project --- source/CMakeLists.txt | 2 +- .../OtmMarkupTablePlugin/CMakeLists.txt | 12 - .../OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.CHR | 26 - .../OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBL | Bin 10267 -> 0 bytes .../OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBX | 873 --- .../OtmMarkupTablePlugin/OTMAMRI/otmamri.mak | 55 - .../OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBL | Bin 778 -> 0 bytes .../OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBX | 20 - .../OTMASCII/OTMASCII.TBL | Bin 778 -> 0 bytes .../OTMASCII/OTMASCII.TBX | 20 - .../OTMASCII/OTMUTF16.TBL | Bin 778 -> 0 bytes .../OTMASCII/OTMUTF16.TBX | 19 - .../OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBL | Bin 778 -> 0 bytes .../OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBX | 20 - .../OTMDQUOT/OTMADQUO.CHR | 26 - .../OTMDQUOT/OTMADQUO.TBL | Bin 1911 -> 0 bytes .../OTMDQUOT/OTMADQUO.TBX | 157 - .../OTMDQUOT/OTMDQUOT.TBL | Bin 1294 -> 0 bytes .../OTMDQUOT/OTMDQUOT.TBX | 81 - .../OTMDQUOT/OTMUDQUO.CHR | 27 - .../OTMDQUOT/OTMUDQUO.TBL | Bin 1294 -> 0 bytes .../OTMDQUOT/OTMUDQUO.TBX | 81 - .../OTMHTM32/OTMHTM32.TBL | Bin 42455 -> 0 bytes .../OTMHTM32/OTMHTM32.TBX | 6246 ----------------- .../OTMHTM32/OTMUHTM3.TBL | Bin 42455 -> 0 bytes .../OTMHTM32/OTMUHTM3.TBX | 6246 ----------------- .../OtmMarkupTablePlugin/OTMHTM32/entity.c | 1533 ---- .../OtmMarkupTablePlugin/OTMHTM32/entity.h | 30 - .../OtmMarkupTablePlugin/OTMHTM32/htmlscri.c | 205 - .../OtmMarkupTablePlugin/OTMHTM32/otmhtm32.c | 1957 ------ .../OtmMarkupTablePlugin/OTMHTM32/otmhtm32.h | 38 - .../OTMHTM32/otmhtm32.mak | 80 - .../OtmMarkupTablePlugin/OTMHTM32/scrptseg.c | 1090 --- .../OtmMarkupTablePlugin/OTMHTM32/scrptseg.h | 24 - .../OTMJDK21/OTMAJDK2.TBL | Bin 25102 -> 0 bytes .../OTMJDK21/OTMJDK21.TBL | Bin 25102 -> 0 bytes .../OTMJDK21/OTMNJDK2.TBL | Bin 25242 -> 0 bytes .../OTMJDK21/OTMUJDK2.TBL | Bin 25102 -> 0 bytes .../OtmMarkupTablePlugin/OTMJDK21/context.c | 164 - .../OtmMarkupTablePlugin/OTMJDK21/javapro.c | 1872 ----- .../OtmMarkupTablePlugin/OTMJDK21/javapro.h | 45 - .../OtmMarkupTablePlugin/OTMJDK21/javaseg.c | 1448 ---- .../OtmMarkupTablePlugin/OTMJDK21/javaseg.h | 33 - .../OTMJDK21/otmajdk2.tbx | 3247 --------- .../OtmMarkupTablePlugin/OTMJDK21/otmjdk11.c | 691 -- .../OtmMarkupTablePlugin/OTMJDK21/otmjdk11.h | 73 - .../OTMJDK21/otmjdk21.mak | 81 - .../OTMJDK21/otmjdk21.tbx | 3247 --------- .../OTMJDK21/otmnjdk2.tbx | 3268 --------- .../OTMJDK21/otmujdk2.tbx | 3247 --------- .../OtmMarkupTablePlugin/OTMMSOFC/CHECK.CPP | 549 -- .../OTMMSOFC/OTMMSOFC.TBL | Bin 3347 -> 0 bytes .../OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c | 125 - .../OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.h | 283 - .../OTMMSOFC/OTMMSOFC.mak | 73 - .../OTMMSOFC/OTMMSOFC.tbx | 326 - .../OtmMarkupTablePlugin/OTMMSOFC/PARSE.CPP | 4126 ----------- .../OtmMarkupTablePlugin/OTMMarkupAllTBL.mak | 84 - .../OTMQUOTE/OTMAQUOT.CHR | 26 - .../OTMQUOTE/OTMAQUOT.TBL | Bin 1264 -> 0 bytes .../OTMQUOTE/OTMAQUOT.TBX | 76 - .../OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.C | 338 - .../OTMQUOTE/OTMQUOTE.CHR | 26 - .../OTMQUOTE/OTMQUOTE.TBL | Bin 1294 -> 0 bytes .../OTMQUOTE/OTMQUOTE.TBX | 80 - .../OTMQUOTE/OTMUQUOT.CHR | 27 - .../OTMQUOTE/OTMUQUOT.TBL | Bin 1293 -> 0 bytes .../OTMQUOTE/OTMUQUOT.TBX | 80 - .../OTMQUOTE/otmquote.mak | 61 - .../OtmMarkupTablePlugin/OTMRTF/OTMRTF.CHR | 220 - .../OtmMarkupTablePlugin/OTMRTF/OTMRTF.H | 1967 ------ .../OtmMarkupTablePlugin/OTMRTF/OTMRTF.TBL | Bin 1766 -> 0 bytes .../OtmMarkupTablePlugin/OTMRTF/OTMRTF.TBX | 132 - .../OtmMarkupTablePlugin/OTMRTF/otmrtf.mak | 56 - .../OTMXHTML/OTMXAHTM.TBL | Bin 11028 -> 0 bytes .../OTMXHTML/OTMXAHTM.XML | 279 - .../OTMXHTML/OTMXUHTM.TBL | Bin 11028 -> 0 bytes .../OTMXHTML/OTMXUHTM.XML | 279 - .../OTMXHTML/otmxahtm.tbx | 1539 ---- .../OTMXHTML/otmxuhtm.tbx | 1539 ---- .../OtmMarkupTablePlugin/OTMXML/OTMXAXML.TBL | Bin 1100 -> 0 bytes .../OtmMarkupTablePlugin/OTMXML/OTMXML.CTL | 17 - .../OtmMarkupTablePlugin/OTMXML/OTMXML.LCL | 60 - .../OtmMarkupTablePlugin/OTMXML/OTMXML.TBL | Bin 1100 -> 0 bytes .../OtmMarkupTablePlugin/OTMXML/OTMXUXML.TBL | Bin 1100 -> 0 bytes .../OtmMarkupTablePlugin/OTMXML/PARSE.CPP | 2325 ------ .../OtmMarkupTablePlugin/OTMXML/otmxaxml.tbx | 73 - .../OtmMarkupTablePlugin/OTMXML/otmxml.c | 565 -- .../OtmMarkupTablePlugin/OTMXML/otmxml.mak | 75 - .../OtmMarkupTablePlugin/OTMXML/otmxml.tbx | 73 - .../OtmMarkupTablePlugin/OTMXML/otmxuxml.tbx | 73 - .../OtmMarkupTablePlugin/OTMXML/parse.h | 22 - .../OtmMarkupTablePlugin/OTMXML/state.cpp | 4753 ------------- .../OtmMarkupTablePlugin/OTMXML/state.h | 580 -- .../OtmMarkupTablePlugin/OTMXML/token.h | 65 - .../OTMXMODC/OTMOPNDC.XML | 982 --- .../OTMXMODC/OTMXMODC.TBL | Bin 1847 -> 0 bytes .../OtmMarkupTablePlugin/OTMXMODC/UNZIP.EXE | Bin 167936 -> 0 bytes .../OtmMarkupTablePlugin/OTMXMODC/ZIP.EXE | Bin 126976 -> 0 bytes .../OtmMarkupTablePlugin/OTMXMODC/export.cpp | 141 - .../OtmMarkupTablePlugin/OTMXMODC/export.h | 23 - .../OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c | 590 -- .../OtmMarkupTablePlugin/OTMXMODC/otmxmodc.h | 51 - .../OTMXMODC/otmxmodc.mak | 73 - .../OTMXMODC/otmxmodc.tbx | 165 - .../OtmMarkupTablePlugin/OTMXMODC/parse.cpp | 203 - .../OtmMarkupTablePlugin/OTMXMODC/parse.h | 23 - .../OTMXMXLF/OTMSDLXLF.XML | 168 - .../OTMXMXLF/OTMXAXLF.TBL | Bin 3660 -> 0 bytes .../OTMXMXLF/OTMXLIFF.XML | 170 - .../OTMXMXLF/OTMXMXLF.TBL | Bin 3660 -> 0 bytes .../OTMXMXLF/OTMXUSDL.TBL | Bin 3660 -> 0 bytes .../OTMXMXLF/OTMXUXLF.TBL | Bin 3660 -> 0 bytes .../OTMXMXLF/otmxaxlf.tbx | 456 -- .../OTMXMXLF/otmxmxlf.tbx | 456 -- .../OTMXMXLF/otmxusdl.tbx | 447 -- .../OTMXMXLF/otmxuxlf.tbx | 456 -- .../OtmMarkupTablePlugin/OtmMarkupTable.cpp | 459 -- .../OtmMarkupTablePlugin/OtmMarkupTable.h | 279 - .../OtmMarkupTablePlugin.cpp | 893 --- .../OtmMarkupTablePlugin.h | 194 - .../OtmMarkupTablePlugin.xml | 336 - .../common/CMakeLists.txt | 20 - .../OtmMarkupTablePlugin/common/common.mak | 62 - .../OtmMarkupTablePlugin/common/eqfcalls.c | 269 - .../OtmMarkupTablePlugin/common/eqfcppom.lib | Bin 74277 -> 0 bytes .../OtmMarkupTablePlugin/common/reseq.c | 149 - .../OtmMarkupTablePlugin/common/reseq.h | 18 - .../OtmMarkupTablePlugin/common/unicode.cpp | 563 -- .../OtmMarkupTablePlugin/common/unicode.h | 233 - .../OtmMarkupTablePlugin/common/usrcalls.c | 1372 ---- .../OtmMarkupTablePlugin/common/usrcalls.h | 96 - .../UserMarkupTablePlugin/UserMarkupTable.cpp | 535 -- .../UserMarkupTablePlugin/UserMarkupTable.h | 280 - .../UserMarkupTablePlugin.cpp | 1337 ---- .../UserMarkupTablePlugin.h | 191 - 136 files changed, 1 insertion(+), 68945 deletions(-) delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/CMakeLists.txt delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.CHR delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/otmamri.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.CHR delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.CHR delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMUHTM3.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMUHTM3.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/entity.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/entity.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/htmlscri.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/otmhtm32.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/otmhtm32.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/otmhtm32.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/scrptseg.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/scrptseg.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/OTMAJDK2.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/OTMJDK21.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/OTMNJDK2.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/OTMUJDK2.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/context.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/javapro.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/javapro.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/javaseg.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/javaseg.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmajdk2.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmnjdk2.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmujdk2.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/CHECK.CPP delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/OTMMSOFC.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMSOFC/PARSE.CPP delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMMarkupAllTBL.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMAQUOT.CHR delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMAQUOT.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMAQUOT.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.C delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.CHR delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMQUOTE.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMUQUOT.CHR delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMUQUOT.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/OTMUQUOT.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMQUOTE/otmquote.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/OTMRTF.CHR delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/OTMRTF.H delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/OTMRTF.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/OTMRTF.TBX delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMRTF/otmrtf.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXHTML/OTMXAHTM.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXHTML/OTMXAHTM.XML delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXHTML/OTMXUHTM.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXHTML/OTMXUHTM.XML delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXHTML/otmxahtm.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXHTML/otmxuhtm.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/OTMXAXML.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/OTMXML.CTL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/OTMXML.LCL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/OTMXML.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/OTMXUXML.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/PARSE.CPP delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/otmxaxml.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/otmxml.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/otmxml.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/otmxml.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/otmxuxml.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/parse.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/state.cpp delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/state.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXML/token.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/OTMOPNDC.XML delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/OTMXMODC.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/UNZIP.EXE delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/ZIP.EXE delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/export.cpp delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/export.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/otmxmodc.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.cpp delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMODC/parse.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/OTMSDLXLF.XML delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/OTMXAXLF.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/OTMXLIFF.XML delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/OTMXMXLF.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/OTMXUSDL.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/OTMXUXLF.TBL delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/otmxaxlf.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/otmxmxlf.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/otmxusdl.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMXMXLF/otmxuxlf.tbx delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTable.cpp delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTable.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.cpp delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/OtmMarkupTablePlugin.xml delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/CMakeLists.txt delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/common.mak delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/eqfcalls.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/eqfcppom.lib delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/reseq.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/reseq.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/unicode.cpp delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/unicode.h delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/usrcalls.c delete mode 100755 source/opentm2/plugins/markup/OtmMarkupTablePlugin/common/usrcalls.h delete mode 100755 source/opentm2/plugins/markup/UserMarkupTablePlugin/UserMarkupTable.cpp delete mode 100755 source/opentm2/plugins/markup/UserMarkupTablePlugin/UserMarkupTable.h delete mode 100755 source/opentm2/plugins/markup/UserMarkupTablePlugin/UserMarkupTablePlugin.cpp delete mode 100755 source/opentm2/plugins/markup/UserMarkupTablePlugin/UserMarkupTablePlugin.h diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 2afaad53..a801c01e 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10) SET(APP_VERSION_GLOBAL 0) SET(APP_VERSION_MAJOR 7) -SET(APP_VERSION_MINOR 28) +SET(APP_VERSION_MINOR 29) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/CMakeLists.txt b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/CMakeLists.txt deleted file mode 100755 index 41f81704..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -cmake_minimum_required (VERSION 3.10) -project(OpenTM2_Markup_OtmPlugins) - -add_subdirectory(common) - -set(OpenTM2_Markup_OtmPlugins_Source_Files - OtmMarkupTable.cpp - OtmMarkupTablePlugin.cpp) - - -add_library(OtmPlugins ${OpenTM2_Markup_OtmPlugins_Source_Files}) - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.CHR b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.CHR deleted file mode 100755 index 7ebd20f5..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.CHR +++ /dev/null @@ -1,26 +0,0 @@ -************************************************************** -* Copyright (C) 1998-2013, International Business Machines -* Corporation and others. All rights reserved -************************************************************** -*********************************************** -*** Parser control file for OTMBMRI Parser **** -*********************************************** -* The following switches can be used: -* -* SINGLEQUOTES=YES to enable detection of strings enclosed in single quotes -* SINGLEQUOTES=NO to disable detection of strings enclosed in single quotes -* -* DOUBLEQUOTES=YES to enable detection of strings enclosed in double quotes -* DOUBLEQUOTES=NO to disable detection of strings enclosed in double quotes -* -* ANSI=YES to enable detection of strings enclosed in double quotes -* ANSINO=NO to disable detection of strings enclosed in double quotes -* -* If a switch is not specified it defaults to NO -* -* I no switch has been specifed DOUBLEQUOTES=YES is used as default -* -* Any other switch or command will be ignored -* -DOUBLEQUOTES=YES -ANSI=YES diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBL deleted file mode 100755 index 13f36d23415eadd19b1e64147bf768723b4ffbaa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10267 zcmeI2d5{#<9mn6?_lS{=919D(P!ToY3Id+c)6=ulZcq2n-NPP1R@7Z25{M9j0Sk%2 zi)chC(Ri$uqDD<5QKAwP4Hm8`C8h)qVgzHUc!a3s5lg{Je&5;YneN~0AO7JVNy@F7 z&u)L__3QV3@Ar-w7=2XEFbk4xOBqcO^*=9(K)w7wF-L}<6&r`g#0d>T+}-l(w`S7s z6B-P=kSUnz|3)7l_0Kw*8Z3RO26rr4a?a9~%f~J2UA}b1va@@~^(|W5yZkHZsQ)Xe z}BS4@g6fDi$l!#eQ}=?A9{;^hWQ$q zY4%NKW|}X@jOLrq%sIXl%&hiZ!OYdZer7iN9$@AX-yfM7@IBAW%f5G*dEa-48NdGs zN`S#k80_gt{{&`^_D^NzM1O{vyx(M|(|;~AOZ;n?S?9lsnT`H_W;XjDVCE7305d!N zFEaCr|7~X8^B-cyA2^Is-QD+wz-VU12PQEyC2%q`QovxQ6gZQazQ8hOE(rXXnacv# zGIK-VR%Y%DJjl$pz>~~86L^uCR|5N)IS}}inJ)sNpf?(c;1SG>47M;cDR?|HGlN-X zP7hklbOjeMvoN@VnbpDd%xnnW!pt4P`dtzR2*1x9_7OlbD$jnaxZtVl&en>0@Sb7+uE91<`fP ztdHKnOn>wqX0}APG4ptI7c+aJ2blRVdWac+ED`he{qWdWW+udrXJ%$B$BY*1VCIb2 zxy&qytzu?v>}qDNi}f?JIra-??vFjn%=Xwz%#mrnDzlNFX<99N1UwkVwzl(2Y=BfBDX7AdnS(>_tnM+dFFmrwCHfHWhJ;=x2tZ0K@p^iKD!c<&H zmSxBuQL+s=ugL8gqf=DMkTRvLYGg{bZRi!7v3jOjwqz@{Yl#XDq zZdaulmA*DC)*ez{SystOCC#oC$t#-c232;qxuEK**XTyZFtdtTyI(itl4%*HrD%%m zs#iCPhN|0&nWF+$t$fi`3zFGo*=kIP>@;EC}xcgXO!GZQ!S@TXiF^=YqxrboT|=HSey)?2-=dWS7Rj8 z!!@-i=T$9hD%EEVT8mnw>TOOSsBF%Z3e`4RJ$co(m0DA*o`Nw~(G6S8bvYfedWsTN zY~v{|SE8Eb+O;LkRVY1NPeSULO4gJ*T&22nxk#EmaCv63LYwWQu$g zvs#;~%UUVxbcuwmP%LY6LCK8Z07J4?&>eL(6%@VXY5^@C^+icXNvsKd6HYd1v7A-Pe_7E1S-=W%h znz7Z9M0skgqUgL9i&i$RNDHK`R@bz0Y08tj&`|XBtYkW4S}xaRu^!RVQ!uiUlXoa& zOIEvUsVFPe)|8CoERj%i71Q=cl+u{*F1f+fb_d#Gqm3l38*Gw%tS!%QVa?!CwuTBa zG~X_9^$?i>TM$)vO*`LKHo#WN9O+ zWFg5s&t-B2+W%Bq%{>KmL&~0BvTRD2+TG`6mL;_*?j{vfsi(Z-R@VlY ztZHGZJE;X#UgO<@_6F7V6wwX3ub^5^Yv|7&&+c9pw2rh_np?fL6xt3XIx@K5C?7_K zdZ!#^S4r1#05u|3u2}%Zg90~H%@*xz)h)AJh*qfTj#L(<46D0tb4M|E&}$W924QB` zW?BJ@x=yEGXYoCM8`#;Sjeen^%Pi9n&dVtZK+IAk*m{lNxWpeCb48sn=q$tTO+o+B?EgULZ?wo4p2{Dg8KF-tWWec%Jh>5NA zI-Xur={1dBC(vs;y=KsBCcRD+qS-HIHcSqPQyWeS3#CC4Vh(T?umbpLLq>?3fGxn2 zz&_vuKzx(%4WI>R1!e(rfU|+sz((L+;4xq~@D}h15Ia&9Vl2=KbO8lSkhWujkP3akfy23!kl0&WEQf!l$*TFT$R;eT&b|E!oV`|Cdp>I&=t diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBX deleted file mode 100755 index c5831af3..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/OTMAMRI.TBX +++ /dev/null @@ -1,873 +0,0 @@ - - - - - -OTMBMRI - Windows resource files - OTMAMRI - * - ? - ASCII - NO - - "" - 2 - - - ... - 3 - - - :NONE. - 6 - - - ACCELTABLE - ' ,|\r\n' - - - ASSOCTABLE - ' ,|\r\n' - - - AUTOCHECKBOX - ' ,|\r\n' - - - AUTORADIOBUTTON - ' ,|\r\n' - - - BS_3STATE - ' ,|\r\n' - - - BS_AUTO3STATE - ' ,|\r\n' - - - BS_AUTOCHECKBOX - ' ,|\r\n' - - - BS_AUTORADIOBUTTON - ' ,|\r\n' - - - BS_AUTOSIZE - ' ,|\r\n' - - - BS_BITMAP - ' ,|\r\n' - - - BS_CHECKBOX - ' ,|\r\n' - - - BS_DEFAULT - ' ,|\r\n' - - - BS_HELP - ' ,|\r\n' - - - BS_ICON - ' ,|\r\n' - - - BS_MINIICON - ' ,|\r\n' - - - BS_NOBORDER - ' ,|\r\n' - - - BS_NOCURSORSELECT - ' ,|\r\n' - - - BS_NOPOINTERFOCUS - ' ,|\r\n' - - - BS_PRIMARYSTYLES - ' ,|\r\n' - - - BS_PUSHBUTTON - ' ,|\r\n' - - - BS_RADIOBUTTON - ' ,|\r\n' - - - BS_SYSCOMMAND - ' ,|\r\n' - - - BS_TEXT - ' ,|\r\n' - - - BS_USERBUTTON - ' ,|\r\n' - - - CBS_COMPATIBLE - ' ,|\r\n' - - - CBS_DROPDOWN - ' ,|\r\n' - - - CBS_DROPDOWNLIST - ' ,|\r\n' - - - CBS_SIMPLE - ' ,|\r\n' - - - CHECKBOX - ' ,|\r\n' - - - COMBOBOX - ' ,|\r\n' - - - CONTAINER - ' ,|\r\n' - - - CS_CLIPCHILDREN - ' ,|\r\n' - - - CS_CLIPSIBLINGS - ' ,|\r\n' - - - CS_FRAME - ' ,|\r\n' - - - CS_HITTEST - ' ,|\r\n' - - - CS_MOVENOTIFY - ' ,|\r\n' - - - CS_PARENTCLIP - ' ,|\r\n' - - - CS_PUBLIC - ' ,|\r\n' - - - CS_SAVEBITS - ' ,|\r\n' - - - CS_SIZEREDRAW - ' ,|\r\n' - - - CS_SYNCPAINT - ' ,|\r\n' - - - CTEXT - ' ,|\r\n' - - - CTLDATA - ' ,|\r\n' - - - DEFAULTICON - ' ,|\r\n' - - - DEFPUSHBUTTON - ' ,|\r\n' - - - DLGINCLUDE - ' ,|\r\n' - - - DLGTEMPLATE - ' ,|\r\n' - - - EDITTEXT - ' ,|\r\n' - - - ENTRYFIELD - ' ,|\r\n' - - - ES_ANY - ' ,|\r\n' - - - ES_AUTOSCROLL - ' ,|\r\n' - - - ES_AUTOSIZE - ' ,|\r\n' - - - ES_AUTOTAB - ' ,|\r\n' - - - ES_CENTER - ' ,|\r\n' - - - ES_COMMAND - ' ,|\r\n' - - - ES_DBCS - ' ,|\r\n' - - - ES_LEFT - ' ,|\r\n' - - - ES_MARGIN - ' ,|\r\n' - - - ES_MIXED - ' ,|\r\n' - - - ES_READONLY - ' ,|\r\n' - - - ES_RIGHT - ' ,|\r\n' - - - ES_SBCS - ' ,|\r\n' - - - ES_UNREADABLE - ' ,|\r\n' - - - FCF_ACCELTABLE - ' ,|\r\n' - - - FCF_AUTOICON - ' ,|\r\n' - - - FCF_BORDER - ' ,|\r\n' - - - FCF_DBE_APPSTAT - ' ,|\r\n' - - - FCF_DLGBORDER - ' ,|\r\n' - - - FCF_HIDEBUTTON - ' ,|\r\n' - - - FCF_HIDEMAX - ' ,|\r\n' - - - FCF_HORZSCROLL - ' ,|\r\n' - - - FCF_ICON - ' ,|\r\n' - - - FCF_MAXBUTTON - ' ,|\r\n' - - - FCF_MENU - ' ,|\r\n' - - - FCF_MINBUTTON - ' ,|\r\n' - - - FCF_MINMAX - ' ,|\r\n' - - - FCF_MOUSEALIGN - ' ,|\r\n' - - - FCF_NOBYTEALIGN - ' ,|\r\n' - - - FCF_NOMOVEWITHOWNER - ' ,|\r\n' - - - FCF_SCREENALIGN - ' ,|\r\n' - - - FCF_SHELLPOSITION - ' ,|\r\n' - - - FCF_SIZEBORDER - ' ,|\r\n' - - - FCF_STANDARD - ' ,|\r\n' - - - FCF_SYSMENU - ' ,|\r\n' - - - FCF_SYSMODAL - ' ,|\r\n' - - - FCF_TASKLIST - ' ,|\r\n' - - - FCF_TITLEBAR - ' ,|\r\n' - - - FCF_VERTSCROLL - ' ,|\r\n' - - - FS_ACCELTABLE - ' ,|\r\n' - - - FS_AUTOICON - ' ,|\r\n' - - - FS_BORDER - ' ,|\r\n' - - - FS_DBE_APPSTAT - ' ,|\r\n' - - - FS_DLGBORDER - ' ,|\r\n' - - - FS_ICON - ' ,|\r\n' - - - FS_MOUSEALIGN - ' ,|\r\n' - - - FS_NOBYTEALIGN - ' ,|\r\n' - - - FS_NOMOVEWITHOWNER - ' ,|\r\n' - - - FS_SCREENALIGN - ' ,|\r\n' - - - FS_SHELLPOSITION - ' ,|\r\n' - - - FS_SIZEBORDER - ' ,|\r\n' - - - FS_STANDARD - ' ,|\r\n' - - - FS_SYSMODAL - ' ,|\r\n' - - - FS_TASKLIST - ' ,|\r\n' - - - GROUPBOX - ' ,|\r\n' - - - HELPTABLE - ' ,|\r\n' - - - LISTBOX - ' ,|\r\n' - - - LS_EXTENDEDSEL - ' ,|\r\n' - - - LS_HORZSCROLL - ' ,|\r\n' - - - LS_MULTIPLESEL - ' ,|\r\n' - - - LS_NOADJUSTPOS - ' ,|\r\n' - - - LS_OWNERDRAW - ' ,|\r\n' - - - LTEXT - ' ,|\r\n' - - - MESSAGETABLE - ' ,|\r\n' - - - MIA_CHECKED - ' ,|\r\n' - - - MIA_DISABLED - ' ,|\r\n' - - - MIA_FRAMED - ' ,|\r\n' - - - MIA_HILITED - ' ,|\r\n' - - - MIA_NODISMISS - ' ,|\r\n' - - - MIS_BITMAP - ' ,|\r\n' - - - MIS_BREAK - ' ,|\r\n' - - - MIS_BREAKSEPARATOR - ' ,|\r\n' - - - MIS_BUTTONSEPARATOR - ' ,|\r\n' - - - MIS_GROUP - ' ,|\r\n' - - - MIS_HELP - ' ,|\r\n' - - - MIS_MULTMENU - ' ,|\r\n' - - - MIS_OWNERDRAW - ' ,|\r\n' - - - MIS_SEPARATOR - ' ,|\r\n' - - - MIS_SINGLE - ' ,|\r\n' - - - MIS_STATIC - ' ,|\r\n' - - - MIS_SUBMENU - ' ,|\r\n' - - - MIS_SYSCOMMAND - ' ,|\r\n' - - - MIS_TEXT - ' ,|\r\n' - - - MLE - ' ,|\r\n' - - - MS_ACTIONBAR - ' ,|\r\n' - - - MS_CONDITIONALCASCADE - ' ,|\r\n' - - - MS_TITLEBUTTON - ' ,|\r\n' - - - MS_VERTICALFLIP - ' ,|\r\n' - - - NOTEBOOK - ' ,|\r\n' - - - POINTER - ' ,|\r\n' - - - PUSHBUTTON - ' ,|\r\n' - - - RADIOBUTTON - ' ,|\r\n' - - - RCDATA - ' ,|\r\n' - - - RTEXT - ' ,|\r\n' - - - SBS_AUTOSIZE - ' ,|\r\n' - - - SBS_AUTOTRACK - ' ,|\r\n' - - - SBS_HORZ - ' ,|\r\n' - - - SBS_THUMBSIZE - ' ,|\r\n' - - - SBS_VERT - ' ,|\r\n' - - - SLIDER - ' ,|\r\n' - - - SPINBUTTON - ' ,|\r\n' - - - SS_AUTOSIZE - ' ,|\r\n' - - - SS_BITMAP - ' ,|\r\n' - - - SS_BKGNDFRAME - ' ,|\r\n' - - - SS_BKGNDRECT - ' ,|\r\n' - - - SS_FGNDFRAME - ' ,|\r\n' - - - SS_FGNDRECT - ' ,|\r\n' - - - SS_GROUPBOX - ' ,|\r\n' - - - SS_HALFTONEFRAME - ' ,|\r\n' - - - SS_HALFTONERECT - ' ,|\r\n' - - - SS_ICON - ' ,|\r\n' - - - SS_SYSICON - ' ,|\r\n' - - - SS_TEXT - ' ,|\r\n' - - - STRINGTABLE - ' ,|\r\n' - - - WC_APPSTAT - ' ,|\r\n' - - - WC_BUTTON - ' ,|\r\n' - - - WC_CIRCULARSLIDER - ' ,|\r\n' - - - WC_COMBOBOX - ' ,|\r\n' - - - WC_CONTAINER - ' ,|\r\n' - - - WC_DBE_KKPOPUP - ' ,|\r\n' - - - WC_ENTRYFIELD - ' ,|\r\n' - - - WC_FRAME - ' ,|\r\n' - - - WC_KBDSTAT - ' ,|\r\n' - - - WC_LISTBOX - ' ,|\r\n' - - - WC_MENU - ' ,|\r\n' - - - WC_MLE - ' ,|\r\n' - - - WC_MMPMFIRST - ' ,|\r\n' - - - WC_MMPMLAST - ' ,|\r\n' - - - WC_NOTEBOOK - ' ,|\r\n' - - - WC_PECIC - ' ,|\r\n' - - - WC_SCROLLBAR - ' ,|\r\n' - - - WC_SLIDER - ' ,|\r\n' - - - WC_SPINBUTTON - ' ,|\r\n' - - - WC_STATIC - ' ,|\r\n' - - - WC_TITLEBAR - ' ,|\r\n' - - - WC_VALUESET - ' ,|\r\n' - - - WS_ANIMATE - ' ,|\r\n' - - - WS_CLIPCHILDREN - ' ,|\r\n' - - - WS_CLIPSIBLINGS - ' ,|\r\n' - - - WS_DISABLED - ' ,|\r\n' - - - WS_GROUP - ' ,|\r\n' - - - WS_MAXIMIZED - ' ,|\r\n' - - - WS_MINIMIZED - ' ,|\r\n' - - - WS_MULTISELECT - ' ,|\r\n' - - - WS_PARENTCLIP - ' ,|\r\n' - - - WS_SAVEBITS - ' ,|\r\n' - - - WS_SYNCPAINT - ' ,|\r\n' - - - WS_TABSTOP - ' ,|\r\n' - - - WS_VISIBLE - ' ,|\r\n' - - - \\ - 1 - - - \\" - 2 - - - \\N - 2 - - - \\T - 2 - - - ~ - 1 - - - %1 - 2 - - - %2 - 2 - - - %3 - 2 - - - %4 - 2 - - - %5 - 2 - - - %6 - 2 - - - %7 - 2 - - - %8 - 2 - - - %9 - 2 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/otmamri.mak b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/otmamri.mak deleted file mode 100755 index cd0d709d..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMAMRI/otmamri.mak +++ /dev/null @@ -1,55 +0,0 @@ -# --------------------------------------------------------------------------- -# Copyright (C) 1998-2017, International Business Machines -# Corporation and others. All rights reserved -# --------------------------------------------------------------------------- -SRCPATH = $(_OTMMARKUP_SRC)\otmamri -CMNPATH = $(_OTMMARKUP_SRC)\common -INCLUDEOPT = /I$(SRCPATH) /I$(CMNPATH) - - -#CFLAGS = $(_OTMMARKUP_CFLAGS) $(INCLUDEOPT) -CFLAGS = $(_CL_CPP_OPTIONS_DLL) $(INCLUDEOPT) -LINKFLAGS = $(_OTMMARKUP_LINKFLAGS) - - -.all: \ - $(_OTMMARKUP_OBJ)\otmbmri.obj \ - $(_OBJ)\eqfparse.obj \ - $(_OTMMARKUP_DLL)\otmbmri.dll - -#--------------------------------------- -# Compile OBJs -#--------------------------------------- -$(_OTMMARKUP_OBJ)\otmbmri.obj: \ - $(SRCPATH)\otmbmri.c \ - $(_INC)\eqfparse.h \ - $(CMNPATH)\usrcalls.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\otmbmri $(SRCPATH)\otmbmri.c /Fd$(_BIN)\EQFSTART.PDB /Fe$(_BIN)\ - - -#--------------------------------------- -# Link DLL -#--------------------------------------- -$(_OTMMARKUP_DLL)\otmbmri.dll: \ - $(_OTMMARKUP_OBJ)\otmbmri.obj \ - $(_OTMMARKUP_OBJ)\eqfcalls.obj \ - $(_OTMMARKUP_OBJ)\usrcalls.obj \ - $(_OBJ)\eqfparse.obj - @if exist $(_OTMMARKUP_DLL)\otmbmri.dll erase $(_OTMMARKUP_DLL)\otmbmri.dll - @echo ---- Linking $(_OTMMARKUP_DLL)\otmbmri.dll - @echo ---- Linking $(_OTMMARKUP_DLL)\otmbmri.dll >>$(_ERR) - @$(_LINKER) >>$(_ERR) @<< -$(_OTMMARKUP_OBJ)\otmbmri.obj -$(_OTMMARKUP_OBJ)\eqfcalls.obj -$(_OTMMARKUP_OBJ)\usrcalls.obj -$(_OBJ)\eqfparse.obj -/OUT:$(_OTMMARKUP_DLL)\otmbmri.dll /nologo /MACHINE:IX86 /ALIGN:0X1000 /DRIVER /DLL /NOD -/MAP:$(_OTMMARKUP_MAP)\otmbmri.map -$(_LINK_LIB_CRT) $(_LIB)\otmbase.lib $(_LIB)\OTMLinguistic.lib -<< - @copy $(_OTMMARKUP_DLL)\otmbmri.dll $(_OTMMARKUP_RELEASE_DIR)\BIN /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmamri\otmamri.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmamri\otmamri.chr $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMANSI.TBL deleted file mode 100755 index 38b32b3defec460a2bf29c93ea570663d64453d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 778 zcmd;LcJmMNbqon{bnoRz{|q$hRKSJ;R_ - - - - - Plain text (ANSI) - OTMANSI - * - ? - ANSI - - \r\n\r\n - SEGRESET - 4 - - - :NONE. - 6 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMASCII.TBL deleted file mode 100755 index acd9f6e1f047976476895a1cb2aed44b780d9652..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 778 zcmd;LcJmMNbqon{bn - - - - - Plain text (ASCII) - OTMASCII - % - ? - ASCII - - \r\n\r\n - SEGRESET - 4 - - - :NONE. - 6 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF16.TBL deleted file mode 100755 index 8d33573f3b205c54f5598ac7e5b0eb5d3360fb01..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 778 zcmd;LcJmMNbqon{bn89*_|0(K!A122$f1dFhNMR>V* dxfmF%{QUh~^%xj - - - - - Plain text (UTF-16) - Plain text (UTF-16) - YES - UNICODE - - \r\n\r\n - SEGRESET - 4 - - - :NONE. - 6 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMASCII/OTMUTF8.TBL deleted file mode 100755 index 7026dec1b23c477543c166bb4924a50d19d3e3dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 778 zcmd;LcJmMNbqon{bna(D?fifS3L#>UKWNoOjc|RUl5V~h6w - - - - - Plain text (UTF-8) - OTMUTF8 - % - ? - UTF8 - - \r\n\r\n - SEGRESET - 4 - - - :NONE. - 6 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.CHR b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.CHR deleted file mode 100755 index 7ebd20f5..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.CHR +++ /dev/null @@ -1,26 +0,0 @@ -************************************************************** -* Copyright (C) 1998-2013, International Business Machines -* Corporation and others. All rights reserved -************************************************************** -*********************************************** -*** Parser control file for OTMBMRI Parser **** -*********************************************** -* The following switches can be used: -* -* SINGLEQUOTES=YES to enable detection of strings enclosed in single quotes -* SINGLEQUOTES=NO to disable detection of strings enclosed in single quotes -* -* DOUBLEQUOTES=YES to enable detection of strings enclosed in double quotes -* DOUBLEQUOTES=NO to disable detection of strings enclosed in double quotes -* -* ANSI=YES to enable detection of strings enclosed in double quotes -* ANSINO=NO to disable detection of strings enclosed in double quotes -* -* If a switch is not specified it defaults to NO -* -* I no switch has been specifed DOUBLEQUOTES=YES is used as default -* -* Any other switch or command will be ignored -* -DOUBLEQUOTES=YES -ANSI=YES diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBL deleted file mode 100755 index 7fbb7797f3020595caf32ce29513099b35fc6311..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1911 zcmd^;%}Z2K7>9pzXYTO>jKC1eMVG-7`N2$@Sb3KA3)3<~@Q{4)eMZCbT*)v8tWzURVSye@{8+JXDr!^8V~@41|FUX|U?WYdXU zF45iJi_@bH;nVV*n7(i=n(Yc4*}$^8Q+7=6BRi8zcc-(dv+<9I*1ZN=a3VSQFjI2H z$*JPlMBW)JP8IS`okD)5;9O6*L#g_b$&v-3k`@0$0bi96EFwQZ1m#5E_}8ccRl;0A zbzmYi11{mRFjsI@7zbLIYiJav3C+T^piP)|+!dw^J;EgMATa)4q>vWIMOK(0JQn5& zJYhyKD$E$33G*B;gqgrgVWu$?7)3K+7IT5|&*nT91LN<`8>|TP7Vm^v#rweceb%rZ z7{AX4YzVW7kHUPyXJNLmEzAyfh1tWtFkkRhm~S``CK?4|u^12^xM4uYfTk^mHw{}2 zZyDY;q=P{sp6EYbSF$ohe^`_iZQ*o!8aeL`|w7njE+jHs30SkbpE}EKX>Y=HH zrVg4KXv%AfYf48g9NV!lzT5FIe%SE{elkuNzZkz6e;5%(sAODZ)F?{SG8~3xG%(_f zRz?S-i_xPx75bDHf~!0WS;iyAuv!RVL}?47j67qUG0B)_6dALO*Ni3qjPWfGrnCqD GGX4ThcwlV+ diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBX deleted file mode 100755 index 05a293e2..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMADQUO.TBX +++ /dev/null @@ -1,157 +0,0 @@ - - - - - -OTMBMRI - Double Quoted text (ANSI) - OTMADQUO - * - ? - ASCII - YES - - "" - 2 - - - ... - 3 - - - /P0 - 3 - - - /P01 - 4 - - - /P02 - 4 - - - /P03 - 4 - - - /P04 - 4 - - - /P05 - 4 - - - /P06 - 4 - - - /P07 - 4 - - - /P08 - 4 - - - /P09 - 4 - - - /P1 - 3 - - - /P2 - 3 - - - /P3 - 3 - - - /P4 - 3 - - - /P5 - 3 - - - /P6 - 3 - - - /P7 - 3 - - - /P8 - 3 - - - /P9 - 3 - - - :NONE. - 6 - - - \\ - 1 - - - \\" - 2 - - - \\N - 2 - - - \\T - 2 - - - ~ - 1 - - - %1 - 2 - - - %2 - 2 - - - %3 - 2 - - - %4 - 2 - - - %5 - 2 - - - %6 - 2 - - - %7 - 2 - - - %8 - 2 - - - %9 - 2 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBL deleted file mode 100755 index 2293a277898ca95f1f1e77373f1dbe08630fcc9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1294 zcmds$%}T>S6opUHv9uepC@3fxBJrm}q&8Yx!IJa`18rj4h>HdUwYX5o!WO&gEBHXZ zg)gM{8VNF8#HBjqoEyHmxtxK?;zHZ+ckLjsolXn$tANZr{3qmH&EX}JM@axv@g^*) zCE*8Mr`vb`#u;DLdXWM*dzXHYnz$KFZXQQkZxW41ciMO~9c#Pxpy9fEsh_4bC}|D9 zcAu}e0+pNv2;|6P5?}p1mJ*WX78J0Ykoe>)SWQSgXASET*}$emH0TM5=WJm+A@Q6Y z6eY3`Qz8dAlt>xJ5~-jjkyD&Wqz*?S4YVbqQ)JVm1U1k1T2w=Nh=m@B?%)hCN+e2T zlt~uph(vje3h9JY1#}gbFf*u%S{Akl6*NVt;+(C+b|Gd82EtHr#WrNSX1is(XM13a S*q+#)>G%Cy-(J7SPs<0eYb`(k diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBX deleted file mode 100755 index 06283f7c..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMDQUOT.TBX +++ /dev/null @@ -1,81 +0,0 @@ - - - - - -OTMBMRI - Double Quoted text (ASCII) - OTMDQUOT - * - ? - ASCII - YES - - "" - 2 - - - ... - 3 - - - :NONE. - 6 - - - \\ - 1 - - - \\" - 2 - - - \\N - 2 - - - \\T - 2 - - - ~ - 1 - - - %1 - 2 - - - %2 - 2 - - - %3 - 2 - - - %4 - 2 - - - %5 - 2 - - - %6 - 2 - - - %7 - 2 - - - %8 - 2 - - - %9 - 2 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.CHR b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.CHR deleted file mode 100755 index bd66e8ae..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.CHR +++ /dev/null @@ -1,27 +0,0 @@ -************************************************************** -* Copyright (C) 1998-2013, International Business Machines -* Corporation and others. All rights reserved -************************************************************** -*********************************************** -*** Parser control file for OTMQUOTE Parser **** -*********************************************** -* The following switches can be used: -* -* SINGLEQUOTES=YES to enable detection of strings enclosed in single quotes -* SINGLEQUOTES=NO to disable detection of strings enclosed in single quotes -* -* DOUBLEQUOTES=YES to enable detection of strings enclosed in double quotes -* DOUBLEQUOTES=NO to disable detection of strings enclosed in double quotes -* -* ESCAPECHARS=YES to enable escape character handling -* ESCAPECHARS=NO to disable escape character handling -* -* If a switch is not specified it defaults to NO -* -* I no switch has been specifed DOUBLEQUOTES=YES is used as default -* -* Any other switch or command will be ignored -* -DOUBLEQUOTES=YES -ESCAPECHARS=YES - \ No newline at end of file diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBL deleted file mode 100755 index d89b20967cd4f248a09f5926f679b473757bd279..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1294 zcmds$%}T>S6opUHv9uepC@3fxBJrnUB(~Zg!IIhr7j0v8iVFjR8eAxJVT)b$6?`Dy z!WYte9SJfO)SWuyoSS?kmvbk~^i&X@w0$Z4daH@al|yP0{uAzDKbf^W1@1K{@|Wov{o& zHRLt230EWA*wIJ$BanJ<#!eTOV%U$#=^u2}>=L diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBX deleted file mode 100755 index 2e16443b..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMDQUOT/OTMUDQUO.TBX +++ /dev/null @@ -1,81 +0,0 @@ - - - - - -OTMQUOTE - Double Quoted text (UTF-8) - OTMUDQUO - * - ? - UTF8 - YES - - "" - 2 - - - ... - 3 - - - :NONE. - 6 - - - \\ - 1 - - - \\" - 2 - - - \\N - 2 - - - \\T - 2 - - - ~ - 1 - - - %1 - 2 - - - %2 - 2 - - - %3 - 2 - - - %4 - 2 - - - %5 - 2 - - - %6 - 2 - - - %7 - 2 - - - %8 - 2 - - - %9 - 2 - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBL b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBL deleted file mode 100755 index cf62acb405e8c1ec052e20234a4ff34b6b80eecc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 42455 zcmeIbcXU<7`aXWrd+$I(AUz~Gq!*Hq(~=x>a!xqsq#=l+fDJn$7r|@cqNt#P1v~bF z4GW5>=(VCKh>D5@EZ9X6QL*ry_nEzC&mO$%bARi%K5KpcAZz7)-uIbzo|!#+@0r~K z+ivgl`Fm}_pv~@aSq}brD651<<6$rVJBXH6u(!76@9X>@W1kGQEc~xK%r?uDUt3jUt*aVe{eNsj{+GIw#&lB-{r>+R zgH$$#^m}M5|K~Aa!-R1nlz(Dbp#KuD`#Q?Gx=&^5&;P{9<)N(dfBGd|Ial`8Hs$!g z&C~VeeBCFMe`S5ur~cEAp0E3a3G6pU`X!Ejh|_=mu3RtEQO?&nLgJ%jLgN(5)E<@5 zLiV42^}2eSaXO))&dW4$w5|G&+9#dY@$WSBvj2osN3N&(daUOCou0ms%ei{}zsuM6 zEg}8LvCi=l!~PxrKXG#Te>yMZ<4tc(uBUTYw=j6W1^{HGR z@Benr)-?~|p%FMV0*6N6&N&T^_? zc+P2-d4l0N|FE1V7_NN2Wr<*T&c&AHg5fzUEY~Z{4VF6u!`r&kvRW|V=+AwYhXoS> z<`K)2g5hngw>&Qx-qs72O$zh6Wt(7lTkl#v5)4=V*s@D7T=@&jZozQnA1uEKhU@IN zgt8Yrd|&XKu#kAc@SKE@Oohn`$rB83YjjA7V7N|cNR?oC53L~$g5k9qL)rwxbJ{~( zg5f!xAzs07WnajUV0aIQLyiy(ANgq^GX%qxXNH_47_M`2$XSBnI`cx#6%5ZgFXUpu z@SICRmJ5dWa7D;<3UhtPZGz!Cw};%PF!zV76%1E?Jme|C@U3_{Y+>MWL4|%;ljg1;drE4ZTS) zd^Bzjy+bfu`OeUL1;fX1b?8Ha;kDL;J}wwu>xs|}g5mvnHuPnMc_nm0zS; z!*g=OiUq@SO2VoH!&|q8H428;Y6|O67+ct3g5f$nVUq;Id*}}v77SOO9CoB&c+OE_ z#|nn$92a(~V0i1Nh0PZXZ|m%^3k1V+7KJSp49{5>cBNo=TUUkMAQ;}(jbXP5hUeTK zwpuVe=f1E<1;g8VEbJM<@U}LDy(Ac3>*cV`g5kBcgl!WHZ|mK#j|IbP?FjoqVZIFe zK`>nB$FO~Z;m5`=VSfsSEB_T17Oo#HJSRLnRxms#E<8nHQp0lu!}~KLe6(P=PJVc? zV7N|6c%{Nrh1Uy)>okP73WnEe3wH{J=eWXq1jBPY;Q_($oM8Cj3Nt1A7{T!InGt@X zV0f+B;d2DTbIuH3AQ+yrF#KPF;W>-LmkNgGEDOI{Fg)j)@S7E8Rro!E;m5|k;cEoL zbsi3XQZQU+efaYV^FsLRg5f%Eguf*izAxSm-!2$FVjqNmqA;I^e=Qi^`ZwV}35K`6 zH~de*@V5R64~tNapNLq&@V4S2QWPdNB1bU1tq~EU1*0855ygVxIwcX63R4wPFBq=V znARe$X>sXiX^3cz2+^*!xK3MySJdJ;eG$_X#+KI2K4J)yu9go=8&H^|B4$MBSKVCa z%!maFb9uyd3bQ6+J?oFGyfxwj(L=7YCt{ycD=IP}(%e>lWU<0DN7@u-AoA$QP}4R4 zg2)RPBe#A{ zj)P0m>V}%Vd~;0DvU2KpfJb8 zox&J-?_3nOOkr+`yF;n9Htxwd^H#hTw^d=ji2FgQ6&artZ|-wZd}X{@#~$xin5pqI z73S>tMGA9W{3?ZcBz~R3Y>IzdY5nW?JqmLmK0LwP!_0&+3e%imQy6bTKw)Mi%vP9x zB%G%(S0`MrF!v`soM0Y}7ZP4knD-MtR%-p2@QacYo0yzv-lqkL#ff|!)W5-~NUTwq zrbL^D@y@rO8(*IoBuOsxWsX-={J9nessL zS|w*)^3%!YbLaWwjY?aaliyQneVqK6!hD_lgTm}f{zGAs(nf1Xls>AzrN$`CC#gTN zk4fdZB{U_TeWoNcSESvdTtBC#WGgv&DMcyfy)!PQPRr3pqb0?u)H*CBs4#;m(-dZU z%1nhhG388!nU`{|!Yoc%qA-`GT&*xSq}-)2_oqC}7;{yvHLR=NB8BO*k(w@Ql~p)eJZwF=V}*{3kaMb1_3{+2{uq1^r5 z8hM{`XZK9xMuqt}@+*b;BQjjMlgf%3tuQrF?aKXEPt+uZnHDuuVa|w}ujpJFwL;0c zBkDedc{=KOg?T$_yTW`MwMS8oh)z-Nt@5Kw6{a=XrQH1uMIWi;%#A)@VOB(6t1$OQ zKcq0vMZc^t??r#CFh4}^Q<$ij1m#XDKcB|HVuqCaxZ`3@S9BJ{T%a&l z#H>_m-5s+=$$2{FC53q_<|BpqCgw+liHc2C?zReJ$0Jh50o$T)C4`VJyVg5)95#Iyw{)DDSDa>cCuUxcl?sn}3 zW_OM3qQ7j}?>c}uqBFLW%9A@YfFnEefyJHUfYqH1z}8M1u&dJp^mh&dr*=*U9@}{; zaHs22rW<0J+j%zP(5$E|(oe|B1dhro1eRyj0GqNV0$o`ipg(IW@aU}Bz|*tN174VQ z9dK3F!@wu9UIlK+`ULo8)*j%mSs~ffUQ~7pFe|$VI4-*h*q+@3oRmEkcy#vhz*Dm4 z1JBLA73~4XMYC#I(rZB*X+N5;W>#p)Q|L>Twp;? z1+XTk1=x|}1Rj>-&!OXOC}$dQM$U=A({j!NF3kBC@-E4_26$u6-M|NO)&V!d1Vn&jF(#UM!r1T-=pWSs1`kOW)8+m0T>rp;oWDn09c_eV=$T`6IBQF9j z8+jdY)yRi{Ye&8ad~M`*;Lee|fk&jzKs!H=Jb*Z2RN5%&N6x5XV8y6r;KWfqz)7P9 zfKx`z0G=@Fbl|*E=K>dxS_)h->U!WUqwWVjGU^%N3!^pz-x;+7xNFq+z`dg^xpW*v z=B5C%a>oKob8CPnq@RI){v-W7U{h{KF7>lB*AE=ZJsNmi?is*;cz)g`z$@~u1Fp)u8~8xpBfxce{|3I2w*|N@?_=QSdEWx} z9X;QOOL0e(69JK#^F{{V*O$L3S}$@$s9y!;~IxcpjRbG{4MlRp5Ql0OSL zJAW>4Mf&v^p9T4g5MP{sHSmV~yMXuSKLLCu{}tdH`P+aW=6{t>$I0&e{Xol@xG@x` zj2Q{MCH-FLju}%vhS8SsW7>d@F+O1bn4^HR#+(K`Ys^C61!I;1SB$vLJ8zy)J30xlbS9dOmy)xd|+pTPD#ll~HL&Dh7mKRxzk;Ok?z0Y4nO z3;6BWy}OeO?f|YXcpUh2!A9Wbg6+T$(!W6d(yq%fJ_%z}(Xa2)e*ph&!CvI;FNiFp ze#RGO0dosWfmMY~z^l8iMg5%xyO8fL^nwo-P6ZxacmnX$!g;`R3jYP%m;O82xuozi z#8(yG2)wQEUf@H8j{~1Bd>**5a5M0o!X3a}g}Z@27ybndFN!asex?@X0Sk-jfGtIC zps(l%;4wv~0naL006f3w65tg@*8*=Yx(j%J(W5->x)s~kUg$tOAsG=F)Xs}Vn^3;B z=p*1~Mc)JW7X1YbFHR_?yt}(rqyCdc8^Grk7lW@TZUVL!yMeyq$-pCvj|ZMoJQuj2 z_(I^);%k667T*QDzj!S$E+ZBF$T}cP9vt$x*u;d8fF(oGh=aifSTvW0QcxB0rz}re506tpsJaA*lyTFf1 zz6I_nIRGroC__KjbUoTd?WYxIqd&=|X{8kBl;#7AODljirA@&0QU}nQQIGoW(muok zrAGm0m7WTmTY4^Vap`5iOFNf$lJ3gRTY-0Xt^q#Y`7H3I&NqNpmEMB=XE&T-eW$8ZPgbW+n|D!atjC7;RQi0iJV}K=PK%JAk{&e**qi9x;yE ziyM~(%pF$>e5dO@jLYP*Y3O(JxK8k%aZ`auk2@VWZ`?(|W#euH-Zt(*;A7)n1im(I zJ8fn;tOIVScm?=I#k;_dDs}-c%D4>q-&XvD__vC1`s~{hTbT|VQCSEq zudD?&SK5GGl>y*z8mp<+s2+m6j^%M`TqRFsG`dijJ4csuo~JRWC45H4QkU>Qvy|s`G#sRxJmvthyC= zchy?xJXN(3xVh>>;HOo)fwyL?Mt^^<`U`QmHN{HpXITq?W!8G&1ZyYopb8&*2Ta}tt)`nTJHfqXnh{I(fT3qQ|mt9AJ)Y2)b5&$C(xht z@gotB8D9ac8Q%)DkM{um;|~Wulkp<*U(476+?KHexGUp3;7=L*ftJi@+~*uM{&>_o zW&8r*`Qw)XuN=P$c<1;vz{Jc9jc-3H)`bz6Yj>UILZsQVH4OCA0g#1dJb zT2Jj~*OvmT>f3;hdOvWeekSn5`m=xw>z4vo)ZYrcyZ&+D)Ag?b->m--_-Xwf;IH+O z4Wu96Fant0PzkJUm%m)@XRs$OwZNRR^KHxy((ZJ&x=K$w7E(TuOcn$Ez#yf%cHLe40 zXnYO$X5&Y|&l-0Fe{TE>7~Yi9MD1oZ6#&bc>VXrQI)OiS?ZtT&Q5}opd$8#U@W(Wr z44l(+4scP^GT@a>Hv&gy7G{!ed1f`RF|!SLThl!#f3RsCa6{88z&D!S1%A}D3+Tw~ zM*g==dlBz%ifksm_~tBNZgVNHs<{c+-s}SQG!Hb>d_ARkCh)}OGl6F}F9PoG`jgKS zO_QJ-)*abRx)(QJhWg&jAljXrc_eUE^PS-DYhDX{s`*9WYt3&1w>N(ZoSAtt^1o{S z4)IUTzXL;BqFYEmu_XgIvSkc#PUb@7Uy!*3Skf{Me057RaAHd*(9;qC4!2ALUY2m4sia2MZk+EtN>m+;b!0+ z6CMOUHsKlI3lm-kzBS8a#16Q`*4!pPZG2r^vmw=mEw{br65w!nd>n_CKw*Ce@&>Gi9 z?WeTm0t?!zfOTyXfVMUd(BC!%IKAye;Aw61f#3nnfFu9$cO@Yac|foD!!2wXF9 z9dKRdv)DfyCccdL^@;DJy&V(30sc7g4`66VdcFogIe( zGrF^JT-zsf;dnc|;~4PAcgzJY=(q^Dtm8W1s*csbH62d?pXt~L+}!a#a7V}2z#lq( z1HRq)KJK@6bbi%I{o3963-Hg*h%Sobx>A8JWxj^(JJ1nsW3(mKmIlnRjRBU}tiXC( zE6{H126}Bnwh)$ogl!gZw(U&d*|rOSOKg`SZ@KMS;LWx>fU9k5fREdr0{+{!3Aoj^ z6ZO8Z?FH_)McG4`eu6y*_-5ui=;vsAp`H3!Zm$J4+ik!uyARlJp90M59)t1i>*(*G z{Mq)?!Oydw54^~J1@IdCD&U>=2Y`>-p9DT@e-*gJ{yuPr{VU*Z`!B#h?com6jdf%I zM>>jt;~e$C2@WT)q`Msbm~Nkidi{0G{u-6u8`RJ@6LCUBLStk2`1_ zo_4$h+~jx%_?}}Y@C(O}$os_+;iUe=IkSOz&T+tMXB*Jr>;X=4P6ZzAJPCM)^BmwJ z=Q7|2nL9DgS30jlyvlhm@FC|q;0EU_z&D)RfFC+P1AguN8Th+1!i9dgGJqppg~0LM zwHSxP96t2-i_9O177I59C)?sCgAO^`+yI- z)&ZNlThUIr%ZhkZR}t3Vaa~O~-rKtl1NL=I0Z#9l4YYT=P;Z0l1=Q>5_9A}OwHa}+ zdl2zEuI-4=>s*ZUtHAAbc>0};D{wP{8K;Y*mpd*Gc$X(YIWDggVLvY|v{(uY7h3WQ ztI85$W7;jnHk+eA$PzV&O%261huhDFj3Xzs6iNlb+F%=5Y$F8G<+lw$1rDg*KFa9r zL*RFNQGpOd(CwjZD7L`?)!*wOf&qng|A5`*7Y!F z2)(;*(OvhXepi5Gxq~d;Ter4{Jhn;wK9&Haa*w~4TLe_sgD&;j9JC_|sHE5J@KZe? zC!9_nrT4nYA_r&x*aEcAd~iVNP5X?d`Jni;LFu#k+453MKo$Bt{efOO%(w%}$IeA!2#H{$ zNE=T;pk1G-2*Pi}`4J=}qF>(!+ySNSgMLB~`ab9v`+yIh zHhTS9q=mz}BzH(*7$+^W2zdIimOgp#FqXkAl}#S>^2keJ>zK+)c3DTzV{`fjrUh+X z3o9!v?Vtu-cDpZ_uMzC}BOjAv>lDMa+U$0}#7mKt9oIhZaIaKqCagB6)9(rdqy#fz zwe|IRTtTVa6kF{^iEyoUr%$4dz-ni6v)BMBu-ZLtZ;w)#>u^WX-WUJEcxbfz|1jdMgF2U_g>K0;|&_@xr${rCo0LR+n@r z8NRjC?eaJS>@dLy%aOIy$JRPA5ORc~N#-&UvO+dxOZAM%%GM%wrppJV%2?ThMp4)B zt=%;eEqrTttwamo+Fd8n!nbzUOSJH<-3<~gd~3JMChb%?veG}PE!B}CtJ~m(YjtBY z`4}6%)$I*v9CxfecBw@vuzGBE{YVlnBZt``jvXgltH;&FF3&Vp_*ReT%oNFIZXifk zQDVpy!rE)=GYX}^+UxT6OC?fd?d|sj-R$g;iWSD{^~n>T9HB@YOjFDui(I{Smo!KU z!4mipWwfm_EG6LZyZf|5UXH9j`(d>DVMq<%DsPJ6TYY^&X*!T2t2AiBwf1>zZZ<0o zNoUnTl(o+<%|23KmBvK4R%!Ydt~J0ekd5t^BdfOMVt1QkYry59?Kg@|u{F?Z^GJKu z6j}p)Hfd^;BWob&ce5KKL)H{iu}?k*&9ODmFV?3ia=?6fGmXH?|5pRioGG*hvD#o4 z3)dQy4k^R8($2E^UDDdEGS*yUi#NP!px`JzR+N>Q#!%vT`tK#DzT zx3N1)tsmse%Zzj}B3*HqE;tUpz>u#cj5$WTjL2tWuUsjZFAU@>0r^4zvl&&^uK-BN zSm$YV2V<=6(h@GM;KrJ5EX~GhY?|A|T1>+7u#hii0C}cVQQrN&Zfte(b# zX{?dPvM8^8-Quv37eTSQXsa9b+1)GF6}PvqUmUWu>Y+T~X3r0X^t~@FcRj9QwutEk ze8Hm{VwKYtIdGzB`N~sS#-xQyUblK(L2*>8i(Ic*JrD^5eP zP=GdF$BOxkZuP`P)Ip1cv@D2jiRoXS@6GeMX})F?w8OYSmZxj;Y;B&5rMXy|eN7WA zpHykvjEPj8HTk5895Flc$q}491sb!SG0}<1&6v-m8Enuv?T3uZwBA6s%hT5q^f^ou zu15RW6^k?tYb-ks2ic9gejpl?A-n461r}zjVQXM(Ve4S)VH;o@VVhu^nJt{pw`f6Cg@tBgc1r0T zV*gK&%jt1->ZdVczq_kj5NxEx0mbfogFgKLHchdPZkMCS?$b|ugyMqE;cHl| zhp4Vj+v@iA>%DQZhhywwL>nv?dD!ZrNA3XquSWWAa|QwdPo>AW;S)1Ur&|nnm$)bC zW{-4omKj2dssian(owZWifB*rzPv3s3MJGuWgtg4_YI= z>_KC{K5q2nO7sx(6T6|;64|ul@ePWg+s~fY@|oZ36Z_2TV^0X&>`}B{!v4!AeazTA zMNe1tl0Y{-dC{Xmzu2ZeTOc5v1byxy_7GT0#KffgKK5Xb*JlrJ-1=h#zs)PQn_Wmb zeO`|~K7N;nJ*X9ZU|khEK|Zc%eC%w0YT|rK6ML6UNWE^oVd>gB=+k$~fXlC!57s&= z<&(^|u#F$@?H1{xhTQx%cK-6M;h`Lv%M1zj1ex7cXtfZ?G8!R-5LY_7ZGL0BZ6@-d z+In`r(x%Q!9Wx0j>1vv(I1pg7p0p8&m7ZvvyeXi}!kjXaqz5+ayk<{tOsbqPXUlq2 zZgiAAQPi&wY~;&QJQhk6N3Dn0Iux5yO*-BlsZi{9Ptc^nB10i4*AE&6 zen+=)9@uy!tLThS;)JB85h7v8t)HVd=E_UFt~%NN$Pr*u?1{e(+(Dz;mnFGl*Ifd9lxo_*Qu{+q{1Ra2#l=72(ug!a)L3} zYBR5*$jZ&VV}x0e2t$!w%!su}yOuF2qFh!4AuDo1C>qx(bT2AvA(ZnZ6;@_+ioIdd z&wh4Y6p&AUEfy+NEXg1=6}t6v9yQ%gxkHpqj8Z|-b33I3>GF;)^-OAr4J{VRN)(Dh zYMp~~q~03S zBc@7IKUtZy6KE!tr6@F-64!1_#&lpHl=Dm)#=1cDj3<3sWbO`?n>s{3NyWJrqaF{^laLAMSpGAU||p$I}&H@XGn_r%Eul7nv|roloiPea(~P>uiCw=Nrz|1UE++as4@B&xQu=3^5Xe~wm-B; z(ie=V%PSb_T87x-PD8wtJyg<^$(3YzsB2PW2qF?i;zE!;+mr?5S&kJ+)ukCueg+C( zR-m_!#;~DSQlU~~n<(Pfh5GgwjH=1=6lL|I63uC(*jB%INYcq%lP-mFp4?MkuPEv= z2EU8FD;FmP4O6<6oCwpI)>`QWlJ;%PSb#T88L!w@Z3(!~>Ha zl^R>m6J#|iH^hu*AM_YR7D6a`#Ue?BBDu)Jo>XYNnq(+cq&TJ4F-ua8AxoFs`uWg} zK$g@9A&6L#qllsLM0NsL+MJ4j!nv%H46TNjCdt6-gnJ)O`Ydv;VC!UZ< zS9tV9Uo5;lkUFOmddGMy38A*pA$mPPPou?-p$ZzK$O=~0)<|;$Iw=yR2Y7DsWn~_l zQYAGi6}>&o7``6E^++z>;qyCaJR6i6(qg?KghoR=G!^P+BZ}n3UMI}iik7k1OGmwO z#G8+H9M@uxBQjd>cJ&(%M5&BI)7I&fT1V~|&k^G(z0Ogd%nKfE8vF6l4w^O`a=4oIK zYDi=Mrn1@$GL3Ds=}+c+Z9^ti78z zy7dQ}%#%Atkx+3<_fss-DDXMQ9rSRuS0aUi;H0h?oGDFtzzR9@Lsmg4Vy1_!lw`gn z7L44S>C#xsFm2y}%|o|YVltu}jgxgi{Y{pkMIR*@Z&dUbEV8neAq~97NV+a#QM87~$Sd*|R-{`qgMQ+Yi!)V$3 zhKXJ`$?c$2l1D75@>p!4FUa2Yhz~Z|N7o2Vf;y$vkwtlq)JN$}mDfjsNs<$WAjPsA zV!haB>^S|sJ-*oGfuySwq9VQ;Gjw^m=%mkUW0(EN=ro^Vm?h}=vO;W7P7G|a9l5hGA zO06TQYAQxYU6Z`xQ@sEJxtkhc2vW8zhgcMM+l<{zzAVLKQ>r9HrADWWul?wjQC2}H zL|7zgP$Y_a++y^ZD@!1h^CSsYW=I(4sB(hxP)N|3C{A7~GbGqwi))8~-y!#bB2nM( zkgBsl6a;PT-a!A1Ex;pLMQ4->k{ZvD6~!$a)AP$SycSDZe)-&?Y)QxX#uuL=noCT5 zWSP>o(#OGa|0onq_V==ffns}kBx~u6QbE$=8HS?x)6>AP{9X!?P#KnwXcp)NL1Q*# z--hF?Q0kmPiRxM%qkDRe(E@!>tKUY)~kT9j%iqSTP2Kh@U`tA6rL zI*1J&7Rovlio$@)_=KC?|CwI_>4Z63(n19xHehTgdjM&CrOE@ThdQC_^4QSTVyRes z{LYF@4?wlplnSAC3yO@+QAOHG+Nhya*`pF?5;X5yk!5JJH^jyRK6VdBk-WV+BUCvd zZLvm(1nCXXfbt4Ri%qGL5@j1bV+)IT3p;>7?w3XgDa4W%kM-g~WBY zjzz3=JQR|~$M(ttO!MPQ^TSKiqst+qSHp%o*>Kq(tP~hgq2Z1<+-k$EG2B|itux$u z!)-9!M#F6~+-Ae2e=4D1A^S%Y_|o2CluR>RdAWcu?JX_VhYq2!uXv~zvTq^uzoZ~n zQsJSTXJ}BFICQLR+}Pg`wa`0FtCUk3OxB%(2uppC znbu%Iq{SI52(#ppR}?IWvQz~NqAl4JdV+-^7GIG4TS7rHBabZ(wYX*#M_FbBi=!tBK`Q%+d-X-K+N!|_Q-9hGoA=axW zhm@S!w(*^>3IJeBgv@nh9wYNAnVn?zl1V;-nMyKUWM-0CL}nG44P-tc6ETgMJTlc} zykw@6Ifu-RWF956k<2G#4v@(^k{SH_mqX;uA+v(agJfPI^BI|tqnJq}GoDNzndxNC zCUZHNd&xXU=3O%1lZl+p%xE(8WNc*m$V?+Mo6I~i=aE@T=0-AilX-;9vt+iC`IyXC zWPTwNb~H07WOB)rk*OnNBhyVLK<02VGs&DvWhC6h)cY!#VaGDnh`LuLt?Tgj{^vz5%(WJ2hK%_q}HCP?OFGFOnfo6H6>+sJ%R zCW6k@QDhp)_{q#7a~_%HWbPvKESY!6d`HGYr|uXsEoA!1oJD3inFq;iA+wiE8lAA^ zWZK9~Av2rI6=d!uvw_UpWPTu%M5k^QnI1AT$t)ss8=0rbyhG+EGVyf6jwWL#b2yok z$SfvvGnutyUMKS@nf+wa=xnT*-X3DH(Jp&$R#CWR2bo=Dz9qAV%&%nrA`^ZrGqGfn T$z&c&|JWq^kz7vvf3*J(l80!9 diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBX b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBX deleted file mode 100755 index a9635681..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMHTM32/OTMHTM32.TBX +++ /dev/null @@ -1,6246 +0,0 @@ - - - - - -HTML 3.2/4.01 -OTMHTM32 -OTMHTM32 -~ - - #INCLUDE - TSL - 8 - 1 - - - #IF - TSL - 3 - 1 - - - #IFDEF - TSL - 6 - 1 - - - #IFNDEF - TSL - 7 - 1 - - - #ELIF - TSL - 5 - 1 - - - #ELSE - TSL - 5 - 1 - - - #ENDIF - TSL - 6 - 1 - - - #UNDEF - TSL - 6 - 1 - - - $ - 1 - ENDNEUTRAL - - - $$~ - ' $.)\r\n\t>' - ENDNEUTRAL - CLS_TRANSLVAR - - - &AACUTE - STNEUTRAL - 7 - CLS_VAR - - - &AACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ACIRC - STNEUTRAL - 6 - CLS_VAR - - - &ACIRC; - STNEUTRAL - 8 - CLS_VAR - - - &ACUTE - STNEUTRAL - 6 - CLS_VAR - - - &ACUTE; - STNEUTRAL - 7 - CLS_VAR - - - &AELIG - STNEUTRAL - 6 - CLS_VAR - - - &AELIG; - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - & - STNEUTRAL - 4 - CLS_VAR - - - & - STNEUTRAL - 5 - CLS_VAR - - - &ARING - STNEUTRAL - 6 - CLS_VAR - - - &ARING; - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE; - STNEUTRAL - 8 - CLS_VAR - - - &AUML - STNEUTRAL - 5 - CLS_VAR - - - &AUML; - STNEUTRAL - 6 - CLS_VAR - - - &BRVBAR - STNEUTRAL - 7 - CLS_VAR - - - &BRVBAR; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL - STNEUTRAL - 7 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CEDIL - STNEUTRAL - 6 - CLS_VAR - - - &CEDIL; - STNEUTRAL - 7 - CLS_VAR - - - &CENT - STNEUTRAL - 5 - CLS_VAR - - - &CENT; - STNEUTRAL - 6 - CLS_VAR - - - © - STNEUTRAL - 5 - CLS_VAR - - - © - STNEUTRAL - 6 - CLS_VAR - - - &CURREN - STNEUTRAL - 7 - CLS_VAR - - - &CURREN; - STNEUTRAL - 8 - CLS_VAR - - - &DEG - STNEUTRAL - 4 - CLS_VAR - - - &DEG; - STNEUTRAL - 5 - CLS_VAR - - - &DIVIDE - STNEUTRAL - 7 - CLS_VAR - - - &DIVIDE; - STNEUTRAL - 8 - CLS_VAR - - - &EACUTE - STNEUTRAL - 7 - CLS_VAR - - - &EACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC; - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - Ð - STNEUTRAL - 4 - CLS_VAR - - - Ð - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML; - STNEUTRAL - 6 - CLS_VAR - - - &FRAC12 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC12; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC14 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC14; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC34 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC34; - STNEUTRAL - 8 - CLS_VAR - - - > - STNEUTRAL - 3 - CLS_VAR - - - > - STNEUTRAL - 4 - CLS_VAR - - - &IACUTE - STNEUTRAL - 7 - CLS_VAR - - - &IACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ICIRC - STNEUTRAL - 6 - CLS_VAR - - - &ICIRC; - STNEUTRAL - 7 - CLS_VAR - - - &IEXCL - STNEUTRAL - 5 - CLS_VAR - - - &IEXCL; - STNEUTRAL - 6 - CLS_VAR - - - &IGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &IGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &IQUEST - STNEUTRAL - 7 - CLS_VAR - - - &IQUEST; - STNEUTRAL - 8 - CLS_VAR - - - &IUML - STNEUTRAL - 5 - CLS_VAR - - - &IUML; - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO; - STNEUTRAL - 7 - CLS_VAR - - - &LRM - STNEUTRAL - 4 - CLS_VAR - - - &LRM; - STNEUTRAL - 5 - CLS_VAR - - - < - STNEUTRAL - 3 - CLS_VAR - - - < - STNEUTRAL - 4 - CLS_VAR - - - &MACR - STNEUTRAL - 5 - CLS_VAR - - - &MACR; - STNEUTRAL - 6 - CLS_VAR - - - &MICRO - STNEUTRAL - 6 - CLS_VAR - - - &MICRO; - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT; - STNEUTRAL - 8 - CLS_VAR - - - &NBSP - STNEUTRAL - 5 - CLS_WHITESPACE - - - &NBSP; - STNEUTRAL - 6 - CLS_WHITESPACE - - - &NOT - STNEUTRAL - 4 - CLS_VAR - - - &NOT; - STNEUTRAL - 5 - CLS_VAR - - - &NTILDE - STNEUTRAL - 7 - CLS_VAR - - - &NTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OACUTE - STNEUTRAL - 7 - CLS_VAR - - - &OACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &OCIRC - STNEUTRAL - 6 - CLS_VAR - - - &OCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &ORDF - STNEUTRAL - 5 - CLS_VAR - - - &ORDF; - STNEUTRAL - 6 - CLS_VAR - - - &ORDM - STNEUTRAL - 5 - CLS_VAR - - - &ORDM; - STNEUTRAL - 6 - CLS_VAR - - - &OSLASH - STNEUTRAL - 7 - CLS_VAR - - - &OSLASH; - STNEUTRAL - 8 - CLS_VAR - - - &OTILDE - STNEUTRAL - 7 - CLS_VAR - - - &OTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OUML - STNEUTRAL - 5 - CLS_VAR - - - &OUML; - STNEUTRAL - 6 - CLS_VAR - - - &PARA - STNEUTRAL - 5 - CLS_VAR - - - &PARA; - STNEUTRAL - 6 - CLS_VAR - - - &PLUSMN - STNEUTRAL - 7 - CLS_VAR - - - &PLUSMN; - STNEUTRAL - 8 - CLS_VAR - - - &POUND - STNEUTRAL - 6 - CLS_VAR - - - &POUND; - STNEUTRAL - 7 - CLS_VAR - - - " - STNEUTRAL - 5 - CLS_VAR - - - " - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO; - STNEUTRAL - 7 - CLS_VAR - - - ® - STNEUTRAL - 4 - CLS_VAR - - - ® - STNEUTRAL - 5 - CLS_VAR - - - &RLM - STNEUTRAL - 4 - CLS_VAR - - - &RLM; - STNEUTRAL - 5 - CLS_VAR - - - &SECT - STNEUTRAL - 5 - CLS_VAR - - - &SECT; - STNEUTRAL - 6 - CLS_VAR - - - &SHY - STNEUTRAL - 4 - CLS_VAR - - - &SHY; - STNEUTRAL - 5 - CLS_VAR - - - &SUP1 - STNEUTRAL - 5 - CLS_VAR - - - &SUP1; - STNEUTRAL - 6 - CLS_VAR - - - &SUP2 - STNEUTRAL - 5 - CLS_VAR - - - &SUP2; - STNEUTRAL - 6 - CLS_VAR - - - &SUP3 - STNEUTRAL - 5 - CLS_VAR - - - &SUP3; - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG; - STNEUTRAL - 7 - CLS_VAR - - - Þ - STNEUTRAL - 6 - CLS_VAR - - - Þ - STNEUTRAL - 7 - CLS_VAR - - - &TIMES - STNEUTRAL - 6 - CLS_VAR - - - &TIMES; - STNEUTRAL - 7 - CLS_VAR - - - &TRADE - STNEUTRAL - 6 - CLS_VAR - - - ™ - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &UCIRC - STNEUTRAL - 6 - CLS_VAR - - - &UCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &UML - STNEUTRAL - 4 - CLS_VAR - - - &UML; - STNEUTRAL - 5 - CLS_VAR - - - &UUML - STNEUTRAL - 5 - CLS_VAR - - - &UUML; - STNEUTRAL - 6 - CLS_VAR - - - &YACUTE - STNEUTRAL - 7 - CLS_VAR - - - &YACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &YEN - STNEUTRAL - 4 - CLS_VAR - - - &YEN; - STNEUTRAL - 5 - CLS_VAR - - - &YUML - STNEUTRAL - 5 - CLS_VAR - - - &YUML; - STNEUTRAL - 6 - CLS_VAR - - - &ZWJ - STNEUTRAL - 4 - CLS_VAR - - - &ZWJ; - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ; - STNEUTRAL - 6 - CLS_VAR - - - - ENDDEL - 5 - - - - - - - -HTML 3.2/4.01 (UTF-8) -OTMUHTM3 -OTMHTM32 -~ - - #INCLUDE - TSL - 8 - 1 - - - #IF - TSL - 3 - 1 - - - #IFDEF - TSL - 6 - 1 - - - #IFNDEF - TSL - 7 - 1 - - - #ELIF - TSL - 5 - 1 - - - #ELSE - TSL - 5 - 1 - - - #ENDIF - TSL - 6 - 1 - - - #UNDEF - TSL - 6 - 1 - - - $ - 1 - ENDNEUTRAL - - - $$~ - ' $.)\r\n\t>' - ENDNEUTRAL - CLS_TRANSLVAR - - - &AACUTE - STNEUTRAL - 7 - CLS_VAR - - - &AACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ACIRC - STNEUTRAL - 6 - CLS_VAR - - - &ACIRC; - STNEUTRAL - 8 - CLS_VAR - - - &ACUTE - STNEUTRAL - 6 - CLS_VAR - - - &ACUTE; - STNEUTRAL - 7 - CLS_VAR - - - &AELIG - STNEUTRAL - 6 - CLS_VAR - - - &AELIG; - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - & - STNEUTRAL - 4 - CLS_VAR - - - & - STNEUTRAL - 5 - CLS_VAR - - - &ARING - STNEUTRAL - 6 - CLS_VAR - - - &ARING; - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE; - STNEUTRAL - 8 - CLS_VAR - - - &AUML - STNEUTRAL - 5 - CLS_VAR - - - &AUML; - STNEUTRAL - 6 - CLS_VAR - - - &BRVBAR - STNEUTRAL - 7 - CLS_VAR - - - &BRVBAR; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL - STNEUTRAL - 7 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CEDIL - STNEUTRAL - 6 - CLS_VAR - - - &CEDIL; - STNEUTRAL - 7 - CLS_VAR - - - &CENT - STNEUTRAL - 5 - CLS_VAR - - - &CENT; - STNEUTRAL - 6 - CLS_VAR - - - © - STNEUTRAL - 5 - CLS_VAR - - - © - STNEUTRAL - 6 - CLS_VAR - - - &CURREN - STNEUTRAL - 7 - CLS_VAR - - - &CURREN; - STNEUTRAL - 8 - CLS_VAR - - - &DEG - STNEUTRAL - 4 - CLS_VAR - - - &DEG; - STNEUTRAL - 5 - CLS_VAR - - - &DIVIDE - STNEUTRAL - 7 - CLS_VAR - - - &DIVIDE; - STNEUTRAL - 8 - CLS_VAR - - - &EACUTE - STNEUTRAL - 7 - CLS_VAR - - - &EACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC; - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - Ð - STNEUTRAL - 4 - CLS_VAR - - - Ð - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML; - STNEUTRAL - 6 - CLS_VAR - - - &FRAC12 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC12; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC14 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC14; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC34 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC34; - STNEUTRAL - 8 - CLS_VAR - - - > - STNEUTRAL - 3 - CLS_VAR - - - > - STNEUTRAL - 4 - CLS_VAR - - - &IACUTE - STNEUTRAL - 7 - CLS_VAR - - - &IACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ICIRC - STNEUTRAL - 6 - CLS_VAR - - - &ICIRC; - STNEUTRAL - 7 - CLS_VAR - - - &IEXCL - STNEUTRAL - 5 - CLS_VAR - - - &IEXCL; - STNEUTRAL - 6 - CLS_VAR - - - &IGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &IGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &IQUEST - STNEUTRAL - 7 - CLS_VAR - - - &IQUEST; - STNEUTRAL - 8 - CLS_VAR - - - &IUML - STNEUTRAL - 5 - CLS_VAR - - - &IUML; - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO; - STNEUTRAL - 7 - CLS_VAR - - - &LRM - STNEUTRAL - 4 - CLS_VAR - - - &LRM; - STNEUTRAL - 5 - CLS_VAR - - - < - STNEUTRAL - 3 - CLS_VAR - - - < - STNEUTRAL - 4 - CLS_VAR - - - &MACR - STNEUTRAL - 5 - CLS_VAR - - - &MACR; - STNEUTRAL - 6 - CLS_VAR - - - &MICRO - STNEUTRAL - 6 - CLS_VAR - - - &MICRO; - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT; - STNEUTRAL - 8 - CLS_VAR - - - &NBSP - STNEUTRAL - 5 - CLS_WHITESPACE - - - &NBSP; - STNEUTRAL - 6 - CLS_WHITESPACE - - - &NOT - STNEUTRAL - 4 - CLS_VAR - - - &NOT; - STNEUTRAL - 5 - CLS_VAR - - - &NTILDE - STNEUTRAL - 7 - CLS_VAR - - - &NTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OACUTE - STNEUTRAL - 7 - CLS_VAR - - - &OACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &OCIRC - STNEUTRAL - 6 - CLS_VAR - - - &OCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &ORDF - STNEUTRAL - 5 - CLS_VAR - - - &ORDF; - STNEUTRAL - 6 - CLS_VAR - - - &ORDM - STNEUTRAL - 5 - CLS_VAR - - - &ORDM; - STNEUTRAL - 6 - CLS_VAR - - - &OSLASH - STNEUTRAL - 7 - CLS_VAR - - - &OSLASH; - STNEUTRAL - 8 - CLS_VAR - - - &OTILDE - STNEUTRAL - 7 - CLS_VAR - - - &OTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OUML - STNEUTRAL - 5 - CLS_VAR - - - &OUML; - STNEUTRAL - 6 - CLS_VAR - - - &PARA - STNEUTRAL - 5 - CLS_VAR - - - &PARA; - STNEUTRAL - 6 - CLS_VAR - - - &PLUSMN - STNEUTRAL - 7 - CLS_VAR - - - &PLUSMN; - STNEUTRAL - 8 - CLS_VAR - - - &POUND - STNEUTRAL - 6 - CLS_VAR - - - &POUND; - STNEUTRAL - 7 - CLS_VAR - - - " - STNEUTRAL - 5 - CLS_VAR - - - " - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO; - STNEUTRAL - 7 - CLS_VAR - - - ® - STNEUTRAL - 4 - CLS_VAR - - - ® - STNEUTRAL - 5 - CLS_VAR - - - &RLM - STNEUTRAL - 4 - CLS_VAR - - - &RLM; - STNEUTRAL - 5 - CLS_VAR - - - &SECT - STNEUTRAL - 5 - CLS_VAR - - - &SECT; - STNEUTRAL - 6 - CLS_VAR - - - &SHY - STNEUTRAL - 4 - CLS_VAR - - - &SHY; - STNEUTRAL - 5 - CLS_VAR - - - &SUP1 - STNEUTRAL - 5 - CLS_VAR - - - &SUP1; - STNEUTRAL - 6 - CLS_VAR - - - &SUP2 - STNEUTRAL - 5 - CLS_VAR - - - &SUP2; - STNEUTRAL - 6 - CLS_VAR - - - &SUP3 - STNEUTRAL - 5 - CLS_VAR - - - &SUP3; - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG; - STNEUTRAL - 7 - CLS_VAR - - - Þ - STNEUTRAL - 6 - CLS_VAR - - - Þ - STNEUTRAL - 7 - CLS_VAR - - - &TIMES - STNEUTRAL - 6 - CLS_VAR - - - &TIMES; - STNEUTRAL - 7 - CLS_VAR - - - &TRADE - STNEUTRAL - 6 - CLS_VAR - - - ™ - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &UCIRC - STNEUTRAL - 6 - CLS_VAR - - - &UCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &UML - STNEUTRAL - 4 - CLS_VAR - - - &UML; - STNEUTRAL - 5 - CLS_VAR - - - &UUML - STNEUTRAL - 5 - CLS_VAR - - - &UUML; - STNEUTRAL - 6 - CLS_VAR - - - &YACUTE - STNEUTRAL - 7 - CLS_VAR - - - &YACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &YEN - STNEUTRAL - 4 - CLS_VAR - - - &YEN; - STNEUTRAL - 5 - CLS_VAR - - - &YUML - STNEUTRAL - 5 - CLS_VAR - - - &YUML; - STNEUTRAL - 6 - CLS_VAR - - - &ZWJ - STNEUTRAL - 4 - CLS_VAR - - - &ZWJ; - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ; - STNEUTRAL - 6 - CLS_VAR - - - - ENDDEL - 5 - - - " ) ; - if ( str2 ) { - for( i=0 ; str1", 8 ) ; - } - } - if ( ! wcsicmp( temp_str, L"STARTNON-TRANSLATABLEVALUE" ) ) { - *bInNonTransAttrValue = TRUE ; - } else { - if ( ! wcsicmp( temp_str, L"ENDNON-TRANSLATABLEVALUE" ) ) { - *bInNonTransAttrValue = FALSE ; - } - } - } - } - - if ( ( *bInNonTrans ) && - ( ! bSkipStmt ) ) { - for( str1=sub_str ; *str1 ; ++str1 ) { - if ( IsDBCS( *str1 ) ) { - ++str1 ; - continue ; - } - if ( ! wcsncmp( str1, L"", 8 ) ; - str1 += 8 ; - } else - if ( ! wcsncmp( str1, L"-->", 3 ) ) { - str1 += 3 ; - memmove( str1+9, str1, (wcslen(str1)+1)*sizeof(wchar_t) ) ; - wcsncpy( str1, L"", 3 ) ) { - sTagState = TAG_NONE ; - sPrevTagState = TAG_NORMAL ; - if ( bInCommentCheck ) { /* Change '--->' to '- -->' */ - bInCommentCheck = FALSE ; - if ( ( ptrChar > Text ) && - ( *(ptrChar-1) == '-' ) ) { - memmove( ptrChar+1, ptrChar, (wcslen(ptrChar)+1)*sizeof(wchar_t) ) ; - *ptrChar = ' ' ; - } - } - ptrChar += 2 ; - } else { - if ( ! wcsncmp( ptrChar, L" ", 6 ) ) { - ptrChar += 5 ; - } else { - if ( ! wcsncmp( ptrChar, L"[TWB", 4 ) ) { /* JSP internal tags */ - ptrChar2 = wcschr( ptrChar, ']' ) ; - if ( ptrChar2 ) { - if ( sTagState == TAG_SCRIPT ) { - if ( ! wcsncmp( ptrChar, L"[TWBSTART] [TWBLT]%", 19 ) ) { - memmove( ptrChar+7, ptrChar-1, (wcslen(ptrChar-1)+1)*sizeof(wchar_t) ) ; - wcsncpy( ptrChar-1, L"{TWBSCR}", 8 ) ; - ptrChar += 8 ; - sTagState = TAG_JSP ; - sPrevTagState = TAG_SCRIPT ; - } - } else { - if ( sTagState == TAG_JSP ) { - if ( ! wcsncmp( ptrChar, L"[TWBGT] [TWBSTOP] ", 18 ) ) { - memmove( ptrChar+18+10, ptrChar+18, (wcslen(ptrChar+18)+1)*sizeof(wchar_t) ) ; - wcsncpy( ptrChar+18, L" MAXENTITY_READ-2 ) || // (1024-2 ) - ( bIn2K ) ) && - ( !wcschr( Text, '\n' ) ) ) { - if ( !wcschr( Text, '>' ) ) { - ptrChar = wcschr( Text, '<' ) ; - if ( ptrChar ) - ptrChar2 = wcschr( ptrChar+1, '<' ) ; - else - ptrChar2 = 0 ; - if ( ( bIn2K ) || - ( ( ptrChar ) && - ( !ptrChar2 ) && - ( ( iswalpha( *(ptrChar+1) ) ) || - ( ( *(ptrChar+1) == '/' ) && - ( iswalpha( *(ptrChar+2) ) ) ) || - ( ( *(ptrChar+1) == '\\' ) && /* <\/xxxx> */ - ( *(ptrChar+2) == '/' ) && - ( iswalpha( *(ptrChar+3) ) ) ) ) ) ) { - if ( ! bIn2K ) - wcscpy( szQuote, L" " ) ; - bIn2K = TRUE ; - for( ptrChar=Text ; *ptrChar ; ++ptrChar ) { - if ( ( *(ptrChar) == '\\' ) && /* Skip escaped quotes */ - ( ( *(ptrChar+1) == '\"' ) || - ( *(ptrChar+1) == '\'' ) ) ) { - ++ptrChar ; - continue ; - } - if ( ( *ptrChar == '\"' ) || /* Process single/double quote */ - ( *ptrChar == '\'' ) ) { - if ( *(ptrChar+1) == *ptrChar ) { /* Skip consecutive quotes */ - ++ptrChar ; - continue ; - } - if ( szQuote[1] == *ptrChar ) {/* If 1st quote is a match, */ - if ( szQuote[2] == ' ' ) /* If no 2nd quote char, */ - szQuote[1] = ' ' ; /* then end of 1st quote */ - } else { - if ( szQuote[2] == *ptrChar ) /* If 2nd quote is a match,*/ - szQuote[2] = ' ' ; /* then end of 2nd quote */ - else - if ( szQuote[1] == ' ' ) /* If no 1st quote yet, */ - szQuote[1] = *ptrChar ;/* Set 1st quote char */ - else - szQuote[2] = *ptrChar ;/* Set 2nd quote char */ - } - } - } - wcscat( Text, L"{TWB2K}" ) ; - memset( szTemp, 0, 4*sizeof(wchar_t) ) ; - if ( szQuote[2] != ' ' ) { - szTemp[2] = szQuote[2] ; - wcscat( Text, &szTemp[2] ) ; - } - if ( szQuote[1] != ' ' ) { - szTemp[0] = szQuote[1] ; - wcscat( Text, szTemp ) ; - } - wcscat( Text, L">\n Text ) && - ( ( iswalpha( *(ptrChar+1) ) ) || /* */ - ( ( *(ptrChar+1) == '/' ) && /* */ - ( iswalpha( *(ptrChar+2) ) ) ) || - ( ( *(ptrChar+1) == '\\' ) && /* <\/xxxx> */ - ( *(ptrChar+2) == '/' ) && - ( iswalpha( *(ptrChar+3) ) ) ) ) ) { - wcsncpy( szTagName2, ptrChar+1, sizeof(szTagName2)/sizeof(wchar_t) ) ; - szTagName2[sizeof(szTagName2)/sizeof(wchar_t)-1] = 0 ; - wcstok( szTagName2, L" >\n\t\r" ) ; - wcsupr( szTagName2 ) ; - if ( szTagName2[0] == '/' ) - swprintf( szTempTag, L" %s ", &szTagName2[1] ) ; - else - if ( szTagName2[0] == '\\' ) - swprintf( szTempTag, L" %s ", &szTagName2[2] ) ; - else - swprintf( szTempTag, L" %s ", szTagName2 ) ; - if ( wcsstr( TAGLIST_NEUTRAL , szTempTag ) ) { - memmove( ptrChar+15, ptrChar, (wcslen(ptrChar)+1)*sizeof(wchar_t) ) ; - wcsncpy( ptrChar, L"{TWB2K}\n", 15 ) ; - } - } - } - } else { - bIn2K = FALSE ; - } -} - -//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -/***************************************************************************/ -/* Handle , - 8 - - - - 6 - ENDNEUTRAL - - - - 6 - ENDNEUTRAL - - -
- 8 - - - - 8 - - - - 5 - - - - 11 - - - - 8 - - - - 5 - - - - 8 - - - - 8 - - - - 5 - - - - 5 - ENDNEUTRAL - - - - 4 - ENDNEUTRAL - - - - 5 - - - - 6 - ENDNEUTRAL - - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.c b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.c deleted file mode 100755 index 64178b75..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.c +++ /dev/null @@ -1,691 +0,0 @@ -/* -* -* Copyright (C) 1998-2013 International Business Machines -* Corporation and others. All rights reserved -* -*/ -/****************************************************************************/ -/* */ -/* otmjdk11.c */ -/* */ -/****************************************************************************/ -/* */ -/* Description: */ -/* This function will examine a Java Resource Bundle and do the */ -/* segmentation processing. This DLL will have 4 entry points. */ -/* These entry points are listed below and are defined by TM2. */ -/* */ -/* */ -/* EQF_BOOL APIENTRY16 EQFPRESEG (PSZ pTagTable, */ -/* PSZ pEdit, */ -/* PSZ pProgPath, */ -/* PSZ pSource, */ -/* PSZ pTempSource, */ -/* PEQF_BOOL pfNoSegment) */ -/* EQF_BOOL APIENTRY16 EQFPOSTSEG (PSZ pTagTable, */ -/* PSZ pEdit, */ -/* PSZ pProgPath, */ -/* PSZ pSource, */ -/* PSZ pSegTarget, */ -/* PTATAG pTATAG) */ -/* EQF_BOOL APIENTRY16 EQFPREUNSEG (PSZ pTagTable, */ -/* PSZ pEdit, */ -/* PSZ pProgPath, */ -/* PSZ pSegTarget, */ -/* PSZ pTemp, */ -/* PTATAG pTATAG, */ -/* PEQF_BOOL pfNoUnseg) */ -/* EQF_BOOL APIENTRY16 EQFPOSTUNSEG (PSZ pTagTable, */ -/* PSZ pEdit, */ -/* PSZ pProgPath, */ -/* PSZ pTarget, */ -/* PTATAG pTATAG) */ -/* */ -/*==========================================================================*/ -/* Date : Who : Description */ -/*==========================================================================*/ -/* 7/08/2010: IBM : Original Source */ -/*==========================================================================*/ -/****************************************************************************/ - -#include "otmjdk11.h" - - -extern char szDocTargetLanguage[80]; /* From USRCALLS.C */ -extern char szDocSourceLanguage[80]; /* From USRCALLS.C */ -extern short sTPVersion ; /* From USRCALLS.C */ - BOOL bDBCSSrc ; - BOOL bDBCSTgt ; - BOOL bDBCSInfo ; - BOOL bJapanese ; - BOOL bKorean ; - BOOL bSChinese ; - BOOL bTChinese ; - BOOL bNoDBCS ; - - - -/****************************************************************************/ -/* */ -/* EQFPRESEG */ -/* */ -/****************************************************************************/ -/* */ -/* Description: */ -/* called after text segmentation is invoked, to change the */ -/* segmented source and target file before translation takes place. */ -/* */ -/* Arguments: PSZ ... pointer to markup table name. */ -/* */ -/* PSZ ... pointer to editor name. */ -/* */ -/* PSZ ... pointer to program path. */ -/* */ -/* PSZ ... pointer to segmented source file name. */ -/* */ -/* PSZ ... pointer to the segmented target file name. */ -/* */ -/* PTATAG ... pointer to tags inserted by text segmentation,*/ -/* ( see layout of tag structure ). */ -/* */ -/* Return: BOOL ... TRUE:processing was OK. */ -/* FALSE: an error occured during processing. */ -/* */ -/* Note: It is vital that the name of the segmented source and target */ -/* file is not changed! */ -/* */ -/* */ -/****************************************************************************/ - - -/*************************************************************************/ -/* Pre-segmentation, including access to progress window. */ -/*************************************************************************/ - - -EQF_BOOL /*APIENTRY*/ EQFPOSTSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pSegSource, - PSZ pSegTarget, - PTATAG_W pTATAGW, - HWND hSlider, - PEQF_BOOL pfKill ) - -{ - - return(TRUE); - -} - - -/*************************************************************************/ -/* Pre-unsegmentation, including access to progress window. */ -/*************************************************************************/ - -EQF_BOOL /*APIENTRY*/ EQFPREUNSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pSegTarget, - PSZ pTemp, - PTATAG_W pTATagW, - PEQF_BOOL pfNoUnseg, - PEQF_BOOL pfKill ) -{ - UCHAR tmpfilename[1024]; - UCHAR tmpfilename2[1024]; - char *szAltTempExt3 = ".$$3"; - char *szAltTempExt4 = ".$$4"; - short sType ; - BOOL bReturn = TRUE ; - BOOL bDBCS ; - - - PrepDocLanguageInfo( pSegTarget ) ; /* Set language unqiue processing, like DBCS */ - bDBCSSrc = SetDBCSLang( szDocSourceLanguage ) ; - bDBCSTgt = SetDBCSLang( szDocTargetLanguage ) ; - bDBCS = IsDBCS( '\x90' ) ; - - *pfNoUnseg = FALSE ; /* Unsegmentation is required. */ - - TM2ConvertShortToLong( pSegTarget, tmpfilename ); - - if (tmpfilename[0] == 0) { - strcpy(tmpfilename, pProgPath); - } - - // Check if LRB or Property file and process as required. - - CreateTempFileName2( pTemp, pSegTarget, szAltTempExt3, TEMPNAME_STARGET ) ; - bReturn = ConvertImport( pSegTarget, pTemp, EQF_UTF162UTF8 ) ; /* UTF-16 -> UTF-8 */ - - - if ( bReturn ) { - CreateTempFileName2( tmpfilename2, pSegTarget, szAltTempExt4, TEMPNAME_STARGET ) ; - - if ( ! chk_LRB_File( tmpfilename, pTemp, &sType, pTagTable ) ) { - - bReturn = ! ExportFile( pTemp, tmpfilename2 ); - - if ( bReturn ) { - bReturn = ConvertImport( tmpfilename2, pTemp, EQF_UTF82UTF16 ) ; /* UTF-8 to UTF-16 */ - } - - } else { - - bReturn = ExportLRBFile( pTemp, tmpfilename2, pTemp ) ; - - if ( bReturn ) { - bReturn = ConvertImport( tmpfilename2, pTemp, EQF_UTF82UTF16 ) ; /* UTF-8 to UTF-16 */ - } - - } - remove( tmpfilename2 ) ; - } - - - return( bReturn ) ; - -} /* EQFPREUNSEG2 */ - - - - - - -/*************************************************************************/ -/* Post-unsegmentation, including access to progress window. */ -/*************************************************************************/ - -EQF_BOOL /*APIENTRY*/ EQFPOSTUNSEGW( - PSZ pTagTable, - PSZ pEdit, - PSZ pProgPath, - PSZ pTarget, - PTATAG pTATag, - PEQF_BOOL pfKill ) -{ - char szTempFile[256] ; - char szMarkupTable[80] ; - UCHAR tmpfilename[1024]; - char *szAltTempExt5 = ".$$5"; - short sType ; - USHORT usCPType = EXPORT_CP_NONE ; - BOOL bReturn = TRUE ; - - - PrepDocLanguageInfo( pTarget ) ; /* Set language unqiue processing, like DBCS */ - bDBCSSrc = SetDBCSLang( szDocSourceLanguage ) ; - bDBCSTgt = SetDBCSLang( szDocTargetLanguage ) ; - - TM2ConvertShortToLong( pTarget, tmpfilename ); - - if (tmpfilename[0] == 0) - strcpy(tmpfilename, pProgPath); - - /*************************************************************************/ - /* Convert ASCII file into UTF-8 for export. */ - /*************************************************************************/ - strcpy( szMarkupTable, pTagTable ) ; - strupr( szMarkupTable ) ; - if ( ! strncmp( szMarkupTable, "OTMU", 4 ) ) { - usCPType = EXPORT_CP_UTF8 ; - bReturn = ConvertExport( pTarget, EQF_UTF162UTF8 ) ; /* UTF-16 -> UTF-8 */ - } else { - if ( ! strncmp( szMarkupTable, "OTMA", 4 ) ) { /* ANSI markup table */ - usCPType = EXPORT_CP_DBCS_ASCII ; - bReturn = ConvertExport( pTarget, EQF_UTF162ANSI ) ; /* UTF-16 -> ANSI */ - if ( ! strcmp( szDocTargetLanguage, "Kazakh" ) ) /* Kazakh */ - usCPType = EXPORT_CP_UTF8 ; - } else { - if ( ! strncmp( szMarkupTable, "OTMNJDK", 7 ) ) { - bReturn = ConvertToEscapedUnicode( pTarget ) ; /* UTF-16 -> Escaped Unicode */ - } else { - usCPType = EXPORT_CP_DBCS_ASCII ; - bReturn = ConvertExport( pTarget, EQF_UTF162ASCII ) ;/* UTF-16 -> ASCII */ - if ( ! strcmp( szDocTargetLanguage, "Kazakh" ) ) /* Kazakh */ - usCPType = EXPORT_CP_UTF8 ; - } - } - } - - - // Check if Property file, then process as required. - - if ( ! chk_LRB_File(tmpfilename, pTarget, &sType, pTagTable ) ) { - - CreateTempFileName2( szTempFile, pTarget, szAltTempExt5, TEMPNAME_TARGET ) ; - - ExportFile2( pTarget, szTempFile, usCPType ); - - - remove( szTempFile ) ; - - } - - return( bReturn ); -} - - -/*******************************************************************/ -/* */ -/* Check if file is a ListResourceBundle Type file */ -/* */ -/*******************************************************************/ - - -BOOL chk_LRB_File(char *sourceName, char *shortName, short *sType, - char* MarkupName ) -{ - FILE *sourceFile_ptr = NULL; - char line[MAX_LEN]; - char upperCaseBuff[MAX_LEN+1]; - PSZ pName, pDot ; - BOOL bLRB = FALSE ; - - *sType = LRB_NONE ; - memset(upperCaseBuff, 0, sizeof(upperCaseBuff)); - - strcpy(upperCaseBuff, sourceName); - StrUpr(upperCaseBuff); - - pName = strrchr( upperCaseBuff, '\\' ); /* Find file name */ - pDot = strrchr( upperCaseBuff, '.' ); /* Find file extension */ - - if (pName == NULL) { - pName = upperCaseBuff; - } - - if ( ( pDot > pName ) && /* 10-25-01 */ - ( ( ! strcmp( pDot, ".JAV" ) ) || - ( ! strcmp( pDot, ".JAVA" ) ) ) ) { - bLRB = TRUE; - } - - sourceFile_ptr = fopen (shortName,"rb"); - while (fgets(line, MAX_LEN, sourceFile_ptr)) { - memset(upperCaseBuff, 0, sizeof(upperCaseBuff)); - strcpy(upperCaseBuff, line); - StrUpr(upperCaseBuff); - if ( ( strstr( upperCaseBuff, "IMPORT JAVA." ) != NULL ) && - ( strchr( upperCaseBuff, ';' ) != NULL ) ) { /* 4-3-08 */ - bLRB = TRUE; - } - if ( strstr( upperCaseBuff, "EXTENDS" ) != NULL ) { - if ( strstr( upperCaseBuff, "RESOURCEBUNDLE" ) != NULL ) { - bLRB = TRUE; - } - break ; - } - } - fclose(sourceFile_ptr); - - - if ( *sType == LRB_NONE ) { - if ( bLRB ) { - *sType = LRB_STANDARD ; - } else { - *sType = PRB_STANDARD ; - } - } - return( bLRB ) ; -} - - - -/****************************************************************************/ -/* */ -/* TM2ConvertShortToLong */ -/* */ -/* Convert the TM/2 short file name to the long file name. */ -/* */ -/* szPath - Complete short name path, including TM/2 folder, file */ -/* szFileName - pointer to Empty string */ -/* */ -/* */ -/****************************************************************************/ -USHORT TM2ConvertShortToLong( UCHAR *szPath, UCHAR *szLong ) -{ - UCHAR szTempPath[MAX_LEN]; - UCHAR szShortName[MAX_LEN]; - UCHAR *path_ptr ; - UCHAR *short_ptr ; - UCHAR *temp_ptr ; - USHORT rc ; - - strcpy( szTempPath, szPath ) ; - strcpy( szShortName, szPath ) ; - - /* Pull folder path out of the full path name name FOLDER_DRIVE:\EQF\FOLDER_NAME.F00 */ - - path_ptr = (strstr( szTempPath, szFOLDER_EXT )); - if ( path_ptr ) { - *(path_ptr+2)= 0 ; - } else { - rc=10; // Return fail no folder path - return rc; - } - - -/* Pull short name out of the full path name name NAME.000 */ - - short_ptr = strrchr(szShortName, '\\'); - - if (short_ptr ) { - short_ptr += 1 ; - } else { - rc = 11; // Return fail no short filename - return rc; - } - temp_ptr = strrchr( short_ptr, '.' ) ; - if ( ( temp_ptr ) && - ( strlen(temp_ptr) == 5 ) ) { - *(temp_ptr+4) = 0 ; // Remove trailing letter from name NAME.000A - } - szLong[0] = 0 ; // Clear the long filename buff TM/2 requires to know which conversion to do - - -//kml : P007994 do conversion to appropriate filename - rc = EQFCONVERTFILENAMES( szTempPath, szLong, short_ptr ) ; - if ( (rc == 0) && (szLong[0] != 0) ){ - strcpy( short_ptr, szLong ) ; - } else { - strcpy( szLong, short_ptr ) ; - } - - return rc ; -} - - - -/****************************************************************************/ -/* */ -/* ConvertFromEscapedUnicode */ -/* */ -/* Convert the Java escaped Unicode data into UTF-16. */ -/* */ -/* szInFile - Input file (escaped Unicode). */ -/* szOutFile - Output file (UTF-16). */ -/* */ -/****************************************************************************/ -USHORT ConvertFromEscapedUnicode( UCHAR *szInFile, UCHAR *szOutFile ) -{ - FILE *fInput ; - FILE *fOutput ; - CHAR szInChars[10] ; - CHAR cChar ; - LONG lFilePos ; - USHORT usChar ; - USHORT usChar1 ; - USHORT usChar2 ; - USHORT i ; - USHORT rc = 1 ; - BOOL bSkipRead = FALSE ; - BOOL bLoop = TRUE ; - - - fInput = fopen( szInFile, "rb" ) ; - fOutput = fopen( szOutFile, "wb" ) ; - fprintf( fOutput, "\xFF\xFE" ) ; /* Write BOM x'FFFE" for UTF-16 file */ - - /*------------------------------------------------------------------------*/ - /* Process all characters in this file. */ - /*------------------------------------------------------------------------*/ - while( bLoop==TRUE && rc==1 ) { - if ( ! bSkipRead ) { - usChar1 = fgetc( fInput ) ; - if ( feof(fInput) ) - break ; - } - bSkipRead = FALSE ; - - if ( usChar1 == '\\' ) { /* Escaped character */ - usChar2 = fgetc( fInput ) ; /* Get next character */ - if ( usChar2 != 'u' ) { /* If not Unicode, */ - fprintf( fOutput, "%c", usChar1 ) ; /* then write out '\' */ - fputc( '\x00', fOutput ) ; - if ( usChar2 == '\\' ) { /* If escaped backslash, */ - fprintf( fOutput, "%c", usChar2 ) ; - fputc( '\x00', fOutput ) ; - } else { - usChar1 = usChar2 ; - bSkipRead = TRUE ; /* Next char already read */ - } - continue ; - } - lFilePos = ftell( fInput ) ; - for( i=0 ; i<4 ; ++i ) { /* Escaped Unicode \u____ */ - cChar = (UCHAR)fgetc(fInput) ; /* Get next character */ - if ( isxdigit(cChar) ) { /* If valid hex digit */ - if ( isdigit(cChar) ) { /* Set hex digit value */ - usChar = cChar - '0' ; /* 0 - 9 */ - } else - if (cChar <= 'F' ) { - usChar = cChar - 'A' + 10 ; /* A - F */ - } else { - usChar = cChar - 'a' + 10 ; /* a - f */ - } - szInChars[i] = usChar ; /* Save value */ - } else { - rc = 0 ; - fseek( fInput, lFilePos, 0 ) ; /* Reset file reading pos */ - fputc( '\\', fOutput ) ; - fputc( '\x00', fOutput ) ; - fputc( 'u', fOutput ) ; - fputc( '\x00', fOutput ) ; - break ; - } - } - - if ( rc != 0 ) { /* Write UTF-16 value */ - if ( ( szInChars[2] == 5 ) && - ( szInChars[3] == 12 ) && - ( szInChars[0] == 0 ) && - ( szInChars[1] == 0 ) ) { - fwprintf( fOutput, L"\\u005%c",cChar ) ; - } else - if ( ( ( szInChars[3] == 0 ) || - ( szInChars[3] == 10 ) || - ( szInChars[3] == 13 ) ) && - ( szInChars[0] == 0 ) && - ( szInChars[1] == 0 ) && - ( szInChars[2] == 0 ) ) { - fwprintf( fOutput, L"\\u000%c",cChar ) ; - } else - if ( ( szInChars[0] == 15 ) && - ( szInChars[1] == 15 ) && - ( szInChars[2] == 15 ) && - ( ( szInChars[3] == 12 ) || - ( szInChars[3] == 13 ) ) ) { - fwprintf( fOutput, L"\\uFFF%c",cChar ) ; - } else { - fprintf( fOutput, "%c", (UCHAR)(szInChars[2]*16+szInChars[3]) ) ; - fprintf( fOutput, "%c", (UCHAR)(szInChars[0]*16+szInChars[1]) ) ; - } - } else { - rc = 1 ; - } - } else { - fprintf( fOutput, "%c", usChar1 ) ; /* Write non-escaped char */ - fputc( '\x00', fOutput ) ; - } - } - fclose( fInput ) ; - fclose( fOutput ) ; - - return rc ; -} - - - -/****************************************************************************/ -/* */ -/* ConvertToEscapedUnicode */ -/* */ -/* Convert the UTF-16 data into Java escaped Unicode. */ -/* */ -/* szFile - Input file (UTF-16). */ -/* */ -/****************************************************************************/ -USHORT ConvertToEscapedUnicode( UCHAR *szFile ) -{ - FILE *fInput ; - FILE *fOutput ; - UCHAR szTempFile[256]; - char *szAltTempExt6 = ".$$6"; - CHAR szOutChars[10] ; - USHORT usChar1 ; - USHORT usChar2 ; - USHORT rc = 1 ; - BOOL bLoop = TRUE ; - - CreateTempFileName2( szTempFile, szFile, szAltTempExt6, TEMPNAME_TARGET ) ; - - fInput = fopen( szFile, "rb" ) ; - fOutput = fopen( szTempFile, "wb" ) ; - usChar1 = fgetc( fInput ) ; /* Skip BOM x'FFFE' for UTF-16 file */ - usChar2 = fgetc( fInput ) ; - - /*----------------------------------------------------------------------*/ - /* Process all characters in this file. */ - /*----------------------------------------------------------------------*/ - while( bLoop == TRUE ){ - usChar1 = fgetc( fInput ) ; /* Get next UTF-16 char */ - usChar2 = fgetc( fInput ) ; - if ( feof(fInput) ) - break ; - - if ( ( usChar1 < 128 ) && /* If non-escaped char, */ - ( usChar2 == 0 ) ) { - szOutChars[0] = usChar1 ; /* Write out 1 character */ - szOutChars[1] = 0 ; - } else { /* Write escaped Unicode */ - sprintf(szOutChars, "\\u%2.2x%2.2x", usChar2, usChar1 ) ; - } - fputs( szOutChars,fOutput ) ; - - } - fclose( fInput ) ; - fclose( fOutput ) ; - - CopyFile( szTempFile, szFile, FALSE); - remove( szTempFile ) ; - - return rc ; -} - - - -/****************************************************************************/ -/* */ -/* GetCharLength */ -/* */ -/* Determine the length of this character. */ -/* */ -/* Check for differences between ANSI/ASCII and UTF-8. */ -/* Unicode UTF-8 */ -/* Min Max Byte Sequence in Binary */ -/* --------------- ----------------------------------- */ -/* 0000 007F 0vvvvvvv */ -/* 0080 07FF 110vvvvv 10vvvvvv */ -/* 0800 FFFF 1110vvvv 10vvvvvv 10vvvvvv */ -/* 10000 10FFFF 11110vvv 10vvvvvv 10vvvvvv 10vvvvvv */ -/* */ -/* Input: szInput - Input string to check. */ -/* Return: - If UTF-8 char, then return the length of char. */ -/* - If note UTF-8, return 1. */ -/* */ -/****************************************************************************/ - -SHORT GetCharLength( UCHAR * szCheck ) -{ - SHORT sLength = 1 ; - - if ( szCheck[0] >= (UCHAR)'\x80' ) { /* Possible UTF-8 */ - if ( ( szCheck[1] >= (UCHAR)'\x80' ) && - ( szCheck[1] <= (UCHAR)'\xBF' ) ) { - sLength = 2 ; /* Valid 2-byte UTF8 */ - if ( szCheck[0] >= (UCHAR)'\xE0' ) { - if ( ( szCheck[2] >= (UCHAR)'\x80' ) && - ( szCheck[2] <= (UCHAR)'\xBF' ) ) { - sLength = 3 ; /* Valid 3-byte UTF8 */ - if ( szCheck[0] >= (UCHAR)'\xF0' ) { - if ( ( szCheck[3] >= (UCHAR)'\x80' ) && - ( szCheck[3] <= (UCHAR)'\xBF' ) ) { - sLength = 4 ; /* Valid 4-byte UTF8 */ - if ( szCheck[0] >= (UCHAR)'\xF8' ) { - sLength = 1 ; /* Not UTF-8 character */ - } - } else { - sLength = 1 ; /* Not UTF-8 character */ - } - } - } else { - sLength = 1 ; /* Not UTF-8 character */ - } - } - } else { - sLength = 1 ; /* Not UTF-8 character */ - } - } -//printf("CC: %d ##%s##\n",sLength,szCheck); - - return( sLength ) ; -} - -/****************************************************************************/ -/* */ -/* SetDBCSLang */ -/* */ -/* Determine if language is DBCS. */ -/* */ -/* Input: szLangauge - Language to chekc. */ -/* Return: - TRUE. Language is DBCS. */ -/* - FALSE. Language is not DBCS. */ -/* */ -/****************************************************************************/ - -BOOL SetDBCSLang( UCHAR * szLanguage ) -{ - if ( ( ! stricmp( szLanguage, "Japanese" ) ) || - ( ! stricmp( szLanguage, "Korean" ) ) || - ( ! stricmp( szLanguage, "Chinese(simpl.)" ) ) || - ( ! stricmp( szLanguage, "Chinese(trad.)" ) ) ) - return( TRUE ) ; - - return( FALSE ) ; -} - - -/******************************************************************************* -* -* function: EQFQUERYEXITINFO -* -* ----------------------------------------------------------------------------- -* Description: -* Determine the files which are required for this markup table. -* Parameters: -* PSZ // name of the markup table, e.g. "OTMHTM32" -* USHORT // type of information being queried -* PSZ // buffer area receiving the information returned by the exit -* USHORT // length of buffer area -* Return: -* EQF_BOOL // 0=Successful -*******************************************************************************/ - - -USHORT /*APIENTRY*/ EQFQUERYEXITINFO( PSZ pszTagTable, // name of the markup table, e.g. "OTMHTM32" - USHORT usMode, // type of information being queried - PSZ pszBuffer, // buffer area receiving the information returned by the exit - USHORT usBufLen // length of buffer area -) -{ - - if( usMode == QUERYEXIT_ADDFILES) { - QueryExportFiles(pszTagTable, pszBuffer, usBufLen, FALSE); - } - - return 0; -} diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.h b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.h deleted file mode 100755 index a39fb220..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk11.h +++ /dev/null @@ -1,73 +0,0 @@ -/* -* -* Copyright (C) 1998-2013, International Business Machines -* Corporation and others. All rights reserved -* -*/ -/********************************************************************* - * - * FUNCTION: - * - * Segmentation routine to segment JDK 1.1 Property files - * for Translation in TM/2 - * - ********************************************************************/ - -/*==========================================================================*/ -/* Date : Who : Description */ -/*==========================================================================*/ -/* 7/08/2010: IBM : Original Source */ -/*==========================================================================*/ - -/* Header file for otmjdk11.dll */ - -#ifndef _OTMJDK11_H_INCLUDE_ - #define _OTMJDK11_H_INCLUDE_ - - #define INCL_DOSNLS - #define INCL_WINWINDOWMGR - #define INCL_WININPUT - #define INCL_DOSFILEMGR - #define INCL_DOSERRORS - #define INCL_BASE - #define MAX_SIZE 144 - #define MAX_LEN 1024 - #define MAX_SEGMENT_SIZE 2048 - #define MAX_CONTEXT_SIZE 2048 - #define szFOLDER_EXT "00\\" - - #define LRB_NONE 0 - #define LRB_STANDARD 1 - - #define PRB_STANDARD 1 - - #define EXPORT_CP_NONE 0 - #define EXPORT_CP_DBCS_ASCII 1 - #define EXPORT_CP_UTF8 2 - - #include - #include - #include - #include - #include - #include - #include "usrcalls.h" - #include "reseq.h" - #include "javaseg.h" - #include "javapro.h" - - #include - #include - - - -BOOL chk_LRB_File(char *sourceName, char *shortName, short *Type, char*MarkupName ); -USHORT TM2ConvertShortToLong( UCHAR *szPath, UCHAR *szFileName ); -USHORT ConvertFromEscapedUnicode( UCHAR *szInFile, UCHAR *szOutFile ); -USHORT ConvertToEscapedUnicode( UCHAR *szFile ); -SHORT GetCharLength( UCHAR *szText ); -BOOL SetDBCSLang( UCHAR *szLanguage ); -BOOL IsHTMLTag( UCHAR *, USHORT * ) ; - -#endif - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.mak b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.mak deleted file mode 100755 index 4fa7435b..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.mak +++ /dev/null @@ -1,81 +0,0 @@ -# --------------------------------------------------------------------------- -# Copyright (C) 1998-2013, International Business Machines -# Corporation and others. All rights reserved -# --------------------------------------------------------------------------- -SRCPATH = $(_OTMMARKUP_SRC)\otmjdk21 -CMNPATH = $(_OTMMARKUP_SRC)\common -INCLUDEOPT = /I$(SRCPATH) - - -CFLAGS = $(_OTMMARKUP_CFLAGS) $(INCLUDEOPT) -LINKFLAGS = $(_OTMMARKUP_LINKFLAGS) - - -.all: \ - $(_OTMMARKUP_OBJ)\otmjdk11.obj \ - $(_OTMMARKUP_OBJ)\javapro.obj \ - $(_OTMMARKUP_OBJ)\javaseg.obj \ - $(_OTMMARKUP_OBJ)\jcontext.obj \ - $(_OTMMARKUP_DLL)\otmjdk11.dll - -#--------------------------------------- -# Compile OBJs -#--------------------------------------- -$(_OTMMARKUP_OBJ)\otmjdk11.obj: \ - $(SRCPATH)\otmjdk11.c \ - $(SRCPATH)\otmjdk11.h \ - $(SRCPATH)\javapro.h \ - $(SRCPATH)\javaseg.h \ - $(CMNPATH)\usrcalls.h \ - $(CMNPATH)\reseq.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\otmjdk11 $(SRCPATH)\otmjdk11.c - -$(_OTMMARKUP_OBJ)\javapro.obj: \ - $(SRCPATH)\javapro.c \ - $(SRCPATH)\javapro.h \ - $(SRCPATH)\otmjdk11.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\javapro $(SRCPATH)\javapro.c - -$(_OTMMARKUP_OBJ)\javaseg.obj: \ - $(SRCPATH)\javaseg.c \ - $(SRCPATH)\javaseg.h \ - $(SRCPATH)\otmjdk11.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\javaseg $(SRCPATH)\javaseg.c - -$(_OTMMARKUP_OBJ)\jcontext.obj: \ - $(SRCPATH)\context.c \ - $(SRCPATH)\otmjdk11.h - $(_COMPILER) $(CFLAGS) /Fo$(_OTMMARKUP_OBJ)\jcontext $(SRCPATH)\context.c - -#--------------------------------------- -# Link DLL -#--------------------------------------- -$(_OTMMARKUP_DLL)\otmjdk11.dll: \ - $(_OTMMARKUP_OBJ)\otmjdk11.obj \ - $(_OTMMARKUP_OBJ)\javapro.obj \ - $(_OTMMARKUP_OBJ)\javaseg.obj \ - $(_OTMMARKUP_OBJ)\jcontext.obj \ - $(_OTMMARKUP_OBJ)\usrcalls.obj \ - $(_OTMMARKUP_OBJ)\reseq.obj \ - $(_OTMMARKUP_OBJ)\eqfcalls.obj - @if exist $(_OTMMARKUP_DLL)\otmjdk11.dll erase $(_OTMMARKUP_DLL)\otmjdk11.dll - @echo ---- Linking $(_OTMMARKUP_DLL)\otmjdk11.dll - @echo ---- Linking $(_OTMMARKUP_DLL)\otmjdk11.dll >>$(_ERR) - @$(_LINKER) >>$(_ERR) @<< -$(_OTMMARKUP_OBJ)\otmjdk11.obj -$(_OTMMARKUP_OBJ)\javapro.obj -$(_OTMMARKUP_OBJ)\javaseg.obj -$(_OTMMARKUP_OBJ)\jcontext.obj -$(_OTMMARKUP_OBJ)\usrcalls.obj -$(_OTMMARKUP_OBJ)\reseq.obj -$(_OTMMARKUP_OBJ)\eqfcalls.obj -/OUT:$(_OTMMARKUP_DLL)\otmjdk11.dll $(LINKFLAGS) -/MAP:$(_OTMMARKUP_MAP)\otmjdk11.map -user32.lib -<< - @copy $(_OTMMARKUP_DLL)\otmjdk11.dll $(_OTMMARKUP_RELEASE_DIR)\BIN /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmjdk21\otmjdk21.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmjdk21\otmajdk2.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmjdk21\otmnjdk2.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - @copy $(_OTMMARKUP_SRC)\otmjdk21\otmujdk2.tbl $(_OTMMARKUP_RELEASE_DIR)\TABLE /Y>$(_ERR) - diff --git a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.tbx b/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.tbx deleted file mode 100755 index 864d4c4d..00000000 --- a/source/opentm2/plugins/markup/OtmMarkupTablePlugin/OTMJDK21/otmjdk21.tbx +++ /dev/null @@ -1,3247 +0,0 @@ - - - - - -Java List/Prop. ResourceBundle (ASCII) -OTMJDK21 -OTMJDK11 -~ - - - > - STNEUTRAL - - - &TWBQ; - 6 - STNEUTRAL - - - &TWBQ;{ - 7 - STNEUTRAL - - - &TWBEQ; - 7 - STNEUTRAL - - - \\\\ - STNEUTRAL - 2 - - - \\B - STNEUTRAL - 2 - - - \\F - STNEUTRAL - 2 - - - \\N - STNEUTRAL - 2 - - - \\R - STNEUTRAL - 2 - - - \\T - STNEUTRAL - 2 - - - \\V - STNEUTRAL - 2 - - - '\\ ' - STNEUTRAL - 2 - - - \\' - STNEUTRAL - 2 - - - \\" - STNEUTRAL - 2 - - - \\: - STNEUTRAL - 2 - - - \\= - STNEUTRAL - 2 - - - \\# - STNEUTRAL - 2 - - - \\! - STNEUTRAL - 2 - - - \\U000A - STNEUTRAL - 6 - - - \\U000D - STNEUTRAL - 6 - - - {0~ - } - STNEUTRAL - - - {1~ - } - STNEUTRAL - - - {2~ - } - STNEUTRAL - - - {3~ - } - STNEUTRAL - - - {4~ - } - STNEUTRAL - - - {5~ - } - STNEUTRAL - - - {6~ - } - STNEUTRAL - - - {7~ - } - STNEUTRAL - - - {8~ - } - STNEUTRAL - - - {9~ - } - STNEUTRAL - - - ${~ - } - STNEUTRAL - - - {TWB~ - } - STNEUTRAL - - - {TWBP}~$ - } - STNEUTRAL - - - $$?~$ - $ - STNEUTRAL - - - @@?~@ - @ - STNEUTRAL - - - &1; - 3 - STNEUTRAL - CLS_VAR - - - &2; - 3 - STNEUTRAL - CLS_VAR - - - &3; - 3 - STNEUTRAL - CLS_VAR - - - &4; - 3 - STNEUTRAL - CLS_VAR - - - &5; - 3 - STNEUTRAL - CLS_VAR - - - &6; - 3 - STNEUTRAL - CLS_VAR - - - &7; - 3 - STNEUTRAL - CLS_VAR - - - &8; - 3 - STNEUTRAL - CLS_VAR - - - &9; - 3 - STNEUTRAL - CLS_VAR - - - <%=~% - > - STNEUTRAL - CLS_VAR - - - <%= ~% - > - STNEUTRAL - CLS_VAR - - - <%=\n~% - > - STNEUTRAL - CLS_VAR - - - <%=\r~% - > - STNEUTRAL - CLS_VAR - - - <%=\t~% - > - STNEUTRAL - CLS_VAR - - - & - STNEUTRAL - 4 - CLS_VAR - - - & - STNEUTRAL - 5 - CLS_VAR - - - > - STNEUTRAL - 3 - CLS_VAR - - - > - STNEUTRAL - 4 - CLS_VAR - - - < - STNEUTRAL - 3 - CLS_VAR - - - < - STNEUTRAL - 4 - CLS_VAR - - - &NBSP - STNEUTRAL - 5 - CLS_WHITESPACE - - - &NBSP; - STNEUTRAL - 6 - CLS_WHITESPACE - - - " - STNEUTRAL - 5 - CLS_VAR - - - " - STNEUTRAL - 6 - CLS_VAR - - - &AACUTE - STNEUTRAL - 7 - CLS_VAR - - - &AACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ACIRC - STNEUTRAL - 6 - CLS_VAR - - - &ACIRC; - STNEUTRAL - 8 - CLS_VAR - - - &ACUTE - STNEUTRAL - 6 - CLS_VAR - - - &ACUTE; - STNEUTRAL - 7 - CLS_VAR - - - &AELIG - STNEUTRAL - 6 - CLS_VAR - - - &AELIG; - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &AGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - & - STNEUTRAL - 4 - CLS_VAR - - - & - STNEUTRAL - 5 - CLS_VAR - - - &ARING - STNEUTRAL - 6 - CLS_VAR - - - &ARING; - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE - STNEUTRAL - 7 - CLS_VAR - - - &ATILDE; - STNEUTRAL - 8 - CLS_VAR - - - &AUML - STNEUTRAL - 5 - CLS_VAR - - - &AUML; - STNEUTRAL - 6 - CLS_VAR - - - &BRVBAR - STNEUTRAL - 7 - CLS_VAR - - - &BRVBAR; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL - STNEUTRAL - 7 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CCEDIL; - STNEUTRAL - 8 - CLS_VAR - - - &CEDIL - STNEUTRAL - 6 - CLS_VAR - - - &CEDIL; - STNEUTRAL - 7 - CLS_VAR - - - &CENT - STNEUTRAL - 5 - CLS_VAR - - - &CENT; - STNEUTRAL - 6 - CLS_VAR - - - © - STNEUTRAL - 6 - CLS_VAR - - - &CURREN - STNEUTRAL - 7 - CLS_VAR - - - &CURREN; - STNEUTRAL - 8 - CLS_VAR - - - &DEG - STNEUTRAL - 4 - CLS_VAR - - - &DEG; - STNEUTRAL - 5 - CLS_VAR - - - &DIVIDE - STNEUTRAL - 7 - CLS_VAR - - - &DIVIDE; - STNEUTRAL - 8 - CLS_VAR - - - &EACUTE - STNEUTRAL - 7 - CLS_VAR - - - &EACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC - STNEUTRAL - 6 - CLS_VAR - - - &ECIRC; - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &EGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - Ð - STNEUTRAL - 4 - CLS_VAR - - - Ð - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML - STNEUTRAL - 5 - CLS_VAR - - - &EUML; - STNEUTRAL - 6 - CLS_VAR - - - &FRAC12 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC12; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC14 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC14; - STNEUTRAL - 8 - CLS_VAR - - - &FRAC34 - STNEUTRAL - 7 - CLS_VAR - - - &FRAC34; - STNEUTRAL - 8 - CLS_VAR - - - > - STNEUTRAL - 3 - CLS_VAR - - - > - STNEUTRAL - 4 - CLS_VAR - - - &IACUTE - STNEUTRAL - 7 - CLS_VAR - - - &IACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &ICIRC - STNEUTRAL - 6 - CLS_VAR - - - &ICIRC; - STNEUTRAL - 7 - CLS_VAR - - - &IEXCL - STNEUTRAL - 5 - CLS_VAR - - - &IEXCL; - STNEUTRAL - 6 - CLS_VAR - - - &IGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &IGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &IQUEST - STNEUTRAL - 7 - CLS_VAR - - - &IQUEST; - STNEUTRAL - 8 - CLS_VAR - - - &IUML - STNEUTRAL - 5 - CLS_VAR - - - &IUML; - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO - STNEUTRAL - 6 - CLS_VAR - - - &LAQUO; - STNEUTRAL - 7 - CLS_VAR - - - &LRM - STNEUTRAL - 4 - CLS_VAR - - - &LRM; - STNEUTRAL - 5 - CLS_VAR - - - < - STNEUTRAL - 3 - CLS_VAR - - - < - STNEUTRAL - 4 - CLS_VAR - - - &MACR - STNEUTRAL - 5 - CLS_VAR - - - &MACR; - STNEUTRAL - 6 - CLS_VAR - - - &MICRO - STNEUTRAL - 6 - CLS_VAR - - - &MICRO; - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT - STNEUTRAL - 7 - CLS_VAR - - - &MIDDOT; - STNEUTRAL - 8 - CLS_VAR - - - &NBSP - STNEUTRAL - 5 - CLS_WHITESPACE - - - &NBSP; - STNEUTRAL - 6 - CLS_WHITESPACE - - - &NOT - STNEUTRAL - 4 - CLS_VAR - - - &NOT; - STNEUTRAL - 5 - CLS_VAR - - - &NTILDE - STNEUTRAL - 7 - CLS_VAR - - - &NTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OACUTE - STNEUTRAL - 7 - CLS_VAR - - - &OACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &OCIRC - STNEUTRAL - 6 - CLS_VAR - - - &OCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &OGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &ORDF - STNEUTRAL - 5 - CLS_VAR - - - &ORDF; - STNEUTRAL - 6 - CLS_VAR - - - &ORDM - STNEUTRAL - 5 - CLS_VAR - - - &ORDM; - STNEUTRAL - 6 - CLS_VAR - - - &OSLASH - STNEUTRAL - 7 - CLS_VAR - - - &OSLASH; - STNEUTRAL - 8 - CLS_VAR - - - &OTILDE - STNEUTRAL - 7 - CLS_VAR - - - &OTILDE; - STNEUTRAL - 8 - CLS_VAR - - - &OUML - STNEUTRAL - 5 - CLS_VAR - - - &OUML; - STNEUTRAL - 6 - CLS_VAR - - - &PARA - STNEUTRAL - 5 - CLS_VAR - - - &PARA; - STNEUTRAL - 6 - CLS_VAR - - - &PLUSMN - STNEUTRAL - 7 - CLS_VAR - - - &PLUSMN; - STNEUTRAL - 8 - CLS_VAR - - - &POUND - STNEUTRAL - 6 - CLS_VAR - - - &POUND; - STNEUTRAL - 7 - CLS_VAR - - - " - STNEUTRAL - 5 - CLS_VAR - - - " - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO - STNEUTRAL - 6 - CLS_VAR - - - &RAQUO; - STNEUTRAL - 7 - CLS_VAR - - - ® - STNEUTRAL - 4 - CLS_VAR - - - ® - STNEUTRAL - 5 - CLS_VAR - - - &RLM - STNEUTRAL - 4 - CLS_VAR - - - &RLM; - STNEUTRAL - 5 - CLS_VAR - - - &SECT - STNEUTRAL - 5 - CLS_VAR - - - &SECT; - STNEUTRAL - 6 - CLS_VAR - - - &SHY - STNEUTRAL - 4 - CLS_VAR - - - &SHY; - STNEUTRAL - 5 - CLS_VAR - - - &SUP1 - STNEUTRAL - 5 - CLS_VAR - - - &SUP1; - STNEUTRAL - 6 - CLS_VAR - - - &SUP2 - STNEUTRAL - 5 - CLS_VAR - - - &SUP2; - STNEUTRAL - 6 - CLS_VAR - - - &SUP3 - STNEUTRAL - 5 - CLS_VAR - - - &SUP3; - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG - STNEUTRAL - 6 - CLS_VAR - - - &SZLIG; - STNEUTRAL - 7 - CLS_VAR - - - Þ - STNEUTRAL - 6 - CLS_VAR - - - Þ - STNEUTRAL - 7 - CLS_VAR - - - &TIMES - STNEUTRAL - 6 - CLS_VAR - - - &TIMES; - STNEUTRAL - 7 - CLS_VAR - - - &TRADE - STNEUTRAL - 6 - CLS_VAR - - - ™ - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE - STNEUTRAL - 7 - CLS_VAR - - - &UACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &UCIRC - STNEUTRAL - 6 - CLS_VAR - - - &UCIRC; - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE - STNEUTRAL - 7 - CLS_VAR - - - &UGRAVE; - STNEUTRAL - 8 - CLS_VAR - - - &UML - STNEUTRAL - 4 - CLS_VAR - - - &UML; - STNEUTRAL - 5 - CLS_VAR - - - &UUML - STNEUTRAL - 5 - CLS_VAR - - - &UUML; - STNEUTRAL - 6 - CLS_VAR - - - &YACUTE - STNEUTRAL - 7 - CLS_VAR - - - &YACUTE; - STNEUTRAL - 8 - CLS_VAR - - - &YEN - STNEUTRAL - 4 - CLS_VAR - - - &YEN; - STNEUTRAL - 5 - CLS_VAR - - - &YUML - STNEUTRAL - 5 - CLS_VAR - - - &YUML; - STNEUTRAL - 6 - CLS_VAR - - - &ZWJ - STNEUTRAL - 4 - CLS_VAR - - - &ZWJ; - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ - STNEUTRAL - 5 - CLS_VAR - - - &ZWNJ; - STNEUTRAL - 6 - CLS_VAR - - - &#~ - STNEUTRAL - ;\n\r\t> - CLS_VAR - - - - 3 - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - 6 - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - 9 - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - - - > - STNEUTRAL - - -
- 9 - - -
- > - - - - > - - - - > - - - - > - - - - 8 - - - - > - - - - > - - - - > - - - - > - - - - 6 - - - - 7 - - - - > - - - - > - - - - > - - - - > - - - - 3 - STNEUTRAL - - - - 6 - - - - 7 - - - - > - - - - > - - - - > - - - - > - - - - > - - - - > - - - - 5 - STNEUTRAL - - - - > - - - - 6 - - - - > - - - - > - - - - > - - - - > - - -
- 4 - - -
- 5 - - -
- > - - - - > - - - - > - - - - > - - -