TS-9 final change - compare with branch name TS-9

TextSearch/Common/Constants.ecl

@@ -1,4 +1,4 @@
-EXPORT Constants := MODULE
+EXPORT Constants := MODULE
   // Limit Constants
   EXPORT Max_SearchTerms := 1000;
   EXPORT Max_Ops  := 2 * Max_SearchTerms;

TextSearch/Common/Default_Keywording.ecl

@@ -1,10 +1,11 @@
 //Default implementation.  Provides minimal functionality.
-IMPORT Std.Uni;
+//from me this file give defult values to varibles and interface in Ikeywording file
+IMPORT Std.Uni; //to use lower case or upper case (from me)
 IMPORT TextSearch.Common;
 IMPORT TextSearch.Common.Types;
 IMPORT TextSearch.Common.Layouts;
 TermString    := Types.TermString;
-EquivTerm     := Layouts.EquivTerm;
+EquivTerm     := Layouts.EquivTerm; //store all record
 Version       := Types.Version;
 NoEquiv       := DATASET([],EquivTerm);
 ToUpper       := Uni.ToUpperCase;

TextSearch/Common/FileName_Info.ecl

@@ -1,13 +1,8 @@
-//FileName Info structured used for file name generation.
-//This version includes pre-Slice management hack to support tracking update
-//versions with incremental updates.
-EXPORT FileName_Info := INTERFACE
+EXPORT FileName_Info := INTERFACE
   EXPORT STRING Prefix;
   EXPORT STRING Instance;    // the version for an individual instance or the Alias
   EXPORT STRING AliasInstance := 'CURRENT';
-  EXPORT SET OF STRING AliasInstances := [AliasInstance, 'LAST', 'PAST', 'DELETED'];
-  EXPORT UNSIGNED2 Naming := 1;       // version of naming system
-  EXPORT UNSIGNED2 DataVersion := 0;  // placeholder for data version to build
-  EXPORT UNSIGNED1 Levels := 5;
-  EXPORT STRING UseInstance(UNSIGNED indx) := IF(indx=0, Instance, AliasInstances[indx]);
+  EXPORT UNSIGNED2 Naming := 1;
+  EXPORT UNSIGNED2 DataVersion := 0;
+  EXPORT UNSIGNED1 Levels := 5;//from 0 to 4
 END;

TextSearch/Common/FileName_Info_Instance.ecl

@@ -1,14 +1,14 @@
-//Instance of the FileName_Info block.  Used to unify the names used by TextSearch.
+//Instance of the FileName_Info block.  Used to unify the names used by TextSearch.
 IMPORT TextSearch.Common;
 IMPORT STD.Str;
 Info := Common.FileName_Info;
-EXPORT FileName_Info_Instance(STRING aPre, STRING aInst) := MODULE(Info)
+EXPORT FileName_Info_Instance(STRING aPre, STRING aInst) := MODULE(Info)//mean this module will have all field in FileName_Info
   STRING wPrefix := TRIM(Str.ToUpperCase(aPre),ALL);
   EXPORT STRING Prefix := IF(wPrefix<>'',
                              wPrefix,
                              FAIL(STRING,
                                   Common.Constants.No_Prfx_code,
                                   (STRING)Common.Constants.No_Prfx_Msg));
   STRING wInst := TRIM(Str.ToUpperCase(aInst),ALL);
-  EXPORT STRING Instance := IF(wInst<>'', wInst, AliasInstance);
+  EXPORT STRING Instance := IF(wInst<>'', wInst, AliasInstance);//AliasInstance='current'
 END;

TextSearch/Common/FileNames.ecl

@@ -1,4 +1,4 @@
-IMPORT TextSearch.Common;
+IMPORT TextSearch.Common;
 //Creates file names.  The names are both the names of the individual
 //logical files and the container names used as aliases for a group
 //of file instances.
@@ -9,11 +9,11 @@ IMPORT TextSearch.Common;
 // Instance is FileName.Instance; and Suffix is the data type as below.
 FileName_Info := Common.FileName_Info;
 
-EXPORT FileNames(FileName_Info info, UNSIGNED Alias=0) := MODULE
+EXPORT FileNames(FileName_Info info) := MODULE //to set name of doc
   SHARED DocSearchPrefix := '::DocSearch::Level-';
   SHARED Name(STRING suffix, UNSIGNED lvl) := info.Prefix + DocSearchPrefix
                                             + INTFORMAT(lvl, 2, 1) + '::'
-                                            + info.UseInstance(Alias) + '::' + suffix;
+                                            + info.Instance + '::' + suffix;
 
   EXPORT DocumentIndex(UNSIGNED lvl=0) := Name('DocIndx', lvl);
   EXPORT TriGramDictionary(UNSIGNED lvl=0) := Name('TriDctIndx', lvl);
@@ -28,26 +28,4 @@ EXPORT FileNames(FileName_Info info, UNSIGNED Alias=0) := MODULE
   EXPORT TagDictionary(UNSIGNED lvl=0) := Name('TagIndx', lvl);
   EXPORT IdentIndx(UNSIGNED1 lvl=0) := Name('IdentIndx', lvl);
   EXPORT DeleteIndex(UNSIGNED1 lvl=0) := NAME('DelIndx', lvl);
-  EXPORT NameEnum := Common.Types.FileEnum;
-  EXPORT NameByEnum(NameEnum ne, UNSIGNED1 lvl=0)
-      := CASE(ne,
-              NameEnum.DocumentIndex                => DocumentIndex(lvl),
-              NameEnum.TriGramDictionary            => TriGramDictionary(lvl),
-              NameEnum.TermDictionary               => TermDictionary(lvl),
-              NameEnum.TriGramIndex                 => TriGramIndex(lvl),
-              NameEnum.TermIndex                    => TermIndex(lvl),
-              NameEnum.PhraseIndex                  => PhraseIndex(lvl),
-              NameEnum.ElementIndex                 => ElementIndex(lvl),
-              NameEnum.AttributeIndex               => AttributeIndex(lvl),
-              NameEnum.RangeIndex                   => RangeIndex(lvl),
-              NameEnum.NameSpaceDict                => NameSpaceDict(lvl),
-              NameEnum.TagDictionary                => TagDictionary(lvl),
-              NameEnum.IdentIndx                    => IdentIndx(lvl),
-              NameEnum.DeleteIndex                  => DeleteIndex(lvl),
-              Name('BadEnum', lvl));
-  // the currently building keys.  Add triGramDictionary and TriGramIndex when ready
-  EXPORT NameSet := [NameEnum.DocumentIndex, NameEnum.TermDictionary, NameEnum.TermIndex,
-                     NameEnum.PhraseIndex, NameEnum.ElementIndex, NameEnum.AttributeIndex,
-                     NameEnum.RangeIndex, NameEnum.TagDictionary, NameEnum.IdentIndx,
-                     NameEnum.DeleteIndex];
 END;

TextSearch/Common/IKeywording.ecl

@@ -3,11 +3,11 @@
 IMPORT TextSearch.Common.Types;
 IMPORT TextSearch.Common.Layouts;
 TermString    := Types.TermString;
-EquivTerm     := Layouts.EquivTerm;
+EquivTerm     := Layouts.EquivTerm;//record
 Version       := Types.Version;
 
 EXPORT IKeywording := INTERFACE
-  EXPORT Version currentVersion;
+  EXPORT Version currentVersion; //define currentversion from version type fro me
   EXPORT BOOLEAN hasEquivalence(TermString trm, Version v=currentVersion);
   EXPORT TermString SingleKeyword(TermString trm, Version v=currentVersion);
   EXPORT DATASET(EquivTerm) EquivKeywords(TermString trm, Version v=currentVersion);

TextSearch/Common/Layouts.ecl

@@ -2,7 +2,7 @@
 IMPORT TextSearch.Common.Types;
 IMPORT TextSearch.Inverted.Layouts AS InvertedLayouts;
 EXPORT Layouts := MODULE
-  EXPORT DocIndex := RECORD(InvertedLayouts.Document-content)
+  EXPORT DocIndex := RECORD(InvertedLayouts.Document-content)//I think here error -content what is?
     Types.KWP                 keywords;
     Types.Position            docLength;
     STRING18                  wunit;

TextSearch/Common/NumericCollationFormat.ecl

@@ -1,4 +1,4 @@
-/*
+/*
   <Numeric collation format>
 
   Layout in big endian:
@@ -39,7 +39,7 @@ EXPORT NumericCollationFormat := MODULE
     int expnt = 0;
     char ch = '\0';
 
-    for (int i = 0; i < (int)lenNumstr; i++)
+    for (int i = 0; i < lenNumstr; i++)
     {
       if ((ch = numstr[i]) == '.')
       {

TextSearch/Common/Pattern_Definitions.ecl

@@ -1,4 +1,4 @@
-EXPORT Pattern_Definitions := MACRO
+EXPORT Pattern_Definitions := MACRO
   // Pure Whitespace
   PATTERN LowControl    := PATTERN(U'[\u0001-\u0008\u000B\u000C\u000E\u000F]');
   PATTERN HighControl    := PATTERN(U'[\u007F-\u009F]');
@@ -100,6 +100,7 @@ EXPORT Pattern_Definitions := MACRO
   // Composite patterns
   // Word strings
   PATTERN Letter        := PATTERN(U'[[:alpha:]]');
+	//PATTERN NOLetter        := PATTERN(U'[^[:alpha:]]');
   PATTERN LowerCase      := PATTERN(U'[[:lower:]]');
   PATTERN UpperCase      := PATTERN(U'[[:upper:]]');
   PATTERN Digit          := PATTERN(U'[[:digit:]]');

TextSearch/Common/Types.ecl

@@ -1,4 +1,4 @@
-// Types for search system
+// Types for search system
 
 EXPORT Types := MODULE
   EXPORT DocNo            := UNSIGNED4;
@@ -16,8 +16,9 @@ EXPORT Types := MODULE
                                   SymbolChar,      // Ampersand, Section, et cetera
                                   NoiseChar,       // Noise, such as a comma or Tab
                                   WhiteSpace,      // blanks
-                                  SpecialStr);     // special keyword string
-  EXPORT TermTypeAsString(TermType typ) := CASE(typ,
+                                  SpecialStr,			 // special keyword string
+																	AcroStr);     	 //Initialism and Acronyms
+  EXPORT TermTypeAsString(TermType typ) := CASE(typ, //meaning if value is 1 then let value of type=Text string
                     1    =>  V'Text String',
                     2    =>  V'Number',
                     3    =>  V'Date',
@@ -27,9 +28,10 @@ EXPORT Types := MODULE
                     7    =>  V'Noise Character',
                     8    =>  V'White Space',
                     9    =>  V'Special Keyword',
+										10   => V'Initialism and Acronyms',
                     V'Unknown');
   EXPORT KeywordTTypes    := [TermType.TextStr, TermType.Number,
-                              TermType.Date, TermType.SymbolChar];
+                              TermType.Date, TermType.SymbolChar,TermType.AcroStr];
   EXPORT InvertTTypes     := [TermType.TextStr, TermType.Number,
                               TermType.Date, TermType.Meta,
                               TermType.Tag, TermType.SymbolChar,
@@ -87,8 +89,4 @@ EXPORT Types := MODULE
   EXPORT DocIdentifier    := UNICODE;
   EXPORT SequenceKey      := STRING50;
   EXPORT SlugLine         := UNICODE;
-  EXPORT FileEnum := ENUM(UNSIGNED1, Unknown=0, DocumentIndex, TriGramDictionary,
-                          TermDictionary, TriGramIndex, TermIndex, PhraseIndex,
-                          ELementIndex, AttributeIndex, RangeIndex, NameSpaceDict,
-                          TagDictionary, IdentIndx, DeleteIndex);
 END;

TextSearch/Inverted/Base_Data.ecl

@@ -9,7 +9,7 @@ EXPORT Base_Data(Common.FileName_Info info,
                  DATASET(Inv_Layouts.DocumentIngest) docsIn):= MODULE
   // The documents must be enumerated
   SHARED keyword_mod := Common.Default_Keywording;
-  EXPORT enumDocs    := Inverted.EnumeratedDocs(info, docsIn);
+  EXPORT enumDocs    := Inverted.EnumeratedDocs(info, docsIn);//start here 
   EXPORT rawPostings := Inverted.RawPostings(enumDocs);
   EXPORT DocIndex    := Inverted.DocIndex(enumDocs, UNGROUP(rawPostings));
   // Need to get Replaced doc list

TextSearch/Inverted/Build_Slice_Action.ecl

@@ -1,14 +1,10 @@
 // The action for building a slice, given the name of the Ingest file, and the
 //prefix and instance for the file names.
-// Optional parameter is a dataset used to list other files that we want managed.
 IMPORT TextSearch.Common;
 IMPORT TextSearch.Inverted;
 Ingest := Inverted.Layouts.DocumentIngest;
-Managed_File_Names := Inverted.Layouts.Managed_File_Names;
-empty := DATASET([], Managed_File_Names);
 
-EXPORT Build_Slice_Action(STRING ingestName, STRING prfx, STRING inst,
-                          DATASET(Managed_File_Names) mfn=empty) := FUNCTION
+EXPORT Build_Slice_Action(STRING ingestName, STRING prfx, STRING inst) := FUNCTION
   inDocs := DATASET(ingestName, Ingest, THOR);
   info := Common.FileName_Info_Instance(prfx, inst);
   kwm  := Common.Default_Keywording;
@@ -22,7 +18,7 @@ EXPORT Build_Slice_Action(STRING ingestName, STRING prfx, STRING inst,
   TrmDict  := base.TermDict;
   TagDict  := base.TagDict;
   Replaced := base.ReplacedDocs;
-  bc := PARALLEL(
+  ac := PARALLEL(
     BUILD(Common.Keys(info).TermIndex(TrmPosts))
    ,BUILD(Common.Keys(info).ElementIndex(tagposts))
    ,BUILD(Common.Keys(info).PhraseIndex(PhrsPosts))
@@ -34,13 +30,5 @@ EXPORT Build_Slice_Action(STRING ingestName, STRING prfx, STRING inst,
    ,BUILD(Common.Keys(info).IdentIndex(docIndx))
    ,BUILD(Common.Keys(info).DeleteIndex(Replaced))
   );
-  Task_Enum := Inverted.Layouts.Management_Task;
-  good_mfn := ASSERT(mfn, (task=Task_Enum.NoOp)
-                        OR (task=Task_Enum.Replace AND logical_name<>''
-                            AND current_name<>'' AND previous_name<>''
-                            AND past_previous_name<>'' AND deleted_name<>'' ),
-                      'Missing required file names for action', FAIL);
-  key_list := Inverted.Basic_Key_List(info) + good_mfn;
-  ac := SEQUENTIAL(bc, Inverted.Manage_Superkeys(info, key_list));
   RETURN ac;
 END;

TextSearch/Inverted/Layouts.ecl

@@ -7,11 +7,13 @@ EXPORT Layouts := MODULE
     Types.SequenceKey       seqKey;
     Types.SlugLine          slugLine;
     UNICODE                 content;
+	UNICODE init;
+		  //string init_w_pun;
   END;
   EXPORT DocumentNo := RECORD
-    Types.DocNo id;
+    Types.DocNo id; //Types.DocNo mean set the type of DocNo to id
   END;
-  EXPORT Document := RECORD(DocumentIngest)
+  EXPORT Document := RECORD(DocumentIngest) // this mean the Document record will have all the fields in DocumentIngest and all the filelds in DocumentNo
     DocumentNo;
   END;
   // Posting Record, generated by parsing the documents.
@@ -34,16 +36,5 @@ EXPORT Layouts := MODULE
     Types.TermString          tagValue;
     Types.PathString          pathString;
     Types.TermString          parentName;
-  END;
-  // Record for the machinery to manage file names with super keys (super files)
-  EXPORT Management_Task := ENUM(UNSIGNED1, NoOp=0, Replace); // Future
-  EXPORT Managed_File_Names := RECORD
-    STRING logical_name;
-    STRING current_name;
-    STRING previous_name;
-    STRING past_previous_name;
-    STRING deleted_name;
-    BOOLEAN delete_deleted;
-    Management_Task task;
-  END;
-END;
+   END;
+END;