freat(ENGKNOW-2950): isla gene mb workflow that uses versioned link files

gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java

@@ -39,7 +39,7 @@ public class LinkUpdateCommand implements Runnable {
     public void run() {
         var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath);
         try {
-            var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(),  true, this));
+            var linkFile = LinkFile.loadV1(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(),  true, this));
             applyHeaders(linkFile);
             linkFile.appendEntry(linkValue, entryMd5, entryInfo);
             linkFile.save();

gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala

@@ -30,14 +30,15 @@ import org.apache.commons.lang3.StringUtils
 import org.gorpipe.exceptions.GorResourceException
 import org.gorpipe.gor.binsearch.GorIndexType
 import org.gorpipe.gor.driver.linkfile.{LinkFile, LinkFileEntryV1}
-import org.gorpipe.gor.driver.meta.{DataType, SourceReference}
+import org.gorpipe.gor.driver.meta.DataType
 import org.gorpipe.gor.driver.providers.stream.sources.StreamSource
-import org.gorpipe.gor.model.{DriverBackedFileReader, Row}
+import org.gorpipe.gor.model.Row
 import org.gorpipe.gor.session.{GorSession, ProjectContext}
 import org.gorpipe.gor.table.util.PathUtils
 import org.gorpipe.gor.util.DataUtil
 import org.gorpipe.model.gor.RowObj
 import org.gorpipe.util.Strings
+import org.slf4j.{Logger, LoggerFactory}
 
 import java.util.UUID
 import scala.collection.mutable
@@ -125,6 +126,8 @@ case class ForkWrite(forkCol: Int,
                      inHeader: String,
                      options: OutputOptions) extends Analysis {
 
+  private val log: Logger = LoggerFactory.getLogger(ForkWrite.this.getClass)
+
   case class FileHolder(forkValue: String) {
     if (forkCol >= 0 && options.useFolder.isEmpty && !(fullFileName.contains("#{fork}") || fullFileName.contains("""${fork}"""))) {
       throw new GorResourceException("WRITE error: #{fork} of ${fork}missing from filename.", fullFileName)
@@ -159,9 +162,9 @@ case class ForkWrite(forkCol: Int,
                     fullFileName.replace("#{fork}", forkValue).replace("""${fork}""", forkValue)
                   } else {
                     if (fullFileName.isEmpty && options.linkFile.nonEmpty) {
-                      val (linkFileMeta, linkFileInfo) = extractLinkMetaInfo(options.linkFileMeta)
-                      val linkSourceRef = new SourceReference(options.linkFile, null, projectContext.getFileReader.getCommonRoot, null, null, true);
-                      // Infer the full file name from the link (and defautl locations)
+                      val (linkFileMeta, _) = extractLinkMetaInfo(options.linkFileMeta)
+                      val linkSourceRef = projectContext.getFileReader.createSourceReference(options.linkFile, true);
+                      // Infer the full file name from the link (and default locations)
                       LinkFile.inferDataFileNameFromLinkFile(
                         projectContext.getFileReader.resolveDataSource(linkSourceRef).asInstanceOf[StreamSource], linkFileMeta);
                     } else {
@@ -409,12 +412,11 @@ case class ForkWrite(forkCol: Int,
     session.getProjectContext.getFileReader.resolveUrl(FilenameUtils.removeExtension(linkFilePath), true)
 
     // Use the nonsecure driver file reader as this is an exception from the write no links rule.
-    val fileReader = new DriverBackedFileReader(session.getProjectContext.getFileReader.getSecurityContext,
-      session.getProjectContext.getProjectRoot, session.getProjectContext.getFileReader.getQueryTime)
+    val fileReader = session.getProjectContext.getFileReader.unsecure()
 
-    LinkFile.load(fileReader.resolveUrl(linkFilePath, true).asInstanceOf[StreamSource])
+    LinkFile.loadV1(fileReader.resolveUrl(linkFilePath, true).asInstanceOf[StreamSource])
       .appendMeta(linkFileMeta)
       .appendEntry(linkFileContent, md5, linkFileInfo, fileReader)
-      .save(session.getProjectContext.getFileReader.getQueryTime)
+      .save(fileReader.getQueryTime)
   }
 }

gortools/src/main/scala/gorsat/InputSources/Exec.scala

@@ -36,7 +36,7 @@ import scala.collection.mutable.ListBuffer
 /**
  * Execute selected gor <command> commands in NOR context.
  */
-class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, 10, ignoreIllegalArguments=true), isNorCommand = true) {
+class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, ignoreIllegalArguments=true), isNorCommand = true) {
 
   override def processArguments(context: GorContext, argString: String, iargs: Array[String],
                                 args: Array[String]): InputSourceParsingResult = {

gortools/src/main/scala/gorsat/Utilities/IteratorUtilities.scala

@@ -43,7 +43,7 @@ object IteratorUtilities {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
-  private val ALLOW_DUPLICATE_COLUMNS = System.getProperty("gor.iterators.allowDuplicateColumns", "false").toBoolean
+  private val ALLOW_DUPLICATE_COLUMNS = System.getProperty("gor.iterators.allowDuplicateColumns", "true").toBoolean
 
   def getHeader(filename: String, gorRoot: String, context: GorContext): String = {
     val gm: GorMonitor = null

gortools/src/test/java/gorsat/UTestHeaderFlags.java

@@ -83,16 +83,17 @@ public void testValidHeaderUsedKeywordsWithDupAllowingDup() {
         Assert.assertFalse(systemErrRule.getLog().contains("Duplicate column name 'order'"));
     }
 
+    @Ignore("Only works if run alone, as the property is read only on class load")
     @Test
     public void testValidHeaderUsedKeywordsWithDupNotAllowingDup() {
+        System.setProperty("gor.iterators.allowDuplicateColumns", "false");
         String testHeader = "#abc\tstart\tfrom\tselect\tmax\tmin\tfrom\tgroup\trange\torder\trank\torder";
         Assert.assertThrows(GorDataException.class, () -> IteratorUtilities.validHeader(testHeader, false));
     }
 
-    @Ignore("Only works if run alone, as the property is read only on class load")
+
     @Test
     public void testValidHeaderUsedKeywordsWithDupAllowingDupOnlyGlobal() {
-        System.setProperty("gor.iterators.allowDuplicateColumns", "true");
         String testHeader = "#abc\tstart\tfrom\tselect\tmax\tmin\tfrom\tgroup\trange\torder\trank\torder";
         String resultingHeader = IteratorUtilities.validHeader(testHeader, false);
         Assert.assertEquals("#abc\tstart\tfrom\tselect\tmax\tmin\tfromx\tgroup\trange\torder\trank\torderx", resultingHeader);

gortools/src/test/java/gorsat/UTestPartGor.java

@@ -309,6 +309,4 @@ public void testPartGorMixedTagsReplacement() {
         Assert.assertEquals("a,b,c | 'a','b','c' | \"a\",\"b\",\"c\"", result);
     }
 
-
-
 }

gortools/src/test/java/gorsat/UTestRename.java

@@ -25,6 +25,7 @@
 import org.gorpipe.exceptions.GorDataException;
 import org.gorpipe.exceptions.GorParsingException;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -139,7 +140,16 @@ public void renameStrictWithNoMatch() {
     }
 
     @Test
-    public void renameToExisting() {
+    public void renameToExistingAllowDup() {
+        String query = "gorrow chr1,1 | calc Foo 0 | calc Bar 1 | rename Foo Bar | top 0";
+        String result = TestUtils.runGorPipe(query);
+        Assert.assertEquals("chrom\tpos\tBar\tBarx\n", result);
+    }
+
+    @Ignore("Only works if run alone, as the property is read only on class load")
+    @Test
+    public void renameToExistingNotAllowDup() {
+        System.setProperty("gor.iterators.allowDuplicateColumns", "false");
         String query = "gorrow chr1,1 | calc Foo 0 | calc Bar 1 | rename Foo Bar | top 0";
         Assert.assertThrows(GorDataException.class, () -> TestUtils.runGorPipe(query));
     }

gortools/src/test/java/gorsat/parser/UTestCalc.java

@@ -25,6 +25,7 @@
 import gorsat.TestUtils;
 import org.gorpipe.exceptions.GorDataException;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.util.Arrays;
@@ -38,13 +39,17 @@ public void addsNewColumn() {
         Assert.assertEquals(expected, result);
     }
 
+    @Ignore("Only works if run alone, as the property is read only on class load")
     @Test
     public void addsNewExistingColumn() {
+        System.setProperty("gor.iterators.allowDuplicateColumns", "false");
         Assert.assertThrows(GorDataException.class, () -> TestUtils.runGorPipe("gorrow 1,1 | calc pos 42"));
     }
 
+    @Ignore("Only works if run alone, as the property is read only on class load")
     @Test
     public void addsNewColumnWithSuffixWhenItExists() {
+        System.setProperty("gor.iterators.allowDuplicateColumns", "false");
         Assert.assertThrows(GorDataException.class, () -> TestUtils.runGorPipe("gorrow 1,1 | calc data 42 | calc data 3.14"));
     }
 

model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java

@@ -121,62 +121,41 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri
             throw new IllegalArgumentException("Link file path is null or empty.  Can not infer data file name.");
         }
 
-        var linkPath = linkSource.getSourceReference().getUrl();
-
-        // Remove common the root if set.
-        var pathReplacements = System.getenv("GOR_DRIVER_LINK_INFER_REPLACE");
-        if (!Strings.isNullOrEmpty(pathReplacements)) {
-            var parts = pathReplacements.split(";", 2);
-            linkPath = linkPath.replaceAll(parts[0], parts.length > 1 ? parts[1] : "");
-        }
-
-        // Adjust link path so it suitable as part of data file path.
-        if (PathUtils.isAbsolutePath(linkPath)) {
-            throw new IllegalArgumentException("Link file path is absolute and gor.driver.link.common.root is not set.  Can not infer data file name: " + linkSource.getFullPath());
-        }
-
-        var dataFileRootPath = "";
+        var dataFileParentPath = "";
 
         // Get root from the link file
         var link = linkSource.exists()
-                ? load(linkSource).appendMeta(linkFileMeta)
-                : create(linkSource, linkFileMeta);
-        var linkDataFileRootPath = link.getMeta().getProperty(LinkFileMeta.HEADER_CONTENT_LOCATION_MANAGED_KEY);
-        if (!Strings.isNullOrEmpty(linkDataFileRootPath)) {
-            dataFileRootPath = linkDataFileRootPath;
-        }
-
-        // Get root from global const
-        if (Strings.isNullOrEmpty(dataFileRootPath)) {
-            dataFileRootPath = System.getenv(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL);
+                ? LinkFile.load(linkSource).appendMeta(linkFileMeta)
+                : LinkFile.create(linkSource, linkFileMeta);
+
+        var linkDataFileParentPath = link.getMeta().getProperty(LinkFileMeta.HEADER_DATA_LOCATION_KEY);
+        if (!Strings.isNullOrEmpty(linkDataFileParentPath)) {
+            dataFileParentPath = linkDataFileParentPath;
+        } else if (link.getLatestEntry() != null) {
+            dataFileParentPath = PathUtils.getParent(link.getLatestEntryUrl());
         }
 
-        if (Strings.isNullOrEmpty(dataFileRootPath)) {
-            throw new IllegalArgumentException("Link file data root path is not set.  Can not infer data file name from link file: " + linkSource.getFullPath());
+        if (!Strings.isNullOrEmpty(linkDataFileParentPath)) {
+            dataFileParentPath = linkDataFileParentPath;
         }
 
-        // Create file name
-        String randomString = RandomStringUtils.random(8, true, true);
-        var linkPathSplit = linkPath.indexOf('.');
-        if (linkPathSplit > 0) {
-            linkPath = "%s.%s.%s".formatted(
-                    linkPath.substring(0, linkPathSplit),
-                    randomString,
-                    linkPath.substring(linkPathSplit + 1));
-        } else {
-            linkPath = "%s.%s".formatted(linkPath, randomString);
+        // Get root from global const
+        if (Strings.isNullOrEmpty(dataFileParentPath)) {
+            dataFileParentPath = System.getenv(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL);
+
+            // Insert project, only if we use global and global is set
+            if (!Strings.isNullOrEmpty(dataFileParentPath)) {
+                var project = linkSource.getSourceReference().getCommonRoot() != null
+                        ? PathUtils.getFileName(linkSource.getSourceReference().getCommonRoot()) : "";
+                if (!Strings.isNullOrEmpty(project)) {
+                    dataFileParentPath = PathUtils.resolve(dataFileParentPath, project);
+                }
+            }
         }
 
-        linkPath = linkPath.replaceAll("\\.link$", "");
-
-        // Insert project
-        var project = linkSource.getSourceReference().getCommonRoot() != null
-                ? PathUtils.getFileName(linkSource.getSourceReference().getCommonRoot()) : "";
-        if (!Strings.isNullOrEmpty(project)) {
-            dataFileRootPath = PathUtils.resolve(dataFileRootPath, project);
-        }
+        var dataFileName = PathUtils.injectRandomStringIntoFileName(PathUtils.getFileName(linkSource.getFullPath()));
 
-        return PathUtils.resolve(dataFileRootPath, linkPath);
+        return PathUtils.resolve(dataFileParentPath, dataFileName);
     }
 
     protected final StreamSource source;

model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java

@@ -14,11 +14,11 @@ public class LinkFileMeta extends BaseMeta {
     // Max age of entries to keep track of in the link file.
     public static final String HEADER_ENTRIES_AGE_MAX_KEY = "ENTRIES_AGE_MAX";
     // Path if the managed content data location.
-    public static final String HEADER_CONTENT_LOCATION_MANAGED_KEY = "CONTENT_LOCATION_MANAGED";
+    public static final String HEADER_DATA_LOCATION_KEY = "DATA_LOCATION";
     // Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false).
-    public static final String HEADER_CONTENT_LIFECYCLE_MANAGED_KEY = "CONTENT_LIFECYCLE_MANAGED";
+    public static final String HEADER_DATA_LIFECYCLE_MANAGED_KEY = "DATA_LIFECYCLE_MANAGED";
 
-    private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", "1");
+    private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", LinkFileV0.VERSION);
 
     public static final int DEFAULT_ENTRIES_COUNT_MAX = 100;
     public static final long DEFAULT_ENTRIES_AGE_MAX = Long.MAX_VALUE;

model/src/main/java/org/gorpipe/gor/table/TableCache.java

@@ -29,23 +29,14 @@ public T getOrCreateTable(String path, FileReader fileReader) throws IOException
     }
 
     public T getOrCreateTable(String path, FileReader fileReader, boolean useCache) throws IOException {
-        try {
-            return getTable(path, fileReader, useCache);
-        } catch (NoSuchFileException e) {
-            return createTable(path, fileReader);
-        }
+        return getTable(path, fileReader, useCache);
     }
 
-    public synchronized T getTable(String path, FileReader fileReader) throws IOException {
+    public synchronized T getTable(String path, FileReader fileReader) {
         return getTable(path, fileReader, useCache);
     }
 
-    public synchronized T getTable(String path, FileReader fileReader, boolean useCache) throws IOException {
-        // TODO:  To make fewer calls to exists consider caching it in metadata.  Should not need this as getSourceMetaData should throw
-        //        exception if file does not exists.
-        if (!fileReader.exists(path)) {
-            throw new NoSuchFileException(path);
-        }
+    public synchronized T getTable(String path, FileReader fileReader, boolean useCache) {
         // The dict is lazy loaded so the only cost is finding the id.
         T dict = createTable(path, fileReader);
 

model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java

@@ -343,6 +343,22 @@ public static Path getTempFilePath(Path filePath) {
         return tempFileName;
     }
 
+    public static String injectRandomStringIntoFileName(String fileName) {
+        var tempFileName = "";
+        String uniqId = RandomStringUtils.insecure().next(8, true, true);
+        var linkPathSplit = fileName.indexOf('.', fileName.indexOf("/"));
+        if (linkPathSplit > 0) {
+            tempFileName = "%s.%s.%s".formatted(
+                    fileName.substring(0, linkPathSplit),
+                    uniqId,
+                    fileName.substring(linkPathSplit + 1));
+        } else {
+            tempFileName = "%s.%s".formatted(fileName, uniqId);
+        }
+
+        return tempFileName.replaceAll("\\.link$", "");
+    }
+
     public static boolean pathEquals(String path1, String path2) {
         if (path1 == null && path2 == null) {
             return true;