From f11d3c8eef889d51fcfe8aff265c74028feca196 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 23 Dec 2025 11:24:35 +0000 Subject: [PATCH 01/10] fix(ENGKNOW-2950): CONTENT_LOCATION_MANAGED renamed DATA_LOCATION and new means full parent path. --- .../main/scala/gorsat/InputSources/Exec.scala | 2 +- .../src/test/java/gorsat/UTestPartGor.java | 10 +++ .../gorpipe/gor/driver/linkfile/LinkFile.java | 71 +++++++------------ .../gor/driver/linkfile/LinkFileMeta.java | 4 +- .../org/gorpipe/gor/table/util/PathUtils.java | 16 +++++ .../gor/driver/linkfile/LinkFileTest.java | 43 +++-------- 6 files changed, 62 insertions(+), 84 deletions(-) diff --git a/gortools/src/main/scala/gorsat/InputSources/Exec.scala b/gortools/src/main/scala/gorsat/InputSources/Exec.scala index 8db82e16..022da135 100644 --- a/gortools/src/main/scala/gorsat/InputSources/Exec.scala +++ b/gortools/src/main/scala/gorsat/InputSources/Exec.scala @@ -36,7 +36,7 @@ import scala.collection.mutable.ListBuffer /** * Execute selected gor commands in NOR context. */ -class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, 10, ignoreIllegalArguments=true), isNorCommand = true) { +class Exec() extends InputSourceInfo("EXEC", CommandArguments("","", 2, ignoreIllegalArguments=true), isNorCommand = true) { override def processArguments(context: GorContext, argString: String, iargs: Array[String], args: Array[String]): InputSourceParsingResult = { diff --git a/gortools/src/test/java/gorsat/UTestPartGor.java b/gortools/src/test/java/gorsat/UTestPartGor.java index cbcc6705..240ff87c 100644 --- a/gortools/src/test/java/gorsat/UTestPartGor.java +++ b/gortools/src/test/java/gorsat/UTestPartGor.java @@ -309,6 +309,16 @@ public void testPartGorMixedTagsReplacement() { Assert.assertEquals("a,b,c | 'a','b','c' | \"a\",\"b\",\"c\"", result); } + @Test + public void partGorWithFileName() throws IOException { + String contents = "#col1\tcol2\tcol3\tcol4\tcol5\tcol6\tlis_PN\n" + + "data/bucket_1.gorz\tbucket_1\tchr1\t0\tchrZ\t1000000000\tSAMPLE_SIM842_000001\n" + + "data/bucket_10.gorz\tbucket_10\tchr1\t0\tchrZ\t1000000000\tSAMPLE_SIM842_000010\n"; + File dictFile = FileTestUtils.createTempFile(workDir.getRoot(), "variants.gord", contents); + + String query = "partgor -dict " + dictFile.getAbsolutePath() + " <(gorrow chr1,1,1 | calc x \"#{tags:q}\" | calc fn 's3://test/test.gor')"; + String results = TestUtils.runGorPipe(query); + } } diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java index 9db557b6..b69f7a4f 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFile.java @@ -121,62 +121,41 @@ public static String inferDataFileNameFromLinkFile(StreamSource linkSource, Stri throw new IllegalArgumentException("Link file path is null or empty. Can not infer data file name."); } - var linkPath = linkSource.getSourceReference().getUrl(); - - // Remove common the root if set. - var pathReplacements = System.getenv("GOR_DRIVER_LINK_INFER_REPLACE"); - if (!Strings.isNullOrEmpty(pathReplacements)) { - var parts = pathReplacements.split(";", 2); - linkPath = linkPath.replaceAll(parts[0], parts.length > 1 ? parts[1] : ""); - } - - // Adjust link path so it suitable as part of data file path. - if (PathUtils.isAbsolutePath(linkPath)) { - throw new IllegalArgumentException("Link file path is absolute and gor.driver.link.common.root is not set. Can not infer data file name: " + linkSource.getFullPath()); - } - - var dataFileRootPath = ""; + var dataFileParentPath = ""; // Get root from the link file var link = linkSource.exists() - ? load(linkSource).appendMeta(linkFileMeta) - : create(linkSource, linkFileMeta); - var linkDataFileRootPath = link.getMeta().getProperty(LinkFileMeta.HEADER_CONTENT_LOCATION_MANAGED_KEY); - if (!Strings.isNullOrEmpty(linkDataFileRootPath)) { - dataFileRootPath = linkDataFileRootPath; - } - - // Get root from global const - if (Strings.isNullOrEmpty(dataFileRootPath)) { - dataFileRootPath = System.getenv(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL); + ? LinkFile.load(linkSource).appendMeta(linkFileMeta) + : LinkFile.create(linkSource, linkFileMeta); + + var linkDataFileParentPath = link.getMeta().getProperty(LinkFileMeta.HEADER_DATA_LOCATION_KEY); + if (!Strings.isNullOrEmpty(linkDataFileParentPath)) { + dataFileParentPath = linkDataFileParentPath; + } else if (link.getLatestEntry() != null) { + dataFileParentPath = PathUtils.getParent(link.getLatestEntryUrl()); } - if (Strings.isNullOrEmpty(dataFileRootPath)) { - throw new IllegalArgumentException("Link file data root path is not set. Can not infer data file name from link file: " + linkSource.getFullPath()); + if (!Strings.isNullOrEmpty(linkDataFileParentPath)) { + dataFileParentPath = linkDataFileParentPath; } - // Create file name - String randomString = RandomStringUtils.random(8, true, true); - var linkPathSplit = linkPath.indexOf('.'); - if (linkPathSplit > 0) { - linkPath = "%s.%s.%s".formatted( - linkPath.substring(0, linkPathSplit), - randomString, - linkPath.substring(linkPathSplit + 1)); - } else { - linkPath = "%s.%s".formatted(linkPath, randomString); + // Get root from global const + if (Strings.isNullOrEmpty(dataFileParentPath)) { + dataFileParentPath = System.getenv(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL); + + // Insert project, only if we use global and global is set + if (!Strings.isNullOrEmpty(dataFileParentPath)) { + var project = linkSource.getSourceReference().getCommonRoot() != null + ? PathUtils.getFileName(linkSource.getSourceReference().getCommonRoot()) : ""; + if (!Strings.isNullOrEmpty(project)) { + dataFileParentPath = PathUtils.resolve(dataFileParentPath, project); + } + } } - linkPath = linkPath.replaceAll("\\.link$", ""); - - // Insert project - var project = linkSource.getSourceReference().getCommonRoot() != null - ? PathUtils.getFileName(linkSource.getSourceReference().getCommonRoot()) : ""; - if (!Strings.isNullOrEmpty(project)) { - dataFileRootPath = PathUtils.resolve(dataFileRootPath, project); - } + var dataFileName = PathUtils.injectRandomStringIntoFileName(PathUtils.getFileName(linkSource.getFullPath())); - return PathUtils.resolve(dataFileRootPath, linkPath); + return PathUtils.resolve(dataFileParentPath, dataFileName); } protected final StreamSource source; diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java index 66ef262e..a07a5361 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java @@ -14,9 +14,9 @@ public class LinkFileMeta extends BaseMeta { // Max age of entries to keep track of in the link file. public static final String HEADER_ENTRIES_AGE_MAX_KEY = "ENTRIES_AGE_MAX"; // Path if the managed content data location. - public static final String HEADER_CONTENT_LOCATION_MANAGED_KEY = "CONTENT_LOCATION_MANAGED"; + public static final String HEADER_DATA_LOCATION_KEY = "DATA_LOCATION"; // Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false). - public static final String HEADER_CONTENT_LIFECYCLE_MANAGED_KEY = "CONTENT_LIFECYCLE_MANAGED"; + public static final String HEADER_DATA_LIFECYCLE_MANAGED_KEY = "DATA_LIFECYCLE_MANAGED"; private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", "1"); diff --git a/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java b/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java index 1b38e2e5..1c0f76c5 100644 --- a/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java +++ b/model/src/main/java/org/gorpipe/gor/table/util/PathUtils.java @@ -343,6 +343,22 @@ public static Path getTempFilePath(Path filePath) { return tempFileName; } + public static String injectRandomStringIntoFileName(String fileName) { + var tempFileName = ""; + String uniqId = RandomStringUtils.insecure().next(8, true, true); + var linkPathSplit = fileName.indexOf('.', fileName.indexOf("/")); + if (linkPathSplit > 0) { + tempFileName = "%s.%s.%s".formatted( + fileName.substring(0, linkPathSplit), + uniqId, + fileName.substring(linkPathSplit + 1)); + } else { + tempFileName = "%s.%s".formatted(fileName, uniqId); + } + + return tempFileName.replaceAll("\\.link$", ""); + } + public static boolean pathEquals(String path1, String path2) { if (path1 == null && path2 == null) { return true; diff --git a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java index 193e1e8e..77dd81e8 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java +++ b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java @@ -4,10 +4,7 @@ import org.gorpipe.gor.driver.meta.SourceReference; import org.gorpipe.gor.driver.providers.stream.sources.StreamSource; import org.gorpipe.gor.driver.providers.stream.sources.file.FileSource; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.contrib.java.lang.system.EnvironmentVariables; import org.junit.contrib.java.lang.system.RestoreSystemProperties; import org.junit.rules.TemporaryFolder; @@ -200,17 +197,6 @@ public void testInferDataFileNameFromLinkFile_NullOrEmptyPath() throws Exception LinkFile.inferDataFileNameFromLinkFile(new FileSource(""), null); } - @Test(expected = IllegalArgumentException.class) - public void testInferDataFileNameFromLinkFile_AbsolutePath() throws Exception { - LinkFile.inferDataFileNameFromLinkFile(new FileSource("/abs/path/x.link"), null); - } - - @Test(expected = IllegalArgumentException.class) - public void testInferDataFileNameFromLinkFile_NoRootConfigured() throws Exception { - environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL, null); - LinkFile.inferDataFileNameFromLinkFile(new FileSource("x.link"), null); - } - @Test public void testInferDataFileNameFromLinkFile_FromEnvVariable_WithProject() throws Exception { String root = "/managed/root"; @@ -235,18 +221,18 @@ public void testInferDataFileNameFromLinkFile_FromExiting_File() throws Exceptio String root = "/managed/fromfile"; String linkFilePath = "x.gor.link"; Files.createDirectory(workPath.resolve("test")); - Files.writeString(workPath.resolve("test").resolve(linkFilePath), "## " + LinkFileMeta.HEADER_CONTENT_LOCATION_MANAGED_KEY + " = " + root + "\nsource/y.gorz\n"); + Files.writeString(workPath.resolve("test").resolve(linkFilePath), root + "/source/y.gorz\n"); String result = LinkFile.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath, null, workPath.resolve("test").toString(), -1, null, null, false, false)), null); assertNotNull(result); - assertTrue(result.matches((root + "/test/x\\..*\\.gor").replace("/", "\\/"))); + assertTrue(result.matches((root + "/source/x\\..*\\.gor").replace("/", "\\/"))); } @Test public void testInferDataFileNameFromLinkFile_FromMetaParam() throws Exception { String root = "/managed/fromparam"; String linkFilePath = "x.gor.link"; - String linkFileMeta = "## " + LinkFileMeta.HEADER_CONTENT_LOCATION_MANAGED_KEY + " = " + root; + String linkFileMeta = "## " + LinkFileMeta.HEADER_DATA_LOCATION_KEY + " = " + root; String result = LinkFile.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath)), linkFileMeta); assertNotNull(result); @@ -257,10 +243,10 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam() throws Exception { public void testInferDataFileNameFromLinkFile_FromMetaParam_ExistingFile() throws Exception { String fileroot = "/managed/fromfile"; String linkFilePath = "x.gor.link"; - Files.writeString(workPath.resolve(linkFilePath), "## " + LinkFileMeta.HEADER_CONTENT_LOCATION_MANAGED_KEY + " = " + fileroot + "\nsource/y.gorz\n"); + Files.writeString(workPath.resolve(linkFilePath), "## " + LinkFileMeta.HEADER_DATA_LOCATION_KEY + " = " + fileroot + "\nsource/y.gorz\n"); String paramroot = "/managed/fromparam"; - String linkFileMeta = "## " + LinkFileMeta.HEADER_CONTENT_LOCATION_MANAGED_KEY + " = " + paramroot; + String linkFileMeta = "## " + LinkFileMeta.HEADER_DATA_LOCATION_KEY + " = " + paramroot; String result = LinkFile.inferDataFileNameFromLinkFile(new FileSource(new SourceReference(linkFilePath)), linkFileMeta); assertNotNull(result); @@ -268,26 +254,13 @@ public void testInferDataFileNameFromLinkFile_FromMetaParam_ExistingFile() throw } @Test - public void testInferDataFileNameFromLinkFile_PathReplace() throws Exception { - String root = "/managed/root"; - environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL, root); - environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_INFER_REPLACE, "wont;will"); - - String result = LinkFile.inferDataFileNameFromLinkFile(new FileSource("wont/x.gor.link"), null); - - assertNotNull(result); - assertTrue(result.matches((root + "/will/x\\..*\\.gor").replace("/", "\\/"))); - } - - @Test - public void testInferDataFileNameFromLinkFile_AbsolutePathReplace() throws Exception { + public void testInferDataFileNameFromLinkFile_AbsolutePath() throws Exception { String root = "/managed/root"; environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_MANAGED_DATA_FILES_URL, root); - environmentVariables.set(GorDriverConfig.GOR_DRIVER_LINK_INFER_REPLACE, "\\/abs\\/"); String result = LinkFile.inferDataFileNameFromLinkFile(new FileSource("/abs/path/x.gor.link"), null); assertNotNull(result); - assertTrue(result.matches((root + "/path/x\\..*\\.gor").replace("/", "\\/"))); + assertTrue(result.matches((root + "/x\\..*\\.gor").replace("/", "\\/"))); } } From a48f0530737fc7d273d4b30049ac2ac5f4302b0c Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 23 Dec 2025 11:25:44 +0000 Subject: [PATCH 02/10] fix(ENGKNOW-2950): CONTENT_LOCATION_MANAGED renamed DATA_LOCATION and new means full parent path. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6b2aa47d..5c19bcfb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,7 +142,7 @@ jobs: files: '**/TEST-*.xml' publishSnapshot: - if: ${{ github.ref == 'refs/heads/main' }} + #if: ${{ github.ref == 'refs/heads/main' }} needs: [test, slowTest, integrationTest] runs-on: ubuntu-latest steps: From b020aa2c712addf2b8f563c4218ea63f14c866ab Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 23 Dec 2025 12:01:19 +0000 Subject: [PATCH 03/10] fix(ENGKNOW-2950): CONTENT_LOCATION_MANAGED renamed DATA_LOCATION and new means full parent path. --- gortools/src/test/java/gorsat/UTestPartGor.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/gortools/src/test/java/gorsat/UTestPartGor.java b/gortools/src/test/java/gorsat/UTestPartGor.java index 240ff87c..5979af00 100644 --- a/gortools/src/test/java/gorsat/UTestPartGor.java +++ b/gortools/src/test/java/gorsat/UTestPartGor.java @@ -309,16 +309,4 @@ public void testPartGorMixedTagsReplacement() { Assert.assertEquals("a,b,c | 'a','b','c' | \"a\",\"b\",\"c\"", result); } - @Test - public void partGorWithFileName() throws IOException { - String contents = "#col1\tcol2\tcol3\tcol4\tcol5\tcol6\tlis_PN\n" + - "data/bucket_1.gorz\tbucket_1\tchr1\t0\tchrZ\t1000000000\tSAMPLE_SIM842_000001\n" + - "data/bucket_10.gorz\tbucket_10\tchr1\t0\tchrZ\t1000000000\tSAMPLE_SIM842_000010\n"; - - File dictFile = FileTestUtils.createTempFile(workDir.getRoot(), "variants.gord", contents); - - String query = "partgor -dict " + dictFile.getAbsolutePath() + " <(gorrow chr1,1,1 | calc x \"#{tags:q}\" | calc fn 's3://test/test.gor')"; - String results = TestUtils.runGorPipe(query); - } - } From d5512f3ec11479e161ecdf6c18b0177fe1331801 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 23 Dec 2025 14:47:30 +0000 Subject: [PATCH 04/10] fix(ENGKNOW-2950): CONTENT_LOCATION_MANAGED renamed DATA_LOCATION and new means full parent path. --- gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala index 61885b76..d19bcd93 100644 --- a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala +++ b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala @@ -409,12 +409,11 @@ case class ForkWrite(forkCol: Int, session.getProjectContext.getFileReader.resolveUrl(FilenameUtils.removeExtension(linkFilePath), true) // Use the nonsecure driver file reader as this is an exception from the write no links rule. - val fileReader = new DriverBackedFileReader(session.getProjectContext.getFileReader.getSecurityContext, - session.getProjectContext.getProjectRoot, session.getProjectContext.getFileReader.getQueryTime) + val fileReader = session.getProjectContext.getSystemFileReader LinkFile.load(fileReader.resolveUrl(linkFilePath, true).asInstanceOf[StreamSource]) .appendMeta(linkFileMeta) .appendEntry(linkFileContent, md5, linkFileInfo, fileReader) - .save(session.getProjectContext.getFileReader.getQueryTime) + .save(fileReader.getQueryTime) } } From c71c8096595a197a055d93a20173d9da91c54207 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 23 Dec 2025 15:53:35 +0000 Subject: [PATCH 05/10] fix(ENGKNOW-2950): CONTENT_LOCATION_MANAGED renamed DATA_LOCATION and new means full parent path. --- gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala index d19bcd93..cc54c145 100644 --- a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala +++ b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala @@ -409,7 +409,7 @@ case class ForkWrite(forkCol: Int, session.getProjectContext.getFileReader.resolveUrl(FilenameUtils.removeExtension(linkFilePath), true) // Use the nonsecure driver file reader as this is an exception from the write no links rule. - val fileReader = session.getProjectContext.getSystemFileReader + val fileReader = session.getProjectContext.getFileReader.unsecure() LinkFile.load(fileReader.resolveUrl(linkFilePath, true).asInstanceOf[StreamSource]) .appendMeta(linkFileMeta) From a6810c4c90ad1c38ba2094a071c71a1138154b42 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 23 Dec 2025 17:58:24 +0000 Subject: [PATCH 06/10] fix(ENGKNOW-2950): CONTENT_LOCATION_MANAGED renamed DATA_LOCATION and new means full parent path. --- .../src/main/scala/gorsat/Analysis/ForkWrite.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala index cc54c145..a0969c3c 100644 --- a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala +++ b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala @@ -37,7 +37,9 @@ import org.gorpipe.gor.session.{GorSession, ProjectContext} import org.gorpipe.gor.table.util.PathUtils import org.gorpipe.gor.util.DataUtil import org.gorpipe.model.gor.RowObj +import org.gorpipe.model.gor.iterators.RefSeqFromConfig import org.gorpipe.util.Strings +import org.slf4j.{Logger, LoggerFactory} import java.util.UUID import scala.collection.mutable @@ -125,6 +127,8 @@ case class ForkWrite(forkCol: Int, inHeader: String, options: OutputOptions) extends Analysis { + private val log: Logger = LoggerFactory.getLogger(ForkWrite.this.getClass) + case class FileHolder(forkValue: String) { if (forkCol >= 0 && options.useFolder.isEmpty && !(fullFileName.contains("#{fork}") || fullFileName.contains("""${fork}"""))) { throw new GorResourceException("WRITE error: #{fork} of ${fork}missing from filename.", fullFileName) @@ -233,6 +237,7 @@ case class ForkWrite(forkCol: Int, * @return */ def createOutFile(name: String, skipHeader: Boolean): Output = { + log.info("Outfile filereader sec context " + Strings.isNullOrEmpty(session.getProjectContext.getFileReader.getSecurityContext)) if (rowHeader == null || useFork) { OutFile.driver(name, session.getProjectContext.getFileReader, header, skipHeader, options) } else { @@ -411,6 +416,11 @@ case class ForkWrite(forkCol: Int, // Use the nonsecure driver file reader as this is an exception from the write no links rule. val fileReader = session.getProjectContext.getFileReader.unsecure() + val source = fileReader.resolveUrl(linkFilePath, true) + log.info("Filereader Security context: " + Strings.isNullOrEmpty(fileReader.getSecurityContext)) + log.info("Source Security context: " + Strings.isNullOrEmpty(source.getSourceReference.getSecurityContext)) + log.info("Orignal filereader Security context: " + Strings.isNullOrEmpty(session.getProjectContext.getFileReader.getSecurityContext)) + LinkFile.load(fileReader.resolveUrl(linkFilePath, true).asInstanceOf[StreamSource]) .appendMeta(linkFileMeta) .appendEntry(linkFileContent, md5, linkFileInfo, fileReader) From 6e746db8e55acf7473259a1313583a4d3549a6a9 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Tue, 23 Dec 2025 19:09:32 +0000 Subject: [PATCH 07/10] fix(ENGKNOW-2950): CONTENT_LOCATION_MANAGED renamed DATA_LOCATION and new means full parent path. --- .../main/scala/gorsat/Analysis/ForkWrite.scala | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala index a0969c3c..ba2ec65b 100644 --- a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala +++ b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala @@ -30,14 +30,13 @@ import org.apache.commons.lang3.StringUtils import org.gorpipe.exceptions.GorResourceException import org.gorpipe.gor.binsearch.GorIndexType import org.gorpipe.gor.driver.linkfile.{LinkFile, LinkFileEntryV1} -import org.gorpipe.gor.driver.meta.{DataType, SourceReference} +import org.gorpipe.gor.driver.meta.DataType import org.gorpipe.gor.driver.providers.stream.sources.StreamSource -import org.gorpipe.gor.model.{DriverBackedFileReader, Row} +import org.gorpipe.gor.model.Row import org.gorpipe.gor.session.{GorSession, ProjectContext} import org.gorpipe.gor.table.util.PathUtils import org.gorpipe.gor.util.DataUtil import org.gorpipe.model.gor.RowObj -import org.gorpipe.model.gor.iterators.RefSeqFromConfig import org.gorpipe.util.Strings import org.slf4j.{Logger, LoggerFactory} @@ -163,9 +162,9 @@ case class ForkWrite(forkCol: Int, fullFileName.replace("#{fork}", forkValue).replace("""${fork}""", forkValue) } else { if (fullFileName.isEmpty && options.linkFile.nonEmpty) { - val (linkFileMeta, linkFileInfo) = extractLinkMetaInfo(options.linkFileMeta) - val linkSourceRef = new SourceReference(options.linkFile, null, projectContext.getFileReader.getCommonRoot, null, null, true); - // Infer the full file name from the link (and defautl locations) + val (linkFileMeta, _) = extractLinkMetaInfo(options.linkFileMeta) + val linkSourceRef = projectContext.getFileReader.createSourceReference(options.linkFile, true); + // Infer the full file name from the link (and default locations) LinkFile.inferDataFileNameFromLinkFile( projectContext.getFileReader.resolveDataSource(linkSourceRef).asInstanceOf[StreamSource], linkFileMeta); } else { @@ -237,7 +236,6 @@ case class ForkWrite(forkCol: Int, * @return */ def createOutFile(name: String, skipHeader: Boolean): Output = { - log.info("Outfile filereader sec context " + Strings.isNullOrEmpty(session.getProjectContext.getFileReader.getSecurityContext)) if (rowHeader == null || useFork) { OutFile.driver(name, session.getProjectContext.getFileReader, header, skipHeader, options) } else { @@ -417,9 +415,6 @@ case class ForkWrite(forkCol: Int, val fileReader = session.getProjectContext.getFileReader.unsecure() val source = fileReader.resolveUrl(linkFilePath, true) - log.info("Filereader Security context: " + Strings.isNullOrEmpty(fileReader.getSecurityContext)) - log.info("Source Security context: " + Strings.isNullOrEmpty(source.getSourceReference.getSecurityContext)) - log.info("Orignal filereader Security context: " + Strings.isNullOrEmpty(session.getProjectContext.getFileReader.getSecurityContext)) LinkFile.load(fileReader.resolveUrl(linkFilePath, true).asInstanceOf[StreamSource]) .appendMeta(linkFileMeta) From d2a21644d3e679c78ff4c3e149278aa6e7af795c Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Sun, 28 Dec 2025 17:23:12 +0000 Subject: [PATCH 08/10] fix(ENGKNOW-2950): Change defaults: allow duplicated=true, defaul link file = 0. --- .../org/gorpipe/gor/cli/link/LinkUpdateCommand.java | 2 +- .../src/main/scala/gorsat/Analysis/ForkWrite.scala | 4 +--- .../scala/gorsat/Utilities/IteratorUtilities.scala | 2 +- gortools/src/test/java/gorsat/UTestHeaderFlags.java | 5 +++-- gortools/src/test/java/gorsat/UTestRename.java | 12 +++++++++++- gortools/src/test/java/gorsat/parser/UTestCalc.java | 5 +++++ .../gorpipe/gor/driver/linkfile/LinkFileMeta.java | 2 +- .../gorpipe/gor/driver/linkfile/LinkFileTest.java | 4 ++-- 8 files changed, 25 insertions(+), 11 deletions(-) diff --git a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java index 1875043b..2aa7a34b 100644 --- a/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java +++ b/gortools/src/main/java/org/gorpipe/gor/cli/link/LinkUpdateCommand.java @@ -39,7 +39,7 @@ public class LinkUpdateCommand implements Runnable { public void run() { var normalizedLinkPath = LinkFile.validateAndUpdateLinkFileName(linkFilePath); try { - var linkFile = LinkFile.load(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(), true, this)); + var linkFile = LinkFile.loadV1(LinkStreamSourceProvider.resolve(normalizedLinkPath, mainCmd.getSecurityContext(), mainCmd.getProjectRoot(), true, this)); applyHeaders(linkFile); linkFile.appendEntry(linkValue, entryMd5, entryInfo); linkFile.save(); diff --git a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala index ba2ec65b..b2067f6c 100644 --- a/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala +++ b/gortools/src/main/scala/gorsat/Analysis/ForkWrite.scala @@ -414,9 +414,7 @@ case class ForkWrite(forkCol: Int, // Use the nonsecure driver file reader as this is an exception from the write no links rule. val fileReader = session.getProjectContext.getFileReader.unsecure() - val source = fileReader.resolveUrl(linkFilePath, true) - - LinkFile.load(fileReader.resolveUrl(linkFilePath, true).asInstanceOf[StreamSource]) + LinkFile.loadV1(fileReader.resolveUrl(linkFilePath, true).asInstanceOf[StreamSource]) .appendMeta(linkFileMeta) .appendEntry(linkFileContent, md5, linkFileInfo, fileReader) .save(fileReader.getQueryTime) diff --git a/gortools/src/main/scala/gorsat/Utilities/IteratorUtilities.scala b/gortools/src/main/scala/gorsat/Utilities/IteratorUtilities.scala index 79350286..df3a9877 100644 --- a/gortools/src/main/scala/gorsat/Utilities/IteratorUtilities.scala +++ b/gortools/src/main/scala/gorsat/Utilities/IteratorUtilities.scala @@ -43,7 +43,7 @@ object IteratorUtilities { private val logger = LoggerFactory.getLogger(this.getClass) - private val ALLOW_DUPLICATE_COLUMNS = System.getProperty("gor.iterators.allowDuplicateColumns", "false").toBoolean + private val ALLOW_DUPLICATE_COLUMNS = System.getProperty("gor.iterators.allowDuplicateColumns", "true").toBoolean def getHeader(filename: String, gorRoot: String, context: GorContext): String = { val gm: GorMonitor = null diff --git a/gortools/src/test/java/gorsat/UTestHeaderFlags.java b/gortools/src/test/java/gorsat/UTestHeaderFlags.java index 5eb5dc62..d63415e2 100644 --- a/gortools/src/test/java/gorsat/UTestHeaderFlags.java +++ b/gortools/src/test/java/gorsat/UTestHeaderFlags.java @@ -83,16 +83,17 @@ public void testValidHeaderUsedKeywordsWithDupAllowingDup() { Assert.assertFalse(systemErrRule.getLog().contains("Duplicate column name 'order'")); } + @Ignore("Only works if run alone, as the property is read only on class load") @Test public void testValidHeaderUsedKeywordsWithDupNotAllowingDup() { + System.setProperty("gor.iterators.allowDuplicateColumns", "false"); String testHeader = "#abc\tstart\tfrom\tselect\tmax\tmin\tfrom\tgroup\trange\torder\trank\torder"; Assert.assertThrows(GorDataException.class, () -> IteratorUtilities.validHeader(testHeader, false)); } - @Ignore("Only works if run alone, as the property is read only on class load") + @Test public void testValidHeaderUsedKeywordsWithDupAllowingDupOnlyGlobal() { - System.setProperty("gor.iterators.allowDuplicateColumns", "true"); String testHeader = "#abc\tstart\tfrom\tselect\tmax\tmin\tfrom\tgroup\trange\torder\trank\torder"; String resultingHeader = IteratorUtilities.validHeader(testHeader, false); Assert.assertEquals("#abc\tstart\tfrom\tselect\tmax\tmin\tfromx\tgroup\trange\torder\trank\torderx", resultingHeader); diff --git a/gortools/src/test/java/gorsat/UTestRename.java b/gortools/src/test/java/gorsat/UTestRename.java index fa833d7e..347ea669 100644 --- a/gortools/src/test/java/gorsat/UTestRename.java +++ b/gortools/src/test/java/gorsat/UTestRename.java @@ -25,6 +25,7 @@ import org.gorpipe.exceptions.GorDataException; import org.gorpipe.exceptions.GorParsingException; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -139,7 +140,16 @@ public void renameStrictWithNoMatch() { } @Test - public void renameToExisting() { + public void renameToExistingAllowDup() { + String query = "gorrow chr1,1 | calc Foo 0 | calc Bar 1 | rename Foo Bar | top 0"; + String result = TestUtils.runGorPipe(query); + Assert.assertEquals("chrom\tpos\tBar\tBarx\n", result); + } + + @Ignore("Only works if run alone, as the property is read only on class load") + @Test + public void renameToExistingNotAllowDup() { + System.setProperty("gor.iterators.allowDuplicateColumns", "false"); String query = "gorrow chr1,1 | calc Foo 0 | calc Bar 1 | rename Foo Bar | top 0"; Assert.assertThrows(GorDataException.class, () -> TestUtils.runGorPipe(query)); } diff --git a/gortools/src/test/java/gorsat/parser/UTestCalc.java b/gortools/src/test/java/gorsat/parser/UTestCalc.java index f6c2bd6b..530a65b4 100644 --- a/gortools/src/test/java/gorsat/parser/UTestCalc.java +++ b/gortools/src/test/java/gorsat/parser/UTestCalc.java @@ -25,6 +25,7 @@ import gorsat.TestUtils; import org.gorpipe.exceptions.GorDataException; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.util.Arrays; @@ -38,13 +39,17 @@ public void addsNewColumn() { Assert.assertEquals(expected, result); } + @Ignore("Only works if run alone, as the property is read only on class load") @Test public void addsNewExistingColumn() { + System.setProperty("gor.iterators.allowDuplicateColumns", "false"); Assert.assertThrows(GorDataException.class, () -> TestUtils.runGorPipe("gorrow 1,1 | calc pos 42")); } + @Ignore("Only works if run alone, as the property is read only on class load") @Test public void addsNewColumnWithSuffixWhenItExists() { + System.setProperty("gor.iterators.allowDuplicateColumns", "false"); Assert.assertThrows(GorDataException.class, () -> TestUtils.runGorPipe("gorrow 1,1 | calc data 42 | calc data 3.14")); } diff --git a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java index a07a5361..38758e54 100644 --- a/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java +++ b/model/src/main/java/org/gorpipe/gor/driver/linkfile/LinkFileMeta.java @@ -18,7 +18,7 @@ public class LinkFileMeta extends BaseMeta { // Should the content lifecycle be managed (data deleted if the link is removed from the link file) (true or false). public static final String HEADER_DATA_LIFECYCLE_MANAGED_KEY = "DATA_LIFECYCLE_MANAGED"; - private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", "1"); + private static final String DEFAULT_VERSION = System.getProperty("gor.driver.link.default.version", LinkFileV0.VERSION); public static final int DEFAULT_ENTRIES_COUNT_MAX = 100; public static final long DEFAULT_ENTRIES_AGE_MAX = Long.MAX_VALUE; diff --git a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java index 77dd81e8..430ba131 100644 --- a/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java +++ b/model/src/test/java/org/gorpipe/gor/driver/linkfile/LinkFileTest.java @@ -64,8 +64,8 @@ public void testCreateLinkFile() { } @Test - public void testCreateLinkFileSimple() { - LinkFile linkFile = LinkFile.create(mockSource, "test.gorz"); + public void testCreateLinkFileSimple() throws IOException { + LinkFile linkFile = LinkFile.createV1(mockSource, "test.gorz"); assertNotNull(linkFile); assertTrue(linkFile instanceof LinkFileV1); assertEquals("1", linkFile.getMeta().getVersion()); From 0b500da64cc17b1ab10aeb6d5ee4a4d26bf9edc3 Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 29 Dec 2025 02:35:12 +0000 Subject: [PATCH 09/10] fix(ENGKNOW-2950): Change defaults: allow duplicated=true, defaul link file = 0. --- .../java/org/gorpipe/gor/table/TableCache.java | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/model/src/main/java/org/gorpipe/gor/table/TableCache.java b/model/src/main/java/org/gorpipe/gor/table/TableCache.java index 44dfd4ec..7a61f7bb 100644 --- a/model/src/main/java/org/gorpipe/gor/table/TableCache.java +++ b/model/src/main/java/org/gorpipe/gor/table/TableCache.java @@ -29,24 +29,14 @@ public T getOrCreateTable(String path, FileReader fileReader) throws IOException } public T getOrCreateTable(String path, FileReader fileReader, boolean useCache) throws IOException { - try { - return getTable(path, fileReader, useCache); - } catch (NoSuchFileException e) { - return createTable(path, fileReader); - } + return getTable(path, fileReader, useCache); } - public synchronized T getTable(String path, FileReader fileReader) throws IOException { + public synchronized T getTable(String path, FileReader fileReader) { return getTable(path, fileReader, useCache); } - public synchronized T getTable(String path, FileReader fileReader, boolean useCache) throws IOException { - // TODO: To make fewer calls to exists consider caching it in metadata. Should not need this as getSourceMetaData should throw - // exception if file does not exists. - if (!fileReader.exists(path)) { - throw new NoSuchFileException(path); - } - // The dict is lazy loaded so the only cost is finding the id. + public synchronized T getTable(String path, FileReader fileReader, boolean useCache) { T dict = createTable(path, fileReader); if (useCache) { From 90538c8b407f845eaa87455e8813b5be37b1b3ac Mon Sep 17 00:00:00 2001 From: Gisli Magnusson Date: Mon, 29 Dec 2025 11:45:02 +0000 Subject: [PATCH 10/10] fix(ENGKNOW-2950): Prep pr. --- .github/workflows/build.yml | 2 +- model/src/main/java/org/gorpipe/gor/table/TableCache.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c19bcfb..6b2aa47d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,7 +142,7 @@ jobs: files: '**/TEST-*.xml' publishSnapshot: - #if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} needs: [test, slowTest, integrationTest] runs-on: ubuntu-latest steps: diff --git a/model/src/main/java/org/gorpipe/gor/table/TableCache.java b/model/src/main/java/org/gorpipe/gor/table/TableCache.java index 7a61f7bb..37f52ac7 100644 --- a/model/src/main/java/org/gorpipe/gor/table/TableCache.java +++ b/model/src/main/java/org/gorpipe/gor/table/TableCache.java @@ -37,6 +37,7 @@ public synchronized T getTable(String path, FileReader fileReader) { } public synchronized T getTable(String path, FileReader fileReader, boolean useCache) { + // The dict is lazy loaded so the only cost is finding the id. T dict = createTable(path, fileReader); if (useCache) {