From f5961a619293b7b71313967e3b3e033999adea9a Mon Sep 17 00:00:00 2001 From: Femi3211 Date: Mon, 10 Nov 2025 11:24:26 +0100 Subject: [PATCH] modified: Source DBT Yaml reading to be more lenient --- .../rosetta/cli/services/DbtModelService.java | 14 ++++++- .../common/models/dbt/AbstractDbtModel.java | 37 +++++++++++++++++++ .../rosetta/common/models/dbt/DbtColumn.java | 2 +- .../rosetta/common/models/dbt/DbtModel.java | 2 +- .../rosetta/common/models/dbt/DbtSource.java | 2 +- .../rosetta/common/models/dbt/DbtTable.java | 2 +- 6 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/AbstractDbtModel.java diff --git a/cli/src/main/java/com/adaptivescale/rosetta/cli/services/DbtModelService.java b/cli/src/main/java/com/adaptivescale/rosetta/cli/services/DbtModelService.java index d10b6a65..808c3a54 100644 --- a/cli/src/main/java/com/adaptivescale/rosetta/cli/services/DbtModelService.java +++ b/cli/src/main/java/com/adaptivescale/rosetta/cli/services/DbtModelService.java @@ -11,10 +11,16 @@ import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.representer.Representer; +import org.yaml.snakeyaml.constructor.Constructor; import queryhelper.pojo.GenericResponse; import queryhelper.service.DbtAIService; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -324,8 +330,14 @@ public List readDbtModels(Path directory) throws IOException { public List readDbtModels(List filePaths) throws IOException { List result = new ArrayList<>(); + + LoaderOptions loaderOptions = new LoaderOptions(); + loaderOptions.setMaxAliasesForCollections(Integer.MAX_VALUE); + + Yaml yaml = new Yaml(loaderOptions); for (String filePath : filePaths) { - result.add(yamlMapper.readValue(new File(filePath), DbtModel.class)); + Object temp = yaml.load(new FileInputStream(filePath)); + result.add(yamlMapper.convertValue(temp, DbtModel.class)); } return result; } diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/AbstractDbtModel.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/AbstractDbtModel.java new file mode 100644 index 00000000..a251b9b4 --- /dev/null +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/AbstractDbtModel.java @@ -0,0 +1,37 @@ +package com.adaptivescale.rosetta.common.models.dbt; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonInclude; + +import java.util.HashMap; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class AbstractDbtModel { + private Map additionalProperties = new HashMap<>(); + + @JsonAnyGetter + public Map getAdditionalProperties() { + return additionalProperties; + } + + @JsonAnySetter + public void setAdditionalProperties(Map additionalProperties) { + this.additionalProperties = additionalProperties; + } + + public void addProperty(Object name, Object value) { + this.additionalProperties.put(name, value); + } + + public Object getProperty(Object name) { + if (additionalProperties.containsKey(name)) + return this.additionalProperties.get(name); + return null; + } + + public String getPropertyAsString(Object name) { + return (String) getProperty(name); + } +} \ No newline at end of file diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtColumn.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtColumn.java index e5450ca7..117eb1ee 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtColumn.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtColumn.java @@ -8,7 +8,7 @@ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -public class DbtColumn { +public class DbtColumn extends AbstractDbtModel{ private String name; private String description; diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtModel.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtModel.java index 3d2fdba1..8eed47a4 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtModel.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtModel.java @@ -7,7 +7,7 @@ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -public class DbtModel { +public class DbtModel extends AbstractDbtModel{ private Integer version; private Collection sources; //sources -> tables private Collection models; //models (direct list) diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtSource.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtSource.java index 932e6a53..53768fb1 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtSource.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtSource.java @@ -7,7 +7,7 @@ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -public class DbtSource { +public class DbtSource extends AbstractDbtModel{ private String name; private String description; private Collection tables; diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtTable.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtTable.java index 2baeef89..b9915bf8 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtTable.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/dbt/DbtTable.java @@ -7,7 +7,7 @@ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) -public class DbtTable { +public class DbtTable extends AbstractDbtModel{ private String name; private String description; private Collection columns;