From 117dba3b2364a416773be7a8b4750c652cceb012 Mon Sep 17 00:00:00 2001 From: nbesimi Date: Sun, 21 May 2023 22:20:37 +0200 Subject: [PATCH 1/6] data transfer: initial commit --- .../com/adaptivescale/rosetta/cli/Cli.java | 36 ++++++++++ .../rosetta/common/models/Table.java | 20 ++++++ .../rosetta/ddl/utils/TemplateEngine.java | 13 ++++ .../test/assertion/DefaultSqlExecution.java | 67 +++++++++++++++++++ .../rosetta/test/assertion/SqlExecution.java | 2 + 5 files changed, 138 insertions(+) diff --git a/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java b/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java index ac53c19f..40e974e3 100644 --- a/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java +++ b/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java @@ -261,6 +261,42 @@ private void test(@CommandLine.Option(names = {"-s", "--source"}) String sourceN } } + @CommandLine.Command(name = "transfer", description = "Transfer data from source to target", mixinStandardHelpOptions = true) + private void transfer(@CommandLine.Option(names = {"-s", "--source"}) String sourceName, + @CommandLine.Option(names = {"-t", "--target"}) String targetName) throws Exception { + requireConfig(config); + + Optional source = config.getConnection(sourceName); + if (source.isEmpty()) { + throw new RuntimeException("Can not find source with name: " + sourceName + " configured in config."); + } + + Optional target = config.getConnection(targetName); + if (target.isEmpty()) { + throw new RuntimeException("Can not find target with name: " + targetName + " configured in config."); + } + Path sourceWorkspace = Paths.get("./", sourceName); + Path targWorkspace = Paths.get("./", targetName); + + if (!Files.isDirectory(sourceWorkspace)) { + throw new RuntimeException(String.format("Can not find directory: %s for source name: %s to find" + + " models for translation", sourceWorkspace, sourceName)); + } + + if (!Files.isDirectory(targWorkspace)) { + throw new RuntimeException(String.format("Can not find directory: %s for source name: %s to find" + + " models for translation", sourceWorkspace, sourceName)); + } + + List collect = getDatabases(targWorkspace) + .map(AbstractMap.SimpleImmutableEntry::getValue) + .collect(Collectors.toList()); + for (Database database : collect) { + DefaultSqlExecution defaultSqlExecution = new DefaultSqlExecution(source.get(), new DriverManagerDriverProvider(), database, target.get()); + defaultSqlExecution.transfer(); + } + } + @CommandLine.Command(name = "init", description = "Creates a sample config (main.conf) and model directory.", mixinStandardHelpOptions = true) private void init(@CommandLine.Parameters(index = "0", description = "Project name.", defaultValue = "") String projectName) throws IOException { diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java index 2969088f..6f97bad0 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java @@ -17,6 +17,10 @@ public class Table { private Collection columns; + private String extract; + + private String load; + public String getName() { return name; } @@ -74,6 +78,22 @@ public void setIndices(List indices) { this.indices = indices; } + public String getExtract() { + return extract; + } + + public void setExtract(String extract) { + this.extract = extract; + } + + public String getLoad() { + return load; + } + + public void setLoad(String load) { + this.load = load; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/utils/TemplateEngine.java b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/utils/TemplateEngine.java index 2a60b99a..01fa9228 100644 --- a/ddl/src/main/java/com/adaptivescale/rosetta/ddl/utils/TemplateEngine.java +++ b/ddl/src/main/java/com/adaptivescale/rosetta/ddl/utils/TemplateEngine.java @@ -19,12 +19,14 @@ import org.thymeleaf.context.Context; import org.thymeleaf.templatemode.TemplateMode; import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver; +import org.thymeleaf.templateresolver.StringTemplateResolver; import java.util.Map; public class TemplateEngine { private static TemplateEngine instance = null; private org.thymeleaf.TemplateEngine engine = null; + private TemplateEngine() { ClassLoaderTemplateResolver resolver = new ClassLoaderTemplateResolver(); resolver.setTemplateMode(TemplateMode.TEXT); @@ -42,6 +44,17 @@ public static TemplateEngine get() { return TemplateEngine.instance; } + public static String processString(String templateName, Map variables) { + org.thymeleaf.TemplateEngine templateEngine = new org.thymeleaf.TemplateEngine(); + StringTemplateResolver templateResolver = new StringTemplateResolver(); + templateResolver.setTemplateMode(TemplateMode.HTML); + templateEngine.setTemplateResolver(templateResolver); + + Context context = new Context(); + context.setVariables(variables); + return templateEngine.process(templateName, context); + } + public static String process(String templateName, Map variables) { Context context = new Context(); context.setVariables(variables); diff --git a/test/src/main/java/com/adaptivescale/rosetta/test/assertion/DefaultSqlExecution.java b/test/src/main/java/com/adaptivescale/rosetta/test/assertion/DefaultSqlExecution.java index 45dd5343..3b71c857 100644 --- a/test/src/main/java/com/adaptivescale/rosetta/test/assertion/DefaultSqlExecution.java +++ b/test/src/main/java/com/adaptivescale/rosetta/test/assertion/DefaultSqlExecution.java @@ -2,10 +2,14 @@ import com.adaptivescale.rosetta.common.JDBCDriverProvider; import com.adaptivescale.rosetta.common.JDBCUtils; +import com.adaptivescale.rosetta.common.models.Database; import com.adaptivescale.rosetta.common.models.input.Connection; +import com.adaptivescale.rosetta.ddl.utils.TemplateEngine; import lombok.extern.slf4j.Slf4j; import java.sql.*; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; @Slf4j @@ -13,11 +17,22 @@ public class DefaultSqlExecution implements SqlExecution { private final Connection connection; private final JDBCDriverProvider driverProvider; + private Database database; + + private Connection targetConnection; + public DefaultSqlExecution(Connection connection, JDBCDriverProvider driverProvider) { this.connection = connection; this.driverProvider = driverProvider; } + public DefaultSqlExecution(Connection connection, JDBCDriverProvider driverProvider, Database database, Connection targetConnection) { + this.connection = connection; + this.driverProvider = driverProvider; + this.database = database; + this.targetConnection = targetConnection; + } + @Override public String execute(String sql) { java.sql.Connection sqlConnection = null; @@ -44,4 +59,56 @@ public String execute(String sql) { } throw new RuntimeException(String.format("Execution of query: '%s' returns no data", sql)); } + + @Override + public String transfer() { + java.sql.Connection sourceSqlConnection = null; + java.sql.Connection targetSqlConnection = null; + String select = database.getTables().stream().findFirst().get().getExtract(); + String insert = database.getTables().stream().findFirst().get().getLoad(); + try { + Driver sourceDriver = driverProvider.getDriver(connection); + Properties properties = JDBCUtils.setJDBCAuth(connection); + sourceSqlConnection = sourceDriver.connect(connection.getUrl(), properties); + + Driver targetDriver = driverProvider.getDriver(targetConnection); + Properties targetProperties = JDBCUtils.setJDBCAuth(targetConnection); + targetSqlConnection = targetDriver.connect(targetConnection.getUrl(), targetProperties); + + ResultSet resultSet = sourceSqlConnection.createStatement().executeQuery(select); + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = resultSet.getMetaData().getColumnCount(); + + while (resultSet.next()) { + Map insertParam = new HashMap<>(); + for (int i = 0; i < columnCount; i++) { + int index = i+1; + + Object value = resultSet.getObject(metaData.getColumnName(index)); + insertParam.put(metaData.getColumnName(index), value); + } + String insertStm = TemplateEngine.processString(insert, insertParam); + targetSqlConnection.createStatement().execute(insertStm); + } + +// boolean execute = targetSqlConnection.createStatement().execute(insert); +// if (execute.next()) { +// int result = execute.getInt(1); +// return String.valueOf(result); +// } + return "OK"; + } catch (SQLException e) { + log.error("Can not execute query.", e); + throw new RuntimeException(e); + } finally { + if (sourceSqlConnection != null) { + try { + sourceSqlConnection.close(); + } catch (SQLException e) { + log.error("Can not close the connection!", e); + } + } + } +// throw new RuntimeException(String.format("Execution of query: '%s' returns no data", insert)); + } } diff --git a/test/src/main/java/com/adaptivescale/rosetta/test/assertion/SqlExecution.java b/test/src/main/java/com/adaptivescale/rosetta/test/assertion/SqlExecution.java index 2f8148c4..d4130f0b 100644 --- a/test/src/main/java/com/adaptivescale/rosetta/test/assertion/SqlExecution.java +++ b/test/src/main/java/com/adaptivescale/rosetta/test/assertion/SqlExecution.java @@ -2,4 +2,6 @@ public interface SqlExecution { String execute(String sql); + + String transfer(); } From f100a05b3ce66064d6fa3f8dc1600391ccaf2ff5 Mon Sep 17 00:00:00 2001 From: nbesimi Date: Tue, 6 Jun 2023 11:52:49 +0200 Subject: [PATCH 2/6] added: load extract generators during extract --- .../com/adaptivescale/rosetta/cli/Cli.java | 3 +- .../rosetta/common/models/Table.java | 28 +++++++++++++++++++ .../rosetta/source/core/DefaultGenerator.java | 8 ++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java b/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java index 40e974e3..eb0dfa2b 100644 --- a/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java +++ b/cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java @@ -79,7 +79,8 @@ public Void call() { @CommandLine.Command(name = "extract", description = "Extract schema chosen from connection config.", mixinStandardHelpOptions = true) private void extract(@CommandLine.Option(names = {"-s", "--source"}, required = true) String sourceName, - @CommandLine.Option(names = {"-t", "--convert-to"}) String targetName + @CommandLine.Option(names = {"-t", "--convert-to"}) String targetName, + @CommandLine.Option(names = {"--include-data"}) boolean includeData ) throws Exception { requireConfig(config); Connection source = getSourceConnection(sourceName); diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java index 6f97bad0..0e4c2c7c 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java @@ -3,6 +3,8 @@ import java.util.Collection; import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; public class Table { @@ -94,6 +96,32 @@ public void setLoad(String load) { this.load = load; } + public void generateExtractSql() { + final StringBuilder extractSql = new StringBuilder(); + List columnNames = this.columns + .stream() + .map(Column::getName) + .collect(Collectors.toList()); + extractSql.append("SELECT " + String.join(", ", columnNames) + " FROM " + this.name + ";"); + this.extract = extractSql.toString(); + } + + public void generateLoadSql() { + final StringBuilder loadSql = new StringBuilder(); + List columnNames = this.columns + .stream() + .map(Column::getName) + .collect(Collectors.toList()); + loadSql.append("INSERT INTO " + this.name + " (" + String.join(", ", columnNames) + ")"); + + List columnNameTemplates = this.columns + .stream().map(Column::getName) + .map(it -> String.format("'[(${%s})]'", it)) + .collect(Collectors.toList()); + loadSql.append(" VALUES (" + String.join(", ", columnNameTemplates) + ")"); + this.load = loadSql.toString(); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java b/source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java index 4b2334aa..72171e10 100644 --- a/source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java +++ b/source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java @@ -45,7 +45,15 @@ public Database generate(Connection connection) throws Exception { database.setTables(tables); database.setViews(views); database.setDatabaseType(connection.getDbType()); + includeData(tables); connect.close(); return database; } + + private void includeData(Collection tables) { + for (Table table : tables) { + table.generateExtractSql(); + table.generateLoadSql(); + } + } } \ No newline at end of file From b6ec36aba9bb130e28ca4d85dde60c79c7016340 Mon Sep 17 00:00:00 2001 From: nbesimi Date: Tue, 6 Jun 2023 14:39:37 +0200 Subject: [PATCH 3/6] added: batch insert initial version --- .../test/assertion/DefaultSqlExecution.java | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/test/src/main/java/com/adaptivescale/rosetta/test/assertion/DefaultSqlExecution.java b/test/src/main/java/com/adaptivescale/rosetta/test/assertion/DefaultSqlExecution.java index 3b71c857..74b733a5 100644 --- a/test/src/main/java/com/adaptivescale/rosetta/test/assertion/DefaultSqlExecution.java +++ b/test/src/main/java/com/adaptivescale/rosetta/test/assertion/DefaultSqlExecution.java @@ -3,14 +3,13 @@ import com.adaptivescale.rosetta.common.JDBCDriverProvider; import com.adaptivescale.rosetta.common.JDBCUtils; import com.adaptivescale.rosetta.common.models.Database; +import com.adaptivescale.rosetta.common.models.Table; import com.adaptivescale.rosetta.common.models.input.Connection; import com.adaptivescale.rosetta.ddl.utils.TemplateEngine; import lombok.extern.slf4j.Slf4j; import java.sql.*; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; +import java.util.*; @Slf4j public class DefaultSqlExecution implements SqlExecution { @@ -62,10 +61,18 @@ public String execute(String sql) { @Override public String transfer() { + for (Table table : database.getTables()) { + doTransfer(table); + } + + return "OK"; + } + + private String doTransfer(Table table) { java.sql.Connection sourceSqlConnection = null; java.sql.Connection targetSqlConnection = null; - String select = database.getTables().stream().findFirst().get().getExtract(); - String insert = database.getTables().stream().findFirst().get().getLoad(); + String select = table.getExtract(); + String insert = table.getLoad(); try { Driver sourceDriver = driverProvider.getDriver(connection); Properties properties = JDBCUtils.setJDBCAuth(connection); @@ -79,6 +86,9 @@ public String transfer() { ResultSetMetaData metaData = resultSet.getMetaData(); int columnCount = resultSet.getMetaData().getColumnCount(); +// List insertStatements = new ArrayList<>(); + Statement statement = targetSqlConnection.createStatement(); + while (resultSet.next()) { Map insertParam = new HashMap<>(); for (int i = 0; i < columnCount; i++) { @@ -88,15 +98,15 @@ public String transfer() { insertParam.put(metaData.getColumnName(index), value); } String insertStm = TemplateEngine.processString(insert, insertParam); - targetSqlConnection.createStatement().execute(insertStm); +// insertStatements.add(insertStm); + statement.addBatch(insertStm); +// targetSqlConnection.createStatement().execute(insertStm); } -// boolean execute = targetSqlConnection.createStatement().execute(insert); -// if (execute.next()) { -// int result = execute.getInt(1); -// return String.valueOf(result); -// } - return "OK"; + int[] result = statement.executeBatch(); +// targetSqlConnection.commit(); + + return "Rows affected " + result.length; } catch (SQLException e) { log.error("Can not execute query.", e); throw new RuntimeException(e); @@ -104,6 +114,7 @@ public String transfer() { if (sourceSqlConnection != null) { try { sourceSqlConnection.close(); + return "OK"; } catch (SQLException e) { log.error("Can not close the connection!", e); } From f5f7e54a93852b4f01490ee5314805dd97be8364 Mon Sep 17 00:00:00 2001 From: nbesimi Date: Fri, 9 Jun 2023 14:13:14 +0200 Subject: [PATCH 4/6] added: data transfer select and insert generation through thymeleaf --- .../rosetta/common/models/Column.java | 7 +++ source/build.gradle | 2 +- .../rosetta/source/common/TemplateEngine.java | 63 +++++++++++++++++++ .../rosetta/source/core/DefaultGenerator.java | 17 ++++- .../resources/templates/bigquery/insert.sqlt | 21 +++++++ .../resources/templates/bigquery/select.sqlt | 7 +++ .../templates/default/experiments.sqlt | 33 ++++++++++ .../resources/templates/default/insert.sqlt | 18 ++++++ .../resources/templates/default/select.sqlt | 7 +++ .../resources/templates/postgres/insert.sqlt | 17 +++++ 10 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 source/src/main/java/com/adataptivescale/rosetta/source/common/TemplateEngine.java create mode 100644 source/src/main/resources/templates/bigquery/insert.sqlt create mode 100644 source/src/main/resources/templates/bigquery/select.sqlt create mode 100644 source/src/main/resources/templates/default/experiments.sqlt create mode 100644 source/src/main/resources/templates/default/insert.sqlt create mode 100644 source/src/main/resources/templates/default/select.sqlt create mode 100644 source/src/main/resources/templates/postgres/insert.sqlt diff --git a/common/src/main/java/com/adaptivescale/rosetta/common/models/Column.java b/common/src/main/java/com/adaptivescale/rosetta/common/models/Column.java index f1f980ab..8067ac33 100644 --- a/common/src/main/java/com/adaptivescale/rosetta/common/models/Column.java +++ b/common/src/main/java/com/adaptivescale/rosetta/common/models/Column.java @@ -1,6 +1,7 @@ package com.adaptivescale.rosetta.common.models; import com.adaptivescale.rosetta.common.models.test.Tests; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.List; @@ -33,6 +34,11 @@ public void setName(String name) { this.name = name; } + @JsonIgnore + public String getTemplatedName() { + return String.format("[(${%s})]", this.name); + } + public String getLabel() { return label; } @@ -137,4 +143,5 @@ public void setTests(Tests tests) { this.tests = tests; } + } diff --git a/source/build.gradle b/source/build.gradle index 58743921..73df8f61 100644 --- a/source/build.gradle +++ b/source/build.gradle @@ -16,7 +16,7 @@ dependencies { compileOnly 'org.projectlombok:lombok:1.18.12' annotationProcessor 'org.projectlombok:lombok:1.18.12' implementation 'org.apache.commons:commons-lang3:3.12.0' - + implementation 'org.thymeleaf:thymeleaf:3.1.0.RELEASE' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.0' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.0' diff --git a/source/src/main/java/com/adataptivescale/rosetta/source/common/TemplateEngine.java b/source/src/main/java/com/adataptivescale/rosetta/source/common/TemplateEngine.java new file mode 100644 index 00000000..17215c45 --- /dev/null +++ b/source/src/main/java/com/adataptivescale/rosetta/source/common/TemplateEngine.java @@ -0,0 +1,63 @@ +/* + * Copyright 2022 AdaptiveScale + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adataptivescale.rosetta.source.common; + +import org.thymeleaf.context.Context; +import org.thymeleaf.templatemode.TemplateMode; +import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver; +import org.thymeleaf.templateresolver.StringTemplateResolver; + +import java.util.Map; + +public class TemplateEngine { + private static TemplateEngine instance = null; + private org.thymeleaf.TemplateEngine engine = null; + + private TemplateEngine() { + ClassLoaderTemplateResolver resolver = new ClassLoaderTemplateResolver(); + resolver.setTemplateMode(TemplateMode.TEXT); + resolver.setCharacterEncoding("UTF-8"); + resolver.setPrefix("/templates/"); + resolver.setSuffix(".sqlt"); + engine = new org.thymeleaf.TemplateEngine(); + engine.setTemplateResolver(resolver); + } + + public static TemplateEngine get() { + if(TemplateEngine.instance == null) { + TemplateEngine.instance = new TemplateEngine(); + } + return TemplateEngine.instance; + } + + public static String processString(String templateName, Map variables) { + org.thymeleaf.TemplateEngine templateEngine = new org.thymeleaf.TemplateEngine(); + StringTemplateResolver templateResolver = new StringTemplateResolver(); + templateResolver.setTemplateMode(TemplateMode.HTML); + templateEngine.setTemplateResolver(templateResolver); + + Context context = new Context(); + context.setVariables(variables); + return templateEngine.process(templateName, context); + } + + public static String process(String templateName, Map variables) { + Context context = new Context(); + context.setVariables(variables); + return get().engine.process(templateName, context); + } +} diff --git a/source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java b/source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java index 72171e10..05fab2ce 100644 --- a/source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java +++ b/source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java @@ -6,6 +6,7 @@ import com.adaptivescale.rosetta.common.models.Table; import com.adaptivescale.rosetta.common.models.View; import com.adaptivescale.rosetta.common.models.input.Connection; +import com.adataptivescale.rosetta.source.common.TemplateEngine; import com.adataptivescale.rosetta.source.core.interfaces.ColumnExtractor; import com.adataptivescale.rosetta.source.core.interfaces.Generator; import com.adataptivescale.rosetta.source.core.interfaces.TableExtractor; @@ -52,8 +53,20 @@ public Database generate(Connection connection) throws Exception { private void includeData(Collection
tables) { for (Table table : tables) { - table.generateExtractSql(); - table.generateLoadSql(); + StringBuilder extractStringBuilder = new StringBuilder(); + StringBuilder loadStringBuilder = new StringBuilder(); + Map createParams = new HashMap<>(); + + createParams.put("table", table); + extractStringBuilder.append(TemplateEngine.process("bigquery/select", createParams)); + loadStringBuilder.append(TemplateEngine.process("bigquery/insert", createParams)); + + table.setExtract( + extractStringBuilder.toString().replaceAll("(\\r|\\n|\\t)", " ").replaceAll(" +", " ") + ); + table.setLoad( + loadStringBuilder.toString().replaceAll("(\\r|\\n|\\t)", " ").replaceAll(" +", " ") + ); } } } \ No newline at end of file diff --git a/source/src/main/resources/templates/bigquery/insert.sqlt b/source/src/main/resources/templates/bigquery/insert.sqlt new file mode 100644 index 00000000..e19fd36b --- /dev/null +++ b/source/src/main/resources/templates/bigquery/insert.sqlt @@ -0,0 +1,21 @@ +INSERT INTO [[${table.schema}]].[[${table.name}]] +( + [# th:each="column : ${table.columns}"] + [[${column.name}]] + + [# th:if="${columnStat.index < table.columns.size - 1}"],[/] + [/] +) +VALUES +( + [# th:each="column : ${table.columns}"] + [# th:switch="${column.typeName}"] + [# th:case="'INT64'"][[${column.getTemplatedName()}]][/] + [# th:case="*"]'[[${column.getTemplatedName()}]]'[/] + + [# th:if="${columnStat.index < table.columns.size - 1}"],[/] + [/] + [/] +); + + diff --git a/source/src/main/resources/templates/bigquery/select.sqlt b/source/src/main/resources/templates/bigquery/select.sqlt new file mode 100644 index 00000000..cffb0a28 --- /dev/null +++ b/source/src/main/resources/templates/bigquery/select.sqlt @@ -0,0 +1,7 @@ +SELECT + [# th:each="column : ${table.columns}"] + [[${column.name}]] + + [# th:if="${columnStat.index < table.columns.size - 1}"],[/] + [/] +FROM [[${table.schema}]].[[${table.name}]]; \ No newline at end of file diff --git a/source/src/main/resources/templates/default/experiments.sqlt b/source/src/main/resources/templates/default/experiments.sqlt new file mode 100644 index 00000000..ef0de5e8 --- /dev/null +++ b/source/src/main/resources/templates/default/experiments.sqlt @@ -0,0 +1,33 @@ +Accessing object [[$${name}]], + +[[$${#if(status == 'paid')}]] +Your payment has been received. Thank you! +[[$${#else}]] +Please submit your payment by the due date. +[[$${/if}]] + + +[[$${#for(column : table.columns)}]] + [[$${column}]] +[[$${/for}]] + + +-- Generate SQL INSERT statement + +INSERT INTO [[${table.tableName}]] +( + [[#th:each(column : ${table.columns})]] + [[${column.columnName}]], + [[/th:each]] +) +VALUES +( + [[#th:each(column : ${table.columns})]] + [[#if(column.columnType == 'String')]] + '[[${column.columnValue}]]' + [[#else]] + [[${column.columnValue}]] + [[/if]] + [[#if(column.name != ${table.columns.last().name})]],[[/if]] + [[/th:each]] +); \ No newline at end of file diff --git a/source/src/main/resources/templates/default/insert.sqlt b/source/src/main/resources/templates/default/insert.sqlt new file mode 100644 index 00000000..4f39ec1c --- /dev/null +++ b/source/src/main/resources/templates/default/insert.sqlt @@ -0,0 +1,18 @@ +INSERT INTO [[${table.name}]] +( + [# th:each="column : ${table.columns}"] + [[${column.name}]] + + [# th:if="${columnStat.index < table.columns.size - 1}"],[/] + [/] +) +VALUES +( + [# th:each="column : ${table.columns}"] + '[[${column.getTemplatedName()}]]' + + [# th:if="${columnStat.index < table.columns.size - 1}"],[/] + [/] +); + + diff --git a/source/src/main/resources/templates/default/select.sqlt b/source/src/main/resources/templates/default/select.sqlt new file mode 100644 index 00000000..17be5ed4 --- /dev/null +++ b/source/src/main/resources/templates/default/select.sqlt @@ -0,0 +1,7 @@ +SELECT + [# th:each="column : ${table.columns}"] + [[${column.name}]] + + [# th:if="${columnStat.index < table.columns.size - 1}"],[/] + [/] +FROM [[${table.name}]]; \ No newline at end of file diff --git a/source/src/main/resources/templates/postgres/insert.sqlt b/source/src/main/resources/templates/postgres/insert.sqlt new file mode 100644 index 00000000..db8659fc --- /dev/null +++ b/source/src/main/resources/templates/postgres/insert.sqlt @@ -0,0 +1,17 @@ +INSERT INTO [[${table.schemaName}]].[[${table.tableName}]] +( + [[#th:each(column : ${table.columns})]] + [[${column.columnName}]], + [[/th:each]] +) +VALUES +( + [[#th:each(column : ${table.columns})]] + [[#if(column.columnType == 'String')]] + '[[${column.columnValue}]]' + [[#else]] + [[${column.columnValue}]] + [[/if]] + [[#if(column != ${table.columns.last()})]],[[/if]] + [[/th:each]] +); \ No newline at end of file From 7bfabc2d1c16eb7c8f6127e424413cfb25233729 Mon Sep 17 00:00:00 2001 From: nbesimi Date: Fri, 9 Jun 2023 15:28:27 +0200 Subject: [PATCH 5/6] added: data transfer select and insert generation through thymeleaf --- test/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/test/build.gradle b/test/build.gradle index cd6a1f54..f41ac120 100644 --- a/test/build.gradle +++ b/test/build.gradle @@ -13,6 +13,7 @@ dependencies { implementation project(':common') implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.30' implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.7' + implementation project(path: ':ddl') compileOnly 'org.projectlombok:lombok:1.18.12' annotationProcessor 'org.projectlombok:lombok:1.18.12' From eb646dea353890eba856dc766a845272c83f406a Mon Sep 17 00:00:00 2001 From: nbesimi Date: Fri, 16 Jun 2023 12:36:50 +0200 Subject: [PATCH 6/6] added: data transfer select and insert generation through thymeleaf --- .../core/data/extract/DefaultDataExtract.java | 4 +++ .../templates/default/experiments.sqlt | 33 ------------------- 2 files changed, 4 insertions(+), 33 deletions(-) create mode 100644 source/src/main/java/com/adataptivescale/rosetta/source/core/data/extract/DefaultDataExtract.java delete mode 100644 source/src/main/resources/templates/default/experiments.sqlt diff --git a/source/src/main/java/com/adataptivescale/rosetta/source/core/data/extract/DefaultDataExtract.java b/source/src/main/java/com/adataptivescale/rosetta/source/core/data/extract/DefaultDataExtract.java new file mode 100644 index 00000000..6d5fad28 --- /dev/null +++ b/source/src/main/java/com/adataptivescale/rosetta/source/core/data/extract/DefaultDataExtract.java @@ -0,0 +1,4 @@ +package com.adataptivescale.rosetta.source.core.data.extract; + +public class DefaultDataExtract { +} diff --git a/source/src/main/resources/templates/default/experiments.sqlt b/source/src/main/resources/templates/default/experiments.sqlt deleted file mode 100644 index ef0de5e8..00000000 --- a/source/src/main/resources/templates/default/experiments.sqlt +++ /dev/null @@ -1,33 +0,0 @@ -Accessing object [[$${name}]], - -[[$${#if(status == 'paid')}]] -Your payment has been received. Thank you! -[[$${#else}]] -Please submit your payment by the due date. -[[$${/if}]] - - -[[$${#for(column : table.columns)}]] - [[$${column}]] -[[$${/for}]] - - --- Generate SQL INSERT statement - -INSERT INTO [[${table.tableName}]] -( - [[#th:each(column : ${table.columns})]] - [[${column.columnName}]], - [[/th:each]] -) -VALUES -( - [[#th:each(column : ${table.columns})]] - [[#if(column.columnType == 'String')]] - '[[${column.columnValue}]]' - [[#else]] - [[${column.columnValue}]] - [[/if]] - [[#if(column.name != ${table.columns.last().name})]],[[/if]] - [[/th:each]] -); \ No newline at end of file