[WIP]: Feature/data transfer

cli/src/main/java/com/adaptivescale/rosetta/cli/Cli.java

@@ -79,7 +79,8 @@ public Void call() {
 
     @CommandLine.Command(name = "extract", description = "Extract schema chosen from connection config.", mixinStandardHelpOptions = true)
     private void extract(@CommandLine.Option(names = {"-s", "--source"}, required = true) String sourceName,
-                         @CommandLine.Option(names = {"-t", "--convert-to"}) String targetName
+                         @CommandLine.Option(names = {"-t", "--convert-to"}) String targetName,
+                         @CommandLine.Option(names = {"--include-data"}) boolean includeData
     ) throws Exception {
         requireConfig(config);
         Connection source = getSourceConnection(sourceName);
@@ -261,6 +262,42 @@ private void test(@CommandLine.Option(names = {"-s", "--source"}) String sourceN
         }
     }
 
+    @CommandLine.Command(name = "transfer", description = "Transfer data from source to target", mixinStandardHelpOptions = true)
+    private void transfer(@CommandLine.Option(names = {"-s", "--source"}) String sourceName,
+                          @CommandLine.Option(names = {"-t", "--target"}) String targetName) throws Exception {
+        requireConfig(config);
+
+        Optional<Connection> source = config.getConnection(sourceName);
+        if (source.isEmpty()) {
+            throw new RuntimeException("Can not find source with name: " + sourceName + " configured in config.");
+        }
+
+        Optional<Connection> target = config.getConnection(targetName);
+        if (target.isEmpty()) {
+            throw new RuntimeException("Can not find target with name: " + targetName + " configured in config.");
+        }
+        Path sourceWorkspace = Paths.get("./", sourceName);
+        Path targWorkspace = Paths.get("./", targetName);
+
+        if (!Files.isDirectory(sourceWorkspace)) {
+            throw new RuntimeException(String.format("Can not find directory: %s for source name: %s to find" +
+                    " models for translation", sourceWorkspace, sourceName));
+        }
+
+        if (!Files.isDirectory(targWorkspace)) {
+            throw new RuntimeException(String.format("Can not find directory: %s for source name: %s to find" +
+                    " models for translation", sourceWorkspace, sourceName));
+        }
+
+        List<Database> collect = getDatabases(targWorkspace)
+                .map(AbstractMap.SimpleImmutableEntry::getValue)
+                .collect(Collectors.toList());
+        for (Database database : collect) {
+            DefaultSqlExecution defaultSqlExecution = new DefaultSqlExecution(source.get(), new DriverManagerDriverProvider(), database, target.get());
+            defaultSqlExecution.transfer();
+        }
+    }
+
     @CommandLine.Command(name = "init", description = "Creates a sample config (main.conf) and model directory.", mixinStandardHelpOptions = true)
     private void init(@CommandLine.Parameters(index = "0", description = "Project name.", defaultValue = "")
                               String projectName) throws IOException {

common/src/main/java/com/adaptivescale/rosetta/common/models/Column.java

@@ -1,6 +1,7 @@
 package com.adaptivescale.rosetta.common.models;
 
 import com.adaptivescale.rosetta.common.models.test.Tests;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 
 import java.util.List;
 
@@ -33,6 +34,11 @@ public void setName(String name) {
         this.name = name;
     }
 
+    @JsonIgnore
+    public String getTemplatedName() {
+        return String.format("[(${%s})]", this.name);
+    }
+
     public String getLabel() {
         return label;
     }
@@ -137,4 +143,5 @@ public void setTests(Tests tests) {
         this.tests = tests;
     }
 
+
 }

common/src/main/java/com/adaptivescale/rosetta/common/models/Table.java

@@ -3,6 +3,8 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
 
 public class Table {
 
@@ -17,6 +19,10 @@ public class Table {
 
     private Collection<Column> columns;
 
+    private String extract;
+
+    private String load;
+
     public String getName() {
         return name;
     }
@@ -74,6 +80,48 @@ public void setIndices(List<Index> indices) {
         this.indices = indices;
     }
 
+    public String getExtract() {
+        return extract;
+    }
+
+    public void setExtract(String extract) {
+        this.extract = extract;
+    }
+
+    public String getLoad() {
+        return load;
+    }
+
+    public void setLoad(String load) {
+        this.load = load;
+    }
+
+    public void generateExtractSql() {
+        final StringBuilder extractSql = new StringBuilder();
+        List<String> columnNames = this.columns
+            .stream()
+            .map(Column::getName)
+            .collect(Collectors.toList());
+        extractSql.append("SELECT " + String.join(", ", columnNames) + " FROM " + this.name + ";");
+        this.extract = extractSql.toString();
+    }
+
+    public void generateLoadSql() {
+        final StringBuilder loadSql = new StringBuilder();
+        List<String> columnNames = this.columns
+            .stream()
+            .map(Column::getName)
+            .collect(Collectors.toList());
+        loadSql.append("INSERT INTO " + this.name + " (" + String.join(", ", columnNames) + ")");
+
+        List<String> columnNameTemplates = this.columns
+            .stream().map(Column::getName)
+            .map(it -> String.format("'[(${%s})]'", it))
+            .collect(Collectors.toList());
+        loadSql.append(" VALUES (" + String.join(", ", columnNameTemplates) + ")");
+        this.load = loadSql.toString();
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

ddl/src/main/java/com/adaptivescale/rosetta/ddl/utils/TemplateEngine.java

@@ -19,12 +19,14 @@
 import org.thymeleaf.context.Context;
 import org.thymeleaf.templatemode.TemplateMode;
 import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver;
+import org.thymeleaf.templateresolver.StringTemplateResolver;
 
 import java.util.Map;
 
 public class TemplateEngine {
     private static TemplateEngine instance = null;
     private org.thymeleaf.TemplateEngine engine = null;
+
     private TemplateEngine() {
         ClassLoaderTemplateResolver resolver = new ClassLoaderTemplateResolver();
         resolver.setTemplateMode(TemplateMode.TEXT);
@@ -42,6 +44,17 @@ public static TemplateEngine get() {
         return TemplateEngine.instance;
     }
 
+    public static String processString(String templateName, Map<String, Object> variables) {
+        org.thymeleaf.TemplateEngine templateEngine = new org.thymeleaf.TemplateEngine();
+        StringTemplateResolver templateResolver = new   StringTemplateResolver();
+        templateResolver.setTemplateMode(TemplateMode.HTML);
+        templateEngine.setTemplateResolver(templateResolver);
+
+        Context context = new Context();
+        context.setVariables(variables);
+        return templateEngine.process(templateName, context);
+    }
+
     public static String process(String templateName, Map<String, Object> variables) {
         Context context = new Context();
         context.setVariables(variables);

source/build.gradle

@@ -16,7 +16,7 @@ dependencies {
     compileOnly 'org.projectlombok:lombok:1.18.12'
     annotationProcessor 'org.projectlombok:lombok:1.18.12'
     implementation 'org.apache.commons:commons-lang3:3.12.0'
-
+    implementation 'org.thymeleaf:thymeleaf:3.1.0.RELEASE'
 
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.0'
     testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.0'

source/src/main/java/com/adataptivescale/rosetta/source/common/TemplateEngine.java

@@ -0,0 +1,63 @@
+/*
+ *  Copyright 2022 AdaptiveScale
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *            http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+package com.adataptivescale.rosetta.source.common;
+
+import org.thymeleaf.context.Context;
+import org.thymeleaf.templatemode.TemplateMode;
+import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver;
+import org.thymeleaf.templateresolver.StringTemplateResolver;
+
+import java.util.Map;
+
+public class TemplateEngine {
+    private static TemplateEngine instance = null;
+    private org.thymeleaf.TemplateEngine engine = null;
+
+    private TemplateEngine() {
+        ClassLoaderTemplateResolver resolver = new ClassLoaderTemplateResolver();
+        resolver.setTemplateMode(TemplateMode.TEXT);
+        resolver.setCharacterEncoding("UTF-8");
+        resolver.setPrefix("/templates/");
+        resolver.setSuffix(".sqlt");
+        engine = new org.thymeleaf.TemplateEngine();
+        engine.setTemplateResolver(resolver);
+    }
+
+    public static TemplateEngine get() {
+        if(TemplateEngine.instance == null) {
+            TemplateEngine.instance = new TemplateEngine();
+        }
+        return TemplateEngine.instance;
+    }
+
+    public static String processString(String templateName, Map<String, Object> variables) {
+        org.thymeleaf.TemplateEngine templateEngine = new org.thymeleaf.TemplateEngine();
+        StringTemplateResolver templateResolver = new   StringTemplateResolver();
+        templateResolver.setTemplateMode(TemplateMode.HTML);
+        templateEngine.setTemplateResolver(templateResolver);
+
+        Context context = new Context();
+        context.setVariables(variables);
+        return templateEngine.process(templateName, context);
+    }
+
+    public static String process(String templateName, Map<String, Object> variables) {
+        Context context = new Context();
+        context.setVariables(variables);
+        return get().engine.process(templateName, context);
+    }
+}

source/src/main/java/com/adataptivescale/rosetta/source/core/DefaultGenerator.java

@@ -6,6 +6,7 @@
 import com.adaptivescale.rosetta.common.models.Table;
 import com.adaptivescale.rosetta.common.models.View;
 import com.adaptivescale.rosetta.common.models.input.Connection;
+import com.adataptivescale.rosetta.source.common.TemplateEngine;
 import com.adataptivescale.rosetta.source.core.interfaces.ColumnExtractor;
 import com.adataptivescale.rosetta.source.core.interfaces.Generator;
 import com.adataptivescale.rosetta.source.core.interfaces.TableExtractor;
@@ -45,7 +46,27 @@ public Database generate(Connection connection) throws Exception {
         database.setTables(tables);
         database.setViews(views);
         database.setDatabaseType(connection.getDbType());
+        includeData(tables);
         connect.close();
         return database;
     }
+
+    private void includeData(Collection<Table> tables) {
+        for (Table table : tables) {
+            StringBuilder extractStringBuilder = new StringBuilder();
+            StringBuilder loadStringBuilder = new StringBuilder();
+            Map<String, Object> createParams = new HashMap<>();
+
+            createParams.put("table", table);
+            extractStringBuilder.append(TemplateEngine.process("bigquery/select", createParams));
+            loadStringBuilder.append(TemplateEngine.process("bigquery/insert", createParams));
+
+            table.setExtract(
+                extractStringBuilder.toString().replaceAll("(\\r|\\n|\\t)", " ").replaceAll(" +", " ")
+            );
+            table.setLoad(
+                loadStringBuilder.toString().replaceAll("(\\r|\\n|\\t)", " ").replaceAll(" +", " ")
+            );
+        }
+    }
 }

source/src/main/java/com/adataptivescale/rosetta/source/core/data/extract/DefaultDataExtract.java

@@ -0,0 +1,4 @@
+package com.adataptivescale.rosetta.source.core.data.extract;
+
+public class DefaultDataExtract {
+}

source/src/main/resources/templates/bigquery/insert.sqlt

@@ -0,0 +1,21 @@
+INSERT INTO [[${table.schema}]].[[${table.name}]]
+(
+    [# th:each="column : ${table.columns}"]
+        [[${column.name}]]
+
+        [# th:if="${columnStat.index < table.columns.size - 1}"],[/]
+    [/]
+)
+VALUES
+(
+    [# th:each="column : ${table.columns}"]
+        [# th:switch="${column.typeName}"]
+            [# th:case="'INT64'"][[${column.getTemplatedName()}]][/]
+            [# th:case="*"]'[[${column.getTemplatedName()}]]'[/]
+
+            [# th:if="${columnStat.index < table.columns.size - 1}"],[/]
+        [/]
+    [/]
+);
+
+

source/src/main/resources/templates/bigquery/select.sqlt

@@ -0,0 +1,7 @@
+SELECT
+    [# th:each="column : ${table.columns}"]
+       [[${column.name}]]
+
+       [# th:if="${columnStat.index < table.columns.size - 1}"],[/]
+    [/]
+FROM [[${table.schema}]].[[${table.name}]];

source/src/main/resources/templates/default/insert.sqlt

@@ -0,0 +1,18 @@
+INSERT INTO [[${table.name}]]
+(
+    [# th:each="column : ${table.columns}"]
+        [[${column.name}]]
+
+        [# th:if="${columnStat.index < table.columns.size - 1}"],[/]
+    [/]
+)
+VALUES
+(
+    [# th:each="column : ${table.columns}"]
+        '[[${column.getTemplatedName()}]]'
+
+        [# th:if="${columnStat.index < table.columns.size - 1}"],[/]
+    [/]
+);
+
+

source/src/main/resources/templates/default/select.sqlt

@@ -0,0 +1,7 @@
+SELECT
+    [# th:each="column : ${table.columns}"]
+       [[${column.name}]]
+
+       [# th:if="${columnStat.index < table.columns.size - 1}"],[/]
+    [/]
+FROM [[${table.name}]];

source/src/main/resources/templates/postgres/insert.sqlt

@@ -0,0 +1,17 @@
+INSERT INTO [[${table.schemaName}]].[[${table.tableName}]]
+(
+    [[#th:each(column : ${table.columns})]]
+    [[${column.columnName}]],
+    [[/th:each]]
+)
+VALUES
+(
+    [[#th:each(column : ${table.columns})]]
+    [[#if(column.columnType == 'String')]]
+    '[[${column.columnValue}]]'
+    [[#else]]
+    [[${column.columnValue}]]
+    [[/if]]
+    [[#if(column != ${table.columns.last()})]],[[/if]]
+    [[/th:each]]
+);

test/build.gradle

@@ -13,6 +13,7 @@ dependencies {
     implementation project(':common')
     implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.30'
     implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.7'
+    implementation project(path: ':ddl')
     compileOnly 'org.projectlombok:lombok:1.18.12'
     annotationProcessor 'org.projectlombok:lombok:1.18.12'