diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json index 943e2d24e..99102177e 100644 --- a/amazon-redshift-plugin/widgets/Redshift-batchsource.json +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ @@ -228,7 +259,7 @@ "name": "connection" } ] - }, + } ], "jump-config": { "datasets": [ diff --git a/amazon-redshift-plugin/widgets/Redshift-connector.json b/amazon-redshift-plugin/widgets/Redshift-connector.json index 3a2af8e01..5b452e54f 100644 --- a/amazon-redshift-plugin/widgets/Redshift-connector.json +++ b/amazon-redshift-plugin/widgets/Redshift-connector.json @@ -69,6 +69,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-action.json b/aurora-mysql-plugin/widgets/AuroraMysql-action.json index efc5f98ff..050ee9967 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-action.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-action.json @@ -90,6 +90,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json b/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json index a435e4e4f..cd48b44a8 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json @@ -116,6 +116,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json b/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json index 50b435645..5956c7f39 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json @@ -135,6 +135,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json b/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json index cc33cf0a1..eac67041e 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json @@ -105,6 +105,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json index 1f3bca862..7057596f8 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json @@ -79,6 +79,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json index 53979d6d4..8b412bf9f 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json @@ -121,6 +121,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json index 14b00b974..1a614b829 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json @@ -124,6 +124,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json index 3fdb1a14b..4ea223eb3 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json index 66d6ebb85..6be894957 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json @@ -112,6 +112,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json index 89a7d7736..1c0a88d2c 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json @@ -176,6 +176,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json index 4ac7747f4..18949ec52 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json @@ -175,6 +175,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json index b5c2c9993..e5dee9e73 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json index eab240679..658ba414e 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json @@ -112,6 +112,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json index 2fda594dd..a685b6610 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json @@ -192,6 +192,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json index 96ea97ac2..1584199eb 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json @@ -175,6 +175,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json index 9824f91bd..c1058b96e 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 67dc8e82e..b1566c75f 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -41,6 +41,13 @@ guava + + + dev.failsafe + failsafe + 3.3.2 + + io.cdap.cdap @@ -64,6 +71,12 @@ org.mockito mockito-core + + org.junit.jupiter + junit-jupiter + RELEASE + test + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/CommonSchemaReader.java b/database-commons/src/main/java/io/cdap/plugin/db/CommonSchemaReader.java index 28c56db8c..f4cfe8322 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/CommonSchemaReader.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/CommonSchemaReader.java @@ -19,7 +19,6 @@ import com.google.common.collect.Lists; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.common.db.DBUtils; - import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -53,8 +52,8 @@ public List getSchemaFields(ResultSet resultSet) throws SQLExcepti @Override public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLException { return DBUtils.getSchema(metadata.getColumnTypeName(index), metadata.getColumnType(index), - metadata.getPrecision(index), metadata.getScale(index), metadata.getColumnName(index), - metadata.isSigned(index), true); + metadata.getPrecision(index), metadata.getScale(index), metadata.getColumnName(index), + metadata.isSigned(index), true); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 588ed78b8..d17adadde 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -45,6 +45,12 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; + private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 5; + public static final int DEFAULT_MAX_RETRY_COUNT = 80; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 5; @Name(JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + @@ -71,6 +77,41 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC @Macro public String connectionArguments; + @Name(NAME_INITIAL_RETRY_DURATION) + @Description("Time taken for the first retry. Default is 5 seconds.") + @Nullable + @Macro + private Integer initialRetryDuration; + + @Name(NAME_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default is 80 seconds.") + @Nullable + @Macro + private Integer maxRetryDuration; + + @Name(NAME_MAX_RETRY_COUNT) + @Description("Maximum number of retries allowed. Default is 5.") + @Nullable + @Macro + private Integer maxRetryCount; + + + @Nullable + public Integer getInitialRetryDuration() { + return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + } + + @Nullable + public Integer getMaxRetryDuration() { + return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + } + + @Nullable + public Integer getMaxRetryCount() { + return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + } + + public ConnectionConfig() { } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfigAccessor.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfigAccessor.java index 253422e79..49cf0ce10 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfigAccessor.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfigAccessor.java @@ -39,6 +39,8 @@ public class ConnectionConfigAccessor { public static final String OPERATION_NAME = "io.cdap.plugin.db.operation.name"; public static final String RELATION_TABLE_KEY = "io.cdap.plugin.db.relation.table.key"; + + private static final Gson GSON = new Gson(); private static final Type STRING_MAP_TYPE = new TypeToken>() { }.getType(); private static final Type STRING_LIST_TYPE = new TypeToken>() { }.getType(); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/DBConfig.java index d9016bf04..35640a041 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBConfig.java @@ -20,7 +20,6 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.plugin.common.Constants; - import javax.annotation.Nullable; /** diff --git a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java index a5a9fcf5f..4e15e215a 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/DBRecord.java @@ -29,7 +29,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.lib.db.DBWritable; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -154,7 +153,6 @@ public void readFields(ResultSet resultSet) throws SQLException { int sqlType = metadata.getColumnType(columnIndex); int sqlPrecision = metadata.getPrecision(columnIndex); int sqlScale = metadata.getScale(columnIndex); - handleField(resultSet, recordBuilder, field, columnIndex, sqlType, sqlPrecision, sqlScale); } record = recordBuilder.build(); @@ -213,6 +211,7 @@ protected void setField(ResultSet resultSet, StructuredRecord.Builder recordBuil protected void setFieldAccordingToSchema(ResultSet resultSet, StructuredRecord.Builder recordBuilder, Schema.Field field, int columnIndex) throws SQLException { + Schema.Type fieldType = field.getSchema().isNullable() ? field.getSchema().getNonNullable().getType() : field.getSchema().getType(); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java b/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java new file mode 100644 index 000000000..bd5c1faf5 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package io.cdap.plugin.db; + +import java.sql.SQLTimeoutException; +import java.sql.SQLTransactionRollbackException; +import java.sql.SQLTransientException; + + +/** + * Checks whether the given exception or one of its causes is a known retryable SQLException. + */ +public class RetryExceptions { + public static boolean isRetryable(Throwable t) { + while (t != null) { + System.out.println("Checking retryable for: " + t.getClass()); + if (t instanceof SQLTransientException || + t instanceof SQLTransactionRollbackException || + t instanceof SQLTimeoutException) { + return true; + } + t = t.getCause(); + } + return false; + } +} + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java index a2be9cbf0..bef93a0de 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java @@ -16,12 +16,18 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; import io.cdap.cdap.etl.api.action.ActionContext; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Driver; +import java.sql.SQLException; /** * Action that runs a db command. @@ -30,7 +36,6 @@ public abstract class AbstractDBAction extends Action { private static final String JDBC_PLUGIN_ID = "driver"; private final QueryConfig config; private final Boolean enableAutoCommit; - public AbstractDBAction(QueryConfig config, Boolean enableAutoCommit) { this.config = config; this.enableAutoCommit = enableAutoCommit; @@ -40,7 +45,19 @@ public AbstractDBAction(QueryConfig config, Boolean enableAutoCommit) { public void run(ActionContext context) throws Exception { Class driverClass = context.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); - executeQuery.run(); + try { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(()-> executeQuery.run()); + } catch (Exception e) { + if (e instanceof SQLException) { + DBErrorDetailsProvider dbe = new DBErrorDetailsProvider(); + throw dbe.getProgramFailureException((SQLException) e, null); + } + FailureCollector collector = context.getFailureCollector(); + collector.addFailure("Failed to execute query with message: " + e.getMessage(), null) + .withStacktrace(e.getStackTrace()); + collector.getOrThrowException(); + } } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index 5e22abf85..eaa95d039 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -16,6 +16,7 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.StageConfigurer; @@ -25,6 +26,7 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Connection; import java.sql.Driver; @@ -56,10 +58,10 @@ public void run(ActionContext context) throws Exception { @Override public void configurePipeline(PipelineConfigurer pipelineConfigurer) - throws IllegalArgumentException { + throws IllegalArgumentException { DBUtils.validateJDBCPluginPipeline(pipelineConfigurer, config, JDBC_PLUGIN_ID); Class driverClass = DBUtils.getDriverClass( - pipelineConfigurer, config, ConnectionConfig.JDBC_PLUGIN_TYPE); + pipelineConfigurer, config, ConnectionConfig.JDBC_PLUGIN_TYPE); StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); FailureCollector collector = stageConfigurer.getFailureCollector(); config.validate(collector); @@ -70,10 +72,10 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) processArguments(driverClass, collector, null); } catch (SQLException e) { collector.addFailure("SQL error while executing query: " + e.getMessage(), null) - .withStacktrace(e.getStackTrace()); + .withStacktrace(e.getStackTrace()); } catch (IllegalAccessException | InstantiationException e) { collector.addFailure("Unable to instantiate JDBC driver: " + e.getMessage(), null) - .withStacktrace(e.getStackTrace()); + .withStacktrace(e.getStackTrace()); } catch (Exception e) { collector.addFailure(e.getMessage(), null).withStacktrace(e.getStackTrace()); } @@ -92,50 +94,53 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) */ private void processArguments(Class driverClass, FailureCollector failureCollector, SettableArguments settableArguments) - throws SQLException, IllegalAccessException, InstantiationException { - DriverCleanup driverCleanup; - - driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), - config.getJdbcPluginName()); - Properties connectionProperties = new Properties(); - connectionProperties.putAll(config.getConnectionArguments()); - try { - Connection connection = DriverManager - .getConnection(config.getConnectionString(), connectionProperties); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(config.getQuery()); - boolean hasRecord = resultSet.next(); - if (!hasRecord) { - failureCollector.addFailure("No record found.", - "The argument selection conditions must match only one record."); - return; - } - if (settableArguments != null) { - setArguments(resultSet, failureCollector, settableArguments); - } - if (resultSet.next()) { - failureCollector - .addFailure("More than one records found.", - "The argument selection conditions must match only one record."); + throws SQLException, IllegalAccessException, InstantiationException { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + DriverCleanup driverCleanup; + driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), + config.getJdbcPluginName()); + Properties connectionProperties = new Properties(); + connectionProperties.putAll(config.getConnectionArguments()); + try { + Connection connection = DriverManager + .getConnection(config.getConnectionString(), connectionProperties); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(config.getQuery()); + boolean hasRecord = resultSet.next(); + if (!hasRecord) { + failureCollector.addFailure("No record found.", + "The argument selection conditions must match only one record."); + return; + } + if (settableArguments != null) { + setArguments(resultSet, settableArguments); + } + if (resultSet.next()) { + failureCollector + .addFailure("More than one records found.", + "The argument selection conditions must match only one record."); + } + } finally { + driverCleanup.destroy(); } - } finally { - driverCleanup.destroy(); - } + }); } /** * Converts column from jdbc results set into pipeline arguments * * @param resultSet - result set from db {@link ResultSet} - * @param failureCollector - context failure collector @{link FailureCollector} * @param arguments - context argument setter {@link SettableArguments} * @throws SQLException - raises {@link SQLException} when configuration is not valid */ - private void setArguments(ResultSet resultSet, FailureCollector failureCollector, - SettableArguments arguments) throws SQLException { - String[] columns = config.getArgumentsColumns().split(","); - for (String column : columns) { - arguments.set(column, resultSet.getString(column)); - } + private void setArguments(ResultSet resultSet, SettableArguments arguments) { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + String[] columns = config.getArgumentsColumns().split(","); + for (String column : columns) { + arguments.set(column, resultSet.getString(column)); + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java index e4b91adbd..5b57e28ca 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java @@ -16,11 +16,13 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchActionContext; import io.cdap.cdap.etl.api.batch.PostAction; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Driver; @@ -51,7 +53,8 @@ public void run(BatchActionContext batchContext) throws Exception { Class driverClass = batchContext.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); - executeQuery.run(); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run (executeQuery::run); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java index e2ccfc57e..1d99974e0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java @@ -16,8 +16,10 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Connection; import java.sql.Driver; @@ -48,6 +50,7 @@ public DBRun(QueryConfig config, Class driverClass, Boolean en * to use and which connection string to use come from the plugin configuration. */ public void run() throws SQLException, InstantiationException, IllegalAccessException { + DriverCleanup driverCleanup = null; try { driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), @@ -55,18 +58,21 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(config.getConnectionString(), connectionProperties)) { - executeInitQueries(connection, config.getInitQueries()); - if (!enableAutoCommit) { - connection.setAutoCommit(false); - } - try (Statement statement = connection.createStatement()) { - statement.execute(config.query); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(config.getConnectionString(), connectionProperties)) { + executeInitQueries(connection, config.getInitQueries()); if (!enableAutoCommit) { - connection.commit(); + connection.setAutoCommit(false); + } + try (Statement statement = connection.createStatement()) { + statement.execute(config.query); + if (!enableAutoCommit) { + connection.commit(); + } } } - } + }); } finally { if (driverCleanup != null) { driverCleanup.destroy(); @@ -74,11 +80,16 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce } } - private void executeInitQueries(Connection connection, List initQueries) throws SQLException { - for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } - } + private void executeInitQueries(Connection connection, List initQueries) { + + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())) + .run(() -> { + for (String query : initQueries) { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java index 5b92a85f7..3d9ed16ff 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java @@ -155,4 +155,16 @@ public Operation getOperationName() { public String getRelationTableKey() { return relationTableKey; } + + public Integer getInitialRetryDuration() { + return getConnection().getInitialRetryDuration(); + } + + public Integer getMaxRetryDuration() { + return getConnection().getMaxRetryDuration(); + } + + public Integer getMaxRetryCount() { + return getConnection().getMaxRetryCount(); + } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java index 41c577397..28c3b05db 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java @@ -28,7 +28,6 @@ import io.cdap.plugin.db.TransactionIsolationLevel; import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; import io.cdap.plugin.db.source.AbstractDBSource; - import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -268,4 +267,16 @@ public Integer getFetchSize() { return fetchSize; } + public Integer getInitialRetryDuration() { + return getConnection().getInitialRetryDuration(); + } + + public Integer getMaxRetryDuration() { + return getConnection().getMaxRetryDuration(); + } + + public Integer getMaxRetryCount() { + return getConnection().getMaxRetryCount(); + } + } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java index 55cfe363f..e1fde69a1 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java @@ -50,4 +50,10 @@ public interface DatabaseConnectionConfig { */ String getPassword(); + Integer getInitialRetryDuration(); + + Integer getMaxRetryDuration(); + + Integer getMaxRetryCount(); + } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java index 4bee056f8..7ddd65bc8 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java @@ -36,6 +36,12 @@ * */ public abstract class AbstractDBConnectorConfig extends PluginConfig implements DBConnectorProperties { + private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 2; + public static final int DEFAULT_MAX_RETRY_COUNT = 3; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 10; @Name(ConnectionConfig.JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + @@ -63,6 +69,26 @@ public abstract class AbstractDBConnectorConfig extends PluginConfig implements @Macro protected String connectionArguments; + + @Name(NAME_INITIAL_RETRY_DURATION) + @Description("Time taken for the first retry. Default is 2 seconds.") + @Nullable + @Macro + private Integer initialRetryDuration; + + @Name(NAME_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default is 300 seconds.") + @Nullable + @Macro + private Integer maxRetryDuration; + + @Name(NAME_MAX_RETRY_COUNT) + @Description("Maximum number of retries allowed. Default is 3.") + @Nullable + @Macro + private Integer maxRetryCount; + + @Nullable @Override public String getUser() { @@ -75,9 +101,25 @@ public String getPassword() { return password; } + @Nullable + public Integer getInitialRetryDuration() { + return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + } + + @Nullable + public Integer getMaxRetryDuration() { + return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + } + + @Nullable + public Integer getMaxRetryCount() { + return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + } + @Override public Properties getConnectionArgumentsProperties() { - return getConnectionArgumentsProperties(connectionArguments, user, password); + return getConnectionArgumentsProperties(connectionArguments, + user, password); } @Override @@ -90,8 +132,9 @@ public String getConnectionArguments() { return connectionArguments; } - protected static Properties getConnectionArgumentsProperties(@Nullable String connectionArguments, - @Nullable String user, @Nullable String password) { + protected static Properties getConnectionArgumentsProperties (@Nullable String connectionArguments, + @Nullable String user, + @Nullable String password) { KeyValueListParser kvParser = new KeyValueListParser("\\s*;\\s*", "="); Map connectionArgumentsMap = new HashMap<>(); @@ -107,6 +150,7 @@ protected static Properties getConnectionArgumentsProperties(@Nullable String co if (password != null) { connectionArgumentsMap.put("password", password); } + Properties properties = new Properties(); properties.putAll(connectionArgumentsMap); return properties; diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index 8a9b7b6e4..43ba74bcf 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -17,6 +17,7 @@ package io.cdap.plugin.db.connector; import com.google.common.collect.Maps; +import dev.failsafe.Failsafe; import io.cdap.cdap.api.data.batch.InputFormatProvider; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.batch.BatchConnector; @@ -33,11 +34,11 @@ import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.source.DataDrivenETLDBInputFormat; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; import org.apache.hadoop.mapreduce.lib.db.DBWritable; - import java.io.IOException; import java.sql.Connection; import java.sql.ResultSet; @@ -55,6 +56,7 @@ public abstract class AbstractDBSpecificConnector extends AbstractDBConnector implements BatchConnector { + private final AbstractDBConnectorConfig config; protected AbstractDBSpecificConnector(AbstractDBConnectorConfig config) { @@ -172,13 +174,16 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, protected Schema loadTableSchema(Connection connection, String query, @Nullable Integer timeoutSec, String sessionID) throws SQLException { - Statement statement = connection.createStatement(); - statement.setMaxRows(1); - if (timeoutSec != null) { - statement.setQueryTimeout(timeoutSec); - } - ResultSet resultSet = statement.executeQuery(query); - return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).get(() -> { + Statement statement = connection.createStatement(); + statement.setMaxRows(1); + if (timeoutSec != null) { + statement.setQueryTimeout(timeoutSec); + } + ResultSet resultSet = statement.executeQuery(query); + return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + }); } protected void setConnectionProperties(Map properties, ConnectorSpecRequest request) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java index 5c6b08031..5577b3537 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnectorConfig.java @@ -42,6 +42,8 @@ public abstract class AbstractDBSpecificConnectorConfig extends AbstractDBConnec @Nullable protected Integer port; + + public String getHost() { return host; } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 797abfc23..3fccf7b46 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -18,6 +18,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -53,6 +55,7 @@ import io.cdap.plugin.db.config.DatabaseSinkConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.lib.db.DBConfiguration; @@ -301,6 +304,8 @@ private Schema inferSchema(Class driverClass) { dbSinkConfig.getJdbcPluginName()); Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { try (Connection connection = DriverManager.getConnection(dbSinkConfig.getConnectionString(), connectionProperties)) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); @@ -329,6 +334,7 @@ private Schema inferSchema(Class driverClass) { e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); } + }); } catch (IllegalAccessException | InstantiationException | SQLException e) { throw new InvalidStageException("JDBC Driver unavailable: " + dbSinkConfig.getJdbcPluginName(), e); } @@ -368,18 +374,20 @@ private void setResultSetMetadata() throws Exception { Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (Statement statement = connection.createStatement(); - // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata - // that can be used to construct DBRecord objects to sink to the database table. - ResultSet rs = statement.executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", - dbColumns, fullyQualifiedTableName)) - ) { - columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (Statement statement = connection.createStatement(); + // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata + // that can be used to construct DBRecord objects to sink to the database table. + ResultSet rs = statement.executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", + dbColumns, fullyQualifiedTableName)) + ) { + columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); + } } - } - + }); this.columnTypes = Collections.unmodifiableList(columnTypes); } @@ -437,26 +445,31 @@ private void validateSchema(FailureCollector collector, Class Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { - if (!tables.next()) { - collector.addFailure( - String.format("Table '%s' does not exist.", tableName), - String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + - "points to a valid database.", fullyQualifiedTableName, connectionString)) - .withConfigProperty(DBSinkConfig.TABLE_NAME); - return; + try { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { + if (!tables.next()) { + collector.addFailure( + String.format("Table '%s' does not exist.", tableName), + String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + + "points to a valid database.", fullyQualifiedTableName, connectionString)) + .withConfigProperty(DBSinkConfig.TABLE_NAME); + return; + } + } + setColumnsInfo(inputSchema.getFields()); + try (PreparedStatement pStmt = connection.prepareStatement(String.format("SELECT %s FROM %s WHERE 1 = 0", + dbColumns, + fullyQualifiedTableName)); + ResultSet rs = pStmt.executeQuery()) { + getFieldsValidator().validateFields(inputSchema, rs, collector); + } } - } - setColumnsInfo(inputSchema.getFields()); - try (PreparedStatement pStmt = connection.prepareStatement(String.format("SELECT %s FROM %s WHERE 1 = 0", - dbColumns, - fullyQualifiedTableName)); - ResultSet rs = pStmt.executeQuery()) { - getFieldsValidator().validateFields(inputSchema, rs, collector); - } - } catch (SQLException e) { + }); + } catch (FailsafeException e) { LOG.error("Exception while trying to validate schema of database table {} for connection {}.", fullyQualifiedTableName, connectionString, e); collector.addFailure( @@ -485,9 +498,12 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 3fd490ada..0a857b332 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -18,6 +18,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import dev.failsafe.Failsafe; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -52,13 +53,13 @@ import io.cdap.plugin.db.config.DatabaseSourceConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.io.IOException; import java.sql.Connection; import java.sql.Driver; @@ -137,7 +138,6 @@ public Schema getSchema(Class driverClass) throws IllegalAcces SQLException, InstantiationException { DriverCleanup driverCleanup; try { - driverCleanup = loadPluginClassAndGetDriver(driverClass); try { return getSchema(); @@ -168,13 +168,15 @@ public Schema getSchema() throws SQLException { } private Schema loadSchemaFromDB(Connection connection, String query) throws SQLException { - Statement statement = connection.createStatement(); - statement.setMaxRows(1); - if (query.contains("$CONDITIONS")) { - query = removeConditionsClause(query); - } - ResultSet resultSet = statement.executeQuery(query); - return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> { + Statement statement = connection.createStatement(); + statement.setMaxRows(1); + String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; + ResultSet resultSet = statement.executeQuery(finalQuery); + return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + }); } @VisibleForTesting @@ -191,41 +193,52 @@ private Schema loadSchemaFromDB(Class driverClass) String connectionString = sourceConfig.getConnectionString(); DriverCleanup driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, connectionString, sourceConfig.getJdbcPluginName()); - Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, sourceConfig.getInitQueries()); - return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); - - } catch (SQLException e) { - // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc driver in classpath - String errorMessage = - String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), - e.getSQLState(), e.getErrorCode()); - String errorMessageWithDetails = String.format("Error occurred while trying to get schema from database." + - "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessage.endsWith(".")) { - errorMessage = errorMessage + "."; - } - errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); - } - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, - e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), - e.getSQLState(), e.getErrorCode())); - } finally { - driverCleanup.destroy(); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, sourceConfig.getInitQueries()); + return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); + } + catch (SQLException e) { + // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc + // driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", + e.getMessage(), + e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error occurred while trying to" + + " get schema from database." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), + e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; + } + errorMessage = String.format("%s For more details, see %s", errorMessage, + externalDocumentationLink); + } + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), + e.getSQLState(), e.getErrorCode())); + } finally { + driverCleanup.destroy(); + } + }); } - } private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())).run(() -> { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + }); } } @@ -266,7 +279,9 @@ private Connection getConnection() throws SQLException { String connectionString = createConnectionString(); Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - return DriverManager.getConnection(connectionString, connectionProperties); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> DriverManager.getConnection(connectionString, connectionProperties)); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java index f704f2ad5..042dc6cb5 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/CloudSQLUtil.java @@ -34,7 +34,6 @@ public class CloudSQLUtil { public static final String CLOUDSQL_POSTGRESQL = "CloudSQL PostgreSQL"; public static final String CLOUDSQL_MYSQL = "CloudSQL MySQL"; - /** * Utility method to check the Connection Name format of a CloudSQL instance. * diff --git a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java index 34043d513..6e3eabf4b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/DBUtils.java @@ -25,7 +25,6 @@ import io.cdap.plugin.db.config.DatabaseConnectionConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.lang.management.ManagementFactory; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -104,14 +103,13 @@ public static void cleanup(Class driverClass) { public static DriverCleanup ensureJDBCDriverIsAvailable(Class jdbcDriverClass, String connectionString, String jdbcPluginName) throws IllegalAccessException, InstantiationException, SQLException { - try { DriverManager.getDriver(connectionString); return new DriverCleanup(null); } catch (SQLException e) { // Driver not found. We will try to register it with the DriverManager. LOG.debug("Plugin Name: {}; Driver Class: {} not found. Registering JDBC driver via shim {} ", - jdbcPluginName, jdbcDriverClass.getName(), JDBCDriverShim.class.getName()); + jdbcPluginName, jdbcDriverClass.getName(), JDBCDriverShim.class.getName()); final JDBCDriverShim driverShim = new JDBCDriverShim(jdbcDriverClass.newInstance()); try { diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java new file mode 100644 index 000000000..39c5f8384 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package io.cdap.plugin.util; + +import dev.failsafe.RetryPolicy; +import io.cdap.cdap.api.Config; +import io.cdap.plugin.db.RetryExceptions; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLTransientConnectionException; +import java.time.Duration; + +/** + * Utility class for creating standardized {@link dev.failsafe.RetryPolicy} configurations + * to handle transient SQL exceptions using the Failsafe library. + */ +public class RetryPolicyUtil extends Config { + public static final Logger LOG = LoggerFactory.getLogger(RetryPolicyUtil.class); + + /** + * Create a RetryPolicy using custom config values. + */ + public static RetryPolicy createConnectionRetryPolicy(Integer initialRetryDuration, + Integer maxRetryDuration, Integer maxRetryCount) { + return RetryPolicy.builder() + .handleIf((failure) -> RetryExceptions.isRetryable(failure)) + .withBackoff(Duration.ofSeconds(initialRetryDuration), Duration.ofSeconds(maxRetryDuration)) + .withMaxRetries(maxRetryCount) + .onRetry(e -> LOG.debug("Retrying... Attempt {}", + e.getAttemptCount())) + .onFailedAttempt(e -> LOG.debug("Failed Attempt : {}", e.getLastException())) + .onFailure(e -> LOG.debug("Failed after retries." + + " Reason: {}", + e.getException() != null ? e.getException().getMessage() : "Unknown error")) + .build(); + } +} diff --git a/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java b/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java new file mode 100644 index 000000000..71e5ab6e7 --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java @@ -0,0 +1,83 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db; + +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; +import io.cdap.plugin.util.RetryPolicyUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.sql.SQLSyntaxErrorException; +import java.sql.SQLTransientConnectionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RetryPolicyUtilTest { + + private AbstractDBConnectorConfig mockConfig; + + @BeforeEach + public void setup() { + mockConfig = mock(AbstractDBConnectorConfig.class); + when(mockConfig.getInitialRetryDuration()).thenReturn(5); + when(mockConfig.getMaxRetryDuration()).thenReturn(10); + when(mockConfig.getMaxRetryCount()).thenReturn(2); + } + + @Test + public void testCreateConnectionRetryPolicy_Retryable() { + RetryPolicy retryPolicy = RetryPolicyUtil + .createConnectionRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), + mockConfig.getMaxRetryCount()); + + AtomicInteger attemptCounter = new AtomicInteger(); + + FailsafeException ex = assertThrows(FailsafeException.class, () -> Failsafe.with(retryPolicy).run(() -> { + attemptCounter.incrementAndGet(); + throw new SQLTransientConnectionException("Temporary issue"); + })); + + assertTrue(ex.getCause() instanceof SQLTransientConnectionException); + assertEquals(3, attemptCounter.get(), "Expected 2 retries + 1 initial attempt"); + } + + @Test + public void testCreateConnectionRetryPolicy_NonRetryable() { + RetryPolicy retryPolicy = RetryPolicyUtil + .createConnectionRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), + mockConfig.getMaxRetryCount()); + + AtomicInteger attemptCounter = new AtomicInteger(); + + FailsafeException ex = assertThrows(FailsafeException.class, () -> + Failsafe.with(retryPolicy).run(() -> { + attemptCounter.incrementAndGet(); + throw new SQLSyntaxErrorException("Bad SQL syntax"); + })); + assertTrue(ex.getCause() instanceof SQLSyntaxErrorException); + assertEquals(1, attemptCounter.get(), "Should not retry for non-retryable exception"); + } +} + diff --git a/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java b/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java index fa9e371da..23abe08a9 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java @@ -216,10 +216,14 @@ public void testValidateFieldsWithNullable() throws Exception { public void validateFieldCompatible(Schema.Type fieldType, Schema.LogicalType fieldLogicalType, int sqlType, boolean isCompatible, int precision, boolean isSigned) { String errorMessage = String.format("Expected type '%s' is %s with sql type '%d'", - fieldType, - isCompatible ? "compatible" : "not compatible", - sqlType); - Assert.assertEquals(errorMessage, isCompatible, VALIDATOR.isFieldCompatible(fieldType, fieldLogicalType, sqlType, - precision, isSigned)); + fieldType, + isCompatible ? "compatible" : "not compatible", + sqlType); + try { + boolean actualCompatible = VALIDATOR.isFieldCompatible(fieldType, fieldLogicalType, sqlType, precision, isSigned); + Assert.assertEquals(errorMessage, isCompatible, actualCompatible); + } catch (Exception e) { + throw new AssertionError("Unexpected exception during compatibility check: " + e.getMessage(), e); + } } } diff --git a/db2-plugin/widgets/Db2-action.json b/db2-plugin/widgets/Db2-action.json index 3e9159c0d..4d6ba9304 100644 --- a/db2-plugin/widgets/Db2-action.json +++ b/db2-plugin/widgets/Db2-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/db2-plugin/widgets/Db2-batchsink.json b/db2-plugin/widgets/Db2-batchsink.json index 5345f03d5..95d9db933 100644 --- a/db2-plugin/widgets/Db2-batchsink.json +++ b/db2-plugin/widgets/Db2-batchsink.json @@ -100,6 +100,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/db2-plugin/widgets/Db2-batchsource.json b/db2-plugin/widgets/Db2-batchsource.json index 1c221606d..9f9a9b4aa 100644 --- a/db2-plugin/widgets/Db2-batchsource.json +++ b/db2-plugin/widgets/Db2-batchsource.json @@ -119,6 +119,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/db2-plugin/widgets/Db2-postaction.json b/db2-plugin/widgets/Db2-postaction.json index cd75dec04..4a12b684b 100644 --- a/db2-plugin/widgets/Db2-postaction.json +++ b/db2-plugin/widgets/Db2-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-database-plugin/widgets/Database-action.json b/generic-database-plugin/widgets/Database-action.json index c849c4cba..d754b5998 100644 --- a/generic-database-plugin/widgets/Database-action.json +++ b/generic-database-plugin/widgets/Database-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-database-plugin/widgets/Database-batchsink.json b/generic-database-plugin/widgets/Database-batchsink.json index 90332c5c5..518073b28 100644 --- a/generic-database-plugin/widgets/Database-batchsink.json +++ b/generic-database-plugin/widgets/Database-batchsink.json @@ -115,6 +115,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/generic-database-plugin/widgets/Database-batchsource.json b/generic-database-plugin/widgets/Database-batchsource.json index 579b87bd9..5adcbe985 100644 --- a/generic-database-plugin/widgets/Database-batchsource.json +++ b/generic-database-plugin/widgets/Database-batchsource.json @@ -134,6 +134,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/generic-database-plugin/widgets/Database-postaction.json b/generic-database-plugin/widgets/Database-postaction.json index abf02ef22..58c2cef43 100644 --- a/generic-database-plugin/widgets/Database-postaction.json +++ b/generic-database-plugin/widgets/Database-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json b/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json index 423792391..6da1e383c 100644 --- a/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json +++ b/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json @@ -93,6 +93,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mariadb-plugin/widgets/Mariadb-action.json b/mariadb-plugin/widgets/Mariadb-action.json index bb78abb27..3b454174b 100644 --- a/mariadb-plugin/widgets/Mariadb-action.json +++ b/mariadb-plugin/widgets/Mariadb-action.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/memsql-plugin/widgets/Memsql-action.json b/memsql-plugin/widgets/Memsql-action.json index 61cbb1e47..b37e06695 100644 --- a/memsql-plugin/widgets/Memsql-action.json +++ b/memsql-plugin/widgets/Memsql-action.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/memsql-plugin/widgets/Memsql-batchsink.json b/memsql-plugin/widgets/Memsql-batchsink.json index 98c2c1f8e..1f27efb9d 100644 --- a/memsql-plugin/widgets/Memsql-batchsink.json +++ b/memsql-plugin/widgets/Memsql-batchsink.json @@ -170,6 +170,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/memsql-plugin/widgets/Memsql-batchsource.json b/memsql-plugin/widgets/Memsql-batchsource.json index ef5d17b39..090c042ac 100644 --- a/memsql-plugin/widgets/Memsql-batchsource.json +++ b/memsql-plugin/widgets/Memsql-batchsource.json @@ -205,6 +205,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/memsql-plugin/widgets/Memsql-postaction.json b/memsql-plugin/widgets/Memsql-postaction.json index 72e67abd9..0b008d019 100644 --- a/memsql-plugin/widgets/Memsql-postaction.json +++ b/memsql-plugin/widgets/Memsql-postaction.json @@ -172,6 +172,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature index 1385cc8c1..a3b751371 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature @@ -171,3 +171,93 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Enter input plugin property: "referenceName" with value: "targetRef" And Click on the Validate button Then Verify that the Plugin is displaying an error message: "errormessageBlankHost" on the header + + @Mssql_Required + Scenario: Verify required fields missing validation messages + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "SQL Server" + Then Click on the Validate button + Then Verify mandatory property error for below listed properties: + | jdbcPluginName | + | referenceName | + | database | + | tableName | + + @Mssql_Required + Scenario: Verify the validation error message with missing jdbc plugin name + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.jdbcPluginName.message" on the header + + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET @Mssql_Required + Scenario: Verify the validation error message with blank password value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + + @MSSQL_AS_SOURCE @MSSQL_AS_TARGET @Mssql_Required + Scenario: Verify the validation error message with blank host value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Validate "MySQL" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature index 31e731858..60d951651 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTime.feature @@ -60,3 +60,55 @@ Feature: Mssql - Verify Mssql sink data transfer Then Close the pipeline logs Then Validate the values of records transferred to target MsSql table is equal to the values from source BigQuery table + @MSSQL_SOURCE @MSSQL_TARGET @CONNECTION @Mssql_Required + Scenario: To verify data is getting transferred from Mssql to Mssql successfully with use connection using MAX type datatypes + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-SQL Server" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "mssqlOutputDatatypesSchema" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + Then Select connection: "connection.name" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "SQL Server2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Mssql sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table are equal to number of records from the source table diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature index 775df040e..ddfb3d2a3 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/RunTimeWithMacros.feature @@ -203,3 +203,87 @@ Feature: Mssql Sink - Run time scenarios (macro) Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorMessageInvalidCredentials | + + @MSSQL_SOURCE_DATATYPES_UIDTYPE_TEST @MSSQL_TARGET_DATATYPES_UIDTYPE_TEST @Mssql_Required + Scenario: To verify data is getting transferred from MsSQL to MsSQL successfully when macro enabled + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Click on the Macro button of Property: "jdbcPluginName" and set the value to: "SQLServerDriverName" + Then Click on the Macro button of Property: "host" and set the value to: "SQLServerHost" + Then Click on the Macro button of Property: "port" and set the value to: "SQLServerPort" + Then Click on the Macro button of Property: "user" and set the value to: "SQLServerUsername" + Then Click on the Macro button of Property: "password" and set the value to: "SQLServerPassword" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSource" + Then Click on the Macro button of Property: "database" and set the value to: "SQLServerDatabaseName" + Then Click on the Macro button of Property: "importQuery" and set the value in textarea: "SQLServerImportQuery" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Click on the Macro button of Property: "numSplits" and set the value to: "SQLServerNumSplits" + Then Click on the Macro button of Property: "fetchSize" and set the value to: "SQLServerFetchSize" + Then Click on the Macro button of Property: "splitBy" and set the value to: "SQLServerSplitByColumn" + Then Click on the Macro button of Property: "boundingQuery" and set the value in textarea: "SQLServerBoundingQuery" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + Then Click on the Macro button of Property: "jdbcPluginName" and set the value to: "SQLServerDriverName" + Then Click on the Macro button of Property: "host" and set the value to: "SQLServerHost" + Then Click on the Macro button of Property: "port" and set the value to: "SQLServerPort" + Then Click on the Macro button of Property: "user" and set the value to: "SQLServerUsername" + Then Click on the Macro button of Property: "password" and set the value to: "SQLServerPassword" + Then Click on the Macro button of Property: "connectionArguments" and set the value to: "connArgumentsSink" + Then Click on the Macro button of Property: "database" and set the value to: "SQLServerDatabaseName" + Then Enter input plugin property: "referenceName" with value: "targetRef" + And Click on the Macro button of Property: "tableName" and set the value to: "SQLServerTableName" + And Click on the Macro button of Property: "dbSchemaName" and set the value to: "SQLServerSchemaName" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "driverName" for key "SQLServerDriverName" + Then Enter runtime argument value from environment variable "host" for key "SQLServerHost" + Then Enter runtime argument value from environment variable "port" for key "SQLServerPort" + Then Enter runtime argument value from environment variable "username" for key "SQLServerUsername" + Then Enter runtime argument value from environment variable "password" for key "SQLServerPassword" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "fetchSize" for key "SQLServerFetchSize" + Then Enter runtime argument value "splitByField" for key "SQLServerSplitByColumn" + Then Enter runtime argument value "numberOfSplits" for key "SQLServerNumSplits" + Then Enter runtime argument value "selectQuery" for key "SQLServerImportQuery" + Then Enter runtime argument value "databaseName" for key "SQLServerDatabaseName" + Then Enter runtime argument value "boundingQuery" for key "SQLServerBoundingQuery" + Then Enter runtime argument value "targetTable" for key "SQLServerTableName" + Then Enter runtime argument value "schema" for key "SQLServerSchemaName" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "driverName" for key "SQLServerDriverName" + Then Enter runtime argument value from environment variable "host" for key "SQLServerHost" + Then Enter runtime argument value from environment variable "port" for key "SQLServerPort" + Then Enter runtime argument value from environment variable "username" for key "SQLServerUsername" + Then Enter runtime argument value from environment variable "password" for key "SQLServerPassword" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSource" + Then Enter runtime argument value "connectionArguments" for key "connArgumentsSink" + Then Enter runtime argument value "fetchSize" for key "SQLServerFetchSize" + Then Enter runtime argument value "splitByField" for key "SQLServerSplitByColumn" + Then Enter runtime argument value "numberOfSplits" for key "SQLServerNumSplits" + Then Enter runtime argument value "selectQuery" for key "SQLServerImportQuery" + Then Enter runtime argument value "databaseName" for key "SQLServerDatabaseName" + Then Enter runtime argument value "boundingQuery" for key "SQLServerBoundingQuery" + Then Enter runtime argument value "targetTable" for key "SQLServerTableName" + Then Enter runtime argument value "schema" for key "SQLServerSchemaName" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate records transferred to target table are equal to number of records from the source table diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature index 74277fa77..cc3bf8562 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/DesignTimeValidation.feature @@ -200,3 +200,31 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Click on the Validate button Then Verify that the Plugin Property: "boundingQuery" is displaying an in-line error message: "errorMessageBoundingQuery" Then Verify that the Plugin Property: "numSplits" is displaying an in-line error message: "errorMessagenumofSplit" + + @Mssql_Required + Scenario: Verify required fields missing validation messages + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "SQL Server" + Then Click on the Validate button + Then Verify mandatory property error for below listed properties: + | jdbcPluginName | + | referenceName | + | database | + | importQuery | + + @Mssql_Required + Scenario: Verify the validation error message with missing jdbc plugin name + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click plugin property: "switch-useConnection" + Then Click on the Validate button + Then Verify that the Plugin is displaying an error message: "blank.jdbcPluginName.message" on the header \ No newline at end of file diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature index 342a576d3..75d2d810d 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTime.feature @@ -184,3 +184,152 @@ Feature: Mssql - Verify Mssql source data transfer Then Wait till pipeline is in running state Then Open and capture logs And Verify the pipeline status is "Failed" + + @MSSQL_AS_SOURCE @BQ_SINK_TEST @Mssql_Required + Scenario: To verify data is getting transferred from Mssql to BigQuery successfully with bounding Query + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "BigQuery" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "BigQuery" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Enter textarea plugin property: "boundingQuery" with value: "boundingQuery" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputSchema" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + And Navigate to the properties page of plugin: "BigQuery" + And Enter input plugin property: "referenceName" with value: "Reference" + And Replace input plugin property: "project" with value: "projectId" + And Enter input plugin property: "datasetProject" with value: "datasetprojectId" + And Enter input plugin property: "dataset" with value: "dataset" + And Enter input plugin property: "table" with value: "bqtarget.table" + Then Validate "BigQuery" plugin properties + And Close the Plugin Properties page + Then Save the pipeline + And Preview and run the pipeline + And Wait till pipeline preview is in running state + And Open and capture pipeline preview logs + And Verify the preview run status of pipeline in the logs is "succeeded" + And Close the pipeline logs + And Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate the values of records transferred to target BigQuery table is equal to the values from source Table + + @MSSQL_SOURCE_DATATYPES_TEST @MSSQL_TARGET_DATATYPES_TEST @CONNECTION @Mssql_Required + Scenario: To verify data is getting transferred from Mssql to Mssql successfully with use connection + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-SQL Server" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + Then Select connection: "connection.name" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "SQL Server2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Verify the preview of pipeline is "success" + Then Click on preview data for Mssql sink + Then Verify preview output schema matches the outputSchema captured in properties + Then Close the preview data + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target table are equal to number of records from the source table + + @MSSQL_SOURCE_DATATYPES_TEST @MSSQL_TARGET_DATATYPES_TEST @CONNECTION @Mssql_Required + Scenario: To verify pipeline failure message in logs when an invalid bounding query is provided + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "SQL Server" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "SQL Server" from the plugins list as: "Sink" + Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection + Then Navigate to the properties page of plugin: "SQL Server" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + And Click on the Add Connection button + Then Click plugin property: "connector-SQL Server" + And Enter input plugin property: "name" with value: "connection.name" + Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName" + Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields + Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields + Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields + Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields + Then Click on the Test Connection button + And Verify the test connection is successful + Then Click on the Create button + Then Select connection: "connection.name" + Then Enter input plugin property: "referenceName" with value: "sourceRef" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "splitBy" with value: "splitBy" + Then Enter textarea plugin property: "importQuery" with value: "selectQuery" + Then Replace input plugin property: "numSplits" with value: "numberOfSplits" + Then Enter textarea plugin property: "boundingQuery" with value: "invalidBoundingQueryValue" + Then Validate "SQL Server" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "SQL Server2" + And Click plugin property: "switch-useConnection" + And Click on the Browse Connections button + Then Select connection: "connection.name" + Then Replace input plugin property: "database" with value: "databaseName" + Then Replace input plugin property: "tableName" with value: "targetTable" + Then Replace input plugin property: "dbSchemaName" with value: "schema" + Then Enter input plugin property: "referenceName" with value: "targetRef" + Then Validate "SQL Server2" plugin properties + Then Close the Plugin Properties page + And Save and Deploy Pipeline + And Run the Pipeline in Runtime + And Wait till pipeline is in running state + And Open and capture logs + And Verify the pipeline status is "Failed" + And Close the pipeline logs + Then Open Pipeline logs and verify Log entries having below listed Level and Message: + | Level | Message | + | ERROR | errorLogsMessageInvalidBoundingQuery | diff --git a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java index 0d1712658..92c354c48 100644 --- a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java +++ b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java @@ -255,5 +255,36 @@ private static boolean compareResultSetData(ResultSet rsSource, ResultSet rsTarg rsTarget.next()); return true; } + + public static void createMssqlSourceTable(String sourceTable, String schema) + throws SQLException, ClassNotFoundException { + try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) { + String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable + " (" + + "Description VARCHAR(MAX), " + + "FileData VARBINARY(MAX), " + + "Notes NVARCHAR(MAX), " + + "CustomValue NVARCHAR(MAX))"; + statement.executeUpdate(createSourceTableQuery); + String insertDataQuery = "INSERT INTO " + schema + "." + sourceTable + + " (Description, FileData, Notes, CustomValue) VALUES (" + + "'This is a long description stored in VARCHAR(MAX)', " + + "CONVERT(VARBINARY(MAX), 'BinaryDataExample'), " + + "N'This is a Unicode note stored in NVARCHAR(MAX)', " + + "'123.45')"; + statement.executeUpdate(insertDataQuery); + } + } + + public static void createMssqlTargetTable(String targetTable, String schema) throws SQLException, + ClassNotFoundException { + try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) { + String createTargetTableQuery = "CREATE TABLE " + schema + "." + targetTable + " (" + + "Description VARCHAR(MAX), " + + "FileData VARBINARY(MAX), " + + "Notes NVARCHAR(MAX), " + + "CustomValue NVARCHAR(MAX))"; + statement.executeUpdate(createTargetTableQuery); + } + } } diff --git a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java index 74ae9f911..902c82ff7 100644 --- a/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java +++ b/mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java @@ -17,6 +17,8 @@ package io.cdap.plugin.common.stepsdesign; import com.google.cloud.bigquery.BigQueryException; +import io.cdap.e2e.pages.actions.CdfConnectionActions; +import io.cdap.e2e.pages.actions.CdfPluginPropertiesActions; import io.cdap.e2e.utils.BigQueryClient; import io.cdap.e2e.utils.PluginPropertyUtils; import io.cdap.plugin.MssqlClient; @@ -51,8 +53,10 @@ public static void setTableName() { String targetTableName = String.format("TARGETTABLE_%s", randomString); PluginPropertyUtils.addPluginProp("targetTable", targetTableName); targetTable = targetTableName; - PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s.%s", schema, - sourceTable)); + PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s.%s" + + " WHERE $CONDITIONS", schema, sourceTable)); + PluginPropertyUtils.addPluginProp("boundingQuery", String.format("select MIN(ID),MAX(ID)" + + " from %s.%s", schema, sourceTable)); } @Before(order = 2, value = "@MSSQL_AS_SOURCE") @@ -264,4 +268,49 @@ public static void deleteTempTargetBQTable() throws IOException, InterruptedExce } } } + + @Before(order = 1, value = "@CONNECTION") + public static void setNewConnectionName() { + String connectionName = "SQL Server" + RandomStringUtils.randomAlphanumeric(10); + PluginPropertyUtils.addPluginProp("connection.name", connectionName); + BeforeActions.scenario.write("New Connection name: " + connectionName); + } + + private static void deleteConnection(String connectionType, String connectionName) throws IOException { + CdfConnectionActions.openWranglerConnectionsPage(); + CdfConnectionActions.expandConnections(connectionType); + CdfConnectionActions.openConnectionActionMenu(connectionType, connectionName); + CdfConnectionActions.selectConnectionAction(connectionType, connectionName, "Delete"); + CdfPluginPropertiesActions.clickPluginPropertyButton("Delete"); + } + + @After(order = 1, value = "@CONNECTION") + public static void deleteTestConnection() throws IOException { + deleteConnection("SQL Server", "connection.name"); + PluginPropertyUtils.removePluginProp("connection.name"); + } + + @Before(order = 2, value = "@MSSQL_SOURCE") + public static void createMssqlSourceTables() throws SQLException, ClassNotFoundException { + MssqlClient.createMssqlSourceTable(sourceTable, schema); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + sourceTable + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_SOURCE") + public static void dropMssqlSourceTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(schema, sourceTable); + BeforeActions.scenario.write("MSSQL SOURCE Table - " + sourceTable + " deleted successfully"); + } + + @Before(order = 2, value = "@MSSQL_TARGET") + public static void createMssqlTargetTables() throws SQLException, ClassNotFoundException { + MssqlClient.createMssqlTargetTable(targetTable, schema); + BeforeActions.scenario.write("MSSQL TARGET Table - " + targetTable + " created successfully"); + } + + @After(order = 2, value = "@MSSQL_TARGET") + public static void dropMssqlTargetTable() throws SQLException, ClassNotFoundException { + MssqlClient.deleteTable(schema, targetTable); + BeforeActions.scenario.write("MSSQL TARGET Table - " + targetTable + " deleted successfully"); + } } diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index c752d6ec1..e66697fd8 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -21,3 +21,6 @@ errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Stage Error occurred while trying to get schema from database.Error message: 'Incorrect syntax near the keyword 'table'.'. errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.lang.IllegalArgumentException: \ Plugin with id SQL Server:source.jdbc.sqlserver does not exist in program phase-1 of application +errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.io.IOException: Could not find stored procedure +blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. +blank.connection.message=Exception while trying to validate schema of database table diff --git a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties index ede609c0e..4421c0c72 100644 --- a/mssql-plugin/src/e2e-test/resources/pluginParameters.properties +++ b/mssql-plugin/src/e2e-test/resources/pluginParameters.properties @@ -39,6 +39,9 @@ invalid.Host=MSSQL_HOST1 invalid.Port=MSSQL_PORT1 invalid.Username=MSSQL_USERNAME1 invalid.Password=MSSQL_PASSWORD1 +splitByField=ID +numSplits=1 +connectionArguments=queryTimeout=50 #bq queries file path CreateBQTableQueryFile=testData/BigQuery/BigQueryCreateTableQuery.txt @@ -84,3 +87,5 @@ dateTimeValues=VALUES ('User1', '2023-01-01 01:00:00.000', '2023-01-01 01:00:00. '2025-12-10 12:32:10.000 +01:00') outputDatatypesSchema4=[{"key":"ID","value":"string"},{"key":"COL1","value":"datetime"},\ {"key":"COL2","value":"datetime"},{"key":"COL3","value":"datetime"}, {"key":"COL4","value":"timestamp"}] +mssqlOutputDatatypesSchema=[{"key":"Description","value":"string"}, {"key":"FileData","value":"bytes"}, \ + {"key":"Notes","value":"string"}, {"key":"CustomValue","value":"string"}] diff --git a/mssql-plugin/widgets/SQL Server-connector.json b/mssql-plugin/widgets/SQL Server-connector.json index 171076295..833f40c26 100644 --- a/mssql-plugin/widgets/SQL Server-connector.json +++ b/mssql-plugin/widgets/SQL Server-connector.json @@ -83,6 +83,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/mssql-plugin/widgets/SqlServer-action.json b/mssql-plugin/widgets/SqlServer-action.json index 303944d71..70ab3a9e8 100644 --- a/mssql-plugin/widgets/SqlServer-action.json +++ b/mssql-plugin/widgets/SqlServer-action.json @@ -204,6 +204,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index 260c66259..85991039e 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -243,6 +243,37 @@ "name": "currentLanguage" } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/mssql-plugin/widgets/SqlServer-batchsource.json b/mssql-plugin/widgets/SqlServer-batchsource.json index dad5f4708..791ea0fad 100644 --- a/mssql-plugin/widgets/SqlServer-batchsource.json +++ b/mssql-plugin/widgets/SqlServer-batchsource.json @@ -262,6 +262,37 @@ "name": "currentLanguage" } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/mssql-plugin/widgets/SqlServer-postaction.json b/mssql-plugin/widgets/SqlServer-postaction.json index 5cec14b89..57947b03d 100644 --- a/mssql-plugin/widgets/SqlServer-postaction.json +++ b/mssql-plugin/widgets/SqlServer-postaction.json @@ -219,6 +219,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index 9064d1bf6..4714b4c22 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -64,6 +64,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/mysql-plugin/widgets/Mysql-action.json b/mysql-plugin/widgets/Mysql-action.json index ae5d0b555..d62f41d45 100644 --- a/mysql-plugin/widgets/Mysql-action.json +++ b/mysql-plugin/widgets/Mysql-action.json @@ -161,6 +161,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index c525ead40..4d0870ef0 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -203,6 +203,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 9175bd5ed..a2a30a9a8 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -238,6 +238,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/mysql-plugin/widgets/Mysql-postaction.json b/mysql-plugin/widgets/Mysql-postaction.json index e34a40928..283069438 100644 --- a/mysql-plugin/widgets/Mysql-postaction.json +++ b/mysql-plugin/widgets/Mysql-postaction.json @@ -176,6 +176,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/netezza-plugin/widgets/Netezza-action.json b/netezza-plugin/widgets/Netezza-action.json index de523e860..a7172528c 100644 --- a/netezza-plugin/widgets/Netezza-action.json +++ b/netezza-plugin/widgets/Netezza-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/netezza-plugin/widgets/Netezza-batchsink.json b/netezza-plugin/widgets/Netezza-batchsink.json index c8634a58b..eb7747124 100644 --- a/netezza-plugin/widgets/Netezza-batchsink.json +++ b/netezza-plugin/widgets/Netezza-batchsink.json @@ -95,6 +95,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/netezza-plugin/widgets/Netezza-batchsource.json b/netezza-plugin/widgets/Netezza-batchsource.json index c1e0f26c3..2ce431d16 100644 --- a/netezza-plugin/widgets/Netezza-batchsource.json +++ b/netezza-plugin/widgets/Netezza-batchsource.json @@ -110,6 +110,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/netezza-plugin/widgets/Netezza-postaction.json b/netezza-plugin/widgets/Netezza-postaction.json index 85ea43ad9..e54bfd2ef 100644 --- a/netezza-plugin/widgets/Netezza-postaction.json +++ b/netezza-plugin/widgets/Netezza-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/oracle-plugin/widgets/Oracle-action.json b/oracle-plugin/widgets/Oracle-action.json index 815aa4b0f..7ed87fa4c 100644 --- a/oracle-plugin/widgets/Oracle-action.json +++ b/oracle-plugin/widgets/Oracle-action.json @@ -107,6 +107,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 8d6168780..a799c048a 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -220,6 +220,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 5eca20cc4..569098111 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -248,6 +248,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 628027caf..d35311088 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -148,6 +148,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters" : [ diff --git a/oracle-plugin/widgets/Oracle-postaction.json b/oracle-plugin/widgets/Oracle-postaction.json index 9a18077c4..d333765f4 100644 --- a/oracle-plugin/widgets/Oracle-postaction.json +++ b/oracle-plugin/widgets/Oracle-postaction.json @@ -122,6 +122,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/pom.xml b/pom.xml index c7c90fe9c..9bbc3a4a7 100644 --- a/pom.xml +++ b/pom.xml @@ -61,8 +61,8 @@ true UTF-8 - 6.11.0-SNAPSHOT - 2.13.0-SNAPSHOT + 6.11.0 + 2.13.1-SNAPSHOT 13.0.1 3.3.6 2.2.4 @@ -176,6 +176,8 @@ guava ${guava.version} + + org.apache.hadoop hadoop-common @@ -298,6 +300,12 @@ ${junit.version} test + + + dev.failsafe + failsafe + 3.3.2 + org.hsqldb hsqldb diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 091afc972..79a53f9e4 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -69,6 +69,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/postgresql-plugin/widgets/Postgres-action.json b/postgresql-plugin/widgets/Postgres-action.json index 351c023f1..d86ac1e3b 100644 --- a/postgresql-plugin/widgets/Postgres-action.json +++ b/postgresql-plugin/widgets/Postgres-action.json @@ -82,6 +82,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 6aa2dad8a..0955b4014 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 0e4ba28c1..2b42ce03b 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -159,6 +159,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/postgresql-plugin/widgets/Postgres-postaction.json b/postgresql-plugin/widgets/Postgres-postaction.json index 5a0daf595..cf601cfd5 100644 --- a/postgresql-plugin/widgets/Postgres-postaction.json +++ b/postgresql-plugin/widgets/Postgres-postaction.json @@ -97,6 +97,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/saphana-plugin/widgets/SapHana-action.json b/saphana-plugin/widgets/SapHana-action.json index 7e60ac35d..a2127e4b8 100644 --- a/saphana-plugin/widgets/SapHana-action.json +++ b/saphana-plugin/widgets/SapHana-action.json @@ -82,6 +82,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/saphana-plugin/widgets/SapHana-batchsink.json b/saphana-plugin/widgets/SapHana-batchsink.json index a9d8c6343..6b46726af 100644 --- a/saphana-plugin/widgets/SapHana-batchsink.json +++ b/saphana-plugin/widgets/SapHana-batchsink.json @@ -103,6 +103,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/saphana-plugin/widgets/SapHana-batchsource.json b/saphana-plugin/widgets/SapHana-batchsource.json index 9352b02f7..84d81e32b 100644 --- a/saphana-plugin/widgets/SapHana-batchsource.json +++ b/saphana-plugin/widgets/SapHana-batchsource.json @@ -127,6 +127,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/saphana-plugin/widgets/SapHana-postaction.json b/saphana-plugin/widgets/SapHana-postaction.json index ad2c8b938..8df387e3e 100644 --- a/saphana-plugin/widgets/SapHana-postaction.json +++ b/saphana-plugin/widgets/SapHana-postaction.json @@ -97,6 +97,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/teradata-plugin/widgets/Teradata-action.json b/teradata-plugin/widgets/Teradata-action.json index 2ffba361c..2d10dbda0 100644 --- a/teradata-plugin/widgets/Teradata-action.json +++ b/teradata-plugin/widgets/Teradata-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/teradata-plugin/widgets/Teradata-batchsink.json b/teradata-plugin/widgets/Teradata-batchsink.json index f455991d4..f978291a1 100644 --- a/teradata-plugin/widgets/Teradata-batchsink.json +++ b/teradata-plugin/widgets/Teradata-batchsink.json @@ -95,6 +95,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/teradata-plugin/widgets/Teradata-batchsource.json b/teradata-plugin/widgets/Teradata-batchsource.json index 94f5314e5..4b060fb90 100644 --- a/teradata-plugin/widgets/Teradata-batchsource.json +++ b/teradata-plugin/widgets/Teradata-batchsource.json @@ -115,6 +115,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/teradata-plugin/widgets/Teradata-postaction.json b/teradata-plugin/widgets/Teradata-postaction.json index 35ead0013..d3552d336 100644 --- a/teradata-plugin/widgets/Teradata-postaction.json +++ b/teradata-plugin/widgets/Teradata-postaction.json @@ -90,6 +90,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 60, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] }