From 5045a4b19b1187becf49fb888ab1584ea577b4e3 Mon Sep 17 00:00:00 2001 From: seanzhougoogle <69162095+seanzhougoogle@users.noreply.github.com> Date: Mon, 28 Jun 2021 15:35:45 -0700 Subject: [PATCH] migrate to CDAP 6.5.0 and fix failed test suits (#120) * migrate to CDAP 6.5.0 and fix failed test suits --- aurora-mysql-plugin/pom.xml | 36 +++--------- .../mysql/AuroraMysqlPluginTestBase.java | 56 +++++++++---------- .../mysql/AuroraMysqlPluginTestSuite.java | 12 ---- aurora-postgresql-plugin/pom.xml | 30 +--------- .../AuroraPostgresPluginTestBase.java | 56 +++++++++---------- .../AuroraPostgresPluginTestSuite.java | 12 ---- cloudsql-mysql-plugin/pom.xml | 10 ---- cloudsql-postgresql-plugin/pom.xml | 10 ---- database-commons/pom.xml | 40 +------------ .../db/batch/DatabasePluginTestBase.java | 26 +++++++++ db2-plugin/pom.xml | 36 +++--------- .../io/cdap/plugin/db2/Db2PluginTestBase.java | 45 ++++++++------- .../cdap/plugin/db2/Db2PluginTestSuite.java | 12 ---- generic-database-plugin/pom.xml | 30 +--------- generic-db-argument-setter/pom.xml | 30 +--------- mariadb-plugin/pom.xml | 36 +++--------- .../plugin/mariadb/MariadbPluginTestBase.java | 45 ++++++++------- .../mariadb/MariadbPluginTestSuite.java | 12 ---- memsql-plugin/pom.xml | 20 +++---- .../plugin/memsql/MemsqlPluginTestBase.java | 45 ++++++++------- .../plugin/memsql/MemsqlPluginTestSuite.java | 12 ---- mssql-plugin/pom.xml | 23 +++----- .../plugin/mssql/SqlServerPluginTestBase.java | 51 +++++++++-------- .../mssql/SqlServerPluginTestSuite.java | 12 ---- mysql-plugin/pom.xml | 36 +++--------- .../plugin/mysql/MysqlPluginTestBase.java | 46 ++++++++------- .../plugin/mysql/MysqlPluginTestSuite.java | 12 ---- netezza-plugin/pom.xml | 30 +--------- .../plugin/netezza/NetezzaPluginTestBase.java | 45 ++++++++------- .../netezza/NetezzaPluginTestSuite.java | 12 ---- oracle-plugin/pom.xml | 36 +++--------- .../plugin/oracle/OraclePluginTestBase.java | 53 ++++++++++-------- .../plugin/oracle/OraclePluginTestSuite.java | 12 ---- pom.xml | 41 ++++++++++++-- postgresql-plugin/pom.xml | 30 +--------- .../postgres/PostgresPluginTestBase.java | 44 ++++++++------- .../postgres/PostgresPluginTestSuite.java | 12 ---- saphana-plugin/pom.xml | 12 +--- teradata-plugin/pom.xml | 12 +--- .../teradata/TeradataPluginTestBase.java | 48 ++++++++-------- .../teradata/TeradataPluginTestSuite.java | 12 ---- 41 files changed, 403 insertions(+), 787 deletions(-) diff --git a/aurora-mysql-plugin/pom.xml b/aurora-mysql-plugin/pom.xml index d88728068..2149dc13c 100644 --- a/aurora-mysql-plugin/pom.xml +++ b/aurora-mysql-plugin/pom.xml @@ -61,12 +61,18 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit junit + + mysql + mysql-connector-java + 8.0.19 + test + io.cdap.cdap cdap-api @@ -79,36 +85,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestBase.java b/aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestBase.java index cb1b5ee53..f9c197d86 100644 --- a/aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestBase.java +++ b/aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -38,6 +37,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Connection; @@ -49,44 +50,40 @@ import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; import javax.sql.rowset.serial.SerialBlob; public abstract class AuroraMysqlPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(AuroraMysqlPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); protected static final String DRIVER_CLASS = "com.mysql.jdbc.Driver"; protected static final String JDBC_DRIVER_NAME = "mysql"; + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static int year; protected static final int PRECISION = 10; protected static final int SCALE = 6; - protected static boolean tearDown = true; private static int startCount; @ClassRule public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false); - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, getPropertyOrFail("auroraMysql.clusterEndpoint")) - .put(ConnectionConfig.PORT, getPropertyOrFail("auroraMysql.port")) - .put(ConnectionConfig.DATABASE, getPropertyOrFail("auroraMysql.database")) - .put(ConnectionConfig.USER, getPropertyOrFail("auroraMysql.username")) - .put(ConnectionConfig.PASSWORD, getPropertyOrFail("auroraMysql.password")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); - @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); + Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date(CURRENT_TS)); year = calendar.get(Calendar.YEAR); @@ -118,6 +115,15 @@ public static void setupTest() throws Exception { prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("auroraMysql.clusterEndpoint")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("auroraMysql.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("auroraMysql.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("auroraMysql.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("auroraMysql.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create a table that the action will truncate at the end of the run @@ -228,19 +234,8 @@ private static void populateData(PreparedStatement... stmts) throws SQLException } } - private static String getPropertyOrFail(String propertyName) { - String value = System.getProperty(propertyName); - - if (value == null) { - throw new IllegalStateException("There is no value for property " + propertyName); - } - - return value; - } - public static Connection createConnection() { try { - Class.forName(DRIVER_CLASS); return DriverManager.getConnection(connectionUrl, BASE_PROPS.get(ConnectionConfig.USER), BASE_PROPS.get(ConnectionConfig.PASSWORD)); } catch (Exception e) { @@ -249,18 +244,17 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE postActionTest"); - stmt.execute("DROP TABLE dbActionTest"); - stmt.execute("DROP TABLE MY_DEST_TABLE"); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestSuite.java b/aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestSuite.java index 8ab9aa88f..db56629f2 100644 --- a/aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestSuite.java +++ b/aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.aurora.mysql; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ AuroraMysqlPostActionTestRun.class }) public class AuroraMysqlPluginTestSuite extends AuroraMysqlPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/aurora-postgresql-plugin/pom.xml b/aurora-postgresql-plugin/pom.xml index 0f61d88d5..67606c3d5 100644 --- a/aurora-postgresql-plugin/pom.xml +++ b/aurora-postgresql-plugin/pom.xml @@ -67,7 +67,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -85,36 +85,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestBase.java b/aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestBase.java index 8a10c8150..d2c200e8e 100644 --- a/aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestBase.java +++ b/aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -35,6 +34,8 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.postgresql.Driver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Connection; @@ -45,43 +46,39 @@ import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; public abstract class AuroraPostgresPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(AuroraPostgresPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); protected static final String JDBC_DRIVER_NAME = "postrgesql"; + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static int year; protected static final int PRECISION = 10; protected static final int SCALE = 6; - protected static boolean tearDown = true; private static int startCount; @ClassRule public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false); - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, getPropertyOrFail("auroraPostgresql.clusterEndpoint")) - .put(ConnectionConfig.PORT, getPropertyOrFail("auroraPostgresql.port")) - .put(ConnectionConfig.DATABASE, getPropertyOrFail("auroraPostgresql.database")) - .put(ConnectionConfig.USER, getPropertyOrFail("auroraPostgresql.username")) - .put(ConnectionConfig.PASSWORD, getPropertyOrFail("auroraPostgresql.password")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); - @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); + Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date(CURRENT_TS)); year = calendar.get(Calendar.YEAR); @@ -111,6 +108,15 @@ public static void setupTest() throws Exception { prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("auroraPostgresql.clusterEndpoint")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("auroraPostgresql.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("auroraPostgresql.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("auroraPostgresql.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("auroraPostgresql.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create a table that the action will truncate at the end of the run @@ -189,16 +195,6 @@ private static void populateData(PreparedStatement... stmts) throws SQLException } } - private static String getPropertyOrFail(String propertyName) { - String value = System.getProperty(propertyName); - - if (value == null) { - throw new IllegalStateException("There is no value for property " + propertyName); - } - - return value; - } - public static Connection createConnection() { try { Class.forName(Driver.class.getCanonicalName()); @@ -210,18 +206,16 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } - + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE \"postActionTest\""); - stmt.execute("DROP TABLE \"dbActionTest\""); - stmt.execute("DROP TABLE \"MY_DEST_TABLE\""); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestSuite.java b/aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestSuite.java index a96398f58..59da03761 100644 --- a/aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestSuite.java +++ b/aurora-postgresql-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.auroradb.postgres; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ AuroraPostgresPostActionTestRun.class }) public class AuroraPostgresPluginTestSuite extends AuroraPostgresPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/cloudsql-mysql-plugin/pom.xml b/cloudsql-mysql-plugin/pom.xml index f60ac6f48..79bb23f0b 100644 --- a/cloudsql-mysql-plugin/pom.xml +++ b/cloudsql-mysql-plugin/pom.xml @@ -55,8 +55,6 @@ org.apache.felix maven-bundle-plugin - 3.5.0 - true <_exportcontents> @@ -67,14 +65,6 @@ lib - - - package - - bundle - - - io.cdap diff --git a/cloudsql-postgresql-plugin/pom.xml b/cloudsql-postgresql-plugin/pom.xml index 7e85cd7c5..727235b90 100644 --- a/cloudsql-postgresql-plugin/pom.xml +++ b/cloudsql-postgresql-plugin/pom.xml @@ -60,8 +60,6 @@ org.apache.felix maven-bundle-plugin - 3.5.0 - true <_exportcontents> @@ -72,14 +70,6 @@ lib - - - package - - bundle - - - io.cdap diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 29f424bd6..a51cf58b6 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -48,17 +48,12 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit junit - - io.cdap.cdap - cdap-api - compile - com.mockrunner mockrunner-jdbc @@ -68,44 +63,11 @@ org.mockito mockito-core - ${mockito.version} - test - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - - - io.cdap - cdap-maven-plugin - org.apache.maven.plugins maven-jar-plugin diff --git a/database-commons/src/test/java/io/cdap/plugin/db/batch/DatabasePluginTestBase.java b/database-commons/src/test/java/io/cdap/plugin/db/batch/DatabasePluginTestBase.java index 23d2ec26c..fa45dcee6 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/batch/DatabasePluginTestBase.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/batch/DatabasePluginTestBase.java @@ -35,6 +35,8 @@ import io.cdap.cdap.test.DataSetManager; import io.cdap.cdap.test.WorkflowManager; import org.junit.Assert; +import org.junit.Assume; +import org.slf4j.Logger; import java.sql.ResultSet; import java.sql.SQLException; @@ -199,4 +201,28 @@ protected void testInvalidDataWrite(ResultSet resultSet, String columnName) thro } Assert.assertFalse(users.contains("1")); } + + //TODO Most of the test suite of Database plugins in this repo rely on manually setting up a local/remote database + // sever + // Should consider automatically bring up a local database server in test suite, e.g. using docker images via + // docker-maven-plugin + protected static String getPropertyOrSkip(String propertyName) { + String value = System.getProperty(propertyName); + Assume.assumeFalse("There is no value for property " + propertyName, value == null); + return value; + } + + @FunctionalInterface + protected interface Cleanup { + void run() throws Exception; + } + protected static void executeCleanup(List cleanups, Logger logger) { + for (Cleanup cleanup : cleanups) { + try { + cleanup.run(); + } catch (Exception e) { + logger.warn("Fail to cleanup.", e); + } + } + } } diff --git a/db2-plugin/pom.xml b/db2-plugin/pom.xml index ea2e1616d..d1561d26f 100644 --- a/db2-plugin/pom.xml +++ b/db2-plugin/pom.xml @@ -61,7 +61,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -72,6 +72,12 @@ cdap-api provided + + com.ibm.db2.jcc + db2jcc + db2jcc4 + test + org.jetbrains annotations @@ -79,36 +85,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/db2-plugin/src/test/java/io/cdap/plugin/db2/Db2PluginTestBase.java b/db2-plugin/src/test/java/io/cdap/plugin/db2/Db2PluginTestBase.java index e7f14cf2a..854a8b2e3 100644 --- a/db2-plugin/src/test/java/io/cdap/plugin/db2/Db2PluginTestBase.java +++ b/db2-plugin/src/test/java/io/cdap/plugin/db2/Db2PluginTestBase.java @@ -17,7 +17,6 @@ package io.cdap.plugin.db2; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -33,6 +32,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Connection; @@ -43,25 +44,28 @@ import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; import javax.sql.rowset.serial.SerialBlob; import javax.sql.rowset.serial.SerialClob; public abstract class Db2PluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(Db2PluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); private static final String DRIVER_CLASS = "com.ibm.db2.jcc.DB2Driver"; protected static final String JDBC_DRIVER_NAME = "db211"; + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static int year; protected static final int PRECISION = 10; protected static final int SCALE = 6; - protected static boolean tearDown = true; private static int startCount; @ClassRule @@ -73,21 +77,14 @@ public abstract class Db2PluginTestBase extends DatabasePluginTestBase { year = calendar.get(Calendar.YEAR); } - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("db2.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("db2.port", "50000")) - .put(ConnectionConfig.DATABASE, System.getProperty("db2.database", "mydb")) - .put(ConnectionConfig.USER, System.getProperty("db2.username", "db2inst1")) - .put(ConnectionConfig.PASSWORD, System.getProperty("db2.password", "123Qwe123")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); - @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + + getProperties(); Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date(CURRENT_TS)); @@ -119,6 +116,15 @@ public static void setupTest() throws Exception { prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("db2.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("db2.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("db2.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("db2.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("db2.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create a table that the action will truncate at the end of the run @@ -218,18 +224,17 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE postActionTest"); - stmt.execute("DROP TABLE dbActionTest"); - stmt.execute("DROP TABLE MY_DEST_TABLE"); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/db2-plugin/src/test/java/io/cdap/plugin/db2/Db2PluginTestSuite.java b/db2-plugin/src/test/java/io/cdap/plugin/db2/Db2PluginTestSuite.java index be59e6637..5ee6c44da 100644 --- a/db2-plugin/src/test/java/io/cdap/plugin/db2/Db2PluginTestSuite.java +++ b/db2-plugin/src/test/java/io/cdap/plugin/db2/Db2PluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.db2; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ Db2PostActionTestRun.class }) public class Db2PluginTestSuite extends Db2PluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/generic-database-plugin/pom.xml b/generic-database-plugin/pom.xml index 8c4092101..439ab8b41 100644 --- a/generic-database-plugin/pom.xml +++ b/generic-database-plugin/pom.xml @@ -61,7 +61,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -84,36 +84,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/generic-db-argument-setter/pom.xml b/generic-db-argument-setter/pom.xml index a9e3875df..fa3949a9c 100644 --- a/generic-db-argument-setter/pom.xml +++ b/generic-db-argument-setter/pom.xml @@ -61,7 +61,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -84,36 +84,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/mariadb-plugin/pom.xml b/mariadb-plugin/pom.xml index 4ad752f37..582ad2375 100644 --- a/mariadb-plugin/pom.xml +++ b/mariadb-plugin/pom.xml @@ -61,7 +61,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -72,6 +72,12 @@ cdap-api provided + + org.mariadb.jdbc + mariadb-java-client + 2.7.3 + test + org.jetbrains annotations @@ -79,36 +85,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/mariadb-plugin/src/test/java/io/cdap/plugin/mariadb/MariadbPluginTestBase.java b/mariadb-plugin/src/test/java/io/cdap/plugin/mariadb/MariadbPluginTestBase.java index bed732962..12f7f1258 100644 --- a/mariadb-plugin/src/test/java/io/cdap/plugin/mariadb/MariadbPluginTestBase.java +++ b/mariadb-plugin/src/test/java/io/cdap/plugin/mariadb/MariadbPluginTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -34,6 +33,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Connection; @@ -47,13 +48,16 @@ import java.sql.Types; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; import javax.sql.rowset.serial.SerialBlob; public abstract class MariadbPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(MariadbPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); @@ -65,7 +69,7 @@ public abstract class MariadbPluginTestBase extends DatabasePluginTestBase { protected static final int PRECISION = 10; protected static final int SCALE = 6; protected static final ZoneId UTC_ZONE = ZoneId.ofOffset("UTC", ZoneOffset.UTC); - protected static boolean tearDown = true; + protected static final Map BASE_PROPS = new HashMap<>(); private static int startCount; @ClassRule @@ -77,20 +81,14 @@ public abstract class MariadbPluginTestBase extends DatabasePluginTestBase { YEAR = calendar.get(Calendar.YEAR); } - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("mariadb.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("mariadb.port", "3306")) - .put(ConnectionConfig.DATABASE, System.getProperty("mariadb.database", "mydb")) - .put(ConnectionConfig.USER, System.getProperty("mariadb.username", "root")) - .put(ConnectionConfig.PASSWORD, System.getProperty("mariadb.password", "123Qwe123")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); + @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); @@ -121,6 +119,15 @@ public static void setupTest() throws Exception { prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("mariadb.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("mariadb.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("mariadb.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("mariadb.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("mariadb.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create a table that the action will truncate at the end of the run @@ -248,18 +255,16 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } - + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE postActionTest"); - stmt.execute("DROP TABLE dbActionTest"); - stmt.execute("DROP TABLE MY_DEST_TABLE"); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/mariadb-plugin/src/test/java/io/cdap/plugin/mariadb/MariadbPluginTestSuite.java b/mariadb-plugin/src/test/java/io/cdap/plugin/mariadb/MariadbPluginTestSuite.java index 15e0ba2b2..d3a62fb51 100644 --- a/mariadb-plugin/src/test/java/io/cdap/plugin/mariadb/MariadbPluginTestSuite.java +++ b/mariadb-plugin/src/test/java/io/cdap/plugin/mariadb/MariadbPluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.mariadb; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ MariadbPostActionTestRun.class }) public class MariadbPluginTestSuite extends MariadbPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/memsql-plugin/pom.xml b/memsql-plugin/pom.xml index afb4c158e..5fa8b8b67 100644 --- a/memsql-plugin/pom.xml +++ b/memsql-plugin/pom.xml @@ -72,7 +72,13 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 + + + org.mariadb.jdbc + mariadb-java-client + 2.7.3 + test junit @@ -85,8 +91,6 @@ org.apache.felix maven-bundle-plugin - 3.3.0 - true <_exportcontents> @@ -97,14 +101,6 @@ lib - - - package - - bundle - - - io.cdap @@ -112,4 +108,4 @@ - \ No newline at end of file + diff --git a/memsql-plugin/src/test/java/io/cdap/plugin/memsql/MemsqlPluginTestBase.java b/memsql-plugin/src/test/java/io/cdap/plugin/memsql/MemsqlPluginTestBase.java index 036bec23f..5b12e3b1a 100644 --- a/memsql-plugin/src/test/java/io/cdap/plugin/memsql/MemsqlPluginTestBase.java +++ b/memsql-plugin/src/test/java/io/cdap/plugin/memsql/MemsqlPluginTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -38,6 +37,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Connection; @@ -51,25 +52,28 @@ import java.sql.Types; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; import javax.sql.rowset.serial.SerialBlob; public abstract class MemsqlPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(MemsqlPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); protected static final String DRIVER_CLASS = "org.mariadb.jdbc.Driver"; protected static final String JDBC_DRIVER_NAME = "mariadb"; + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static final int YEAR; protected static final int PRECISION = 10; protected static final int SCALE = 6; protected static final ZoneId UTC_ZONE = ZoneId.ofOffset("UTC", ZoneOffset.UTC); - protected static boolean tearDown = true; private static int startCount; @ClassRule @@ -81,21 +85,13 @@ public abstract class MemsqlPluginTestBase extends DatabasePluginTestBase { YEAR = calendar.get(Calendar.YEAR); } - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("memsql.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("memsql.port", "3308")) - .put(ConnectionConfig.DATABASE, System.getProperty("memsql.database", "mydb")) - .put(ConnectionConfig.USER, System.getProperty("memsql.username", "root")) - .put(ConnectionConfig.PASSWORD, System.getProperty("memsql.password", "root")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); - @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); addPluginArtifact(NamespaceId.DEFAULT.artifact(JDBC_DRIVER_NAME, "1.0.0"), @@ -123,6 +119,15 @@ public static void setupTest() throws Exception { prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("memsql.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("memsql.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("memsql.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("memsql.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("memsql.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create a table that the action will truncate at the end of the run @@ -256,18 +261,16 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } - + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE IF EXISTS my_table"); - stmt.execute("DROP TABLE IF EXISTS your_table"); - stmt.execute("DROP TABLE IF EXISTS postActionTest"); - stmt.execute("DROP TABLE IF EXISTS dbActionTest"); - stmt.execute("DROP TABLE IF EXISTS MY_DEST_TABLE"); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/memsql-plugin/src/test/java/io/cdap/plugin/memsql/MemsqlPluginTestSuite.java b/memsql-plugin/src/test/java/io/cdap/plugin/memsql/MemsqlPluginTestSuite.java index a9abe945e..4b657b6db 100644 --- a/memsql-plugin/src/test/java/io/cdap/plugin/memsql/MemsqlPluginTestSuite.java +++ b/memsql-plugin/src/test/java/io/cdap/plugin/memsql/MemsqlPluginTestSuite.java @@ -21,8 +21,6 @@ import io.cdap.plugin.memsql.postaction.MemsqlPostActionTestRun; import io.cdap.plugin.memsql.sink.MemsqlSinkTestRun; import io.cdap.plugin.memsql.source.MemsqlSourceTestRun; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -38,14 +36,4 @@ MemsqlPostActionTestRun.class }) public class MemsqlPluginTestSuite extends MemsqlPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/mssql-plugin/pom.xml b/mssql-plugin/pom.xml index 708350bae..64278e568 100644 --- a/mssql-plugin/pom.xml +++ b/mssql-plugin/pom.xml @@ -38,11 +38,6 @@ database-commons ${project.version} - - com.microsoft.azure - adal4j - 1.6.4 - io.cdap.plugin hydrator-common @@ -66,7 +61,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -77,6 +72,12 @@ cdap-api provided + + com.microsoft.sqlserver + mssql-jdbc + 8.2.1.jre8 + test + org.jetbrains annotations @@ -90,8 +91,6 @@ org.apache.felix maven-bundle-plugin - 3.3.0 - true <_exportcontents> @@ -106,14 +105,6 @@ lib - - - package - - bundle - - - io.cdap diff --git a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerPluginTestBase.java b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerPluginTestBase.java index 9d0e00ca4..490293e86 100644 --- a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerPluginTestBase.java +++ b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerPluginTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -34,6 +33,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -50,12 +51,15 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; public abstract class SqlServerPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(SqlServerPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); @@ -65,6 +69,7 @@ public abstract class SqlServerPluginTestBase extends DatabasePluginTestBase { protected static final List TIMESTAMP_VALUES = new ArrayList<>(); protected static final List GEOMETRY_VALUES = new ArrayList<>(); protected static final List GEOGRAPHY_VALUES = new ArrayList<>(); + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static final int PRECISION = 10; @@ -74,28 +79,20 @@ public abstract class SqlServerPluginTestBase extends DatabasePluginTestBase { protected static final int SMALL_MONEY_PRECISION = 10; protected static final int SMALL_MONEY_SCALE = 4; protected static final LocalTime TIME_MICROS = LocalTime.of(16, 17, 18, 123456000); - protected static boolean tearDown = true; protected static final ZoneId UTC = ZoneId.ofOffset("UTC", ZoneOffset.UTC); private static int startCount; @ClassRule public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false); - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("mssql.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("mssql.port", "1433")) - .put(ConnectionConfig.DATABASE, System.getProperty("mssql.database", "tempdb")) - .put(ConnectionConfig.USER, System.getProperty("mssql.username", "sa")) - .put(ConnectionConfig.PASSWORD, System.getProperty("mssql.password", "123Qwe123")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); - @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); + setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); addPluginArtifact(NamespaceId.DEFAULT.artifact(JDBC_DRIVER_NAME, "1.0.0"), @@ -130,6 +127,15 @@ public static void setupTest() throws Exception { } } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("mssql.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("mssql.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("mssql.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("mssql.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("mssql.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create UDT which is actually alias for varchar(11) @@ -287,20 +293,19 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } - + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE postActionTest"); - stmt.execute("DROP TABLE dbActionTest"); - stmt.execute("DROP TABLE MY_DEST_TABLE"); - stmt.execute("DROP TYPE SSN"); - stmt.execute("DROP TYPE BIG_UDT"); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE"), + () -> stmt.execute("DROP TYPE SSN"), + () -> stmt.execute("DROP TYPE BIG_UDT")), LOGGER); + + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerPluginTestSuite.java b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerPluginTestSuite.java index 1434d77dd..74e931f7b 100644 --- a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerPluginTestSuite.java +++ b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerPluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.mssql; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ SqlServerPostActionTestRun.class }) public class SqlServerPluginTestSuite extends SqlServerPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/mysql-plugin/pom.xml b/mysql-plugin/pom.xml index dc57a3618..46a0ac099 100644 --- a/mysql-plugin/pom.xml +++ b/mysql-plugin/pom.xml @@ -61,7 +61,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -72,6 +72,12 @@ cdap-api provided + + mysql + mysql-connector-java + 8.0.19 + test + org.jetbrains annotations @@ -79,36 +85,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlPluginTestBase.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlPluginTestBase.java index 544b2fe47..2d83c9d3d 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlPluginTestBase.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlPluginTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -34,6 +33,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Connection; @@ -47,17 +48,20 @@ import java.sql.Types; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; import javax.sql.rowset.serial.SerialBlob; public abstract class MysqlPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(MysqlPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); - protected static final String DRIVER_CLASS = "com.mysql.jdbc.Driver"; + protected static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; protected static final String JDBC_DRIVER_NAME = "mysql"; protected static String connectionUrl; @@ -65,8 +69,8 @@ public abstract class MysqlPluginTestBase extends DatabasePluginTestBase { protected static final int PRECISION = 10; protected static final int SCALE = 6; protected static final ZoneId UTC_ZONE = ZoneId.ofOffset("UTC", ZoneOffset.UTC); - protected static boolean tearDown = true; private static int startCount; + protected static final Map BASE_PROPS = new HashMap<>(); @ClassRule public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false); @@ -77,20 +81,13 @@ public abstract class MysqlPluginTestBase extends DatabasePluginTestBase { YEAR = calendar.get(Calendar.YEAR); } - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("mysql.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("mysql.port", "3306")) - .put(ConnectionConfig.DATABASE, System.getProperty("mysql.database", "mydb")) - .put(ConnectionConfig.USER, System.getProperty("mysql.username", "root")) - .put(ConnectionConfig.PASSWORD, System.getProperty("mysql.password", "123Qwe123")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); @@ -118,6 +115,15 @@ public static void setupTest() throws Exception { prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("mysql.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("mysql.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("mysql.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("mysql.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("mysql.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create a table that the action will truncate at the end of the run @@ -245,18 +251,18 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE postActionTest"); - stmt.execute("DROP TABLE dbActionTest"); - stmt.execute("DROP TABLE MY_DEST_TABLE"); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlPluginTestSuite.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlPluginTestSuite.java index 89866ec7f..7d6bcd49d 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlPluginTestSuite.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlPluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.mysql; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ MysqlPostActionTestRun.class }) public class MysqlPluginTestSuite extends MysqlPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/netezza-plugin/pom.xml b/netezza-plugin/pom.xml index 5a6609af6..b3abce6cd 100644 --- a/netezza-plugin/pom.xml +++ b/netezza-plugin/pom.xml @@ -54,7 +54,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 io.cdap.plugin @@ -79,36 +79,8 @@ test - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/netezza-plugin/src/test/java/io/cdap/plugin/netezza/NetezzaPluginTestBase.java b/netezza-plugin/src/test/java/io/cdap/plugin/netezza/NetezzaPluginTestBase.java index 74396b7c4..5c63e97b6 100644 --- a/netezza-plugin/src/test/java/io/cdap/plugin/netezza/NetezzaPluginTestBase.java +++ b/netezza-plugin/src/test/java/io/cdap/plugin/netezza/NetezzaPluginTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -34,6 +33,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.math.MathContext; @@ -46,23 +47,26 @@ import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; public abstract class NetezzaPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(NetezzaPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); protected static final String JDBC_DRIVER_NAME = "netezza"; + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static final int YEAR; protected static final int PRECISION = 16; protected static final int SCALE = 6; - protected static boolean tearDown = true; private static int startCount; @ClassRule @@ -74,20 +78,12 @@ public abstract class NetezzaPluginTestBase extends DatabasePluginTestBase { YEAR = calendar.get(Calendar.YEAR); } - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("netezza.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("netezza.port", "5480")) - .put(ConnectionConfig.DATABASE, System.getProperty("netezza.database", "mydb")) - .put(ConnectionConfig.USER, System.getProperty("netezza.username", "admin")) - .put(ConnectionConfig.PASSWORD, System.getProperty("netezza.password", "password")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); - @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); @@ -116,6 +112,15 @@ public static void setupTest() throws Exception { prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("netezza.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("netezza.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("netezza.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("netezza.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("netezza.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create a table that the action will truncate at the end of the run @@ -219,18 +224,16 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } - + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE post_action_test"); - stmt.execute("DROP TABLE db_action_test"); - stmt.execute("DROP TABLE MY_DEST_TABLE"); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE post_action_test"), + () -> stmt.execute("DROP TABLE db_action_test"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/netezza-plugin/src/test/java/io/cdap/plugin/netezza/NetezzaPluginTestSuite.java b/netezza-plugin/src/test/java/io/cdap/plugin/netezza/NetezzaPluginTestSuite.java index 8a2197329..38bce5731 100644 --- a/netezza-plugin/src/test/java/io/cdap/plugin/netezza/NetezzaPluginTestSuite.java +++ b/netezza-plugin/src/test/java/io/cdap/plugin/netezza/NetezzaPluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.netezza; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ NetezzaPostActionTestRun.class }) public class NetezzaPluginTestSuite extends NetezzaPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/oracle-plugin/pom.xml b/oracle-plugin/pom.xml index b42486277..9c740a580 100644 --- a/oracle-plugin/pom.xml +++ b/oracle-plugin/pom.xml @@ -61,7 +61,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -77,6 +77,12 @@ cdap-api provided + + com.oracle.database.jdbc + ojdbc8 + 21.1.0.0 + test + org.jetbrains annotations @@ -84,36 +90,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OraclePluginTestBase.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OraclePluginTestBase.java index 335e80a96..f1bb45618 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OraclePluginTestBase.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OraclePluginTestBase.java @@ -17,7 +17,6 @@ package io.cdap.plugin.oracle; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -33,6 +32,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Clob; @@ -46,13 +47,16 @@ import java.sql.Timestamp; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.TimeZone; public abstract class OraclePluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(OraclePluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); @@ -65,6 +69,7 @@ public abstract class OraclePluginTestBase extends DatabasePluginTestBase { protected static final String YOUR_TABLE = "your_table"; protected static final String MY_TABLE_FOR_LONG = "my_table_long"; protected static final String MY_DEST_TABLE_FOR_LONG = "MY_DEST_LONG"; + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static final int YEAR; @@ -73,7 +78,6 @@ public abstract class OraclePluginTestBase extends DatabasePluginTestBase { protected static final int SCALE = 6; protected static final ZoneId UTC = ZoneId.ofOffset("UTC", ZoneOffset.UTC); - protected static boolean tearDown = true; private static int startCount; @ClassRule @@ -85,16 +89,16 @@ public abstract class OraclePluginTestBase extends DatabasePluginTestBase { YEAR = calendar.get(Calendar.YEAR); } - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("oracle.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("oracle.port", "1521")) - .put(ConnectionConfig.DATABASE, System.getProperty("oracle.database", "cdap")) - .put(ConnectionConfig.USER, System.getProperty("oracle.username", "SYSTEM")) - .put(ConnectionConfig.PASSWORD, System.getProperty("oracle.password", "123Qwe123")) - .put(OracleConstants.CONNECTION_TYPE, System.getProperty("oracle.connectionType", "sid")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .put(OracleConstants.DEFAULT_BATCH_VALUE, "10") - .build(); + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("oracle.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("oracle.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("oracle.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("oracle.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("oracle.password")); + BASE_PROPS.put(OracleConstants.CONNECTION_TYPE, getPropertyOrSkip("oracle.connectionType")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + BASE_PROPS.put(OracleConstants.DEFAULT_BATCH_VALUE, "10"); + } @BeforeClass public static void setupTest() throws Exception { @@ -102,6 +106,8 @@ public static void setupTest() throws Exception { return; } + getProperties(); + setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); addPluginArtifact(NamespaceId.DEFAULT.artifact(JDBC_DRIVER_NAME, "1.0.0"), @@ -329,22 +335,23 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } + public static void tearDownDB() { String dropTableFormat = "DROP TABLE %s"; try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute(String.format(dropTableFormat, MY_TABLE)); - stmt.execute(String.format(dropTableFormat, MY_TABLE_FOR_LONG)); - stmt.execute(String.format(dropTableFormat, YOUR_TABLE)); - stmt.execute(String.format(dropTableFormat, "postActionTest")); - stmt.execute(String.format(dropTableFormat, "dbActionTest")); - stmt.execute(String.format(dropTableFormat, MY_DEST_TABLE)); - stmt.execute(String.format(dropTableFormat, MY_DEST_TABLE_FOR_LONG)); + executeCleanup(Arrays.asList(() -> stmt.execute(String.format(dropTableFormat, MY_TABLE)), + () -> stmt.execute(String.format(dropTableFormat, MY_TABLE_FOR_LONG)), + () -> stmt.execute(String.format(dropTableFormat, YOUR_TABLE)), + () -> stmt.execute(String.format(dropTableFormat, "postActionTest")), + () -> stmt.execute(String.format(dropTableFormat, "dbActionTest")), + () -> stmt.execute(String.format(dropTableFormat, MY_DEST_TABLE)), + () -> stmt.execute(String.format(dropTableFormat, MY_DEST_TABLE_FOR_LONG))), + LOGGER); + + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OraclePluginTestSuite.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OraclePluginTestSuite.java index f2d6fcd1b..d5d3ebc81 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OraclePluginTestSuite.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OraclePluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.oracle; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ OraclePostActionTestRun.class }) public class OraclePluginTestSuite extends OraclePluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/pom.xml b/pom.xml index 719ec8428..ba13b2dc4 100644 --- a/pom.xml +++ b/pom.xml @@ -60,12 +60,12 @@ true UTF-8 - 6.1.1 - 2.3.4 + 6.5.0-SNAPSHOT + 2.8.0-SNAPSHOT 13.0.1 2.3.0 2.2.4 - 4.11 + 4.13 2.0.1 3.3.3 1.7.5 @@ -129,7 +129,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 ${cdap.version} test @@ -332,6 +332,12 @@ + + org.mockito + mockito-core + ${mockito.version} + test + @@ -482,6 +488,33 @@ + + org.apache.felix + maven-bundle-plugin + 5.1.2 + true + + + <_exportcontents> + io.cdap.plugin.*; + org.apache.commons.lang; + org.apache.commons.logging.*; + org.codehaus.jackson.* + + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + diff --git a/postgresql-plugin/pom.xml b/postgresql-plugin/pom.xml index f8f22c417..ab0c44e1e 100644 --- a/postgresql-plugin/pom.xml +++ b/postgresql-plugin/pom.xml @@ -67,7 +67,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -85,36 +85,8 @@ compile - - - org.apache.felix - maven-bundle-plugin - 3.3.0 - true - - - <_exportcontents> - io.cdap.plugin.*; - org.apache.commons.lang; - org.apache.commons.logging.*; - org.codehaus.jackson.* - - *;inline=false;scope=compile - true - lib - - - - - package - - bundle - - - - io.cdap cdap-maven-plugin diff --git a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresPluginTestBase.java b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresPluginTestBase.java index 517fb0316..8df0a00a7 100644 --- a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresPluginTestBase.java +++ b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresPluginTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -37,6 +36,8 @@ import org.postgresql.Driver; import org.postgresql.util.PGTime; import org.postgresql.util.PGobject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Connection; @@ -51,23 +52,26 @@ import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; public abstract class PostgresPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(PostgresPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); protected static final String JDBC_DRIVER_NAME = "postrgesql"; + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static final int YEAR; protected static final int PRECISION = 10; protected static final int SCALE = 6; - protected static boolean tearDown = true; protected static final OffsetDateTime OFFSET_TIME = OffsetDateTime.of( 1992, 3, @@ -90,20 +94,12 @@ public abstract class PostgresPluginTestBase extends DatabasePluginTestBase { } - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("postgresql.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("postgresql.port", "5432")) - .put(ConnectionConfig.DATABASE, System.getProperty("postgresql.database", "mydb")) - .put(ConnectionConfig.USER, System.getProperty("postgresql.username", "postgres")) - .put(ConnectionConfig.PASSWORD, System.getProperty("postgresql.password", "123Qwe123")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); - @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); @@ -129,6 +125,15 @@ public static void setupTest() throws Exception { prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("postgresql.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("postgresql.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("postgresql.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("postgresql.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("postgresql.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { // create a table that the action will truncate at the end of the run @@ -281,18 +286,17 @@ private static PGobject createPGObject(String type, String value) throws SQLExce } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE \"postActionTest\""); - stmt.execute("DROP TABLE \"dbActionTest\""); - stmt.execute("DROP TABLE \"MY_DEST_TABLE\""); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresPluginTestSuite.java b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresPluginTestSuite.java index 84110a0d4..969970187 100644 --- a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresPluginTestSuite.java +++ b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresPluginTestSuite.java @@ -17,8 +17,6 @@ package io.cdap.plugin.postgres; import io.cdap.cdap.common.test.TestSuite; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -34,14 +32,4 @@ PostgresPostActionTestRun.class }) public class PostgresPluginTestSuite extends PostgresPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } } diff --git a/saphana-plugin/pom.xml b/saphana-plugin/pom.xml index 3882ea845..9a954e32f 100644 --- a/saphana-plugin/pom.xml +++ b/saphana-plugin/pom.xml @@ -57,7 +57,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -81,8 +81,6 @@ org.apache.felix maven-bundle-plugin - 3.3.0 - true <_exportcontents> @@ -96,14 +94,6 @@ lib - - - package - - bundle - - - io.cdap diff --git a/teradata-plugin/pom.xml b/teradata-plugin/pom.xml index 021fb1cfc..24054936d 100644 --- a/teradata-plugin/pom.xml +++ b/teradata-plugin/pom.xml @@ -62,7 +62,7 @@ io.cdap.cdap - cdap-data-pipeline + cdap-data-pipeline2_2.11 junit @@ -86,8 +86,6 @@ org.apache.felix maven-bundle-plugin - 3.3.0 - true <_exportcontents> @@ -100,14 +98,6 @@ lib - - - package - - bundle - - - io.cdap diff --git a/teradata-plugin/src/test/java/io/cdap/plugin/teradata/TeradataPluginTestBase.java b/teradata-plugin/src/test/java/io/cdap/plugin/teradata/TeradataPluginTestBase.java index 99c5caf97..6e70ff338 100644 --- a/teradata-plugin/src/test/java/io/cdap/plugin/teradata/TeradataPluginTestBase.java +++ b/teradata-plugin/src/test/java/io/cdap/plugin/teradata/TeradataPluginTestBase.java @@ -17,7 +17,6 @@ package io.cdap.plugin.teradata; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.cdap.cdap.api.artifact.ArtifactSummary; import io.cdap.cdap.api.plugin.PluginClass; @@ -37,6 +36,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.sql.Connection; @@ -49,25 +50,28 @@ import java.sql.Timestamp; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.Arrays; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.TimeZone; public abstract class TeradataPluginTestBase extends DatabasePluginTestBase { + private static final Logger LOGGER = LoggerFactory.getLogger(TeradataPluginTestBase.class); protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0"); protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); protected static final long CURRENT_TS = System.currentTimeMillis(); protected static final String JDBC_DRIVER_NAME = "teradata"; protected static final String DRIVER_CLASS = "com.teradata.jdbc.TeraDriver"; + protected static final Map BASE_PROPS = new HashMap<>(); protected static String connectionUrl; protected static final int YEAR; protected static final int PRECISION = 10; protected static final int SCALE = 6; protected static final ZoneId UTC_ZONE = ZoneId.ofOffset("UTC", ZoneOffset.UTC); - protected static boolean tearDown = true; private static int startCount; @ClassRule @@ -79,21 +83,14 @@ public abstract class TeradataPluginTestBase extends DatabasePluginTestBase { YEAR = calendar.get(Calendar.YEAR); } - protected static final Map BASE_PROPS = ImmutableMap.builder() - .put(ConnectionConfig.HOST, System.getProperty("teradata.host", "localhost")) - .put(ConnectionConfig.PORT, System.getProperty("teradata.port", "1025")) - .put(ConnectionConfig.DATABASE, System.getProperty("teradata.database", "mydb")) - .put(ConnectionConfig.USER, System.getProperty("teradata.username", "test")) - .put(ConnectionConfig.PASSWORD, System.getProperty("teradata.password", "test")) - .put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME) - .build(); - @BeforeClass public static void setupTest() throws Exception { if (startCount++ > 0) { return; } + getProperties(); + setupBatchArtifacts(DATAPIPELINE_ARTIFACT_ID, DataPipelineApp.class); addPluginArtifact(NamespaceId.DEFAULT.artifact(JDBC_DRIVER_NAME, "1.0.0"), @@ -124,11 +121,19 @@ public static void setupTest() throws Exception { "" ); Connection conn = createConnection(); - createTestTables(conn); prepareTestData(conn); } + private static void getProperties() { + BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("teradata.host")); + BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("teradata.port")); + BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("teradata.database")); + BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("teradata.username")); + BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("teradata.password")); + BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME); + } + protected static void createTestTables(Connection conn) throws SQLException { try (Statement stmt = conn.createStatement()) { String columns = "ID INTEGER NOT NULL, " + @@ -253,18 +258,17 @@ public static Connection createConnection() { } @AfterClass - public static void tearDownDB() throws SQLException { - if (!tearDown) { - return; - } - + public static void tearDownDB() { try (Connection conn = createConnection(); Statement stmt = conn.createStatement()) { - stmt.execute("DROP TABLE my_table"); - stmt.execute("DROP TABLE your_table"); - stmt.execute("DROP TABLE MY_DEST_TABLE"); - stmt.execute("DROP TABLE postActionTest"); - stmt.execute("DROP TABLE dbActionTest"); + executeCleanup(Arrays.asList(() -> stmt.execute("DROP TABLE my_table"), + () -> stmt.execute("DROP TABLE your_table"), + () -> stmt.execute("DROP TABLE MY_DEST_TABLE"), + () -> stmt.execute("DROP TABLE postActionTest"), + () -> stmt.execute("DROP TABLE dbActionTest") + ), LOGGER); + } catch (Exception e) { + LOGGER.warn("Fail to tear down.", e); } } } diff --git a/teradata-plugin/src/test/java/io/cdap/plugin/teradata/TeradataPluginTestSuite.java b/teradata-plugin/src/test/java/io/cdap/plugin/teradata/TeradataPluginTestSuite.java index ac48296e9..ef452775f 100644 --- a/teradata-plugin/src/test/java/io/cdap/plugin/teradata/TeradataPluginTestSuite.java +++ b/teradata-plugin/src/test/java/io/cdap/plugin/teradata/TeradataPluginTestSuite.java @@ -21,8 +21,6 @@ import io.cdap.plugin.teradata.postaction.TeradataPostActionTestRun; import io.cdap.plugin.teradata.sink.TeradataSinkTestRun; import io.cdap.plugin.teradata.source.TeradataSourceTestRun; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -37,14 +35,4 @@ TeradataPostActionTestRun.class }) public class TeradataPluginTestSuite extends TeradataPluginTestBase { - - @BeforeClass - public static void setup() { - tearDown = false; - } - - @AfterClass - public static void tearDown() throws Exception { - tearDown = true; - } }