Skip to content

Commit

Permalink
migrate to CDAP 6.5.0 and fix failed test suits (#120)
Browse files Browse the repository at this point in the history 
* migrate to CDAP 6.5.0 and fix failed test suits
  • Loading branch information
@seanzhougoogle
seanzhougoogle authored Jun 28, 2021
1 parent c652e04 commit 5045a4b
Show file tree
Hide file tree
Showing 41 changed files with 403 additions and 787 deletions.
36 changes: 7 additions & 29 deletions aurora-mysql-plugin/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,18 @@
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-pipeline</artifactId>
<artifactId>cdap-data-pipeline2_2.11</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-api</artifactId>
Expand All @@ -79,36 +85,8 @@
<scope>compile</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<version>3.3.0</version>
<extensions>true</extensions>
<configuration>
<instructions>
<_exportcontents>
io.cdap.plugin.*;
org.apache.commons.lang;
org.apache.commons.logging.*;
org.codehaus.jackson.*
</_exportcontents>
<Embed-Dependency>*;inline=false;scope=compile</Embed-Dependency>
<Embed-Transitive>true</Embed-Transitive>
<Embed-Directory>lib</Embed-Directory>
</instructions>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>bundle</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>io.cdap</groupId>
<artifactId>cdap-maven-plugin</artifactId>
Expand Down
56 changes: 25 additions & 31 deletions aurora-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import io.cdap.cdap.api.artifact.ArtifactSummary;
import io.cdap.cdap.api.plugin.PluginClass;
Expand All @@ -38,6 +37,8 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.math.BigDecimal;
import java.sql.Connection;
Expand All @@ -49,44 +50,40 @@
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.TimeZone;
import javax.sql.rowset.serial.SerialBlob;

public abstract class AuroraMysqlPluginTestBase extends DatabasePluginTestBase {
private static final Logger LOGGER = LoggerFactory.getLogger(AuroraMysqlPluginTestBase.class);
protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0");
protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0");
protected static final long CURRENT_TS = System.currentTimeMillis();
protected static final String DRIVER_CLASS = "com.mysql.jdbc.Driver";
protected static final String JDBC_DRIVER_NAME = "mysql";
protected static final Map<String, String> BASE_PROPS = new HashMap<>();

protected static String connectionUrl;
protected static int year;
protected static final int PRECISION = 10;
protected static final int SCALE = 6;
protected static boolean tearDown = true;
private static int startCount;

@ClassRule
public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false);

protected static final Map<String, String> BASE_PROPS = ImmutableMap.<String, String>builder()
.put(ConnectionConfig.HOST, getPropertyOrFail("auroraMysql.clusterEndpoint"))
.put(ConnectionConfig.PORT, getPropertyOrFail("auroraMysql.port"))
.put(ConnectionConfig.DATABASE, getPropertyOrFail("auroraMysql.database"))
.put(ConnectionConfig.USER, getPropertyOrFail("auroraMysql.username"))
.put(ConnectionConfig.PASSWORD, getPropertyOrFail("auroraMysql.password"))
.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME)
.build();

@BeforeClass
public static void setupTest() throws Exception {
if (startCount++ > 0) {
return;
}

getProperties();

Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date(CURRENT_TS));
year = calendar.get(Calendar.YEAR);
Expand Down Expand Up @@ -118,6 +115,15 @@ public static void setupTest() throws Exception {
prepareTestData(conn);
}

private static void getProperties() {
BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("auroraMysql.clusterEndpoint"));
BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("auroraMysql.port"));
BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("auroraMysql.database"));
BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("auroraMysql.username"));
BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("auroraMysql.password"));
BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME);
}

protected static void createTestTables(Connection conn) throws SQLException {
try (Statement stmt = conn.createStatement()) {
// create a table that the action will truncate at the end of the run
Expand Down Expand Up @@ -228,19 +234,8 @@ private static void populateData(PreparedStatement... stmts) throws SQLException
}
}

private static String getPropertyOrFail(String propertyName) {
String value = System.getProperty(propertyName);

if (value == null) {
throw new IllegalStateException("There is no value for property " + propertyName);
}

return value;
}

public static Connection createConnection() {
try {
Class.forName(DRIVER_CLASS);
return DriverManager.getConnection(connectionUrl, BASE_PROPS.get(ConnectionConfig.USER),
BASE_PROPS.get(ConnectionConfig.PASSWORD));
} catch (Exception e) {
Expand All @@ -249,18 +244,17 @@ public static Connection createConnection() {
}

@AfterClass
public static void tearDownDB() throws SQLException {
if (!tearDown) {
return;
}
public static void tearDownDB() {

try (Connection conn = createConnection();
Statement stmt = conn.createStatement()) {
stmt.execute("DROP TABLE my_table");
stmt.execute("DROP TABLE your_table");
stmt.execute("DROP TABLE postActionTest");
stmt.execute("DROP TABLE dbActionTest");
stmt.execute("DROP TABLE MY_DEST_TABLE");
executeCleanup(Arrays.<Cleanup>asList(() -> stmt.execute("DROP TABLE my_table"),
() -> stmt.execute("DROP TABLE your_table"),
() -> stmt.execute("DROP TABLE postActionTest"),
() -> stmt.execute("DROP TABLE dbActionTest"),
() -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER);
} catch (Exception e) {
LOGGER.warn("Fail to tear down.", e);
}
}
}
12 changes: 0 additions & 12 deletions ...ra-mysql-plugin/src/test/java/io/cdap/plugin/aurora/mysql/AuroraMysqlPluginTestSuite.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
package io.cdap.plugin.aurora.mysql;

import io.cdap.cdap.common.test.TestSuite;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;

Expand All @@ -34,14 +32,4 @@
AuroraMysqlPostActionTestRun.class
})
public class AuroraMysqlPluginTestSuite extends AuroraMysqlPluginTestBase {

@BeforeClass
public static void setup() {
tearDown = false;
}

@AfterClass
public static void tearDown() throws Exception {
tearDown = true;
}
}
30 changes: 1 addition & 29 deletions aurora-postgresql-plugin/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-pipeline</artifactId>
<artifactId>cdap-data-pipeline2_2.11</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
Expand All @@ -85,36 +85,8 @@
<scope>compile</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
<version>3.3.0</version>
<extensions>true</extensions>
<configuration>
<instructions>
<_exportcontents>
io.cdap.plugin.*;
org.apache.commons.lang;
org.apache.commons.logging.*;
org.codehaus.jackson.*
</_exportcontents>
<Embed-Dependency>*;inline=false;scope=compile</Embed-Dependency>
<Embed-Transitive>true</Embed-Transitive>
<Embed-Directory>lib</Embed-Directory>
</instructions>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>bundle</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>io.cdap</groupId>
<artifactId>cdap-maven-plugin</artifactId>
Expand Down
56 changes: 25 additions & 31 deletions ...l-plugin/src/test/java/io/cdap/plugin/auroradb/postgres/AuroraPostgresPluginTestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import io.cdap.cdap.api.artifact.ArtifactSummary;
import io.cdap.cdap.api.plugin.PluginClass;
Expand All @@ -35,6 +34,8 @@
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.postgresql.Driver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.math.BigDecimal;
import java.sql.Connection;
Expand All @@ -45,43 +46,39 @@
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.TimeZone;

public abstract class AuroraPostgresPluginTestBase extends DatabasePluginTestBase {
private static final Logger LOGGER = LoggerFactory.getLogger(AuroraPostgresPluginTestBase.class);
protected static final ArtifactId DATAPIPELINE_ARTIFACT_ID = NamespaceId.DEFAULT.artifact("data-pipeline", "3.2.0");
protected static final ArtifactSummary DATAPIPELINE_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0");
protected static final long CURRENT_TS = System.currentTimeMillis();

protected static final String JDBC_DRIVER_NAME = "postrgesql";
protected static final Map<String, String> BASE_PROPS = new HashMap<>();

protected static String connectionUrl;
protected static int year;
protected static final int PRECISION = 10;
protected static final int SCALE = 6;
protected static boolean tearDown = true;
private static int startCount;

@ClassRule
public static final TestConfiguration CONFIG = new TestConfiguration("explore.enabled", false);

protected static final Map<String, String> BASE_PROPS = ImmutableMap.<String, String>builder()
.put(ConnectionConfig.HOST, getPropertyOrFail("auroraPostgresql.clusterEndpoint"))
.put(ConnectionConfig.PORT, getPropertyOrFail("auroraPostgresql.port"))
.put(ConnectionConfig.DATABASE, getPropertyOrFail("auroraPostgresql.database"))
.put(ConnectionConfig.USER, getPropertyOrFail("auroraPostgresql.username"))
.put(ConnectionConfig.PASSWORD, getPropertyOrFail("auroraPostgresql.password"))
.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME)
.build();

@BeforeClass
public static void setupTest() throws Exception {
if (startCount++ > 0) {
return;
}

getProperties();

Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date(CURRENT_TS));
year = calendar.get(Calendar.YEAR);
Expand Down Expand Up @@ -111,6 +108,15 @@ public static void setupTest() throws Exception {
prepareTestData(conn);
}

private static void getProperties() {
BASE_PROPS.put(ConnectionConfig.HOST, getPropertyOrSkip("auroraPostgresql.clusterEndpoint"));
BASE_PROPS.put(ConnectionConfig.PORT, getPropertyOrSkip("auroraPostgresql.port"));
BASE_PROPS.put(ConnectionConfig.DATABASE, getPropertyOrSkip("auroraPostgresql.database"));
BASE_PROPS.put(ConnectionConfig.USER, getPropertyOrSkip("auroraPostgresql.username"));
BASE_PROPS.put(ConnectionConfig.PASSWORD, getPropertyOrSkip("auroraPostgresql.password"));
BASE_PROPS.put(ConnectionConfig.JDBC_PLUGIN_NAME, JDBC_DRIVER_NAME);
}

protected static void createTestTables(Connection conn) throws SQLException {
try (Statement stmt = conn.createStatement()) {
// create a table that the action will truncate at the end of the run
Expand Down Expand Up @@ -189,16 +195,6 @@ private static void populateData(PreparedStatement... stmts) throws SQLException
}
}

private static String getPropertyOrFail(String propertyName) {
String value = System.getProperty(propertyName);

if (value == null) {
throw new IllegalStateException("There is no value for property " + propertyName);
}

return value;
}

public static Connection createConnection() {
try {
Class.forName(Driver.class.getCanonicalName());
Expand All @@ -210,18 +206,16 @@ public static Connection createConnection() {
}

@AfterClass
public static void tearDownDB() throws SQLException {
if (!tearDown) {
return;
}

public static void tearDownDB() {
try (Connection conn = createConnection();
Statement stmt = conn.createStatement()) {
stmt.execute("DROP TABLE my_table");
stmt.execute("DROP TABLE your_table");
stmt.execute("DROP TABLE \"postActionTest\"");
stmt.execute("DROP TABLE \"dbActionTest\"");
stmt.execute("DROP TABLE \"MY_DEST_TABLE\"");
executeCleanup(Arrays.<Cleanup>asList(() -> stmt.execute("DROP TABLE my_table"),
() -> stmt.execute("DROP TABLE your_table"),
() -> stmt.execute("DROP TABLE postActionTest"),
() -> stmt.execute("DROP TABLE dbActionTest"),
() -> stmt.execute("DROP TABLE MY_DEST_TABLE")), LOGGER);
} catch (Exception e) {
LOGGER.warn("Fail to tear down.", e);
}
}
}
Loading

0 comments on commit 5045a4b

Please sign in to comment.