Skip to content

Commit

Permalink
pgDump azureDatbaseUtils command
Browse files Browse the repository at this point in the history
  • Loading branch information
mspector committed Oct 26, 2023
1 parent fe726d7 commit c64344a
Show file tree
Hide file tree
Showing 11 changed files with 459 additions and 4 deletions.
11 changes: 9 additions & 2 deletions azureDatabaseUtils/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,21 @@
# This file is expected to be part of source control.
ch.qos.logback:logback-classic:1.4.8=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
ch.qos.logback:logback-core:1.4.8=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-core-http-netty:1.13.4=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-core:1.40.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-core-http-netty:1.13.5=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-core:1.41.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-identity-extensions:1.1.4=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-identity:1.9.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-json:1.0.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-storage-blob:12.23.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-storage-common:12.22.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.azure:azure-storage-internal-avro:12.8.0=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.core:jackson-annotations:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.core:jackson-core:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.core:jackson-databind:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.jackson:jackson-bom:2.15.2=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.fasterxml.woodstox:woodstox-core:6.5.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
com.github.docker-java:docker-java-api:3.2.13=testCompileClasspath,testRuntimeClasspath
com.github.docker-java:docker-java-transport-zerodep:3.2.13=testCompileClasspath,testRuntimeClasspath
com.github.docker-java:docker-java-transport:3.2.13=testCompileClasspath,testRuntimeClasspath
Expand Down Expand Up @@ -75,6 +80,7 @@ org.apache.logging.log4j:log4j-to-slf4j:2.20.0=compileClasspath,productionRuntim
org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath
org.assertj:assertj-core:3.24.2=testCompileClasspath,testRuntimeClasspath
org.checkerframework:checker-qual:3.5.0=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
org.codehaus.woodstox:stax2-api:4.2.1=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.dom4j:dom4j:2.1.3=spotbugs
org.hamcrest:hamcrest-core:2.2=testCompileClasspath,testRuntimeClasspath
org.hamcrest:hamcrest:2.2=testCompileClasspath,testRuntimeClasspath
Expand Down Expand Up @@ -107,6 +113,7 @@ org.ow2.asm:asm:9.0=spotbugs
org.ow2.asm:asm:9.2=jacocoAnt
org.ow2.asm:asm:9.3=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.postgresql:postgresql:42.3.3=productionRuntimeClasspath,runtimeClasspath,testRuntimeClasspath
org.postgresql:postgresql:42.6.0=compileClasspath,testCompileClasspath
org.reactivestreams:reactive-streams:1.0.4=compileClasspath,productionRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
org.rnorth.duct-tape:duct-tape:1.0.8=testCompileClasspath,testRuntimeClasspath
org.skyscreamer:jsonassert:1.5.1=testCompileClasspath,testRuntimeClasspath
Expand Down
2 changes: 2 additions & 0 deletions azureDatabaseUtils/gradle/dependencies.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ dependencies {
// Azure dependencies
implementation 'com.azure:azure-identity:1.9.1'
implementation 'com.azure:azure-identity-extensions:1.1.4'
implementation 'org.postgresql:postgresql'
implementation 'com.azure:azure-storage-blob:12.23.0'

runtimeOnly group: "org.postgresql", name: "postgresql", version: "42.3.3"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
package bio.terra.workspace.azureDatabaseUtils.database;

import bio.terra.workspace.azureDatabaseUtils.process.LocalProcessLauncher;
import bio.terra.workspace.azureDatabaseUtils.storage.BlobStorage;
import bio.terra.workspace.azureDatabaseUtils.validation.Validator;
import java.util.Set;
import com.azure.identity.extensions.jdbc.postgresql.AzurePostgresqlAuthenticationPlugin;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import org.postgresql.plugin.AuthenticationRequestType;
import org.postgresql.util.PSQLException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -13,14 +20,16 @@ public class DatabaseService {
private static final Logger logger = LoggerFactory.getLogger(DatabaseService.class);
private final DatabaseDao databaseDao;
private final Validator validator;
private final BlobStorage storage;

@Value("${spring.datasource.username}")
private String datasourceUserName;

@Autowired
public DatabaseService(DatabaseDao databaseDao, Validator validator) {
public DatabaseService(DatabaseDao databaseDao, Validator validator, BlobStorage blobStorage) {
this.databaseDao = databaseDao;
this.validator = validator;
this.storage = blobStorage;
}

public void createDatabaseWithDbRole(String newDbName) {
Expand Down Expand Up @@ -80,4 +89,97 @@ public void restoreNamespaceRoleAccess(String namespaceRole) {

databaseDao.restoreLoginPrivileges(namespaceRole);
}

public void pgDump(
String sourceDbName,
String sourceDbHost,
String sourceDbPort,
String sourceDbUser,
String pgDumpFilename,
String destinationWorkspaceId,
String blobstorageDetails) {
logger.info("running DatabaseService.pgDump against {}", sourceDbName);
logger.info("destinationWorkspaceId: {}", destinationWorkspaceId);
try {
// Grant the database role (sourceDbName) to the workspace identity (sourceDbUser).
// In theory, we should be revoking this role after the operation is complete.
// We are choosing to *not* revoke this role for now, because:
// (1) we could run into concurrency issues if multiple users attempt to clone the same
// workspace at once;
// (2) the workspace identity can grant itself access at any time, so revoking the role
// doesn't protect us.
databaseDao.grantRole(sourceDbUser, sourceDbName);

List<String> commandList =
generateCommandList("pg_dump", sourceDbName, sourceDbHost, sourceDbPort, sourceDbUser);
Map<String, String> envVars = Map.of("PGPASSWORD", determinePassword());
LocalProcessLauncher localProcessLauncher = new LocalProcessLauncher();
localProcessLauncher.launchProcess(commandList, envVars);

storage.streamOutputToBlobStorage(
localProcessLauncher.getInputStream(),
pgDumpFilename,
destinationWorkspaceId,
blobstorageDetails);

String output = checkForError(localProcessLauncher);
logger.info("pg_dump output: {}", output);

} catch (PSQLException ex) {
logger.error("process error: {}", ex.getMessage());
}
}

public List<String> generateCommandList(
String pgDumpPath, String sourceDbName, String dbHost, String dbPort, String dbUser) {
Map<String, String> command = new LinkedHashMap<>();

command.put(pgDumpPath, null);
command.put("-b", null);
command.put("-h", dbHost);
command.put("-p", dbPort);
command.put("-U", dbUser);
command.put("-d", sourceDbName);

List<String> commandList = new ArrayList<>();
for (Map.Entry<String, String> entry : command.entrySet()) {
commandList.add(entry.getKey());
if (entry.getValue() != null) {
commandList.add(entry.getValue());
}
}
commandList.add("-v");
commandList.add("-w");

return commandList;
}

private String checkForError(LocalProcessLauncher localProcessLauncher) {
// materialize only the first 1024 bytes of the error stream to ensure we don't DoS ourselves
int errorLimit = 1024;

int exitCode = localProcessLauncher.waitForTerminate();
if (exitCode != 0) {
InputStream errorStream =
localProcessLauncher.getOutputForProcess(LocalProcessLauncher.Output.ERROR);
try {
String error = new String(errorStream.readNBytes(errorLimit)).trim();
logger.error("process error: {}", error);
return error;
} catch (IOException e) {
logger.warn(
"process failed with exit code {}, but encountered an exception reading the error output: {}",
exitCode,
e.getMessage());
return "Unknown error";
}
}
return "";
}

private String determinePassword() throws PSQLException {
return new String(
new AzurePostgresqlAuthenticationPlugin(new Properties())
.getPassword(AuthenticationRequestType.CLEARTEXT_PASSWORD));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package bio.terra.workspace.azureDatabaseUtils.process;

/**
* Custom exception class for system or internal exceptions. These represent errors that the user
* cannot fix.
*/
public class LaunchProcessException extends RuntimeException {
/**
* Constructs an exception with the given message. The cause is set to null.
*
* @param message description of error that may help with debugging
*/
public LaunchProcessException(String message) {
super(message);
}

/**
* Constructs an exception with the given message and cause.
*
* @param message description of error that may help with debugging
* @param cause underlying exception that can be logged for debugging purposes
*/
public LaunchProcessException(String message, Throwable cause) {
super(message, cause);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package bio.terra.workspace.azureDatabaseUtils.process;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;

/** This class provides utility methods for launching local child processes. */
public class LocalProcessLauncher {
private Process process;

public enum Output {
OUT,
ERROR
}

/**
* Executes a command in a separate process from the current working directory (i.e. the same
* place as this Java process is running).
*
* @param command the command and arguments to execute
* @param envVars the environment variables to set or overwrite if already defined
*/
public void launchProcess(List<String> command, Map<String, String> envVars) {
launchProcess(command, envVars, null);
}

/**
* Executes a command in a separate process from the given working directory, with the given
* environment variables set beforehand.
*
* @param command the command and arguments to execute
* @param envVars the environment variables to set or overwrite if already defined
* @param workingDirectory the working directory to launch the process from
*/
public void launchProcess(
List<String> command, Map<String, String> envVars, Path workingDirectory) {
// build and run process from the specified working directory
ProcessBuilder procBuilder = new ProcessBuilder(command);
if (workingDirectory != null) {
procBuilder.directory(workingDirectory.toFile());
}
if (envVars != null) {
Map<String, String> procEnvVars = procBuilder.environment();
procEnvVars.putAll(envVars);
}

try {
process = procBuilder.start();
} catch (IOException ioEx) {
throw new LaunchProcessException("Error launching local process", ioEx);
}
}

/**
* Stream standard out/err from the child process to the CLI console.
*
* @param type specifies which process stream to get data from
*/
public InputStream getOutputForProcess(Output type) {
if (type == Output.ERROR) {
return process.getErrorStream();
}

return process.getInputStream();
}

/** Block until the child process terminates, then return its exit code. */
public int waitForTerminate() {
try {
return process.waitFor();
} catch (InterruptedException intEx) {
Thread.currentThread().interrupt();
throw new LaunchProcessException("Error waiting for child process to terminate", intEx);
}
}

/** Get stdout input stream from the child process. */
public InputStream getInputStream() {
return process.getInputStream();
}

/** Get stdin output stream from the child process. */
public OutputStream getOutputStream() {
return process.getOutputStream();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package bio.terra.workspace.azureDatabaseUtils.runners;

import bio.terra.workspace.azureDatabaseUtils.database.DatabaseService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;

@Profile("PgDumpDatabase")
@Component
public class PgDumpDatabaseRunner implements ApplicationRunner {

@Value("${env.db.connectToDatabase}")
private String sourceDbName;

@Value("${env.db.url}")
private String sourceDbHost;

@Value("${env.db.port}")
private String sourceDbPort;

@Value("${env.db.user}")
private String sourceDbUser;

@Value("${env.params.dumpfileName}")
private String dumpfileName;

@Value("${env.params.destinationWorkspaceId}")
private String destinationWorkspaceId;

@Value("${env.params.blobstorageDetails}")
private String blobstorageDetails;

private final DatabaseService databaseService;

public PgDumpDatabaseRunner(DatabaseService databaseService) {
this.databaseService = databaseService;
}

@Override
public void run(ApplicationArguments args) {
// should I reuse `newDbName`, or create a new param `cloneDbName`?
databaseService.pgDump(
sourceDbName,
sourceDbHost,
sourceDbPort,
sourceDbUser,
dumpfileName,
destinationWorkspaceId,
blobstorageDetails);
}
}
Loading

0 comments on commit c64344a

Please sign in to comment.