Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cloud upload — rough/draft code #234

Draft
wants to merge 1 commit into
base: dev
Choose a base branch
from
Draft

Cloud upload — rough/draft code #234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions occurrence-cli/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
</distributionManagement>

<properties>
<azure-storage-blob.version>12.4.0</azure-storage-blob.version>
<azure-storage-queue.version>12.3.0</azure-storage-queue.version>
<azure-storage-file-share.version>12.2.0</azure-storage-file-share.version>
<azure-storage-file-datalake.version>12.0.0-preview.6</azure-storage-file-datalake.version>
<google-cloud.version>16.1.0</google-cloud.version>
<google-cloud-storage.verison>1.113.4</google-cloud-storage.verison>
<!-- have to match gbif-cli version -->
<guava.version>23.0</guava.version>
<wagon-ssh.version>2.4</wagon-ssh.version>
Expand Down Expand Up @@ -115,6 +121,18 @@
</repository>
</repositories>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>libraries-bom</artifactId>
<version>${google-cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<dependencies>
<dependency>
<groupId>com.beust</groupId>
Expand Down Expand Up @@ -256,6 +274,11 @@
<artifactId>variables</artifactId>
</dependency>

<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>

<!-- false positive from maven dep:analyze -->
<dependency>
<groupId>org.kohsuke.metainf-services</groupId>
Expand Down Expand Up @@ -323,6 +346,37 @@
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>

<!-- Cloud services -->
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<version>${azure-storage-blob.version}</version>
</dependency>

<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-queue</artifactId>
<version>${azure-storage-queue.version}</version>
</dependency>

<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-file-share</artifactId>
<version>${azure-storage-file-share.version}</version>
</dependency>

<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-file-datalake</artifactId>
<version>${azure-storage-file-datalake.version}</version>
</dependency>

<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>${google-cloud-storage.verison}</version>
</dependency>
</dependencies>

<profiles>
Expand Down
59 changes: 59 additions & 0 deletions occurrence-cli/src/main/java/org/apache/avro/file/ParallelAvroSplitter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package org.apache.avro.file;

import org.apache.avro.generic.GenericContainer;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.io.DatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;

/**
* Tool to split an Avro file into N chunks, preserving the schema.
*
* Usage: ParallelAvroSplitter filename outputFileFormat numberChunks
*/
public class ParallelAvroSplitter {

public static void main(String... args) throws Exception{

InputStream is;
if (args[0].equals("-")) {
is = System.in;
} else {
is = new FileInputStream(args[0]);
}

DatumReader<GenericRecord> datumReader = new GenericDatumReader<>();
ReflectDatumWriter<GenericContainer> rdw = new ReflectDatumWriter<>(GenericContainer.class);

try (DataFileStream<GenericRecord> dfr = new DataFileStream(is, datumReader)) {

int files = Integer.parseInt(args[2]);
RawDataFileWriter<GenericContainer>[] dfws = new RawDataFileWriter[files];
for (int i = 0; i < files; i++) {
FileOutputStream output = new FileOutputStream(String.format(args[1], i));
dfws[i] = new RawDataFileWriter<>(rdw);
dfws[i].setCodec(CodecFactory.deflateCodec(6));
dfws[i].setFlushOnEveryBlock(false);
dfws[i].create(dfr.getSchema(), output);
}

int o = 0;
while (dfr.hasNextBlock()) {
DataFileStream.DataBlock nextBlockRaw = null;
nextBlockRaw = dfr.nextRawBlock(nextBlockRaw);
dfws[o%files].writeRawBlock(nextBlockRaw);

o++;
}

for (RawDataFileWriter<GenericContainer> dfw : dfws) {
dfw.close();
}
}
}
}
Loading