From e962a00a7fd92463486419b847365804f5244ecf Mon Sep 17 00:00:00 2001 From: Victor ADASCALITEI <210908+3Nigma@users.noreply.github.com> Date: Thu, 13 Jan 2022 00:27:32 +0200 Subject: [PATCH 1/8] Initial commit with the logistics in place and the previous S3Client supported. Need to fill in the logic for local-downloads and add tests. --- .../mirror/importer/MirrorProperties.java | 1 + .../config/CloudStorageConfiguration.java | 22 +++- .../CommonDownloaderProperties.java | 1 + .../importer/downloader/Downloader.java | 98 +++-------------- .../importer/downloader/PendingDownload.java | 4 +- .../balance/AccountBalancesDownloader.java | 7 +- .../downloader/client/FileClient.java | 22 ++++ .../client/FileClientWithProperties.java | 37 +++++++ .../downloader/client/LocalFileClient.java | 61 +++++++++++ .../downloader/client/S3FileClient.java | 100 ++++++++++++++++++ .../downloader/event/EventFileDownloader.java | 8 +- .../record/RecordFileDownloader.java | 7 +- .../src/main/resources/addressbook/local | Bin 0 -> 887 bytes .../downloader/AbstractDownloaderTest.java | 8 +- .../AccountBalancesDownloaderTest.java | 4 +- .../event/EventFileDownloaderTest.java | 2 +- .../AbstractRecordFileDownloaderTest.java | 2 +- 17 files changed, 275 insertions(+), 109 deletions(-) create mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClient.java create mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java create mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/LocalFileClient.java create mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/S3FileClient.java create mode 100644 hedera-mirror-importer/src/main/resources/addressbook/local diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/MirrorProperties.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/MirrorProperties.java index 40d16bb3914..7bf80fe3e2d 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/MirrorProperties.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/MirrorProperties.java @@ -71,6 +71,7 @@ public class MirrorProperties { @RequiredArgsConstructor public enum HederaNetwork { DEMO("hedera-demo-streams", true), + LOCAL("/opt/hedera/services/data", true), MAINNET("hedera-mainnet-streams", false), TESTNET("hedera-stable-testnet-streams-2020-08-27", false), PREVIEWNET("hedera-preview-testnet-streams", false), diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java index 84413284d42..b042723f6d5 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java @@ -22,6 +22,11 @@ import java.net.URI; import java.time.Duration; + +import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.LocalFileClient; +import com.hedera.mirror.importer.downloader.client.S3FileClient; + import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.StringUtils; @@ -67,9 +72,18 @@ AwsCredentialsProvider awsCredentialsProvider() { return DefaultCredentialsProvider.create(); } + @Bean + @ConditionalOnProperty(prefix = "hedera.mirror.importer.downloader", name = "cloudProvider", havingValue = "FS") + public FileClientWithProperties.Builder localStorageClientBuilder() { + log.info("Configured to download from FileSystem having base-path '{}'", + downloaderProperties.getBucketName()); + + return new LocalFileClient.Builder(); + } + @Bean @ConditionalOnProperty(prefix = "hedera.mirror.importer.downloader", name = "cloudProvider", havingValue = "GCP") - S3AsyncClient gcpCloudStorageClient() { + public FileClientWithProperties.Builder gcpCloudStorageClientBuilder() { log.info("Configured to download from GCP with bucket name '{}'", downloaderProperties.getBucketName()); // Any valid region for aws client. Ignored by GCP. S3AsyncClientBuilder clientBuilder = asyncClientBuilder("us-east-1") @@ -85,13 +99,13 @@ public SdkHttpRequest modifyHttpRequest( } })); } - return clientBuilder.build(); + return new S3FileClient.Builder(clientBuilder.build()); } @Bean @ConditionalOnProperty(prefix = "hedera.mirror.importer.downloader", name = "cloudProvider", havingValue = "S3", matchIfMissing = true) - public S3AsyncClient s3CloudStorageClient() { + public FileClientWithProperties.Builder s3CloudStorageClientBuilder() { log.info("Configured to download from S3 in region {} with bucket name '{}'", downloaderProperties.getRegion(), downloaderProperties.getBucketName()); S3AsyncClientBuilder clientBuilder = asyncClientBuilder( @@ -101,7 +115,7 @@ public S3AsyncClient s3CloudStorageClient() { log.info("Overriding s3 client endpoint to {}", endpointOverride); clientBuilder.endpointOverride(URI.create(endpointOverride)); } - return clientBuilder.build(); + return new S3FileClient.Builder(clientBuilder.build()); } private S3AsyncClientBuilder asyncClientBuilder(String region) { diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/CommonDownloaderProperties.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/CommonDownloaderProperties.java index a229a44dfc9..b56f0506339 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/CommonDownloaderProperties.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/CommonDownloaderProperties.java @@ -83,6 +83,7 @@ public boolean isAnonymousCredentials() { @Getter @RequiredArgsConstructor public enum CloudProvider { + FS("file://"), S3("https://s3.amazonaws.com"), GCP("https://storage.googleapis.com"); diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java index 9a1dddb3208..7ea84440c8f 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java @@ -21,14 +21,15 @@ */ import static com.hedera.mirror.common.domain.DigestAlgorithm.SHA384; -import static com.hedera.mirror.importer.domain.StreamFilename.FileType.SIGNATURE; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.maxBy; import com.google.common.base.Stopwatch; import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; import com.google.common.collect.TreeMultimap; + +import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.FileClient; + import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; import java.nio.file.Path; @@ -37,7 +38,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -48,15 +48,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import software.amazon.awssdk.core.async.AsyncResponseTransformer; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.RequestPayer; -import software.amazon.awssdk.services.s3.model.S3Object; import com.hedera.mirror.importer.MirrorProperties; import com.hedera.mirror.importer.addressbook.AddressBookService; @@ -69,7 +62,6 @@ import com.hedera.mirror.importer.domain.StreamFilename; import com.hedera.mirror.common.domain.StreamType; import com.hedera.mirror.importer.exception.HashMismatchException; -import com.hedera.mirror.importer.exception.InvalidStreamFileException; import com.hedera.mirror.importer.exception.SignatureVerificationException; import com.hedera.mirror.importer.reader.StreamFileReader; import com.hedera.mirror.importer.reader.signature.SignatureFileReader; @@ -80,12 +72,11 @@ public abstract class Downloader { public static final String STREAM_CLOSE_LATENCY_METRIC_NAME = "hedera.mirror.stream.close.latency"; protected final Logger log = LogManager.getLogger(getClass()); - private final S3AsyncClient s3Client; + private final FileClient fileClient; private final AddressBookService addressBookService; private final ExecutorService signatureDownloadThreadPool; // One per node during the signature download process protected final DownloaderProperties downloaderProperties; private final MirrorProperties mirrorProperties; - private final CommonDownloaderProperties commonDownloaderProperties; protected final NodeSignatureVerifier nodeSignatureVerifier; protected final SignatureFileReader signatureFileReader; protected final StreamFileReader streamFileReader; @@ -102,15 +93,15 @@ public abstract class Downloader { private final Timer streamCloseMetric; private final Timer.Builder streamVerificationMetric; - protected Downloader(S3AsyncClient s3Client, - AddressBookService addressBookService, DownloaderProperties downloaderProperties, + protected Downloader(FileClientWithProperties fileClient, + AddressBookService addressBookService, MeterRegistry meterRegistry, NodeSignatureVerifier nodeSignatureVerifier, SignatureFileReader signatureFileReader, StreamFileReader streamFileReader, StreamFileNotifier streamFileNotifier, MirrorDateRangePropertiesProcessor mirrorDateRangePropertiesProcessor) { - this.s3Client = s3Client; + this.fileClient = fileClient; this.addressBookService = addressBookService; - this.downloaderProperties = downloaderProperties; + this.downloaderProperties = fileClient.getDownloaderProperties(); this.meterRegistry = meterRegistry; this.nodeSignatureVerifier = nodeSignatureVerifier; signatureDownloadThreadPool = Executors.newFixedThreadPool(downloaderProperties.getThreads()); @@ -120,7 +111,6 @@ protected Downloader(S3AsyncClient s3Client, this.mirrorDateRangePropertiesProcessor = mirrorDateRangePropertiesProcessor; Runtime.getRuntime().addShutdownHook(new Thread(signatureDownloadThreadPool::shutdown)); mirrorProperties = downloaderProperties.getMirrorProperties(); - commonDownloaderProperties = downloaderProperties.getCommon(); streamType = downloaderProperties.getStreamType(); @@ -208,8 +198,8 @@ private Multimap downloadAndParseSigFiles(AddressBo Stopwatch stopwatch = Stopwatch.createStarted(); try { - List s3Objects = listFiles(startAfterFilename, nodeAccountIdStr); - List pendingDownloads = downloadSignatureFiles(nodeAccountIdStr, s3Objects); + List filePaths = fileClient.list(nodeAccountIdStr, startAfterFilename); + List pendingDownloads = fileClient.download(nodeAccountIdStr, filePaths); AtomicInteger count = new AtomicInteger(); pendingDownloads.forEach(pendingDownload -> { try { @@ -253,47 +243,6 @@ private Multimap downloadAndParseSigFiles(AddressBo return sigFilesMap; } - private List listFiles(String lastFilename, String nodeAccountId) throws ExecutionException, - InterruptedException { - // batchSize (number of items we plan do download in a single batch) times 2 for file + sig. - int listSize = (downloaderProperties.getBatchSize() * 2); - String s3Prefix = getS3Prefix(nodeAccountId); - // Not using ListObjectsV2Request because it does not work with GCP. - ListObjectsRequest listRequest = ListObjectsRequest.builder() - .bucket(commonDownloaderProperties.getBucketName()) - .prefix(s3Prefix) - .delimiter("/") - .marker(s3Prefix + lastFilename) - .maxKeys(listSize) - .requestPayer(RequestPayer.REQUESTER) - .build(); - return s3Client.listObjects(listRequest).get().contents(); - } - - private List downloadSignatureFiles(String nodeAccountId, List s3Objects) { - // group the signature filenames by its instant - Map> signatureFilenamesByInstant = s3Objects.stream() - .map(S3Object::key) - .map(key -> key.substring(key.lastIndexOf('/') + 1)) - .map(filename -> { - try { - return new StreamFilename(filename); - } catch (InvalidStreamFileException e) { - log.error(e); - return null; - } - }) - .filter(s -> s != null && s.getFileType() == SIGNATURE) - .collect(groupingBy(StreamFilename::getInstant, maxBy(StreamFilename.EXTENSION_COMPARATOR))); - - String s3Prefix = getS3Prefix(nodeAccountId); - return signatureFilenamesByInstant.values() - .stream() - .filter(Optional::isPresent) - .map(s -> pendingDownload(s.get(), s3Prefix)) - .collect(Collectors.toList()); - } - private Optional parseSignatureFile(PendingDownload pendingDownload, EntityId nodeAccountId) throws InterruptedException, ExecutionException { String s3Key = pendingDownload.getS3key(); Stopwatch stopwatch = pendingDownload.getStopwatch(); @@ -332,25 +281,6 @@ private String getStartAfterFilename() { .orElse(""); } - /** - * Returns a PendingDownload for which the caller can waitForCompletion() to wait for the download to complete. This - * either queues or begins the download (depending on the AWS TransferManager). - * - * @param streamFilename - * @param s3Prefix - * @return - */ - private PendingDownload pendingDownload(StreamFilename streamFilename, String s3Prefix) { - String s3Key = s3Prefix + streamFilename.getFilename(); - var request = GetObjectRequest.builder() - .bucket(commonDownloaderProperties.getBucketName()) - .key(s3Key) - .requestPayer(RequestPayer.REQUESTER) - .build(); - var future = s3Client.getObject(request, AsyncResponseTransformer.toBytes()); - return new PendingDownload(future, streamFilename, s3Key); - } - /** * For each group of signature files with the same file name: (1) verify that the signature files are signed by * corresponding node's PublicKey; (2) For valid signature files, we compare their Hashes to see if at least 1/3 of @@ -459,11 +389,7 @@ private void verifySigsAndDownloadDataFiles(Multimap> future; private final StreamFilename streamFilename; @@ -50,7 +50,7 @@ class PendingDownload { @NonFinal private boolean downloadSuccessful = false; - PendingDownload(CompletableFuture> future, StreamFilename streamFilename, + public PendingDownload(CompletableFuture> future, StreamFilename streamFilename, String s3key) { this.future = future; stopwatch = Stopwatch.createStarted(); diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloader.java index 5aeaccb318e..d8f9efd70e3 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloader.java @@ -20,11 +20,12 @@ * ‍ */ +import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; + import io.micrometer.core.instrument.MeterRegistry; import javax.inject.Named; import lombok.extern.log4j.Log4j2; import org.springframework.scheduling.annotation.Scheduled; -import software.amazon.awssdk.services.s3.S3AsyncClient; import com.hedera.mirror.importer.addressbook.AddressBookService; import com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor; @@ -41,13 +42,13 @@ public class AccountBalancesDownloader extends Downloader { public AccountBalancesDownloader( - S3AsyncClient s3Client, AddressBookService addressBookService, + FileClientWithProperties.Builder fileClientBuilder, AddressBookService addressBookService, BalanceDownloaderProperties downloaderProperties, MeterRegistry meterRegistry, NodeSignatureVerifier nodeSignatureVerifier, SignatureFileReader signatureFileReader, BalanceFileReader balanceFileReader, StreamFileNotifier streamFileNotifier, MirrorDateRangePropertiesProcessor mirrorDateRangePropertiesProcessor) { - super(s3Client, addressBookService, downloaderProperties, + super(fileClientBuilder.buildFor(downloaderProperties), addressBookService, meterRegistry, nodeSignatureVerifier, signatureFileReader, balanceFileReader, streamFileNotifier, mirrorDateRangePropertiesProcessor); } diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClient.java new file mode 100644 index 00000000000..55962997255 --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClient.java @@ -0,0 +1,22 @@ +package com.hedera.mirror.importer.downloader.client; + +import com.hedera.mirror.importer.domain.StreamFilename; +import com.hedera.mirror.importer.downloader.PendingDownload; + +import java.util.List; +import java.util.concurrent.ExecutionException; + +public interface FileClient { + interface Builder { + O buildFor(I dependency); + } + + /** + * Returns a PendingDownload for which the caller can waitForCompletion() to wait for the download to complete. This + * either queues or begins the download (depending on the AWS TransferManager). + */ + PendingDownload download(String nodeAccountId, StreamFilename streamFile); + List download(String nodeAccountId, List filePaths); + + List list(String nodeAccountId, String lastFileName) throws ExecutionException, InterruptedException; +} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java new file mode 100644 index 00000000000..c162f79045f --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java @@ -0,0 +1,37 @@ +package com.hedera.mirror.importer.downloader.client; + +import com.hedera.mirror.importer.downloader.DownloaderProperties; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import java.util.List; +import java.util.concurrent.ExecutionException; + +@Getter +@RequiredArgsConstructor(access = AccessLevel.PROTECTED) +public abstract class FileClientWithProperties implements FileClient { + public interface Builder extends FileClient.Builder { } + + private final DownloaderProperties downloaderProperties; + + @Override + public List list(String nodeAccountId, String lastFilename) throws ExecutionException, + InterruptedException { + // batchSize (number of items we plan do download in a single batch) times 2 for file + sig. + int listSize = (downloaderProperties.getBatchSize() * 2); + + return list(nodeAccountId, lastFilename, listSize); + } + + protected abstract List list(String nodeAccountId, String lastFilename, int maxCount) + throws ExecutionException, InterruptedException; + + protected String pathPrefixFor(String nodeAccountId) { + return downloaderProperties.getPrefix() + nodeAccountId + "/"; + } + + protected String rootPath() { + return downloaderProperties.getCommon().getBucketName(); + } +} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/LocalFileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/LocalFileClient.java new file mode 100644 index 00000000000..eb1e7a1618d --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/LocalFileClient.java @@ -0,0 +1,61 @@ +package com.hedera.mirror.importer.downloader.client; + +import com.hedera.mirror.importer.domain.StreamFilename; +import com.hedera.mirror.importer.downloader.DownloaderProperties; +import com.hedera.mirror.importer.downloader.PendingDownload; + +import lombok.RequiredArgsConstructor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.io.FilenameFilter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +public class LocalFileClient extends FileClientWithProperties { + @RequiredArgsConstructor + public static class Builder implements FileClientWithProperties.Builder { + @Override + public FileClientWithProperties buildFor(DownloaderProperties downloaderProperties) { + return new LocalFileClient(downloaderProperties); + } + } + + protected final Logger log = LogManager.getLogger(getClass()); + + protected LocalFileClient(DownloaderProperties downloaderProperties) { + super(downloaderProperties); + } + + @Override + public PendingDownload download(String nodeAccountId, StreamFilename streamFile) { + return null; + } + + @Override + public List download(String nodeAccountId, List filePaths) { + return new ArrayList<>(); + } + + @Override + protected List list(String nodeAccountId, String lastFilename, int maxCount) + throws ExecutionException, InterruptedException { + File localBucketDir = new File(rootPath() + "/" + pathPrefixFor(nodeAccountId)); + String[] fileNames = localBucketDir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return lastFilename.compareTo(name) > 0; + } + }); + + return fileNames == null ? Collections.emptyList() : + Arrays.stream(fileNames) + .limit(maxCount) + .collect(Collectors.toList()); + } +} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/S3FileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/S3FileClient.java new file mode 100644 index 00000000000..a229620c847 --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/S3FileClient.java @@ -0,0 +1,100 @@ +package com.hedera.mirror.importer.downloader.client; + +import com.hedera.mirror.importer.domain.StreamFilename; +import com.hedera.mirror.importer.downloader.DownloaderProperties; +import com.hedera.mirror.importer.downloader.PendingDownload; + +import com.hedera.mirror.importer.exception.InvalidStreamFileException; + +import lombok.RequiredArgsConstructor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.RequestPayer; +import software.amazon.awssdk.services.s3.model.S3Object; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import static com.hedera.mirror.importer.domain.StreamFilename.FileType.SIGNATURE; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.maxBy; + +public class S3FileClient extends FileClientWithProperties { + @RequiredArgsConstructor + public static class Builder implements FileClientWithProperties.Builder { + private final S3AsyncClient s3Client; + + @Override + public FileClientWithProperties buildFor(DownloaderProperties downloaderProperties) { + return new S3FileClient(downloaderProperties, s3Client); + } + } + + protected final Logger log = LogManager.getLogger(getClass()); + private final S3AsyncClient s3Client; + + protected S3FileClient(DownloaderProperties downloaderProperties, S3AsyncClient s3Client) { + super(downloaderProperties); + this.s3Client = s3Client; + } + + @Override + public List download(String nodeAccountId, List filePaths) { + // group the signature filenames by its instant + Map> signatureFilenamesByInstant = filePaths.stream() + .map(filePath -> filePath.substring(filePath.lastIndexOf('/') + 1)) + .map(filename -> { + try { + return new StreamFilename(filename); + } catch (InvalidStreamFileException e) { + log.error(e); + return null; + } + }) + .filter(s -> s != null && s.getFileType() == SIGNATURE) + .collect(groupingBy(StreamFilename::getInstant, maxBy(StreamFilename.EXTENSION_COMPARATOR))); + + return signatureFilenamesByInstant.values() + .stream() + .filter(Optional::isPresent) + .map(s -> download(nodeAccountId, s.get())) + .collect(Collectors.toList()); + } + + @Override + public PendingDownload download(String nodeAccountId, StreamFilename streamFile) { + String s3Key = pathPrefixFor(nodeAccountId) + streamFile.getFilename(); + var request = GetObjectRequest.builder() + .bucket(rootPath()) + .key(s3Key) + .requestPayer(RequestPayer.REQUESTER) + .build(); + var future = s3Client.getObject(request, AsyncResponseTransformer.toBytes()); + return new PendingDownload(future, streamFile, s3Key); + } + + @Override + protected List list(String nodeAccountId, String lastFilename, int maxCount) throws ExecutionException, InterruptedException { + String s3Prefix = pathPrefixFor(nodeAccountId); + // Not using ListObjectsV2Request because it does not work with GCP. + ListObjectsRequest listRequest = ListObjectsRequest.builder() + .bucket(rootPath()) + .prefix(s3Prefix) + .delimiter("/") + .marker(s3Prefix + lastFilename) + .maxKeys(maxCount) + .requestPayer(RequestPayer.REQUESTER) + .build(); + + return s3Client.listObjects(listRequest).get().contents().stream() + .map(S3Object::key) + .collect(Collectors.toList()); + } +} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/event/EventFileDownloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/event/EventFileDownloader.java index 9b8b7a3698b..046c6ddb51e 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/event/EventFileDownloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/event/EventFileDownloader.java @@ -20,10 +20,11 @@ * ‍ */ +import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; + import io.micrometer.core.instrument.MeterRegistry; import javax.inject.Named; import org.springframework.scheduling.annotation.Scheduled; -import software.amazon.awssdk.services.s3.S3AsyncClient; import com.hedera.mirror.importer.addressbook.AddressBookService; import com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor; @@ -39,13 +40,14 @@ public class EventFileDownloader extends Downloader { public EventFileDownloader( - S3AsyncClient s3Client, AddressBookService addressBookService, + FileClientWithProperties.Builder fileClientBuilder, AddressBookService addressBookService, EventDownloaderProperties downloaderProperties, MeterRegistry meterRegistry, EventFileReader eventFileReader, NodeSignatureVerifier nodeSignatureVerifier, SignatureFileReader signatureFileReader, StreamFileNotifier streamFileNotifier, MirrorDateRangePropertiesProcessor mirrorDateRangePropertiesProcessor) { - super(s3Client, addressBookService, downloaderProperties, meterRegistry, + super(fileClientBuilder.buildFor(downloaderProperties), + addressBookService, meterRegistry, nodeSignatureVerifier, signatureFileReader, eventFileReader, streamFileNotifier, mirrorDateRangePropertiesProcessor); } diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/record/RecordFileDownloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/record/RecordFileDownloader.java index 1a391843839..c4d5a1a88e4 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/record/RecordFileDownloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/record/RecordFileDownloader.java @@ -20,10 +20,11 @@ * ‍ */ +import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; + import io.micrometer.core.instrument.MeterRegistry; import javax.inject.Named; import org.springframework.scheduling.annotation.Scheduled; -import software.amazon.awssdk.services.s3.S3AsyncClient; import com.hedera.mirror.importer.addressbook.AddressBookService; import com.hedera.mirror.importer.config.MirrorDateRangePropertiesProcessor; @@ -39,13 +40,13 @@ public class RecordFileDownloader extends Downloader { public RecordFileDownloader( - S3AsyncClient s3Client, AddressBookService addressBookService, + FileClientWithProperties.Builder fileClientBuilder, AddressBookService addressBookService, RecordDownloaderProperties downloaderProperties, MeterRegistry meterRegistry, NodeSignatureVerifier nodeSignatureVerifier, SignatureFileReader signatureFileReader, RecordFileReader recordFileReader, StreamFileNotifier streamFileNotifier, MirrorDateRangePropertiesProcessor mirrorDateRangePropertiesProcessor) { - super(s3Client, addressBookService, downloaderProperties, meterRegistry, + super(fileClientBuilder.buildFor(downloaderProperties), addressBookService, meterRegistry, nodeSignatureVerifier, signatureFileReader, recordFileReader, streamFileNotifier, mirrorDateRangePropertiesProcessor); } diff --git a/hedera-mirror-importer/src/main/resources/addressbook/local b/hedera-mirror-importer/src/main/resources/addressbook/local new file mode 100644 index 0000000000000000000000000000000000000000..34c5f68f183568afbde28b6dc24a0259b82ec485 GIT binary patch literal 887 zcmW-g&5>0(426Lndhu3lsG4F)09}&x>n#zGf@Djv5CkC*3gJuuqt|s^zP7JRNBWNL z;p-}iv^eu4lg zX9W;b;VIFBv%Wi6()U8*9hjgV(lednxZGVLR+zO_D(ZGDyx~D=PxZ*98DiJ(1uzSe zLhArAIIhd$#)-6Z?a(2h%Qv+(H4Kb6xbesaVDC9oM#_~!j_Z)BEL3lYlXTja_Z82> zORk)&m$o5zg#rV$+uBK!pTxP0CVVF?4^u@*OU~Zb#0pTGbQqcoljT~ubq2^`iJcy- zX9MbBf?lhS1H_tGmwLFEkKCG%Iq9LJhYRSrJD<@d5H*@C%BvIh9(`$(0a(atcv79m z7lf%kQoIqb&>6iv>ZOHaNNsG~BFM$c11I|kKBualM8O*H7_?}}O(=D|SUm}iIcqEG zp$-#)OIKArwS;*JtT+>XU;&QpwADp=3$_eiaI=%Vro1j&>EyT6`VL~fAwy(ajrY7k z*DW&=B3>L>MUACo({aM5Tw$?ot?^2PBiv;#XtjOr2*flo8leDgoR&%l=>s`_?OWl~ zpfHVH!8SHf&|q0{ENQb-)mgoVAJfQA1tDY6ij2Gbi7Er^=-nn$&V+z9@#a4OT%Z1Z U`SE!C^XKb_moLA6z5M+6Kd9E{aR2}S literal 0 HcmV?d00001 diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java index be3263ab7f9..c3befa020a5 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.verify; import com.hedera.mirror.common.util.DomainUtils; +import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.logging.LoggingMeterRegistry; @@ -72,7 +73,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.cglib.core.ReflectUtils; import org.springframework.util.ResourceUtils; -import software.amazon.awssdk.services.s3.S3AsyncClient; import com.hedera.mirror.common.domain.StreamFile; import com.hedera.mirror.common.domain.StreamType; @@ -115,7 +115,7 @@ public abstract class AbstractDownloaderTest { protected FileCopier fileCopier; protected CommonDownloaderProperties commonDownloaderProperties; protected MirrorProperties mirrorProperties; - protected S3AsyncClient s3AsyncClient; + protected FileClientWithProperties.Builder fileClientBuilder; protected DownloaderProperties downloaderProperties; protected Downloader downloader; protected MeterRegistry meterRegistry = new LoggingMeterRegistry(); @@ -212,8 +212,8 @@ boolean isStreamFile(Path path) { protected void beforeEach() throws Exception { initProperties(); commonDownloaderProperties.setAllowAnonymousAccess(true); - s3AsyncClient = new CloudStorageConfiguration(commonDownloaderProperties, - new MetricsExecutionInterceptor(meterRegistry)).s3CloudStorageClient(); + fileClientBuilder = new CloudStorageConfiguration(commonDownloaderProperties, + new MetricsExecutionInterceptor(meterRegistry)).s3CloudStorageClientBuilder(); signatureFileReader = new CompositeSignatureFileReader(new SignatureFileReaderV2(), new SignatureFileReaderV5()); diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloaderTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloaderTest.java index a5b7d1c9434..eeebda6af81 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloaderTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloaderTest.java @@ -51,7 +51,7 @@ protected DownloaderProperties getDownloaderProperties() { protected Downloader getDownloader() { BalanceFileReader balanceFileReader = new BalanceFileReaderImplV1(new BalanceParserProperties(), new AccountBalanceLineParserV1(mirrorProperties)); - return new AccountBalancesDownloader(s3AsyncClient, addressBookService, + return new AccountBalancesDownloader(fileClientBuilder, addressBookService, (BalanceDownloaderProperties) downloaderProperties, meterRegistry, nodeSignatureVerifier, signatureFileReader, balanceFileReader, streamFileNotifier, dateRangeProcessor); } @@ -82,7 +82,7 @@ void downloadWithMixedStreamFileExtensions() throws Exception { // .csv_sig files are intentionally made empty so if two account balance files are processed, they must be // the .pb.gz files ProtoBalanceFileReader protoBalanceFileReader = new ProtoBalanceFileReader(); - downloader = new AccountBalancesDownloader(s3AsyncClient, addressBookService, + downloader = new AccountBalancesDownloader(fileClientBuilder, addressBookService, (BalanceDownloaderProperties) downloaderProperties, meterRegistry, nodeSignatureVerifier, signatureFileReader, protoBalanceFileReader, streamFileNotifier, dateRangeProcessor); fileCopier = FileCopier.create(TestUtils.getResource("data").toPath(), s3Path) diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/event/EventFileDownloaderTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/event/EventFileDownloaderTest.java index a1563e6de28..cbbfab8d33c 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/event/EventFileDownloaderTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/event/EventFileDownloaderTest.java @@ -49,7 +49,7 @@ protected DownloaderProperties getDownloaderProperties() { @Override protected Downloader getDownloader() { - return new EventFileDownloader(s3AsyncClient, addressBookService, + return new EventFileDownloader(fileClientBuilder, addressBookService, (EventDownloaderProperties) downloaderProperties, meterRegistry, new EventFileReaderV3(), nodeSignatureVerifier, signatureFileReader, streamFileNotifier, dateRangeProcessor); diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/record/AbstractRecordFileDownloaderTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/record/AbstractRecordFileDownloaderTest.java index 6f44467d353..3f7bb6d5b32 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/record/AbstractRecordFileDownloaderTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/record/AbstractRecordFileDownloaderTest.java @@ -57,7 +57,7 @@ protected DownloaderProperties getDownloaderProperties() { protected Downloader getDownloader() { RecordFileReader recordFileReader = new CompositeRecordFileReader(new RecordFileReaderImplV1(), new RecordFileReaderImplV2(), new RecordFileReaderImplV5()); - return new RecordFileDownloader(s3AsyncClient, addressBookService, + return new RecordFileDownloader(fileClientBuilder, addressBookService, (RecordDownloaderProperties) downloaderProperties, meterRegistry, nodeSignatureVerifier, signatureFileReader, recordFileReader, streamFileNotifier, dateRangeProcessor); } From a1fbe0325b74c77d3fb14512eb645676952b21e9 Mon Sep 17 00:00:00 2001 From: Victor ADASCALITEI <210908+3Nigma@users.noreply.github.com> Date: Fri, 14 Jan 2022 17:17:39 +0200 Subject: [PATCH 2/8] Completed the LocalFileClient functionality & arranged stuff better. --- .../config/CloudStorageConfiguration.java | 4 +- .../importer/downloader/Downloader.java | 24 ++++--- .../downloader/client/DownloadResult.java | 29 ++++++++ .../downloader/client/FileClient.java | 22 +++++- .../client/FileClientWithProperties.java | 30 ++++++-- .../downloader/client/MultiFileClient.java | 71 ++++++++++++++++++ .../{ => client}/PendingDownload.java | 57 ++++++++------- .../client/{ => s3}/S3FileClient.java | 72 ++++++++----------- .../client/s3/S3PendingDownload.java | 64 +++++++++++++++++ 9 files changed, 287 insertions(+), 86 deletions(-) create mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/DownloadResult.java create mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/MultiFileClient.java rename hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/{ => client}/PendingDownload.java (51%) rename hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/{ => s3}/S3FileClient.java (59%) create mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3PendingDownload.java diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java index b042723f6d5..c466f709545 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java @@ -24,8 +24,8 @@ import java.time.Duration; import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; -import com.hedera.mirror.importer.downloader.client.LocalFileClient; -import com.hedera.mirror.importer.downloader.client.S3FileClient; +import com.hedera.mirror.importer.downloader.client.local.LocalFileClient; +import com.hedera.mirror.importer.downloader.client.s3.S3FileClient; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java index 7ea84440c8f..c21310cf219 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java @@ -28,7 +28,9 @@ import com.google.common.collect.TreeMultimap; import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; -import com.hedera.mirror.importer.downloader.client.FileClient; +import com.hedera.mirror.importer.downloader.client.MultiFileClient; + +import com.hedera.mirror.importer.downloader.client.PendingDownload; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; @@ -72,7 +74,7 @@ public abstract class Downloader { public static final String STREAM_CLOSE_LATENCY_METRIC_NAME = "hedera.mirror.stream.close.latency"; protected final Logger log = LogManager.getLogger(getClass()); - private final FileClient fileClient; + private final MultiFileClient fileClient; private final AddressBookService addressBookService; private final ExecutorService signatureDownloadThreadPool; // One per node during the signature download process protected final DownloaderProperties downloaderProperties; @@ -199,7 +201,7 @@ private Multimap downloadAndParseSigFiles(AddressBo try { List filePaths = fileClient.list(nodeAccountIdStr, startAfterFilename); - List pendingDownloads = fileClient.download(nodeAccountIdStr, filePaths); + List pendingDownloads = fileClient.downloadSignatureFiles(nodeAccountIdStr, filePaths); AtomicInteger count = new AtomicInteger(); pendingDownloads.forEach(pendingDownload -> { try { @@ -210,11 +212,11 @@ private Multimap downloadAndParseSigFiles(AddressBo totalDownloads.incrementAndGet(); }); } catch (InterruptedException ex) { - log.warn("Failed downloading {} in {}", pendingDownload.getS3key(), + log.warn("Failed downloading {} in {}", pendingDownload.getRemotePath(), pendingDownload.getStopwatch(), ex); Thread.currentThread().interrupt(); } catch (Exception ex) { - log.warn("Failed to parse signature file {}: {}", pendingDownload.getS3key(), ex); + log.warn("Failed to parse signature file {}: {}", pendingDownload.getRemotePath(), ex); } }); @@ -244,7 +246,7 @@ private Multimap downloadAndParseSigFiles(AddressBo } private Optional parseSignatureFile(PendingDownload pendingDownload, EntityId nodeAccountId) throws InterruptedException, ExecutionException { - String s3Key = pendingDownload.getS3key(); + String s3Key = pendingDownload.getRemotePath(); Stopwatch stopwatch = pendingDownload.getStopwatch(); if (!pendingDownload.waitForCompletion()) { @@ -252,8 +254,8 @@ private Optional parseSignatureFile(PendingDownload pending return Optional.empty(); } - StreamFilename streamFilename = pendingDownload.getStreamFilename(); - StreamFileData streamFileData = new StreamFileData(streamFilename, pendingDownload.getBytes()); + StreamFilename streamFilename = pendingDownload.getLocalFilename(); + StreamFileData streamFileData = new StreamFileData(streamFilename, pendingDownload.getResult().getBytes()); FileStreamSignature fileStreamSignature = signatureFileReader.read(streamFileData); fileStreamSignature.setNodeAccountId(nodeAccountId); fileStreamSignature.setStreamType(streamType); @@ -331,8 +333,8 @@ private void verifySigsAndDownloadDataFiles(Multimap { * either queues or begins the download (depending on the AWS TransferManager). */ PendingDownload download(String nodeAccountId, StreamFilename streamFile); - List download(String nodeAccountId, List filePaths); List list(String nodeAccountId, String lastFileName) throws ExecutionException, InterruptedException; } diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java index c162f79045f..177413c55bd 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java @@ -1,16 +1,38 @@ package com.hedera.mirror.importer.downloader.client; -import com.hedera.mirror.importer.downloader.DownloaderProperties; +/*- + * ‌ + * Hedera Mirror Node + * ​ + * Copyright (C) 2019 - 2021 Hedera Hashgraph, LLC + * ​ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ‍ + */ +import java.util.List; +import java.util.concurrent.ExecutionException; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; -import java.util.List; -import java.util.concurrent.ExecutionException; +import lombok.extern.log4j.Log4j2; + +import com.hedera.mirror.importer.downloader.DownloaderProperties; @Getter +@Log4j2 @RequiredArgsConstructor(access = AccessLevel.PROTECTED) -public abstract class FileClientWithProperties implements FileClient { +public abstract class FileClientWithProperties extends MultiFileClient { public interface Builder extends FileClient.Builder { } private final DownloaderProperties downloaderProperties; diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/MultiFileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/MultiFileClient.java new file mode 100644 index 00000000000..b29b5f1e8d7 --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/MultiFileClient.java @@ -0,0 +1,71 @@ +package com.hedera.mirror.importer.downloader.client; + +/*- + * ‌ + * Hedera Mirror Node + * ​ + * Copyright (C) 2019 - 2021 Hedera Hashgraph, LLC + * ​ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ‍ + */ + +import static com.hedera.mirror.importer.domain.StreamFilename.FileType.SIGNATURE; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.maxBy; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import lombok.extern.log4j.Log4j2; + +import com.hedera.mirror.importer.domain.StreamFilename; +import com.hedera.mirror.importer.exception.InvalidStreamFileException; + +@Log4j2 +public abstract class MultiFileClient implements FileClient { + private static final Predicate SIGNATURE_FILES_FILTER = s -> s != null && s.getFileType() == SIGNATURE; + + private List download(String nodeAccountId, List filePaths, Predicate filter) { + if (filePaths.isEmpty()) { + return Collections.emptyList(); + } + + Map> signatureFilenamesByInstant = filePaths.stream() + .map(filePath -> filePath.substring(filePath.lastIndexOf('/') + 1)) + .map(filename -> { + try { + return new StreamFilename(filename); + } catch (InvalidStreamFileException e) { + log.error(e); + return null; + } + }) + .filter(filter) + .collect(groupingBy(StreamFilename::getInstant, maxBy(StreamFilename.EXTENSION_COMPARATOR))); + + return signatureFilenamesByInstant.values() + .stream() + .filter(Optional::isPresent) + .map(s -> download(nodeAccountId, s.get())) + .collect(Collectors.toList()); + } + + public List downloadSignatureFiles(String nodeAccountId, List filePaths) { + return download(nodeAccountId, filePaths, SIGNATURE_FILES_FILTER); + } +} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/PendingDownload.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/PendingDownload.java similarity index 51% rename from hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/PendingDownload.java rename to hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/PendingDownload.java index be991db52d9..aabec6b864d 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/PendingDownload.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/PendingDownload.java @@ -1,4 +1,4 @@ -package com.hedera.mirror.importer.downloader; +package com.hedera.mirror.importer.downloader.client; /*- * ‌ @@ -23,11 +23,10 @@ import com.google.common.base.Stopwatch; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; + import lombok.Value; import lombok.experimental.NonFinal; import lombok.extern.log4j.Log4j2; -import software.amazon.awssdk.core.ResponseBytes; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; import com.hedera.mirror.importer.domain.StreamFilename; @@ -36,55 +35,61 @@ * complete and get the status of whether it was successful or not. */ @Log4j2 +@NonFinal @Value -public class PendingDownload { +public abstract class PendingDownload { + public static class SimpleResultForwarder extends PendingDownload { + public SimpleResultForwarder(CompletableFuture future, StreamFilename localFilename, String remotePath) { + super(future, localFilename, remotePath); + } - private final CompletableFuture> future; - private final StreamFilename streamFilename; - private final Stopwatch stopwatch; - private final String s3key; + @Override + protected DownloadResult mapResult(DownloadResult resolvedFuture) { + return resolvedFuture; + } + } + + protected CompletableFuture future; + StreamFilename localFilename; + String remotePath; + Stopwatch stopwatch = Stopwatch.createStarted(); @NonFinal - private boolean alreadyWaited = false; // has waitForCompletion been called + boolean alreadyWaited = false; // has waitForCompletion been called @NonFinal - private boolean downloadSuccessful = false; + boolean downloadSuccessful = false; - public PendingDownload(CompletableFuture> future, StreamFilename streamFilename, - String s3key) { + protected PendingDownload(CompletableFuture future, StreamFilename localFilename, String remotePath) { this.future = future; - stopwatch = Stopwatch.createStarted(); - this.streamFilename = streamFilename; - this.s3key = s3key; - } - - byte[] getBytes() throws ExecutionException, InterruptedException { - return future.get().asByteArrayUnsafe(); + this.localFilename = localFilename; + this.remotePath = remotePath; } - GetObjectResponse getObjectResponse() throws ExecutionException, InterruptedException { - return future.get().response(); + public DownloadResult getResult() throws ExecutionException, InterruptedException { + return mapResult(this.future.get()); } + protected abstract DownloadResult mapResult(I resolvedFuture); /** * @return true if the download was successful. */ - boolean waitForCompletion() throws InterruptedException { + public boolean waitForCompletion() throws InterruptedException { if (alreadyWaited) { return downloadSuccessful; } alreadyWaited = true; try { future.get(); - log.debug("Finished downloading {} in {}", s3key, stopwatch); + log.debug("Finished downloading {} in {}", remotePath, stopwatch); downloadSuccessful = true; } catch (InterruptedException e) { - log.warn("Failed downloading {} after {}", s3key, stopwatch, e); + log.warn("Failed downloading {} after {}", remotePath, stopwatch, e); Thread.currentThread().interrupt(); } catch (ExecutionException ex) { - log.warn("Failed downloading {} after {}: {}", s3key, stopwatch, ex.getMessage()); + log.warn("Failed downloading {} after {}: {}", remotePath, stopwatch, ex.getMessage()); } catch (Exception ex) { - log.warn("Failed downloading {} after {}", s3key, stopwatch, ex); + log.warn("Failed downloading {} after {}", remotePath, stopwatch, ex); } return downloadSuccessful; } diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/S3FileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java similarity index 59% rename from hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/S3FileClient.java rename to hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java index a229620c847..66888019c5c 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/S3FileClient.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java @@ -1,31 +1,45 @@ -package com.hedera.mirror.importer.downloader.client; +package com.hedera.mirror.importer.downloader.client.s3; -import com.hedera.mirror.importer.domain.StreamFilename; -import com.hedera.mirror.importer.downloader.DownloaderProperties; -import com.hedera.mirror.importer.downloader.PendingDownload; +/*- + * ‌ + * Hedera Mirror Node + * ​ + * Copyright (C) 2019 - 2021 Hedera Hashgraph, LLC + * ​ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ‍ + */ + +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; -import com.hedera.mirror.importer.exception.InvalidStreamFileException; +import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; import lombok.RequiredArgsConstructor; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.extern.log4j.Log4j2; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsRequest; import software.amazon.awssdk.services.s3.model.RequestPayer; import software.amazon.awssdk.services.s3.model.S3Object; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import static com.hedera.mirror.importer.domain.StreamFilename.FileType.SIGNATURE; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.maxBy; +import com.hedera.mirror.importer.domain.StreamFilename; +import com.hedera.mirror.importer.downloader.DownloaderProperties; +import com.hedera.mirror.importer.downloader.client.PendingDownload; +@Log4j2 public class S3FileClient extends FileClientWithProperties { @RequiredArgsConstructor public static class Builder implements FileClientWithProperties.Builder { @@ -37,7 +51,6 @@ public FileClientWithProperties buildFor(DownloaderProperties downloaderProperti } } - protected final Logger log = LogManager.getLogger(getClass()); private final S3AsyncClient s3Client; protected S3FileClient(DownloaderProperties downloaderProperties, S3AsyncClient s3Client) { @@ -45,29 +58,6 @@ protected S3FileClient(DownloaderProperties downloaderProperties, S3AsyncClient this.s3Client = s3Client; } - @Override - public List download(String nodeAccountId, List filePaths) { - // group the signature filenames by its instant - Map> signatureFilenamesByInstant = filePaths.stream() - .map(filePath -> filePath.substring(filePath.lastIndexOf('/') + 1)) - .map(filename -> { - try { - return new StreamFilename(filename); - } catch (InvalidStreamFileException e) { - log.error(e); - return null; - } - }) - .filter(s -> s != null && s.getFileType() == SIGNATURE) - .collect(groupingBy(StreamFilename::getInstant, maxBy(StreamFilename.EXTENSION_COMPARATOR))); - - return signatureFilenamesByInstant.values() - .stream() - .filter(Optional::isPresent) - .map(s -> download(nodeAccountId, s.get())) - .collect(Collectors.toList()); - } - @Override public PendingDownload download(String nodeAccountId, StreamFilename streamFile) { String s3Key = pathPrefixFor(nodeAccountId) + streamFile.getFilename(); @@ -77,7 +67,7 @@ public PendingDownload download(String nodeAccountId, StreamFilename streamFile) .requestPayer(RequestPayer.REQUESTER) .build(); var future = s3Client.getObject(request, AsyncResponseTransformer.toBytes()); - return new PendingDownload(future, streamFile, s3Key); + return new S3PendingDownload(future, streamFile, s3Key); } @Override diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3PendingDownload.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3PendingDownload.java new file mode 100644 index 00000000000..c7b233e21ef --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3PendingDownload.java @@ -0,0 +1,64 @@ +package com.hedera.mirror.importer.downloader.client.s3; + +/*- + * ‌ + * Hedera Mirror Node + * ​ + * Copyright (C) 2019 - 2021 Hedera Hashgraph, LLC + * ​ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ‍ + */ + +import java.time.Instant; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import com.hedera.mirror.importer.downloader.client.PendingDownload; +import com.hedera.mirror.importer.downloader.client.DownloadResult; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import com.hedera.mirror.importer.domain.StreamFilename; + +@Log4j2 +public class S3PendingDownload extends PendingDownload> { + @RequiredArgsConstructor(access = AccessLevel.PRIVATE) + public static class S3DownloadResult implements DownloadResult { + private final ResponseBytes result; + + @Override + public byte[] getBytes() throws ExecutionException, InterruptedException { + return result.asByteArrayUnsafe(); + } + + @Override + public Instant getLastModified() { + return result.response().lastModified(); + } + } + + + public S3PendingDownload(CompletableFuture> future, StreamFilename streamFile, String s3key) { + super(future, streamFile, s3key); + } + + @Override + protected DownloadResult mapResult(ResponseBytes resolvedFuture) { + return new S3DownloadResult(resolvedFuture); + } +} From d99bb045ed050acaa43afd340ac4bb218662a6e7 Mon Sep 17 00:00:00 2001 From: Victor ADASCALITEI <210908+3Nigma@users.noreply.github.com> Date: Fri, 14 Jan 2022 17:19:08 +0200 Subject: [PATCH 3/8] Added missing LocalFileClient that was skipped by Git, for some reason. --- .../downloader/client/LocalFileClient.java | 61 ------------ .../client/local/LocalFileClient.java | 94 +++++++++++++++++++ 2 files changed, 94 insertions(+), 61 deletions(-) delete mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/LocalFileClient.java create mode 100644 hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/LocalFileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/LocalFileClient.java deleted file mode 100644 index eb1e7a1618d..00000000000 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/LocalFileClient.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.hedera.mirror.importer.downloader.client; - -import com.hedera.mirror.importer.domain.StreamFilename; -import com.hedera.mirror.importer.downloader.DownloaderProperties; -import com.hedera.mirror.importer.downloader.PendingDownload; - -import lombok.RequiredArgsConstructor; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.FilenameFilter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -public class LocalFileClient extends FileClientWithProperties { - @RequiredArgsConstructor - public static class Builder implements FileClientWithProperties.Builder { - @Override - public FileClientWithProperties buildFor(DownloaderProperties downloaderProperties) { - return new LocalFileClient(downloaderProperties); - } - } - - protected final Logger log = LogManager.getLogger(getClass()); - - protected LocalFileClient(DownloaderProperties downloaderProperties) { - super(downloaderProperties); - } - - @Override - public PendingDownload download(String nodeAccountId, StreamFilename streamFile) { - return null; - } - - @Override - public List download(String nodeAccountId, List filePaths) { - return new ArrayList<>(); - } - - @Override - protected List list(String nodeAccountId, String lastFilename, int maxCount) - throws ExecutionException, InterruptedException { - File localBucketDir = new File(rootPath() + "/" + pathPrefixFor(nodeAccountId)); - String[] fileNames = localBucketDir.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return lastFilename.compareTo(name) > 0; - } - }); - - return fileNames == null ? Collections.emptyList() : - Arrays.stream(fileNames) - .limit(maxCount) - .collect(Collectors.toList()); - } -} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java new file mode 100644 index 00000000000..5b793e94b02 --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java @@ -0,0 +1,94 @@ +package com.hedera.mirror.importer.downloader.client.local; + +/*- + * ‌ + * Hedera Mirror Node + * ​ + * Copyright (C) 2019 - 2021 Hedera Hashgraph, LLC + * ​ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ‍ + */ + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; + +import com.hedera.mirror.importer.domain.StreamFilename; +import com.hedera.mirror.importer.downloader.client.DownloadResult; +import com.hedera.mirror.importer.downloader.DownloaderProperties; +import com.hedera.mirror.importer.downloader.client.PendingDownload; + +@Log4j2 +public class LocalFileClient extends FileClientWithProperties { + @RequiredArgsConstructor + public static class Builder implements FileClientWithProperties.Builder { + @Override + public FileClientWithProperties buildFor(DownloaderProperties downloaderProperties) { + return new LocalFileClient(downloaderProperties); + } + } + + protected LocalFileClient(DownloaderProperties downloaderProperties) { + super(downloaderProperties); + } + + @SneakyThrows + @Override + public PendingDownload download(String nodeAccountId, StreamFilename streamFile) { + Path filePath = Paths.get(rootPath(), pathPrefixFor(nodeAccountId), streamFile.getFilename()); + CompletableFuture futureLocalDownload = CompletableFuture.supplyAsync(() -> new DownloadResult() { + @SneakyThrows + @Override + public byte[] getBytes() { + return Files.readAllBytes(filePath); + } + + @SneakyThrows + @Override + public Instant getLastModified() { + BasicFileAttributes fileAttributes = Files.readAttributes(filePath, BasicFileAttributes.class); + return fileAttributes.lastModifiedTime().toInstant(); + } + }); + + return new PendingDownload.SimpleResultForwarder(futureLocalDownload, streamFile, filePath.toString()); + } + + @Override + protected List list(String nodeAccountId, String lastFilename, int maxCount) + throws ExecutionException, InterruptedException { + File localBucketDir = new File(rootPath() + "/" + pathPrefixFor(nodeAccountId)); + String[] fileNames = localBucketDir.list((dir, name) -> lastFilename.compareTo(name) < 0); + + return fileNames == null ? Collections.emptyList() : + Arrays.stream(fileNames) + .limit(maxCount) + .collect(Collectors.toList()); + } +} From fb49a737fd8f341d331698485663a767fbf6100a Mon Sep 17 00:00:00 2001 From: Victor ADASCALITEI <210908+3Nigma@users.noreply.github.com> Date: Fri, 14 Jan 2022 17:54:12 +0200 Subject: [PATCH 4/8] Cleared-up some hanging comments. --- .../hedera/mirror/importer/downloader/client/FileClient.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClient.java index 04e319a6cd5..36ff95dd1ce 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClient.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClient.java @@ -30,11 +30,6 @@ interface Builder { O buildFor(I dependency); } - /** - * Returns a PendingDownload for which the caller can waitForCompletion() to wait for the download to complete. This - * either queues or begins the download (depending on the AWS TransferManager). - */ PendingDownload download(String nodeAccountId, StreamFilename streamFile); - List list(String nodeAccountId, String lastFileName) throws ExecutionException, InterruptedException; } From 88936827750bc481815f1ca86e290e07bcc53376 Mon Sep 17 00:00:00 2001 From: Victor ADASCALITEI <210908+3Nigma@users.noreply.github.com> Date: Fri, 14 Jan 2022 18:27:41 +0200 Subject: [PATCH 5/8] Updated docs. --- charts/README.md | 2 +- charts/marketplace/gcp/schema.yaml | 3 ++- docs/configuration.md | 14 +++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/charts/README.md b/charts/README.md index 2fe6452e9bc..1adf9b6e288 100644 --- a/charts/README.md +++ b/charts/README.md @@ -56,7 +56,7 @@ $ helm upgrade --install "${RELEASE}" charts/hedera-mirror --set postgresql.enab ### Non-Production Environments -When running against a network other than a public network (e.g., demo, previewnet, testnet, or mainnet), the network +When running against a network other than a public network (e.g., demo, previewnet, testnet, local or mainnet), the network must be updated with an initial address book file prior to deploying the chart. 1. First acquire the address book file and encode its contents to Base64: diff --git a/charts/marketplace/gcp/schema.yaml b/charts/marketplace/gcp/schema.yaml index 267dfcab1ef..8d2ec8ae0c7 100644 --- a/charts/marketplace/gcp/schema.yaml +++ b/charts/marketplace/gcp/schema.yaml @@ -137,9 +137,10 @@ properties: importer.config.hedera.mirror.importer.network: default: MAINNET type: string - description: Which Hedera network to use. Can be either DEMO (free), TESTNET or MAINNET. + description: Which Hedera network to use. Can be either DEMO (free), LOCAL (free), MAINNET, TESTNET, OTHER or PREVIEWNET. title: Importer Hedera network enum: + - LOCAL - MAINNET - TESTNET - PREVIEWNET diff --git a/docs/configuration.md b/docs/configuration.md index 6bced0ae67f..7eab06abb63 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -51,8 +51,8 @@ value, it is recommended to only populate overridden properties in the custom `a | `hedera.mirror.importer.downloader.balance.threads` | 15 | The number of threads to search for new files to download | | `hedera.mirror.importer.downloader.balance.writeFiles` | false | Whether to write verified stream files to the filesystem. | | `hedera.mirror.importer.downloader.balance.writeSignatures` | false | Whether to write verified signature files to the filesystem. | -| `hedera.mirror.importer.downloader.bucketName` | | The cloud storage bucket name to download streamed files. This value takes priority over network hardcoded bucket names regardless of `hedera.mirror.importer.network` value.| -| `hedera.mirror.importer.downloader.cloudProvider` | S3 | The cloud provider to download files from. Either `S3` or `GCP` | +| `hedera.mirror.importer.downloader.bucketName` | | The cloud storage bucket name to download streamed files from (if `cloudProvider` is either `GCP` or `S3`) and the local root file-system path to look-up new network entrie on (if `cloudProvider` is `FS`). This value takes priority over network hardcoded bucket names regardless of `hedera.mirror.importer.network` value.| +| `hedera.mirror.importer.downloader.cloudProvider` | S3 | The cloud provider to download files from. Either `FS`, `S3` or `GCP` | | `hedera.mirror.importer.downloader.consensusRatio` | 0.333 | The ratio of verified nodes (nodes used to come to consensus on the signature file hash) to total number of nodes available | | `hedera.mirror.importer.downloader.endpointOverride` | | Can be specified to download streams from a source other than S3 and GCP. Should be S3 compatible | | `hedera.mirror.importer.downloader.event.batchSize` | 100 | The number of signature files to download per node before downloading the signed files | @@ -78,7 +78,7 @@ value, it is recommended to only populate overridden properties in the custom `a | `hedera.mirror.importer.endDate` | 2262-04-11T23:47:16.854775807Z | The end date (inclusive) of the data to import. Items after this date will be ignored. Format: YYYY-MM-ddTHH:mm:ss.nnnnnnnnnZ | | `hedera.mirror.importer.importHistoricalAccountInfo` | true | Import historical account information that occurred before the last stream reset. Skipped if `startDate` is unset or after 2019-09-14T00:00:10Z. | | `hedera.mirror.importer.initialAddressBook` | "" | The path to the bootstrap address book used to override the built-in address book | -| `hedera.mirror.importer.network` | DEMO | Which Hedera network to use. Can be either `DEMO`, `MAINNET`, `TESTNET`, `PREVIEWNET` or `OTHER` | +| `hedera.mirror.importer.network` | DEMO | Which Hedera network to use. Can be either `DEMO`, `LOCAL`, `MAINNET`, `TESTNET`, `PREVIEWNET` or `OTHER` | | `hedera.mirror.importer.parser.balance.batchSize` | 200000 | The number of balances to store in memory before saving to the database | | `hedera.mirror.importer.parser.balance.enabled` | true | Whether to enable balance file parsing | | `hedera.mirror.importer.parser.balance.fileBufferSize` | 200000 | The size of the buffer to use when reading in the balance file | @@ -233,7 +233,7 @@ Name | Default | D `hedera.mirror.monitor.mirrorNode.grpc.port` | 5600 | The port of the mirror node's gRPC API `hedera.mirror.monitor.mirrorNode.rest.host` | "" | The hostname of the mirror node's REST API `hedera.mirror.monitor.mirrorNode.rest.port` | 443 | The port of the mirror node's REST API -`hedera.mirror.monitor.network` | TESTNET | Which network to connect to. Automatically populates the main node & mirror node endpoints. Can be `MAINNET`, `PREVIEWNET`, `TESTNET` or `OTHER` +`hedera.mirror.monitor.network` | TESTNET | Which network to connect to. Automatically populates the main node & mirror node endpoints. Can be `LOCAL`, `MAINNET`, `PREVIEWNET`, `TESTNET` or `OTHER` `hedera.mirror.monitor.nodes[].accountId` | "" | The main node's account ID `hedera.mirror.monitor.nodes[].host` | "" | The main node's hostname `hedera.mirror.monitor.nodes[].port` | 50211 | The main node's port @@ -338,10 +338,10 @@ value, it is recommended to only populate overridden properties in the custom `a | `hedera.mirror.rest.stateproof.enabled` | false | Whether to enable stateproof REST API or not | | `hedera.mirror.rest.stateproof.streams.accessKey` | "" | The cloud storage access key | | `hedera.mirror.rest.stateproof.streams.bucketName` | | The cloud storage bucket name to download streamed files. This value takes priority over network hardcoded bucket names regardless of `hedera.mirror.rest.stateproof.streams.network` | -| `hedera.mirror.rest.stateproof.streams.cloudProvider` | S3 | The cloud provider to download files from. Either `S3` or `GCP` | +| `hedera.mirror.rest.stateproof.streams.cloudProvider` | S3 | The cloud provider to download files from. Either `FS`, `S3` or `GCP` | | `hedera.mirror.rest.stateproof.streams.endpointOverride` | | Can be specified to download streams from a source other than S3 and GCP. Should be S3 compatible | | `hedera.mirror.rest.stateproof.streams.gcpProjectId` | | GCP project id to bill for requests to GCS bucket which has Requester Pays enabled. | -| `hedera.mirror.rest.stateproof.streams.network` | DEMO | Which Hedera network to use. Can be either `DEMO`, `MAINNET`, `TESTNET`, `PREVIEWNET` or `OTHER` | +| `hedera.mirror.rest.stateproof.streams.network` | DEMO | Which Hedera network to use. Can be either `DEMO`, `LOCAL`, `MAINNET`, `TESTNET`, `PREVIEWNET` or `OTHER` | | `hedera.mirror.rest.stateproof.streams.region` | us-east-1 | The region associated with the bucket | | `hedera.mirror.rest.stateproof.streams.secretKey` | "" | The cloud storage secret key | @@ -394,7 +394,7 @@ Name | Default `hedera.mirror.rosetta.db.statementTimeout` | 20 | The number of seconds to wait before timing out a query statement `hedera.mirror.rosetta.db.username` | mirror_rosetta | The username the processor uses to connect to the database `hedera.mirror.rosetta.log.level` | info | The log level -`hedera.mirror.rosetta.network` | DEMO | Which Hedera network to use. Can be either `DEMO`, `MAINNET`, `PREVIEWNET`, `TESTNET` or `OTHER` +`hedera.mirror.rosetta.network` | DEMO | Which Hedera network to use. Can be either `DEMO`, `LOCAL`, `MAINNET`, `PREVIEWNET`, `TESTNET` or `OTHER` `hedera.mirror.rosetta.nodes` | {} | A map of main nodes with its service endpoint as the key and the node account id as its value `hedera.mirror.rosetta.nodeVersion` | 0 | The default canonical version of the node runtime `hedera.mirror.rosetta.online` | true | The default online mode of the Rosetta interface From bd8c8b97a6fbe65fa87f2b09d3b05af031e04193 Mon Sep 17 00:00:00 2001 From: Victor ADASCALITEI <210908+3Nigma@users.noreply.github.com> Date: Sat, 15 Jan 2022 10:04:50 +0200 Subject: [PATCH 6/8] Renamed the FileClientWitProperties into ParameterizedFileClient. --- .../mirror/importer/config/CloudStorageConfiguration.java | 8 ++++---- .../com/hedera/mirror/importer/downloader/Downloader.java | 4 ++-- .../downloader/balance/AccountBalancesDownloader.java | 4 ++-- ...ntWithProperties.java => ParameterizedFileClient.java} | 4 ++-- .../importer/downloader/client/local/LocalFileClient.java | 8 ++++---- .../importer/downloader/client/s3/S3FileClient.java | 8 ++++---- .../importer/downloader/event/EventFileDownloader.java | 4 ++-- .../importer/downloader/record/RecordFileDownloader.java | 4 ++-- .../importer/downloader/AbstractDownloaderTest.java | 4 ++-- 9 files changed, 24 insertions(+), 24 deletions(-) rename hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/{FileClientWithProperties.java => ParameterizedFileClient.java} (94%) diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java index c466f709545..f6ef4146a8b 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/CloudStorageConfiguration.java @@ -23,7 +23,7 @@ import java.net.URI; import java.time.Duration; -import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import com.hedera.mirror.importer.downloader.client.local.LocalFileClient; import com.hedera.mirror.importer.downloader.client.s3.S3FileClient; @@ -74,7 +74,7 @@ AwsCredentialsProvider awsCredentialsProvider() { @Bean @ConditionalOnProperty(prefix = "hedera.mirror.importer.downloader", name = "cloudProvider", havingValue = "FS") - public FileClientWithProperties.Builder localStorageClientBuilder() { + public ParameterizedFileClient.Builder localStorageClientBuilder() { log.info("Configured to download from FileSystem having base-path '{}'", downloaderProperties.getBucketName()); @@ -83,7 +83,7 @@ public FileClientWithProperties.Builder localStorageClientBuilder() { @Bean @ConditionalOnProperty(prefix = "hedera.mirror.importer.downloader", name = "cloudProvider", havingValue = "GCP") - public FileClientWithProperties.Builder gcpCloudStorageClientBuilder() { + public ParameterizedFileClient.Builder gcpCloudStorageClientBuilder() { log.info("Configured to download from GCP with bucket name '{}'", downloaderProperties.getBucketName()); // Any valid region for aws client. Ignored by GCP. S3AsyncClientBuilder clientBuilder = asyncClientBuilder("us-east-1") @@ -105,7 +105,7 @@ public SdkHttpRequest modifyHttpRequest( @Bean @ConditionalOnProperty(prefix = "hedera.mirror.importer.downloader", name = "cloudProvider", havingValue = "S3", matchIfMissing = true) - public FileClientWithProperties.Builder s3CloudStorageClientBuilder() { + public ParameterizedFileClient.Builder s3CloudStorageClientBuilder() { log.info("Configured to download from S3 in region {} with bucket name '{}'", downloaderProperties.getRegion(), downloaderProperties.getBucketName()); S3AsyncClientBuilder clientBuilder = asyncClientBuilder( diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java index c21310cf219..fbc458298d1 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/Downloader.java @@ -27,7 +27,7 @@ import com.google.common.collect.Multimaps; import com.google.common.collect.TreeMultimap; -import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import com.hedera.mirror.importer.downloader.client.MultiFileClient; import com.hedera.mirror.importer.downloader.client.PendingDownload; @@ -95,7 +95,7 @@ public abstract class Downloader { private final Timer streamCloseMetric; private final Timer.Builder streamVerificationMetric; - protected Downloader(FileClientWithProperties fileClient, + protected Downloader(ParameterizedFileClient fileClient, AddressBookService addressBookService, MeterRegistry meterRegistry, NodeSignatureVerifier nodeSignatureVerifier, SignatureFileReader signatureFileReader, StreamFileReader streamFileReader, diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloader.java index d8f9efd70e3..94337246ce1 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/balance/AccountBalancesDownloader.java @@ -20,7 +20,7 @@ * ‍ */ -import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import io.micrometer.core.instrument.MeterRegistry; import javax.inject.Named; @@ -42,7 +42,7 @@ public class AccountBalancesDownloader extends Downloader { public AccountBalancesDownloader( - FileClientWithProperties.Builder fileClientBuilder, AddressBookService addressBookService, + ParameterizedFileClient.Builder fileClientBuilder, AddressBookService addressBookService, BalanceDownloaderProperties downloaderProperties, MeterRegistry meterRegistry, NodeSignatureVerifier nodeSignatureVerifier, SignatureFileReader signatureFileReader, BalanceFileReader balanceFileReader, diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/ParameterizedFileClient.java similarity index 94% rename from hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java rename to hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/ParameterizedFileClient.java index 177413c55bd..44912d18abf 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/FileClientWithProperties.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/ParameterizedFileClient.java @@ -32,8 +32,8 @@ @Getter @Log4j2 @RequiredArgsConstructor(access = AccessLevel.PROTECTED) -public abstract class FileClientWithProperties extends MultiFileClient { - public interface Builder extends FileClient.Builder { } +public abstract class ParameterizedFileClient extends MultiFileClient { + public interface Builder extends FileClient.Builder { } private final DownloaderProperties downloaderProperties; diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java index 5b793e94b02..84d4416fc4e 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java @@ -33,7 +33,7 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -45,11 +45,11 @@ import com.hedera.mirror.importer.downloader.client.PendingDownload; @Log4j2 -public class LocalFileClient extends FileClientWithProperties { +public class LocalFileClient extends ParameterizedFileClient { @RequiredArgsConstructor - public static class Builder implements FileClientWithProperties.Builder { + public static class Builder implements ParameterizedFileClient.Builder { @Override - public FileClientWithProperties buildFor(DownloaderProperties downloaderProperties) { + public ParameterizedFileClient buildFor(DownloaderProperties downloaderProperties) { return new LocalFileClient(downloaderProperties); } } diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java index 66888019c5c..21abea7173b 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java @@ -24,7 +24,7 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; @@ -40,13 +40,13 @@ import com.hedera.mirror.importer.downloader.client.PendingDownload; @Log4j2 -public class S3FileClient extends FileClientWithProperties { +public class S3FileClient extends ParameterizedFileClient { @RequiredArgsConstructor - public static class Builder implements FileClientWithProperties.Builder { + public static class Builder implements ParameterizedFileClient.Builder { private final S3AsyncClient s3Client; @Override - public FileClientWithProperties buildFor(DownloaderProperties downloaderProperties) { + public ParameterizedFileClient buildFor(DownloaderProperties downloaderProperties) { return new S3FileClient(downloaderProperties, s3Client); } } diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/event/EventFileDownloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/event/EventFileDownloader.java index 046c6ddb51e..381d628e9b2 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/event/EventFileDownloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/event/EventFileDownloader.java @@ -20,7 +20,7 @@ * ‍ */ -import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import io.micrometer.core.instrument.MeterRegistry; import javax.inject.Named; @@ -40,7 +40,7 @@ public class EventFileDownloader extends Downloader { public EventFileDownloader( - FileClientWithProperties.Builder fileClientBuilder, AddressBookService addressBookService, + ParameterizedFileClient.Builder fileClientBuilder, AddressBookService addressBookService, EventDownloaderProperties downloaderProperties, MeterRegistry meterRegistry, EventFileReader eventFileReader, NodeSignatureVerifier nodeSignatureVerifier, SignatureFileReader signatureFileReader, diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/record/RecordFileDownloader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/record/RecordFileDownloader.java index c4d5a1a88e4..46160fedb75 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/record/RecordFileDownloader.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/record/RecordFileDownloader.java @@ -20,7 +20,7 @@ * ‍ */ -import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import io.micrometer.core.instrument.MeterRegistry; import javax.inject.Named; @@ -40,7 +40,7 @@ public class RecordFileDownloader extends Downloader { public RecordFileDownloader( - FileClientWithProperties.Builder fileClientBuilder, AddressBookService addressBookService, + ParameterizedFileClient.Builder fileClientBuilder, AddressBookService addressBookService, RecordDownloaderProperties downloaderProperties, MeterRegistry meterRegistry, NodeSignatureVerifier nodeSignatureVerifier, SignatureFileReader signatureFileReader, RecordFileReader recordFileReader, diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java index c3befa020a5..1ec127985a2 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/AbstractDownloaderTest.java @@ -28,7 +28,7 @@ import static org.mockito.Mockito.verify; import com.hedera.mirror.common.util.DomainUtils; -import com.hedera.mirror.importer.downloader.client.FileClientWithProperties; +import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.logging.LoggingMeterRegistry; @@ -115,7 +115,7 @@ public abstract class AbstractDownloaderTest { protected FileCopier fileCopier; protected CommonDownloaderProperties commonDownloaderProperties; protected MirrorProperties mirrorProperties; - protected FileClientWithProperties.Builder fileClientBuilder; + protected ParameterizedFileClient.Builder fileClientBuilder; protected DownloaderProperties downloaderProperties; protected Downloader downloader; protected MeterRegistry meterRegistry = new LoggingMeterRegistry(); From b1f1c2f28b67225a4714623ec1726debb440f04b Mon Sep 17 00:00:00 2001 From: Victor ADASCALITEI <210908+3Nigma@users.noreply.github.com> Date: Sun, 16 Jan 2022 11:01:53 +0200 Subject: [PATCH 7/8] Added the complete 3-node addressbook taken from dockerized hedera-services runtime for local cloudProvider. --- .../src/main/resources/addressbook/local | Bin 887 -> 2665 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/hedera-mirror-importer/src/main/resources/addressbook/local b/hedera-mirror-importer/src/main/resources/addressbook/local index 34c5f68f183568afbde28b6dc24a0259b82ec485..8495ed8c2a067baa4521fd39857b089350cd0fb6 100644 GIT binary patch literal 2665 zcmb`JIkIdy5r*l}cou<0U}9l)WGa=JlFJI%f>Og;SOiO8DJ+M;=Bw8oK^vGIb)#-o zopUt&qrUw6$1h(r+fO=vlAr8PKgj3bpa1ov$!MahnaGil+17|YM!e*^eto%!*ypbC z`0(}qv7~+dMMcVYTVma=?(4CY&u6agHD*)mb7@J8<&U?0`#Sx->z-HZ&hWeLd8_GJ zV_D>9_Ejxvd7g2ONaixXns14x>=7;3S$kFPliRk5oqj4_RIHS=zWUScQ>2f_Z_(AN zmD|v^EyA0|ctdvCe%Z!5K5F;o$M=b2GgZc2Azkc^Y_p8@`P!#zdOs{{tc*~ptX^;C zvmPRMT7Y2=@0QhC&EQXbcVDx{>anM20F%7d-X0>WG1jpME;*R4aj{sQ4jpT_#6Gh= zqx({g^(@yN_m#}a{QlAb@M@3LT`@YmsIdE0KA!4v+ss!x#*M2Ga;H9>?%HpU4mkC; z>*wv>$3i8a=uUYa%TLLTp-1myL6FmCXl#2c-R0Pch*5F9cQ$y}W_Z97g?5`U+~0J4 z4!kllclE~h-py9S#<`L20CAk0jxNzm( z*BZlTlNUMnVSIn_D3|QLQSvLEydv|*i{JJqM zJ6H}yZPhw$e4wf&x#P@qw!;w87!Mzawr~IV-Cw_a`Tn1O_}zEE|Ht3I``b5P{_|_w z^`C9me-qm+>(}(7f2@>od$n2|kz&(DT%OG4l5uF}pt;w_rT$TbZvPQ&Z{ zUWw{ak_yK_Jh2c1&HEnqo3?bXytB{Vioh6-d!h~qp}7}wpR6&lvY!)>c$lhnTSG>; z7c;Pg#qBd?tyAaC5j&^OQyz*cKN4~DdncNjF{iW~xfh~Iu6ditZq7DL=7twNfhjiR zy(VN(?q_&3Cf*LS^@x&k1fbcWbG9>5YM}&Nxmd$|dx!0N0HrZ8PG%#ruxe%B%EMQh zV>6r~MO@+574hm}YJK+sn)(SDHR>$VCb@7W5US~szV5Ah=a!#K z)B-Lc!~E}dwwG!c)U#>kZwqw`xCfm?kslZTNqGe9~$ajwFLw+&Q+`%ptgC7+eQ2PJG5QBEAWm5mwYD zIRI$}OaE5=(>aq3n&M zo-jht5+E|-~T`3`On7l-^6$*E!rpi<|l(kY0?F^ z=qU%!Cq)wCq-gl?5N1wIO~iu=ob9+RLQ5&1q0nV)@DY||&=weh;DT$uh8s1r8Ps*a z5L!h88XfgN9tv`+Qxm(PyGw3i=$zIlY2VK`HSs~5GQo2%kB(kzG{V$klBEb7Z)exx zYB@W+7g1% zAS4p8m-J{PJU6t-2_fjc=`{fWAo`@?4prH2APGXDK~o22TCq{b*y4Px`U|dnxr8Lg z3@B|_7NhPeB6Z@q;K65uKwjvZ^FUVuy4_2nJg^DFNa0mrN1t=ZK}qB|WRlfDt%T(y zkAtfm!Y|lo{`~3?qp_8Q3hs#IUURai~xUR2NVDR From a1e0d2569bf395ca5958060d1b854206bd6cd1e5 Mon Sep 17 00:00:00 2001 From: Victor ADASCALITEI <210908+3Nigma@users.noreply.github.com> Date: Mon, 17 Jan 2022 10:50:12 +0200 Subject: [PATCH 8/8] Removed hanging @Log4j adnotations --- .../importer/downloader/client/ParameterizedFileClient.java | 2 -- .../importer/downloader/client/local/LocalFileClient.java | 2 -- .../mirror/importer/downloader/client/s3/S3FileClient.java | 2 -- .../mirror/importer/downloader/client/s3/S3PendingDownload.java | 2 -- 4 files changed, 8 deletions(-) diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/ParameterizedFileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/ParameterizedFileClient.java index 44912d18abf..79011cfa92e 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/ParameterizedFileClient.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/ParameterizedFileClient.java @@ -25,12 +25,10 @@ import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; import com.hedera.mirror.importer.downloader.DownloaderProperties; @Getter -@Log4j2 @RequiredArgsConstructor(access = AccessLevel.PROTECTED) public abstract class ParameterizedFileClient extends MultiFileClient { public interface Builder extends FileClient.Builder { } diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java index 84d4416fc4e..644a2a5b3b6 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/local/LocalFileClient.java @@ -37,14 +37,12 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; -import lombok.extern.log4j.Log4j2; import com.hedera.mirror.importer.domain.StreamFilename; import com.hedera.mirror.importer.downloader.client.DownloadResult; import com.hedera.mirror.importer.downloader.DownloaderProperties; import com.hedera.mirror.importer.downloader.client.PendingDownload; -@Log4j2 public class LocalFileClient extends ParameterizedFileClient { @RequiredArgsConstructor public static class Builder implements ParameterizedFileClient.Builder { diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java index 21abea7173b..7aa5ec4b86f 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3FileClient.java @@ -27,7 +27,6 @@ import com.hedera.mirror.importer.downloader.client.ParameterizedFileClient; import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectRequest; @@ -39,7 +38,6 @@ import com.hedera.mirror.importer.downloader.DownloaderProperties; import com.hedera.mirror.importer.downloader.client.PendingDownload; -@Log4j2 public class S3FileClient extends ParameterizedFileClient { @RequiredArgsConstructor public static class Builder implements ParameterizedFileClient.Builder { diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3PendingDownload.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3PendingDownload.java index c7b233e21ef..9aafcadf6fa 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3PendingDownload.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/client/s3/S3PendingDownload.java @@ -29,13 +29,11 @@ import lombok.AccessLevel; import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import com.hedera.mirror.importer.domain.StreamFilename; -@Log4j2 public class S3PendingDownload extends PendingDownload> { @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public static class S3DownloadResult implements DownloadResult {