From 6837496892e750be0194d94380839a891efebd2c Mon Sep 17 00:00:00 2001 From: Alessio Cialini Date: Thu, 8 Aug 2024 15:42:38 +0200 Subject: [PATCH 1/3] fix: updated compression method to avoid mac-os related errors --- .../storage/NoticeStorageClient.java | 43 ++++++++++++------- .../utils/WorkingDirectoryUtils.java | 35 +++++++++++++++ 2 files changed, 62 insertions(+), 16 deletions(-) create mode 100644 src/main/java/it/gov/pagopa/print/payment/notice/functions/utils/WorkingDirectoryUtils.java diff --git a/src/main/java/it/gov/pagopa/print/payment/notice/functions/storage/NoticeStorageClient.java b/src/main/java/it/gov/pagopa/print/payment/notice/functions/storage/NoticeStorageClient.java index ce83fd0..62ee99e 100644 --- a/src/main/java/it/gov/pagopa/print/payment/notice/functions/storage/NoticeStorageClient.java +++ b/src/main/java/it/gov/pagopa/print/payment/notice/functions/storage/NoticeStorageClient.java @@ -6,6 +6,7 @@ import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.ListBlobsOptions; import it.gov.pagopa.print.payment.notice.functions.model.response.BlobStorageResponse; +import it.gov.pagopa.print.payment.notice.functions.utils.WorkingDirectoryUtils; import lombok.extern.slf4j.Slf4j; import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Autowired; @@ -13,12 +14,14 @@ import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static it.gov.pagopa.print.payment.notice.functions.utils.WorkingDirectoryUtils.createWorkingDirectory; + @Component @Slf4j public class NoticeStorageClient { @@ -48,11 +51,16 @@ public NoticeStorageClient( public BlobStorageResponse compressFolder(String folderId) throws IOException { log.info("Create Zip file. Request {}", folderId); - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - + File workingDirectory = createWorkingDirectory(); + Path tempDirectory = Files.createTempDirectory(workingDirectory.toPath(), "notice-generation-function") + .normalize() + .toAbsolutePath(); + File pathFile = File.createTempFile("tempFile", ".zip", tempDirectory.toFile()); - try (ZipOutputStream zipStream = new ZipOutputStream(outputStream)) { -// List> futures = new ArrayList<>(); + try { + try (FileOutputStream fos = new FileOutputStream(pathFile); + BufferedOutputStream outputStream = new BufferedOutputStream(fos); + ZipOutputStream zipStream = new ZipOutputStream(outputStream)) { String delimiter = "/"; ListBlobsOptions options = new ListBlobsOptions() @@ -73,8 +81,6 @@ public BlobStorageResponse compressFolder(String folderId) throws IOException { log.info("Get info file {} from blob. Request {}", blobItem.getName(), folderId); final BlobClient blobClient = blobContainerClient.getBlobClient(blobItem.getName()); - -// CompletableFuture future = CompletableFuture.runAsync(() -> { try (ByteArrayOutputStream fileOutputStream = new ByteArrayOutputStream()) { if (blobClient.exists()) { log.info("put file {} into zipStream. Request {}", blobItem.getName(), folderId); @@ -82,6 +88,7 @@ public BlobStorageResponse compressFolder(String folderId) throws IOException { zipStream.putNextEntry(new ZipEntry(finalSingleFileName)); zipStream.write(fileOutputStream.toByteArray()); zipStream.closeEntry(); + zipStream.flush(); } else { log.error("file not found: {}", finalSingleFileName); throw new RuntimeException("File not found: " + finalSingleFilepath); @@ -91,22 +98,24 @@ public BlobStorageResponse compressFolder(String folderId) throws IOException { throw new RuntimeException("Error processing file: " + finalSingleFileName, e); } log.info("Download file completed. Request {}", folderId); -// }); -// futures.add(future); } }); -// CompletableFuture allOf = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); -// allOf.join(); - // TODO : zip library is not thread-safe. Make multithread in the future with another library + zipStream.finish(); + + + } catch (IOException e) { + throw new RuntimeException(e); + } + try (FileInputStream fis = new FileInputStream(pathFile); + BufferedInputStream bif = new BufferedInputStream(fis)) { BlobClient zipFileClient = blobContainerClient.getBlobClient( folderId + "/" + folderId.concat(".zip")); - zipFileClient.upload(new ByteArrayInputStream( - outputStream.toByteArray()), outputStream.size(), true); + zipFileClient.upload(bif, true); log.info("Zip file uploaded. Request {}", folderId); } catch (IOException e) { throw new RuntimeException(e); @@ -115,6 +124,8 @@ public BlobStorageResponse compressFolder(String folderId) throws IOException { BlobStorageResponse blobStorageResponse = new BlobStorageResponse(); blobStorageResponse.setStatusCode(HttpStatus.OK.value()); return blobStorageResponse; + } finally { + WorkingDirectoryUtils.clearTempDirectory(tempDirectory); } } } diff --git a/src/main/java/it/gov/pagopa/print/payment/notice/functions/utils/WorkingDirectoryUtils.java b/src/main/java/it/gov/pagopa/print/payment/notice/functions/utils/WorkingDirectoryUtils.java new file mode 100644 index 0000000..b51b956 --- /dev/null +++ b/src/main/java/it/gov/pagopa/print/payment/notice/functions/utils/WorkingDirectoryUtils.java @@ -0,0 +1,35 @@ +package it.gov.pagopa.print.payment.notice.functions.utils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +/** + * Utils methods for working directory + */ +@Slf4j +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class WorkingDirectoryUtils { + + public static File createWorkingDirectory() throws IOException { + File workingDirectory = new File("temp"); + if(!workingDirectory.exists()) { + Files.createDirectory(workingDirectory.toPath()); + } + return workingDirectory; + } + + public static void clearTempDirectory(java.nio.file.Path workingDirPath) { + try { + FileUtils.deleteDirectory(workingDirPath.toFile()); + } catch (IOException e) { + log.warn("Unable to clear working directory", e); + } + } + +} From e7facbb461bf01bb80077dbe0d60be6f0fa2c5a2 Mon Sep 17 00:00:00 2001 From: Alessio Cialini Date: Thu, 8 Aug 2024 15:43:34 +0200 Subject: [PATCH 2/3] fix: updated compression method to avoid mac-os related errors --- openapi/openapi.json | 65 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/openapi/openapi.json b/openapi/openapi.json index 8b13789..ad9f67e 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -1 +1,64 @@ - +{ + "openapi" : "3.0.1", + "info" : { + "description" : "PagoPA Print Payment Notices Functions", + "termsOfService" : "https://www.pagopa.gov.it/", + "title" : "pagopa-print-payment-notice-functions", + "version" : "1.0.0" + }, + "servers" : [ { + "url" : "http://localhost", + "description" : "Generated server url" + } ], + "paths" : { + "/info" : { + "get" : { + "description" : "Return OK if application is started", + "operationId" : "healthCheck", + "responses" : { + "200" : { + "content" : { + "*/*" : { + "schema" : { + "$ref" : "#/components/schemas/AppInfo" + } + } + }, + "description" : "OK" + } + }, + "security" : [ { + "ApiKey" : [ ] + } ], + "summary" : "health check", + "tags" : [ "Home" ] + } + } + }, + "components" : { + "schemas" : { + "AppInfo" : { + "type" : "object", + "properties" : { + "environment" : { + "type" : "string" + }, + "name" : { + "type" : "string" + }, + "version" : { + "type" : "string" + } + } + } + }, + "securitySchemes" : { + "ApiKey" : { + "description" : "The API key to access this function app.", + "in" : "header", + "name" : "Ocp-Apim-Subscription-Key", + "type" : "apiKey" + } + } + } +} \ No newline at end of file From b296f721b6443a64e0bf35a229c6d3cb55a25aa7 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Thu, 8 Aug 2024 13:55:08 +0000 Subject: [PATCH 3/3] Bump to version 1.0.0-1-hotfix-zip-function [skip ci] --- helm/Chart.yaml | 4 +- helm/values-dev.yaml | 2 +- helm/values-prod.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi.json | 96 +++++++++++++++++++++++-------------------- pom.xml | 2 +- 6 files changed, 57 insertions(+), 51 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index efd2931..2475f5e 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-print-payment-notice-functions description: Microservice that handles services for notice print generation type: application -version: 0.38.0 -appVersion: 1.0.0 +version: 0.39.0 +appVersion: 1.0.0-1-hotfix-zip-function dependencies: - name: microservice-chart version: 2.8.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 4cd3a80..ad7561d 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "print-payment-notice-functions" image: repository: ghcr.io/pagopa/pagopa-print-payment-notice-functions - tag: "1.0.0" + tag: "1.0.0-1-hotfix-zip-function" pullPolicy: Always livenessProbe: httpGet: diff --git a/helm/values-prod.yaml b/helm/values-prod.yaml index 7c45359..65b317d 100644 --- a/helm/values-prod.yaml +++ b/helm/values-prod.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "print-payment-notice-functions" image: repository: ghcr.io/pagopa/pagopa-print-payment-notice-functions - tag: "1.0.0" + tag: "1.0.0-1-hotfix-zip-function" pullPolicy: Always livenessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 4362e7d..0490195 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "print-payment-notice-functions" image: repository: ghcr.io/pagopa/pagopa-print-payment-notice-functions - tag: "1.0.0" + tag: "1.0.0-1-hotfix-zip-function" pullPolicy: Always livenessProbe: httpGet: diff --git a/openapi/openapi.json b/openapi/openapi.json index ad9f67e..c817e70 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -1,64 +1,70 @@ { - "openapi" : "3.0.1", - "info" : { - "description" : "PagoPA Print Payment Notices Functions", - "termsOfService" : "https://www.pagopa.gov.it/", - "title" : "pagopa-print-payment-notice-functions", - "version" : "1.0.0" + "openapi": "3.0.1", + "info": { + "description": "PagoPA Print Payment Notices Functions", + "termsOfService": "https://www.pagopa.gov.it/", + "title": "pagopa-print-payment-notice-functions", + "version": "1.0.0-1-hotfix-zip-function" }, - "servers" : [ { - "url" : "http://localhost", - "description" : "Generated server url" - } ], - "paths" : { - "/info" : { - "get" : { - "description" : "Return OK if application is started", - "operationId" : "healthCheck", - "responses" : { - "200" : { - "content" : { - "*/*" : { - "schema" : { - "$ref" : "#/components/schemas/AppInfo" + "servers": [ + { + "url": "http://localhost", + "description": "Generated server url" + } + ], + "paths": { + "/info": { + "get": { + "description": "Return OK if application is started", + "operationId": "healthCheck", + "responses": { + "200": { + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/AppInfo" } } }, - "description" : "OK" + "description": "OK" } }, - "security" : [ { - "ApiKey" : [ ] - } ], - "summary" : "health check", - "tags" : [ "Home" ] + "security": [ + { + "ApiKey": [] + } + ], + "summary": "health check", + "tags": [ + "Home" + ] } } }, - "components" : { - "schemas" : { - "AppInfo" : { - "type" : "object", - "properties" : { - "environment" : { - "type" : "string" + "components": { + "schemas": { + "AppInfo": { + "type": "object", + "properties": { + "environment": { + "type": "string" }, - "name" : { - "type" : "string" + "name": { + "type": "string" }, - "version" : { - "type" : "string" + "version": { + "type": "string" } } } }, - "securitySchemes" : { - "ApiKey" : { - "description" : "The API key to access this function app.", - "in" : "header", - "name" : "Ocp-Apim-Subscription-Key", - "type" : "apiKey" + "securitySchemes": { + "ApiKey": { + "description": "The API key to access this function app.", + "in": "header", + "name": "Ocp-Apim-Subscription-Key", + "type": "apiKey" } } } -} \ No newline at end of file +} diff --git a/pom.xml b/pom.xml index 00c9fc3..2498b90 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ it.gov.pagopa print-payment-notice-functions - 1.0.0 + 1.0.0-1-hotfix-zip-function pagopa-print-payment-notice-functions PagoPA Print Payment Notices Functions