Merge pull request #3 from pagopa/PAGOPA-1360-refactoring

ref: fix repository issue and update API [PAGOPA-1360]

.identity/00_data.tf

@@ -1,8 +1,3 @@
-data "azurerm_storage_account" "tf_storage_account"{
-  name                = "pagopainfraterraform${var.env}"
-  resource_group_name = "io-infra-rg"
-}
-
 data "azurerm_resource_group" "dashboards" {
   name = "dashboards"
 }

README.md

@@ -59,6 +59,16 @@ Start the micronaut application with this command:
 
 `mvn mn:run`
 
+### Rule of thumb
+
+Prevent more than 1 file in with macOS
+
+`zip -d filename.zip __MACOSX/\*`
+
+`zip -d filename.zip \*/.DS_Store`
+
+`zip file.zip uncompressed`
+
 ### Micronaut Profiles
 
 - **local**: to develop locally.

helm/Chart.yaml

@@ -1,9 +1,9 @@
 apiVersion: v2
-name: pagopa-afm-calculator
-description: Microservice that handles calculation for pagoPA Advanced Fees Management
+name: pagopa-gpd-upload
+description: Microservice that handles file upload of massive debt positions JSON object
 type: application
-version: 0.15.0
-appVersion: 0.1.1
+version: 0.19.0
+appVersion: 0.1.1-4-PAGOPA-1360-refactoring
 dependencies:
   - name: microservice-chart
     version: 2.8.0

helm/values-dev.yaml

@@ -4,7 +4,7 @@ microservice-chart:
   fullnameOverride: ""
   image:
     repository: ghcr.io/pagopa/pagopa-gpd-upload
-    tag: "0.1.1"
+    tag: "0.1.1-4-PAGOPA-1360-refactoring"
     pullPolicy: Always
   livenessProbe:
     httpGet:
@@ -66,6 +66,11 @@ microservice-chart:
     ENV: 'azure-kubernetes-dev'
     APP_LOGGING_LEVEL: 'DEBUG'
     DEFAULT_LOGGING_LEVEL: 'INFO'
+    COSMOS_URI: "https://pagopa-d-weu-gps-cosmos-account.documents.azure.com:443/"
+    DB_NAME: "gpd_db"
+    CONTAINER_NAME: "gpd_upload_status"
+    POST_FILE_RETRY_AFTER: '10000'
+    LOG_LEVEL: "DEBUG"
     #    CORS_CONFIGURATION: '{"origins": ["*"], "methods": ["*"]}'
     #    OTEL_SERVICE_NAME: # TODO
     #    OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=dev"
@@ -77,12 +82,8 @@ microservice-chart:
   envSecret:
     #    APPLICATIONINSIGHTS_CONNECTION_STRING: 'ai-d-connection-string'
     #    OTEL_EXPORTER_OTLP_HEADERS: elastic-apm-secret-token
-    BLOB_SAS_URL: 'gpd-upload-blob-sas-url'
-    BLOB_SAS_TOKEN: 'gpd-upload-blob-sas-token'
-    COSMOS_URI: 'gpd-upload-cosmos-uri'
+    BLOB_CONNECTION_STRING: 'gpd-upload-sa-connection-string'
     COSMOS_KEY: 'gpd-upload-cosmos-key'
-    DB_NAME: 'gpd-upload-cosmos-db-name'
-    CONTAINER_NAME: 'gpd-upload-cosmos-db-container-name'
   keyvault:
     name: "pagopa-d-gps-kv"
     tenantId: "7788edaf-0346-4068-9d79-c868aed15b3d"

helm/values-prod.yaml

@@ -4,7 +4,7 @@ microservice-chart:
   fullnameOverride: ""
   image:
     repository: ghcr.io/pagopa/yourname # TODO
-    tag: "0.1.1"
+    tag: "0.1.1-4-PAGOPA-1360-refactoring"
     pullPolicy: Always
   livenessProbe:
     httpGet:

helm/values-uat.yaml

@@ -4,7 +4,7 @@ microservice-chart:
   fullnameOverride: ""
   image:
     repository: ghcr.io/pagopa/yourname # TODO
-    tag: "0.1.1"
+    tag: "0.1.1-4-PAGOPA-1360-refactoring"
     pullPolicy: Always
   livenessProbe:
     httpGet:

openapi/openapi.json

@@ -1,5 +1,5 @@
 {
   "info": {
-    "version": "0.1.1"
+    "version": "0.1.1-4-PAGOPA-1360-refactoring"
   }
 }

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>it.gov.pagopa.gpd.upload</groupId>
     <artifactId>pagopa-gpd-upload</artifactId>
-    <version>0.1.1</version>
+    <version>0.1.1-4-PAGOPA-1360-refactoring</version>
     <packaging>${packaging}</packaging>
 
     <parent>
@@ -18,6 +18,7 @@
         <release.version>17</release.version>
         <micronaut.version>4.2.0</micronaut.version>
         <lombok.version>1.18.22</lombok.version>
+        <mockito-junit.version>4.1.0</mockito-junit.version>
         <micronaut.aot.enabled>false</micronaut.aot.enabled>
         <micronaut.aot.packageName>it.gov.pagopa.gpd.upload.aot.generated</micronaut.aot.packageName>
         <micronaut.runtime>netty</micronaut.runtime>
@@ -32,6 +33,12 @@
     </repositories>
 
     <dependencies>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-junit-jupiter</artifactId>
+            <version>${mockito-junit.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>io.micronaut</groupId>
             <artifactId>micronaut-http-server-netty</artifactId>

src/main/java/it/gov/pagopa/gpd/upload/controller/FileController.java

@@ -1,5 +1,9 @@
 package it.gov.pagopa.gpd.upload.controller;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.micronaut.context.annotation.Value;
+import io.micronaut.http.HttpHeaders;
 import io.micronaut.http.HttpResponse;
 import io.micronaut.http.HttpStatus;
 import io.micronaut.http.MediaType;
@@ -18,12 +22,16 @@
 import it.gov.pagopa.gpd.upload.model.ProblemJson;
 import it.gov.pagopa.gpd.upload.service.FileStatusService;
 import it.gov.pagopa.gpd.upload.service.FileUploadService;
+import jakarta.annotation.PostConstruct;
 import jakarta.inject.Inject;
 import jakarta.validation.constraints.NotBlank;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 
 import java.io.IOException;
+import java.net.URI;
+import java.text.SimpleDateFormat;
 
 @ExecuteOn(TaskExecutors.IO)
 @Controller()
@@ -36,12 +44,24 @@ public class FileController {
     @Inject
     FileStatusService fileStatusService;
 
+    @Value("${post.file.response.headers.retry_after.millis}")
+    private int retryAfter;
+    private ObjectMapper objectMapper;
+
+    @PostConstruct
+    public void init() {
+        objectMapper = new ObjectMapper();
+        objectMapper.registerModule(new JavaTimeModule());
+        objectMapper.setDateFormat(new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS"));
+    }
+
+    @SneakyThrows
     @Operation(summary = "The Organization creates the debt positions listed in the file.", security = {@SecurityRequirement(name = "ApiKey"), @SecurityRequirement(name = "Authorization")}, operationId = "createMassivePositions")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "201", description = "Request created."),
+            @ApiResponse(responseCode = "202", description = "Request accepted."),
             @ApiResponse(responseCode = "400", description = "Malformed request.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ProblemJson.class))),
             @ApiResponse(responseCode = "401", description = "Wrong or missing function key.", content = @Content(schema = @Schema())),
-            @ApiResponse(responseCode = "409", description = "Conflict: duplicate debt position found.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ProblemJson.class))),
+            @ApiResponse(responseCode = "409", description = "Conflict: duplicate file found.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ProblemJson.class))),
             @ApiResponse(responseCode = "500", description = "Service unavailable.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ProblemJson.class)))})
     @Consumes(MediaType.MULTIPART_FORM_DATA)
     @Post("/organizations/{organizationFiscalCode}/debtpositions/file")
@@ -50,30 +70,33 @@ public HttpResponse uploadDebtPositionFile(@Parameter(description = "The organiz
         String key = fileUploadService.upload(organizationFiscalCode, file);
         log.debug("A file with name: " + file.getFilename() + " has been uploaded");
 
-        return HttpResponse.status(HttpStatus.OK)
-                .body(key);
+        String uri = "/organizations/" + organizationFiscalCode +"/debtpositions/file/" + key +"/status";
+        HttpResponse response = HttpResponse.accepted(new URI(uri));
+
+        return response.toMutableResponse().header(HttpHeaders.RETRY_AFTER, retryAfter + " ms");
     }
 
+    @SneakyThrows
     @Operation(summary = "Returns the upload status of debt positions uploaded via file.", security = {@SecurityRequirement(name = "ApiKey"), @SecurityRequirement(name = "Authorization")}, operationId = "createMassivePositions")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "201", description = "Request created."),
+            @ApiResponse(responseCode = "200", description = "Upload found."),
             @ApiResponse(responseCode = "400", description = "Malformed request.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ProblemJson.class))),
             @ApiResponse(responseCode = "401", description = "Wrong or missing function key.", content = @Content(schema = @Schema())),
-            @ApiResponse(responseCode = "409", description = "Conflict: duplicate debt position found.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ProblemJson.class))),
+            @ApiResponse(responseCode = "404", description = "Upload not found.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ProblemJson.class))),
             @ApiResponse(responseCode = "500", description = "Service unavailable.", content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ProblemJson.class)))})
     @Get(value = "/organizations/{organizationFiscalCode}/debtpositions/file/{fileId}/status",
             produces = MediaType.APPLICATION_JSON)
-    HttpResponse<FileStatus> getFileSatus(
+    HttpResponse<String> getFileSatus(
             @Parameter(description = "The organization fiscal code", required = true)
             @NotBlank @PathVariable String organizationFiscalCode,
             @Parameter(description = "The fiscal code of the Organization.", required = true)
             @NotBlank @PathVariable String fileId) {
 
-        FileStatus fileStatus = fileStatusService.getStatus(fileId);
+        FileStatus fileStatus = fileStatusService.getStatus(fileId, organizationFiscalCode);
         log.debug("The Status related to file-id " + fileId + " has been found");
 
         return HttpResponse.status(HttpStatus.OK)
-                .body(fileStatus);
+                .contentType(MediaType.APPLICATION_JSON)
+                .body(objectMapper.writeValueAsString(fileStatus));
     }
-
 }

src/main/java/it/gov/pagopa/gpd/upload/entity/FailedIUPD.java

@@ -0,0 +1,16 @@
+package it.gov.pagopa.gpd.upload.entity;
+
+import lombok.*;
+
+import java.util.List;
+
+@Builder(toBuilder = true)
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class FailedIUPD {
+    private int errorCode;
+    private String details;
+    private List<String> skippedIUPDs;
+}

src/main/java/it/gov/pagopa/gpd/upload/entity/Upload.java

@@ -15,9 +15,13 @@ public class Upload {
     private int current;
     private int total;
     private ArrayList<String> successIUPD;
-    private ArrayList<String> failedIUPD;
+    private ArrayList<FailedIUPD> failedIUPDs;
     @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss")
     private LocalDateTime start;
     @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss")
     private LocalDateTime end;
+
+    public void addFailures(FailedIUPD failedIUPD) {
+        this.failedIUPDs.add(failedIUPD);
+    }
 }

src/main/java/it/gov/pagopa/gpd/upload/exception/ErrorHandler.java

@@ -17,7 +17,7 @@ public class ErrorHandler implements ExceptionHandler<AppException, HttpResponse
 
     @Override
     public HttpResponse handle(HttpRequest request, AppException exception) {
-        log.error("[ERROR] AppException raised: ", exception);
+        log.error("[ERROR] AppException raised: ", exception.toString());
 
         ProblemJson errorResponse = ProblemJson.builder()
                 .status(exception.getHttpStatus().getCode())

src/main/java/it/gov/pagopa/gpd/upload/model/FileStatus.java

@@ -10,8 +10,11 @@
 @Setter
 @NoArgsConstructor
 @AllArgsConstructor
+@ToString
 public class FileStatus {
     public String fileId;
+    public int processed;
+    public int uploaded;
     private ArrayList<String> successIUPD;
     private ArrayList<String> failedIUPD;
     private LocalDateTime uploadTime;

src/main/java/it/gov/pagopa/gpd/upload/repository/BlobStorageRepository.java

@@ -26,11 +26,8 @@
 @Slf4j
 public class BlobStorageRepository implements FileRepository {
 
-    @Value("${blob.sas.url}")
-    private String blobURL;
-
-    @Value("${blob.sas.token}")
-    private String blobToken;
+    @Value("${blob.sas.connection}")
+    private String connectionString;
 
     @Value("${blob.container.input}")
     private String inputContainer;
@@ -40,8 +37,7 @@ public class BlobStorageRepository implements FileRepository {
     @PostConstruct
     public void init() {
         blobServiceClient = new BlobServiceClientBuilder()
-                .endpoint(blobURL)
-                .sasToken(blobToken)
+                .connectionString(connectionString)
                 .buildClient();
     }
 

src/main/java/it/gov/pagopa/gpd/upload/repository/StatusRepository.java

@@ -3,6 +3,7 @@
 import com.azure.cosmos.CosmosClientBuilder;
 import com.azure.cosmos.CosmosClient;
 import com.azure.cosmos.CosmosContainer;
+import com.azure.cosmos.CosmosException;
 import com.azure.cosmos.models.CosmosItemResponse;
 import com.azure.cosmos.models.PartitionKey;
 import io.micronaut.context.annotation.Context;
@@ -44,20 +45,28 @@ public void init() {
     }
 
     public Status saveStatus(Status status) {
-        CosmosItemResponse<Status> response = container.createItem(status);
-        if (response.getStatusCode() != HttpStatus.CREATED.getCode()) {
-            log.error("the Status saving was not successful: " + response);
-            throw new AppException(HttpStatus.SERVICE_UNAVAILABLE, "COSMOS UNAVAILABLE", "the Status saving was not successful");
+        try {
+            CosmosItemResponse<Status> response = container.createItem(status);
+            return response.getItem();
+        } catch (CosmosException ex) {
+            log.error("the Status retrieval was not successful: " + ex.getStatusCode());
+            if(ex.getStatusCode() == HttpStatus.INTERNAL_SERVER_ERROR.getCode())
+                throw new AppException(HttpStatus.INTERNAL_SERVER_ERROR, HttpStatus.INTERNAL_SERVER_ERROR.name(), "Status saving unavailable");
+            else
+                throw new AppException(HttpStatus.valueOf(ex.getStatusCode()), String.valueOf(ex.getStatusCode()), "Status saving failed");
         }
-        return response.getItem();
     }
 
-    public Status findStatusById(String id) {
-        CosmosItemResponse<Status> response = container.readItem(id, PartitionKey.NONE, Status.class);
-        if (response.getStatusCode() != HttpStatus.OK.getCode()) {
-            log.error("the Status retrieval was not successful: " + response);
-            throw new AppException(HttpStatus.SERVICE_UNAVAILABLE, "COSMOS UNAVAILABLE", "the Status retrieval was not successful");
+    public Status findStatusById(String id, String fiscalCode) {
+        try {
+            CosmosItemResponse<Status> response = container.readItem(id, new PartitionKey(fiscalCode), Status.class);
+            return response.getItem();
+        } catch (CosmosException ex) {
+            log.error("the Status retrieval was not successful: " + ex.getStatusCode());
+            if(ex.getStatusCode() == HttpStatus.INTERNAL_SERVER_ERROR.getCode())
+                throw new AppException(HttpStatus.INTERNAL_SERVER_ERROR, HttpStatus.INTERNAL_SERVER_ERROR.name(), "Status saving unavailable");
+            else
+                throw new AppException(HttpStatus.valueOf(ex.getStatusCode()), String.valueOf(ex.getStatusCode()), "Status retrieval failed");
         }
-        return response.getItem();
     }
 }