Skip to content

Commit

Permalink
Merge pull request #30 from pagopa/PAGOPA-1595-elk
Browse files Browse the repository at this point in the history
feat: Add OpenTelemetry agent [PAGOPA-1595]
  • Loading branch information
pasqualespica authored May 13, 2024
2 parents 8dade7c + 25ba141 commit 35de6f8
Show file tree
Hide file tree
Showing 14 changed files with 193 additions and 54 deletions.
10 changes: 10 additions & 0 deletions .env.sample
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
export BLOB_SAS_URL="<to-set>"
export BLOB_SAS_TOKEN="<to-set>"
export BLOB_CONNECTION_STRING="<to-set>"
COSMOS_URI="<to-set>"
COSMOS_KEY="<to-set>"
DB_NAME="<gpd-db-name>"
CONTAINER_NAME="gpd-upload-status-container-name"
POST_FILE_RETRY_AFTER="10000"
LOG_LEVEL=INFO
OTEL_SERVICE_NAME=pagopa-gpd-upload
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ hs_err_pid*
# Project files
/target/
**/node_modules
**/.env
**/**.env
**.env
.cache_ggshield

# Helm
Expand Down
15 changes: 8 additions & 7 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
#
# Build stage
#
FROM maven:3.8.2-openjdk-17-slim AS build
COPY src /home/app/src
COPY pom.xml /home/app
RUN mvn -f /home/app/pom.xml clean package -Dmaven.test.skip=true
FROM maven:3.8.4-openjdk-17-slim AS build
WORKDIR /app
COPY pom.xml .
COPY src ./src
RUN mvn -B -DskipTests=true clean package -Dmaven.test.skip=true

#
# Package stage
#
FROM openjdk:17-alpine
COPY --from=build /home/app/target/pagopa-gpd-upload*.jar /usr/local/lib/app.jar
RUN true
ADD https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.25.1/opentelemetry-javaagent.jar /opt/opentelemetry-javaagent.jar
COPY --from=build /app/target/pagopa-gpd-upload*.jar /app/app.jar
EXPOSE 8080
ENTRYPOINT ["java","-jar","/usr/local/lib/app.jar"]
ENTRYPOINT ["java", "-javaagent:/opt/opentelemetry-javaagent.jar", "-jar", "/app/app.jar"]
18 changes: 13 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
## GPD Massive Upload µ-service

[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=TODO-set-your-id&metric=alert_status)](https://sonarcloud.io/dashboard?id=TODO-set-your-id)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=pagopa_pagopa-gpd-upload&metric=alert_status)](https://sonarcloud.io/dashboard?id=pagopa_pagopa-gpd-upload)
[![Integration Tests](https://github.com/pagopa/pagopa-gpd-upload/actions/workflows/integration_test.yml/badge.svg)](https://github.com/pagopa/pagopa-gpd-upload/actions/workflows/integration_test.yml)


Expand All @@ -10,10 +10,6 @@ It allows the creditor institutions to:
- Get Debt Positions massive Upload status
- Get Debt Positions massive Upload report

TODO: generate a index with this tool: https://ecotrust-canada.github.io/markdown-toc/

TODO: resolve all the TODOs in this template

---

## Api Documentation 📖
Expand Down Expand Up @@ -59,6 +55,18 @@ docker run -p 8080:8080 --env-file <docker-env-file> <container-name>

### Run the project

```
cp .env.sample .env
```

After setting all required environment variables

```
set -o allexport
source .env
set +o allexport
```

Start the micronaut application with this command:

`mvn mn:run`
Expand Down
4 changes: 2 additions & 2 deletions helm/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ apiVersion: v2
name: pagopa-gpd-upload
description: Microservice that handles file upload of massive debt positions JSON object
type: application
version: 0.68.0
appVersion: 0.1.23
version: 0.74.0
appVersion: 0.1.23-6-PAGOPA-1595-elk
dependencies:
- name: microservice-chart
version: 2.8.0
Expand Down
17 changes: 7 additions & 10 deletions helm/values-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ microservice-chart:
fullnameOverride: ""
image:
repository: ghcr.io/pagopa/pagopa-gpd-upload
tag: "0.1.23"
tag: "0.1.23-6-PAGOPA-1595-elk"
pullPolicy: Always
livenessProbe:
httpGet:
Expand Down Expand Up @@ -73,17 +73,14 @@ microservice-chart:
CONTAINER_NAME: "gpd_upload_status"
POST_FILE_RETRY_AFTER: '10000'
LOG_LEVEL: "DEBUG"
# CORS_CONFIGURATION: '{"origins": ["*"], "methods": ["*"]}'
# OTEL_SERVICE_NAME: # TODO
# OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=dev"
# OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.elastic-system.svc:4317"
# OTEL_TRACES_EXPORTER: otlp
# OTEL_METRICS_EXPORTER: otlp
# OTEL_LOGS_EXPORTER: none
# OTEL_TRACES_SAMPLER: "always_on"
OTEL_SERVICE_NAME: "pagopa-gpd-upload"
OTEL_RESOURCE_ATTRIBUTES: "service.name=pagopa-gpd-upload,deployment.environment=dev"
OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.elastic-system.svc:4317"
OTEL_LOGS_EXPORTER: none
OTEL_TRACES_SAMPLER: "always_on"
envSecret:
# APPLICATIONINSIGHTS_CONNECTION_STRING: 'ai-d-connection-string'
# OTEL_EXPORTER_OTLP_HEADERS: elastic-apm-secret-token
OTEL_EXPORTER_OTLP_HEADERS: elastic-apm-secret-token
BLOB_CONNECTION_STRING: 'gpd-upload-sa-connection-string'
COSMOS_KEY: 'gpd-upload-db-key'
keyvault:
Expand Down
17 changes: 7 additions & 10 deletions helm/values-prod.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ microservice-chart:
fullnameOverride: ""
image:
repository: ghcr.io/pagopa/pagopa-gpd-upload
tag: "0.1.23"
tag: "0.1.23-6-PAGOPA-1595-elk"
pullPolicy: Always
livenessProbe:
httpGet:
Expand Down Expand Up @@ -78,17 +78,14 @@ microservice-chart:
CONTAINER_NAME: "gpd_upload_status"
POST_FILE_RETRY_AFTER: '2000'
LOG_LEVEL: "INFO"
# CORS_CONFIGURATION: '{"origins": ["*"], "methods": ["*"]}'
# OTEL_SERVICE_NAME: # TODO
# OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=prod"
# OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.elastic-system.svc:4317"
# OTEL_TRACES_EXPORTER: otlp
# OTEL_METRICS_EXPORTER: otlp
# OTEL_LOGS_EXPORTER: none
# OTEL_TRACES_SAMPLER: "always_on"
OTEL_SERVICE_NAME: "pagopa-gpd-upload"
OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=prod"
OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.elastic-system.svc:4317"
OTEL_LOGS_EXPORTER: none
OTEL_TRACES_SAMPLER: "always_on"
envSecret:
# APPLICATIONINSIGHTS_CONNECTION_STRING: 'ai-p-connection-string'
# OTEL_EXPORTER_OTLP_HEADERS: elastic-apm-secret-token
OTEL_EXPORTER_OTLP_HEADERS: elastic-apm-secret-token
BLOB_CONNECTION_STRING: 'gpd-upload-sa-connection-string'
COSMOS_KEY: 'gpd-upload-db-key'
keyvault:
Expand Down
17 changes: 7 additions & 10 deletions helm/values-uat.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ microservice-chart:
fullnameOverride: ""
image:
repository: ghcr.io/pagopa/pagopa-gpd-upload
tag: "0.1.23"
tag: "0.1.23-6-PAGOPA-1595-elk"
pullPolicy: Always
livenessProbe:
httpGet:
Expand Down Expand Up @@ -73,17 +73,14 @@ microservice-chart:
CONTAINER_NAME: "gpd_upload_status"
POST_FILE_RETRY_AFTER: '10000'
LOG_LEVEL: "INFO"
# CORS_CONFIGURATION: '{"origins": ["*"], "methods": ["*"]}'
# OTEL_SERVICE_NAME: # TODO
# OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=dev"
# OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.elastic-system.svc:4317"
# OTEL_TRACES_EXPORTER: otlp
# OTEL_METRICS_EXPORTER: otlp
# OTEL_LOGS_EXPORTER: none
# OTEL_TRACES_SAMPLER: "always_on"
OTEL_SERVICE_NAME: "pagopa-gpd-upload"
OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=uat"
OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.elastic-system.svc:4317"
OTEL_LOGS_EXPORTER: none
OTEL_TRACES_SAMPLER: "always_on"
envSecret:
# APPLICATIONINSIGHTS_CONNECTION_STRING: 'ai-d-connection-string'
# OTEL_EXPORTER_OTLP_HEADERS: elastic-apm-secret-token
OTEL_EXPORTER_OTLP_HEADERS: elastic-apm-secret-token
BLOB_CONNECTION_STRING: 'gpd-upload-sa-connection-string'
COSMOS_KEY: 'gpd-upload-db-key'
keyvault:
Expand Down
2 changes: 1 addition & 1 deletion openapi/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"description": "Microservice to manage PagoPA GPD Upload",
"termsOfService": "https://www.pagopa.gov.it/",
"title": "pagoPA GPD Upload",
"version": "0.1.23"
"version": "0.1.23-6-PAGOPA-1595-elk"
},
"servers": [
{
Expand Down
14 changes: 13 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>it.gov.pagopa.gpd.upload</groupId>
<artifactId>pagopa-gpd-upload</artifactId>
<version>0.1.23</version>
<version>0.1.23-6-PAGOPA-1595-elk</version>
<packaging>${packaging}</packaging>

<parent>
Expand All @@ -23,6 +23,8 @@
<jackson.datatype.version>2.17.0</jackson.datatype.version>
<mockito-junit.version>4.1.0</mockito-junit.version>
<micronaut.aot.enabled>false</micronaut.aot.enabled>
<logback.ecs.version>1.6.0</logback.ecs.version>
<reactivex.rxjava.version>3.1.8</reactivex.rxjava.version>
<micronaut.aot.packageName>it.gov.pagopa.gpd.upload.aot.generated</micronaut.aot.packageName>
<micronaut.runtime>netty</micronaut.runtime>
<exec.mainClass>it.gov.pagopa.gpd.upload.Application</exec.mainClass>
Expand All @@ -36,6 +38,11 @@
</repositories>

<dependencies>
<dependency>
<groupId>io.reactivex.rxjava3</groupId>
<artifactId>rxjava</artifactId>
<version>${reactivex.rxjava.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
Expand Down Expand Up @@ -126,6 +133,11 @@
<version>5.0.0</version>
</dependency>
<!-- validation END -->
<dependency>
<groupId>co.elastic.logging</groupId>
<artifactId>logback-ecs-encoder</artifactId>
<version>${logback.ecs.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package it.gov.pagopa.gpd.upload.config;

import io.micronaut.aop.Around;
import java.lang.annotation.*;
import static java.lang.annotation.ElementType.*;
import static java.lang.annotation.RetentionPolicy.RUNTIME;

@Documented
@Retention(RUNTIME)
@Target({TYPE, METHOD})
@Around
public @interface ExampleAnnotation {
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package it.gov.pagopa.gpd.upload.config;

import io.micronaut.aop.InterceptorBean;
import io.micronaut.aop.MethodInterceptor;
import io.micronaut.aop.MethodInvocationContext;
import io.micronaut.core.annotation.Nullable;

import jakarta.inject.Singleton;


@Singleton
@InterceptorBean(ExampleAnnotation.class)
public class ExampleInterceptor implements MethodInterceptor<Object, Object> {

@Nullable
@Override
public Object intercept(MethodInvocationContext<Object, Object> context) {
// <ExampleInterceptor proxy code here>
// @ExampleAnnotation to use it around method
return context.proceed();
}
}
77 changes: 77 additions & 0 deletions src/main/java/it/gov/pagopa/gpd/upload/config/LogAspect.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package it.gov.pagopa.gpd.upload.config;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.micronaut.http.HttpRequest;
import io.micronaut.http.MutableHttpResponse;
import io.micronaut.http.annotation.Filter;
import io.micronaut.http.filter.HttpServerFilter;
import io.micronaut.http.filter.ServerFilterChain;
import io.reactivex.rxjava3.core.Flowable;
import lombok.extern.slf4j.Slf4j;
import org.reactivestreams.Publisher;
import org.slf4j.MDC;

import java.util.UUID;


@Filter("/*")
@Slf4j
public class LogAspect implements HttpServerFilter {
public static final String START_TIME = "startTime";
public static final String METHOD = "method";
public static final String STATUS = "status";
public static final String CODE = "httpCode";
public static final String RESPONSE_TIME = "responseTime";
public static final String RESPONSE = "response";
public static final String FAULT_CODE = "faultCode";
public static final String FAULT_DETAIL = "faultDetail";
public static final String REQUEST_ID = "requestId";
public static final String OPERATION_ID = "operationId";
public static final String ARGS = "args";

@Override
public Publisher<MutableHttpResponse<?>> doFilter(HttpRequest<?> request, ServerFilterChain chain) {
long startTime = System.currentTimeMillis();
String path = request.getPath();
MDC.put(METHOD, request.getMethod() + " " + path.substring(path.lastIndexOf('/')));
MDC.put(START_TIME, String.valueOf(System.currentTimeMillis()));
MDC.put(OPERATION_ID, UUID.randomUUID().toString());
if(MDC.get(REQUEST_ID) == null) {
var requestId = UUID.randomUUID().toString();
MDC.put(REQUEST_ID, requestId);
}
String params = request.getParameters().asMap().toString();
MDC.put(ARGS, params);

log.info("Invoking API operation {} - args: {}", request.getMethodName(), params);

return Flowable.fromPublisher(chain.proceed(request)).flatMap(response -> {

MDC.put(STATUS, "OK");
MDC.put(CODE, String.valueOf(response.getStatus().getCode()));
MDC.put(RESPONSE_TIME, String.valueOf(System.currentTimeMillis() - startTime));
MDC.put(RESPONSE, toJsonString(response.getBody().toString()));
log.info("Successful API operation {} - result: {}", request.getMethodName(), response);
MDC.remove(RESPONSE);
MDC.remove(STATUS);
MDC.remove(CODE);
MDC.remove(RESPONSE_TIME);
MDC.remove(START_TIME);

return Flowable.just(response);
});
}

private static String toJsonString(Object param) {
try {
return new ObjectMapper()
.registerModule(new JavaTimeModule())
.writeValueAsString(param);
} catch (JsonProcessingException e) {
log.warn("An error occurred when trying to parse a parameter", e);
return "parsing error";
}
}
}
18 changes: 11 additions & 7 deletions src/main/resources/logback.xml
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>

<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%cyan(%d{HH:mm:ss.SSS}) %gray([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n</pattern>
<encoder class="co.elastic.logging.logback.EcsEncoder">
<serviceName>${OTEL_SERVICE_NAME}</serviceName>
<serviceVersion>${project.version}</serviceVersion>
<serviceEnvironment>${ENV}</serviceEnvironment>
</encoder>
</appender>

<root level="info">
<appender-ref ref="STDOUT" />
<logger name="io.micronaut" level="INFO"/>

<root level="INFO">
<appender-ref ref="STDOUT"/>
</root>
</configuration>

</configuration>

0 comments on commit 35de6f8

Please sign in to comment.