From f48abc4e010eb1f20c0338e871b809998eefa931 Mon Sep 17 00:00:00 2001 From: Tomi Katajanheimo Date: Tue, 31 Dec 2024 12:40:00 +0200 Subject: [PATCH] OY-4608 Tee assume role ennen jokaista Lampi-tiedoston siirtoa [skip ci] --- scripts/lampi-export/Dockerfile | 2 +- scripts/lampi-export/lampi-export.sh | 47 +++++++++++----------- scripts/lampi-export/send-error-message.sh | 4 +- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/scripts/lampi-export/Dockerfile b/scripts/lampi-export/Dockerfile index ccf01401b..db90e7511 100644 --- a/scripts/lampi-export/Dockerfile +++ b/scripts/lampi-export/Dockerfile @@ -20,4 +20,4 @@ WORKDIR /opt/export COPY scripts/lampi-export/lampi-export.sh /opt/export COPY scripts/lampi-export/send-error-message.sh /opt/export -ENTRYPOINT ["./lampi-export.sh", "||", "./send-error-message.sh"] +ENTRYPOINT exec ./lampi-export.sh || ./send-error-message.sh diff --git a/scripts/lampi-export/lampi-export.sh b/scripts/lampi-export/lampi-export.sh index bbbc5aec3..121e100be 100755 --- a/scripts/lampi-export/lampi-export.sh +++ b/scripts/lampi-export/lampi-export.sh @@ -17,6 +17,9 @@ else fi db_name="ehoks" db_user="app" +assume_access_key_id="" +assume_secret_access_key="" +assume_session_token="" case $ENV_NAME in "sade") @@ -29,18 +32,21 @@ case $ENV_NAME in lampi_s3_bucket="oph-lampi-dev" esac +refresh_assume_role() { + local -r credentials_json=$(aws sts assume-role --role-arn "$assume_role_arn" --external-id "$ssm_external_id" --role-session-name "ehoks-lampi-export-$ENV_NAME") + assume_access_key_id=$(jq -r '.Credentials.AccessKeyId' <<<"${credentials_json}") + assume_secret_access_key=$(jq -r '.Credentials.SecretAccessKey' <<<"${credentials_json}") + assume_session_token=$(jq -r '.Credentials.SessionToken' <<<"${credentials_json}") + local -r assume_expiration=$(jq -r '.Credentials.Expiration' <<<"${credentials_json}") + log "INFO" "Assumed temporary access key ${assume_access_key_id} (expiration ${assume_expiration})" +} + dump_and_upload_db_to_lampi() { log "INFO" "Start dump_and_upload_db_to_lampi" log "DEBUG" "Params: db_hostname ${db_hostname}, db_name ${db_name}, ENV_NAME $ENV_NAME, local_s3_bucket $local_s3_bucket, lampi_s3_bucket $lampi_s3_bucket assume_role_arn $assume_role_arn" log "INFO" "Starting ${db_name} database data dump" - local -r credentials_json=$(aws sts assume-role --role-arn "$assume_role_arn" --external-id "$ssm_external_id" --role-session-name "ehoks-lampi-export-$ENV_NAME") - local -r assume_role_access_key_id=$(jq -r '.Credentials.AccessKeyId' <<<"${credentials_json}") - local -r assume_role_secret_access_key=$(jq -r '.Credentials.SecretAccessKey' <<<"${credentials_json}") - local -r assume_role_session_token=$(jq -r '.Credentials.SessionToken' <<<"${credentials_json}") - log "INFO" "Assumed temporary access key ${assume_role_access_key_id}" - local -r db_password="$ssm_app_user_password" # aws s3 extension and granting privileges need to be created with master user: @@ -59,11 +65,11 @@ dump_and_upload_db_to_lampi() { log "INFO" "Exporting table $db_table to local S3 $s3_url" local files_uploaded=$(pg_command "$db_password" "SELECT files_uploaded FROM aws_s3.query_export_to_s3('SELECT * FROM ${reporting_schema_name}.${db_table}', aws_commons.create_s3_uri('$local_s3_bucket', '$s3_key', 'eu-west-1'), options := 'format csv, header true')" 1 1) log "INFO" "Successfully exported table ${reporting_schema_name}.${db_table} to local S3 $s3_url" - copy_table_to_lampi "$s3_url" $files_uploaded "$assume_role_access_key_id" "$assume_role_secret_access_key" "$assume_role_session_token" + copy_table_to_lampi "$s3_url" $files_uploaded done - generate_and_upload_schema_file "$db_password" "$assume_role_access_key_id" "$assume_role_secret_access_key" "$assume_role_session_token" - upload_file_to_lampi "$lampi_manifest_file" "$assume_role_access_key_id" "$assume_role_secret_access_key" "$assume_role_session_token" > /dev/null + generate_and_upload_schema_file "$db_password" + upload_file_to_lampi "$lampi_manifest_file" > /dev/null log "INFO" "manifest.json" $(cat "$lampi_manifest_file") } @@ -87,10 +93,7 @@ pg_command() { copy_table_to_lampi() { local -r source_url="$1" local -r file_parts=$2 - local -r access_key_id="$3" - local -r secret_access_key="$4" - local -r session_token="$5" - + refresh_assume_role local -r s3_key="fulldump/$system_name/$version/$(basename "${source_url}").gz" local -r target_url="s3://$lampi_s3_bucket/$s3_key" @@ -100,10 +103,10 @@ copy_table_to_lampi() { for i in $(seq 2 "$file_parts"); do aws s3 cp "${source_url}_part${i}" - done - fi) | gzip | AWS_ACCESS_KEY_ID="$access_key_id" AWS_SECRET_ACCESS_KEY="$secret_access_key" AWS_SESSION_TOKEN="$session_token" aws s3 cp - "$target_url" + fi) | gzip | AWS_ACCESS_KEY_ID="$assume_access_key_id" AWS_SECRET_ACCESS_KEY="$assume_secret_access_key" AWS_SESSION_TOKEN="$assume_session_token" aws s3 cp - "$target_url" log "INFO" "Fetch object version for $s3_key in $lampi_s3_bucket" - local -r obj_version=$(AWS_ACCESS_KEY_ID="$access_key_id" AWS_SECRET_ACCESS_KEY="$secret_access_key" AWS_SESSION_TOKEN="$session_token" aws s3api head-object --region eu-west-1 --bucket "$lampi_s3_bucket" --key "$s3_key" --output json | jq -r .VersionId) + local -r obj_version=$(AWS_ACCESS_KEY_ID="$assume_access_key_id" AWS_SECRET_ACCESS_KEY="$assume_secret_access_key" AWS_SESSION_TOKEN="$assume_session_token" aws s3api head-object --region eu-west-1 --bucket "$lampi_s3_bucket" --key "$s3_key" --output json | jq -r .VersionId) log "INFO" "Update received object version to manifest: $s3_key = $obj_version" local -r item=$(lampi_manifest_item "$s3_key" "$obj_version") @@ -128,27 +131,23 @@ lampi_manifest_item() { generate_and_upload_schema_file() { local -r db_password="$1" - local -r access_key_id="$2" - local -r secret_access_key="$3" - local -r session_token="$4" + refresh_assume_role local -r schema_file="ehoks-$reporting_schema_name.schema" touch "$schema_file" log "INFO" "Generating schema file $schema_file" PGPASSWORD="${db_password}" pg_dump -Fc --section=pre-data --section=post-data --no-comments --no-privilege --no-owner --host "$db_hostname" --username "$db_user" --schema-only --file "$schema_file" --schema "$reporting_schema_name" -t "$reporting_schema_name.*" "$db_name" - local -r schema=$(upload_file_to_lampi "$schema_file" "$access_key_id" "$secret_access_key" "$session_token") + local -r schema=$(upload_file_to_lampi "$schema_file") add_to_manifest ".schema += $schema" } upload_file_to_lampi() { local -r file="$1" - local -r access_key_id="$2" - local -r secret_access_key="$3" - local -r session_token="$4" + refresh_assume_role > /dev/null local -r file_s3_key="fulldump/$system_name/$version/$file" local -r file_s3_url="s3://$local_s3_bucket/$file_s3_key" local -r put_result=$(aws s3api put-object --region eu-west-1 --body "$file" --bucket "$local_s3_bucket" --key "$file_s3_key" --output json) - local -r cp_result=$(aws s3 cp "$file_s3_url" - | AWS_ACCESS_KEY_ID="$access_key_id" AWS_SECRET_ACCESS_KEY="$secret_access_key" AWS_SESSION_TOKEN="$session_token" aws s3 cp - "s3://$lampi_s3_bucket/$file_s3_key") - local -r obj_version=$(AWS_ACCESS_KEY_ID="$access_key_id" AWS_SECRET_ACCESS_KEY="$secret_access_key" AWS_SESSION_TOKEN="$session_token" aws s3api head-object --region eu-west-1 --bucket "$lampi_s3_bucket" --key "$file_s3_key" --output json | jq -r .VersionId) + local -r cp_result=$(aws s3 cp "$file_s3_url" - | AWS_ACCESS_KEY_ID="$assume_access_key_id" AWS_SECRET_ACCESS_KEY="$assume_secret_access_key" AWS_SESSION_TOKEN="$assume_session_token" aws s3 cp - "s3://$lampi_s3_bucket/$file_s3_key") + local -r obj_version=$(AWS_ACCESS_KEY_ID="$assume_access_key_id" AWS_SECRET_ACCESS_KEY="$assume_secret_access_key" AWS_SESSION_TOKEN="$assume_session_token" aws s3api head-object --region eu-west-1 --bucket "$lampi_s3_bucket" --key "$file_s3_key" --output json | jq -r .VersionId) echo $(lampi_manifest_item "$file_s3_key" "$obj_version") } diff --git a/scripts/lampi-export/send-error-message.sh b/scripts/lampi-export/send-error-message.sh index 0ba2e470e..fcb40855a 100755 --- a/scripts/lampi-export/send-error-message.sh +++ b/scripts/lampi-export/send-error-message.sh @@ -1,5 +1,5 @@ #!/bin/bash +echo "Sending error message to Slack webhook $ssm_monitor_slack_webhook" set -eu - -curl -X POST --data '{"blocks":[{"type":"header","text":{"type":"plain_text","text":"⚠️ Error in eHOKS -> Lampi export: check logs","emoji":true}}]}' -H 'content-type: application/json' -H 'accept: application/json' $ssm_monitor_slack_webhook +curl -X POST --data '{"blocks":[{"type":"header","text":{"type":"plain_text","text":"⚠️ Error in eHOKS -> Lampi export: check logs","emoji":true}}]}' -H 'content-type: application/json' -H 'accept: application/json' "$ssm_monitor_slack_webhook"