Merge branch 'aws-migration'

.github/actions/push_image/action.yml

@@ -0,0 +1,41 @@
+name: Push image
+description: Upload built image to AWS ECR
+inputs:
+  username:
+    description: ghcr.io username
+    required: true
+  password:
+    description: ghcr.io password
+    required: true
+  aws-arn:
+    description: aws util arn
+    required: true
+  image-tag:
+    description: image tag for container image
+    required: true
+
+
+
+runs:
+  using: composite
+  steps:
+    - name: Login to GitHub Container Registry
+      uses: docker/[email protected]
+      with:
+        registry: ghcr.io
+        username: ${{ inputs.username }}
+        password: ${{ inputs.password }}
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: ${{ inputs.aws-arn }}
+        role-session-name: aoe-runner-ecr-push
+        aws-region: eu-west-1
+
+    - name: 03-push-image.sh
+      working-directory: ./
+      shell: bash
+      env:
+        IMAGE_TAG: ${{ inputs.image-tag }}
+      run: ./deploy-scripts/03-push-image.sh

.github/workflows/build.yml

@@ -1,14 +1,71 @@
 name: Build
 
 on:
+  workflow_dispatch:
   pull_request:
     branches:
       - '**'
-  workflow_dispatch:
+
+env:
+  DOCKER_BUILDKIT: '1'
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+  packages: write
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    name: 01-build.sh
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run 01-build.sh
+        run: ./deploy-scripts/01-build.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          IMAGE_TAG: ga-${{ github.run_number }}
+          TRUST_STORE_PASSWORD: ${{ secrets.TRUST_STORE_PASSWORD }}
+
+  lint:
+    name: 02-lint.sh
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/[email protected]
+      - name: Run 02-lint.sh
+        run: ./deploy-scripts/02-lint.sh
+
+  push_image:
+    name: 03-push-image.sh
+    if: github.event.pull_request.draft == false
+    needs: [ lint, build ]
+    permissions:
+      packages: write
+      id-token: write
+      contents: read
+    runs-on: 'ubuntu-24.04'
     steps:
-      - name: Output
-        run: echo "===============> Would now start building AOE"
+      - name: Fetch history for all branches and tags
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Push image
+        uses: ./.github/actions/push_image
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          aws-arn: ${{ secrets.AWS_AOE_UTILITY_ROLE_ARN }}
+          image-tag: ga-${{ github.run_number }}

.github/workflows/deploy_dev.yml

@@ -6,10 +6,51 @@ on:
       branch:
         description: Branch to deploy
         required: true
+      proceedDeploy:
+        description: Set to true to skip CDK diff and directly proceed with deployment. Default is false.
+        required: true
+        default: 'false'
+
+concurrency: deploy-dev
+
+env:
+  DOCKER_BUILDKIT: '1'
+
+defaults:
+  run:
+    shell: bash
 
 jobs:
   deploy_dev:
-    runs-on: ubuntu-latest
+    name: 04-deploy-dev.sh
+    permissions:
+      packages: read
+      id-token: write
+      contents: write
+    runs-on: 'ubuntu-24.04'
     steps:
-      - name: Output
-        run: echo "===============> Would now start deploying AOE to DEV"
+      - uses: actions/[email protected]
+      - name: Checkout all branches and tags with full history
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+          fetch-depth: 0
+      - name: Configure dev AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_AOE_DEV_ROLE_ARN }}
+          role-session-name: aoe-runner-deploy-dev
+          aws-region: eu-west-1
+
+      - name: Run CDK Diff
+        if: ${{ github.event.inputs.proceedDeploy == 'false' }}
+        run: ./deploy-scripts/04-deploy-dev.sh diff
+        env:
+          UTILITY_ACCOUNT_ID: ${{ secrets.UTILITY_ACCOUNT_ID }}
+
+      - name: Run CDK Deploy
+        if: ${{ github.event.inputs.proceedDeploy == 'true' }}
+        run: ./deploy-scripts/04-deploy-dev.sh deploy
+        env:
+          UTILITY_ACCOUNT_ID: ${{ secrets.UTILITY_ACCOUNT_ID }}
+

.github/workflows/deploy_prod.yml

@@ -6,10 +6,51 @@ on:
       branch:
         description: Branch to deploy
         required: true
+      proceedDeploy:
+        description: Set to true to skip CDK diff and directly proceed with deployment. Default is false.
+        required: true
+        default: 'false'
+
+concurrency: deploy-prod
+
+env:
+  DOCKER_BUILDKIT: '1'
+
+defaults:
+  run:
+    shell: bash
 
 jobs:
   deploy_prod:
-    runs-on: ubuntu-latest
+    name: 06-deploy-prod.sh
+    permissions:
+      packages: read
+      id-token: write
+      contents: write
+    runs-on: 'ubuntu-24.04'
     steps:
-      - name: Output
-        run: echo "===============> Would now start deploying AOE to PROD"
+      - uses: actions/[email protected]
+      - name: Checkout all branches and tags with full history
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+          fetch-depth: 0
+      - name: Configure prod AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_AOE_PROD_ROLE_ARN }}
+          role-session-name: aoe-runner-deploy-prod
+          aws-region: eu-west-1
+
+      - name: Run CDK Diff
+        if: ${{ github.event.inputs.proceedDeploy == 'false' }}
+        run: ./deploy-scripts/06-deploy-prod.sh diff
+        env:
+          UTILITY_ACCOUNT_ID: ${{ secrets.UTILITY_ACCOUNT_ID }}
+
+      - name: Run CDK Deploy
+        if: ${{ github.event.inputs.proceedDeploy == 'true' }}
+        run: ./deploy-scripts/06-deploy-prod.sh deploy
+        env:
+          UTILITY_ACCOUNT_ID: ${{ secrets.UTILITY_ACCOUNT_ID }}
+

.github/workflows/deploy_qa.yml

@@ -6,10 +6,51 @@ on:
       branch:
         description: Branch to deploy
         required: true
+      proceedDeploy:
+        description: Set to true to skip CDK diff and directly proceed with deployment. Default is false.
+        required: true
+        default: 'false'
+
+concurrency: deploy-qa
+
+env:
+  DOCKER_BUILDKIT: '1'
+
+defaults:
+  run:
+    shell: bash
 
 jobs:
   deploy_qa:
-    runs-on: ubuntu-latest
+    name: 05-deploy-qa.sh
+    permissions:
+      packages: read
+      id-token: write
+      contents: write
+    runs-on: 'ubuntu-24.04'
     steps:
-      - name: Output
-        run: echo "===============> Would now start deploying AOE to QA"
+      - uses: actions/[email protected]
+      - name: Checkout all branches and tags with full history
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+          fetch-depth: 0
+      - name: Configure qa AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_AOE_QA_ROLE_ARN }}
+          role-session-name: aoe-runner-deploy-qa
+          aws-region: eu-west-1
+
+      - name: Run CDK Diff
+        if: ${{ github.event.inputs.proceedDeploy == 'false' }}
+        run: ./deploy-scripts/05-deploy-qa.sh diff
+        env:
+          UTILITY_ACCOUNT_ID: ${{ secrets.UTILITY_ACCOUNT_ID }}
+
+      - name: Run CDK Deploy
+        if: ${{ github.event.inputs.proceedDeploy == 'true' }}
+        run: ./deploy-scripts/05-deploy-qa.sh deploy
+        env:
+          UTILITY_ACCOUNT_ID: ${{ secrets.UTILITY_ACCOUNT_ID }}
+

.gitignore

@@ -0,0 +1,4 @@
+.idea
+
+# exclude container mounted folders
+/docker/dev

.nvmrc

@@ -0,0 +1 @@
+v23.1.0

README.md

@@ -0,0 +1,56 @@
+# AOE
+
+## Teknologiat
+Nämä ovat keskeiset AOE järjestelmän käytettävät teknologiat
+
+- PostgreSQL
+- MongoDB
+- Redis
+- ElasticSearch
+- Apache Kafka
+- AWS S3
+- Palvelinteknologiat
+  - Spring boot
+  - NodeJs
+  - Maven
+- Web-sovelluksen frontend-teknologiat
+  - Angular 14.3.0
+  - npm-työkalu riippuvuuksien hakuun
+  - TypeScript
+
+## Kehitysympäristö
+
+### .env tiedostojen hakeminen
+AOE:n service projektit vaatii .env tiedostot jotka voi ladata `./scripts/fetch_secrets.sh` skriptin avulla AWS:stä.
+Skripti kopioi AWS S3:sta .env tiedostot projekteihin. Huom skripti tarvitsee toimiakseen aws sso sisäänkirjautumisen.
+
+### S3 .env datan päivittäminen
+AWS S3 .env datan voi päivitää käyttäen `./scripts/update_secrets.sh` skriptiä
+1) Päivitä .env tiedosto(t) projektien juuressa.
+- `/aoe-web-backend/.env`
+- `/aoe-streaming-app/.env`
+- `/aoe-data-analytics/.env`
+- `/aoe-semantic-apis/.env`
+- `/aoe-data-services/.env`
+2) Aja `./scripts/update_secrets.sh`
+
+#### Ajaminen Dockerilla
+Koko AOE:n saa paikallisesti ajoon `./start-local-env.sh` skiptin avulla. 
+Skripti käynnistää docker compose:n avulla kaikki AOE palvelut, frontin ja riippuvuudet docker kontteihin.
+
+Ympäristössä on myös NGINX, jonka kautta kaikki web sovelluksen https-pyynnöt menevät.
+
+Lokaali AOE käyttää mock OIDC palvelua, jota vasten AOE tekee autentikaation. OIDC mock service:ssä on konfiguroitu yksi käyttäjä: aoeuser/password123.
+
+1) Lisää oman koneesi host tiedostoon seuraavat rivit
+- 127.0.0.1       aoe-oidc-server
+- 127.0.0.1       demo.aoe.fi
+
+2) Aja projektin juuressa `./start-local-env.sh`
+- Skripti lataa tarvittaessa .env tiedostot AWS S3:sta
+- Skipti luo itseallekirjoitetun varmenteen hakemistoon `docker/dev/nginx/nginx-selfsigned.crt`
+- Lisää varmenne `docker/dev/nginx/nginx-selfsigned.crt` oman koneesi luotettuihin varmenteisiin, selaimat sallivat itseallekirjoitetun varmenteen käytön.
+
+4. Selaimella AOE web sovellukseen pääsee url:lla https://demo.aoe.fi/
+
+

aoe-data-analytics/.env.template

@@ -0,0 +1,21 @@
+spring.datasource.primary.username=
+spring.datasource.primary.password=
+
+kafka.bootstrap-servers=
+spring.kafka.bootstrap-servers=
+spring.kafka.consumer.bootstrap-servers=
+spring.kafka.producer.bootstrap-servers=
+
+mongodb.primary.host=
+mongodb.primary.port=
+mongodb.primary.database=
+mongodb.primary.username=
+mongodb.primary.password=
+
+mongodb.primary.enable.ssl=false
+
+kafka.enabled=true
+kafka.sasl.enable=false
+kafka.consumer.auto.startup=true
+trust.store.pass=
+trust.store.location=

aoe-data-analytics/.gitignore

@@ -1,7 +1,7 @@
 ### Build Directory
 target/
 
-### Environment Variables
+# env file
 .env
 
 ### Markup files