From 32f3462a81f22f2450d10e81d3de730888487872 Mon Sep 17 00:00:00 2001 From: Vincent Date: Thu, 11 Apr 2024 15:50:50 +0200 Subject: [PATCH] Group deployment files in a single folder --- Makefile | 19 +++----- .../.data-warehouse} | 0 {forklift => infra/deployment}/.env.template | 0 infra/deployment/Makefile | 5 ++ infra/deployment/README.md | 46 +++++++++++++++++++ infra/{forklift => deployment}/backend.toml | 0 .../docker-compose.yml | 1 - infra/deployment/prefect-agent/.prefect-agent | 1 + .../deployment/prefect-agent/.python-version | 1 + .../prefect-agent/prefectdockeragent.service | 13 ++++++ .../prefect-agent/prefectdockeragent.sh | 4 ++ infra/deployment/register-flows.sh | 11 +++++ infra/forklift/register-flows.sh | 11 ----- .../docker-compose-dev-metabase.yml | 1 - .../docker-compose-test-data.yml | 1 - 15 files changed, 88 insertions(+), 26 deletions(-) rename infra/{data_warehouse/.data-warehouse-template => deployment/.data-warehouse} (100%) rename {forklift => infra/deployment}/.env.template (100%) create mode 100644 infra/deployment/Makefile create mode 100644 infra/deployment/README.md rename infra/{forklift => deployment}/backend.toml (100%) rename infra/{data_warehouse => deployment}/docker-compose.yml (98%) create mode 100644 infra/deployment/prefect-agent/.prefect-agent create mode 100644 infra/deployment/prefect-agent/.python-version create mode 100644 infra/deployment/prefect-agent/prefectdockeragent.service create mode 100755 infra/deployment/prefect-agent/prefectdockeragent.sh create mode 100755 infra/deployment/register-flows.sh delete mode 100755 infra/forklift/register-flows.sh rename infra/{data_warehouse => testing}/docker-compose-dev-metabase.yml (98%) rename infra/{data_warehouse => testing}/docker-compose-test-data.yml (99%) diff --git a/Makefile b/Makefile index 768386f..a2d9722 100644 --- a/Makefile +++ b/Makefile @@ -4,18 +4,20 @@ dev-run-data-warehouse: git clone --depth=1 --branch=main https://github.com/MTES-MCT/monitorenv.git ./forklift/tests/test_data/external/monitorenv || echo "Monitorenv repository already present - skipping git clone" && \ export DATA_WAREHOUSE_PASSWORD=password && \ export DATA_WAREHOUSE_USER=clickhouse_user && \ - docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml up -d --remove-orphans + docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml up -d --remove-orphans dev-run-metabase: - docker compose -f ./infra/data_warehouse/docker-compose-dev-metabase.yml up -d && \ + docker compose -f ./infra/testing/docker-compose-dev-metabase.yml up -d && \ docker cp ./infra/metabase_plugins/clickhouse.metabase-driver.jar metabase:/plugins && \ docker restart metabase dev-stop-data-warehouse: - docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml down + export DATA_WAREHOUSE_PASSWORD=password && \ + export DATA_WAREHOUSE_USER=clickhouse_user && \ + docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml down dev-stop-metabase: - docker compose -f ./infra/data_warehouse/docker-compose-dev-metabase.yml down + docker compose -f ./infra/testing/docker-compose-dev-metabase.yml down dev-erase-data-warehouse-databases: docker volume rm data_warehouse_data-warehouse-db data_warehouse_data-warehouse-logs data_warehouse_monitorfish-db-data data_warehouse_monitorenv-db-data || exit 0 @@ -31,17 +33,10 @@ docker-run-data-warehouse: git clone --depth=1 --branch=main https://github.com/MTES-MCT/monitorenv.git ./forklift/tests/test_data/external/monitorenv || echo "Monitorenv repository already present - skipping git clone" && \ export DATA_WAREHOUSE_PASSWORD=password && \ export DATA_WAREHOUSE_USER=clickhouse_user && \ - docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml up -d --remove-orphans + docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml up -d --remove-orphans docker-test-forklift: docker-run-data-warehouse docker run --network host -v /var/run/docker.sock:/var/run/docker.sock -u forklift:$(DOCKER_GROUP) --env-file forklift/.env.test forklift:$(VERSION) coverage run -m pytest --pdb tests docker-tag-forklift: docker tag forklift:$(VERSION) ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(VERSION) docker-push-forklift: docker push ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(VERSION) - -# RUN commands -run-datawarehouse: - docker compose -f ./infra/data_warehouse/docker-compose.yml up -d -register-forklift-flows: - docker pull ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(FORKLIFT_VERSION) && \ - infra/forklift/register-flows.sh diff --git a/infra/data_warehouse/.data-warehouse-template b/infra/deployment/.data-warehouse similarity index 100% rename from infra/data_warehouse/.data-warehouse-template rename to infra/deployment/.data-warehouse diff --git a/forklift/.env.template b/infra/deployment/.env.template similarity index 100% rename from forklift/.env.template rename to infra/deployment/.env.template diff --git a/infra/deployment/Makefile b/infra/deployment/Makefile new file mode 100644 index 0000000..1bfaa35 --- /dev/null +++ b/infra/deployment/Makefile @@ -0,0 +1,5 @@ +run-datawarehouse: + docker compose -f ./docker-compose.yml up -d +register-forklift-flows: + docker pull ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(FORKLIFT_VERSION) && \ + ./register-flows.sh diff --git a/infra/deployment/README.md b/infra/deployment/README.md new file mode 100644 index 0000000..5677e8f --- /dev/null +++ b/infra/deployment/README.md @@ -0,0 +1,46 @@ +## Data Warehouse Deployment +### 1. Purpose and Architecture + +This Data Warehouse is meant to collect data from Monitorfish, Monitorenv and Rapportnav V2 applications and provide +a centralized database for statistical analysis through data querying and visualization tools such as Metabase. + +It is composed of 2 dockerized services : +- a Clickhouse database +- a set of python data flows orchestrated by Prefect, named `Forklift` + +### 2. Deployment +#### 2.1. Prerequisites + +- Install docker : https://docs.docker.com/engine/install/debian/#install-using-the-repository +- Install GNU make if not already installed : + `sudo apt-get update` + `sudo apt-get install make` +- Install pyenv : https://github.com/pyenv/pyenv-installer +- Install python : `pyenv install 3.12.3` +- Copy the contents of the `infra/deployment` folder onto the host machine, in the `~` folder. +- Create a virtual environment in the `~/prefect-agent` folder and install `prefect` in this virtual environment : + - `cd prefect-agent` + - `pyenv local 3.12.3` + - `python -m venv .venv` + - `source .venv/bin/activate` + - `pip install prefect=1.3.0` + - Run `prefect backend server` to switch Prefect into server mode (as opposed to cloud mode, the default). A file `backend.toml` should appear at `~/.prefect/backend.toml` containing `backend = "server"`. +- Define and start a Prefect agent service: + - Fill in the placeholders in `.prefect-agent` and `prefectdockeragent.service`. + - Move `prefectdockeragent.service` into `/etc/systemd/system`. + - Run `sudo systemctl enable prefectdockeragent.service` to enable the service. + - Run `sudo systemctl start prefectdockeragent.service` to start the service. + - Run `systemctl status prefectdockeragent.service` to check the service is running. +- Rename `.env.template` as `.env` and fill in the environement variables : these are the environment variables that will be used by the flow runners, inside the forklift runner containers. + +#### 2.2. Running the Data Warehouse + +- Update the `DATA_WAREHOUSE_USER` and `DATA_WAREHOUSE_PASSWORD` environment variables in the `.data-warehouse` file. +- Run `source .data-warehouse`. +- Run `make run-datawarehouse`. + +#### 2.3. Running or updating `Forklift` + +- Update the `FORKLIFT_VERSION` variable in the `.data-warehouse` file. +- Run `source .data-warehouse`. +- Run `make register-forklift-flows`. diff --git a/infra/forklift/backend.toml b/infra/deployment/backend.toml similarity index 100% rename from infra/forklift/backend.toml rename to infra/deployment/backend.toml diff --git a/infra/data_warehouse/docker-compose.yml b/infra/deployment/docker-compose.yml similarity index 98% rename from infra/data_warehouse/docker-compose.yml rename to infra/deployment/docker-compose.yml index bf44243..bf1059b 100644 --- a/infra/data_warehouse/docker-compose.yml +++ b/infra/deployment/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: db: image: clickhouse/clickhouse-server:23.11.4.24 diff --git a/infra/deployment/prefect-agent/.prefect-agent b/infra/deployment/prefect-agent/.prefect-agent new file mode 100644 index 0000000..d379f1c --- /dev/null +++ b/infra/deployment/prefect-agent/.prefect-agent @@ -0,0 +1 @@ +PREFECT_SERVER_URL= \ No newline at end of file diff --git a/infra/deployment/prefect-agent/.python-version b/infra/deployment/prefect-agent/.python-version new file mode 100644 index 0000000..4c8b864 --- /dev/null +++ b/infra/deployment/prefect-agent/.python-version @@ -0,0 +1 @@ +3.12.3 \ No newline at end of file diff --git a/infra/deployment/prefect-agent/prefectdockeragent.service b/infra/deployment/prefect-agent/prefectdockeragent.service new file mode 100644 index 0000000..928c210 --- /dev/null +++ b/infra/deployment/prefect-agent/prefectdockeragent.service @@ -0,0 +1,13 @@ +[Unit] +Description=Prefect Docker Agent + +[Service] +Type=simple + +User= +ExecStart=/prefectdockeragent.sh +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/infra/deployment/prefect-agent/prefectdockeragent.sh b/infra/deployment/prefect-agent/prefectdockeragent.sh new file mode 100755 index 0000000..92da2fd --- /dev/null +++ b/infra/deployment/prefect-agent/prefectdockeragent.sh @@ -0,0 +1,4 @@ +#!/bin/bash +source ~/prefect-agent/.venv/bin/activate && \ +source ~/prefect-agent/.prefect-agent && \ +prefect agent docker start --api "${PREFECT_SERVER_URL}" --label forklift; diff --git a/infra/deployment/register-flows.sh b/infra/deployment/register-flows.sh new file mode 100755 index 0000000..4ba8586 --- /dev/null +++ b/infra/deployment/register-flows.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +docker run -t --rm --network=host --name forklift-register-flows \ + -v /opt2/sacrois-data:/opt2/sacrois-data \ + -v "$(pwd)"/.env:/home/forklift/data_warehouse/.env \ + -v "$(pwd)"/backend.toml:/home/forklift/.prefect/backend.toml \ + --env-file .env \ + -e FORKLIFT_VERSION \ + -e SACROIS_FILES_GID \ + ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$FORKLIFT_VERSION \ + python forklift/main.py diff --git a/infra/forklift/register-flows.sh b/infra/forklift/register-flows.sh deleted file mode 100755 index f039116..0000000 --- a/infra/forklift/register-flows.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -docker run -t --rm --network=host --name forklift-register-flows \ - -v /opt2/sacrois-data:/opt2/sacrois-data \ - -v "$(pwd)"/data_warehouse/.env:/home/forklift/data_warehouse/.env \ - -v "$(pwd)"/infra/forklift/backend.toml:/home/forklift/.prefect/backend.toml \ - --env-file data_warehouse/.env \ - -e FORKLIFT_VERSION \ - -e SACROIS_FILES_GID \ - ghcr.io/mtes-mct/monitorfish/forklift:$FORKLIFT_VERSION \ - python forklift/main.py diff --git a/infra/data_warehouse/docker-compose-dev-metabase.yml b/infra/testing/docker-compose-dev-metabase.yml similarity index 98% rename from infra/data_warehouse/docker-compose-dev-metabase.yml rename to infra/testing/docker-compose-dev-metabase.yml index 6414bf8..e3eed84 100644 --- a/infra/data_warehouse/docker-compose-dev-metabase.yml +++ b/infra/testing/docker-compose-dev-metabase.yml @@ -1,4 +1,3 @@ -version: '3' services: metabase-db: diff --git a/infra/data_warehouse/docker-compose-test-data.yml b/infra/testing/docker-compose-test-data.yml similarity index 99% rename from infra/data_warehouse/docker-compose-test-data.yml rename to infra/testing/docker-compose-test-data.yml index c55d53f..fc7f303 100644 --- a/infra/data_warehouse/docker-compose-test-data.yml +++ b/infra/testing/docker-compose-test-data.yml @@ -1,4 +1,3 @@ -version: '3' services: monitorfish-db: