Group deployment files in a single folder

Makefile

@@ -4,18 +4,20 @@ dev-run-data-warehouse:
 	git clone --depth=1 --branch=main https://github.com/MTES-MCT/monitorenv.git ./forklift/tests/test_data/external/monitorenv || echo "Monitorenv repository already present - skipping git clone" && \
 	export DATA_WAREHOUSE_PASSWORD=password && \
 	export DATA_WAREHOUSE_USER=clickhouse_user && \
-	docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml up -d --remove-orphans
+	docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml up -d --remove-orphans
 
 dev-run-metabase:
-	docker compose -f ./infra/data_warehouse/docker-compose-dev-metabase.yml up -d && \
+	docker compose -f ./infra/testing/docker-compose-dev-metabase.yml up -d && \
 	docker cp ./infra/metabase_plugins/clickhouse.metabase-driver.jar metabase:/plugins && \
 	docker restart metabase
 
 dev-stop-data-warehouse:
-	docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml down
+	export DATA_WAREHOUSE_PASSWORD=password && \
+	export DATA_WAREHOUSE_USER=clickhouse_user && \
+	docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml down
 
 dev-stop-metabase:
-	docker compose -f ./infra/data_warehouse/docker-compose-dev-metabase.yml down
+	docker compose -f ./infra/testing/docker-compose-dev-metabase.yml down
 
 dev-erase-data-warehouse-databases:
 	docker volume rm data_warehouse_data-warehouse-db data_warehouse_data-warehouse-logs data_warehouse_monitorfish-db-data data_warehouse_monitorenv-db-data || exit 0
@@ -31,17 +33,10 @@ docker-run-data-warehouse:
 	git clone --depth=1 --branch=main https://github.com/MTES-MCT/monitorenv.git ./forklift/tests/test_data/external/monitorenv || echo "Monitorenv repository already present - skipping git clone" && \
 	export DATA_WAREHOUSE_PASSWORD=password && \
 	export DATA_WAREHOUSE_USER=clickhouse_user && \
-	docker compose -f ./infra/data_warehouse/docker-compose.yml -f ./infra/data_warehouse/docker-compose-test-data.yml up -d --remove-orphans
+	docker compose -f ./infra/deployment/docker-compose.yml -f ./infra/testing/docker-compose-test-data.yml up -d --remove-orphans
 docker-test-forklift: docker-run-data-warehouse
 	docker run --network host -v /var/run/docker.sock:/var/run/docker.sock -u forklift:$(DOCKER_GROUP) --env-file forklift/.env.test forklift:$(VERSION) coverage run -m pytest --pdb tests
 docker-tag-forklift:
 	docker tag forklift:$(VERSION) ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(VERSION)
 docker-push-forklift:
 	docker push ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(VERSION)
-
-# RUN commands
-run-datawarehouse:
-	docker compose -f ./infra/data_warehouse/docker-compose.yml up -d
-register-forklift-flows:
-	docker pull ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(FORKLIFT_VERSION) && \
-	infra/forklift/register-flows.sh

infra/deployment/Makefile

@@ -0,0 +1,5 @@
+run-datawarehouse:
+	docker compose -f ./docker-compose.yml up -d
+register-forklift-flows:
+	docker pull ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$(FORKLIFT_VERSION) && \
+	./register-flows.sh

infra/deployment/README.md

@@ -0,0 +1,46 @@
+## Data Warehouse Deployment
+### 1. Purpose and Architecture
+
+This Data Warehouse is meant to collect data from Monitorfish, Monitorenv and Rapportnav V2 applications and provide
+a centralized database for statistical analysis through data querying and visualization tools such as Metabase.
+
+It is composed of 2 dockerized services :
+- a Clickhouse database
+- a set of python data flows orchestrated by Prefect, named `Forklift`
+
+### 2. Deployment
+#### 2.1. Prerequisites
+
+- Install docker : https://docs.docker.com/engine/install/debian/#install-using-the-repository
+- Install GNU make if not already installed :   
+    `sudo apt-get update`  
+    `sudo apt-get install make`
+- Install pyenv : https://github.com/pyenv/pyenv-installer
+- Install python : `pyenv install 3.12.3`
+- Copy the contents of the `infra/deployment` folder onto the host machine, in the `~` folder.
+- Create a virtual environment in the `~/prefect-agent` folder and install `prefect` in this virtual environment :
+  - `cd prefect-agent`
+  - `pyenv local 3.12.3`
+  - `python -m venv .venv`
+  - `source .venv/bin/activate`
+  - `pip install prefect=1.3.0`
+  - Run `prefect backend server` to switch Prefect into server mode (as opposed to cloud mode, the default). A file `backend.toml` should appear at `~/.prefect/backend.toml` containing `backend = "server"`.
+- Define and start a Prefect agent service:
+  - Fill in the placeholders in `.prefect-agent` and `prefectdockeragent.service`.
+  - Move `prefectdockeragent.service` into `/etc/systemd/system`.
+  - Run `sudo systemctl enable prefectdockeragent.service` to enable the service.
+  - Run `sudo systemctl start prefectdockeragent.service` to start the service.
+  - Run `systemctl status prefectdockeragent.service` to check the service is running.
+- Rename `.env.template` as `.env` and fill in the environement variables : these are the environment variables that will be used by the flow runners, inside the forklift runner containers.
+
+#### 2.2. Running the Data Warehouse
+
+- Update the `DATA_WAREHOUSE_USER` and `DATA_WAREHOUSE_PASSWORD` environment variables in the `.data-warehouse` file.
+- Run `source .data-warehouse`.
+- Run `make run-datawarehouse`.
+
+#### 2.3. Running or updating `Forklift`
+
+- Update the `FORKLIFT_VERSION` variable in the `.data-warehouse` file.
+- Run `source .data-warehouse`.
+- Run `make register-forklift-flows`.

infra/data_warehouse/docker-compose.yml → infra/deployment/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   db:
     image: clickhouse/clickhouse-server:23.11.4.24

infra/deployment/prefect-agent/.prefect-agent

@@ -0,0 +1 @@
+PREFECT_SERVER_URL=

infra/deployment/prefect-agent/.python-version

@@ -0,0 +1 @@
+3.12.3

infra/deployment/prefect-agent/prefectdockeragent.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Prefect Docker Agent
+
+[Service]
+Type=simple
+
+User=<USER-TO-CHANGE>
+ExecStart=<SCRIPT-LOCATION-TO-CHANGE>/prefectdockeragent.sh
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

infra/deployment/prefect-agent/prefectdockeragent.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+source ~/prefect-agent/.venv/bin/activate && \
+source ~/prefect-agent/.prefect-agent && \
+prefect agent docker start --api "${PREFECT_SERVER_URL}" --label forklift;

infra/deployment/register-flows.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+docker run -t --rm --network=host --name forklift-register-flows \
+        -v /opt2/sacrois-data:/opt2/sacrois-data \
+        -v "$(pwd)"/.env:/home/forklift/data_warehouse/.env \
+	    -v "$(pwd)"/backend.toml:/home/forklift/.prefect/backend.toml \
+        --env-file .env \
+        -e FORKLIFT_VERSION \
+        -e SACROIS_FILES_GID \
+        ghcr.io/mtes-mct/fisheries-and-environment-data-warehouse/forklift:$FORKLIFT_VERSION \
+        python forklift/main.py

infra/data_warehouse/docker-compose-dev-metabase.yml → infra/testing/docker-compose-dev-metabase.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
 
   metabase-db:

infra/data_warehouse/docker-compose-test-data.yml → infra/testing/docker-compose-test-data.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
 
   monitorfish-db: