From 6c59aa0dadafcee4a5bb1001047eecf160403258 Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer <94527853+knard-intel@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:56:26 +0100 Subject: [PATCH] DAOS-14226 docker: deployment of vcluster with md-on-ssd (#13087) Update DAOS docker vcluster scripts for being able to deploy a minimal docker DAOS system using the md-on-ssd feature. This PR also fix miscellaneous minor issues such as default DAOS rpms repos, variables naming, etc. The Doc-only pragma as been used as there is nothing yet tested by the CI related to this docker stuff. Signed-off-by: Cedric Koch-Hofer --- docs/QSG/docker.md | 178 ++++-------------- utils/docker/examples/.env | 10 +- utils/docker/examples/README.md | 12 +- .../docker/examples/daos-admin/el8/Dockerfile | 26 ++- .../examples/daos-admin/el8/daos-bash.sh | 2 +- .../docker/examples/daos-agent/el8/Dockerfile | 26 ++- .../examples/daos-agent/el8/run-daos_agent.sh | 2 +- .../docker/examples/daos-base/el8/Dockerfile | 35 ++-- .../examples/daos-client/el8/Dockerfile | 29 +-- .../examples/daos-client_agent/el8/Dockerfile | 29 +-- .../daos-client_agent/el8/daos-bash.sh | 2 +- .../examples/daos-server/el8/Dockerfile | 26 ++- .../daos-server/el8/run-daos_server.sh | 2 +- .../examples/daos-spdk_setup/el8/Dockerfile | 26 ++- utils/docker/examples/deploy-docker_stack.sh | 5 +- utils/docker/vcluster/.env | 25 ++- .../docker/vcluster/daos-admin/el8/Dockerfile | 61 +++--- .../daos-admin/el8/daos_control.yml.in | 8 +- .../docker/vcluster/daos-base/el8/Dockerfile | 91 +++++---- .../vcluster/daos-client/el8/Dockerfile | 81 +++++--- .../daos-client/el8/daos_agent.yml.in | 12 +- utils/docker/vcluster/daos-cm.sh | 59 ++++-- .../vcluster/daos-server/el8/Dockerfile | 100 +++++----- .../daos-server/el8/daos_server.yml.in | 38 ++-- utils/docker/vcluster/docker-compose.yml | 62 +++--- 25 files changed, 488 insertions(+), 459 deletions(-) diff --git a/docs/QSG/docker.md b/docs/QSG/docker.md index fd44eea7da3..ed8ca764691 100644 --- a/docs/QSG/docker.md +++ b/docs/QSG/docker.md @@ -2,14 +2,12 @@ This section describes how to build and deploy Docker images allowing to simulate a small cluster using DAOS as backend storage. This small cluster is composed of the following three nodes: - - The `daos-server` node running a DAOS server daemon managing data storage devices such as SCM or NVMe disks. - The `daos-admin` node allowing to manage the DAOS server thanks to `dmg`command. - The `daos-client` node using the the DAOS server to store data. At this time only emulated hardware storage are supported by this Docker platform: - - SCM (i.e. Storage Class Memory) are emulated with standard RAM memory. - NVMe disks are emulated with a file device. @@ -45,7 +43,7 @@ The platform was tested and validated with the following dependencies: [RPMs](https://download.docker.com/linux/centos/docker-ce.repo) - [DAOS 2.6](https://docs.daos.io/v2.6/) local RPMS builds from [DAOS master branch](https://github.com/daos-stack/daos/tree/master) -- [rockylinux/rockylinux:8.6](https://hub.docker.com/r/rockylinux/rockylinux/) official docker +- [rockylinux/rockylinux:8.9](https://hub.docker.com/r/rockylinux/rockylinux/) official docker images. ### Configuring HugePages @@ -75,93 +73,58 @@ $ sysctl -p ### Base DAOS Image The first image to create is the `daos-base` image which is not intetended to be used as it, but as -a base image for building the other three daos images. This first image could be built directly -from GitHub with the following command: - -```bash -$ docker build --tag daos-base:rocky8.6 \ - https://github.com/daos-stack/daos.git#master:utils/docker/vcluster/daos-base/el8 -``` - -This Docker file accept the following arguments: - -- `RHEL_BASE_IMAGE`: Base docker image to use (default "rockylinux/rockylinux") -- `RHEL_BASE_VERSION`: Version of the base docker image to use (default "8.6") +a base image for building the other three daos images. The easiest way is to use the `docker +compose` sub command from a local DAOS source file tree. The first step is to update the docker +environment file "utils/docker/examples/.env" according to the targeted DAOS system. The following +environment variables allow to customize the Docker image to build: +- `LINUX_DISTRO`: Linux distribution identifier (default "el8") +- `DAOS_DOCKER_IMAGE_NSP`: Namespace identifier of the base DAOS docker image (default "daos") +- `DAOS_DOCKER_IMAGE_TAG`: Tag identifier of the base DAOS docker image (default "v2.4.1") - `BUST_CACHE`: Manage docker building cache (default ""). To invalidate the cache, a random value - such as the date of the day shall be given. -- `DAOS_AUTH`: Enable DAOS authentication when set to "yes" (default "yes") + such as the date of day shall be given. +- `LINUX_IMAGE_NAME`: Base docker image name to use (default "rockylinux/rockylinux") +- `LINUX_IMAGE_TAG`: Tag identifier of the base docker image to use (default "8.9") - `DAOS_REPOS`: Space separated list of repos needed to install DAOS (default - "https://packages.daos.io/v2.2/EL8/packages/x86\_64/") + "https://packages.daos.io/v2.4.1/EL8/packages/x86\_64/") - `DAOS_GPG_KEYS`: Space separated list of GPG keys associated with DAOS repos (default - "https://packages.daos.io/RPM-GPG-KEY") + "https://packages.daos.io/v2.4.1/RPM-GPG-KEY-2023") - `DAOS_REPOS_NOAUTH`: Space separated list of repos to use without GPG authentication - (default "") - -For example, building a DAOS base image, with authentication disabled, could be done with the -following command: - -```bash -$ docker build --tag daos-base:rocky8.6 --build-arg DAOS_AUTH=no \ - https://github.com/daos-stack/daos.git#master:utils/docker/vcluster/daos-base/el8 -``` - -It is also possible to build the `daos-base` image from a local tree with the following command: - -```bash -$ docker build --tag daos-base:rocky8.6 utils/docker/vcluster/daos-base/el8 -``` - -From a local tree, a more straightforward way to build these images could be done with -`docker compose`: + (default "") +- `DAOS_VERSION`: Version of DAOS to use (default "2.4.1-2.el8") +- `DAOS_AUTH`: Enable DAOS authentication when set to "yes" (default "yes") +When the environment file has been properly filled, run the following command to build the base DAOS +docker image. ```bash $ docker compose --file utils/docker/vcluster/docker-compose.yml build daos_base ``` -The same arguments are accepted but they have to be defined in the Docker Compose environment file -`utils/docker/vcluster/.env`. +!!! warning + For working properly, the DAOS authentication have to be enabled in all the images (i.e. nodes + images and base image). ### DAOS Nodes Images -The three images `daos-server`, `daos-admin` and `daos-client` could be built directly from GitHub -or from a local tree in the same way as for the `daos-base` image. Following command could be used -to build directly the three images from GitHub: - -```bash -$ for image in daos-server daos-admin daos-client ; do \ - docker build --tag "$image:rocky8.6" \ - "https://github.com/daos-stack/daos.git#master:utils/docker/vcluster/$image/el8"; \ - done -``` - -The Docker file of the `daos-server` image accept the following arguments: - -- `DAOS_BASE_IMAGE`: Base docker image to use (default "daos-base") -- `DAOS_BASE_VERSION`: Version of the base docker image to use (default "rocky8.6") +To build the the three docker images `daos-server`, `daos-admin` and `daos-client`, the first step +is to update the docker environment file "utils/docker/examples/.env" according to the targeted DAOS +system. The `daos-server`,`daos-client` and `daos-admin` images can be customize with the following +environment variables: +- `DAOS_DOCKER_IMAGE_TAG`: Tag identifier of the base DAOS docker image to use (default "v2.4.1") +- `DAOS_VERSION`: Version of DAOS to use (default "2.4.1-2.el8") - `DAOS_AUTH`: Enable DAOS authentication when set to "yes" (default "yes") + +The `daos-server` image is also using the following environment variables: - `DAOS_HUGEPAGES_NBR`: Number of huge pages to allocate for SPDK (default 4096) - `DAOS_SCM_SIZE`: Size in GB of the RAM emulating SCM devices (default 4) - `DAOS_BDEV_SIZE`: Size in GB of the file created to emulate NVMe devices (default 16) - `DAOS_IFACE_NAME`: Fabric network interface used by the DAOS engine (default "eth0") +- `DAOS_MD_ON_SSD`: Enable DAOS MD-on-SSD feature when set to "yes" (default "no") !!! note The IP address of the network interface referenced by the `DAOS_IFACE_NAME` argument will be required when starting DAOS. -The Dockerfile of the `daos-client` and `daos-admin` images accept the following arguments: - -- `DAOS_BASE_IMAGE`: Base docker image to use (default "daos-base") -- `DAOS_BASE_VERSION`: Version of the base docker image to use (default "rocky8.6") -- `DAOS_AUTH`: Enable DAOS authentication when set to "yes" (default "yes") -- `DAOS_ADMIN_USER`: Name or uid of the daos administrattor user (default "root") -- `DAOS_ADMIN_GROUP`: Name or gid of the daos administrattor group (default "root") - -!!! warning - For working properly, the DAOS authentication have to be enabled in all the images (i.e. nodes - images and base image). - -The Dockerfile of the `daos-client` image accept the following arguments: - +The `daos-client` image is also using the following environment variables: - `DAOS_AGENT_IFACE_CFG`: Enable manual configuration of the interface to use by the agent (default "yes") - `DAOS_AGENT_IFACE_NUMA_NODE`: Numa node of the interface to use by the agent (default "0"). @@ -175,88 +138,21 @@ The Dockerfile of the `daos-client` image accept the following arguments: On most of the system the`DAOS_IFACE_CFG` should be enabled: The DAOS Network Interface auto-detection could not yet be properly done inside a DAOS Agent Docker container. -From a local tree, a more straightforward way to build these images could be done with -`docker compose`: - +When the environment file has been properly filled, run the following command to build the docker +images: ```bash $ docker compose --file utils/docker/vcluster/docker-compose.yml build daos_server daos_admin daos_client ``` -The same arguments are accepted but they have to be defined in the Docker Compose environment file -`utils/docker/vcluster/.env`. - -!!! warning - For working properly, the DAOS authentication have to be enabled in all the images (i.e. nodes - images and base image). - ## Running the DAOS Containers -### Via Docker Commands - -Once the images are created, the containers could be directly started with docker with the following -commands: - -```bash -$ export DAOS_IFACE_IP=x.x.x.x -$ docker run --detach --privileged --name=daos-server --hostname=daos-server \ - --add-host "daos-server:$DAOS_IFACE_IP" --add-host "daos-admin:$DAOS_IFACE_IP" \ - --add-host "daos-client:$DAOS_IFACE_IP" --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \ - --volume=/dev/hugepages:/dev/hugepages --tmpfs=/run --network=host \ - daos-server:rocky8.6 -$ docker run --detach --privileged --name=daos-agent --hostname=daos-agent \ - --add-host "daos-server:$DAOS_IFACE_IP" --add-host "daos-admin:$DAOS_IFACE_IP" \ - --add-host "daos-client:$DAOS_IFACE_IP" --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \ - --tmpfs=/run --network=host daos-agent:rocky8.6 -$ docker run --detach --privileged --name=daos-client --hostname=daos-client \ - --add-host "daos-server:$DAOS_IFACE_IP" --add-host "daos-admin:$DAOS_IFACE_IP" \ - --add-host "daos-client:$DAOS_IFACE_IP" --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \ - --tmpfs=/run --network=host daos-client:rocky8.6 -``` - -The value of the `DAOS_IFACE_IP` shall be replaced with the one of the network interface which was -provided when the images have been built. - -Once started, the DAOS server waits for the administrator to format the system. -This can be done using the following command: - -```bash -$ docker exec daos-admin dmg -i storage format -``` - -Upon successful completion of the format, the storage engine is started, and pools -can be created using the daos admin tool. For more advanced configurations and usage refer to the -section [DAOS Tour](https://docs.daos.io/v2.6/QSG/tour/). - - -### Via Docker Compose - -From a local tree, a more straightforward way to start the containers could be done with -`docker compose`: - -```bash -$ docker compose --file utils/docker/vcluster/docker-compose.yml up --detach -``` - -!!! note - Before starting the containers with `docker compose`, the IP address of the network interface, - which was provided when the images have been built, shall be defined in the Docker - Compose environment file `utils/docker/vcluster/.env`. - -As with the docker command, the system shall be formatted, pools created, etc.. - - -### Via Custom Scripts - -From a local tree, the bash script `utils/docker/vcluster/daos-cm.sh` could be used to start the -containers and setup a simple DAOS system composed of the following elements: - +Once the images are created, the bash script `utils/docker/vcluster/daos-cm.sh` can be used to to +start the containers and setup a simple DAOS system composed of the following elements: - 1 DAOS pool of 10GB (i.e. size of the pool is configurable) - 1 DAOS POSIX container mounted on /mnt/daos-posix-fs +This script can also be used to respectively stop and monitor the containers. -This script could also be used to respectively stop and monitor the containers. - -More details on the usage of `daos-cm.sh` command could be found with running the following command: - +To get more details on the usage of `daos-cm.sh` run the following command: ```bash $ utils/docker/vcluster/daos-cm.sh --help ``` diff --git a/utils/docker/examples/.env b/utils/docker/examples/.env index 8ac0c2fd4aa..36334a34c14 100644 --- a/utils/docker/examples/.env +++ b/utils/docker/examples/.env @@ -21,10 +21,10 @@ DAOS_AGENT_CERTS_TXZ="secrets/daos_agent-certs.txz" BUST_CACHE="" LINUX_DISTRO="el8" LINUX_IMAGE_NAME="rockylinux/rockylinux" -LINUX_IMAGE_TAG="8.8" -DAOS_REPOS="https://packages.daos.io/v2.4/EL8/packages/x86_64/" -DAOS_GPG_KEYS="https://packages.daos.io/v2.4.0/RPM-GPG-KEY-2023" +LINUX_IMAGE_TAG="8.9" +DAOS_REPOS="https://packages.daos.io/v2.4.1/EL8/packages/x86_64/" +DAOS_GPG_KEYS="https://packages.daos.io/v2.4.1/RPM-GPG-KEY-2023" DAOS_REPOS_NOAUTH="" -DAOS_VERSION="2.4.0-2.el8" +DAOS_VERSION="2.4.1-2.el8" DAOS_DOCKER_IMAGE_NSP="daos" -DAOS_DOCKER_IMAGE_TAG="v2.4.0" +DAOS_DOCKER_IMAGE_TAG="v2.4.1" diff --git a/utils/docker/examples/README.md b/utils/docker/examples/README.md index 667b0d2461f..c998ba2c268 100644 --- a/utils/docker/examples/README.md +++ b/utils/docker/examples/README.md @@ -15,7 +15,7 @@ The platform was tested and validated with the following dependencies: - [Docker CE](https://docs.docker.com/engine/install/centos/) latest [RPMs](https://download.docker.com/linux/centos/docker-ce.repo) - [DAOS 2.4](https://docs.daos.io/v2.4/) official [RPMS](https://packages.daos.io/v2.4/) -- [rockylinux/rockylinux:8.8](https://hub.docker.com/r/rockylinux/rockylinux/) official docker +- [rockylinux/rockylinux:8.9](https://hub.docker.com/r/rockylinux/rockylinux/) official docker images. @@ -34,18 +34,18 @@ properly build a docker image: The following environment variables allow to customize the Docker image to build: - `LINUX_DISTRO`: Linux distribution identifier (default "el8") - `DAOS_DOCKER_IMAGE_NSP`: Namespace identifier of the base DAOS docker image (default "daos") -- `DAOS_DOCKER_IMAGE_TAG`: Tag identifier of the base DAOS docker image (default "v2.4.0") +- `DAOS_DOCKER_IMAGE_TAG`: Tag identifier of the base DAOS docker image (default "v2.4.1") - `BUST_CACHE`: Manage docker building cache (default ""). To invalidate the cache, a random value such as the date of day shall be given. - `LINUX_IMAGE_NAME`: Base docker image name to use (default "rockylinux/rockylinux") -- `LINUX_IMAGE_TAG`: Tag identifier of the base docker image to use (default "8.8") +- `LINUX_IMAGE_TAG`: Tag identifier of the base docker image to use (default "8.9") - `DAOS_REPOS`: Space separated list of repos needed to install DAOS (default - "https://packages.daos.io/v2.4/EL8/packages/x86\_64/") + "https://packages.daos.io/v2.4.1/EL8/packages/x86\_64/") - `DAOS_GPG_KEYS`: Space separated list of GPG keys associated with DAOS repos (default - "https://packages.daos.io/v2.4.0/RPM-GPG-KEY-2023") + "https://packages.daos.io/v2.4.1/RPM-GPG-KEY-2023") - `DAOS_REPOS_NOAUTH`: Space separated list of repos to use without GPG authentication (default "") -- `DAOS_VERSION`: Version of DAOS to use (default "2.4.0-2.el8") +- `DAOS_VERSION`: Version of DAOS to use (default "2.4.1-2.el8") When the environment file has been properly filled, run the following command to build the base DAOS docker image. diff --git a/utils/docker/examples/daos-admin/el8/Dockerfile b/utils/docker/examples/daos-admin/el8/Dockerfile index 0d1b5b306c3..918d59de882 100644 --- a/utils/docker/examples/daos-admin/el8/Dockerfile +++ b/utils/docker/examples/daos-admin/el8/Dockerfile @@ -1,24 +1,30 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a base RHEL DAOS admin image # # This Dockerfile accept the following input build arguments: -# - LINUX_DISTRO Linux distribution identifier (default "el8") -# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (default "daos") -# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (default "v2.4.0") -# - DAOS_VERSION Version of DAOS to use (default "2.4.0-2.el8") +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) # Pull base image -ARG LINUX_DISTRO="el8" -ARG DAOS_DOCKER_IMAGE_NSP="daos" -ARG DAOS_DOCKER_IMAGE_TAG="v2.4.0" +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" FROM "$DAOS_DOCKER_IMAGE_NSP/daos-base-$LINUX_DISTRO:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install DAOS package -ARG DAOS_VERSION="2.4.0-2.el8" -RUN echo "[INFO] Installing DAOS containerization dependencies" ; \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + echo "[INFO] Installing DAOS containerization dependencies" ; \ dnf install \ sudo \ xz && \ diff --git a/utils/docker/examples/daos-admin/el8/daos-bash.sh b/utils/docker/examples/daos-admin/el8/daos-bash.sh index bfd6ff41b3b..edb8ae0acc4 100644 --- a/utils/docker/examples/daos-admin/el8/daos-bash.sh +++ b/utils/docker/examples/daos-admin/el8/daos-bash.sh @@ -1,7 +1,7 @@ #!/bin/bash # set -x -set -e -o pipefail +set -u -e -o pipefail if [[ "$(id -u)" != "0" ]] ; then echo "[ERROR] daos-bash can only be run as root" diff --git a/utils/docker/examples/daos-agent/el8/Dockerfile b/utils/docker/examples/daos-agent/el8/Dockerfile index dd0eec77ee6..7177e1d314a 100644 --- a/utils/docker/examples/daos-agent/el8/Dockerfile +++ b/utils/docker/examples/daos-agent/el8/Dockerfile @@ -1,24 +1,30 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a base RHEL DAOS client docker image # # This Dockerfile accept the following input build arguments: -# - LINUX_DISTRO Linux distribution identifier (default "el8") -# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (default "daos") -# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (default "v2.4.0") -# - DAOS_VERSION Version of DAOS to use (default "2.4.0-2.el8") +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) # Pull base image -ARG LINUX_DISTRO="el8" -ARG DAOS_DOCKER_IMAGE_NSP="daos" -ARG DAOS_DOCKER_IMAGE_TAG="v2.4.0" +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" FROM "$DAOS_DOCKER_IMAGE_NSP/daos-base-$LINUX_DISTRO:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install DAOS package -ARG DAOS_VERSION="2.4.0-2.el8" -RUN echo "[INFO] Installing DAOS containerization dependencies" ; \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + echo "[INFO] Installing DAOS containerization dependencies" ; \ dnf install \ sudo \ xz && \ diff --git a/utils/docker/examples/daos-agent/el8/run-daos_agent.sh b/utils/docker/examples/daos-agent/el8/run-daos_agent.sh index 048ee802460..150fb31d7d9 100644 --- a/utils/docker/examples/daos-agent/el8/run-daos_agent.sh +++ b/utils/docker/examples/daos-agent/el8/run-daos_agent.sh @@ -1,7 +1,7 @@ #!/bin/bash # set -x -set -e -o pipefail +set -u -e -o pipefail if [[ "$(id -u)" != "0" ]] ; then echo "[ERROR] run-daos_agent can only be run as root" diff --git a/utils/docker/examples/daos-base/el8/Dockerfile b/utils/docker/examples/daos-base/el8/Dockerfile index a9f4b054206..182def7ef00 100644 --- a/utils/docker/examples/daos-base/el8/Dockerfile +++ b/utils/docker/examples/daos-base/el8/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a base RHEL DAOS client docker image @@ -8,21 +8,18 @@ # - DAOS_CLIENT_UID User id of the client (mandatory) # - DAOS_CLIENT_GNAME Group name of the client (mandatory) # - DAOS_CLIENT_GID Group id of the client (mandatory) -# - LINUX_IMAGE_NAME Base docker image name to use (default "rockylinux/rockylinux") -# - LINUX_IMAGE_TAG Tag identifier of the base docker image to use (default "8.8") +# - LINUX_IMAGE_NAME Base docker image name to use (mandatory) +# - LINUX_IMAGE_TAG Tag identifier of the base docker image to use (mandatory) # - BUST_CACHE Manage docker building cache (default undefined). To invalidate the # cache, a random value such as the date of day shall be given. -# - DAOS_REPOS Space separated list of repos needed to install DAOS (default -# "https://packages.daos.io/v2.4/EL8/packages/x86_64/") -# - DAOS_GPG_KEYS Space separated list of GPG keys associated with DAOS repos (default -# "https://packages.daos.io/v2.4.0/RPM-GPG-KEY-2023") -# - DAOS_REPOS_NOAUTH Space separated list of repos to use without GPG authentication -# (optional) -# - DAOS_VERSION Version of DAOS to use (default "2.4.0-2.el8") +# - DAOS_REPOS Space separated list of repos needed to install DAOS (mandatory) +# - DAOS_GPG_KEYS Space separated list of GPG keys associated with DAOS repos mandatory) +# - DAOS_REPOS_NOAUTH Space separated list of repos to use without GPG authentication (optional) +# - DAOS_VERSION Version of DAOS to use (mandatory) # Pull base image -ARG LINUX_IMAGE_NAME="rockylinux/rockylinux" -ARG LINUX_IMAGE_TAG="8.8" +ARG LINUX_IMAGE_NAME="" +ARG LINUX_IMAGE_TAG="" FROM "$LINUX_IMAGE_NAME:$LINUX_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" @@ -42,11 +39,17 @@ RUN dnf clean all && # XXX NOTE XXX option and thus to not update all rpms. To work properly a random value such as the # XXX NOTE XXX date of the day should be given. ARG BUST_CACHE="" -ARG DAOS_REPOS="https://packages.daos.io/v2.4/EL8/packages/x86_64/" -ARG DAOS_GPG_KEYS="https://packages.daos.io/v2.4.0/RPM-GPG-KEY-2023" +ARG DAOS_REPOS="" +ARG DAOS_GPG_KEYS="" ARG DAOS_REPOS_NOAUTH="" -ARG DAOS_VERSION="2.4.0-2.el8" -RUN if [ -n "$BUST_CACHE" ] ; then \ +ARG DAOS_VERSION="" +RUN for it in DAOS_REPOS DAOS_GPG_KEYS DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + if [ -n "$BUST_CACHE" ] ; then \ echo "[INFO] Busting cache" ; \ dnf update ; \ fi && \ diff --git a/utils/docker/examples/daos-client/el8/Dockerfile b/utils/docker/examples/daos-client/el8/Dockerfile index cf0ceeb54c6..1b9913c2bf4 100644 --- a/utils/docker/examples/daos-client/el8/Dockerfile +++ b/utils/docker/examples/daos-client/el8/Dockerfile @@ -1,27 +1,32 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a base RHEL DAOS client docker image # # This Dockerfile accept the following input build arguments: -# - LINUX_DISTRO Linux distribution identifier (default "el8") -# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (default "daos") -# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (default "v2.4.0") -# - DAOS_VERSION Version of DAOS to use (default "2.4.0-2.el8") +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) # Pull base image -ARG LINUX_DISTRO="el8" -ARG DAOS_DOCKER_IMAGE_NSP="daos" -ARG DAOS_DOCKER_IMAGE_TAG="v2.4.0" +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" FROM "$DAOS_DOCKER_IMAGE_NSP/daos-base-$LINUX_DISTRO:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install base DAOS package -ARG DAOS_VERSION="2.4.0-2.el8" -RUN echo "[INFO] Installing DAOS" ; \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + echo "[INFO] Installing DAOS" ; \ dnf install \ - daos-client-${DAOS_VERSION} \ - daos-client-tests-${DAOS_VERSION} && \ + daos-client-${DAOS_VERSION} && \ dnf clean all # Install minimal DAOS agent configuration file diff --git a/utils/docker/examples/daos-client_agent/el8/Dockerfile b/utils/docker/examples/daos-client_agent/el8/Dockerfile index aa4b567e079..a30394094ac 100644 --- a/utils/docker/examples/daos-client_agent/el8/Dockerfile +++ b/utils/docker/examples/daos-client_agent/el8/Dockerfile @@ -1,32 +1,35 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a base RHEL DAOS client docker image # # This Dockerfile accept the following input build arguments: -# - LINUX_DISTRO Linux distribution identifier (default "el8") -# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (default "daos") -# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (default "v2.4.0") -# - DAOS_VERSION Version of DAOS to use (default "2.4.0-2.el8") +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) # - DAOS_CLIENT_UNAME User name of the client (mandatory) # - DAOS_CLIENT_GNAME Group name of the client (mandatory) # Pull base image -ARG LINUX_DISTRO="el8" -ARG DAOS_DOCKER_IMAGE_NSP="daos" -ARG DAOS_DOCKER_IMAGE_TAG="v2.4.0" +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" FROM "$DAOS_DOCKER_IMAGE_NSP/daos-agent-$LINUX_DISTRO:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install base DAOS package -ARG DAOS_VERSION="2.4.0-2.el8" -RUN echo "[INFO] Installing DAOS containerization dependencies" ; \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + echo "[INFO] Installing DAOS containerization dependencies" ; \ dnf install \ sudo \ xz && \ - echo "[INFO] Installing DAOS" ; \ - dnf install \ - daos-client-tests-${DAOS_VERSION} && \ dnf clean all # Install DAOS Bash launcher diff --git a/utils/docker/examples/daos-client_agent/el8/daos-bash.sh b/utils/docker/examples/daos-client_agent/el8/daos-bash.sh index d2999fe9afe..3170740a467 100644 --- a/utils/docker/examples/daos-client_agent/el8/daos-bash.sh +++ b/utils/docker/examples/daos-client_agent/el8/daos-bash.sh @@ -1,7 +1,7 @@ #!/bin/bash # set -x -set -e -o pipefail +set -u -e -o pipefail if [[ "$(id -u)" != "0" ]] ; then echo "[ERROR] daos-bash can only be run as root" diff --git a/utils/docker/examples/daos-server/el8/Dockerfile b/utils/docker/examples/daos-server/el8/Dockerfile index d5a003f8ca3..57db3348705 100644 --- a/utils/docker/examples/daos-server/el8/Dockerfile +++ b/utils/docker/examples/daos-server/el8/Dockerfile @@ -1,24 +1,30 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a base RHEL DAOS server docker image # # This Dockerfile accept the following input build arguments: -# - LINUX_DISTRO Linux distribution identifier (default "el8") -# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (default "daos") -# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS base docker image (default "v2.4.0") -# - DAOS_VERSION Version of DAOS to use (default "2.4.0-2.el8") +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) # Pull base image -ARG LINUX_DISTRO="el8" -ARG DAOS_DOCKER_IMAGE_NSP="daos" -ARG DAOS_DOCKER_IMAGE_TAG="v2.4.0" +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" FROM "$DAOS_DOCKER_IMAGE_NSP/daos-base-$LINUX_DISTRO:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install DAOS package -ARG DAOS_VERSION="2.4.0-2.el8" -RUN echo "[INFO] Installing DAOS containerization dependencies" ; \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + echo "[INFO] Installing DAOS containerization dependencies" ; \ dnf install \ kmod \ sudo \ diff --git a/utils/docker/examples/daos-server/el8/run-daos_server.sh b/utils/docker/examples/daos-server/el8/run-daos_server.sh index c9e4f1f15a8..4461b1eeee4 100644 --- a/utils/docker/examples/daos-server/el8/run-daos_server.sh +++ b/utils/docker/examples/daos-server/el8/run-daos_server.sh @@ -1,7 +1,7 @@ #!/bin/bash # set -x -set -e -o pipefail +set -u -e -o pipefail if [[ "$(id -u)" != "0" ]] ; then echo "[ERROR] run-daos_server can only be run as root" diff --git a/utils/docker/examples/daos-spdk_setup/el8/Dockerfile b/utils/docker/examples/daos-spdk_setup/el8/Dockerfile index 47169d8da74..e48a512569f 100644 --- a/utils/docker/examples/daos-spdk_setup/el8/Dockerfile +++ b/utils/docker/examples/daos-spdk_setup/el8/Dockerfile @@ -1,24 +1,30 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a base RHEL SPDK setup script wrapper # # This Dockerfile accept the following input build arguments: -# - LINUX_DISTRO Linux distribution identifier (default "el8") -# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (default "daos") -# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS base docker image (default "v2.4.0") -# - DAOS_VERSION Version of DAOS to use (default "2.4.0-2.el8") +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) # Pull base image -ARG LINUX_DISTRO="el8" -ARG DAOS_DOCKER_IMAGE_NSP="daos" -ARG DAOS_DOCKER_IMAGE_TAG="v2.4.0" +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" FROM "$DAOS_DOCKER_IMAGE_NSP/daos-base-$LINUX_DISTRO:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install DAOS package -ARG DAOS_VERSION="2.4.0-2.el8" -RUN echo "[INFO] Installing SPDK tools dependencies" ; \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + echo "[INFO] Installing SPDK tools dependencies" ; \ dnf install \ kmod && \ echo "[INFO] Installing SPDK tools" ; \ diff --git a/utils/docker/examples/deploy-docker_stack.sh b/utils/docker/examples/deploy-docker_stack.sh index 92f7eed6dc5..64666305c00 100644 --- a/utils/docker/examples/deploy-docker_stack.sh +++ b/utils/docker/examples/deploy-docker_stack.sh @@ -1,9 +1,10 @@ #!/bin/bash # set -x -set -e -o pipefail +set -u -e -o pipefail -CWD="$(realpath "${0%/*}")" +CWD="$(realpath "${0%}")" +CWD="${CWD%/*}" set -a # shellcheck disable=SC1091 diff --git a/utils/docker/vcluster/.env b/utils/docker/vcluster/.env index a5c6384dc88..412a5e3763e 100644 --- a/utils/docker/vcluster/.env +++ b/utils/docker/vcluster/.env @@ -7,28 +7,25 @@ DAOS_IFACE_IP="x.x.x.x" # Image building configuration -RHEL_BASE_IMAGE=rockylinux/rockylinux -RHEL_BASE_VERSION=8.6 BUST_CACHE="" -DAOS_REPOS="https://packages.daos.io/v2.2/EL8/packages/x86_64/" -DAOS_GPG_KEYS="https://packages.daos.io/RPM-GPG-KEY" +LINUX_DISTRO="el8" +LINUX_IMAGE_NAME="rockylinux/rockylinux" +LINUX_IMAGE_TAG="8.9" +DAOS_REPOS="https://packages.daos.io/v2.4.1/EL8/packages/x86_64/" +DAOS_GPG_KEYS="https://packages.daos.io/v2.4.1/RPM-GPG-KEY-2023" DAOS_REPOS_NOAUTH="" -DAOS_BASE_IMAGE="daos-base" -DAOS_BASE_VERSION="rocky8.6" +DAOS_VERSION="2.4.1-2.el8" +DAOS_DOCKER_IMAGE_NSP="daos" +DAOS_DOCKER_IMAGE_TAG="v2.4.1" DAOS_AUTH="yes" DAOS_HUGEPAGES_NBR=4096 DAOS_IFACE_NAME="eth0" -DAOS_SCM_SIZE=4 -DAOS_BDEV_SIZE=16 +DAOS_SCM_SIZE=6 +DAOS_BDEV_SIZE=24 DAOS_ADMIN_USER="root" DAOS_ADMIN_GROUP="root" DAOS_AGENT_IFACE_CFG="yes" DAOS_AGENT_IFACE_NUMA_NODE="0" DAOS_AGENT_IFACE_NAME="eth0" DAOS_AGENT_IFACE_DOMAIN_NAME="eth0" - -# Image TAG configuration -# XXX Should not be changed XXX -DAOS_SERVER_VERSION="${DAOS_BASE_VERSION}" -DAOS_ADMIN_VERSION="${DAOS_BASE_VERSION}" -DAOS_CLIENT_VERSION="${DAOS_BASE_VERSION}" +DAOS_MD_ON_SSD="no" diff --git a/utils/docker/vcluster/daos-admin/el8/Dockerfile b/utils/docker/vcluster/daos-admin/el8/Dockerfile index f83aa851dc1..e948d9d95dd 100644 --- a/utils/docker/vcluster/daos-admin/el8/Dockerfile +++ b/utils/docker/vcluster/daos-admin/el8/Dockerfile @@ -1,53 +1,60 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a RHEL variant docker image of a DAOS administrator node # -# - DAOS_BASE_IMAGE Base docker image to use (default "daos-base") -# - DAOS_BASE_VERSION Version of the base docker image to use (default "rocky8.6") -# - DAOS_AUTH Enable DAOS authentication when set to "yes" (default "yes") -# - DAOS_ADMIN_USER Name or uid of the daos administrattor user (default "root") -# - DAOS_ADMIN_GROUP Name or gid of the daos administrattor group (default "root") +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) +# - DAOS_AUTH Enable DAOS authentication when set to "yes" (mandatory) # Pull base image -ARG DAOS_BASE_IMAGE=daos-base -ARG DAOS_BASE_VERSION=rocky8.6 -FROM $DAOS_BASE_IMAGE:$DAOS_BASE_VERSION +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" +FROM "${DAOS_DOCKER_IMAGE_NSP}/daos-base-${LINUX_DISTRO}:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install DAOS client package -RUN dnf install daos-admin && \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + dnf install daos-admin-${DAOS_VERSION} && \ dnf clean all # Install certificates -ARG DAOS_AUTH=yes -ARG DAOS_ADMIN_USER=root -ARG DAOS_ADMIN_GROUP=root -# FIXME Should be provided through volumes (or Secrets for K8S) -# XXX NOTE XXX With a production platform, this configuration file should be provided with a volume -# XXX NOTE XXX (or ConfigMaps for K8S). COPY daos_control.yml.in /tmp/daos_control.yml.in -RUN if [ "$DAOS_AUTH" == yes ] ; then \ +ARG DAOS_AUTH="" +RUN for it in DAOS_AUTH ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + if [ "$DAOS_AUTH" == yes ] ; then \ sed --regexp-extended \ - --expression '/^@DAOS_NOAUTH_SECTION_BEGIN@$/,/^@DAOS_NOAUTH_SECTION_END@/d' \ - --expression '/(^@DAOS_AUTH_SECTION_BEGIN@$)|(^@DAOS_AUTH_SECTION_END@$)/d' \ + --expression '/^@DAOS_NOAUTH_BEGIN@$/,/^@DAOS_NOAUTH_END@/d' \ + --expression '/(^@DAOS_AUTH_BEGIN@$)|(^@DAOS_AUTH_END@$)/d' \ /tmp/daos_control.yml.in > /etc/daos/daos_control.yml && \ -# XXX WARNING XXX With a production platform, these certificates should be provided with a volume -# XXX WARNING XXX (or Secrets with K8S). chmod 644 /root/daosCA/certs/daosCA.crt && \ chmod 644 /root/daosCA/certs/admin.crt && \ - chmod 600 /root/daosCA/certs/admin.key && \ - chown "$DAOS_ADMIN_USER:$DAOS_ADMIN_GROUP" /root/daosCA/certs/daosCA.crt && \ - chown "$DAOS_ADMIN_USER:$DAOS_ADMIN_GROUP" /root/daosCA/certs/admin.crt && \ - chown "$DAOS_ADMIN_USER:$DAOS_ADMIN_GROUP" /root/daosCA/certs/admin.key && \ + chmod 400 /root/daosCA/certs/admin.key && \ + chown root:root /root/daosCA/certs/daosCA.crt && \ + chown root:root /root/daosCA/certs/admin.crt && \ + chown root:root /root/daosCA/certs/admin.key && \ mv /root/daosCA/certs/daosCA.crt /etc/daos/certs/. && \ mv /root/daosCA/certs/admin.crt /etc/daos/certs/. && \ mv /root/daosCA/certs/admin.key /etc/daos/certs/. && \ rm -fr /root/daosCA ; \ else \ sed --regexp-extended \ - --expression '/^@DAOS_AUTH_SECTION_BEGIN@$/,/^@DAOS_AUTH_SECTION_END@/d' \ - --expression '/(^@DAOS_NOAUTH_SECTION_BEGIN@$)|(^@DAOS_NOAUTH_SECTION_END@$)/d'\ + --expression '/^@DAOS_AUTH_BEGIN@$/,/^@DAOS_AUTH_END@/d' \ + --expression '/(^@DAOS_NOAUTH_BEGIN@$)|(^@DAOS_NOAUTH_END@$)/d'\ /tmp/daos_control.yml.in > /etc/daos/daos_control.yml ; \ fi && \ rm -f /tmp/daos_control.yml.in diff --git a/utils/docker/vcluster/daos-admin/el8/daos_control.yml.in b/utils/docker/vcluster/daos-admin/el8/daos_control.yml.in index 0369743231b..9de48e82f67 100644 --- a/utils/docker/vcluster/daos-admin/el8/daos_control.yml.in +++ b/utils/docker/vcluster/daos-admin/el8/daos_control.yml.in @@ -5,12 +5,12 @@ hostlist: ['daos-server'] port: 10001 transport_config: -@DAOS_NOAUTH_SECTION_BEGIN@ +@DAOS_NOAUTH_BEGIN@ allow_insecure: true -@DAOS_NOAUTH_SECTION_END@ -@DAOS_AUTH_SECTION_BEGIN@ +@DAOS_NOAUTH_END@ +@DAOS_AUTH_BEGIN@ allow_insecure: false ca_cert: /etc/daos/certs/daosCA.crt cert: /etc/daos/certs/admin.crt key: /etc/daos/certs/admin.key -@DAOS_AUTH_SECTION_END@ +@DAOS_AUTH_END@ diff --git a/utils/docker/vcluster/daos-base/el8/Dockerfile b/utils/docker/vcluster/daos-base/el8/Dockerfile index 125a0cb9ac6..5837a0a90e6 100644 --- a/utils/docker/vcluster/daos-base/el8/Dockerfile +++ b/utils/docker/vcluster/daos-base/el8/Dockerfile @@ -1,48 +1,25 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a base RHEL variant docker image of a DAOS node # # This Dockerfile accept the following input build arguments: -# - RHEL_BASE_IMAGE Base docker image to use (default "rockylinux/rockylinux") -# - RHEL_BASE_VERSION Version of the base docker image to use (default "8.6") +# - LINUX_IMAGE_NAME Base docker image name to use (mandatory) +# - LINUX_IMAGE_TAG Tag identifier of the base docker image to use (mandatory) # - BUST_CACHE Manage docker building cache (default ""). To invalidate the cache, # a random value such as the date of day shall be given. -# - DAOS_AUTH Enable DAOS authentication when set to "yes" (default "yes") -# - DAOS_REPOS Space separated list of repos needed to install DAOS (default -# "https://packages.daos.io/v2.2/EL8/packages/x86_64/") -# - DAOS_GPG_KEYS Space separated list of GPG keys associated with DAOS repos (default -# "https://packages.daos.io/RPM-GPG-KEY") -# - DAOS_REPOS_NOAUTH Space separated list of repos to use without GPG authentication -# (default "") +# - DAOS_AUTH Enable DAOS authentication when set to "yes" (mandatory) +# - DAOS_REPOS Space separated list of repos needed to install DAOS (mandatory) +# - DAOS_GPG_KEYS Space separated list of GPG keys associated with DAOS repos mandatory) +# - DAOS_REPOS_NOAUTH Space separated list of repos to use without GPG authentication (optional) +# - DAOS_VERSION Version of DAOS to use (mandatory) # Pull base image -ARG RHEL_BASE_IMAGE=rockylinux/rockylinux -ARG RHEL_BASE_VERSION=8.6 -FROM $RHEL_BASE_IMAGE:$RHEL_BASE_VERSION +ARG LINUX_IMAGE_NAME="" +ARG LINUX_IMAGE_TAG="" +FROM $LINUX_IMAGE_NAME:$LINUX_IMAGE_TAG LABEL maintainer="daos@daos.groups.io" -# Configure systemd: more details could be found at following URL: -# https://markandruth.co.uk/2020/10/10/running-systemd-inside-a-centos-8-docker-container -# XXX FIXME XXX Should be removed in production with application dedicated entry point -VOLUME [ "/sys/fs/cgroup" ] -RUN systemctl mask systemd-remount-fs.service graphical.target kdump.service \ - systemd-logind.service dev-hugepages.mount && \ - pushd /lib/systemd/system/sysinit.target.wants && \ - for item in * ; do \ - [ "$item" == systemd-tmpfiles-setup.service ] || rm -f "$item" ; \ - done && \ - popd && \ - rm -f /lib/systemd/system/multi-user.target.wants/* && \ - rm -f /etc/systemd/system/*.wants/* && \ - rm -f /lib/systemd/system/local-fs.target.wants/* && \ - rm -f /lib/systemd/system/sockets.target.wants/*udev* && \ - rm -f /lib/systemd/system/sockets.target.wants/*initctl* && \ - rm -f /lib/systemd/system/basic.target.wants/* && \ - rm -f /lib/systemd/system/anaconda.target.wants/* -STOPSIGNAL SIGRTMIN+3 -ENTRYPOINT [ "/sbin/init" ] - # Base configuration of dnf and system update RUN dnf clean all && \ dnf makecache && \ @@ -59,10 +36,17 @@ RUN dnf clean all && # XXX NOTE XXX to not update all rpms. To work properly a random value such as the date of the day # XXX NOTE XXX should be given. ARG BUST_CACHE="" -ARG DAOS_REPOS="https://packages.daos.io/v2.2/EL8/packages/x86_64/" -ARG DAOS_GPG_KEYS="https://packages.daos.io/RPM-GPG-KEY" +ARG DAOS_REPOS="" +ARG DAOS_GPG_KEYS="" +ARG DAOS_VERSION="" ARG DAOS_REPOS_NOAUTH="" -RUN if [ -n "$BUST_CACHE" ] ; then \ +RUN for it in DAOS_REPOS DAOS_GPG_KEYS DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + if [ -n "$BUST_CACHE" ] ; then \ echo "[INFO] Busting cache" && \ dnf update ; \ fi && \ @@ -79,12 +63,18 @@ RUN if [ -n "$BUST_CACHE" ] ; then dnf config-manager --save --setopt="${repo}.gpgcheck=0" ; \ done && \ echo "[INFO] Installing DAOS" && \ - dnf install daos && \ + dnf install daos-${DAOS_VERSION} && \ dnf clean all # Generate GPG authentication certificates for using DAOS authentication -ARG DAOS_AUTH=yes -RUN if [ "$DAOS_AUTH" == yes ] ; then \ +ARG DAOS_AUTH="" +RUN for it in DAOS_AUTH ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + if [ "$DAOS_AUTH" == yes ] ; then \ echo "[INFO] Generating authentication certificates" && \ if [ ! -d /etc/daos/certs ] ; then \ mkdir -d /etc/daos/certs && \ @@ -95,3 +85,24 @@ RUN if [ "$DAOS_AUTH" == yes ] ; then # XXX WARNING XXX (or Secrets with K8S). cd /root && /usr/lib64/daos/certgen/gen_certificates.sh ; \ fi + +# Configure systemd: more details could be found at following URL: +# https://markandruth.co.uk/2020/10/10/running-systemd-inside-a-centos-8-docker-container +# XXX FIXME XXX Should be removed in production with application dedicated entry point +VOLUME [ "/sys/fs/cgroup" ] +RUN systemctl mask systemd-remount-fs.service graphical.target kdump.service \ + systemd-logind.service dev-hugepages.mount && \ + pushd /lib/systemd/system/sysinit.target.wants && \ + for item in * ; do \ + [ "$item" == systemd-tmpfiles-setup.service ] || rm -f "$item" ; \ + done && \ + popd && \ + rm -f /lib/systemd/system/multi-user.target.wants/* && \ + rm -f /etc/systemd/system/*.wants/* && \ + rm -f /lib/systemd/system/local-fs.target.wants/* && \ + rm -f /lib/systemd/system/sockets.target.wants/*udev* && \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl* && \ + rm -f /lib/systemd/system/basic.target.wants/* && \ + rm -f /lib/systemd/system/anaconda.target.wants/* +STOPSIGNAL SIGRTMIN+3 +ENTRYPOINT [ "/sbin/init" ] diff --git a/utils/docker/vcluster/daos-client/el8/Dockerfile b/utils/docker/vcluster/daos-client/el8/Dockerfile index 6b5b2e76ba8..60036ab614f 100644 --- a/utils/docker/vcluster/daos-client/el8/Dockerfile +++ b/utils/docker/vcluster/daos-client/el8/Dockerfile @@ -1,49 +1,63 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a RHEL variant docker image of a DAOS client node # # This Dockerfile accept the following input build arguments: -# - DAOS_BASE_IMAGE Base docker image to use (default "daos-base") -# - DAOS_BASE_VERSION Version of the base docker image to use (default "rocky8.6") -# - DAOS_AUTH Enable DAOS authentication when set to "yes" (default "yes") -# - DAOS_ADMIN_USER Name or uid of the daos administrattor user (default "root") -# - DAOS_ADMIN_GROUP Name or gid of the daos administrattor group (default "root") +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) +# - DAOS_AUTH Enable DAOS authentication when set to "yes" (mandatory) # - DAOS_AGENT_IFACE_CFG Enable manual configuration of the interface to use by the agent -# (default "yes") -# - DAOS_AGENT_IFACE_NUMA_NODE Numa node of the interface to use by the agent (default "0") -# - DAOS_AGENT_IFACE_NAME Name of the interface to use by the agent (default "eth0") -# - DAOS_AGENT_IFACE_DOMAIN_NAME Domain name of the interface to use by the agent (default "eth0") +# (mandatory) +# - DAOS_AGENT_IFACE_NUMA_NODE Numa node of the interface to use by the agent (mandatory) +# - DAOS_AGENT_IFACE_NAME Name of the interface to use by the agent (mandatory) +# - DAOS_AGENT_IFACE_DOMAIN_NAME Domain name of the interface to use by the agent (mandatory) # Pull base image -ARG DAOS_BASE_IMAGE=daos-base -ARG DAOS_BASE_VERSION=rocky8.6 -FROM $DAOS_BASE_IMAGE:$DAOS_BASE_VERSION +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" +FROM "${DAOS_DOCKER_IMAGE_NSP}/daos-base-${LINUX_DISTRO}:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install DAOS client package -RUN dnf install daos-client daos-tests && \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + dnf install \ + daos-client-${DAOS_VERSION} && \ dnf clean all && \ systemctl enable daos_agent # Install certificates -ARG DAOS_AUTH=yes -ARG DAOS_ADMIN_USER=root -ARG DAOS_ADMIN_GROUP=root -ARG DAOS_AGENT_IFACE_CFG="yes" -ARG DAOS_AGENT_IFACE_NUMA_NODE="0" -ARG DAOS_AGENT_IFACE_NAME="eth0" -ARG DAOS_AGENT_IFACE_DOMAIN_NAME="eth0" +ARG DAOS_AUTH="" +ARG DAOS_AGENT_IFACE_CFG="" +ARG DAOS_AGENT_IFACE_NUMA_NODE="" +ARG DAOS_AGENT_IFACE_NAME="" +ARG DAOS_AGENT_IFACE_DOMAIN_NAME="" COPY daos_agent.yml.in /tmp/daos_agent.yml.in -RUN if [ "$DAOS_AUTH" == yes ] ; then \ +RUN for it in DAOS_AUTH DAOS_AGENT_IFACE_CFG DAOS_AGENT_IFACE_NUMA_NODE DAOS_AGENT_IFACE_NAME \ + DAOS_AGENT_IFACE_DOMAIN_NAME ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + if [ "$DAOS_AUTH" == yes ] ; then \ sed --in-place --regexp-extended \ - --expression '/^@DAOS_NOAUTH_SECTION_BEGIN@$/,/^@DAOS_NOAUTH_SECTION_END@/d' \ - --expression '/(^@DAOS_AUTH_SECTION_BEGIN@$)|(^@DAOS_AUTH_SECTION_END@$)/d' \ + --expression '/^@DAOS_NOAUTH_BEGIN@$/,/^@DAOS_NOAUTH_END@/d' \ + --expression '/(^@DAOS_AUTH_BEGIN@$)|(^@DAOS_AUTH_END@$)/d' \ /tmp/daos_agent.yml.in && \ chmod 644 /root/daosCA/certs/daosCA.crt && \ chmod 644 /root/daosCA/certs/agent.crt && \ - chmod 600 /root/daosCA/certs/agent.key && \ - chown "$DAOS_ADMIN_USER:$DAOS_ADMIN_GROUP" /root/daosCA/certs/daosCA.crt && \ + chmod 400 /root/daosCA/certs/agent.key && \ + chown root:root /root/daosCA/certs/daosCA.crt && \ chown daos_agent:daos_agent /root/daosCA/certs/agent.crt && \ chown daos_agent:daos_agent /root/daosCA/certs/agent.key && \ mv /root/daosCA/certs/daosCA.crt /etc/daos/certs/. && \ @@ -52,26 +66,31 @@ RUN if [ "$DAOS_AUTH" == yes ] ; then rm -fr /root/daosCA ; \ else \ sed --in-place --regexp-extended \ - --expression '/^@DAOS_AUTH_SECTION_BEGIN@$/,/^@DAOS_AUTH_SECTION_END@/d' \ - --expression '/(^@DAOS_NOAUTH_SECTION_BEGIN@$)|(^@DAOS_NOAUTH_SECTION_END@$)/d' \ + --expression '/^@DAOS_AUTH_BEGIN@$/,/^@DAOS_AUTH_END@/d' \ + --expression '/(^@DAOS_NOAUTH_BEGIN@$)|(^@DAOS_NOAUTH_END@$)/d' \ /tmp/daos_agent.yml.in ; \ fi && \ if [[ "${DAOS_AGENT_IFACE_CFG}" == yes ]] ; then \ - for it in DAOS_AGENT_IFACE_NUMA_NODE DAOS_AGENT_IFACE_NAME DAOS_AGENT_IFACE_DOMAIN_NAME ; do \ + args=( \ + DAOS_AGENT_IFACE_NUMA_NODE \ + DAOS_AGENT_IFACE_NAME \ + DAOS_AGENT_IFACE_DOMAIN_NAME \ + ) ; \ + for it in $args; do \ if eval "[[ -z \"\$$it\" ]]" ; then \ echo "[ERROR] Docker build argument $it is not defined" ; \ exit 1 ; \ fi ; \ done ; \ sed --in-place --regexp-extended \ - --expression '/(^@DAOS_IFACE_SECTION_BEGIN@$)|(^@DAOS_IFACE_SECTION_END@$)/d' \ + --expression '/(^@DAOS_IFACE_BEGIN@$)|(^@DAOS_IFACE_END@$)/d' \ --expression "s/@DAOS_IFACE_NUMA_NODE@/${DAOS_AGENT_IFACE_NUMA_NODE}/" \ --expression "s/@DAOS_IFACE_NAME@/${DAOS_AGENT_IFACE_NAME}/" \ --expression "s/@DAOS_IFACE_DOMAIN_NAME@/${DAOS_AGENT_IFACE_DOMAIN_NAME}/" \ /tmp/daos_agent.yml.in ; \ else \ sed --in-place --regexp-extended \ - --expression '/^@DAOS_IFACE_SECTION_BEGIN@$/,/^@DAOS_IFACE_SECTION_END@/d' \ + --expression '/^@DAOS_IFACE_BEGIN@$/,/^@DAOS_IFACE_END@/d' \ /tmp/daos_agent.yml.in ; \ fi && \ mv -f /tmp/daos_agent.yml.in /etc/daos/daos_agent.yml diff --git a/utils/docker/vcluster/daos-client/el8/daos_agent.yml.in b/utils/docker/vcluster/daos-client/el8/daos_agent.yml.in index f8ea938096d..5d2b591a57d 100644 --- a/utils/docker/vcluster/daos-client/el8/daos_agent.yml.in +++ b/utils/docker/vcluster/daos-client/el8/daos_agent.yml.in @@ -9,20 +9,20 @@ log_file: /tmp/daos_agent.log control_log_mask: info transport_config: -@DAOS_NOAUTH_SECTION_BEGIN@ +@DAOS_NOAUTH_BEGIN@ allow_insecure: true -@DAOS_NOAUTH_SECTION_END@ -@DAOS_AUTH_SECTION_BEGIN@ +@DAOS_NOAUTH_END@ +@DAOS_AUTH_BEGIN@ allow_insecure: false ca_cert: /etc/daos/certs/daosCA.crt cert: /etc/daos/certs/agent.crt key: /etc/daos/certs/agent.key -@DAOS_AUTH_SECTION_END@ +@DAOS_AUTH_END@ -@DAOS_IFACE_SECTION_BEGIN@ +@DAOS_IFACE_BEGIN@ fabric_ifaces: - numa_node: @DAOS_IFACE_NUMA_NODE@ devices: - iface: @DAOS_IFACE_NAME@ domain: @DAOS_IFACE_DOMAIN_NAME@ -@DAOS_IFACE_SECTION_END@ +@DAOS_IFACE_END@ diff --git a/utils/docker/vcluster/daos-cm.sh b/utils/docker/vcluster/daos-cm.sh index 40acd028bc2..7fba639248c 100755 --- a/utils/docker/vcluster/daos-cm.sh +++ b/utils/docker/vcluster/daos-cm.sh @@ -2,13 +2,13 @@ # shellcheck disable=SC2034,SC2145,SC2086,SC2068 -set -e -o pipefail +set -u -e -o pipefail -VERSION=0.2 -# shellcheck disable=SC2046 -CWD="$(realpath $(dirname $0))" +VERSION=0.3 +CWD="$(realpath "${0%}")" +CWD="${CWD%/*}" -DAOS_POOL_SIZE=10G +DAOS_POOL_SIZE=22G ANSI_COLOR_BLACK=30 ANSI_COLOR_RED=31 @@ -135,8 +135,8 @@ function stop function start { - DAOS_IFACE_IP="$1" - DAOS_POOL_SIZE="$2" + DAOS_IFACE_IP="${1:?Network Interface IP has to be defined}" + DAOS_POOL_SIZE="${2:?Pool size has to be defined}" info "Starting DAOS virtual cluster containers" if ! run env DAOS_IFACE_IP="$DAOS_IFACE_IP" docker compose up --detach daos_server daos_admin daos_client ; then @@ -144,22 +144,41 @@ function start fi info "Waiting for daos-server services to be started" - timeout_counter=0 - until [[ $timeout_counter -ge 5 ]] || run docker exec daos-server systemctl --quiet is-active daos_server ; do - info "daos-server not yet ready: waiting 1s" + timeout_counter=5 + until docker exec daos-server systemctl --quiet is-active daos_server > /dev/null 2>&1 ; do + info "daos-server not yet ready: timeout=$timeout_counter" sleep 1 - (( timeout_counter++ )) + if ! (( timeout_counter-- )) ; then + fatal "DAOS server could not be started" + fi done - if [[ $timeout_counter -ge 5 ]] ; then - fatal "DAOS server could not be started" - fi + + timeout_counter=10 + until docker exec daos-server grep -q -e "format required" /tmp/daos_server.log > /dev/null 2>&1 ; do + info "Waiting DAOS file system for being ready to be formatted : timeout=$timeout_counter" + sleep 1 + if ! (( timeout_counter-- )) ; then + fatal "DAOS file system could not be formatted" + fi + done + info "DAOS file system ready to be formatted" info "Formatting DAOS storage" if ! run docker exec daos-admin dmg storage format --host-list=daos-server ; then fatal "DAOS storage could not be formatted" fi - info "Checking system" + timeout_counter=10 + until docker exec daos-server grep -q -e "DAOS I/O Engine .* started on rank" /tmp/daos_server.log > /dev/null 2>&1 ; do + info "Waiting DAOS file system to be formatted : timeout=$timeout_counter" + sleep 1 + if ! (( timeout_counter-- )) ; then + fatal "DAOS file system could not be formatted" + fi + done + info "DAOS file system formatted" + + info "Checking system state" if ! run docker exec daos-admin dmg system query --verbose ; then fatal "DAOS system not healthy" fi @@ -206,9 +225,9 @@ function start ================================================================================ Mount point /mnt/daos-posix-fs is ready on daos-client container. - fio could be run on DAOS POSIX container with the following command: + dd could be run on DAOS POSIX container with the following command: - docker exec daos-client /usr/bin/fio --name=random-write --ioengine=pvsync --rw=randwrite --bs=4k --size=128M --nrfiles=4 --directory=/mnt/daos-posix-fs --numjobs=8 --iodepth=16 --runtime=60 --time_based --direct=1 --buffered=0 --randrepeat=0 --norandommap --refill_buffers --group_reporting + docker exec daos-client /usr/bin/dd if=/dev/urandom of=/mnt/daos-posix-fs/blob bs=1M count=100 ================================================================================ EOF } @@ -229,10 +248,10 @@ do esac done -[[ $1 ]] || fatal "Command not defined: start, stop or state" -[[ $1 != "start" || $2 ]] || fatal "Start command: missing IP address" +[[ ${1:+x} ]] || fatal "Command not defined: start, stop or state" +[[ "$1" != "start" || ${2:+x} ]] || fatal "Start command: missing IP address" CMD="$1" -DAOS_IFACE_IP="$2" +[[ ${2:+x} ]] && DAOS_IFACE_IP="$2" cd "$CWD" case "$CMD" in diff --git a/utils/docker/vcluster/daos-server/el8/Dockerfile b/utils/docker/vcluster/daos-server/el8/Dockerfile index 0cbd6517c16..0cbc8fd1154 100644 --- a/utils/docker/vcluster/daos-server/el8/Dockerfile +++ b/utils/docker/vcluster/daos-server/el8/Dockerfile @@ -1,59 +1,85 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # 'recipe' for building a RHEL variant docker image of a DAOS server node # # This Dockerfile accept the following input build arguments: -# - DAOS_BASE_IMAGE Base docker image to use (default "daos-base") -# - DAOS_BASE_VERSION Version of the base docker image to use (default "rocky8.6") -# - DAOS_AUTH Enable DAOS authentication when set to "yes" (default "yes") -# - DAOS_HUGEPAGES_NBR Number of huge pages to allocate for SPDK (default 4096) -# - DAOS_IFACE_NAME Fabric network interface used by the DAOS engine (default "eth0") -# - DAOS_SCM_SIZE Size in GB of the RAM emulating SCM devices (default 4) -# - DAOS_BDEV_SIZE Size in GB of the file created to emulate NVMe devices (derault 16) +# - LINUX_DISTRO Linux distribution identifier (mandatory) +# - DAOS_DOCKER_IMAGE_NSP Namespace identifier of the base DAOS docker image (mandatory) +# - DAOS_DOCKER_IMAGE_TAG Tag identifier of the DAOS client docker image (mandatory) +# - DAOS_VERSION Version of DAOS to use (mandatory) +# - DAOS_AUTH Enable DAOS authentication when set to "yes" (mandatory) +# - DAOS_HUGEPAGES_NBR Number of huge pages to allocate for SPDK (mandatory) +# - DAOS_IFACE_NAME Fabric network interface used by the DAOS engine (mandatory) +# - DAOS_SCM_SIZE Size in GB of the RAM emulating SCM devices (mandatory) +# - DAOS_BDEV_SIZE Size in GB of the file created to emulate NVMe devices (mandatory) +# - DAOS_MD_ON_SSD Enable DAOS MD-on-SSD feature when set to "yes" (mandatory) # Pull base image -ARG DAOS_BASE_IMAGE=daos-base -ARG DAOS_BASE_VERSION=rocky8.6 -FROM $DAOS_BASE_IMAGE:$DAOS_BASE_VERSION +ARG LINUX_DISTRO="" +ARG DAOS_DOCKER_IMAGE_NSP="" +ARG DAOS_DOCKER_IMAGE_TAG="" +FROM "${DAOS_DOCKER_IMAGE_NSP}/daos-base-${LINUX_DISTRO}:$DAOS_DOCKER_IMAGE_TAG" LABEL maintainer="daos@daos.groups.io" # Install DAOS server package -RUN dnf install daos-server && \ +ARG DAOS_VERSION="" +RUN for it in DAOS_VERSION ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ + dnf install daos-server-${DAOS_VERSION} && \ dnf clean all && \ systemctl enable daos_server # Configuration of the server -ARG DAOS_AUTH=yes -ARG DAOS_HUGEPAGES_NBR=4096 -ARG DAOS_IFACE_NAME=eth0 -ARG DAOS_SCM_SIZE=4 -ARG DAOS_BDEV_SIZE=16 -# XXX NOTE XXX With a production platform, this configuration file should be provided with a volume -# XXX NOTE XXX (or ConfigMaps for K8S). COPY daos_server.yml.in /tmp/daos_server.yml.in +ARG DAOS_AUTH="" +ARG DAOS_HUGEPAGES_NBR="" +ARG DAOS_IFACE_NAME="" +ARG DAOS_SCM_SIZE="" +ARG DAOS_BDEV_SIZE="" +ARG DAOS_MD_ON_SSD="" +RUN for it in DAOS_AUTH DAOS_HUGEPAGES_NBR DAOS_IFACE_NAME DAOS_SCM_SIZE DAOS_BDEV_SIZE \ + DAOS_MD_ON_SSD ; do \ + if eval "[[ -z \$$it ]]" ; then \ + echo "[ERROR] Docker build argument $it is not defined" ; \ + exit 1 ; \ + fi ; \ + done && \ # XXX NOTE XXX HugePages and vfio seem to not be well managed with docker and latest version of the # XXX NOTE XXX kernel. More details could be found at: # XXX NOTE XXX - https://stackoverflow.com/questions/64707324/dpdk-sample-application-aborts-after-eal-couldnt-get-fd-on-hugepage-file # XXX NOTE XXX - https://doc.dpdk.org/guides/linux_gsg/enable_func.html -RUN sed --regexp-extended --in-place \ + sed --regexp-extended --in-place \ --expression "s/^User=daos_server$/User=root/" \ --expression "s/^Group=daos_server$/Group=root/" \ /usr/lib/systemd/system/daos_server.service && \ + sed_opts="--regexp-extended" ; \ + sed_opts+=" --expression s/@DAOS_HUGEPAGES_NBR@/${DAOS_HUGEPAGES_NBR}/" ; \ + sed_opts+=" --expression s/@DAOS_IFACE_NAME@/${DAOS_IFACE_NAME}/" ; \ + sed_opts+=" --expression s/@DAOS_SCM_SIZE@/${DAOS_SCM_SIZE}/" ; \ + sed_opts+=" --expression s/@DAOS_BDEV_SIZE@/${DAOS_BDEV_SIZE}/" ; \ + if [ "$DAOS_AUTH" == yes ] ; then \ + sed_opts+=' --expression /^@DAOS_NOAUTH_BEGIN@$/,/^@DAOS_NOAUTH_END@/d' ; \ + sed_opts+=' --expression /(^@DAOS_AUTH_BEGIN@$)|(^@DAOS_AUTH_END@$)/d' ; \ + else \ + sed_opts+=' --expression /^@DAOS_AUTH_BEGIN@$/,/^@DAOS_AUTH_END@/d' ; \ + sed_opts+=' --expression /(^@DAOS_NOAUTH_BEGIN@$)|(^@DAOS_NOAUTH_END@$)/d' ; \ + fi ; \ + if [ "$DAOS_MD_ON_SSD" == yes ] ; then \ + sed_opts+=' --expression /(^@DAOS_MD_ON_SSD_BEGIN@$)|(^@DAOS_MD_ON_SSD_END@$)/d' ; \ + else \ + sed_opts+=' --expression /^@DAOS_MD_ON_SSD_BEGIN@$/,/^@DAOS_MD_ON_SSD_END@/d' ; \ + fi ; \ + sed $sed_opts /tmp/daos_server.yml.in > /etc/daos/daos_server.yml && \ + rm -f /tmp/daos_server.yml.in && \ if [ "$DAOS_AUTH" == yes ] ; then \ - sed --regexp-extended \ - --expression "s/@DAOS_HUGEPAGES_NBR@/${DAOS_HUGEPAGES_NBR}/" \ - --expression "s/@DAOS_IFACE_NAME@/${DAOS_IFACE_NAME}/" \ - --expression "s/@DAOS_SCM_SIZE@/${DAOS_SCM_SIZE}/" \ - --expression "s/@DAOS_BDEV_SIZE@/${DAOS_BDEV_SIZE}/" \ - --expression '/^@DAOS_NOAUTH_SECTION_BEGIN@$/,/^@DAOS_NOAUTH_SECTION_END@/d' \ - --expression '/(^@DAOS_AUTH_SECTION_BEGIN@$)|(^@DAOS_AUTH_SECTION_END@$)/d' \ - /tmp/daos_server.yml.in > /etc/daos/daos_server.yml && \ -# XXX WARNING XXX With a production platform, these certificates should be provided with a volume -# XXX WARNING XXX (or Secrets with K8S). chmod 644 /root/daosCA/certs/daosCA.crt && \ chmod 644 /root/daosCA/certs/server.crt && \ - chmod 600 /root/daosCA/certs/server.key && \ + chmod 400 /root/daosCA/certs/server.key && \ chmod 644 /root/daosCA/certs/agent.crt && \ chown daos_server:daos_server /root/daosCA/certs/daosCA.crt && \ chown daos_server:daos_server /root/daosCA/certs/server.crt && \ @@ -64,14 +90,4 @@ RUN sed --regexp-extended --in-place mv /root/daosCA/certs/server.key /etc/daos/certs/. && \ mv /root/daosCA/certs/agent.crt /etc/daos/certs/clients/. && \ rm -fr /root/daosCA ; \ - else \ - sed --regexp-extended \ - --expression "s/@DAOS_HUGEPAGES_NBR@/${DAOS_HUGEPAGES_NBR}/" \ - --expression "s/@DAOS_IFACE_NAME@/${DAOS_IFACE_NAME}/" \ - --expression "s/@DAOS_SCM_SIZE@/${DAOS_SCM_SIZE}/" \ - --expression "s/@DAOS_BDEV_SIZE@/${DAOS_BDEV_SIZE}/" \ - --expression '/^@DAOS_AUTH_SECTION_BEGIN@$/,/^@DAOS_AUTH_SECTION_END@/d' \ - --expression '/(^@DAOS_NOAUTH_SECTION_BEGIN@$)|(^@DAOS_NOAUTH_SECTION_END@$)/d'\ - /tmp/daos_server.yml.in > /etc/daos/daos_server.yml ; \ - fi && \ - rm -f /tmp/daos_server.yml.in + fi diff --git a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in index 982deac8670..0d9ea2086f9 100644 --- a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in +++ b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in @@ -9,31 +9,43 @@ socket_dir: /var/run/daos_server nr_hugepages: @DAOS_HUGEPAGES_NBR@ transport_config: -@DAOS_NOAUTH_SECTION_BEGIN@ +@DAOS_NOAUTH_BEGIN@ allow_insecure: true -@DAOS_NOAUTH_SECTION_END@ -@DAOS_AUTH_SECTION_BEGIN@ +@DAOS_NOAUTH_END@ +@DAOS_AUTH_BEGIN@ allow_insecure: false client_cert_dir: /etc/daos/certs/clients ca_cert: /etc/daos/certs/daosCA.crt cert: /etc/daos/certs/server.crt key: /etc/daos/certs/server.key -@DAOS_AUTH_SECTION_END@ +@DAOS_AUTH_END@ +helper_log_file: /tmp/daos_server_helper.log +control_log_file: /tmp/daos_server.log control_log_mask: INFO +@DAOS_MD_ON_SSD_BEGIN@ +control_metadata: + path: /var/db/daos_server +@DAOS_MD_ON_SSD_END@ engines: - targets: 1 - first_core: 0 + pinned_numa_node: 0 nr_xs_helpers: 0 fabric_iface: @DAOS_IFACE_NAME@ fabric_iface_port: 31416 log_file: /tmp/daos_engine_0.log - - scm_class: ram - scm_mount: /mnt/daos - scm_size: @DAOS_SCM_SIZE@ - - bdev_class: file - bdev_list: [/tmp/daos-bdev] - bdev_size: @DAOS_BDEV_SIZE@ + log_mask: INFO + storage: + - class: ram + scm_mount: /mnt/daos + scm_size: @DAOS_SCM_SIZE@ + - class: file + bdev_list: [/tmp/daos-bdev] + bdev_size: @DAOS_BDEV_SIZE@ +@DAOS_MD_ON_SSD_BEGIN@ + bdev_roles: + - meta + - wal + - data +@DAOS_MD_ON_SSD_END@ diff --git a/utils/docker/vcluster/docker-compose.yml b/utils/docker/vcluster/docker-compose.yml index a75fcec356f..34f463fcc58 100644 --- a/utils/docker/vcluster/docker-compose.yml +++ b/utils/docker/vcluster/docker-compose.yml @@ -1,4 +1,4 @@ -# Copyright 2021-2023 Intel Corporation +# Copyright 2021-2024 Intel Corporation # All rights reserved. # # Docker Compose file allowing to build and deploy locally a DAOS virtual cluster @@ -8,37 +8,51 @@ version: "3.8" services: daos_base: - image: "daos-base:${DAOS_SERVER_VERSION}" + image: "${DAOS_DOCKER_IMAGE_NSP}/daos-base-${LINUX_DISTRO}:${DAOS_DOCKER_IMAGE_TAG}" build: context: "daos-base/el8" args: - - "RHEL_BASE_IMAGE=${RHEL_BASE_IMAGE}" - - "RHEL_BASE_VERSION=${RHEL_BASE_VERSION}" - "BUST_CACHE=${BUST_CACHE}" - - "DAOS_AUTH=${DAOS_AUTH}" + - "LINUX_IMAGE_NAME=${LINUX_IMAGE_NAME}" + - "LINUX_IMAGE_TAG=${LINUX_IMAGE_TAG}" - "DAOS_REPOS=${DAOS_REPOS}" - "DAOS_GPG_KEYS=${DAOS_GPG_KEYS}" - "DAOS_REPOS_NOAUTH=${DAOS_REPOS_NOAUTH}" + - "DAOS_VERSION=${DAOS_VERSION}" + - "DAOS_AUTH=${DAOS_AUTH}" + privileged: true + cgroup: host + volumes: + - type: bind + read_only: true + source: /sys/fs/cgroup + target: /sys/fs/cgroup + - type: tmpfs + target: /run daos_server: - image: "daos-server:${DAOS_SERVER_VERSION}" + image: "${DAOS_DOCKER_IMAGE_NSP}/daos-server-${LINUX_DISTRO}:${DAOS_DOCKER_IMAGE_TAG}" build: context: "daos-server/el8" args: - - "DAOS_BASE_IMAGE=${DAOS_BASE_IMAGE}" - - "DAOS_BASE_VERSION=${DAOS_BASE_VERSION}" + - "LINUX_DISTRO=${LINUX_DISTRO}" + - "DAOS_DOCKER_IMAGE_NSP=${DAOS_DOCKER_IMAGE_NSP}" + - "DAOS_DOCKER_IMAGE_TAG=${DAOS_DOCKER_IMAGE_TAG}" + - "DAOS_VERSION=${DAOS_VERSION}" - "DAOS_AUTH=${DAOS_AUTH}" - "DAOS_HUGEPAGES_NBR=${DAOS_HUGEPAGES_NBR}" - "DAOS_IFACE_NAME=${DAOS_IFACE_NAME}" - "DAOS_SCM_SIZE=${DAOS_SCM_SIZE}" - "DAOS_BDEV_SIZE=${DAOS_BDEV_SIZE}" + - "DAOS_MD_ON_SSD=${DAOS_MD_ON_SSD}" container_name: daos-server hostname: daos-server - # FIXME Refine needed capabilities privileged: true - # FIXME virtual network such as bridge are not yet supported + # XXX Needed for some linux distro + cgroup: host + # XXX NOTE virtual network such as bridge are not yet supported network_mode: host - # FIXME List of hosts needed until virtual network will be supported + # XXX NOTE List of hosts needed until virtual network will be supported extra_hosts: - "daos-server:${DAOS_IFACE_IP}" - "daos-admin:${DAOS_IFACE_IP}" @@ -64,19 +78,20 @@ services: target: /run daos_admin: - image: "daos-admin:${DAOS_ADMIN_VERSION}" + image: "${DAOS_DOCKER_IMAGE_NSP}/daos-admin-${LINUX_DISTRO}:${DAOS_DOCKER_IMAGE_TAG}" build: context: "daos-admin/el8" args: - - "DAOS_BASE_IMAGE=${DAOS_BASE_IMAGE}" - - "DAOS_BASE_VERSION=${DAOS_BASE_VERSION}" + - "LINUX_DISTRO=${LINUX_DISTRO}" + - "DAOS_DOCKER_IMAGE_NSP=${DAOS_DOCKER_IMAGE_NSP}" + - "DAOS_DOCKER_IMAGE_TAG=${DAOS_DOCKER_IMAGE_TAG}" + - "DAOS_VERSION=${DAOS_VERSION}" - "DAOS_AUTH=${DAOS_AUTH}" - - "DAOS_ADMIN_USER=${DAOS_ADMIN_USER}" - - "DAOS_ADMIN_GROUP=${DAOS_ADMIN_GROUP}" container_name: daos-admin hostname: daos-admin - # FIXME Refine needed capabilities privileged: true + # XXX Needed for some linux distro + cgroup: host # FIXME virtual network such as bridge are not yet supported network_mode: host # FIXME List of hosts needed until virtual network will be supported @@ -95,23 +110,24 @@ services: - daos_server daos_client: - image: "daos-client:${DAOS_CLIENT_VERSION}" + image: "${DAOS_DOCKER_IMAGE_NSP}/daos-client-${LINUX_DISTRO}:${DAOS_DOCKER_IMAGE_TAG}" build: context: "daos-client/el8" args: - - "DAOS_BASE_IMAGE=${DAOS_BASE_IMAGE}" - - "DAOS_BASE_VERSION=${DAOS_BASE_VERSION}" + - "LINUX_DISTRO=${LINUX_DISTRO}" + - "DAOS_DOCKER_IMAGE_NSP=${DAOS_DOCKER_IMAGE_NSP}" + - "DAOS_DOCKER_IMAGE_TAG=${DAOS_DOCKER_IMAGE_TAG}" + - "DAOS_VERSION=${DAOS_VERSION}" - "DAOS_AUTH=${DAOS_AUTH}" - - "DAOS_ADMIN_USER=${DAOS_ADMIN_USER}" - - "DAOS_ADMIN_GROUP=${DAOS_ADMIN_GROUP}" - "DAOS_AGENT_IFACE_CFG=${DAOS_AGENT_IFACE_CFG}" - "DAOS_AGENT_IFACE_NUMA_NODE=${DAOS_AGENT_IFACE_NUMA_NODE}" - "DAOS_AGENT_IFACE_NAME=${DAOS_AGENT_IFACE_NAME}" - "DAOS_AGENT_IFACE_DOMAIN_NAME=${DAOS_AGENT_IFACE_DOMAIN_NAME}" container_name: daos-client hostname: daos-client - # FIXME Refine needed capabilities privileged: true + # XXX Needed for some linux distro + cgroup: host # FIXME virtual network such as bridge are not yet supported network_mode: host # FIXME List of hosts needed until virtual network will be supported