From fb32c024a98f55ea66665fb8ecf3def269119ebf Mon Sep 17 00:00:00 2001 From: FabioLuporini Date: Mon, 21 Nov 2022 09:55:27 +0000 Subject: [PATCH 1/4] misc: Add Omnitrace to Dockerfile.amd --- docker/Dockerfile.amd | 10 ++++++++++ docker/entrypoint.sh | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.amd b/docker/Dockerfile.amd index 763d164de0..6022b94791 100644 --- a/docker/Dockerfile.amd +++ b/docker/Dockerfile.amd @@ -45,6 +45,16 @@ ENV PATH=${PATH}:/opt/openmpi/bin:$AOMP/bin ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/openmpi/lib:$AOMP/lib ENV OMPI_CC=$AOMP/bin/clang +# Install Omnitrace, an AMDResearch profiling tool which is de facto the best profiler from AMD +RUN wget https://github.com/AMDResearch/omnitrace/releases/download/v1.7.3/omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh -q +RUN mv omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh omnitrace-installer.sh && \ + chmod +x omnitrace-installer.sh && \ + mkdir -p /opt/omnitrace && \ + mv omnitrace-installer.sh /opt/omnitrace && \ + cd /opt/omnitrace && \ + ./omnitrace-installer.sh --skip-license && \ + cd - + # Devito env ENV DEVITO_ARCH="aomp" ENV DEVITO_PLATFORM="amdgpuX" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ac90d34180..b269107120 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -13,4 +13,9 @@ if [[ -z "${DEPLOY_ENV}" ]]; then ./codecov -t -t ${CODECOV_TOKEN} -F "${DEVITO_ARCH}-${DEVITO-PLATFORM}" else exec "$@" -fi \ No newline at end of file +fi + +if [[ -n "${ROCM_VERSION}" ]]; then + echo "sourcing Omnitrace" + source /opt/omnitrace/share/omnitrace/setup-env.sh +fi From 4f015eba23e30b5598bd760b1b6205dde44d3ba7 Mon Sep 17 00:00:00 2001 From: FabioLuporini Date: Tue, 22 Nov 2022 12:02:23 +0000 Subject: [PATCH 2/4] misc: Add Omniperf to Dockerfile.amd --- docker/Dockerfile.amd | 27 +++++++++++++++++++-------- docker/entrypoint.sh | 5 ++++- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile.amd b/docker/Dockerfile.amd index 6022b94791..b7a277161e 100644 --- a/docker/Dockerfile.amd +++ b/docker/Dockerfile.amd @@ -45,15 +45,26 @@ ENV PATH=${PATH}:/opt/openmpi/bin:$AOMP/bin ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/openmpi/lib:$AOMP/lib ENV OMPI_CC=$AOMP/bin/clang -# Install Omnitrace, an AMDResearch profiling tool which is de facto the best profiler from AMD +# Install Omnitrace, an AMDResearch profiling tool akin to Nvidia's Nsight-systems RUN wget https://github.com/AMDResearch/omnitrace/releases/download/v1.7.3/omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh -q -RUN mv omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh omnitrace-installer.sh && \ - chmod +x omnitrace-installer.sh && \ - mkdir -p /opt/omnitrace && \ - mv omnitrace-installer.sh /opt/omnitrace && \ - cd /opt/omnitrace && \ - ./omnitrace-installer.sh --skip-license && \ - cd - +RUN ./omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh --prefix=/opt/omnitrace --exclude-subdir --skip-license && \ + rm omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh + +# Install Omniperf, an AMDResearch profiling tool akin to Nvidia's Nsight-compute +RUN apt-get update && apt-get install -y software-properties-common && \ # cmake3.19 required by Omniperf... + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ + apt-get update && \ + apt install -y -q cmake +RUN apt-get update && apt-get install lmod # to eventually bring it in PATH via the entrypoint +ENV OMNIPERF_DIR=/app/omniperf +RUN wget https://github.com/AMDResearch/omniperf/releases/download/v1.0.4/omniperf-1.0.4.tar.gz -q +RUN tar xfz omniperf-1.0.4.tar.gz && \ + cd omniperf-1.0.4/ && \ + /venv/bin/pip install --no-cache-dir -r requirements.txt && \ + mkdir build && cd build/ && \ + cmake -DCMAKE_INSTALL_PREFIX=${OMNIPERF_DIR}/1.0.4 -DPYTHON_DEPS=${OMNIPERF_DIR}/python-libs -DMOD_INSTALL_PATH=${OMNIPERF_DIR}/modulefiles .. && \ + make install # Devito env ENV DEVITO_ARCH="aomp" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b269107120..5587f0dc26 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -16,6 +16,9 @@ else fi if [[ -n "${ROCM_VERSION}" ]]; then - echo "sourcing Omnitrace" + echo "configuring Omnitrace..." source /opt/omnitrace/share/omnitrace/setup-env.sh + echo "configuring Omniperf..." + module use /opt/omniperf/modulefiles + module load omniperf fi From d1a6194c728c442f91ea92f6e100ed712d4c5b86 Mon Sep 17 00:00:00 2001 From: FabioLuporini Date: Wed, 23 Nov 2022 07:42:21 +0000 Subject: [PATCH 3/4] misc: Refactor Dockerfile.amd --- docker/Dockerfile.amd | 20 ++++++++++++-------- docker/entrypoint.sh | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile.amd b/docker/Dockerfile.amd index b7a277161e..3769f9990e 100644 --- a/docker/Dockerfile.amd +++ b/docker/Dockerfile.amd @@ -11,8 +11,17 @@ ARG ROCM_VERSION ENV rocm=/opt/rocm-${ROCM_VERSION} ENV AOMP=/opt/rocm-${ROCM_VERSION}/llvm +# Bring in cmake3.19 apt repo +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ + # Some utils needed -RUN apt-get update && apt-get install -y wget git autoconf dh-autoreconf flex python3-venv python3-dev vim libnuma1 tmux +# * lmod required to source Omniperf via `module use` +# * software-properties-common required to compile cmake3.19, which in turn is required to compile Omniperf +RUN apt-get update && \ + apt-get install -y wget git autoconf dh-autoreconf \ + flex python3-venv python3-dev vim lmod \ + software-properties-common cmake libnuma1 tmux # Install tmpi RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi @@ -51,12 +60,6 @@ RUN ./omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh --prefix=/opt rm omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh # Install Omniperf, an AMDResearch profiling tool akin to Nvidia's Nsight-compute -RUN apt-get update && apt-get install -y software-properties-common && \ # cmake3.19 required by Omniperf... - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ - apt-get update && \ - apt install -y -q cmake -RUN apt-get update && apt-get install lmod # to eventually bring it in PATH via the entrypoint ENV OMNIPERF_DIR=/app/omniperf RUN wget https://github.com/AMDResearch/omniperf/releases/download/v1.0.4/omniperf-1.0.4.tar.gz -q RUN tar xfz omniperf-1.0.4.tar.gz && \ @@ -64,7 +67,8 @@ RUN tar xfz omniperf-1.0.4.tar.gz && \ /venv/bin/pip install --no-cache-dir -r requirements.txt && \ mkdir build && cd build/ && \ cmake -DCMAKE_INSTALL_PREFIX=${OMNIPERF_DIR}/1.0.4 -DPYTHON_DEPS=${OMNIPERF_DIR}/python-libs -DMOD_INSTALL_PATH=${OMNIPERF_DIR}/modulefiles .. && \ - make install + make install && \ + rm -rf omniperf-1.0.4.tar.gz omniperf-1.0.4/ # Devito env ENV DEVITO_ARCH="aomp" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 5587f0dc26..e50f5d0a55 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -17,7 +17,7 @@ fi if [[ -n "${ROCM_VERSION}" ]]; then echo "configuring Omnitrace..." - source /opt/omnitrace/share/omnitrace/setup-env.sh + module use /opt/omnitrace/share/modulefiles echo "configuring Omniperf..." module use /opt/omniperf/modulefiles module load omniperf From d4e9dc36ff92299644aada824f0ec3786d2f9fef Mon Sep 17 00:00:00 2001 From: FabioLuporini Date: Tue, 29 Nov 2022 08:49:22 +0000 Subject: [PATCH 4/4] misc: Patch Omniperf install in Dockerfile.amd --- docker/Dockerfile.amd | 64 +++++++++++++++++++++++++++++++++---------- docker/entrypoint.sh | 15 +++++----- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/docker/Dockerfile.amd b/docker/Dockerfile.amd index 3769f9990e..5f07291d2e 100644 --- a/docker/Dockerfile.amd +++ b/docker/Dockerfile.amd @@ -1,9 +1,13 @@ -############################################################## -# This Dockerfile contains AMD compilers -############################################################## +#################################################################################### +# This Dockerfile contains the AMD ROCm SDK as well as other useful tools for Devito +#################################################################################### +ARG arch="aomp" ARG ROCM_VERSION=5.1.3 -FROM rocm/dev-ubuntu-20.04:${ROCM_VERSION}-complete +######################################################################## +# Build base image with apt setup and common env +######################################################################## +FROM rocm/dev-ubuntu-20.04:${ROCM_VERSION}-complete as sdk-base ENV DEBIAN_FRONTEND noninteractive @@ -12,16 +16,21 @@ ENV rocm=/opt/rocm-${ROCM_VERSION} ENV AOMP=/opt/rocm-${ROCM_VERSION}/llvm # Bring in cmake3.19 apt repo -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ +RUN apt-get update && \ + apt-get install -y wget software-properties-common && \ + wget https://apt.kitware.com/keys/kitware-archive-latest.asc && \ + apt-key add kitware-archive-latest.asc && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' # Some utils needed # * lmod required to source Omniperf via `module use` -# * software-properties-common required to compile cmake3.19, which in turn is required to compile Omniperf RUN apt-get update && \ - apt-get install -y wget git autoconf dh-autoreconf \ - flex python3-venv python3-dev vim lmod \ - software-properties-common cmake libnuma1 tmux + apt-get install -y git autoconf dh-autoreconf flex \ + python3-venv python3-dev vim lmod cmake libnuma1 tmux + +# MongoDB utils is ultimately used by Omniperf +RUN wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb && \ + apt install ./mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb # Install tmpi RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi @@ -55,26 +64,51 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/openmpi/lib:$AOMP/lib ENV OMPI_CC=$AOMP/bin/clang # Install Omnitrace, an AMDResearch profiling tool akin to Nvidia's Nsight-systems -RUN wget https://github.com/AMDResearch/omnitrace/releases/download/v1.7.3/omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh -q -RUN ./omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh --prefix=/opt/omnitrace --exclude-subdir --skip-license && \ +ENV OMNITRACE_DIR=/opt/omnitrace +RUN wget https://github.com/AMDResearch/omnitrace/releases/download/v1.7.3/omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh -q && \ + chmod +x omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh && \ + mkdir -p ${OMNITRACE_DIR} && \ + ./omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh --prefix=${OMNITRACE_DIR} --exclude-subdir --skip-license && \ rm omnitrace-1.7.3-ubuntu-20.04-ROCm-50100-PAPI-OMPT-Python3.sh # Install Omniperf, an AMDResearch profiling tool akin to Nvidia's Nsight-compute -ENV OMNIPERF_DIR=/app/omniperf +ENV OMNIPERF_DIR=/opt/omniperf RUN wget https://github.com/AMDResearch/omniperf/releases/download/v1.0.4/omniperf-1.0.4.tar.gz -q RUN tar xfz omniperf-1.0.4.tar.gz && \ cd omniperf-1.0.4/ && \ - /venv/bin/pip install --no-cache-dir -r requirements.txt && \ + mkdir -p ${OMNIPERF_DIR} && \ + python3 -m pip install -t ${OMNIPERF_DIR}/python-libs -r requirements.txt && \ mkdir build && cd build/ && \ cmake -DCMAKE_INSTALL_PREFIX=${OMNIPERF_DIR}/1.0.4 -DPYTHON_DEPS=${OMNIPERF_DIR}/python-libs -DMOD_INSTALL_PATH=${OMNIPERF_DIR}/modulefiles .. && \ make install && \ rm -rf omniperf-1.0.4.tar.gz omniperf-1.0.4/ +ENV PYTHONPATH ${OMNIPERF_DIR}/python-libs:${PYTHONPATH} + +# This will only trigger if arch is aomp since the final stage depends on it +######################################################################## +# AOMP for GPUs via OpenMP config +######################################################################## +FROM sdk-base as aomp -# Devito env ENV DEVITO_ARCH="aomp" ENV DEVITO_PLATFORM="amdgpuX" ENV DEVITO_LANGUAGE="openmp" +# This will only trigger if arch is hip since the final stage depends on it +######################################################################## +# HIPCC for GPUs via HIP config +######################################################################## +FROM sdk-base as hip + +ENV DEVITO_ARCH="hip" +ENV DEVITO_PLATFORM="amdgpuX" +ENV DEVITO_LANGUAGE="hip" + +######################################################################## +# Final image +######################################################################## +FROM ${arch} as final + RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e50f5d0a55..eb60aa443d 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -8,17 +8,18 @@ if [[ "$MPIVER" = "HPCX" ]]; then hpcx_load fi -if [[ -z "${DEPLOY_ENV}" ]]; then - exec "$@" - ./codecov -t -t ${CODECOV_TOKEN} -F "${DEVITO_ARCH}-${DEVITO-PLATFORM}" -else - exec "$@" -fi - if [[ -n "${ROCM_VERSION}" ]]; then + source /usr/share/lmod/lmod/init/profile echo "configuring Omnitrace..." module use /opt/omnitrace/share/modulefiles echo "configuring Omniperf..." module use /opt/omniperf/modulefiles module load omniperf fi + +if [[ -z "${DEPLOY_ENV}" ]]; then + exec "$@" + ./codecov -t -t ${CODECOV_TOKEN} -F "${DEVITO_ARCH}-${DEVITO-PLATFORM}" +else + exec "$@" +fi