Merge develop-1.3 into develop #43

Merge develop-1.3 into develop

.github/workflows/build.yml

@@ -19,7 +19,7 @@ on:
 env:
   ECWAM_TOOLS: ${{ github.workspace }}/.github/tools
   CTEST_PARALLEL_LEVEL: 1
-  CACHE_SUFFIX: v0         # Increase to force new cache to be created
+  CACHE_SUFFIX: v1         # Increase to force new cache to be created
 
 jobs:
   ci:
@@ -31,12 +31,13 @@ jobs:
       matrix:
         #build_type: [Release,Debug] # Debug tests takes too long
         build_type: [Release]
+        prec: ['DP', 'SP']
         name:
           - linux gnu-10
-          - linux clang-12
+          - linux gnu-14
+          - linux nvhpc-23.5
           - linux intel-classic
           - macos
-          # - linux nvhpc-21.9
 
         include:
 
@@ -46,41 +47,35 @@ jobs:
             compiler_cc: gcc-10
             compiler_cxx: g++-10
             compiler_fc: gfortran-10
+            python-version: '3.8'
             caching: true
 
-          - name: linux clang-12
-            os: ubuntu-20.04
-            compiler: clang-12
-            compiler_cc: clang-12
-            compiler_cxx: clang++-12
-            compiler_fc: gfortran-10
+          - name: linux gnu-14
+            os: ubuntu-24.04
+            compiler: gnu-14
+            compiler_cc: gcc-14
+            compiler_cxx: g++-14
+            compiler_fc: gfortran-14
+            python-version: '3.11'
             caching: true
 
-          - name: linux clang-12
-            build_type: Release
+          - name: linux nvhpc-23.5
             os: ubuntu-20.04
-            compiler: clang-12
-            compiler_cc: clang-12
-            compiler_cxx: clang++-12
-            compiler_fc: gfortran-10
+            compiler: nvhpc-23.5
+            compiler_cc: nvc
+            compiler_cxx: nvc++
+            compiler_fc: nvfortran
+            cmake_options: -DCMAKE_CXX_FLAGS=--diag_suppress177
+            python-version: '3.8'
             caching: true
 
-# Disable due to problematic environment
-#          - name: linux nvhpc-21.9
-#            os: ubuntu-20.04
-#            compiler: nvhpc-21.9
-#            compiler_cc: nvc
-#            compiler_cxx: nvc++
-#            compiler_fc: nvfortran
-#            cmake_options: -DCMAKE_CXX_FLAGS=--diag_suppress177
-#            caching: true
-
           - name : linux intel-classic
             os: ubuntu-20.04
             compiler: intel-classic
             compiler_cc: icc
             compiler_cxx: icpc
             compiler_fc: ifort
+            python-version: '3.8'
             caching: true
 
           - name: macos
@@ -90,10 +85,16 @@ jobs:
             compiler_cc: ~
             compiler_cxx: ~
             compiler_fc: gfortran-13
+            python-version: '3.11'
             caching: true
 
     runs-on: ${{ matrix.os }}
     steps:
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
     - name: Checkout Repository
       uses: actions/checkout@v2
 
@@ -108,19 +109,15 @@ jobs:
           export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
           export HOMEBREW_NO_AUTO_UPDATE=1
           export HOMEBREW_NO_INSTALL_CLEANUP=1
+          export SDKROOT=$(xcrun --show-sdk-path)
           echo "HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1" >> $GITHUB_ENV
           echo "HOMEBREW_NO_AUTO_UPDATE=1"                >> $GITHUB_ENV
           echo "HOMEBREW_NO_INSTALL_CLEANUP=1"            >> $GITHUB_ENV
+          echo "SDKROOT=$(xcrun --show-sdk-path)"         >> $GITHUB_ENV
           brew install ninja
           brew install libomp
           brew install libaec
           brew install coreutils
-          brew install pyenv
-
-          pyenv install 3.11.4
-          pyenv global 3.11.4
-
-          echo "LOKI_PYTHON_ROOT_DIR=$HOME/.pyenv/versions/3.11.4/bin" >> $GITHUB_ENV
         else
           sudo apt-get update
           sudo apt-get install libaec-dev
@@ -141,11 +138,30 @@ jobs:
         path: ${{ env.DEPS_DIR }}
         key: deps-${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.build_type }}-${{ env.CACHE_SUFFIX }}
 
+    # Free up disk space for nvhpc
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      if: contains( matrix.compiler, 'nvhpc' )
+      continue-on-error: true
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
+
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: true
+        swap-storage: true
+
     - name: Install NVHPC compiler
       if: contains( matrix.compiler, 'nvhpc' )
       shell: bash -eux {0}
       run: |
-        ${ECWAM_TOOLS}/install-nvhpc.sh --prefix /opt/nvhpc
+        ${ECWAM_TOOLS}/install-nvhpc.sh --prefix /opt/nvhpc --version 23.5
         source /opt/nvhpc/env.sh
         echo "${NVHPC_DIR}/compilers/bin"                   >> $GITHUB_PATH
         [ -z ${MPI_HOME+x} ] || echo "MPI_HOME=${MPI_HOME}" >> $GITHUB_ENV
@@ -181,22 +197,22 @@ jobs:
         self_coverage: false
         force_build: true
         cache_suffix: "${{ matrix.build_type }}-${{ env.CACHE_SUFFIX }}"
-        recreate_cache: true
+        recreate_cache: ${{ matrix.caching == false }}
         dependencies: |
           ecmwf/ecbuild
           ecmwf/eccodes
           ecmwf/fckit@refs/tags/0.13.0
           ecmwf-ifs/fiat@refs/tags/1.4.1
           ecmwf-ifs/field_api@refs/tags/v0.3.1
-          ecmwf-ifs/loki@refs/tags/v0.2.4
+          ecmwf-ifs/loki@refs/tags/v0.2.9
         dependency_branch: develop
         dependency_cmake_options: |
           ecmwf/fckit: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF -DENABLE_FCKIT_VENV=ON"
-          ecmwf-ifs/fiat: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF"
-          ecmwf/eccodes: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_MEMFS=ON -DENABLE_JPG=OFF"
-          ecmwf-ifs/field_api: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF -DENABLE_ACC=OFF -DENABLE_SINGLE_PRECISION=OFF"
-          ecmwf-ifs/loki: "-G Ninja -DENABLE_TESTS=OFF -DENABLE_EDITABLE=ON -DPython3_ROOT_DIR=${{ env.LOKI_PYTHON_ROOT_DIR }}
-        cmake_options: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ${{ matrix.cmake_options }} -DENABLE_MPI=ON -DENABLE_LOKI=ON -DLOKI_MODE=idem-stack"
+          ecmwf-ifs/fiat: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF -DENABLE_SINGLE_PRECISION=${{ matrix.prec == 'SP' }} -DENABLE_DOUBLE_PRECISION=${{ matrix.prec == 'DP' }}"
+          ecmwf/eccodes: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_MEMFS=ON -DENABLE_JPG=OFF -DENABLE_PNG=OFF"
+          ecmwf-ifs/field_api: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF -DENABLE_ACC=OFF -DENABLE_SINGLE_PRECISION=${{ matrix.prec == 'SP' }} -DENABLE_DOUBLE_PRECISION=${{ matrix.prec == 'DP' }}"
+          ecmwf-ifs/loki: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DENABLE_TESTS=OFF"
+        cmake_options: "-G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ${{ matrix.cmake_options }} -DENABLE_MPI=ON -DENABLE_LOKI=ON -DLOKI_MODE=idem-stack -DENABLE_SINGLE_PRECISION=${{ matrix.prec == 'SP' }}"
         ctest_options: "${{ matrix.ctest_options }}"
 
     - name: Verify tools

CMakeLists.txt

@@ -44,7 +44,7 @@ ecbuild_add_option( FEATURE MPI
 
 ecbuild_add_option( FEATURE MULTIO
                     DESCRIPTION "Support for IO to the FDB"
-                    REQUIRED_PACKAGES "multio" )
+                    REQUIRED_PACKAGES "multio VERSION 2.1" )
 
 ecbuild_add_option( FEATURE SINGLE_PRECISION
                     DEFAULT OFF
@@ -64,7 +64,7 @@ if( NOT DEFINED OCEAN_PREC )
 endif()
 string( TOLOWER "${OCEAN_PREC}" ocean_prec )
 
-ecbuild_add_option( FEATURE OCEAN_COUPLING 
+ecbuild_add_option( FEATURE OCEAN_COUPLING
                     DEFAULT ON
                     DESCRIPTION "Support ocean coupling"
                     REQUIRED_PACKAGES "nemo_${ocean_prec}" )
@@ -110,10 +110,11 @@ ecbuild_add_option( FEATURE LOKI
                     REQUIRED_PACKAGES "loki" )
 
 # Set default Loki transformation mode
-set( LOKI_MODE "scc-hoist" CACHE STRING "Transformation mode for Loki source transformations" )
+set( LOKI_MODE "scc-stack" CACHE STRING "Transformation mode for Loki source transformations" )
 
 ### OpenACC
-if( ${CMAKE_VERSION} VERSION_LESS "3.25" AND (NOT DEFINED ENABLE_ACC OR ENABLE_ACC ) )
+if( ${CMAKE_VERSION} VERSION_LESS "3.25" AND HAVE_LOKI AND NOT LOKI_MODE MATCHES "idem|idem-stack" )
+  if ( ${PNAME}_ENABLE_ACC OR (NOT DEFINED ${PNAME}_ENABLE_ACC AND ENABLE_ACC) )
     # Incredibly inconvenient: FindOpenACC does _not_ set OpenACC_FOUND, only
     # the language-specific components OpenACC_Fortran_FOUND and OpenACC_C_FOUND.
     # This means, even internally CMake considers OpenACC as not found.
@@ -124,19 +125,30 @@ if( ${CMAKE_VERSION} VERSION_LESS "3.25" AND (NOT DEFINED ENABLE_ACC OR ENABLE_A
     # and rectifies CMake's internal bookkeeping in the process.
     # This has been fixed in CMake 3.25
     find_package( OpenACC )
-    if( OpenACC_Fortran_FOUND AND OpenACC_C_FOUND )
+    if( OpenACC_Fortran_FOUND )
         set( OpenACC_FOUND ON )
     endif()
+  endif()
 endif()
 ecbuild_add_option( FEATURE ACC
                     DESCRIPTION "OpenACC" DEFAULT OFF
-                    REQUIRED_PACKAGES "OpenACC" 
+                    REQUIRED_PACKAGES "OpenACC COMPONENTS Fortran"
                     CONDITION HAVE_LOKI AND NOT LOKI_MODE MATCHES "idem|idem-stack" )
 
-### CUDA-aware MPI
+### CUDA
+include(CheckLanguage)
+check_language(CUDA)
+ecbuild_add_option( FEATURE CUDA
+    DESCRIPTION "CUDA" DEFAULT OFF
+    CONDITION CMAKE_CUDA_COMPILER AND HAVE_ACC )
+if( HAVE_CUDA )
+    enable_language( CUDA )
+endif()
+
+### GPU-aware MPI
 ecbuild_add_option( FEATURE GPU_AWARE_MPI
                     DEFAULT OFF
-                    DESCRIPTION "Enable GPU-aware MPI" 
+                    DESCRIPTION "Enable GPU-aware MPI"
                     CONDITION HAVE_ACC AND MPI_Fortran_HAVE_F08_MODULE )
 
 ### Sources
@@ -161,4 +173,3 @@ add_subdirectory(doc)
 ecbuild_install_project( NAME ${PROJECT_NAME} )
 
 ecbuild_print_summary()
-

README.md

@@ -234,16 +234,15 @@ Building
 The recommended option for building the GPU enabled ecWAM is to use the provided bundle, and pass the
 `--with-loki --with-acc` options. Different Loki transformations can also be chosen at build-time via the following 
 bundle option: `--loki-mode=<trafo>`. Direct GPU-to-GPU MPI communications can be enabled by passing the 
-`--with-gpu-aware-mpi` option.
+`--with-gpu-aware-mpi` option. CPU to GPU data transfers can be accelerated (via pinning of host-side allocations)
+by building with the `--with-cuda` option.
 
 The ecwam-bundle also provides appropriate arch files for the nvhpc suite on the ECMWF ATOS system.
 
 Running
 -------
 No extra run-time options are needed to run the GPU enabled ecWam. Please note that this means that if ecWam is built
 using the `--with-loki` and `--with-acc` bundle arguments, it will necessarily be offloaded for GPU execution.
-For multi-GPU runs, the number of GPUs maps to the number of MPI ranks. Thus multiple GPUs can be requested by
-launching with multiple MPI ranks. The mapping of MPI ranks to GPUs assumes at most 4 GPUs per host node.
 
 Environment variables
 ---------------------

cmake/ecwam_expand_drv_types.cmake

@@ -0,0 +1,61 @@
+# (C) Copyright 2024- ECMWF.
+# 
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+# Read src/ecwam/yowdrvtype_config.yml and expand the derived-types accordingly.
+# We create one sourcefile per derived-type because of *very* slow compilation times
+# with nvfortran if they are all placed in the same module. Unfortunately the problem
+# still persists even if compiler optimisations are disabled.
+
+macro( ecwam_expand_drv_types )
+
+   if( ${OCEAN_PREC} STREQUAL SP )
+      list(APPEND FYPP_ARGS -DPARKIND1_SINGLE_NEMO)
+   endif()
+
+   if( HAVE_LOKI AND NOT LOKI_MODE MATCHES "idem|idem-stack" )
+      list(APPEND FYPP_ARGS -DWAM_GPU)
+   endif()
+
+   execute_process(
+       COMMAND ${ECWAM_PYTHON_INTERP} -c
+       "import sys; sys.path.append('${CMAKE_CURRENT_SOURCE_DIR}/../../share/ecwam/scripts'); \
+       from ecwam_yaml_reader import yaml; f = open('${CMAKE_CURRENT_SOURCE_DIR}/yowdrvtype_config.yml'); \
+       yml = f.read(); f.close(); objtypes = yaml.safe_load(yml)['objtypes']; print(list(objtypes))"
+       RESULT_VARIABLE EXIT_CODE
+       OUTPUT_VARIABLE TYPE_NAMES
+       ERROR_QUIET
+   )
+
+   if( NOT EXIT_CODE EQUAL 0 )
+      ecbuild_critical("${ECWAM_PROJECT_NAME} FAILED TO READ yowdrvtype_config.yml")
+   endif()
+
+   string(REGEX MATCHALL "\'[A-Za-z0-9_]+\'" type_names "${TYPE_NAMES}")
+
+   foreach(type IN LISTS type_names)
+      string(REPLACE "'" "" _type ${type})
+      add_custom_command(
+          OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${_type}_type_mod.F90
+          COMMAND ${FYPP} ${FYPP_ARGS} -m io -m os -DTYPE_NAME='${_type}'
+                  -M ${CMAKE_CURRENT_SOURCE_DIR}/../../share/ecwam/scripts -m ecwam_yaml_reader
+                  ${CMAKE_CURRENT_SOURCE_DIR}/drvtype_mod.fypp > ${_type}_type_mod.F90
+          DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/drvtype_mod.fypp
+          VERBATIM)
+      list( APPEND ecwam_srcs ${CMAKE_CURRENT_BINARY_DIR}/${_type}_type_mod.F90)
+   endforeach()
+
+   add_custom_command(
+       OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/yowdrvtype.F90
+       COMMAND ${FYPP} -m io -m os -M ${CMAKE_CURRENT_SOURCE_DIR}/../../share/ecwam/scripts -m ecwam_yaml_reader
+               ${CMAKE_CURRENT_SOURCE_DIR}/yowdrvtype.fypp > yowdrvtype.F90
+       DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/yowdrvtype.fypp
+       VERBATIM)
+   list( APPEND ecwam_srcs ${CMAKE_CURRENT_BINARY_DIR}/yowdrvtype.F90)
+
+endmacro()

cmake/ecwam_macros.cmake

@@ -42,3 +42,4 @@ include( ecwam_target_compile_definitions_FILENAME )
 include( ecwam_add_test )
 include( ecwam_cache_bathymetry )
 include( ecwam_find_python_mods )
+include( ecwam_expand_drv_types )

package/bundle/bundle.yml

@@ -39,7 +39,7 @@ projects :
 
     - loki :
         git     : https://github.com/ecmwf-ifs/loki
-        version : v0.2.4
+        version : v0.2.9
         optional: true
         require : ecbuild
         cmake   : >
@@ -75,6 +75,11 @@ options :
         cmake : >
             ENABLE_ACC=ON
 
+    - with-cuda :
+        help  : Enable FIELD_API CUDA backend
+        cmake : >
+            ENABLE_CUDA=ON
+
     - without-loki-install :
         help  : Skip installation of Loki (Requires Loki to be on the PATH)
         cmake : >

requirements.txt

@@ -0,0 +1 @@
+.