Add index-index mapping for arrays (#207)

* add index map to C API * add file-based index mapping test * add Mappings struct for arrays of name-index maps * update Fortran mapping sets * add index mapping test in fortran * finish index mapping fortran tests * update python dependencies to include yaml-cpp * make yaml-cpp private * update actions to build with current commit * add yaml-cpp to export set * remove yaml-cpp from tuv export set * fix index mapping returned errors * update python dockerfile and action * output error message in python test * debugging * fix temporary string bug * remove debug code and update docker files * install yaml-cpp for python wrapper * try updated TUV * addres fortran memory issues * try to fix pip dependency issue on ubunutu * try to fix pip build * try again with pip action * update actions * update mac gcc action versions * update ubuntu action * remove older fortran compiler tests bc of ice * update python install script * fix merge problem and update TUV-x commit * remove python wrapper tests from CMake; let the pip action test them instead * debugging * one last desparate attempt to get the test to pass * add const for config arg
NCAR · Aug 29, 2024 · 7ebb513 · 7ebb513
1 parent aa0854e
commit 7ebb513
Show file tree

Hide file tree

Showing 37 changed files with 1,135 additions and 298 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -27,6 +27,8 @@ jobs:
           - Dockerfile.mpi
           - Dockerfile.mpi_openmp
           - Dockerfile.python
+          - Dockerfile.pip
+        build_type: [Release, Debug]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -37,10 +39,10 @@ jobs:
         run: rm -rf /opt/hostedtoolcache
 
       - name: Build Docker image
-        run: docker build -t musica -f docker/${{ matrix.dockerfile }} .
+        run: docker build -t musica -f docker/${{ matrix.dockerfile }} . --build-arg MUSICA_GIT_TAG=${{ github.sha }} --build-arg BUILD_TYPE=${{ matrix.build_type }}
 
       - name: Run tests in container
-        if: matrix.dockerfile != 'Dockerfile.coverage'
+        if: matrix.dockerfile != 'Dockerfile.coverage' && matrix.dockerfile != 'Dockerfile.pip'
         run: docker run --name test-container -t musica bash -c 'make test ARGS="--rerun-failed --output-on-failure -j8"'
 
       - name: Run coverage tests in container

diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
@@ -20,6 +20,7 @@ jobs:
         compiler:
           - { cpp: g++-12, c: gcc-12, fc: gfortran-12}
           - { cpp: g++-13, c: gcc-13, fc: gfortran-13}
+          - { cpp: g++-14, c: gcc-14, fc: gfortran-14}
           - { cpp: clang++, c: clang, fc: gfortran-12}
           - { cpp: g++-14, c: gcc-14, fc: gfortran-14}
         build_type: [Release]
@@ -44,6 +45,7 @@ jobs:
             cmake -S . -B build \
                 -D CMAKE_CXX_FLAGS=-Wl,-ld_classic \
                 -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+                -D MUSICA_GIT_TAG=${{ github.sha }} \
                 -D MUSICA_ENABLE_PYTHON_LIBRARY=ON \
                 -D Python3_EXECUTABLE=$PYTHON_PATH 
 
@@ -57,6 +59,7 @@ jobs:
             cmake -S . -B build \
                 -D CMAKE_CXX_FLAGS=-Wl,-ld_classic \
                 -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+                -D MUSICA_GIT_TAG=${{ github.sha }} \
                 -D MUSICA_ENABLE_PYTHON_LIBRARY=ON \
                 -D Python3_EXECUTABLE=$PYTHON_PATH \
                 -D CMAKE_OSX_SYSROOT=""
@@ -73,7 +76,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        gcc_version: [12, 13, 14]
+        gcc_version: [13, 14]
         build_type: [Release]
     env:
       FC: gfortran-${{ matrix.gcc_version }}
@@ -85,7 +88,7 @@ jobs:
         run: brew install netcdf netcdf-fortran
 
       - name: Run Cmake 
-        run: cmake -S . -B build -D CMAKE_CXX_FLAGS=-Wl,-ld_classic -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D MUSICA_BUILD_FORTRAN_INTERFACE=ON
+        run: cmake -S . -B build -D CMAKE_CXX_FLAGS=-Wl,-ld_classic -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D MUSICA_GIT_TAG=${{ github.sha }} -D MUSICA_BUILD_FORTRAN_INTERFACE=ON
 
       - name: Build
         run: cmake --build build --verbose --parallel 10

diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml
@@ -34,4 +34,4 @@ jobs:
       run: pip install --verbose .[test]
 
     - name: Test
-      run: python -m unittest discover -s python/test
+      run: python -m unittest discover -v -s python/test
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -41,7 +41,7 @@ jobs:
           sudo apt-get install -y python3-numpy
 
       - name: Run Cmake 
-        run: cmake -S . -B build -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D MUSICA_ENABLE_PYTHON_LIBRARY=${{ matrix.options.python }} -D MUSICA_ENABLE_TUVX=${{ matrix.matrix.options.tuvx }}
+        run: cmake -S . -B build -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D MUSICA_GIT_TAG=${{ github.sha }} -D MUSICA_ENABLE_PYTHON_LIBRARY=${{ matrix.options.python }} -D MUSICA_ENABLE_TUVX=${{ matrix.matrix.options.tuvx }}
 
       - name: Build
         run: cmake --build build --verbose
@@ -52,9 +52,10 @@ jobs:
           ctest -C ${{ matrix.build_type }} --rerun-failed --output-on-failure . --verbose
   fortran:
     runs-on: ubuntu-24.04
+    continue-on-error: true
     strategy:
       matrix:
-        gcc_version: [12, 13, 14]
+        gcc_version: [14]
         build_type: [Debug, Release]
     env:
       CXX: g++-${{ matrix.gcc_version }}
@@ -70,7 +71,7 @@ jobs:
           sudo apt-get install -y libnetcdf-dev netcdf-bin libnetcdff-dev liblapack-dev libblas-dev
 
       - name: Run Cmake 
-        run: cmake -S . -B build -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D MUSICA_BUILD_FORTRAN_INTERFACE=ON
+        run: cmake -S . -B build -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D MUSICA_GIT_TAG=${{ github.sha }} -D MUSICA_BUILD_FORTRAN_INTERFACE=ON
 
       - name: Build
         run: cmake --build build --verbose --parallel 1

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -30,7 +30,7 @@ jobs:
       - run: pip install numpy
 
       - name: Run Cmake 
-        run: cmake -S . -B build -G "MinGW Makefiles" -D MUSICA_ENABLE_TUVX=OFF -D MUSICA_ENABLE_PYTHON_LIBRARY=ON
+        run: cmake -S . -B build -G "MinGW Makefiles" -D MUSICA_GIT_TAG=${{ github.sha }} -D MUSICA_ENABLE_TUVX=OFF -D MUSICA_ENABLE_PYTHON_LIBRARY=ON
 
       - name: Build
         run: cmake --build build --parallel
@@ -61,7 +61,7 @@ jobs:
     - run: pip install numpy
 
     - name: Run CMake
-      run: cmake -S . -B build -G "Visual Studio 17 2022" -A ${{ matrix.architecture }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -D MUSICA_ENABLE_TUVX=OFF -D MUSICA_ENABLE_PYTHON_LIBRARY=ON
+      run: cmake -S . -B build -G "Visual Studio 17 2022" -A ${{ matrix.architecture }} -D MUSICA_GIT_TAG=${{ github.sha }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -D MUSICA_ENABLE_TUVX=OFF -D MUSICA_ENABLE_PYTHON_LIBRARY=ON
 
     - name: Build
       run: cmake --build build --config ${{ matrix.build_type }} --parallel
@@ -86,7 +86,7 @@ jobs:
       - run: pip install numpy
 
       - name: Run CMake
-        run: cmake -S . -B build -DCMAKE_CXX_COMPILER="C:/Program Files/LLVM/bin/clang++.exe" -DCMAKE_C_COMPILER="C:/Program Files/LLVM/bin/clang.exe" -G"MinGW Makefiles" -D MUSICA_ENABLE_TUVX=OFF -D MUSICA_ENABLE_PYTHON_LIBRARY=ON
+        run: cmake -S . -B build -DCMAKE_CXX_COMPILER="C:/Program Files/LLVM/bin/clang++.exe" -DCMAKE_C_COMPILER="C:/Program Files/LLVM/bin/clang.exe" -G"MinGW Makefiles" -D MUSICA_GIT_TAG=${{ github.sha }} -D MUSICA_ENABLE_TUVX=OFF -D MUSICA_ENABLE_PYTHON_LIBRARY=ON
 
       - name: Build
         run: cmake --build build --parallel
@@ -115,7 +115,7 @@ jobs:
       - run: pip install numpy
 
       - name: Run CMake
-        run: cmake -S . -B build -G "Visual Studio 16 2019" -A ${{ matrix.architecture }} -T ClangCL -D MUSICA_ENABLE_TUVX=OFF -D MUSICA_ENABLE_PYTHON_LIBRARY=ON
+        run: cmake -S . -B build -G "Visual Studio 16 2019" -A ${{ matrix.architecture }} -T ClangCL -D MUSICA_GIT_TAG=${{ github.sha }} -D MUSICA_ENABLE_TUVX=OFF -D MUSICA_ENABLE_PYTHON_LIBRARY=ON
 
       - name: Build
         run: cmake --build build --config Debug --parallel

diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake
@@ -20,6 +20,17 @@ if (MUSICA_BUILD_FORTRAN_INTERFACE)
   pkg_check_modules(netcdfc IMPORTED_TARGET REQUIRED netcdf)
 endif()
 
+# ##############################################################################
+# yaml-cpp
+
+FetchContent_Declare(
+  yaml-cpp
+  GIT_REPOSITORY https://github.com/jbeder/yaml-cpp/
+  GIT_TAG 0.8.0
+  GIT_PROGRESS NOT
+  ${FETCHCONTENT_QUIET})
+FetchContent_MakeAvailable(yaml-cpp)
+
 ################################################################################
 # google test
 if(MUSICA_ENABLE_TESTS)
@@ -75,7 +86,7 @@ if (MUSICA_ENABLE_TUVX AND MUSICA_BUILD_C_CXX_INTERFACE)
   set(TUVX_INSTALL_INCLUDE_DIR ${MUSICA_INSTALL_INCLUDE_DIR} CACHE STRING "" FORCE)
 
   set_git_default(TUVX_GIT_REPOSITORY https://github.com/NCAR/tuv-x.git)
-  set_git_default(TUVX_GIT_TAG 674ee1e72853bb44d23c36602fa73c955b2f021d)
+  set_git_default(TUVX_GIT_TAG ccda76c0553064a9b7b0eba73162ddeee6d8eaff)
 
   FetchContent_Declare(tuvx
     GIT_REPOSITORY ${TUVX_GIT_REPOSITORY}

diff --git a/cmake/test_util.cmake b/cmake/test_util.cmake
@@ -41,7 +41,7 @@ function(create_standard_test_cxx)
   include(CMakeParseArguments)
   cmake_parse_arguments(${prefix} " " "${singleValues}" "${multiValues}" ${ARGN})
   add_executable(test_${TEST_NAME} ${TEST_SOURCES})
-  target_link_libraries(test_${TEST_NAME} PUBLIC musica::musica GTest::gtest_main)
+  target_link_libraries(test_${TEST_NAME} PUBLIC musica::musica GTest::gtest_main yaml-cpp)
   if(MUSICA_ENABLE_OPENMP)
     target_link_libraries(test_${TEST_NAME} PUBLIC OpenMP::OpenMP_CXX)
   endif()

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,5 +1,6 @@
-FROM fedora:35
+FROM fedora:latest
 
+ARG MUSICA_GIT_TAG=main
 ARG BUILD_TYPE=Release
 
 RUN dnf -y update \
@@ -25,6 +26,7 @@ RUN cd musica \
     && cmake -S . \
              -B build \
              -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+             -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
     && cd build \
     && make install -j 8 
 

diff --git a/docker/Dockerfile.docs b/docker/Dockerfile.docs
@@ -1,4 +1,7 @@
-FROM fedora:37
+FROM fedora:latest
+
+ARG MUSICA_GIT_TAG=main
+ARG BUILD_TYPE=Release
 
 RUN dnf -y update \
     && dnf -y install \
@@ -17,7 +20,8 @@ RUN dnf -y update \
 
 COPY . /musica/
 
-RUN pip3 install -r /musica/docs/requirements.txt
+RUN python3 -m pip install --upgrade pip setuptools \
+    && python3 -m pip install -r /musica/docs/requirements.txt
 
 ARG SUFFIX=""
 ENV SWITCHER_SUFFIX=$SUFFIX
@@ -28,6 +32,8 @@ RUN mkdir /build \
       && cd /build \
       && cmake -D MUSICA_BUILD_DOCS=ON \
                -D MUSICA_ENABLE_TUVX=OFF \
+                -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
+                -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
                /musica \
       && make docs
 

diff --git a/docker/Dockerfile.fortran-gcc b/docker/Dockerfile.fortran-gcc
@@ -1,5 +1,6 @@
-FROM fedora:35
+FROM fedora:latest
 
+ARG MUSICA_GIT_TAG=main
 ARG BUILD_TYPE=Release
 
 RUN dnf -y update \

diff --git a/docker/Dockerfile.fortran-gcc.integration b/docker/Dockerfile.fortran-gcc.integration
@@ -1,4 +1,4 @@
-FROM fedora:35
+FROM fedora:latest
 
 ARG MUSICA_GIT_TAG=main
 ARG BUILD_TYPE=Release
@@ -38,6 +38,7 @@ RUN cd musica \
     && cmake -S . \
              -B build \
              -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+              -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
     && cd build \
     && make install -j
 

diff --git a/docker/Dockerfile.fortran-intel b/docker/Dockerfile.fortran-intel
@@ -8,6 +8,7 @@ RUN wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \
   gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics-archive-keyring.gpg
 
 ARG MUSICA_GIT_TAG=main
+ARG BUILD_TYPE=Release
 
 RUN apt update \
     && apt -y install \
@@ -52,6 +53,7 @@ RUN cd musica \
     && cmake -S . \
              -B build \
              -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+             -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
     && cd build \
     && make install -j
 
@@ -63,7 +65,7 @@ ENV FC=ifx
 RUN cd musica/fortran/test/fetch_content_integration \
     && mkdir build && cd build \
     && cmake .. \
-        -D CMAKE_BUILD_TYPE=Release \
+        -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
         -D CMAKE_EXE_LINKER_FLAGS="-Wl,--copy-dt-needed-entries" \
         -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
         -D MUSICA_ENABLE_MEMCHECK=ON \

diff --git a/docker/Dockerfile.fortran-nvhpc b/docker/Dockerfile.fortran-nvhpc
@@ -49,23 +49,25 @@ COPY . musica
 
 # Build and install MUSICA
 RUN cd musica \
-    && cmake -S . \
-             -B build \
-             -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
-    && cd build \
-    && make install -j
+&& cmake -S . \
+-B build \
+-D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
+&& cd build \
+&& make install -j
 
 # Set environment variables to build MUSICA-Fortran using nvidia compilers
 ENV CXX=nvc++
 ENV CC=nvc
 ENV FC=nvfortran
 
+# (There is a runtime test failure when building MUSICA with nvhpc in Release mode, so we build in Debug mode)
 RUN cd musica/fortran/test/fetch_content_integration \
     && mkdir build && cd build \
     && cmake .. \
-        -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
-        -D CMAKE_EXE_LINKER_FLAGS="-Wl,--copy-dt-needed-entries" \
+        -D CMAKE_BUILD_TYPE=Debug \
         -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
+        -D CMAKE_EXE_LINKER_FLAGS="-Wl,--copy-dt-needed-entries" \
         -D MUSICA_ENABLE_MEMCHECK=ON \
     && make -j
 

diff --git a/docker/Dockerfile.memcheck b/docker/Dockerfile.memcheck
@@ -1,5 +1,6 @@
-FROM fedora:35
+FROM fedora:latest
 
+ARG MUSICA_GIT_TAG=main
 ARG BUILD_TYPE=Debug
 
 RUN dnf -y update \
@@ -24,6 +25,7 @@ RUN cd musica \
     && cmake -S . \
              -B build \
              -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+             -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
              -D MUSICA_ENABLE_MEMCHECK=ON \
     && cd build \
     && make install -j 8 

diff --git a/docker/Dockerfile.mpi b/docker/Dockerfile.mpi
@@ -1,5 +1,6 @@
-FROM fedora:35
+FROM fedora:latest
 
+ARG MUSICA_GIT_TAG=main
 ARG BUILD_TYPE=Debug
 
 RUN dnf -y update \
@@ -40,6 +41,7 @@ RUN cd musica \
     && cmake -S . \
              -B build \
              -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+             -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
              -D MUSICA_ENABLE_TESTS=ON \
              -D MUSICA_ENABLE_MPI=ON \
              -D CMAKE_Fortran_COMPILER=/usr/lib64/openmpi/bin/mpif90 \

diff --git a/docker/Dockerfile.mpi_openmp b/docker/Dockerfile.mpi_openmp
@@ -1,5 +1,6 @@
-FROM fedora:35
+FROM fedora:latest
 
+ARG MUSICA_GIT_TAG=main
 ARG BUILD_TYPE=Debug
 
 RUN dnf -y update \
@@ -40,6 +41,7 @@ RUN cd musica \
     && cmake -S . \
              -B build \
              -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+             -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
              -D MUSICA_ENABLE_MPI=ON \
              -D MUSICA_ENABLE_OPENMP=ON \
              -D MUSICA_ENABLE_TESTS=ON \

diff --git a/docker/Dockerfile.openmp b/docker/Dockerfile.openmp
@@ -1,5 +1,6 @@
-FROM fedora:35
+FROM fedora:latest
 
+ARG MUSICA_GIT_TAG=main
 ARG BUILD_TYPE=Debug
 
 RUN dnf -y update \
@@ -39,6 +40,7 @@ RUN cd musica \
     && cmake -S . \
              -B build \
              -D CMAKE_BUILD_TYPE=${BUILD_TYPE} \
+             -D MUSICA_GIT_TAG=${MUSICA_GIT_TAG} \
              -D MUSICA_ENABLE_OPENMP=ON \
              -D MUSICA_ENABLE_TESTS=ON \
              -D CMAKE_Fortran_COMPILER=/usr/lib64/openmpi/bin/mpif90 \