Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Various fixes for single-precision and switch to new FIELD_API #4

Merged
merged 11 commits into from
Nov 27, 2023
Merged
14 changes: 7 additions & 7 deletions .github/tools/install-intel-oneapi.sh
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
#!/bin/sh
#!/usr/bin/env bash

KEY=GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
version=2023.2.0
KEY=GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
wget https://apt.repos.intel.com/intel-gpg-keys/$KEY
sudo apt-key add $KEY
rm $KEY
echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
sudo apt-get update
sudo apt-get install \
intel-oneapi-compiler-fortran \
intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic \
intel-oneapi-mpi \
intel-oneapi-mpi-devel \
intel-oneapi-mkl
intel-oneapi-compiler-fortran-$version \
intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-$version \
intel-oneapi-mpi-devel-2021.10.0 \
intel-oneapi-mkl-$version
8 changes: 4 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
name:
- linux gnu-10
- linux clang-12
- linux intel
- linux intel-classic
- macos
# - linux nvhpc-21.9

Expand Down Expand Up @@ -75,9 +75,9 @@ jobs:
# cmake_options: -DCMAKE_CXX_FLAGS=--diag_suppress177
# caching: true

- name : linux intel
- name : linux intel-classic
os: ubuntu-20.04
compiler: intel-oneapi
compiler: intel-classic
compiler_cc: icc
compiler_cxx: icpc
compiler_fc: ifort
Expand Down Expand Up @@ -153,7 +153,7 @@ jobs:
${ECWAM_TOOLS}/install-intel-oneapi.sh
source /opt/intel/oneapi/setvars.sh
printenv >> $GITHUB_ENV
echo "CACHE_SUFFIX=$(icc -dumpversion)" >> $GITHUB_ENV
echo "CACHE_SUFFIX=$CC-$($CC -dumpversion)" >> $GITHUB_ENV

- name: Install MPI
shell: bash -eux {0}
Expand Down
11 changes: 5 additions & 6 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.

cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
cmake_minimum_required( VERSION 3.24 FATAL_ERROR )
find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild )

if( NOT ECWAM_PROJECT_NAME )
Expand Down Expand Up @@ -54,11 +54,6 @@ ecbuild_add_option( FEATURE ECWAM_LAUNCH
DEFAULT ON
DESCRIPTION "Use share/ecwam/scripts/ecwam-launch script to run ecwam tests" )

ecbuild_add_option( FEATURE FIELD_API
DEFAULT ON
DESCRIPTION "Build using field_api"
REQUIRED_PACKAGES "field_api" )

if( HAVE_ECWAM_LAUNCH )
# Overwrite meaning of mpiexec
set( ECWAM_LAUNCH ${CMAKE_CURRENT_SOURCE_DIR}/share/ecwam/scripts/ecwam-launch )
Expand All @@ -68,6 +63,10 @@ if( HAVE_ECWAM_LAUNCH )
set( CMAKE_CROSSCOMPILING_EMULATOR ${ECWAM_LAUNCH} ) # For serial runs
endif()

### field_api

include(cmake/ecwam_fetchcontent_field_api.cmake)
ecbuild_find_package( NAME field_api REQUIRED )

### Optional dependencies

Expand Down
7 changes: 3 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@ Requirements
- ecbuild (see https://github.com/ecmwf/ecbuild)
- fiat (see https://github.com/ecmwf-ifs/fiat)
- eccodes (see https://github.com/ecmwf/eccodes)
- field_api (see https://github.com/ecmwf-ifs/field_api)

Further optional dependencies:
- MPI Fortran libraries
- multio (see https://github.com/ecmwf/multio)
- ocean model (e.g. NEMO or FESOM)
- fypp (see https://github.com/aradi/fypp)
- field_api (see https://git.ecmwf.int/projects/RDX/repos/field_api/browse)

Some driver scripts to run tests and validate results rely on availability of:
- md5sum (part of GNU Coreutils; on MacOS, install with `brew install coreutils`)
Expand All @@ -65,7 +65,7 @@ Environment variables
$ export MPI_HOME=<path-to-MPI>
$ export fiat_ROOT=<path-to-fiat>
$ export eccodes_ROOT=<path-to-eccodes>
$ export field_api_ROOT=<path-to-field_api> (optional)
$ export field_api_ROOT=<path-to-field_api> (if not set, field_api will be cloned)
$ export CC=<path-to-C-compiler>
$ export FC=<path-to-Fortran-compiler>
$ export CXX=<path-to-C++-compiler>
Expand All @@ -88,7 +88,6 @@ Extra options can be added to the `cmake` command to control the build:
- `-DENABLE_TESTS=<ON|OFF>`
- `-DENABLE_MPI=<ON|OFF>`
- `-DENABLE_OMP=<ON|OFF>`
- `-DENABLE_FIELD_API=<ON|OFF>`
- `-DCMAKE_INSTALL_PREFIX=<install-prefix>`

More options to control compilation flags, only when defaults are not sufficient
Expand Down Expand Up @@ -123,7 +122,7 @@ The bundle also facilitates setting environment variables and compiler flags rel
The following options can also be configured during the bundle build step:
- `--without-mpi` - Disable MPI
- `--without-omp` - Disable OpenMP
- `--with-field_api` - Build using FIELD_API repo in `source/field_api`
- `--single-precision` - Build single-precision variant of ecWAM

Finally, additional `CMake` options can also be set during the bundle build step:

Expand Down
5 changes: 3 additions & 2 deletions cmake/ecwam_compile_flags.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")

elseif(CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC")
set(autopromote_flags "-r8")
set(checkbounds_flags "-Mbounds")
set(fpe_flags "-Ktrap=fp")
# set(checkbounds_flags "-Mbounds") # Added by default by CMake in NVHPC debug builds

elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Flang")
set(autopromote_flags "-fdefault-real-8")
Expand All @@ -39,7 +39,8 @@ if( NOT HAVE_SINGLE_PRECISION )
ecbuild_add_fortran_flags( "${autopromote_flags}" NAME autopromote )
endif()

if( CMAKE_BUILD_TYPE MATCHES "Debug" )
## Debug flags for NVHPC are applied selectively to sourcefiles in src/ecwam/CMakeLists.txt
if( CMAKE_BUILD_TYPE MATCHES "Debug" AND NOT CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
foreach( debug_flag fpe initsnan checkbounds )
if( ${debug_flag}_flags )
ecbuild_add_fortran_flags( "${${debug_flag}_flags}" NAME ${debug_flag} )
Expand Down
35 changes: 35 additions & 0 deletions cmake/ecwam_fetchcontent_field_api.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# (C) Copyright 2020- ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.


ecbuild_find_package(NAME field_api QUIET)
set(clone_field_api TRUE)

## Test if field_api was cloned by ecWAM
if(field_api_FOUND)
cmake_path(RELATIVE_PATH field_api_DIR BASE_DIRECTORY ${CMAKE_BINARY_DIR} OUTPUT_VARIABLE path_var)
cmake_path(GET path_var PARENT_PATH parent_path)
string(FIND ${parent_path} "../" result_var)

# If field_api is found but was not cloned by ecWAM, clone_field_api is set to FALSE
string(COMPARE EQUAL ${result_var} "-1" clone_field_api)
Copy link
Contributor Author

@awnawab awnawab Aug 28, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Once we build ecWAM and run FetchContent on FIELD_API, the cache variable field_api_FOUND is set to TRUE. If we then run make clean and re-configure the build without deleting the CMake cache, field_api_FOUND will still be true but the FIELD_API builddir will be empty and the build will fail at compile-time when trying to use libfield_api.

If on the other hand field_api_FOUND is TRUE not because ecWAM cloned FIELD_API but because it is coming from a bundle or system installation, then the above mechanism ensures we don't run FetchContent on FIELD_API.

endif()

if( clone_field_api )
include(FetchContent)
FetchContent_Declare(
field_api
GIT_REPOSITORY https://github.com/ecmwf-ifs/field_api.git
GIT_TAG v0.2.1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[no action] Eventually we might want a mechanism to define this in the top-level CMakeLists.txt, but not needed for this PR, I think

OVERRIDE_FIND_PACKAGE
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires CMake version 3.24 i.e. cmake_minimum_required(VERSION 3.24). I guess this is fine, but I strongly recommend to adapt the CMake file. The error you get with 3.22 is extremely unclear.

Copy link
Contributor Author

@awnawab awnawab Sep 8, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks a lot for pointing this out! FIELD_API itself requires CMake 3.25 for fixes to the cmake find_package(OpenACC), and given that this PR makes ecWAM hard reliant on FIELD_API, it makes sense to update the minimum CMake version in ecWAM to 3.25 too.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the dependency on 3.25 is only due to the OpenACC fixes, I'd argue it would be better to simply add the following workaround and make that a less strict requirement:

https://github.com/ecmwf-ifs/ectrans/blob/e7a738749fa9f6e28cc1ecce35f1643700d2bb19/CMakeLists.txt#L38-L47

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OVERRIDE_FIND_PACKAGE already needs CMake 3.24 as Lukas pointed out. When reconfiguring after a make clean, OVERRIDE_FIND_PACKAGE makes the FetchContent mechanism a bit more robust. So the decision would be between 3.24 and 3.25. In that context, I think 3.25 isn't too restrictive. What are your thoughts on this @wdeconinck ?

Copy link
Collaborator

@wdeconinck wdeconinck Sep 8, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think 3.24 is my current preference, using @reuterbal 's suggestion to adapt FIELD_API, as Leonardo currently uses cmake 3.24 as latest version. Both LUMI and our own HPC have 3.25 however.

)

set(FIELD_API_ENABLE_TESTS OFF)
set(FIELD_API_ENABLE_ACC OFF)
FetchContent_MakeAvailable(field_api)
endif()
6 changes: 3 additions & 3 deletions cmake/ecwam_find_python_mods.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ macro( ecwam_find_python_mods )
set(PYYAML_FOUND OFF)

# Look for fypp pre-processor
find_program( FYPP_PATH fypp QUIET)
if( FYPP_PATH )
find_program( FYPP fypp HINTS ${fypp_ROOT} )
if( FYPP )
ecbuild_info( "${ECWAM_PROJECT_NAME} FOUND fypp" )
set(FYPP_FOUND ON)
else()
ecbuild_info( "${ECWAM_PROJECT_NAME} FAILED to find optional package fypp" )
endif()
# We do a QUIET ecbuild_find_package to update the ecbuild project summary
ecbuild_find_package( fypp QUIET)
ecbuild_find_package( fypp QUIET )

# Look for python interpreter and pyyaml package
ecbuild_find_python()
Expand Down
2 changes: 1 addition & 1 deletion package/bundle/arch/ecmwf/hpc2020/default/env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ module_load fftw/3.3.9
module_load netcdf4/4.7.4
module_load hdf5/1.10.6
module_load eigen/3.3.7
module_load cmake/3.20.2
module_load cmake/3.25.2
module_load ninja/1.10.0
module_load fcm/2019.05.0
module_load aec/1.0.4
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ module_load fftw/3.3.9
module_load netcdf4/4.7.4
module_load hdf5/1.10.6
module_load eigen/3.3.7
module_load cmake/3.20.2
module_load cmake/3.25.2
module_load ninja/1.10.0
module_load fcm/2019.05.0
module_load aec/1.0.4
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ module_load fftw/3.3.9
module_load netcdf4/4.7.4
module_load hdf5/1.10.6
module_load eigen/3.3.7
module_load cmake/3.20.2
module_load cmake/3.25.2
module_load ninja/1.10.0
module_load fcm/2019.05.0
module_load aec/1.0.4
Expand Down
32 changes: 32 additions & 0 deletions package/bundle/arch/ecmwf/hpc2020/nvhpc/22.11/env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Source me to get the correct configure/build/run environment

# Store tracing and disable (module is *way* too verbose)
{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null

module_load() {
echo "+ module load $*"
module load $*
}
module_unload() {
echo "+ module unload $*"
module unload $*
}
module_purge() {
echo "+ module purge"
module purge
}

# Delete existing modules
module_purge

# Load modules
module_load prgenv/nvidia
module_load nvidia/22.11
module_load cmake/3.25.2
module_load python3/3.10.10-01
module_load hpcx-openmpi/2.10.0

# Restore tracing to stored setting
{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null

export ECBUILD_TOOLCHAIN="./toolchain.cmake"
26 changes: 26 additions & 0 deletions package/bundle/arch/ecmwf/hpc2020/nvhpc/22.11/toolchain.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# (C) Copyright 2022- ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.

####################################################################
# OpenMP FLAGS
####################################################################

# Note: OpenMP_Fortran_FLAGS gets overwritten by the FindOpenMP module
# unless its stored as a cache variable
set( OpenMP_Fortran_FLAGS "-mp=bind,allcores,numa" CACHE STRING "" FORCE)

####################################################################
# COMMON FLAGS
####################################################################

set(ECBUILD_Fortran_FLAGS "-Mframe")
set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mbyteswapio")
set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mstack_arrays")
set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mrecursive")
set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Kieee")
set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mdaz")
17 changes: 4 additions & 13 deletions package/bundle/bundle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,22 +23,17 @@ projects :
git : https://github.com/ecmwf-ifs/fiat
version : main

- field_api :
git : ${BITBUCKET}/RDX/field_api
version : master
optional: true
cmake : >
BUILD_field_api=OFF
ENABLE_CUDA=OFF
ENABLE_FIELD_API_TESTS=OFF

- ecwam :
dir : $PWD
version : main
require : fiat

options :

- with-single-precision :
help : Enable single precision build of the dwarf
cmake : ENABLE_SINGLE_PRECISION=ON

- without-mpi :
help : Disable MPI
cmake : ENABLE_MPI=OFF
Expand All @@ -47,7 +42,3 @@ options :
help : Disable OpenMP
cmake : ENABLE_OMP=OFF

- with-field_api :
help : Enable FIELD_API
cmake : BUILD_field_api=ON

38 changes: 32 additions & 6 deletions src/ecwam/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -347,21 +347,23 @@ list( APPEND ecwam_srcs
z0wave.F90
)

# check for mixed precision build
set(MIXED_PRECISION 0)
if(${HAVE_SINGLE_PRECISION} AND NOT ${PNAME}_OCEANMODEL_HAVE_SINGLE_PRECISION)
set(MIXED_PRECISION 1)
endif()

if( HAVE_GEN_DERIV_TYPES )
list( APPEND ecwam_srcs ${CMAKE_CURRENT_BINARY_DIR}/yowfield_mod.F90)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/yowfield_mod.F90
COMMAND fypp -m io -m yaml -m os ${CMAKE_CURRENT_SOURCE_DIR}/yowfield_mod.fypp > yowfield_mod.F90
COMMAND ${FYPP} -m io -m yaml -m os -DMIXED_PRECISION=${MIXED_PRECISION} ${CMAKE_CURRENT_SOURCE_DIR}/yowfield_mod.fypp > yowfield_mod.F90
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/yowfield_mod.fypp
VERBATIM)
else()
list( APPEND ecwam_srcs yowfield_mod.F90)
endif()

if( NOT HAVE_FIELD_API )
list( APPEND ecwam_srcs field_module.F90)
endif()

unset( MPI_Fortran_LIBRARIES )
if( HAVE_UNWAM )
list(APPEND ecwam_srcs ${unwam_srcs})
Expand Down Expand Up @@ -418,7 +420,7 @@ ecbuild_add_library(
PRIVATE_LIBS eccodes_f90
${MULTIO_LIBRARIES}
${OpenMP_Fortran_LIBRARIES}
$<${HAVE_FIELD_API}:field_api_${prec}>
field_api_${prec}
PUBLIC_INCLUDES $<INSTALL_INTERFACE:include>
PRIVATE_INCLUDES ${${PNAME}_OCEANMODEL_INCLUDE_DIRS}
PUBLIC_DEFINITIONS ${ECWAM_DEFINITIONS}
Expand All @@ -443,3 +445,27 @@ elseif( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
set_source_files_properties( mubuf.F90 PROPERTIES COMPILE_OPTIONS "-ffp-contract=off" )
endif()

### The file grib2wgrid.F90 is sensitive to optimizations in single precision builds.
#  This leads to non-neglibible differences
#  of average 'swh' when running "chief".

if( CMAKE_Fortran_COMPILER_ID MATCHES Intel AND HAVE_SINGLE_PRECISION )
set_source_files_properties( grib2wgrid.F90 PROPERTIES COMPILE_OPTIONS "-fp-model;strict" )
endif()

### NVHPC debug flags applied selectively to sourcefiles to avoid single-precision overflow
if( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC AND CMAKE_BUILD_TYPE MATCHES "Debug" )
foreach( debug_flag initsnan checkbounds )
if( ${debug_flag}_flags )
ecbuild_add_fortran_flags( "${${debug_flag}_flags}" NAME ${debug_flag} )
endif()
endforeach()

list(REMOVE_ITEM ecwam_srcs outbeta.F90 secondhh.F90)
if( fpe_flags )
set_source_files_properties( ${ecwam_srcs} PROPERTIES COMPILE_OPTIONS ${fpe_flags} )
set_source_files_properties( outbeta.F90 PROPERTIES COMPILE_OPTIONS "-Ktrap=divz" )
set_source_files_properties( secondhh.F90 PROPERTIES COMPILE_OPTIONS "-Ktrap=inv,ovf" )
endif()
list(APPEND ecwam_srcs outbeta.F90 secondhh.F90)
endif()
Loading
Loading