diff --git a/checkout-versions.yaml b/checkout-versions.yaml index 2d7d07ce6..6c3973512 100644 --- a/checkout-versions.yaml +++ b/checkout-versions.yaml @@ -5,4 +5,4 @@ versions: ramble: bb664f142b2cbdb2b2ea39e70a8535c9f27c1179 - spack: 31de670bd26beca979ebd75dcb0ce90c535a78c4 + spack: c2eef8bab26adb00b250992e29d697b4706356a0 diff --git a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml index 0cb4fc3b5..7aa44dea9 100644 --- a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml @@ -16,9 +16,9 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 - buildable: false + - spec: cmake@3.26.3 + prefix: /usr/tce/packages/cmake/cmake-3.26.3 + buildable: false gmake: externals: - spec: gmake@4.2.1 diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml index 1b64a0ee3..129876c20 100644 --- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml @@ -16,8 +16,8 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 + - spec: cmake@3.26.3 + prefix: /usr/tce/packages/cmake/cmake-3.26.3 buildable: false gmake: externals: @@ -48,6 +48,11 @@ packages: prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/ - spec: mvapich2@2.3.7-clang-14.0.6 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6 + cub: + buildable: false + externals: + - spec: cub@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ cuda: buildable: false externals: diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml index d27a632c1..4e4c457a1 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml @@ -4,6 +4,22 @@ # SPDX-License-Identifier: Apache-2.0 compilers: +- compiler: + spec: xl@16.1.1-2023.06.28-cuda-11.8.0-gcc-11.2.1 + paths: + cc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlc + cxx: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlC + f77: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf + fc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf + flags: + cflags: -g -O2 + cxxflags: -g -O2 -std=c++14 + fflags: -g -O2 + operating_system: rhel7 + target: ppc64le + modules: [cuda/11.8.0, xl/2023.06.28-cuda-11.8.0-gcc-11.2.1] + environment: {} + extra_rpaths: [] - compiler: spec: xl@16.1.1-2022.08.19-cuda-11.7.0 paths: @@ -29,7 +45,7 @@ compilers: fc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf flags: cflags: -g -O2 - cxxflags: -g -O2 -std=c++11 + cxxflags: -g -O2 -std=c++14 fflags: -g -O2 operating_system: rhel7 target: ppc64le @@ -45,7 +61,7 @@ compilers: fc: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlf flags: # TODO: Fix spack concretization bug cflags: -g -O2 - cxxflags: -g -O2 -std=c++11 + cxxflags: -g -O2 -std=c++14 fflags: -g -O2 operating_system: rhel7 target: ppc64le diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml index 61c85f62d..5d26658c1 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml @@ -11,8 +11,8 @@ packages: buildable: false cmake: externals: - - spec: cmake@3.23.1 - prefix: /usr/tce/packages/cmake/cmake-3.23.1 + - spec: cmake@3.29.2 + prefix: /usr/tce/packages/cmake/cmake-3.29.2 buildable: false gmake: externals: @@ -41,6 +41,11 @@ packages: - spec: cusparse@11.8.0 prefix: /usr/tce/packages/cuda/cuda-11.8.0 buildable: false + cub: + buildable: false + externals: + - spec: cub@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ cuda: externals: - spec: cuda@10.1.243 @@ -72,6 +77,10 @@ packages: buildable: false mpi: externals: + - spec: spectrum-mpi@2023.06.28-cuda-11.8.0-gcc-11.2.1 + prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2023.06.28-cuda-11.8.0-gcc-11.2.1 + extra_attributes: + ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm" - spec: spectrum-mpi@2022.08.19-cuda-11.8.0 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-11.8.0 extra_attributes: diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml index ed1edcb60..7b2f4257b 100644 --- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml +++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml @@ -6,9 +6,9 @@ spack: packages: default-compiler: - spack_spec: xl@16.1.1-2022.08.19-cuda{default_cuda_version} + spack_spec: xl@16.1.1-2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1 default-mpi: - spack_spec: spectrum-mpi@2022.08.19 + spack_spec: spectrum-mpi@2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1 compiler-gcc: spack_spec: gcc@8.3.1 compiler-clang: diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml index af604fd29..a9d63480b 100644 --- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml +++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml @@ -41,6 +41,7 @@ packages: prefix: /usr - spec: cmake@3.23.1 prefix: /usr/tce + buildable: false pkgconf: externals: - spec: pkgconf@1.4.2 diff --git a/experiments/kripke/cuda/execute_experiment.tpl b/experiments/kripke/cuda/execute_experiment.tpl index cda7e5bcb..89e73cf49 100755 --- a/experiments/kripke/cuda/execute_experiment.tpl +++ b/experiments/kripke/cuda/execute_experiment.tpl @@ -1,5 +1,4 @@ #!/bin/bash -# # Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # @@ -11,8 +10,4 @@ cd {experiment_run_dir} -{spack_setup} - -{experiment_setup} - {command} diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml index 0eec22b58..b4771bdba 100644 --- a/experiments/kripke/cuda/ramble.yaml +++ b/experiments/kripke/cuda/ramble.yaml @@ -7,6 +7,7 @@ ramble: include: - ./configs/spack.yaml - ./configs/variables.yaml + - ./configs/modifier.yaml config: deprecated: true @@ -17,38 +18,37 @@ ramble: applications: kripke: workloads: - problem1: + kripke: env_vars: variables: - p: 2 - px: '{p}' - py: '{p}' - pz: '{p}' - n: ['55', '110'] - nx: '{n}' - ny: '{n}' - nz: '{n}' - experiment_setup: '' - processes_per_node: ['4'] - n_nodes: ['1', '2'] n_ranks: '{processes_per_node} * {n_nodes}' + experiment_setup: '' + processes_per_node: '2' + n_nodes: '2' + arch: 'CUDA' experiments: - kripke_rocm_problem1_{n_nodes}_{n_ranks}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: + kripke_cuda_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: variables: - env_name: kripke - matrices: - - size_jobs: - - n # TODO: Filter matrix - - n_nodes # TODO: Filter matrix - + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '64' + nzz: '32' + npx: '2' + npy: '2' + npz: '1' spack: concretized: true packages: - kripke-cuda: - spack_spec: kripke@updateRCU2024.02 +cuda + kripke: + spack_spec: kripke@1.2.7.0 +mpi+cuda{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: - kripke-cuda: + kripke: packages: - default-mpi - - kripke-cuda + - kripke + - '{modifier_package_name}' diff --git a/experiments/kripke/openmp/execute_experiment.tpl b/experiments/kripke/openmp/execute_experiment.tpl index cda7e5bcb..89e73cf49 100755 --- a/experiments/kripke/openmp/execute_experiment.tpl +++ b/experiments/kripke/openmp/execute_experiment.tpl @@ -1,5 +1,4 @@ #!/bin/bash -# # Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # @@ -11,8 +10,4 @@ cd {experiment_run_dir} -{spack_setup} - -{experiment_setup} - {command} diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml index 0ede5c91f..afdbaa127 100644 --- a/experiments/kripke/openmp/ramble.yaml +++ b/experiments/kripke/openmp/ramble.yaml @@ -7,6 +7,7 @@ ramble: include: - ./configs/spack.yaml - ./configs/variables.yaml + - ./configs/modifier.yaml config: deprecated: true @@ -17,41 +18,40 @@ ramble: applications: kripke: workloads: - problem1: + kripke: env_vars: set: OMP_NUM_THREADS: '{omp_num_threads}' variables: n_ranks: '{processes_per_node} * {n_nodes}' - p: 2 - px: '{p}' - py: '{p}' - pz: '{p}' - n: ['55', '110'] - nx: '{n}' - ny: '{n}' - nz: '{n}' experiment_setup: '' - processes_per_node: ['8', '4'] - n_nodes: ['1', '2'] - threads_per_node_core: ['8', '10', '13'] #TODO: Specify n_threads according to available n_nodes and n_ranks - omp_num_threads: '{threads_per_node_core} * {n_nodes}' + processes_per_node: ['1', '16'] + n_nodes: '1' + omp_num_threads: ['32', '2'] + arch: 'OpenMP' experiments: - kripke_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: + kripke_omp_{n_nodes}_{omp_num_threads}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: variables: - env_name: kripke - matrices: - - size_threads: - - n # TODO: Filter matrix - - threads_per_node_core # TODO: Filter matrix + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '32' + nzz: '32' + npx: ['1', '4'] + npy: ['1', '2'] + npz: ['1', '2'] spack: concretized: true packages: - kripke-omp: - spack_spec: kripke@develop +mpi+openmp + kripke: + spack_spec: kripke@1.2.7.0 +mpi+openmp{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: - kripke-omp: + kripke: packages: - default-mpi - - kripke-omp + - kripke + - '{modifier_package_name}' diff --git a/experiments/kripke/rocm/execute_experiment.tpl b/experiments/kripke/rocm/execute_experiment.tpl index cda7e5bcb..89e73cf49 100755 --- a/experiments/kripke/rocm/execute_experiment.tpl +++ b/experiments/kripke/rocm/execute_experiment.tpl @@ -1,5 +1,4 @@ #!/bin/bash -# # Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # @@ -11,8 +10,4 @@ cd {experiment_run_dir} -{spack_setup} - -{experiment_setup} - {command} diff --git a/experiments/kripke/rocm/ramble.yaml b/experiments/kripke/rocm/ramble.yaml index d22da9f3d..7439b57dc 100644 --- a/experiments/kripke/rocm/ramble.yaml +++ b/experiments/kripke/rocm/ramble.yaml @@ -7,6 +7,7 @@ ramble: include: - ./configs/spack.yaml - ./configs/variables.yaml + - ./configs/modifier.yaml config: deprecated: true @@ -17,37 +18,37 @@ ramble: applications: kripke: workloads: - problem1: + kripke: env_vars: variables: - p: 2 - px: '{p}' - py: '{p}' - pz: '{p}' - n: ['55', '110'] - nx: '{n}' - ny: '{n}' - nz: '{n}' - experiment_setup: '' - processes_per_node: ['4'] - n_nodes: ['1', '2'] n_ranks: '{processes_per_node} * {n_nodes}' + experiment_setup: '' + processes_per_node: ['8'] + n_nodes: '1' + arch: 'HIP' experiments: - kripke_rocm_problem1_{n_nodes}_{n_ranks}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: + kripke_rocm_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}: variables: - env_name: kripke - matrices: - - size_jobs: - - n # TODO: Filter matrix - - n_nodes # TODO: Filter matrix + ngroups: 64 + gs: 1 + nquad: 128 + ds: 128 + lorder: 4 + nzx: '64' + nzy: '64' + nzz: '64' + npx: '2' + npy: '2' + npz: '2' spack: concretized: true packages: - kripke-rocm: - spack_spec: kripke@updateRCU2024.02 +mpi+rocm + kripke: + spack_spec: kripke@1.2.7.0 +mpi+rocm{modifier_spack_variant} ^chai@2024.02 compiler: default-compiler environments: - kripke-rocm: + kripke: packages: - default-mpi - - kripke-rocm + - kripke + - '{modifier_package_name}' diff --git a/repo/kripke/application.py b/repo/kripke/application.py index e9d0ba2a3..61a79f7f6 100644 --- a/repo/kripke/application.py +++ b/repo/kripke/application.py @@ -17,44 +17,102 @@ class Kripke(SpackApplication): 'high-memory-bandwidth','large-memory-footprint','regular-memory-access', 'mpi','network-latency-bound','network-collectives'] - executable('p1', 'kripke' + - ' -P {px} {py} {pz}' + - ' -n {nx} {ny} {nz}' + - ' -problem 1' + - ' -keepT', use_mpi=True) - - executable('p2', 'kripke' + - ' -P {px} {py} {pz}' + - ' -n {nx} {ny} {nz}' + - ' -problem 2' + - ' -keepT', use_mpi=True) - - workload('problem1', executables=['p1']) - workload('problem2', executables=['p2']) - - workload_variable('px', default='2', - description='px', - workloads=['problem1', 'problem2']) - workload_variable('py', default='2', - description='py', - workloads=['problem1', 'problem2']) - workload_variable('pz', default='2', - description='pz', - workloads=['problem1', 'problem2']) - workload_variable('nx', default='220', - description='nx', - workloads=['problem1', 'problem2']) - workload_variable('ny', default='220', - description='ny', - workloads=['problem1', 'problem2']) - workload_variable('nz', default='220', - description='nz', - workloads=['problem1', 'problem2']) - - figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + executable('kripke', 'kripke.exe' + + ' --groups {ngroups}' + + ' --legendre {lorder}' + + ' --quad {nquad}' + + ' --zones {nzx},{nzy},{nzz}' + + ' --sigt {sigt0},{sigt1},{sigt2}' + + ' --sigs {sigs0},{sigs1},{sigs2}' + + ' --arch {arch}' + + ' --layout {layout}' + +# ' --pdist {lout}' + + ' --procs {npx},{npy},{npz}' + + ' --dset {ds}' + + ' --gset {gs}' + + ' --zset {nzsetx},{nzsety},{nzsetz}' + + ' --niter {niter}' + + ' --pmethod {method}', use_mpi=True) - #TODO: Fix the FOM success_criteria(...) - success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') + workload('kripke', executables=['kripke']) + + workload_variable('ngroups', default='32', + description='Number of energy groups. (Default: --groups 32)', + workloads=['kripke']) + workload_variable('lorder', default='4', + description='Scattering Legendre Expansion Order (0, 1, ...). (Default: --legendre 4)', + workloads=['kripke']) + workload_variable('nquad', default='96', + description='Define the quadrature set to use either a fake S2 with points (ndirs), OR Gauss-Legendre with by points (polar:azim). (Default: --quad 96)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in x. (Default: 16)', + workloads=['kripke']) + workload_variable('nzx', default='16', + description='Number of zones in y. (Default: 16)', + workloads=['kripke']) + workload_variable('nzz', default='16', + description='Number of zones in z. (Default: 16)', + workloads=['kripke']) + workload_variable('sigt0', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt1', default='0.0001', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigt2', default='0.1', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs0', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs1', default='0.00005', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('sigs2', default='0.05', + description='Total material cross-sections', + workloads=['kripke']) + workload_variable('arch', default='OpenMP', + description='Architecture selection. Selects the back-end used for computation, available are Sequential, OpenMP, CUDA and HIP. The default depends on capabilities selected by the build system and is selected from list of increasing precedence: Sequential, OpenMP, CUDA and HIP.', + workloads=['kripke']) + workload_variable('layout', default='DGZ', + description='Data layout selection. This determines the data layout and kernel implementation details (such as loop nesting order). The layouts are determined by the order of unknowns in the angular flux: Direction, Group, and Zone. Available layouts are DGZ, DZG, GDZ, GZD, ZDG, and ZGD. The order is specified left-to-right in longest-to-shortest stride. For example: DGZ means that Directions are the longest stride, and Zones are stride-1. (Default: --nest DGZ)', + workloads=['kripke']) +# workload_variable('lout', default='0', +# description='Layout of spatial subdomains over mpi ranks. 0 for "Blocked" where local zone sets represent adjacent regions of space. 1 for "Scattered" where adjacent regions of space are distributed to adjacent MPI ranks. (Default: --layout 0)', +# workloads=['kripke']) + workload_variable('npx', default='1', + description='Number of MPI ranks in x dimension', + workloads=['kripke']) + workload_variable('npy', default='1', + description='Number of MPI ranks in y dimension', + workloads=['kripke']) + workload_variable('npz', default='1', + description='Number of MPI ranks in z dimension', + workloads=['kripke']) + workload_variable('ds', default='8', + description='Number of direction-sets. Must be a factor of 8, and divide evenly the number of quadrature points. (Default: --dset 8)', + workloads=['kripke']) + workload_variable('gs', default='1', + description='Number of energy group-sets. Must divide evenly the number energy groups. (Default: --gset 1)', + workloads=['kripke']) + workload_variable('nzsetx', default='1', + description='Number of zone-sets in x', + workloads=['kripke']) + workload_variable('nzsety', default='1', + description='Number of zone-sets in y', + workloads=['kripke']) + workload_variable('nzsetz', default='1', + description='Number of zone-sets in z', + workloads=['kripke']) + workload_variable('niter', default='10', + description='Number of solver iterations to run. (Default: --niter 10)', + workloads=['kripke']) + workload_variable('method', default='sweep', + description='Parallel solver method. "sweep" for full up-wind sweep (wavefront algorithm). "bj" for Block Jacobi. (Default: --pmethod sweep)', + workloads=['kripke']) - def evaluate_success(self): - return True + #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') + + #TODO: Fix the FOM success_criteria(...) + #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') diff --git a/repo/kripke/package.py b/repo/kripke/package.py index 6bf335f35..7f1269550 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -22,7 +22,8 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): version("develop", branch="develop", submodules=False) version( - "updateRCU2024.02", branch="task/chen59/updateRCU2024.02", submodules=False) + "1.2.7.0", submodules=False, commit="db920c1f5e1dcbb9e949d120e7d86efcdb777635" + ) version( "1.2.4", submodules=False, tag="v1.2.4", commit="d85c6bc462f17a2382b11ba363059febc487f771" ) @@ -55,18 +56,35 @@ class Kripke(CMakePackage, CudaPackage, ROCmPackage): depends_on("mpi", when="+mpi") depends_on("caliper", when="+caliper") depends_on("adiak@0.4:", when="+caliper") - + conflicts("^blt@:0.3.6", when="+rocm") + aligned_versions = ["2024.02"] - constrain_chai = list(f"^chai@{v}" for v in aligned_versions) - requires(*aligned_versions, policy="one_of", msg="Pick a chai version with proper alignment w/RAJA & Umpire") - for v in aligned_versions: depends_on(f"raja@{v}~exercises~examples", when=f"^chai@{v}") depends_on(f"umpire@{v}~examples", when=f"^chai@{v}") depends_on(f"chai@{v}~examples+raja", when=f"^chai@{v}") - - depends_on("blt@0.6.2", type="build") - conflicts("^blt@:0.3.6", when="+rocm") + depends_on(f"camp@{v}", when=f"^chai@{v}") + + depends_on("blt@0.6.2:", type="build", when=f"@1.2.7:") + + depends_on("chai+openmp", when="+openmp") + depends_on("chai~openmp", when="~openmp") + depends_on("chai+cuda", when="+cuda") + depends_on("chai~cuda", when="~cuda") + depends_on("chai+rocm", when="+rocm") + depends_on("chai~rocm", when="~rocm") + + depends_on("umpire+openmp", when="+openmp") + depends_on("umpire~openmp", when="~openmp") + depends_on("umpire+cuda", when="+cuda") + depends_on("umpire~cuda", when="~cuda") + depends_on("umpire+rocm", when="+rocm") + depends_on("umpire~rocm", when="~rocm") + + def setup_build_environment(self, env): + spec = self.spec + if "+cuda" in spec: + env.set("CUDAHOSTCXX", self.spec["mpi"].mpicxx) def cmake_args(self): spec = self.spec @@ -83,6 +101,9 @@ def cmake_args(self): ] ) + if "+openmp" in spec: + args.append("-DENABLE_OPENMP=ON") + if "+caliper" in spec: args.append("-DENABLE_CALIPER=ON") @@ -109,10 +130,10 @@ def cmake_args(self): cuda_arch = spec.variants["cuda_arch"].value args.append("-DCUDA_ARCH={0}".format(cuda_arch[0])) args.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) - args.append( - "-DCMAKE_CUDA_FLAGS=--expt-extended-lambda -I%s -I=%s" - % (self.spec["cub"].prefix.include, self.spec["mpi"].prefix.include) - ) + args.append( + "-DCMAKE_CUDA_FLAGS=--extended-lambda -I%s -I=%s" + % (self.spec["cub"].prefix.include, self.spec["mpi"].prefix.include) + ) else: args.append("-DENABLE_CUDA=OFF")