From 3212294af06da026db30d70cc76f922e7cb46b14 Mon Sep 17 00:00:00 2001 From: Elliott Slaughter Date: Thu, 18 Jul 2024 18:39:19 -0400 Subject: [PATCH] Fix NIC binding for Frontier. --- experiment/frontier/sbatch_circuit.sh | 9 +++++++++ experiment/frontier/sbatch_cpp_circuit.sh | 11 ++++++++++- experiment/frontier/sbatch_cpp_pennant.sh | 11 ++++++++++- experiment/frontier/sbatch_pennant.sh | 11 ++++++++++- experiment/frontier/sbatch_stencil.sh | 11 ++++++++++- 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/experiment/frontier/sbatch_circuit.sh b/experiment/frontier/sbatch_circuit.sh index f1b1166..2d2d5a8 100644 --- a/experiment/frontier/sbatch_circuit.sh +++ b/experiment/frontier/sbatch_circuit.sh @@ -12,6 +12,15 @@ root_dir="$PWD" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD" export SCRATCH="$MEMBERWORK/cmb103" +export FI_MR_CACHE_MONITOR=memhooks +export FI_CXI_RX_MATCH_MODE=software +export GASNET_OFI_DEVICE_0=cxi2 +export GASNET_OFI_DEVICE_1=cxi1 +export GASNET_OFI_DEVICE_2=cxi3 +export GASNET_OFI_DEVICE_3=cxi0 +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_NUM_RECEIVE_BUFFS=32M + ulimit -S -c 0 # disable core dumps slurm_flags= diff --git a/experiment/frontier/sbatch_cpp_circuit.sh b/experiment/frontier/sbatch_cpp_circuit.sh index 1ffeea5..a99812f 100644 --- a/experiment/frontier/sbatch_cpp_circuit.sh +++ b/experiment/frontier/sbatch_cpp_circuit.sh @@ -2,7 +2,7 @@ #SBATCH -A CMB103 #SBATCH --partition=batch #SBATCH --dependency=singleton -#SBATCH --job-name=circuit_test +#SBATCH --job-name=cpp_circuit_test #SBATCH --time=01:00:00 #SBATCH --mail-type=ALL #SBATCH -C nvme @@ -12,6 +12,15 @@ root_dir="$PWD" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD" export SCRATCH="$MEMBERWORK/cmb103" +export FI_MR_CACHE_MONITOR=memhooks +export FI_CXI_RX_MATCH_MODE=software +export GASNET_OFI_DEVICE_0=cxi2 +export GASNET_OFI_DEVICE_1=cxi1 +export GASNET_OFI_DEVICE_2=cxi3 +export GASNET_OFI_DEVICE_3=cxi0 +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_NUM_RECEIVE_BUFFS=32M + ulimit -S -c 0 # disable core dumps slurm_flags= diff --git a/experiment/frontier/sbatch_cpp_pennant.sh b/experiment/frontier/sbatch_cpp_pennant.sh index 814b52b..15269da 100644 --- a/experiment/frontier/sbatch_cpp_pennant.sh +++ b/experiment/frontier/sbatch_cpp_pennant.sh @@ -2,7 +2,7 @@ #SBATCH -A CMB103 #SBATCH --partition=batch #SBATCH --dependency=singleton -#SBATCH --job-name=circuit_test +#SBATCH --job-name=cpp_pennant_test #SBATCH --time=01:00:00 #SBATCH --mail-type=ALL #SBATCH -C nvme @@ -12,6 +12,15 @@ root_dir="$PWD" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD" export SCRATCH="$MEMBERWORK/cmb103" +export FI_MR_CACHE_MONITOR=memhooks +export FI_CXI_RX_MATCH_MODE=software +export GASNET_OFI_DEVICE_0=cxi2 +export GASNET_OFI_DEVICE_1=cxi1 +export GASNET_OFI_DEVICE_2=cxi3 +export GASNET_OFI_DEVICE_3=cxi0 +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_NUM_RECEIVE_BUFFS=32M + ulimit -S -c 0 # disable core dumps slurm_flags= diff --git a/experiment/frontier/sbatch_pennant.sh b/experiment/frontier/sbatch_pennant.sh index a7ba166..8c4da3b 100644 --- a/experiment/frontier/sbatch_pennant.sh +++ b/experiment/frontier/sbatch_pennant.sh @@ -2,7 +2,7 @@ #SBATCH -A CMB103 #SBATCH --partition=batch #SBATCH --dependency=singleton -#SBATCH --job-name=circuit_test +#SBATCH --job-name=pennant_test #SBATCH --time=01:00:00 #SBATCH --mail-type=ALL #SBATCH -C nvme @@ -12,6 +12,15 @@ root_dir="$PWD" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD" export SCRATCH="$MEMBERWORK/cmb103" +export FI_MR_CACHE_MONITOR=memhooks +export FI_CXI_RX_MATCH_MODE=software +export GASNET_OFI_DEVICE_0=cxi2 +export GASNET_OFI_DEVICE_1=cxi1 +export GASNET_OFI_DEVICE_2=cxi3 +export GASNET_OFI_DEVICE_3=cxi0 +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_NUM_RECEIVE_BUFFS=32M + ulimit -S -c 0 # disable core dumps slurm_flags= diff --git a/experiment/frontier/sbatch_stencil.sh b/experiment/frontier/sbatch_stencil.sh index 4ee712e..da7d999 100644 --- a/experiment/frontier/sbatch_stencil.sh +++ b/experiment/frontier/sbatch_stencil.sh @@ -2,7 +2,7 @@ #SBATCH -A CMB103 #SBATCH --partition=batch #SBATCH --dependency=singleton -#SBATCH --job-name=circuit_test +#SBATCH --job-name=stencil_test #SBATCH --time=01:00:00 #SBATCH --mail-type=ALL #SBATCH -C nvme @@ -12,6 +12,15 @@ root_dir="$PWD" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD" export SCRATCH="$MEMBERWORK/cmb103" +export FI_MR_CACHE_MONITOR=memhooks +export FI_CXI_RX_MATCH_MODE=software +export GASNET_OFI_DEVICE_0=cxi2 +export GASNET_OFI_DEVICE_1=cxi1 +export GASNET_OFI_DEVICE_2=cxi3 +export GASNET_OFI_DEVICE_3=cxi0 +export GASNET_OFI_DEVICE_TYPE=Node +export GASNET_OFI_NUM_RECEIVE_BUFFS=32M + ulimit -S -c 0 # disable core dumps slurm_flags=