Fix NIC binding for Frontier.

StanfordLegion · Jul 18, 2024 · 3212294 · 3212294
1 parent 54b6e0f
commit 3212294
Show file tree

Hide file tree

Showing 5 changed files with 49 additions and 4 deletions.
diff --git a/experiment/frontier/sbatch_circuit.sh b/experiment/frontier/sbatch_circuit.sh
@@ -12,6 +12,15 @@ root_dir="$PWD"
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD"
 export SCRATCH="$MEMBERWORK/cmb103"
 
+export FI_MR_CACHE_MONITOR=memhooks
+export FI_CXI_RX_MATCH_MODE=software
+export GASNET_OFI_DEVICE_0=cxi2
+export GASNET_OFI_DEVICE_1=cxi1
+export GASNET_OFI_DEVICE_2=cxi3
+export GASNET_OFI_DEVICE_3=cxi0
+export GASNET_OFI_DEVICE_TYPE=Node
+export GASNET_OFI_NUM_RECEIVE_BUFFS=32M
+
 ulimit -S -c 0 # disable core dumps
 
 slurm_flags=

diff --git a/experiment/frontier/sbatch_cpp_circuit.sh b/experiment/frontier/sbatch_cpp_circuit.sh
@@ -2,7 +2,7 @@
 #SBATCH -A CMB103
 #SBATCH --partition=batch
 #SBATCH --dependency=singleton
-#SBATCH --job-name=circuit_test
+#SBATCH --job-name=cpp_circuit_test
 #SBATCH --time=01:00:00
 #SBATCH --mail-type=ALL
 #SBATCH -C nvme
@@ -12,6 +12,15 @@ root_dir="$PWD"
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD"
 export SCRATCH="$MEMBERWORK/cmb103"
 
+export FI_MR_CACHE_MONITOR=memhooks
+export FI_CXI_RX_MATCH_MODE=software
+export GASNET_OFI_DEVICE_0=cxi2
+export GASNET_OFI_DEVICE_1=cxi1
+export GASNET_OFI_DEVICE_2=cxi3
+export GASNET_OFI_DEVICE_3=cxi0
+export GASNET_OFI_DEVICE_TYPE=Node
+export GASNET_OFI_NUM_RECEIVE_BUFFS=32M
+
 ulimit -S -c 0 # disable core dumps
 
 slurm_flags=

diff --git a/experiment/frontier/sbatch_cpp_pennant.sh b/experiment/frontier/sbatch_cpp_pennant.sh
@@ -2,7 +2,7 @@
 #SBATCH -A CMB103
 #SBATCH --partition=batch
 #SBATCH --dependency=singleton
-#SBATCH --job-name=circuit_test
+#SBATCH --job-name=cpp_pennant_test
 #SBATCH --time=01:00:00
 #SBATCH --mail-type=ALL
 #SBATCH -C nvme
@@ -12,6 +12,15 @@ root_dir="$PWD"
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD"
 export SCRATCH="$MEMBERWORK/cmb103"
 
+export FI_MR_CACHE_MONITOR=memhooks
+export FI_CXI_RX_MATCH_MODE=software
+export GASNET_OFI_DEVICE_0=cxi2
+export GASNET_OFI_DEVICE_1=cxi1
+export GASNET_OFI_DEVICE_2=cxi3
+export GASNET_OFI_DEVICE_3=cxi0
+export GASNET_OFI_DEVICE_TYPE=Node
+export GASNET_OFI_NUM_RECEIVE_BUFFS=32M
+
 ulimit -S -c 0 # disable core dumps
 
 slurm_flags=

diff --git a/experiment/frontier/sbatch_pennant.sh b/experiment/frontier/sbatch_pennant.sh
@@ -2,7 +2,7 @@
 #SBATCH -A CMB103
 #SBATCH --partition=batch
 #SBATCH --dependency=singleton
-#SBATCH --job-name=circuit_test
+#SBATCH --job-name=pennant_test
 #SBATCH --time=01:00:00
 #SBATCH --mail-type=ALL
 #SBATCH -C nvme
@@ -12,6 +12,15 @@ root_dir="$PWD"
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD"
 export SCRATCH="$MEMBERWORK/cmb103"
 
+export FI_MR_CACHE_MONITOR=memhooks
+export FI_CXI_RX_MATCH_MODE=software
+export GASNET_OFI_DEVICE_0=cxi2
+export GASNET_OFI_DEVICE_1=cxi1
+export GASNET_OFI_DEVICE_2=cxi3
+export GASNET_OFI_DEVICE_3=cxi0
+export GASNET_OFI_DEVICE_TYPE=Node
+export GASNET_OFI_NUM_RECEIVE_BUFFS=32M
+
 ulimit -S -c 0 # disable core dumps
 
 slurm_flags=

diff --git a/experiment/frontier/sbatch_stencil.sh b/experiment/frontier/sbatch_stencil.sh
@@ -2,7 +2,7 @@
 #SBATCH -A CMB103
 #SBATCH --partition=batch
 #SBATCH --dependency=singleton
-#SBATCH --job-name=circuit_test
+#SBATCH --job-name=stencil_test
 #SBATCH --time=01:00:00
 #SBATCH --mail-type=ALL
 #SBATCH -C nvme
@@ -12,6 +12,15 @@ root_dir="$PWD"
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD"
 export SCRATCH="$MEMBERWORK/cmb103"
 
+export FI_MR_CACHE_MONITOR=memhooks
+export FI_CXI_RX_MATCH_MODE=software
+export GASNET_OFI_DEVICE_0=cxi2
+export GASNET_OFI_DEVICE_1=cxi1
+export GASNET_OFI_DEVICE_2=cxi3
+export GASNET_OFI_DEVICE_3=cxi0
+export GASNET_OFI_DEVICE_TYPE=Node
+export GASNET_OFI_NUM_RECEIVE_BUFFS=32M
+
 ulimit -S -c 0 # disable core dumps
 
 slurm_flags=