From d19fcf17c49eb030f963541416eaaefb299631fe Mon Sep 17 00:00:00 2001
From: Carl Pearson <cwpears@sandia.gov>
Date: Mon, 9 Oct 2023 12:07:41 -0400
Subject: [PATCH] frontier scripts

---
 scripts/build.sh |  3 +--
 scripts/run.sh   | 24 -----------------
 scripts/run_1.sh | 68 ++++++++++++++++++++++++++++++++++++++++++++++++
 scripts/run_2.sh | 68 ++++++++++++++++++++++++++++++++++++++++++++++++
 scripts/run_3.sh | 68 ++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 205 insertions(+), 26 deletions(-)
 delete mode 100644 scripts/run.sh
 create mode 100755 scripts/run_1.sh
 create mode 100755 scripts/run_2.sh
 create mode 100755 scripts/run_3.sh

diff --git a/scripts/build.sh b/scripts/build.sh
index 89d4296..f9c5cbb 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -2,8 +2,7 @@
 
 set -eou pipefail
 
-# export ROOT=/lustre/orion/CSC465/scratch/cpearson/frontier-gpu-bandwidth
-export ROOT=$HOME/frontier-gpu-bandwidth
+export ROOT=/lustre/orion/csc465/scratch/cpearson/frontier-gpu-bandwidth
 export SCOPE_SRC=${ROOT}/comm_scope
 export SCOPE_BUILD=${ROOT}/build
 
diff --git a/scripts/run.sh b/scripts/run.sh
deleted file mode 100644
index 4abd0e5..0000000
--- a/scripts/run.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-#SBATCH -A CSC465
-#SBATCH -J frontier-gpu-bandwidth
-#SBATCH -o %x-%j.out
-#SBATCH -t 1:00:00
-#SBATCH -p batch
-#SBATCH -N 1
-
-# export ROOT=/lustre/orion/CSC465/scratch/cpearson/frontier-gpu-bandwidth
-export ROOT=$HOME/frontier-gpu-bandwidth
-export SCOPE_SRC=${ROOT}/comm_scope
-export SCOPE_BUILD=${ROOT}/build
-export SCOPE_RESULTS=${ROOT}/run
-
-module load PrgEnv-amd/8.3.3
-export HSA_XNACK=1
-
-mkdir -p $SCOPE_RESULTS
-module list > $SCOPE_RESULTS/modules.$SLURM_JOBID.txt 2>&1
-env > $SCOPE_RESULTS/env.$SLURM_JOBID.txt
-
-srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
---benchmark_list_tests
diff --git a/scripts/run_1.sh b/scripts/run_1.sh
new file mode 100755
index 0000000..d7c3f91
--- /dev/null
+++ b/scripts/run_1.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#SBATCH -A CSC465
+#SBATCH -J r1
+#SBATCH -o %x-%j.out
+#SBATCH -t 2:00:00
+#SBATCH -p batch
+#SBATCH -N 1
+
+# salloc -A CSC465 -N 1 -p batch -t 2:00:00
+
+export ROOT=/lustre/orion/csc465/scratch/cpearson/frontier-gpu-bandwidth
+export SCOPE_SRC=${ROOT}/comm_scope
+export SCOPE_BUILD=${ROOT}/build
+export SCOPE_RESULTS=${ROOT}/run
+
+module load PrgEnv-amd/8.3.3
+export HSA_XNACK=1
+
+mkdir -p $SCOPE_RESULTS
+module list > $SCOPE_RESULTS/modules.r1.$SLURM_JOBID.txt 2>&1
+env > $SCOPE_RESULTS/env.r1.$SLURM_JOBID.txt
+
+date
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*hipMemcpyAsync_GPUToPinned/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/hipMemcpyAsync_GPUToPinned.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*hipMemcpyAsync_PinnedToGPU/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/hipMemcpyAsync_PinnedToGPU"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*hipMemcpyAsync_GPUToGPU/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/hipMemcpyAsync_GPUToGPU.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*hipMemcpyAsync_GPUToPageable/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/hipMemcpyAsync_GPUToPageable.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*hipMemcpyAsync_PageableToGPU/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/hipMemcpyAsync_PageableToGPU.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*prefetch_managed_GPUToHost/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/prefetch_managed_GPUToHost.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*prefetch_managed_HostToGPU/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/prefetch_managed_HostToGPU.json"
+
+date
diff --git a/scripts/run_2.sh b/scripts/run_2.sh
new file mode 100755
index 0000000..6cd8beb
--- /dev/null
+++ b/scripts/run_2.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#SBATCH -A CSC465
+#SBATCH -J r2
+#SBATCH -o %x-%j.out
+#SBATCH -t 2:00:00
+#SBATCH -p batch
+#SBATCH -N 1
+
+# salloc -A CSC465 -N 1 -p batch -t 2:00:00
+
+export ROOT=/lustre/orion/csc465/scratch/cpearson/frontier-gpu-bandwidth
+export SCOPE_SRC=${ROOT}/comm_scope
+export SCOPE_BUILD=${ROOT}/build
+export SCOPE_RESULTS=${ROOT}/run
+
+module load PrgEnv-amd/8.3.3
+export HSA_XNACK=1
+
+mkdir -p $SCOPE_RESULTS
+module list > $SCOPE_RESULTS/modules.r2.$SLURM_JOBID.txt 2>&1
+env > $SCOPE_RESULTS/env.r2.$SLURM_JOBID.txt
+
+date
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_mapped_GPURdHost/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_mapped_GPURdHost.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_managed_HostWrGPU_fine/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_managed_HostWrGPU_fine.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_managed_HostWrGPU_coarse/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_managed_HostWrGPU_coarse.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_managed_GPUWrHost_fine/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_managed_GPUWrHost_fine.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_managed_GPUWrHost_coarse/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_managed_GPUWrHost_coarse.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_mapped_GPUWrHost/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_mapped_GPUWrHost.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_mapped_GPUWrGPU/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_mapped_GPUWrGPU.json"
+
+date
diff --git a/scripts/run_3.sh b/scripts/run_3.sh
new file mode 100755
index 0000000..06248f0
--- /dev/null
+++ b/scripts/run_3.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#SBATCH -A CSC465
+#SBATCH -J r3
+#SBATCH -o %x-%j.out
+#SBATCH -t 2:00:00
+#SBATCH -p batch
+#SBATCH -N 1
+
+# salloc -A CSC465 -N 1 -p batch -t 2:00:00
+
+export ROOT=/lustre/orion/csc465/scratch/cpearson/frontier-gpu-bandwidth
+export SCOPE_SRC=${ROOT}/comm_scope
+export SCOPE_BUILD=${ROOT}/build
+export SCOPE_RESULTS=${ROOT}/run
+
+module load PrgEnv-amd/8.3.3
+export HSA_XNACK=1
+
+mkdir -p $SCOPE_RESULTS
+module list > $SCOPE_RESULTS/modules.r3.$SLURM_JOBID.txt 2>&1
+env > $SCOPE_RESULTS/env.r3.$SLURM_JOBID.txt
+
+date
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_managed_GPUWrGPU_fine/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_managed_GPUWrGPU_fine.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_managed_GPUWrGPU_coarse/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_managed_GPUWrGPU_coarse.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_managed_GPURdHost_fine/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_managed_GPURdHost_fine.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_managed_GPURdHost_coarse/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_managed_GPURdHost_coarse.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*hipManaged_HostToGPUWriteDst/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/hipManaged_HostToGPUWriteDst.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*prefetch_managed_GPUToGPU/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/prefetch_managed_GPUToGPU.json"
+
+srun -c 56 -n 1 --gpus 8 $SCOPE_BUILD/comm_scope \
+--benchmark_repetitions=5 \
+--benchmark_filter='.*implicit_mapped_HostWrGPU/0/.*' \
+--benchmark_out_format=json \
+--benchmark_out="$SCOPE_RESULTS/implicit_mapped_HostWrGPU.json"
+
+date