From 28cf332ad8b2607fae77b96ce4bcab5d6297cbb9 Mon Sep 17 00:00:00 2001
From: Carl Pearson <cwpears@sandia.gov>
Date: Thu, 30 Nov 2023 15:14:56 -0700
Subject: [PATCH] add Blake SPR scripts

---
 README.md             |  2 +-
 blake-spr/download.sh | 45 +++++++++++++++++++++++++++++++++++++++++++
 blake-spr/env.sh      | 14 ++++++++++++++
 blake-spr/run.sh      | 24 +++++++++++++++++++++++
 frontier/download.sh  |  6 +++---
 5 files changed, 87 insertions(+), 4 deletions(-)
 create mode 100755 blake-spr/download.sh
 create mode 100644 blake-spr/env.sh
 create mode 100755 blake-spr/run.sh

diff --git a/README.md b/README.md
index d0261de..6c67023 100644
--- a/README.md
+++ b/README.md
@@ -7,5 +7,5 @@ Some examples of building and running Kokkos Kernels on Exascale-relevant Hardwa
 | `frontier` | AMD MI250x |
 | `blake-spr` | Intel Sapphire Rapids |
 | `blake-pvc` | Intel Ponte Vecchio |
-| `lumen` | Nvidia H100 |
+| `blake-h100` | Nvidia H100 |
 | `perlmutter` | Nvidia A100 |
\ No newline at end of file
diff --git a/blake-spr/download.sh b/blake-spr/download.sh
new file mode 100755
index 0000000..96da9f4
--- /dev/null
+++ b/blake-spr/download.sh
@@ -0,0 +1,45 @@
+#! /bin/bash
+
+set -eou pipefail
+
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build"
+export LOG_DIR
+mkdir -p "$LOG_DIR"
+
+# intel blows up SSH for some reason?
+module del intel/oneAPI/hpc-toolkit/2022.1.2
+git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true
+(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true
+git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true
+(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true
+
+# re-set up our environment
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+module list |& tee "$LOG_DIR/module-list.log"
+lscpu |& tee "$LOG_DIR/lscpu.log"
+hostname |& tee "$LOG_DIR/hostname.log"
+
+## Configure Kokkos
+cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \
+-DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \
+-DCMAKE_CXX_STANDARD=17 \
+-DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_CXX_COMPILER=icpx \
+-DKokkos_ENABLE_OPENMP=ON \
+-DKokkos_ARCH_NATIVE=ON
+
+## Build & Install Kokkos
+cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install
+
+## Configure Kernels
+cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \
+-DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \
+-DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_CXX_COMPILER=icpx \
+-DKokkosKernels_ENABLE_TPL_MKL=ON
+
+## Build Kernels
+cmake --build "$KERNELS_BUILD" -j "$(nproc)"
\ No newline at end of file
diff --git a/blake-spr/env.sh b/blake-spr/env.sh
new file mode 100644
index 0000000..231c733
--- /dev/null
+++ b/blake-spr/env.sh
@@ -0,0 +1,14 @@
+export ROOT_DIR=/projects/cwpears/kug-2023/blake-spr
+
+export KOKKOS_SHA=f8788ef2ae1940b627cc6ebc6abeef2c34e7e8dc # 2023 11 30
+export KOKKOS_SRC="$ROOT_DIR/kokkos-${KOKKOS_SHA:0:8}"
+export KOKKOS_BUILD="$ROOT_DIR/kokkos-build-${KOKKOS_SHA:0:8}"
+export KOKKOS_INSTALL="$ROOT_DIR/kokkos-install-${KOKKOS_SHA:0:8}"
+
+export KERNELS_SHA=a80eb9114ddda2d9454e4f3cc8a3dd5143ecdfc8 # 2023 11 30
+export KERNELS_SRC="$ROOT_DIR/kernels-${KERNELS_SHA:0:8}"
+export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}"
+
+export MODULEPATH="$MODULEPATH:/projects/x86-64/modulefiles"
+module load intel/oneAPI/hpc-toolkit/2022.1.2
+module load cmake
\ No newline at end of file
diff --git a/blake-spr/run.sh b/blake-spr/run.sh
new file mode 100755
index 0000000..89174dc
--- /dev/null
+++ b/blake-spr/run.sh
@@ -0,0 +1,24 @@
+#! /bin/bash
+#SBATCH -A csc465
+#SBATCH -N 1
+
+set -eou pipefail
+
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+LOG_DIR="$(date +"%Y%m%d_%H%M%S")_run"
+export LOG_DIR
+mkdir -p "$LOG_DIR"
+
+srun -n1 -t 1 /opt/rocm-5.7.0/bin/rocm-smi \
+  --showdriverversion \
+  --showproductname \
+  --showmclkrange \
+  -v \
+  --showsclkrange \
+  --showfwinfo \
+  |& tee "$LOG_DIR/rocm-smi.log"  || true
+srun -n1 -t 1 /opt/rocm-5.7.0/bin/rocminfo |& tee "$LOG_DIR/rocminfo.log" || true
+srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true
+srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true
+srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true
\ No newline at end of file
diff --git a/frontier/download.sh b/frontier/download.sh
index 12c5ea2..a5eee7f 100755
--- a/frontier/download.sh
+++ b/frontier/download.sh
@@ -14,9 +14,9 @@ git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true
 git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true
 (cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true
 
-srun -n1 -t 1 module list |& tee "$LOG_DIR/module-list.log"
-srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log"
-srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log"
+module list |& tee "$LOG_DIR/module-list.log"
+lscpu |& tee "$LOG_DIR/lscpu.log"
+hostname |& tee "$LOG_DIR/hostname.log"
 
 ## Configure Kokkos
 cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \