draft perlmutter scripts

.gitignore

@@ -1,4 +1,5 @@
 **/kokkos-*
 **/kernels-*
 **/*_run
-**/*_build
+**/*_build
+**/matrix.mm

README.md

@@ -8,4 +8,4 @@ Some examples of building and running Kokkos Kernels on Exascale-relevant Hardwa
 | `blake-spr` | Intel Sapphire Rapids |
 | `blake-pvc` | Intel Ponte Vecchio |
 | `blake-h100` | Nvidia H100 |
-| `perlmutter` | Nvidia A100 |
+| `perlmutter-a100` | Nvidia A100 |

perlmutter-a100/download.sh

@@ -0,0 +1,56 @@
+#! /bin/bash
+
+set -eou pipefail
+
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build"
+export LOG_DIR
+mkdir -p "$LOG_DIR"
+
+git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true
+(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true
+git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true
+(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true
+
+# re-set up our environment
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+module list |& tee "$LOG_DIR/module-list.log"
+lscpu |& tee "$LOG_DIR/lscpu.log"
+hostname |& tee "$LOG_DIR/hostname.log"
+env |& tee "$LOG_DIR/env.log" || true
+
+## Configure Kokkos
+cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \
+-DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \
+-DCMAKE_CXX_STANDARD=17 \
+-DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_CXX_COMPILER="$KOKKOS_SRC"/bin/nvcc_wrapper \
+-DKokkos_ENABLE_CUDA=ON \
+-DKokkos_ARCH_NATIVE=ON \
+-DKokkos_ARCH_AMPERE80=ON \
+|& tee "$LOG_DIR/kokkos-config.log"
+
+## Build & Install Kokkos
+cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install \
+|& tee "$LOG_DIR/kokkos-build.log"
+
+## Configure Kernels
+cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \
+-DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \
+-DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_CXX_COMPILER="$KOKKOS_SRC"/bin/nvcc_wrapper \
+-DKokkosKernels_ENABLE_TPL_CUSPARSE=ON \
+-DKokkosKernels_ENABLE_TPL_CUBLASE=ON \
+-DKokkosKernels_ENABLE_TESTS=ON \
+-DKokkosKernels_ENABLE_PERFTESTS=ON \
+-DKokkosKernels_ENABLE_BENCHMARK=ON \
+-DTPLCUBLAS_ROOT=/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/math_libs/11.7/lib64 \
+|& tee "$LOG_DIR/kernels-config.log"
+
+## Build Kernels
+VERBOSE=1 make -C "$KERNELS_BUILD" -j "$(nproc)" \
+KokkosKernels_Blas3_gemm_benchmark \
+KokkosKernels_sparse_spmv_benchmark \
+|& tee "$LOG_DIR/kernels-build.log"

perlmutter-a100/env.sh

@@ -0,0 +1,11 @@
+export ROOT_DIR=$PSCRATCH/kug-2023/perlmutter-a100
+
+export KOKKOS_SHA=f8788ef2ae1940b627cc6ebc6abeef2c34e7e8dc # 2023 11 30
+export KOKKOS_SRC="$ROOT_DIR/kokkos-${KOKKOS_SHA:0:8}"
+export KOKKOS_BUILD="$ROOT_DIR/kokkos-build-${KOKKOS_SHA:0:8}"
+export KOKKOS_INSTALL="$ROOT_DIR/kokkos-install-${KOKKOS_SHA:0:8}"
+
+export KERNELS_SHA=a80eb9114ddda2d9454e4f3cc8a3dd5143ecdfc8 # 2023 11 30
+export KERNELS_SRC="$ROOT_DIR/kernels-${KERNELS_SHA:0:8}"
+export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}"
+

perlmutter-a100/run-worker.sh

@@ -0,0 +1,18 @@
+#! /bin/bash
+
+set -eou pipefail
+
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+LOG_DIR="$(date +"%Y%m%d_%H%M%S")_run"
+export LOG_DIR
+mkdir -p "$LOG_DIR"
+
+lscpu |& tee "$LOG_DIR/lscpu.log" || true
+hostname |& tee "$LOG_DIR/hostname.log" || true
+cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true
+env |& tee "$LOG_DIR/env.log" || true
+
+
+"$KERNELS_BUILD"/perf_test/blas/blas3/KokkosKernels_Blas3_gemm_benchmark --cuda 0 |& tee "$LOG_DIR/gemm.log"
+"$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f "$ROOT_DIR"/matrix.mm |& tee "$LOG_DIR/spmv.log"

perlmutter-a100/run.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -eou pipefail
+
+srun -A m3953 -C "gpu&hbm40g" --qos regular -N 1 -t 120 ./run-worker.sh