diff --git a/.gitignore b/.gitignore index 49e967d..d96a44c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ **/kokkos-* **/kernels-* **/*_run -**/*_build \ No newline at end of file +**/*_build +**/matrix.mm \ No newline at end of file diff --git a/README.md b/README.md index 6c67023..2ae9da4 100644 --- a/README.md +++ b/README.md @@ -8,4 +8,4 @@ Some examples of building and running Kokkos Kernels on Exascale-relevant Hardwa | `blake-spr` | Intel Sapphire Rapids | | `blake-pvc` | Intel Ponte Vecchio | | `blake-h100` | Nvidia H100 | -| `perlmutter` | Nvidia A100 | \ No newline at end of file +| `perlmutter-a100` | Nvidia A100 | \ No newline at end of file diff --git a/perlmutter-a100/download.sh b/perlmutter-a100/download.sh new file mode 100755 index 0000000..a0f9efe --- /dev/null +++ b/perlmutter-a100/download.sh @@ -0,0 +1,56 @@ +#! /bin/bash + +set -eou pipefail + +source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh + +LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build" +export LOG_DIR +mkdir -p "$LOG_DIR" + +git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true +(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true +git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true +(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true + +# re-set up our environment +source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh + +module list |& tee "$LOG_DIR/module-list.log" +lscpu |& tee "$LOG_DIR/lscpu.log" +hostname |& tee "$LOG_DIR/hostname.log" +env |& tee "$LOG_DIR/env.log" || true + +## Configure Kokkos +cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ +-DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \ +-DCMAKE_CXX_STANDARD=17 \ +-DCMAKE_BUILD_TYPE=Release \ +-DCMAKE_CXX_COMPILER="$KOKKOS_SRC"/bin/nvcc_wrapper \ +-DKokkos_ENABLE_CUDA=ON \ +-DKokkos_ARCH_NATIVE=ON \ +-DKokkos_ARCH_AMPERE80=ON \ +|& tee "$LOG_DIR/kokkos-config.log" + +## Build & Install Kokkos +cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install \ +|& tee "$LOG_DIR/kokkos-build.log" + +## Configure Kernels +cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ +-DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \ +-DCMAKE_BUILD_TYPE=Release \ +-DCMAKE_CXX_COMPILER="$KOKKOS_SRC"/bin/nvcc_wrapper \ +-DKokkosKernels_ENABLE_TPL_CUSPARSE=ON \ +-DKokkosKernels_ENABLE_TPL_CUBLASE=ON \ +-DKokkosKernels_ENABLE_TESTS=ON \ +-DKokkosKernels_ENABLE_PERFTESTS=ON \ +-DKokkosKernels_ENABLE_BENCHMARK=ON \ +-DTPLCUBLAS_ROOT=/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/math_libs/11.7/lib64 \ +|& tee "$LOG_DIR/kernels-config.log" + +## Build Kernels +VERBOSE=1 make -C "$KERNELS_BUILD" -j "$(nproc)" \ +KokkosKernels_Blas3_gemm_benchmark \ +KokkosKernels_sparse_spmv_benchmark \ +|& tee "$LOG_DIR/kernels-build.log" diff --git a/perlmutter-a100/env.sh b/perlmutter-a100/env.sh new file mode 100644 index 0000000..6a8de06 --- /dev/null +++ b/perlmutter-a100/env.sh @@ -0,0 +1,11 @@ +export ROOT_DIR=$PSCRATCH/kug-2023/perlmutter-a100 + +export KOKKOS_SHA=f8788ef2ae1940b627cc6ebc6abeef2c34e7e8dc # 2023 11 30 +export KOKKOS_SRC="$ROOT_DIR/kokkos-${KOKKOS_SHA:0:8}" +export KOKKOS_BUILD="$ROOT_DIR/kokkos-build-${KOKKOS_SHA:0:8}" +export KOKKOS_INSTALL="$ROOT_DIR/kokkos-install-${KOKKOS_SHA:0:8}" + +export KERNELS_SHA=a80eb9114ddda2d9454e4f3cc8a3dd5143ecdfc8 # 2023 11 30 +export KERNELS_SRC="$ROOT_DIR/kernels-${KERNELS_SHA:0:8}" +export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}" + diff --git a/perlmutter-a100/run-worker.sh b/perlmutter-a100/run-worker.sh new file mode 100755 index 0000000..55d0242 --- /dev/null +++ b/perlmutter-a100/run-worker.sh @@ -0,0 +1,18 @@ +#! /bin/bash + +set -eou pipefail + +source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh + +LOG_DIR="$(date +"%Y%m%d_%H%M%S")_run" +export LOG_DIR +mkdir -p "$LOG_DIR" + +lscpu |& tee "$LOG_DIR/lscpu.log" || true +hostname |& tee "$LOG_DIR/hostname.log" || true +cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true +env |& tee "$LOG_DIR/env.log" || true + + +"$KERNELS_BUILD"/perf_test/blas/blas3/KokkosKernels_Blas3_gemm_benchmark --cuda 0 |& tee "$LOG_DIR/gemm.log" +"$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f "$ROOT_DIR"/matrix.mm |& tee "$LOG_DIR/spmv.log" diff --git a/perlmutter-a100/run.sh b/perlmutter-a100/run.sh new file mode 100755 index 0000000..6dd4dec --- /dev/null +++ b/perlmutter-a100/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eou pipefail + +srun -A m3953 -C "gpu&hbm40g" --qos regular -N 1 -t 120 ./run-worker.sh