PVC and H100 builds

blake-h100/download.sh

@@ -0,0 +1,47 @@
+#! /bin/bash
+
+set -eou pipefail
+
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build"
+export LOG_DIR
+mkdir -p "$LOG_DIR"
+
+# intel blows up SSH for some reason?
+module del intel/oneAPI/hpc-toolkit/2022.1.2
+git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true
+(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true
+git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true
+(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true
+
+# re-set up our environment
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+module list |& tee "$LOG_DIR/module-list.log"
+lscpu |& tee "$LOG_DIR/lscpu.log"
+hostname |& tee "$LOG_DIR/hostname.log"
+
+## Configure Kokkos
+cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \
+-DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \
+-DCMAKE_CXX_STANDARD=17 \
+-DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_CXX_COMPILER="$KOKKOS_SRC"/bin/nvcc_wrapper \
+-DKokkos_ENABLE_CUDA=ON \
+-DKokkos_ARCH_NATIVE=ON \
+-DKokkos_ARCH_HOPPER90=ON
+
+## Build & Install Kokkos
+cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install
+
+## Configure Kernels
+cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \
+-DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \
+-DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_CXX_COMPILER="$KOKKOS_SRC"/bin/nvcc_wrapper \
+-DKokkosKernels_ENABLE_TPL_CUSPARSE=ON \
+-DKokkosKernels_ENABLE_TPL_CUBLASE=ON
+
+## Build Kernels
+cmake --build "$KERNELS_BUILD" -j "$(nproc)"

blake-h100/env.sh

@@ -0,0 +1,14 @@
+export ROOT_DIR=/projects/cwpears/kug-2023/blake-h100
+
+export KOKKOS_SHA=f8788ef2ae1940b627cc6ebc6abeef2c34e7e8dc # 2023 11 30
+export KOKKOS_SRC="$ROOT_DIR/kokkos-${KOKKOS_SHA:0:8}"
+export KOKKOS_BUILD="$ROOT_DIR/kokkos-build-${KOKKOS_SHA:0:8}"
+export KOKKOS_INSTALL="$ROOT_DIR/kokkos-install-${KOKKOS_SHA:0:8}"
+
+export KERNELS_SHA=a80eb9114ddda2d9454e4f3cc8a3dd5143ecdfc8 # 2023 11 30
+export KERNELS_SRC="$ROOT_DIR/kernels-${KERNELS_SHA:0:8}"
+export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}"
+
+source /projects/x86-64-icelake-rocky8/spack-config/blake-setup-user-module-env.sh
+module load gcc/11.3.0 cuda/11.8.0
+module load cmake

blake-h100/run.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+#SBATCH -N 1
+#SBATCH -p all
+
+set -eou pipefail
+
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+LOG_DIR="$(date +"%Y%m%d_%H%M%S")_run"
+export LOG_DIR
+mkdir -p "$LOG_DIR"
+
+
+srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true
+srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true
+srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true

blake-pvc/download.sh

@@ -0,0 +1,48 @@
+#! /bin/bash
+
+set -eou pipefail
+
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build"
+export LOG_DIR
+mkdir -p "$LOG_DIR"
+
+# intel blows up SSH for some reason?
+module del intel/oneAPI/hpc-toolkit/2022.1.2
+git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true
+(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true
+git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true
+(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true
+
+# re-set up our environment
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+module list |& tee "$LOG_DIR/module-list.log"
+lscpu |& tee "$LOG_DIR/lscpu.log"
+hostname |& tee "$LOG_DIR/hostname.log"
+
+## Configure Kokkos
+cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \
+-DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \
+-DCMAKE_CXX_STANDARD=17 \
+-DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_CXX_COMPILER=icpx \
+-DKokkos_ENABLE_SYCL=ON \
+-DKokkos_ARCH_NATIVE=ON \
+-DKokkos_ARCH_INTEL_PVC=ON \
+
+
+## Build & Install Kokkos
+cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install
+
+## Configure Kernels
+cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \
+-DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \
+-DCMAKE_BUILD_TYPE=Release \
+-DCMAKE_CXX_COMPILER=icpx \
+-DKokkosKernels_ENABLE_TPL_MKL=ON \
+-DCMAKE_CXX_FLAGS="-fp-model=precise"
+
+## Build Kernels
+cmake --build "$KERNELS_BUILD" -j "$(nproc)"

blake-pvc/env.sh

@@ -0,0 +1,17 @@
+export ROOT_DIR=/projects/cwpears/kug-2023/blake-pvc
+
+export KOKKOS_SHA=f8788ef2ae1940b627cc6ebc6abeef2c34e7e8dc # 2023 11 30
+export KOKKOS_SRC="$ROOT_DIR/kokkos-${KOKKOS_SHA:0:8}"
+export KOKKOS_BUILD="$ROOT_DIR/kokkos-build-${KOKKOS_SHA:0:8}"
+export KOKKOS_INSTALL="$ROOT_DIR/kokkos-install-${KOKKOS_SHA:0:8}"
+
+export KERNELS_SHA=a80eb9114ddda2d9454e4f3cc8a3dd5143ecdfc8 # 2023 11 30
+export KERNELS_SRC="$ROOT_DIR/kernels-${KERNELS_SHA:0:8}"
+export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}"
+
+source /projects/x86-64-icelake-rocky8/spack-config/blake-setup-user-module-env.sh
+module load intel-oneapi-compilers/2023.1.0 intel-oneapi-dpl/2022.1.0
+module load cmake
+
+# Required for the hashmap accumulator
+export ZES_ENABLE_SYSMAN=1

blake-pvc/run.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+#SBATCH -N 1
+#SBATCH -p all
+
+set -eou pipefail
+
+source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh
+
+LOG_DIR="$(date +"%Y%m%d_%H%M%S")_run"
+export LOG_DIR
+mkdir -p "$LOG_DIR"
+
+
+srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true
+srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true
+srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true

blake-spr/run.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
-#SBATCH -A csc465
 #SBATCH -N 1
+#SBATCH -p all
 
 set -eou pipefail
 
@@ -10,15 +10,7 @@ LOG_DIR="$(date +"%Y%m%d_%H%M%S")_run"
 export LOG_DIR
 mkdir -p "$LOG_DIR"
 
-srun -n1 -t 1 /opt/rocm-5.7.0/bin/rocm-smi \
+
-  --showdriverversion \
-  --showproductname \
-  --showmclkrange \
-  -v \
-  --showsclkrange \
-  --showfwinfo \
-  |& tee "$LOG_DIR/rocm-smi.log"  || true
-srun -n1 -t 1 /opt/rocm-5.7.0/bin/rocminfo |& tee "$LOG_DIR/rocminfo.log" || true
 srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true
 srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true
 srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true