From 0b7739471a968216803860610e1e9e54f2c2a913 Mon Sep 17 00:00:00 2001 From: Carl Pearson Date: Mon, 4 Dec 2023 16:48:30 -0700 Subject: [PATCH] . --- blake-h100/run.sh | 9 ++++-- blake-pvc/download-worker.sh | 53 ++++++++++++++++++++++++++++++++++++ blake-pvc/download.sh | 53 +----------------------------------- blake-pvc/env.sh | 4 ++- blake-pvc/run.sh | 10 +++---- blake-spr/download-worker.sh | 49 +++++++++++++++++++++++++++++++++ blake-spr/download.sh | 49 +-------------------------------- blake-spr/env.sh | 3 +- blake-spr/run.sh | 6 ++-- 9 files changed, 124 insertions(+), 112 deletions(-) create mode 100755 blake-pvc/download-worker.sh create mode 100755 blake-spr/download-worker.sh diff --git a/blake-h100/run.sh b/blake-h100/run.sh index e1e9b51..edfd78e 100755 --- a/blake-h100/run.sh +++ b/blake-h100/run.sh @@ -1,6 +1,6 @@ #! /bin/bash #SBATCH -N 1 -#SBATCH -p all +#SBATCH -p H100 set -eou pipefail @@ -13,4 +13,9 @@ mkdir -p "$LOG_DIR" srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true -srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true \ No newline at end of file +srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true + + +srun -N 1 -p H100 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm + +# srun -n1 -t 60 $ROOT/erf_test/blas/blas3/KokkosKernels_Blas3_gemm_benchmark \ No newline at end of file diff --git a/blake-pvc/download-worker.sh b/blake-pvc/download-worker.sh new file mode 100755 index 0000000..fda2f84 --- /dev/null +++ b/blake-pvc/download-worker.sh @@ -0,0 +1,53 @@ +#! /bin/bash + +set -eou pipefail + +source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh + +LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build" +export LOG_DIR +mkdir -p "$LOG_DIR" + +# intel blows up SSH for some reason? +module del intel/oneAPI/hpc-toolkit/2022.1.2 +git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true +(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true +git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true +(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true + +# re-set up our environment +source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh + +module list |& tee "$LOG_DIR/module-list.log" +lscpu |& tee "$LOG_DIR/lscpu.log" +hostname |& tee "$LOG_DIR/hostname.log" + +## Configure Kokkos +cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ +-DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \ +-DCMAKE_CXX_STANDARD=17 \ +-DCMAKE_BUILD_TYPE=Release \ +-DCMAKE_CXX_COMPILER=icpx \ +-DKokkos_ENABLE_SYCL=ON \ +-DKokkos_ARCH_NATIVE=ON \ +-DKokkos_ARCH_INTEL_PVC=ON \ +-DKokkos_ENABLE_ONEDPL=OFF \ +-DCMAKE_CXX_FLAGS="-fp-model=precise -fno-finite-math-only" \ +-DBUILD_SHARED_LIBS=ON + +## Build & Install Kokkos +cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install + +## Configure Kernels +cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ +-DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \ +-DCMAKE_BUILD_TYPE=Release \ +-DCMAKE_CXX_COMPILER=icpx \ +-DKokkosKernels_ENABLE_TESTS=ON \ +-DKokkosKernels_ENABLE_PERFTESTS=ON \ +-DKokkosKernels_ENABLE_BENCHMARK=ON \ +-DCMAKE_CXX_FLAGS="-fp-model=precise" \ +-DBUILD_SHARED_LIBS=ON + +## Build Kernels +cmake --build "$KERNELS_BUILD" -j "$(nproc)" diff --git a/blake-pvc/download.sh b/blake-pvc/download.sh index 6d0b4cd..0d219bf 100755 --- a/blake-pvc/download.sh +++ b/blake-pvc/download.sh @@ -1,54 +1,3 @@ #! /bin/bash -set -eou pipefail - -source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh - -LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build" -export LOG_DIR -mkdir -p "$LOG_DIR" - -# intel blows up SSH for some reason? -module del intel/oneAPI/hpc-toolkit/2022.1.2 -git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true -(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true -git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true -(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true - -# re-set up our environment -source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh - -module list |& tee "$LOG_DIR/module-list.log" -lscpu |& tee "$LOG_DIR/lscpu.log" -hostname |& tee "$LOG_DIR/hostname.log" - -## Configure Kokkos -cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ --DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \ --DCMAKE_CXX_STANDARD=17 \ --DCMAKE_BUILD_TYPE=Release \ --DCMAKE_CXX_COMPILER=icpx \ --DKokkos_ENABLE_SYCL=ON \ --DKokkos_ARCH_NATIVE=ON \ --DKokkos_ARCH_INTEL_PVC=ON \ --DBUILD_SHARED_LIBS=ON - - -## Build & Install Kokkos -cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install - -## Configure Kernels -cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ --DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \ --DCMAKE_BUILD_TYPE=Release \ --DCMAKE_CXX_COMPILER=icpx \ --DKokkosKernels_ENABLE_TESTS=ON \ --DKokkosKernels_ENABLE_PERFTESTS=ON \ --DKokkosKernels_ENABLE_BENCHMARK=ON \ --DCMAKE_CXX_FLAGS="-fp-model=precise" \ --DBUILD_SHARED_LIBS=ON - -## Build Kernels -cmake --build "$KERNELS_BUILD" -j "$(nproc)" - -srun -N 1 -p PV "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm \ No newline at end of file +srun -N 1 -p PV --exclude=blake15 -n 1 -t 120 ./download-worker.sh \ No newline at end of file diff --git a/blake-pvc/env.sh b/blake-pvc/env.sh index 7051ab2..41f0626 100644 --- a/blake-pvc/env.sh +++ b/blake-pvc/env.sh @@ -1,17 +1,19 @@ export ROOT_DIR=/projects/cwpears/kug-2023/blake-pvc export KOKKOS_SHA=f8788ef2ae1940b627cc6ebc6abeef2c34e7e8dc # 2023 11 30 +# export KOKKOS_SHA="4.2.00" export KOKKOS_SRC="$ROOT_DIR/kokkos-${KOKKOS_SHA:0:8}" export KOKKOS_BUILD="$ROOT_DIR/kokkos-build-${KOKKOS_SHA:0:8}" export KOKKOS_INSTALL="$ROOT_DIR/kokkos-install-${KOKKOS_SHA:0:8}" export KERNELS_SHA=a80eb9114ddda2d9454e4f3cc8a3dd5143ecdfc8 # 2023 11 30 +# export KERNELS_SHA="4.2.00" export KERNELS_SRC="$ROOT_DIR/kernels-${KERNELS_SHA:0:8}" export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}" source /projects/x86-64-icelake-rocky8/spack-config/blake-setup-user-module-env.sh -module load intel-oneapi-compilers/2023.1.0 intel-oneapi-dpl/2022.1.0 module load cmake +module load intel-oneapi-compilers/2023.1.0 intel-oneapi-dpl/2022.1.0 intel-oneapi-mkl/2023.1.0 # Required for the hashmap accumulator export ZES_ENABLE_SYSMAN=1 \ No newline at end of file diff --git a/blake-pvc/run.sh b/blake-pvc/run.sh index e1e9b51..49c3d5d 100755 --- a/blake-pvc/run.sh +++ b/blake-pvc/run.sh @@ -1,6 +1,4 @@ #! /bin/bash -#SBATCH -N 1 -#SBATCH -p all set -eou pipefail @@ -11,6 +9,8 @@ export LOG_DIR mkdir -p "$LOG_DIR" -srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true -srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true -srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true \ No newline at end of file +srun -N 1 -p PV --exclude=blake15 -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true +srun -N 1 -p PV --exclude=blake15 -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true +srun -N 1 -p PV --exclude=blake15 -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true + +srun -N 1 -p PV --exclude=blake15 -n1 -t 120 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm diff --git a/blake-spr/download-worker.sh b/blake-spr/download-worker.sh new file mode 100755 index 0000000..c578358 --- /dev/null +++ b/blake-spr/download-worker.sh @@ -0,0 +1,49 @@ +#! /bin/bash + +set -eou pipefail + +source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh + +LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build" +export LOG_DIR +mkdir -p "$LOG_DIR" + +# intel blows up SSH for some reason? +module del intel/oneAPI/hpc-toolkit/2022.1.2 +git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true +(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true +git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true +(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true + +# re-set up our environment +source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh + +module list |& tee "$LOG_DIR/module-list.log" +lscpu |& tee "$LOG_DIR/lscpu.log" +hostname |& tee "$LOG_DIR/hostname.log" + +## Configure Kokkos +cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ +-DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \ +-DCMAKE_CXX_STANDARD=17 \ +-DCMAKE_BUILD_TYPE=Release \ +-DCMAKE_CXX_COMPILER=icpx \ +-DKokkos_ENABLE_OPENMP=ON \ +-DKokkos_ARCH_SPR=ON \ +-DKokkos_ENABLE_PRAGMA_IVDEP=ON + +## Build & Install Kokkos +cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install + +## Configure Kernels +cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ +-DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \ +-DCMAKE_BUILD_TYPE=Release \ +-DCMAKE_CXX_COMPILER=icpx \ +-DKokkosKernels_ENABLE_TPL_MKL=ON \ +-DKokkosKernels_ENABLE_TESTS=ON \ +-DKokkosKernels_ENABLE_PERFTESTS=ON \ +-DKokkosKernels_ENABLE_BENCHMARK=ON + +## Build Kernels +VERBOSE=1 cmake --build "$KERNELS_BUILD" -j "$(nproc)" diff --git a/blake-spr/download.sh b/blake-spr/download.sh index bdc156c..27425e7 100755 --- a/blake-spr/download.sh +++ b/blake-spr/download.sh @@ -1,50 +1,3 @@ #! /bin/bash -set -eou pipefail - -source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh - -LOG_DIR="$(date +"%Y%m%d_%H%M%S")_build" -export LOG_DIR -mkdir -p "$LOG_DIR" - -# intel blows up SSH for some reason? -module del intel/oneAPI/hpc-toolkit/2022.1.2 -git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true -(cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true -git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true -(cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true - -# re-set up our environment -source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh - -module list |& tee "$LOG_DIR/module-list.log" -lscpu |& tee "$LOG_DIR/lscpu.log" -hostname |& tee "$LOG_DIR/hostname.log" - -## Configure Kokkos -cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ --DCMAKE_INSTALL_PREFIX="$KOKKOS_INSTALL" \ --DCMAKE_CXX_STANDARD=17 \ --DCMAKE_BUILD_TYPE=Release \ --DCMAKE_CXX_COMPILER=icpx \ --DKokkos_ENABLE_OPENMP=ON \ --DKokkos_ARCH_NATIVE=ON - -## Build & Install Kokkos -cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install - -## Configure Kernels -cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ --DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \ --DCMAKE_BUILD_TYPE=Release \ --DCMAKE_CXX_COMPILER=icpx \ --DKokkosKernels_ENABLE_TPL_MKL=ON \ --DKokkosKernels_ENABLE_TESTS=ON \ --DKokkosKernels_ENABLE_PERFTESTS=ON \ --DKokkosKernels_ENABLE_BENCHMARK=ON - -## Build Kernels -cmake --build "$KERNELS_BUILD" -j "$(nproc)" - -srun -N 1 -p H100 "$KERNELS_BUILD"/perf_test/sparse/sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm \ No newline at end of file +srun -N 1 -p all -n 1 -t 120 ./download-worker.sh \ No newline at end of file diff --git a/blake-spr/env.sh b/blake-spr/env.sh index 231c733..ed5a256 100644 --- a/blake-spr/env.sh +++ b/blake-spr/env.sh @@ -11,4 +11,5 @@ export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}" export MODULEPATH="$MODULEPATH:/projects/x86-64/modulefiles" module load intel/oneAPI/hpc-toolkit/2022.1.2 -module load cmake \ No newline at end of file +module load cmake +module load gcc/7.2.0 \ No newline at end of file diff --git a/blake-spr/run.sh b/blake-spr/run.sh index e1e9b51..a99216c 100755 --- a/blake-spr/run.sh +++ b/blake-spr/run.sh @@ -1,6 +1,4 @@ #! /bin/bash -#SBATCH -N 1 -#SBATCH -p all set -eou pipefail @@ -13,4 +11,6 @@ mkdir -p "$LOG_DIR" srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true -srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true \ No newline at end of file +srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true + +srun -N 1 -p all -n1 -t 120 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm \ No newline at end of file