diff --git a/blake-h100/download.sh b/blake-h100/download.sh index df602eb..268dd1b 100755 --- a/blake-h100/download.sh +++ b/blake-h100/download.sh @@ -21,6 +21,7 @@ source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh module list |& tee "$LOG_DIR/module-list.log" lscpu |& tee "$LOG_DIR/lscpu.log" hostname |& tee "$LOG_DIR/hostname.log" +env |& tee "$LOG_DIR/env.log" || true ## Configure Kokkos cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ @@ -30,10 +31,12 @@ cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ -DCMAKE_CXX_COMPILER="$KOKKOS_SRC"/bin/nvcc_wrapper \ -DKokkos_ENABLE_CUDA=ON \ -DKokkos_ARCH_NATIVE=ON \ --DKokkos_ARCH_HOPPER90=ON +-DKokkos_ARCH_HOPPER90=ON \ +|& tee "$LOG_DIR/kokkos-config.log" ## Build & Install Kokkos -cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install +cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install \ +|& tee "$LOG_DIR/kokkos-build.log" ## Configure Kernels cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ @@ -44,9 +47,11 @@ cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ -DKokkosKernels_ENABLE_TPL_CUBLASE=ON \ -DKokkosKernels_ENABLE_TESTS=ON \ -DKokkosKernels_ENABLE_PERFTESTS=ON \ --DKokkosKernels_ENABLE_BENCHMARK=ON +-DKokkosKernels_ENABLE_BENCHMARK=ON \ +|& tee "$LOG_DIR/kernels-config.log" ## Build Kernels -cmake --build "$KERNELS_BUILD" -j "$(nproc)" - -srun -N 1 -p H100 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm \ No newline at end of file +VERBOSE=1 make -C "$KERNELS_BUILD" -j "$(nproc)" \ +KokkosKernels_Blas3_gemm_benchmark \ +KokkosKernels_sparse_spmv_benchmark \ +|& tee "$LOG_DIR/kernels-build.log" diff --git a/blake-h100/run.sh b/blake-h100/run.sh index edfd78e..5a77b5f 100755 --- a/blake-h100/run.sh +++ b/blake-h100/run.sh @@ -14,8 +14,8 @@ mkdir -p "$LOG_DIR" srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true +srun -n1 -t 1 env |& tee "$LOG_DIR/env.log" || true -srun -N 1 -p H100 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm - -# srun -n1 -t 60 $ROOT/erf_test/blas/blas3/KokkosKernels_Blas3_gemm_benchmark \ No newline at end of file +srun -N 1 -p H100 -n 1 -t 60 "$KERNELS_BUILD"/perf_test/blas/blas3/KokkosKernels_Blas3_gemm_benchmark --cuda 0 |& tee "$LOG_DIR/gemm.log" +srun -N 1 -p H100 -n 1 -t 60 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm |& tee "$LOG_DIR/spmv.log" \ No newline at end of file diff --git a/blake-pvc/download-worker.sh b/blake-pvc/download-worker.sh index fda2f84..220dd62 100755 --- a/blake-pvc/download-worker.sh +++ b/blake-pvc/download-worker.sh @@ -21,6 +21,7 @@ source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh module list |& tee "$LOG_DIR/module-list.log" lscpu |& tee "$LOG_DIR/lscpu.log" hostname |& tee "$LOG_DIR/hostname.log" +env |& tee "$LOG_DIR/env.log" ## Configure Kokkos cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ @@ -29,10 +30,10 @@ cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=icpx \ -DKokkos_ENABLE_SYCL=ON \ --DKokkos_ARCH_NATIVE=ON \ +-DKokkos_ARCH_SPR=ON \ -DKokkos_ARCH_INTEL_PVC=ON \ -DKokkos_ENABLE_ONEDPL=OFF \ --DCMAKE_CXX_FLAGS="-fp-model=precise -fno-finite-math-only" \ +-DCMAKE_CXX_FLAGS="-fp-model=precise -fno-finite-math-only -mavx512f" \ -DBUILD_SHARED_LIBS=ON ## Build & Install Kokkos @@ -46,8 +47,11 @@ cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ -DKokkosKernels_ENABLE_TESTS=ON \ -DKokkosKernels_ENABLE_PERFTESTS=ON \ -DKokkosKernels_ENABLE_BENCHMARK=ON \ --DCMAKE_CXX_FLAGS="-fp-model=precise" \ +-DCMAKE_CXX_FLAGS="-fp-model=precise -mavx512f" \ -DBUILD_SHARED_LIBS=ON ## Build Kernels -cmake --build "$KERNELS_BUILD" -j "$(nproc)" +VERBOSE=1 make -C "$KERNELS_BUILD" -j "$(nproc)" \ +KokkosKernels_Blas3_gemm_benchmark \ +KokkosKernels_sparse_spmv_benchmark \ +|& tee "$LOG_DIR/kernels-build.log" diff --git a/blake-pvc/run.sh b/blake-pvc/run.sh index 49c3d5d..880cb95 100755 --- a/blake-pvc/run.sh +++ b/blake-pvc/run.sh @@ -12,5 +12,7 @@ mkdir -p "$LOG_DIR" srun -N 1 -p PV --exclude=blake15 -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true srun -N 1 -p PV --exclude=blake15 -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true srun -N 1 -p PV --exclude=blake15 -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true +srun -N 1 -p PV --exclude=blake15 -n1 -t 1 env |& tee "$LOG_DIR/env.log" || true -srun -N 1 -p PV --exclude=blake15 -n1 -t 120 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm +srun -N 1 -p PV --exclude=blake15 -n 1 -t 60 "$KERNELS_BUILD"/perf_test/blas/blas3/KokkosKernels_Blas3_gemm_benchmark --sycl 0 |& tee "$LOG_DIR/gemm.log" +srun -N 1 -p PV --exclude=blake15 -n 1 -t 60 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm |& tee "$LOG_DIR/spmv.log" diff --git a/blake-spr/download-worker.sh b/blake-spr/download-worker.sh index c578358..bbff4e7 100755 --- a/blake-spr/download-worker.sh +++ b/blake-spr/download-worker.sh @@ -9,18 +9,19 @@ export LOG_DIR mkdir -p "$LOG_DIR" # intel blows up SSH for some reason? -module del intel/oneAPI/hpc-toolkit/2022.1.2 +# module del intel/oneAPI/hpc-toolkit/2022.1.2 git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true (cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true (cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true # re-set up our environment -source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh +# source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh module list |& tee "$LOG_DIR/module-list.log" lscpu |& tee "$LOG_DIR/lscpu.log" hostname |& tee "$LOG_DIR/hostname.log" +env |& tee "$LOG_DIR/env.log" || true ## Configure Kokkos cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ @@ -28,12 +29,15 @@ cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ -DCMAKE_CXX_STANDARD=17 \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=icpx \ --DKokkos_ENABLE_OPENMP=ON \ +-DKokkos_ENABLE_SERIAL=ON \ +-DKokkos_ENABLE_OPENMP=OFF \ -DKokkos_ARCH_SPR=ON \ --DKokkos_ENABLE_PRAGMA_IVDEP=ON +-DKokkos_ENABLE_PRAGMA_IVDEP=ON \ +|& tee "$LOG_DIR/kokkos-config.log" ## Build & Install Kokkos -cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install +cmake --build "$KOKKOS_BUILD" -j "$(nproc)" -t install \ +|& tee "$LOG_DIR/kokkos-build.log" ## Configure Kernels cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ @@ -43,7 +47,11 @@ cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ -DKokkosKernels_ENABLE_TPL_MKL=ON \ -DKokkosKernels_ENABLE_TESTS=ON \ -DKokkosKernels_ENABLE_PERFTESTS=ON \ --DKokkosKernels_ENABLE_BENCHMARK=ON +-DKokkosKernels_ENABLE_BENCHMARK=ON \ +|& tee "$LOG_DIR/kernels-config.log" ## Build Kernels -VERBOSE=1 cmake --build "$KERNELS_BUILD" -j "$(nproc)" +VERBOSE=1 make -C "$KERNELS_BUILD" -j "$(nproc)" \ +KokkosKernels_Blas3_gemm_benchmark \ +KokkosKernels_sparse_spmv_benchmark \ +|& tee "$LOG_DIR/kernels-build.log" diff --git a/blake-spr/env.sh b/blake-spr/env.sh index ed5a256..c3c7895 100644 --- a/blake-spr/env.sh +++ b/blake-spr/env.sh @@ -9,7 +9,6 @@ export KERNELS_SHA=a80eb9114ddda2d9454e4f3cc8a3dd5143ecdfc8 # 2023 11 30 export KERNELS_SRC="$ROOT_DIR/kernels-${KERNELS_SHA:0:8}" export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}" -export MODULEPATH="$MODULEPATH:/projects/x86-64/modulefiles" -module load intel/oneAPI/hpc-toolkit/2022.1.2 +source /projects/x86-64-icelake-rocky8/spack-config/blake-setup-user-module-env.sh module load cmake -module load gcc/7.2.0 \ No newline at end of file +module load intel-oneapi-compilers/2023.1.0 intel-oneapi-dpl/2022.1.0 intel-oneapi-mkl/2023.1.0 \ No newline at end of file diff --git a/blake-spr/run.sh b/blake-spr/run.sh index a99216c..9737070 100755 --- a/blake-spr/run.sh +++ b/blake-spr/run.sh @@ -12,5 +12,7 @@ mkdir -p "$LOG_DIR" srun -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true srun -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true srun -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true +srun -n1 -t 1 env |& tee "$LOG_DIR/env.log" || true -srun -N 1 -p all -n1 -t 120 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm \ No newline at end of file +srun -N 1 -p all -n1 -t 60 "$KERNELS_BUILD"/perf_test/blas/blas3/KokkosKernels_Blas3_gemm_benchmark |& tee "$LOG_DIR/gemm.log" +srun -N 1 -p all -n1 -t 60 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm |& tee "$LOG_DIR/spmv.log" \ No newline at end of file