diff --git a/blake-pvc/download-worker.sh b/blake-pvc/download-worker.sh index 220dd62..886f0e1 100755 --- a/blake-pvc/download-worker.sh +++ b/blake-pvc/download-worker.sh @@ -9,19 +9,31 @@ export LOG_DIR mkdir -p "$LOG_DIR" # intel blows up SSH for some reason? -module del intel/oneAPI/hpc-toolkit/2022.1.2 +#module del intel/oneAPI/hpc-toolkit/2022.1.2 git clone git@github.com:kokkos/kokkos.git "$KOKKOS_SRC" || true (cd "$KOKKOS_SRC" && git checkout $KOKKOS_SHA) || true -git clone git@github.com:kokkos/kokkos-kernels.git "$KERNELS_SRC" || true +git clone "$KERNELS_REMOTE" "$KERNELS_SRC" || true (cd "$KERNELS_SRC" && git checkout $KERNELS_SHA) || true +git clone "$PTI_REMOTE" "$PTI_SRC" || true +(cd "$PTI_SRC" && git checkout $PTI_SHA) || true # re-set up our environment -source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh +#source "$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/env.sh module list |& tee "$LOG_DIR/module-list.log" lscpu |& tee "$LOG_DIR/lscpu.log" hostname |& tee "$LOG_DIR/hostname.log" -env |& tee "$LOG_DIR/env.log" +env 2>&1 >> "$LOG_DIR/env.log" + +## Configure onetrace +cmake -S "$PTI_SRC/tools/onetrace" -B "$PTI_BUILD" \ +-DCMAKE_BUILD_TYPE=Release \ +-DCMAKE_CXX_COMPILER=icpx \ +|& tee "$LOG_DIR"/onetrace-config.log + +## build onetrace +make -C "$PTI_BUILD" \ +|& tee "$LOG_DIR/onetrace-build.log" ## Configure Kokkos cmake -S "$KOKKOS_SRC" -B "$KOKKOS_BUILD" \ @@ -44,6 +56,7 @@ cmake -S "$KERNELS_SRC" -B "$KERNELS_BUILD" \ -DKokkos_DIR="$KOKKOS_INSTALL/lib64/cmake/Kokkos" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=icpx \ +-DKokkosKernels_ENABLE_TPL_MKL=ON \ -DKokkosKernels_ENABLE_TESTS=ON \ -DKokkosKernels_ENABLE_PERFTESTS=ON \ -DKokkosKernels_ENABLE_BENCHMARK=ON \ diff --git a/blake-pvc/env.sh b/blake-pvc/env.sh index db5b2e5..4838287 100644 --- a/blake-pvc/env.sh +++ b/blake-pvc/env.sh @@ -5,13 +5,19 @@ export KOKKOS_SRC="$ROOT_DIR/kokkos-${KOKKOS_SHA:0:8}" export KOKKOS_BUILD="$ROOT_DIR/kokkos-build-${KOKKOS_SHA:0:8}" export KOKKOS_INSTALL="$ROOT_DIR/kokkos-install-${KOKKOS_SHA:0:8}" -export KERNELS_SHA="4ce619e161efd731b87df4160821328cff17a4ee" # 2023 12 06 +export KERNELS_REMOTE="git@github.com:cwpearson/kokkos-kernels.git" +export KERNELS_SHA=a91a1f24560692d6a294e764f82f497dc10aeb7d # 2023 12 08 export KERNELS_SRC="$ROOT_DIR/kernels-${KERNELS_SHA:0:8}" export KERNELS_BUILD="$ROOT_DIR/kernels-build-${KERNELS_SHA:0:8}" +export PTI_REMOTE="git@github.com:intel/pti-gpu.git" +export PTI_SHA="f27829883defd69c18e7dc2e4cdd727a2483c40c" # 2023 12 11 +export PTI_SRC="$ROOT_DIR/pti-gpu-${PTI_SHA:0:8}" +export PTI_BUILD="$ROOT_DIR/pti-gpu-build-${PTI_SHA:0:8}" + source /projects/x86-64-icelake-rocky8/spack-config/blake-setup-user-module-env.sh module load cmake module load intel-oneapi-compilers/2023.1.0 intel-oneapi-dpl/2022.1.0 intel-oneapi-mkl/2023.1.0 # Required for the hashmap accumulator -export ZES_ENABLE_SYSMAN=1 \ No newline at end of file +export ZES_ENABLE_SYSMAN=1 diff --git a/blake-pvc/run.sh b/blake-pvc/run.sh index 880cb95..332d945 100755 --- a/blake-pvc/run.sh +++ b/blake-pvc/run.sh @@ -11,8 +11,10 @@ mkdir -p "$LOG_DIR" srun -N 1 -p PV --exclude=blake15 -n1 -t 1 lscpu |& tee "$LOG_DIR/lscpu.log" || true srun -N 1 -p PV --exclude=blake15 -n1 -t 1 hostname |& tee "$LOG_DIR/hostname.log" || true -srun -N 1 -p PV --exclude=blake15 -n1 -t 1 cat /proc/cpuinfo |& tee "$LOG_DIR/cpuinfo.log" || true -srun -N 1 -p PV --exclude=blake15 -n1 -t 1 env |& tee "$LOG_DIR/env.log" || true +srun -N 1 -p PV --exclude=blake15 -n1 -t 1 cat /proc/cpuinfo 2>&1 > "$LOG_DIR/cpuinfo.log" || true +srun -N 1 -p PV --exclude=blake15 -n1 -t 1 env 2>&1 > "$LOG_DIR/env.log" || true +srun -N 1 -p PV --exclude=blake15 -n1 -t 60 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/kug-2023/dielFilterV3real/dielFilterV3real.mtx |& tee "$LOG_DIR/spmv2.log" +srun -N 1 -p PV --exclude=blake15 -n1 -t 60 "$PTI_BUILD"/onetrace -d "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/kug-2023/dielFilterV3real/dielFilterV3real.mtx |& tee "$LOG_DIR/spmv2-onetrace.log" srun -N 1 -p PV --exclude=blake15 -n 1 -t 60 "$KERNELS_BUILD"/perf_test/blas/blas3/KokkosKernels_Blas3_gemm_benchmark --sycl 0 |& tee "$LOG_DIR/gemm.log" -srun -N 1 -p PV --exclude=blake15 -n 1 -t 60 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm |& tee "$LOG_DIR/spmv.log" +srun -N 1 -p PV --exclude=blake15 -n 1 -t 60 "$KERNELS_BUILD"/perf_test/sparse/KokkosKernels_sparse_spmv_benchmark -f /projects/cwpears/sparc_gpu_problems/single_gpu/matrix.mm |& tee "$LOG_DIR/spmv1.log"