diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index b2a03985..402cff84 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -81,6 +81,11 @@ jobs: # SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU # RUNS_ON: ubuntu-latest + - name: Ubuntu (24.04, amdclang 19 + ROCm 6.4.3) + DOCKER_REPOSITORY: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3 + TPL_DOCKERFILE: docker/tpl-ubuntu-hip.Dockerfile + DOCKER_ROOT_IMAGE: rocm/dev-ubuntu-24.04:6.4.3 + RUNS_ON: ubuntu-latest steps: - name: Checkout diff --git a/docker/spack.yaml b/docker/spack.yaml index 1a977a5c..40da6744 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -6,8 +6,7 @@ spack: all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' misc_cache: $spack/../misc_cache test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage + build_stage: $spack/../build_stage # Regular TPLs do not need views view: false @@ -18,6 +17,15 @@ spack: - ../versions.yaml toolchains: + amdclang-19: + - spec: '%[virtuals=c]llvm-amdgpu@6.4.3' + when: '%c' + - spec: '%[virtuals=cxx]llvm-amdgpu@6.4.3' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' clang-15: - spec: '%[virtuals=c]llvm@15.0.7+clang~flang~lld~lldb' when: '%c' @@ -80,15 +88,26 @@ spack: when: '%fortran' - spec: '%openmpi@4.1.2' when: '%mpi' + gcc-13: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@13.3.0' + when: '%c' + - spec: '%cxx=gcc@13.3.0' + when: '%cxx' + - spec: '%fortran=gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' packages: all: target: [x86_64] + # Require mpi: require: - openmpi - blas: require: - "netlib-lapack" @@ -96,6 +115,7 @@ spack: require: - "netlib-lapack" + # Compiler packages llvm: externals: - spec: llvm@10.0.0+clang~flang~lld~lldb @@ -110,7 +130,15 @@ spack: compilers: c: /usr/bin/clang-15 cxx: /usr/bin/clang++-15 - + llvm-amdgpu: + buildable: false + externals: + - spec: llvm-amdgpu@6.4.3 + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/amdclang + cxx: /usr/bin/amdclang++ gcc: externals: - spec: gcc@9.5.0 languages:='c,c++,fortran' @@ -141,7 +169,118 @@ spack: c: /usr/bin/gcc-12 cxx: /usr/bin/g++-12 fortran: /usr/bin/gfortran-12 + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + fortran: /usr/bin/gfortran-13 + + # MPI + openmpi: + externals: + - spec: openmpi@4.1.2 %llvm + prefix: /usr/lib/x86_64-linux-gnu/openmpi + modules: [mpi] + - spec: openmpi@4.1.2 %llvm-amdgpu@6.4.3 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@9.5.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@10.5.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@11.4.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@12.3.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@13.3.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + + # CUDA packages + cuda: + buildable: False + externals: + - spec: cuda@11.8.0 +allow-unsupported-compilers + prefix: /usr/local/cuda + + # ROCm packages + hip: + buildable: false + externals: + - spec: hip@6.4.3 + prefix: /opt/rocm-6.4.3 + rocprim: + buildable: false + externals: + - spec: rocprim@6.4.3 + prefix: /opt/rocm-6.4.3 + rocsparse: + buildable: false + externals: + - spec: rocsparse@6.4.3 + prefix: /opt/rocm-6.4.3 + roctracer: + buildable: false + externals: + - spec: roctracer@6.4.3 + prefix: /opt/rocm-6.4.3 + rocblas: + buildable: false + externals: + - spec: rocblas@6.4.3 + prefix: /opt/rocm-6.4.3 + rocrand: + buildable: false + externals: + - spec: rocrand@6.4.3 + prefix: /opt/rocm-6.4.3 + rocsolver: + buildable: false + externals: + - spec: rocsolver@6.4.3 + prefix: /opt/rocm-6.4.3 + rocthrust: + buildable: false + externals: + - spec: rocthrust@6.4.3 + prefix: /opt/rocm-6.4.3 + hipblas: + buildable: false + externals: + - spec: hipblas@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + hipsparse: + buildable: false + externals: + - spec: hipsparse@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + hipfft: + buildable: false + externals: + - spec: hipfft@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + hipsolver: + buildable: false + externals: + - spec: hipsolver@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + hiprand: + buildable: false + externals: + - spec: hiprand@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + rocm-device-libs: + buildable: false + externals: + - spec: rocm-device-libs@6.4.3 + prefix: /opt/rocm-6.4.3 + hsa-rocr-dev: + buildable: false + externals: + - spec: hsa-rocr-dev@6.4.3 + prefix: /opt/rocm-6.4.3 + # System packages autoconf: version: [2.71] buildable: false @@ -160,11 +299,6 @@ spack: externals: - spec: cmake@3.28.3 prefix: /usr/local - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/local/cuda findutils: version: [4.7.0] buildable: false @@ -181,29 +315,10 @@ spack: externals: - spec: mpfr@6.0.2 prefix: /usr - netlib-lapack: buildable: false externals: - - spec: netlib-lapack@3.10.0 - prefix: /usr - netlib-blas: - buildable: false - externals: - - spec: netlib-blas@3.10.0 - prefix: /usr - openmpi: - externals: - - spec: openmpi@4.1.2 %llvm - prefix: /usr - modules: [mpi] - - spec: openmpi@4.1.2 %gcc@9.5.0 - prefix: /usr - - spec: openmpi@4.1.2 %gcc@10.5.0 - prefix: /usr - - spec: openmpi@4.1.2 %gcc@11.4.0 - prefix: /usr - - spec: openmpi@4.1.2 %gcc@12.3.0 + - spec: netlib-lapack@3.12.0 prefix: /usr perl: buildable: false @@ -250,3 +365,59 @@ spack: externals: - spec: zlib@1.2.11 prefix: /usr + z3: + buildable: false + externals: + - spec: z3@4.8.12 + prefix: /usr + gettext: + buildable: false + externals: + - spec: gettext@0.21 + prefix: /usr + bison: + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /usr + # Rely on spack to build a newer flex for scotch + # flex: + # buildable: false + # externals: + # - spec: flex@2.6.4 + # prefix: /usr + bzip2: + buildable: false + externals: + - spec: bzip2@1.0.8 + prefix: /usr + help2man: + buildable: false + externals: + - spec: help2man@1.49.3 + prefix: /usr + libtool: + buildable: false + externals: + - spec: libtool@2.4.7 + prefix: /usr + gmp: + buildable: false + externals: + - spec: gmp@6.3.0 + prefix: /usr + pugixml: + buildable: false + externals: + - spec: pugixml@1.13.0 + prefix: /usr + gmake: + buildable: false + externals: + - spec: gmake@4.4.1 + prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.11.1 + prefix: /usr diff --git a/docker/tpl-centos-gcc-cuda.Dockerfile b/docker/tpl-centos-gcc-cuda.Dockerfile index f99c9f19..dcfbbe0e 100644 --- a/docker/tpl-centos-gcc-cuda.Dockerfile +++ b/docker/tpl-centos-gcc-cuda.Dockerfile @@ -1,5 +1,5 @@ # NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments -ARG TMP_DIR=/tmp +ARG TMP_DIR=/tmp ARG SRC_DIR=$TMP_DIR/thirdPartyLibs ARG BLD_DIR=$TMP_DIR/build @@ -11,12 +11,12 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo + sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo # Using gcc 8.3.1 provided by the Software Collections (SCL). RUN yum install -y \ centos-release-scl - + # Modify the SCLo repository configuration RUN sed -i 's|^mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo && \ sed -i 's|^baseurl=http://mirror.centos.org/centos/\$releasever/sclo/\$basearch/rh|baseurl=http://vault.centos.org/7.9.2009/sclo/x86_64/rh|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo && \ @@ -25,7 +25,7 @@ RUN sed -i 's|^mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo # Install necessary tools and update the system RUN yum -y update && \ - yum -y install yum-utils + yum -y install yum-utils RUN yum install -y \ devtoolset-8-gcc \ @@ -48,6 +48,10 @@ RUN yum -y install \ # xz \ unzip \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ gnupg \ && pip3 install virtualenv @@ -65,6 +69,8 @@ ARG BLD_DIR RUN yum install -y \ tbb-devel \ make \ + autopoint \ + autotools-dev \ bc \ file \ patch \ diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index f761368c..ede0c379 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -13,7 +13,7 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR RUN dnf clean all && \ dnf -y update && \ dnf -y install \ - which \ + which \ clang-17.0.6 \ gcc-toolset-13 \ python3 \ @@ -28,7 +28,11 @@ RUN dnf clean all && \ unzip \ mpfr-devel \ bzip2 \ + flex \ + bison \ + gettext \ gnupg \ + libtool \ xz \ python3-virtualenv diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index d73e87ff..b5a6127b 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -26,7 +26,11 @@ RUN dnf clean all && \ unzip \ mpfr-devel \ bzip2 \ + flex \ + bison \ + gettext \ gnupg \ + libtool \ xz \ python3-virtualenv diff --git a/docker/tpl-ubuntu-clang-cuda.Dockerfile b/docker/tpl-ubuntu-clang-cuda.Dockerfile index 1a1b3628..37c3a98b 100644 --- a/docker/tpl-ubuntu-clang-cuda.Dockerfile +++ b/docker/tpl-ubuntu-clang-cuda.Dockerfile @@ -34,6 +34,11 @@ RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \ libmpfr-dev \ lbzip2 \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ + libtool \ gnupg \ virtualenv @@ -52,10 +57,25 @@ RUN apt-get install -y --no-install-recommends \ libtbb-dev \ bc \ file \ + autopoint \ + autotools-dev \ + automake \ patch \ ca-certificates \ git +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables +ENV CC=/usr/bin/clang \ + CXX=/usr/bin/clang++ \ + MPICC=/usr/bin/mpicc \ + MPICXX=/usr/bin/mpicxx \ + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/clang \ + OMPI_CXX=/usr/bin/clang++ + # Run uberenv # Have to create install directory first for uberenv # -k flag is to ignore SSL errors diff --git a/docker/tpl-ubuntu-clang.Dockerfile b/docker/tpl-ubuntu-clang.Dockerfile index 4e0058ad..574353ea 100644 --- a/docker/tpl-ubuntu-clang.Dockerfile +++ b/docker/tpl-ubuntu-clang.Dockerfile @@ -39,6 +39,14 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ libmpfr-dev \ lbzip2 \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ + libtool \ + autopoint \ + autotools-dev \ + automake \ gnupg \ virtualenv @@ -69,16 +77,17 @@ RUN apt-get install -y --no-install-recommends \ ca-certificates \ git -# Add MPI environment path info -ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ - CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables +ENV CC=/usr/bin/clang-$CLANG_MAJOR_VERSION \ + CXX=/usr/bin/clang++-$CLANG_MAJOR_VERSION \ MPICC=/usr/bin/mpicc \ MPICXX=/usr/bin/mpicxx \ - MPIEXEC=/usr/bin/mpirun -# The multi-line definition of arguments does not seem happy -# when a variable uses the value of another variable previously defined on the same line. -ENV OMPI_CC=$CC \ - OMPI_CXX=$CXX + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/clang-$CLANG_MAJOR_VERSION \ + OMPI_CXX=/usr/bin/clang++-$CLANG_MAJOR_VERSION # Run uberenv # Have to create install directory first for uberenv diff --git a/docker/tpl-ubuntu-gcc.Dockerfile b/docker/tpl-ubuntu-gcc.Dockerfile index 2f74d46c..68fbd86e 100644 --- a/docker/tpl-ubuntu-gcc.Dockerfile +++ b/docker/tpl-ubuntu-gcc.Dockerfile @@ -64,6 +64,11 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ libmpfr-dev \ lbzip2 \ bzip2 \ + gettext \ + bison \ + flex \ + help2man \ + libtool \ gnupg \ virtualenv @@ -73,15 +78,17 @@ RUN python3 -m pip install --upgrade pip && \ RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ MPICC=/usr/bin/mpicc \ MPICXX=/usr/bin/mpicxx \ - MPIEXEC=/usr/bin/mpirun -# The multi-line definition of arguments does not seem happy -# when a variable uses the value of another variable previously defined on the same line. -ENV OMPI_CC=$CC \ - OMPI_CXX=$CXX + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ + OMPI_CXX=/usr/bin/g++-$GCC_MAJOR_VERSION # This stage is dedicated to TPLs uniquely. # A multi-stage build patern will allow to extract what we need for the GEOSX build. @@ -96,6 +103,9 @@ ARG GCC_MAJOR_VERSION RUN apt-get install -y --no-install-recommends \ libtbb-dev \ make \ + autopoint \ + autotools-dev \ + automake \ bc \ file \ # GEOS patches some tpl. Remove when it's not the case anymore. diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile new file mode 100644 index 00000000..ee104b67 --- /dev/null +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -0,0 +1,168 @@ +# NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build + +# Base image is set by workflow via DOCKER_ROOT_IMAGE +ARG DOCKER_ROOT_IMAGE=rocm/dev-ubuntu-24.04:6.4.3 + +FROM ${DOCKER_ROOT_IMAGE} AS tpl_toolchain_intersect_geosx_toolchain +ARG SRC_DIR + +ARG INSTALL_DIR +ENV GEOSX_TPL_DIR=$INSTALL_DIR + +# Parameters +ARG AMDGPU_TARGET=gfx942 +ARG ROCM_VERSION=6.4.3 + +# Allow changing the number of cores used for building code via spack +ARG SPACK_BUILD_JOBS=4 +ENV SPACK_BUILD_JOBS=${SPACK_BUILD_JOBS} + +RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \ + apt-get update + +# Install system packages +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget \ + gnupg \ + ca-certificates \ + libtbb12 \ + libtbbmalloc2 \ + libblas-dev \ + liblapack-dev \ + libz3-dev \ + zlib1g-dev \ + openmpi-bin \ + libopenmpi-dev \ + python3 \ + python3-dev \ + python3-pip \ + python3-sphinx \ + doxygen \ + pkg-config \ + xz-utils \ + gettext \ + bison \ + flex \ + bzip2 \ + help2man \ + libtool \ + libgmp-dev \ + unzip \ + libmpfr-dev \ + lbzip2 \ + bzip2 \ + gnupg \ + virtualenv \ + libpugixml-dev \ + roctracer-dev \ + rocsparse-dev \ + rocsolver-dev \ + rocblas-dev \ + rocprim-dev \ + rocrand-dev \ + rocthrust-dev \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Install clingo for Spack (use pip without upgrading pip to avoid Debian conflict) +RUN python3 -m pip install clingo --break-system-packages + +# Install CMake +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh + +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables +ENV CC=/usr/bin/amdclang \ + CXX=/usr/bin/amdclang++ \ + MPICC=/usr/bin/mpicc \ + MPICXX=/usr/bin/mpicxx \ + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/amdclang \ + OMPI_CXX=/usr/bin/amdclang++ + +# Installing TPLs +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +ARG SRC_DIR +ARG BLD_DIR + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libtbb-dev \ + make \ + autopoint \ + autotools-dev \ + automake \ + ninja-build \ + bc \ + file \ + patch \ + ca-certificates \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +# --spack-debug to debug build +RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ + mkdir -p ${GEOSX_TPL_DIR} && \ + ./scripts/uberenv/uberenv.py \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ + --spack-env-file=${SRC_DIR}/docker/spack.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k && \ +# Remove host-config generated for LvArray + rm lvarray* && \ +# Rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated.cmake && \ +# Remove extraneous spack files + cd ${GEOSX_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + +# Extract only TPLs from previous stage +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +ARG SRC_DIR + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR + +# Extract the generated host-config +COPY --from=tpl_toolchain /spack-generated.cmake / + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get install -y --no-install-recommends \ + openssh-client \ + ca-certificates \ + curl \ + python3 \ + texlive \ + texlive-latex-extra \ + graphviz \ + libxml2-utils \ + git \ + ghostscript \ + ninja-build \ + python3-dev \ + python3-sphinx \ + python3-mpi4py \ + python3-scipy \ + python3-virtualenv \ + python3-matplotlib \ + python3-venv \ + python3-pytest && \ + rm -rf /var/lib/apt/lists/* + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache + +# Helpful environment defaults for HIP +ENV ROCM_PATH=/opt/rocm-${ROCM_VERSION} +ENV HIP_PATH=${ROCM_PATH}/hip +ENV PATH=${ROCM_PATH}/bin:${ROCM_PATH}/llvm/bin:${PATH} +ENV LD_LIBRARY_PATH=${ROCM_PATH}/lib:${ROCM_PATH}/lib64:${ROCM_PATH}/llvm/lib +ENV CMAKE_HIP_ARCHITECTURES=${AMDGPU_TARGET} \ No newline at end of file diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index 50c5d4c2..ded3556d 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -89,16 +89,16 @@ function launch_jobs() { case "$machine" in dane) ALLOC_CMD="salloc -N 1 --exclusive -t 60 -A vortex -ppdebug" - "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %gcc-12 ${COMMON}" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %gcc-13 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %gcc-12 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %gcc-13 ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane llvm-14 "+docs %clang-14 ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane llvm-19 "+docs %clang-19 ${COMMON}" "${ALLOC_CMD}" "$@" & ;; matrix) ALLOC_CMD="salloc -N 1 --exclusive -t 60 -A vortex -ppdebug" - "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %gcc-12 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %gcc-12 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" matrix llvm-14-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %clang-14 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" matrix llvm-19-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %clang-19 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & ;; diff --git a/scripts/spack_configs/pangea-4/spack.yaml b/scripts/spack_configs/pangea-4/spack.yaml index aabf1f14..49d3725a 100644 --- a/scripts/spack_configs/pangea-4/spack.yaml +++ b/scripts/spack_configs/pangea-4/spack.yaml @@ -68,7 +68,7 @@ spack: require: "@3.26.1" grpc: require: "@1.64.0-full-clone" - + #### # external compiler gcc: diff --git a/scripts/spack_configs/versions.yaml b/scripts/spack_configs/versions.yaml index 37052ab1..e798f2be 100644 --- a/scripts/spack_configs/versions.yaml +++ b/scripts/spack_configs/versions.yaml @@ -14,29 +14,29 @@ # This file lists the package versions for geos's dependencies # packages: - # v0.6.2 + # v0.7.1 + CUDA CMP0146 fix blt: - require: "@git.9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35=develop" + require: "@git.fe075e596c5fde30acbe4e05f40baaf0a258ff54=develop" # master - 06/27/25 hypre: - require: "@git.907a2d07b64fe47bdde4540c54665c83ced83a2c" + require: "@git.907a2d07b64fe47bdde4540c54665c83ced83a2c=master" - # v2025.0.3.0 + # v2025.09.1 chai: - require: "@git.4b9060b18b9bec1167026cfb3132bd540c4bd56b=develop" + require: "@git.51e66b8a5a6caac41aa329c8643180b8e2b33e5f=develop" - # v2025.0.3.0 + # v2025.09.0 umpire: - require: "@git.1ed0669c57f041baa1f1070693991c3a7a43e7ee=develop" + require: "@git.6b0ea9edbbbc741c8a429768d946549cd3bd7d33=develop" - # v2025.0.3.0 + # v2025.09.0 raja: - require: "@git.1d70abf171474d331f1409908bdf1b1c3fe19222=develop" + require: "@git.ca756788dbdd43fec2a3840389126ae94a905d5f=develop" - # v2025.0.3.0 + # v2025.09.2 camp: - require: "@git.ee0a3069a7ae72da8bcea63c06260fad34901d43=main" + require: "@git.4070ce93a802849d61037310a87c50cc24c9e498=main" # v2.12.0 caliper: diff --git a/scripts/spack_packages/packages/camp/package.py b/scripts/spack_packages/packages/camp/package.py new file mode 100644 index 00000000..4fd65fd4 --- /dev/null +++ b/scripts/spack_packages/packages/camp/package.py @@ -0,0 +1,15 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +from spack.package import * +from spack_repo.builtin.packages.camp.package import Camp as BuiltinCamp + +class Camp(BuiltinCamp): + depends_on("cxx", type="build") + + # From radiuss-packages (PR #143) + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") diff --git a/scripts/spack_packages/packages/chai/package.py b/scripts/spack_packages/packages/chai/package.py index 16b68a56..bdc3f94b 100644 --- a/scripts/spack_packages/packages/chai/package.py +++ b/scripts/spack_packages/packages/chai/package.py @@ -1,3 +1,7 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + import os from spack.package import * @@ -6,4 +10,12 @@ class Chai(BuiltinChai): # Bypass llnl_link_helpers failure depends_on("fortran") + depends_on("c", type="build") + depends_on("cxx", type="build") + # From radiuss-packages (PR #143) + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") + depends_on("umpire@2025.09.0:", when="@2025.09.0:") + depends_on("umpire@2025.03.0:", when="@2025.03.0:") + depends_on("umpire@2024.07.0", when="@2024.07.0") diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 2fd7fbe0..875906af 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -9,27 +9,21 @@ import socket import os -import llnl.util.tty as tty - from os import environ as env from os.path import join as pjoin -from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_path - from spack_repo.builtin.build_systems.cmake import CMakePackage from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.cached_cmake import ( + CachedCMakePackage, + cmake_cache_option, + cmake_cache_path, + cmake_cache_string, +) # Tested specs are located at scripts/spack_configs/<$SYS_TYPE>/spack.yaml (e.g. %clang@10.0.1) - # WARNING: +petsc variant is yet to be tested. - -def cmake_cache_entry(name, value, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE PATH "%s")\n\n' % (name, value, comment) - - def cmake_cache_list(name, value, comment=""): """Generate a list for a cmake cache variable""" @@ -38,19 +32,6 @@ def cmake_cache_list(name, value, comment=""): return 'set(%s %s CACHE STRING "%s")\n\n' % (name, join_str.join(value), comment) -def cmake_cache_string(name, string, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE STRING "%s")\n\n' % (name, string, comment) - - -def cmake_cache_option(name, boolean_value, comment=""): - """Generate a string for a cmake configuration option""" - - value = "ON" if boolean_value else "OFF" - return 'set(%s %s CACHE BOOL "%s")\n\n' % (name, value, comment) - - class Geosx(CMakePackage, CudaPackage, ROCmPackage): """GEOSX simulation framework.""" @@ -78,6 +59,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): multi=False) variant('grpc', default=False, description='Enable gRPC.') variant('pygeosx', default=True, description='Enable pygeosx.') + variant('cxxstd', default='17', description='CXX standard.') # SPHINX_END_VARIANTS @@ -112,15 +94,15 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # # Performance portability # - depends_on('raja ~examples~exercises~shared') + depends_on("raja ~examples~exercises~shared") depends_on("raja~openmp", when="~openmp") depends_on("raja+openmp", when="+openmp") - depends_on('umpire +c~examples+fortran~device_alloc~shared') + depends_on("umpire +c~examples+fortran~device_alloc~shared") depends_on("umpire~openmp", when="~openmp") depends_on("umpire+openmp", when="+openmp") - depends_on('chai +raja~examples~shared') + depends_on("chai +raja~examples~shared") depends_on("chai~openmp", when="~openmp") depends_on("chai+openmp", when="+openmp") @@ -158,14 +140,14 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on('pugixml@1.13 ~shared') - depends_on('fmt@10.0.0 cxxstd=14') + depends_on('fmt@11') depends_on('vtk@9.4.2', when='+vtk') # # Math # - depends_on("parmetis@4.0.3+int64~shared cflags='-fPIC' cxxflags='-fPIC'") - depends_on("metis +int64~shared cflags='-fPIC' cxxflags='-fPIC'") + depends_on("parmetis@4.0.3+int64+shared cflags='-fPIC' cxxflags='-fPIC'") + depends_on("metis +int64+shared cflags='-fPIC' cxxflags='-fPIC'") depends_on("superlu-dist +int64 fflags='-fPIC'") depends_on("superlu-dist~openmp", when="~openmp") @@ -190,6 +172,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on("hypre +rocm+superlu-dist+mixedint+mpi+umpire+unified-memory~shared cflags='-fPIC' cxxflags='-fPIC'", when='+rocm') depends_on("hypre ~openmp", when="~openmp") depends_on("hypre +openmp", when="+openmp") + depends_on("hypre +caliper", when="+caliper") depends_on('petsc@3.19.4~hdf5~hypre+int64', when='+petsc') depends_on('petsc+ptscotch', when='+petsc+scotch') @@ -230,6 +213,12 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): phases = ['geos_hostconfig', 'lvarray_hostconfig'] #phases = ['hostconfig', 'cmake', 'build', 'install'] + # This method is called to finalize dependencies + def dependencies(self): + super().dependencies() + # Add a conditional dependency on fmt with the cxxstd variant + self.depends_on(f"fmt@11 cxxstd=c++{self.spec.variants['cxxstd'].value}") + @run_after('build') @on_package_attributes(run_tests=True) def check(self): @@ -254,22 +243,22 @@ def _get_sys_type(self, spec): return sys_type def _get_host_config_path(self, spec, lvarray=False): - var = '' - - if '+cuda' in spec: - var = '-'.join([var, 'cuda']) - var += "@" + str(spec['cuda'].version) - elif '+rocm' in spec: - var = '-'.join([var, 'rocm']) - var += "@" + str(spec['hip'].version) - + gpu_backend = "" + if "+cuda" in spec: + gpu_backend = f"cuda@{spec['cuda'].version}" + elif "+rocm" in spec: + gpu_backend = f"rocm@{spec['hip'].version}" hostname = socket.gethostname().rstrip('1234567890') if lvarray: hostname = "lvarray-" + hostname - host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, self._get_sys_type(spec), (str(spec.compiler.name)), str(spec.compiler.version),var) + host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, + self._get_sys_type(spec), + str(spec.compiler.name), + str(spec.compiler.version), + gpu_backend) dest_dir = self.stage.source_path host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) @@ -331,15 +320,15 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n".format("-" * 80)) cfg.write("# Compilers\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write(cmake_cache_path("CMAKE_C_COMPILER", c_compiler)) cflags = ' '.join(spec.compiler_flags['cflags']) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + cfg.write(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + cfg.write(cmake_cache_path("CMAKE_CXX_COMPILER", cpp_compiler)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) + cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) release_flags = "-O3 -DNDEBUG" if "clang" in self.compiler.cxx: @@ -351,21 +340,21 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) + cfg.write(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_string("BLT_CXX_STD", "c++17")) + cfg.write(cmake_cache_string("BLT_CXX_STD", f"c++{spec.variants['cxxstd'].value}")) cfg.write("#{0}\n".format("-" * 80)) cfg.write("# MPI\n") cfg.write("#{0}\n\n".format("-" * 80)) cfg.write(cmake_cache_option('ENABLE_MPI', True)) - cfg.write(cmake_cache_entry('MPI_C_COMPILER', spec['mpi'].mpicc)) - cfg.write(cmake_cache_entry('MPI_CXX_COMPILER', spec['mpi'].mpicxx)) + cfg.write(cmake_cache_path('MPI_C_COMPILER', spec['mpi'].mpicc)) + cfg.write(cmake_cache_path('MPI_CXX_COMPILER', spec['mpi'].mpicxx)) hostname = socket.gethostname().rstrip('1234567890') @@ -373,14 +362,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): and hostname != 'p3dev': cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) + cfg.write(cmake_cache_string('MPIEXEC', 'lrun')) + cfg.write(cmake_cache_string('MPIEXEC_NUMPROC_FLAG', '-n')) else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) + cfg.write(cmake_cache_string('MPIEXEC', 'jsrun')) cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) elif sys_type in ('toss_4_x86_64_ib_cray'): - cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) + cfg.write(cmake_cache_string('MPIEXEC', 'srun')) + cfg.write(cmake_cache_string('MPIEXEC_NUMPROC_FLAG', '-n')) else: # Taken from cached_cmake class: # https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/cached_cmake.py#L180-234 @@ -441,12 +430,12 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+cuda' in spec: cfg.write(cmake_cache_option('ENABLE_CUDA', True)) - cfg.write(cmake_cache_entry('CMAKE_CUDA_STANDARD', 17)) + cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value)) cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) + cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' - cfg.write(cmake_cache_entry('CMAKE_CUDA_COMPILER', cudacompiler)) + cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' 'cross-execution-space-call,reorder,' @@ -476,7 +465,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cuda_stack_size = int(spec.variants['cuda_stack_size'].value) if 0 < cuda_stack_size: cfg.write(cmake_cache_option('ENABLE_CUDA_STACK_SIZE', True, "Adjust the CUDA stack size limit")) - cfg.write(cmake_cache_entry('CUDA_STACK_SIZE', cuda_stack_size, "CUDA stack size in KB")) + cfg.write(cmake_cache_string('CUDA_STACK_SIZE', cuda_stack_size, "CUDA stack size in KB")) else: cfg.write(cmake_cache_option('ENABLE_CUDA', False)) @@ -486,14 +475,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+rocm' in spec: cfg.write(cmake_cache_option('ENABLE_HIP', True)) - cfg.write(cmake_cache_string('CMAKE_HIP_STANDARD', 17)) - cfg.write(cmake_cache_entry('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) + cfg.write(cmake_cache_string('CMAKE_HIP_STANDARD', spec.variants['cxxstd'].value)) + cfg.write(cmake_cache_path('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) if not spec.satisfies('amdgpu_target=none'): cmake_hip_archs = ";".join(spec.variants["amdgpu_target"].value) cfg.write(cmake_cache_string('CMAKE_HIP_ARCHITECTURES', cmake_hip_archs)) - cfg.write(cmake_cache_entry('ROCM_PATH', spec['hip'].prefix)) + cfg.write(cmake_cache_path('ROCM_PATH', spec['hip'].prefix)) else: cfg.write(cmake_cache_option('ENABLE_HIP', False)) @@ -502,14 +491,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_CHAI', True)) - cfg.write(cmake_cache_entry('CHAI_DIR', spec['chai'].prefix)) + cfg.write(cmake_cache_path('CHAI_DIR', spec['chai'].prefix)) - cfg.write(cmake_cache_entry('RAJA_DIR', spec['raja'].prefix)) + cfg.write(cmake_cache_path('RAJA_DIR', spec['raja'].prefix)) cfg.write(cmake_cache_option('ENABLE_UMPIRE', True)) - cfg.write(cmake_cache_entry('UMPIRE_DIR', spec['umpire'].prefix)) + cfg.write(cmake_cache_path('UMPIRE_DIR', spec['umpire'].prefix)) - cfg.write(cmake_cache_entry('CAMP_DIR', spec['camp'].prefix)) + cfg.write(cmake_cache_path('CAMP_DIR', spec['camp'].prefix)) # yapf: disable io_tpls = ( @@ -529,12 +518,12 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+caliper' in spec: cfg.write(cmake_cache_option('ENABLE_CALIPER', True)) - cfg.write(cmake_cache_entry('CALIPER_DIR', spec['caliper'].prefix)) - cfg.write(cmake_cache_entry('ADIAK_DIR', spec['adiak'].prefix)) + cfg.write(cmake_cache_path('CALIPER_DIR', spec['caliper'].prefix)) + cfg.write(cmake_cache_path('ADIAK_DIR', spec['adiak'].prefix)) for tpl, cmake_name, enable in io_tpls: if enable: - cfg.write(cmake_cache_entry('{}_DIR'.format(cmake_name), spec[tpl].prefix)) + cfg.write(cmake_cache_path('{}_DIR'.format(cmake_name), spec[tpl].prefix)) else: cfg.write(cmake_cache_option('ENABLE_{}'.format(cmake_name), False)) @@ -544,15 +533,15 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if spec["blas"].name == "intel-oneapi-mkl": cfg.write(cmake_cache_option('ENABLE_MKL', True)) - cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-oneapi-mkl'].prefix.include)) + cfg.write(cmake_cache_path('MKL_INCLUDE_DIRS', spec['intel-oneapi-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-oneapi-mkl'].libs)) elif spec["blas"].name == "mkl": cfg.write(cmake_cache_option('ENABLE_MKL', True)) - cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) + cfg.write(cmake_cache_path('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) elif spec["blas"].name == "essl": cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) + cfg.write(cmake_cache_path('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) @@ -578,7 +567,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) for tpl, cmake_name, enable in math_tpls: if enable: - cfg.write(cmake_cache_entry('{}_DIR'.format(cmake_name), spec[tpl].prefix)) + cfg.write(cmake_cache_path('{}_DIR'.format(cmake_name), spec[tpl].prefix)) if tpl == 'hypre' and '+cuda' in spec: cfg.write(cmake_cache_string('ENABLE_HYPRE_DEVICE', "CUDA")) @@ -601,8 +590,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('# Python\n') cfg.write('#{0}\n\n'.format('-' * 80)) - cfg.write(cmake_cache_entry('Python3_ROOT_DIR', os.path.join(spec['python'].prefix))) - cfg.write(cmake_cache_entry('Python3_EXECUTABLE', os.path.join(spec['python'].prefix.bin, 'python3'))) + cfg.write(cmake_cache_path('Python3_ROOT_DIR', os.path.join(spec['python'].prefix))) + cfg.write(cmake_cache_path('Python3_EXECUTABLE', os.path.join(spec['python'].prefix.bin, 'python3'))) if '+pygeosx' in spec: cfg.write(cmake_cache_option('ENABLE_PYGEOSX', True)) @@ -614,10 +603,10 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+docs' in spec: sphinx_bin_dir = spec['py-sphinx'].prefix.bin - cfg.write(cmake_cache_entry('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) + cfg.write(cmake_cache_path('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) doxygen_bin_dir = spec['doxygen'].prefix.bin - cfg.write(cmake_cache_entry('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) + cfg.write(cmake_cache_path('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) else: cfg.write(cmake_cache_option('ENABLE_DOCS', False)) cfg.write(cmake_cache_option('ENABLE_DOXYGEN', False)) @@ -630,14 +619,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_UNCRUSTIFY', '+uncrustify' in spec)) if '+uncrustify' in spec: cfg.write( - cmake_cache_entry('UNCRUSTIFY_EXECUTABLE', os.path.join(spec['uncrustify'].prefix.bin, 'uncrustify'))) + cmake_cache_path('UNCRUSTIFY_EXECUTABLE', os.path.join(spec['uncrustify'].prefix.bin, 'uncrustify'))) if '+addr2line' in spec: cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# addr2line\n') cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_ADDR2LINE', True)) - cfg.write(cmake_cache_entry('ADDR2LINE_EXEC ', '/usr/bin/addr2line')) + cfg.write(cmake_cache_path('ADDR2LINE_EXEC ', '/usr/bin/addr2line')) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Other\n') @@ -645,7 +634,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+mathpresso' in spec: cfg.write(cmake_cache_option('ENABLE_MATHPRESSO', True)) - cfg.write(cmake_cache_entry('MATHPRESSO_DIR', spec['mathpresso'].prefix)) + cfg.write(cmake_cache_path('MATHPRESSO_DIR', spec['mathpresso'].prefix)) cfg.write(cmake_cache_option('ENABLE_XML_UPDATES', True)) else: cfg.write(cmake_cache_option('ENABLE_MATHPRESSO', False)) @@ -731,15 +720,15 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n".format("-" * 80)) cfg.write("# Compilers\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write(cmake_cache_path("CMAKE_C_COMPILER", c_compiler)) cflags = ' '.join(spec.compiler_flags['cflags']) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + cfg.write(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + cfg.write(cmake_cache_path("CMAKE_CXX_COMPILER", cpp_compiler)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) + cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) release_flags = "-O3 -DNDEBUG" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", release_flags)) @@ -749,19 +738,19 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) + cfg.write(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') cfg.write('#{0}\n\n'.format('-' * 80)) if '+cuda' in spec: cfg.write(cmake_cache_option('ENABLE_CUDA', True)) - cfg.write(cmake_cache_entry('CMAKE_CUDA_STANDARD', 17)) + cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value)) cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) + cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' - cfg.write(cmake_cache_entry('CMAKE_CUDA_COMPILER', cudacompiler)) + cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' 'cross-execution-space-call,reorder,' @@ -792,14 +781,14 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_CHAI', True)) - cfg.write(cmake_cache_entry('CHAI_DIR', spec['chai'].prefix)) + cfg.write(cmake_cache_path('CHAI_DIR', spec['chai'].prefix)) - cfg.write(cmake_cache_entry('RAJA_DIR', spec['raja'].prefix)) + cfg.write(cmake_cache_path('RAJA_DIR', spec['raja'].prefix)) cfg.write(cmake_cache_option('ENABLE_UMPIRE', True)) - cfg.write(cmake_cache_entry('UMPIRE_DIR', spec['umpire'].prefix)) + cfg.write(cmake_cache_path('UMPIRE_DIR', spec['umpire'].prefix)) - cfg.write(cmake_cache_entry('CAMP_DIR', spec['camp'].prefix)) + cfg.write(cmake_cache_path('CAMP_DIR', spec['camp'].prefix)) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# IO TPLs\n') @@ -807,18 +796,18 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+caliper' in spec: cfg.write(cmake_cache_option('ENABLE_CALIPER', True)) - cfg.write(cmake_cache_entry('CALIPER_DIR', spec['caliper'].prefix)) - cfg.write(cmake_cache_entry('adiak_DIR', spec['adiak'].prefix + '/lib/cmake/adiak')) + cfg.write(cmake_cache_path('CALIPER_DIR', spec['caliper'].prefix)) + cfg.write(cmake_cache_path('adiak_DIR', spec['adiak'].prefix + '/lib/cmake/adiak')) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Documentation\n') cfg.write('#{0}\n\n'.format('-' * 80)) if '+docs' in spec: sphinx_bin_dir = spec['py-sphinx'].prefix.bin - cfg.write(cmake_cache_entry('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) + cfg.write(cmake_cache_path('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) doxygen_bin_dir = spec['doxygen'].prefix.bin - cfg.write(cmake_cache_entry('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) + cfg.write(cmake_cache_path('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) else: cfg.write(cmake_cache_option('ENABLE_DOXYGEN', False)) cfg.write(cmake_cache_option('ENABLE_SPHINX', False)) diff --git a/scripts/spack_packages/packages/raja/package.py b/scripts/spack_packages/packages/raja/package.py new file mode 100644 index 00000000..3ef41bba --- /dev/null +++ b/scripts/spack_packages/packages/raja/package.py @@ -0,0 +1,174 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +from spack.package import * +from spack_repo.builtin.packages.raja.package import Raja as BuiltinRaja + +class Raja(BuiltinRaja): + depends_on("cxx", type="build") + + # From radiuss-packages (PR #143) + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") + depends_on("camp@main", when="@develop") + depends_on("camp@2025.09.2:", when="@2025.09.0:") + depends_on("camp@2025.03.0:", when="@2025.03.0:") + depends_on("camp@2024.07.0:", when="@2024.07.0:") + depends_on("camp@2024.02.1:", when="@2024.02.1:") + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + entries.append("#------------------{0}".format("-" * 30)) + entries.append("# Package custom hardware settings") + entries.append("#------------------{0}\n".format("-" * 30)) + + entries.append(cmake_cache_option("ENABLE_OPENMP", spec.satisfies("+openmp"))) + entries.append(cmake_cache_option("ENABLE_CUDA", spec.satisfies("+cuda"))) + + if spec.satisfies("+rocm"): + entries.append(cmake_cache_option("ENABLE_HIP", True)) + hipcc_flags = [] + if self.spec.satisfies("@2025.09.0:"): + hipcc_flags.append("-std=c++17") + elif self.spec.satisfies("@0.14.0:2025.09.0"): + hipcc_flags.append("-std=c++14") + entries.append(cmake_cache_string("HIP_HIPCC_FLAGS", " ".join(hipcc_flags))) + else: + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "RAJA_" if spec.satisfies("@0.14.0:") else "" + + # TPL locations + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# TPLs") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + if "camp" in self.spec: + entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) + + # Build options + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# Build Options") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_string("CMAKE_BUILD_TYPE", spec.variants["build_type"].value)) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", spec.satisfies("+shared"))) + + entries.append(cmake_cache_option("RAJA_ENABLE_DESUL_ATOMICS", spec.satisfies("+desul"))) + + entries.append( + cmake_cache_option("RAJA_ENABLE_VECTORIZATION", spec.satisfies("+vectorization")) + ) + + entries.append(cmake_cache_option("RAJA_ENABLE_OPENMP_TASK", spec.satisfies("+omptask"))) + + entries.append( + cmake_cache_option("RAJA_ENABLE_TARGET_OPENMP", spec.satisfies("+omptarget")) + ) + + entries.append(cmake_cache_option("RAJA_ENABLE_SYCL", spec.satisfies("+sycl"))) + entries.append( + cmake_cache_option("RAJA_ENABLE_NV_TOOLS_EXT", spec.satisfies("+gpu-profiling +cuda")) + ) + entries.append( + cmake_cache_option("RAJA_ENABLE_ROCTX", spec.satisfies("+gpu-profiling +rocm")) + ) + + if spec.satisfies("+lowopttest"): + entries.append(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", "-O1")) + + # C++17 + if (spec.satisfies("@2025.09.0:") or + (spec.satisfies("@2024.07.0:") and spec.satisfies("+sycl"))): + entries.append(cmake_cache_string("BLT_CXX_STD", "c++17")) + # C++14 + elif spec.satisfies("@0.14.0:2025.09.0"): + entries.append(cmake_cache_string("BLT_CXX_STD", "c++14")) + + if spec.satisfies("+desul"): + if spec.satisfies("+cuda"): + entries.append(cmake_cache_string("CMAKE_CUDA_STANDARD", "14")) + + entries.append( + cmake_cache_option("RAJA_ENABLE_RUNTIME_PLUGINS", spec.satisfies("+plugins")) + ) + + if spec.satisfies("+omptarget"): + entries.append( + cmake_cache_string( + "BLT_OPENMP_COMPILE_FLAGS", "-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" + ) + ) + entries.append( + cmake_cache_string( + "BLT_OPENMP_LINK_FLAGS", "-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" + ) + ) + + entries.append( + cmake_cache_option( + "{}ENABLE_EXAMPLES".format(option_prefix), spec.satisfies("+examples") + ) + ) + if spec.satisfies("@0.14.0:"): + entries.append( + cmake_cache_option( + "{}ENABLE_EXERCISES".format(option_prefix), spec.satisfies("+exercises") + ) + ) + else: + entries.append(cmake_cache_option("ENABLE_EXERCISES", spec.satisfies("+exercises"))) + + # TODO: Treat the workaround when building tests with spack wrapper + # For now, removing it to test CI, which builds tests outside of wrapper. + # Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS which + # is used by the spack compiler wrapper. This can go away when BLT + # removes -Werror from GTest flags + # + # if self.spec.satisfies("%clang target=ppc64le:") + # or (not self.run_tests and not spec.satisfies("+tests")): + if not self.run_tests and not spec.satisfies("+tests"): + entries.append(cmake_cache_option("ENABLE_TESTS", False)) + else: + entries.append(cmake_cache_option("ENABLE_TESTS", True)) + if not spec.satisfies("+run-all-tests"): + if spec.satisfies("%clang@12.0.0:13.9.999"): + entries.append( + cmake_cache_string( + "CTEST_CUSTOM_TESTS_IGNORE", + "test-algorithm-sort-OpenMP.exe;test-algorithm-stable-sort-OpenMP.exe", + ) + ) + excluded_tests = [ + "test-algorithm-sort-Cuda.exe", + "test-algorithm-stable-sort-Cuda.exe", + "test-algorithm-sort-OpenMP.exe", + "test-algorithm-stable-sort-OpenMP.exe", + ] + if spec.satisfies("+cuda %clang@12.0.0:13.9.999"): + entries.append( + cmake_cache_string("CTEST_CUSTOM_TESTS_IGNORE", ";".join(excluded_tests)) + ) + if spec.satisfies("+cuda %xl@16.1.1.12"): + entries.append( + cmake_cache_string( + "CTEST_CUSTOM_TESTS_IGNORE", + "test-algorithm-sort-Cuda.exe;test-algorithm-stable-sort-Cuda.exe", + ) + ) + + entries.append(cmake_cache_option("RAJA_HOST_CONFIG_LOADED", True)) + + return entries diff --git a/scripts/spack_packages/packages/umpire/package.py b/scripts/spack_packages/packages/umpire/package.py new file mode 100644 index 00000000..4037e59d --- /dev/null +++ b/scripts/spack_packages/packages/umpire/package.py @@ -0,0 +1,20 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +from spack.package import * +from spack_repo.builtin.packages.umpire.package import Umpire as BuiltinUmpire + +class Umpire(BuiltinUmpire): + depends_on("c", type="build") # generated + depends_on("cxx", type="build") # generated + depends_on("fortran", type="build") # generated + + # From radiuss-packages (PR #143) + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("camp@2025.09.2:", when="@2025.09.0:") + depends_on("camp@2025.03.0:", when="@2025.03.0:") + + conflicts("+ipc_shmem", when="+mpi3_shmem @:2025.03.0")