From ce104e48d3a7a5c842640d39ada57700ebf21b9b Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 2 Oct 2025 09:14:21 -0700 Subject: [PATCH 01/55] Update uberenv commit --- scripts/uberenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uberenv b/scripts/uberenv index a76a2942..eb185358 160000 --- a/scripts/uberenv +++ b/scripts/uberenv @@ -1 +1 @@ -Subproject commit a76a2942f0b575f7d9047dfb87d699e2b6d18f12 +Subproject commit eb1853586821360613f1c6c415ebf65d370a47e7 From 63448c52431f8da94d34708f3b91482c2b6e91fc Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 2 Oct 2025 09:20:32 -0700 Subject: [PATCH 02/55] Lock in main spack repo and spack-packages version & hash --- .uberenv_config.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.uberenv_config.json b/.uberenv_config.json index a50db3f9..1bee4505 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,7 +4,8 @@ "package_final_phase": "lvarray_hostconfig", "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2bfcc69fa870d3c6919be87593f22647981b648a", +"spack_commit": "734c5db2121b01c373eed6538e452f18887e9e44", "spack_configs_path": "scripts/spack_configs", -"spack_packages_path": "scripts/spack_packages/packages" +"spack_packages_path": "scripts/spack_packages/packages", +"spack_packages_commit": "a75a7f75182ffc7a51c6ca7f0fec4bf9b2705be8" } From a0f1c7d361e7ce09455088c68d0b6f913c0b1d00 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 2 Oct 2025 09:54:15 -0700 Subject: [PATCH 03/55] Dockerfile, environment file, recipe changes for toss_4/Ubuntu/Rockylinux/pangea --- docker/TotalEnergies/Pangea3.Dockerfile | 6 +- docker/pangea-spack.yaml | 64 ++--- docker/rocky-spack.yaml | 94 ++++--- docker/spack.yaml | 236 +++++++++-------- .../tpl-rockylinux-clang-cuda-12.Dockerfile | 6 +- docker/tpl-rockylinux-gcc-cuda-12.Dockerfile | 4 +- docker/tpl-ubuntu-clang-cuda.Dockerfile | 4 +- docker/tpl-ubuntu-clang.Dockerfile | 18 +- docker/tpl-ubuntu-gcc.Dockerfile | 4 +- .../blueos_3_ppc64le_ib_p9/spack.yaml | 164 +++++++----- .../spack_configs/toss_4_x86_64_ib/spack.yaml | 245 ++++++++++-------- .../spack_packages/packages/essl/package.py | 2 +- .../spack_packages/packages/geosx/package.py | 18 +- .../spack_packages/packages/hdf5/package.py | 8 +- .../packages/mathpresso/package.py | 1 + .../packages/pygeosx/package.py | 3 +- .../spack_packages/packages/scotch/package.py | 6 +- .../packages/trilinos/package.py | 26 +- .../spack_packages/packages/vtk/package.py | 4 +- 19 files changed, 517 insertions(+), 396 deletions(-) diff --git a/docker/TotalEnergies/Pangea3.Dockerfile b/docker/TotalEnergies/Pangea3.Dockerfile index 61434384..3d67d82e 100644 --- a/docker/TotalEnergies/Pangea3.Dockerfile +++ b/docker/TotalEnergies/Pangea3.Dockerfile @@ -48,9 +48,11 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ ln -s /usr/lib64/libnsl.so.2.0.0 /usr/lib64/libnsl.so && \ ln -s /usr/lib64/librdmacm.so.1.3.48.0 /usr/lib64/librdmacm.so && \ ln -s /usr/lib64/liblustreapi.so.1 /usr/lib64/liblustreapi.so && \ + ln -s /data_local/sw/lsf/10.1/linux3.10-glibc2.17-ppc64le/lib/libbat.so /usr/lib64/libbat.so && \ + ln -s /data_local/sw/lsf/10.1/linux3.10-glibc2.17-ppc64le/lib/liblsf.so /usr/lib64/liblsf.so && \ # Run uberenv ./scripts/uberenv/uberenv.py \ - --spec "%gcc@9.4.0+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+cuda~uncrustify~openmp~pygeosx cuda_arch=70 %gcc-9 ^cuda@11.5.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/pangea-spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -61,7 +63,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated-wave-solver-only.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Build only the wave solver for Pangea 3 RUN echo 'set ( GEOS_ENABLE_CONTACT OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ diff --git a/docker/pangea-spack.yaml b/docker/pangea-spack.yaml index a505347d..274a1dce 100644 --- a/docker/pangea-spack.yaml +++ b/docker/pangea-spack.yaml @@ -17,31 +17,41 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers:: - - compiler: - spec: gcc@=9.4.0 - paths: - cc: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gcc - cxx: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/g++ - f77: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gfortran - fc: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gfortran - flags: {} - operating_system: almalinux8 - target: ppc64le - modules: [] - environment: - prepend_path: - LD_LIBRARY_PATH: /data_local/sw/lsf/10.1/linux3.10-glibc2.17-ppc64le/lib - extra_rpaths: [] + toolchains: + gcc-9: + - spec: '%c=gcc@9.4.0' + when: '%c' + - spec: '%cxx=gcc@9.4.0' + when: '%cxx' + - spec: '%fortran=gcc@9.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' packages: all: target: [x86_64] - compiler: [gcc] - providers: - blas: [openblas] - lapack: [openblas] - mpi: [openmpi] + + mpi: + require: + - openmpi + + blas: + require: + - "openblas" + lapack: + require: + - "openblas" + + gcc: + externals: + - spec: gcc@9.4.0 languages:='c,c++,fortran' + prefix: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v + extra_attributes: + compilers: + c: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gcc + cxx: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/g++ + fortran: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gfortran autoconf: externals: @@ -94,10 +104,6 @@ spack: - spec: openblas@0.3.18 prefix: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-9.4.0/openblas-0.3.18-vk36pzksytuhylqesg4cca7667np5sjp buildable: false - blas: - buildable: false - lapack: - buildable: false libtool: externals: @@ -110,13 +116,9 @@ spack: prefix: /usr buildable: false - # Spack may grab for mpi & we don't want to use them - mpi: - buildable: false - openmpi: externals: - - spec: openmpi@4.1.2%gcc@9.4.0 +cuda~cxx~cxx_exceptions~java+lustre~memchecker+pmi+static~wrapper-rpath schedulers=lsf + - spec: openmpi@4.1.2 +cuda~cxx~cxx_exceptions~java+lustre~memchecker+pmi+static~wrapper-rpath schedulers=lsf %gcc@9.4.0 prefix: /data_local/sw/openmpi/4.1.2/env/gcc-8.4.1 # Built-in ncurses causes spack failure @@ -137,7 +139,7 @@ spack: buildable: false python: externals: - - spec: python@3.8.17+bz2+crypt+ctypes+dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib + - spec: python@3.8.17+bz2+crypt+ctypes+dbm+lzma+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib prefix: /usr buildable: false readline: diff --git a/docker/rocky-spack.yaml b/docker/rocky-spack.yaml index 94025319..4a8c51bd 100644 --- a/docker/rocky-spack.yaml +++ b/docker/rocky-spack.yaml @@ -17,44 +17,62 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers:: - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: rocky8 - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - spec: clang@17.0.6 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: rocky8 - paths: - cc: /opt/rh/gcc-toolset-13/root/usr/bin/gcc - cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ - f77: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - fc: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - spec: gcc@13.3.1 - target: x86_64 + toolchains: + gcc-13: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@13.3.1' + when: '%c' + - spec: '%cxx=gcc@13.3.1' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%openmpi@4.1.1' + when: '%mpi' + clang-17: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%[virtuals=c]llvm@17.0.6+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@17.0.6+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@13.3.1' + when: '%fortran' + - spec: '%openmpi@4.1.1' + when: '%mpi' packages: all: target: [x86_64] - compiler: [gcc, clang] - providers: - blas: [netlib-lapack] - lapack: [netlib-lapack] - mpi: [openmpi] + + mpi: + require: + - openmpi + + blas: + require: + - "netlib-lapack" + lapack: + require: + - "netlib-lapack" + + llvm: + externals: + - spec: llvm@17.0.6+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + gcc: + externals: + - spec: gcc@13.3.1 languages:='c,c++,fortran' + prefix: /opt/rh/gcc-toolset-13/root/usr + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran autoconf: version: [2.71] @@ -77,9 +95,9 @@ spack: cuda: buildable: False externals: - - spec: cuda@12.9.1%gcc@13.3.1 +allow-unsupported-compilers + - spec: cuda@12.9.1 +allow-unsupported-compilers %gcc@13.3.1 prefix: /usr/local/cuda - - spec: cuda@12.9.1%clang@17.0.6 +allow-unsupported-compilers + - spec: cuda@12.9.1 +allow-unsupported-compilers %llvm@17.0.6 prefix: /usr/local/cuda m4: buildable: false @@ -87,10 +105,6 @@ spack: - spec: m4@1.4.18 prefix: /usr - # Spack may grab for mpi & we don't want to use them - mpi: - buildable: false - netlib-lapack: buildable: false externals: diff --git a/docker/spack.yaml b/docker/spack.yaml index 463e8ead..8db0a342 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -17,111 +17,129 @@ spack: - ../defaults.yaml - ../versions.yaml - - compilers:: - - compiler: - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu22.04 - paths: - cc: /usr/bin/clang-15 - cxx: /usr/bin/clang++-15 - f77: /usr/bin/gfortran-11 - fc: /usr/bin/gfortran-11 - spec: clang@15 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu20.04 - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - spec: clang@10 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: ubuntu22.04 - paths: - cc: /usr/bin/gcc-12 - cxx: /usr/bin/g++-12 - f77: /usr/bin/gfortran-12 - fc: /usr/bin/gfortran-12 - spec: gcc@12 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: ubuntu22.04 - paths: - cc: /usr/bin/gcc-11 - cxx: /usr/bin/g++-11 - f77: /usr/bin/gfortran-11 - fc: /usr/bin/gfortran-11 - spec: gcc@11 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: ubuntu20.04 - paths: - cc: /usr/bin/gcc-10 - cxx: /usr/bin/g++-10 - f77: /usr/bin/gfortran-10 - fc: /usr/bin/gfortran-10 - spec: gcc@10 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: ubuntu20.04 - paths: - cc: /usr/bin/gcc-9 - cxx: /usr/bin/g++-9 - f77: /usr/bin/gfortran-9 - fc: /usr/bin/gfortran-9 - spec: gcc@9 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: centos7 - paths: - cc: /opt/rh/devtoolset-8/root/usr/bin/gcc - cxx: /opt/rh/devtoolset-8/root/usr/bin/g++ - f77: /opt/rh/devtoolset-8/root/usr/bin/gfortran - fc: /opt/rh/devtoolset-8/root/usr/bin/gfortran - spec: gcc@8 - target: x86_64 + toolchains: + clang-15: + - spec: '%[virtuals=c]llvm@15.0.7+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@15.0.7+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@11.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + clang-10: + - spec: '%[virtuals=c]llvm@10.0.0+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@10.0.0+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@9.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + gcc-9: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@9.5.0' + when: '%c' + - spec: '%cxx=gcc@9.5.0' + when: '%cxx' + - spec: '%fortran=gcc@9.5.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + gcc-10: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@10.5.0' + when: '%c' + - spec: '%cxx=gcc@10.5.0' + when: '%cxx' + - spec: '%fortran=gcc@10.5.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + gcc-11: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@11.4.0' + when: '%c' + - spec: '%cxx=gcc@11.4.0' + when: '%cxx' + - spec: '%fortran=gcc@11.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + gcc-12: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@12.3.0' + when: '%c' + - spec: '%cxx=gcc@12.3.0' + when: '%cxx' + - spec: '%fortran=gcc@12.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' packages: all: target: [x86_64] - compiler: [gcc, clang] - providers: - blas: [netlib-lapack] - lapack: [netlib-lapack] - mpi: [openmpi] + + mpi: + require: + - openmpi + + blas: + require: + - "netlib-lapack" + lapack: + require: + - "netlib-lapack" + + llvm: + externals: + - spec: llvm@10.0.0+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + - spec: llvm@15.0.7+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-15 + cxx: /usr/bin/clang++-15 + + gcc: + externals: + - spec: gcc@9.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-9 + cxx: /usr/bin/g++-9 + fortran: /usr/bin/gfortran-9 + - spec: gcc@10.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-10 + cxx: /usr/bin/g++-10 + fortran: /usr/bin/gfortran-10 + - spec: gcc@11.4.0 languages:=c,fortran + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-11 + fortran: /usr/bin/gfortran-11 + - spec: gcc@12.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + fortran: /usr/bin/gfortran-12 autoconf: version: [2.71] @@ -163,10 +181,6 @@ spack: - spec: mpfr@6.0.2 prefix: /usr - # Spack may grab for mpi & we don't want to use them - mpi: - buildable: false - netlib-lapack: buildable: false externals: @@ -179,18 +193,16 @@ spack: prefix: /usr openmpi: externals: - - spec: openmpi@4.1.2%clang + - spec: openmpi@4.1.2 %llvm prefix: /usr - - spec: openmpi@1.10.7%gcc@8 - prefix: /usr/lib64/openmpi modules: [mpi] - - spec: openmpi@4.1.2%gcc@9 + - spec: openmpi@4.1.2 %gcc@9.5.0 prefix: /usr - - spec: openmpi@4.1.2%gcc@10 + - spec: openmpi@4.1.2 %gcc@10.5.0 prefix: /usr - - spec: openmpi@4.1.2%gcc@11 + - spec: openmpi@4.1.2 %gcc@11.4.0 prefix: /usr - - spec: openmpi@4.1.2%gcc@12 + - spec: openmpi@4.1.2 %gcc@12.3.0 prefix: /usr perl: buildable: false diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index bffe8879..f761368c 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -15,7 +15,7 @@ RUN dnf clean all && \ dnf -y install \ which \ clang-17.0.6 \ - gcc-gfortran \ + gcc-toolset-13 \ python3 \ zlib-devel \ tbb \ @@ -70,7 +70,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ ln -s /usr/lib64/libblas.so.3 /usr/lib64/libblas.so && \ ln -s /usr/lib64/liblapack.so.3 /usr/lib64/liblapack.so && \ ./scripts/uberenv/uberenv.py \ - --spec "%clang@17.0.6+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+cuda~uncrustify~openmp~pygeosx cuda_arch=70 %clang-17 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/rocky-spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -81,7 +81,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Extract only TPL's from the previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index faf56712..d73e87ff 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -67,7 +67,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ ln -s /usr/lib64/libblas.so.3 /usr/lib64/libblas.so && \ ln -s /usr/lib64/liblapack.so.3 /usr/lib64/liblapack.so && \ ./scripts/uberenv/uberenv.py \ - --spec "%gcc@13.3.1+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+cuda~uncrustify~openmp~pygeosx cuda_arch=70 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/rocky-spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -78,7 +78,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Extract only TPL's from the previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/docker/tpl-ubuntu-clang-cuda.Dockerfile b/docker/tpl-ubuntu-clang-cuda.Dockerfile index 0fe92b43..1a1b3628 100644 --- a/docker/tpl-ubuntu-clang-cuda.Dockerfile +++ b/docker/tpl-ubuntu-clang-cuda.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get install -y --no-install-recommends \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "%clang@10+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+cuda~uncrustify~openmp~pygeosx cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -73,7 +73,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Extract only TPL's from previous stage diff --git a/docker/tpl-ubuntu-clang.Dockerfile b/docker/tpl-ubuntu-clang.Dockerfile index 55094101..0ef16273 100644 --- a/docker/tpl-ubuntu-clang.Dockerfile +++ b/docker/tpl-ubuntu-clang.Dockerfile @@ -58,6 +58,7 @@ ARG GCC_MAJOR_VERSION RUN apt-get install -y --no-install-recommends \ gfortran-$GCC_MAJOR_VERSION \ + g++-$GCC_MAJOR_VERSION \ libtbb-dev \ make \ bc \ @@ -68,13 +69,26 @@ RUN apt-get install -y --no-install-recommends \ ca-certificates \ git +# Add MPI environment path info +ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ + CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ + MPICC=/usr/bin/mpicc \ + MPICXX=/usr/bin/mpicxx \ + MPIEXEC=/usr/bin/mpirun +# The multi-line definition of arguments does not seem happy +# when a variable uses the value of another variable previously defined on the same line. +ENV OMPI_CC=$CC \ + OMPI_CXX=$CXX + # Run uberenv # Have to create install directory first for uberenv # -k flag is to ignore SSL errors RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ +# Create symlinks to g++ libraries + ln -s /usr/bin/g++-${GCC_MAJOR_VERSION} /usr/bin/g++ && \ ./scripts/uberenv/uberenv.py \ - --spec "%clang@${CLANG_MAJOR_VERSION} ~shared~openmp+docs ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "~shared~openmp+docs %clang@${CLANG_MAJOR_VERSION} ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -85,7 +99,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Extract only TPLs from previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/docker/tpl-ubuntu-gcc.Dockerfile b/docker/tpl-ubuntu-gcc.Dockerfile index 736ca370..2f74d46c 100644 --- a/docker/tpl-ubuntu-gcc.Dockerfile +++ b/docker/tpl-ubuntu-gcc.Dockerfile @@ -111,7 +111,7 @@ RUN apt-get install -y --no-install-recommends \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "%gcc@${GCC_MAJOR_VERSION} ~pygeosx +docs" \ + --spec "~pygeosx +docs %gcc-${GCC_MAJOR_VERSION}" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=${SRC_DIR}/.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -122,7 +122,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack-env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Last step is setting everything for a complete slave that will build GEOSX. FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml index 71c542f7..7e898860 100644 --- a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml +++ b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml @@ -16,13 +16,13 @@ # geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers # # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="%clang@10.0.1+cuda~uncrustify cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers" +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers" # -# python3 ./scripts/uberenv/uberenv.py --spec="%gcc@8.3.1+cuda~uncrustify cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers" +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %gcc-8 ^cuda@11.8.0+allow-unsupported-compilers" # -# python3 ./scripts/uberenv/uberenv.py --spec="%clang@13.0.1+cuda~uncrustify cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers" +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-13 ^cuda@11.8.0+allow-unsupported-compilers" # -# python3 ./scripts/uberenv/uberenv.py --spec="%clang@13.0.1+cuda~openmp~uncrustify cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers" +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~openmp~uncrustify cuda_arch=70 %clang-13 ^cuda@12.2.2+allow-unsupported-compilers" spack: @@ -44,79 +44,109 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers:: - - compiler: - spec: clang@10.0.1 - paths: - cc: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - extra_rpaths: [] - - compiler: - spec: clang@13.0.1 - paths: - cc: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@8.3.1 - paths: - cc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - operating_system: rhel7 - target: ppc64le - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@12.2.1 - paths: - cc: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ - f77: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - operating_system: rhel7 - target: ppc64le - modules: [] - extra_rpaths: [] + toolchains: + clang-13: + - spec: '%[virtuals=c]llvm@13.0.1+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@13.0.1+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@8.3.1' + when: '%fortran' + - spec: '%spectrum-mpi@release.clang.13' + when: '%mpi' + clang-10: + - spec: '%[virtuals=c]llvm@10.0.1+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@10.0.1+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@8.3.1' + when: '%fortran' + - spec: '%spectrum-mpi@release.clang.10' + when: '%mpi' + gcc-12: + - spec: '%c=gcc@12.1.1' + when: '%c' + - spec: '%cxx=gcc@12.1.1' + when: '%cxx' + - spec: '%fortran=gcc@12.1.1' + when: '%fortran' + - spec: '%spectrum-mpi@release.gcc.12' + when: '%mpi' + gcc-8: + - spec: '%c=gcc@8.3.1 ' + when: '%c' + - spec: '%cxx=gcc@8.3.1 ' + when: '%cxx' + - spec: '%fortran=gcc@8.3.1 ' + when: '%fortran' + - spec: '%spectrum-mpi@release.gcc.8' + when: '%mpi' packages: - all: - compiler: [gcc, clang] - providers: - mpi: [spectrum-mpi] - blas: [essl] - lapack: [essl] + mpi: + require: + - spectrum-mpi + blas: + require: + - essl + lapack: + require: + - essl + + llvm: + externals: + - spec: llvm@10.0.1+clang~flang~lld~lldb + prefix: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1 + extra_attributes: + compilers: + c: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang + cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ + flags: {} + extra_rpaths: [] + - spec: llvm@13.0.1+clang~flang~lld~lldb + prefix: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 + extra_attributes: + compilers: + c: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang + cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ + flags: {} + extra_rpaths: [] + gcc: + externals: + - spec: gcc@8.3.1 languages:='c,c++,fortran' + prefix: /usr/tce/packages/gcc/gcc-8.3.1 + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran + flags: + # Fix for "undefined reference to `_gfortran_transfer_integer_write'" + ldlibs: -lgfortran + extra_rpaths: [] + - spec: gcc@12.2.1 languages:='c,c++,fortran' + prefix: /usr/tce/packages/gcc/gcc-12.2.1 + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran + flags: + # Fix for "undefined reference to `_gfortran_transfer_integer_write'" + ldlibs: -lgfortran + extra_rpaths: [] spectrum-mpi: buildable: False externals: - - spec: spectrum-mpi@release%gcc@8.3.1 + - spec: spectrum-mpi@release.gcc.8 %gcc@8.3.1 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1 - - spec: spectrum-mpi@release%gcc@12.2.1 + - spec: spectrum-mpi@release.gcc.12 %gcc@12.2.1 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1 # Previously clang@upstream - clingo doesn't like "upstream" version - - spec: spectrum-mpi@release%clang@10.0.1 + - spec: spectrum-mpi@release.clang.10 %llvm@10.0.1 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1 - - spec: spectrum-mpi@release%clang@13.0.1 + - spec: spectrum-mpi@release.clang.13 %llvm@13.0.1 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1 essl: diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index ebb477b9..02d82253 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -10,17 +10,23 @@ # See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. #------------------------------------------------------------------------------------------------------------ -# geosx@develop%clang@14.0.6 -# geosx@develop%gcc@12.1.1 -# geosx@develop%gcc@noAVX +# geosx@develop %clang-14 +# geosx@develop %gcc-12 +# geosx@develop %gcc-13 +# geosx@develop %gcc-12-noAVX +# geosx@develop %gcc-13-noAVX # # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="%clang@14.0.6 +docs" +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %clang-14" # -# python3 ./scripts/uberenv/uberenv.py --spec="%gcc@12.1.1 +docs" +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-12" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-13" # # No AVX instructions -# python3 ./scripts/uberenv/uberenv.py --spec="%gcc@12noAVX +docs" +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-12-noAVX" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-13-noAVX" spack: @@ -42,122 +48,131 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers:: - - compiler: - spec: clang@14.0.6 - paths: - cc: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang - cxx: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] - - compiler: - spec: clang@19.1.3 - paths: - cc: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang - cxx: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@12.1.1 - paths: - cc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ - f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@12noAVX - paths: - cc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ - f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - flags: - cxxflags: -march=x86-64-v2 -mno-avx512f - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@13.3.1 - paths: - cc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/g++ - f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - operating_system: rhel8 - target: x86_64 - modules: [] - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: [] - - compiler: - spec: gcc@13noAVX - paths: - cc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/g++ - f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - flags: - cxxflags: -march=x86-64-v2 -mno-avx512f - operating_system: rhel8 - target: x86_64 - modules: [] - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: [] + toolchains: + clang-14: + - spec: '%[virtuals=c]llvm@14.0.6+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@14.0.6+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@10.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.clang' + when: '%mpi' + gcc-12: + - spec: '%c=gcc@12.1.1' + when: '%c' + - spec: '%cxx=gcc@12.1.1' + when: '%cxx' + - spec: '%fortran=gcc@12.1.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' + gcc-12-noAVX: + - spec: cxxflags='-march=x86-64-v2 -mno-avx512f' + - spec: '%c=gcc@12.1.1' + when: '%c' + - spec: '%cxx=gcc@12.1.1' + when: '%cxx' + - spec: '%fortran=gcc@12.1.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' + gcc-13: + - spec: '%c=gcc@13.3.1' + when: '%c' + - spec: '%cxx=gcc@13.3.1' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' + gcc-13-noAVX: + - spec: cxxflags='-march=x86-64-v2 -mno-avx512f' + - spec: '%c=gcc@13.3.1' + when: '%c' + - spec: '%cxx=gcc@13.3.1' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' packages: all: target: [ivybridge] - compiler: [gcc, clang] - providers: - mpi: [mvapich2] - blas: [intel-oneapi-mkl] - lapack: [intel-oneapi-mkl] - # Lock down which MPI we are using + mpi: + require: + - mvapich2 + + zlib-api: + require: + - zlib + + blas: + require: + - "openblas" + lapack: + require: + - "openblas" + + # Compilers + llvm: + externals: + - spec: llvm@14.0.6+clang~flang~lld~lldb + prefix: /usr/tce/packages/clang/clang-14.0.6-magic/ + extra_attributes: + compilers: + c: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang + cxx: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang++ + extra_rpaths: [] + gcc: + externals: + - spec: gcc@10.3.1 languages:='c,c++,fortran' + prefix: /usr/tce/packages/gcc/gcc-10.3.1 + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-10.3.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-10.3.1/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran + flags: {} + environment: {} + extra_rpaths: [] + - spec: gcc@12.1.1 languages:=c,c++,fortran + prefix: /usr/tce/packages/gcc/gcc-12.1.1-magic/ + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + extra_rpaths: [] + - spec: gcc@13.3.1 languages:=c,c++,fortran + prefix: /usr/tce/packages/gcc/gcc-13.3.1-magic/ + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran + extra_rpaths: [] + + # Lock down which MPI we are using mvapich2: buildable: False externals: - - spec: mvapich2@2.3%gcc@12.1.1 process_managers=slurm + - spec: mvapich2@2.3.7.gcc process_managers=slurm %gcc@12.1.1 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic - - spec: mvapich2@2.3%gcc@13.3.1 process_managers=slurm + - spec: mvapich2@2.3.7.gcc process_managers=slurm %gcc@13.3.1 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-13.3.1-magic - - spec: mvapich2@2.3%gcc@12noAVX process_managers=slurm - prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic - - spec: mvapich2@2.3%clang@14.0.6 process_managers=slurm + - spec: mvapich2@2.3.7.clang process_managers=slurm %clang@14.0.6 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6-magic - - spec: mvapich2@2.3%clang@19.1.3 process_managers=slurm - prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3-magic - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2022.1.0 threads=openmp - prefix: /usr/tce/packages/mkl/mkl-2022.1.0/ - buildable: False - cuda: - buildable: False - externals: - - spec: cuda@12.6.0 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-12.6.0 - - spec: cuda@12.9.1 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-12.9.1 + # Spack issue prevents compilation: https://github.com/spack/spack/issues/49994 + # intel-oneapi-mkl: + # externals: + # - spec: intel-oneapi-mkl@2022.1.0 threads=openmp %gcc@12.1.1 + # prefix: /usr/tce/packages/mkl/mkl-2022.1.0/ + # buildable: False + # System level packages to not build papi: buildable: False @@ -230,3 +245,13 @@ spack: externals: - spec: gettext@0.19.8.1 prefix: /usr/bin/ + openblas: + buildable: false + externals: + - prefix: /usr + spec: openblas@0.3.15 + zlib: + buildable: false + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/scripts/spack_packages/packages/essl/package.py b/scripts/spack_packages/packages/essl/package.py index f9d0564d..fa2a2cb7 100644 --- a/scripts/spack_packages/packages/essl/package.py +++ b/scripts/spack_packages/packages/essl/package.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package import * - +from spack_repo.builtin.build_systems.bundle import BundlePackage # Recipe for pre-built essl library on blueos machines. # Defines additonal flags for blueos: diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 1e40d064..07116776 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * import warnings import socket @@ -14,6 +14,11 @@ from os import environ as env from os.path import join as pjoin +from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_path + +from spack_repo.builtin.build_systems.cmake import CMakePackage +from spack_repo.builtin.build_systems.cuda import CudaPackage + # Tested specs are located at scripts/spack_configs/<$SYS_TYPE>/spack.yaml (e.g. %clang@10.0.1) # WARNING: +petsc variant is yet to be tested. @@ -84,10 +89,13 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): description='Add support for addr2line.') variant('mathpresso', default=True, description='Build mathpresso.') - variant('cuda_stack_size', default=0, description="Defines the adjusted cuda stack \ + variant('cuda_stack_size', default="0", description="Defines the adjusted cuda stack \ size limit if required. Zero or negative keep default behavior") # SPHINX_BEGIN_DEPENDS + depends_on("c", type="build") + depends_on("cxx", type="build") + depends_on("fortran", type="build") depends_on('cmake@3.24:', type='build') @@ -245,6 +253,10 @@ def _get_sys_type(self, spec): def _get_host_config_path(self, spec, lvarray=False): var = '' + + if 'no-avx' in str(spec.compiler_flags): + var += "-noAVX" + if '+cuda' in spec: var = '-'.join([var, 'cuda']) var += "@" + str(spec['cuda'].version) @@ -258,7 +270,7 @@ def _get_host_config_path(self, spec, lvarray=False): if lvarray: hostname = "lvarray-" + hostname - host_config_path = "%s-%s-%s%s.cmake" % (hostname, self._get_sys_type(spec), (str(spec.compiler)).replace('=',''), var) + host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, self._get_sys_type(spec), (str(spec.compiler.name)), str(spec.compiler.version),var) dest_dir = self.stage.source_path host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) diff --git a/scripts/spack_packages/packages/hdf5/package.py b/scripts/spack_packages/packages/hdf5/package.py index 061251f5..1a2db178 100644 --- a/scripts/spack_packages/packages/hdf5/package.py +++ b/scripts/spack_packages/packages/hdf5/package.py @@ -11,10 +11,8 @@ import os import shutil -import llnl.util.tty as tty - from spack.package import * - +from spack_repo.builtin.build_systems.cmake import CMakePackage class Hdf5(CMakePackage): """HDF5 is a data model, library, and file format for storing and managing @@ -199,6 +197,10 @@ class Hdf5(CMakePackage): multi=False, ) + depends_on("c", type="build") + depends_on("cxx", type="build") + depends_on("fortran", type="build", when="+fortran") + depends_on("cmake@3.12:", type="build") depends_on("mpi", when="+mpi") diff --git a/scripts/spack_packages/packages/mathpresso/package.py b/scripts/spack_packages/packages/mathpresso/package.py index b050bb6b..c3fa63b8 100644 --- a/scripts/spack_packages/packages/mathpresso/package.py +++ b/scripts/spack_packages/packages/mathpresso/package.py @@ -1,5 +1,6 @@ from spack.package import * import subprocess +from spack_repo.builtin.build_systems.cmake import CMakePackage class Mathpresso(CMakePackage): """MathPresso is a mathematical expression parser and JIT compiler.""" diff --git a/scripts/spack_packages/packages/pygeosx/package.py b/scripts/spack_packages/packages/pygeosx/package.py index aaa1ddc4..acf56fa0 100644 --- a/scripts/spack_packages/packages/pygeosx/package.py +++ b/scripts/spack_packages/packages/pygeosx/package.py @@ -3,7 +3,8 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * +from spack_repo.builtin.build_systems.bundle import BundlePackage class Pygeosx(BundlePackage): """This is a set of libraries necessary for the pygeosx ATS environment. diff --git a/scripts/spack_packages/packages/scotch/package.py b/scripts/spack_packages/packages/scotch/package.py index c79025a8..ace6057a 100644 --- a/scripts/spack_packages/packages/scotch/package.py +++ b/scripts/spack_packages/packages/scotch/package.py @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.build_systems import cmake, makefile -from spack.build_systems.cmake import CMakePackage -from spack.build_systems.makefile import MakefilePackage +from spack_repo.builtin.build_systems import cmake, makefile +from spack_repo.builtin.build_systems.cmake import CMakePackage +from spack_repo.builtin.build_systems.makefile import MakefilePackage from spack.package import * diff --git a/scripts/spack_packages/packages/trilinos/package.py b/scripts/spack_packages/packages/trilinos/package.py index 20600933..f3f90123 100644 --- a/scripts/spack_packages/packages/trilinos/package.py +++ b/scripts/spack_packages/packages/trilinos/package.py @@ -12,11 +12,12 @@ import os import sys -from spack import * -from spack.build_environment import dso_suffix -from spack.error import NoHeadersError -from spack.operating_systems.mac_os import macos_version -from spack.pkg.builtin.kokkos import Kokkos +from spack.package import * + +from spack_repo.builtin.build_systems.cmake import CMakePackage +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage +from spack_repo.builtin.packages.kokkos.package import Kokkos # Trilinos is complicated to build, as an inspiration a couple of links to # other repositories which build it: @@ -312,11 +313,12 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage): conflicts('gotype=all', when='@12.15:') # CUDA without wrapper requires clang - for _compiler in spack.compilers.supported_compilers(): - if _compiler != 'clang': - conflicts('+cuda', when='~wrapper %' + _compiler, - msg='trilinos~wrapper+cuda can only be built with the ' - 'Clang compiler') + requires( + "%clang", + when="+cuda~wrapper", + msg="trilinos~wrapper+cuda can only be built with the Clang compiler", + ) + conflicts('+cuda_rdc', when='~cuda') conflicts('+rocm_rdc', when='~rocm') conflicts('+wrapper', when='~cuda') @@ -335,6 +337,10 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage): conflicts('+stokhos', when='%xl_r') # ###################### Dependencies ########################## + depends_on("c", type="build") + depends_on("cxx", type="build") + depends_on("fortran", type="build", when="+fortran") + depends_on('adios2', when='+adios2') depends_on('blas') diff --git a/scripts/spack_packages/packages/vtk/package.py b/scripts/spack_packages/packages/vtk/package.py index 39be1dfb..aa0ee9d8 100644 --- a/scripts/spack_packages/packages/vtk/package.py +++ b/scripts/spack_packages/packages/vtk/package.py @@ -6,8 +6,8 @@ import os import sys -from spack import * - +from spack.package import * +from spack_repo.builtin.build_systems.cmake import CMakePackage class Vtk(CMakePackage): """The Visualization Toolkit (VTK) is an open-source, freely From 5675848d185dca8f3700647e0840d11f5dca7312 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 2 Oct 2025 09:55:59 -0700 Subject: [PATCH 04/55] Add uberenv_libs to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 228b4823..89c70d36 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ .vscode/ spack-*.txt __pycache__ +uberenv_libs From 87ed03191c085002f7657e13276606ee3d8c25cd Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 2 Oct 2025 14:19:49 -0700 Subject: [PATCH 05/55] Update Sherlock specs --- .github/workflows/docker_build_tpls.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index b4181e62..2e733d0a 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -71,14 +71,14 @@ jobs: DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 TPL_DOCKERFILE: docker/Stanford/Dockerfile DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "%gcc@10.1.0~pygeosx~openmp" + SPEC: "~pygeosx~openmp %gcc@10.1.0" INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU RUNS_ON: ubuntu-latest - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 TPL_DOCKERFILE: docker/Stanford/Dockerfile DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "%gcc@10.1.0~pygeosx~openmp cuda_arch=70 ^cuda@12.4.0+allow-unsupported-compilers" + SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU RUNS_ON: ubuntu-latest From b9181ff644f38f7e4151aca73f77b057d76f2f20 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 2 Oct 2025 14:32:32 -0700 Subject: [PATCH 06/55] Fix gcc version --- docker/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/spack.yaml b/docker/spack.yaml index 8db0a342..f9074ed1 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -32,7 +32,7 @@ spack: when: '%c' - spec: '%[virtuals=cxx]llvm@10.0.0+clang~flang~lld~lldb' when: '%cxx' - - spec: '%[virtuals=fortran]gcc@9.4.0' + - spec: '%[virtuals=fortran]gcc@9.5.0' when: '%fortran' - spec: '%openmpi@4.1.2' when: '%mpi' From 7a722e592f1babd51a7a79ea916c5bb750e76c73 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 2 Oct 2025 14:56:15 -0700 Subject: [PATCH 07/55] Fill in missing cxx line for gcc@11 --- docker/spack.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/spack.yaml b/docker/spack.yaml index f9074ed1..1a977a5c 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -127,11 +127,12 @@ spack: c: /usr/bin/gcc-10 cxx: /usr/bin/g++-10 fortran: /usr/bin/gfortran-10 - - spec: gcc@11.4.0 languages:=c,fortran + - spec: gcc@11.4.0 languages:='c,c++,fortran' prefix: /usr extra_attributes: compilers: c: /usr/bin/gcc-11 + cxx: /usr/bin/g++-11 fortran: /usr/bin/gfortran-11 - spec: gcc@12.3.0 languages:='c,c++,fortran' prefix: /usr From e4a993bbc579b0611fd4950909c9f62d4c9871b1 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 3 Oct 2025 02:50:14 -0400 Subject: [PATCH 08/55] Remove deprecated llnl module --- scripts/spack_packages/packages/geosx/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 07116776..bcfdc70a 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -9,8 +9,6 @@ import socket import os -import llnl.util.tty as tty - from os import environ as env from os.path import join as pjoin From f02d90ae9e4345d8fb345bc7fc0349cca677d24e Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 3 Oct 2025 02:50:31 -0400 Subject: [PATCH 09/55] Comment out Sherlock jobs --- .github/workflows/docker_build_tpls.yml | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 2e733d0a..b2a03985 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -67,20 +67,20 @@ jobs: DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 HOST_ARCH: ppc64le RUNS_ON: streak2 - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - TPL_DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "~pygeosx~openmp %gcc@10.1.0" - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU - RUNS_ON: ubuntu-latest - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 - TPL_DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU - RUNS_ON: ubuntu-latest + # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + # TPL_DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + # SPEC: "~pygeosx~openmp %gcc@10.1.0" + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU + # RUNS_ON: ubuntu-latest + # - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 + # TPL_DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + # SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU + # RUNS_ON: ubuntu-latest steps: - name: Checkout From 6293d0120a4924f19c1d2f9bec97877677a4dffb Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 3 Oct 2025 02:50:50 -0400 Subject: [PATCH 10/55] Add RAJA 2025.09 suite --- .../spack_packages/packages/camp/package.py | 158 +++++ .../spack_packages/packages/chai/package.py | 352 +++++++++++ .../spack_packages/packages/raja/package.py | 585 ++++++++++++++++++ .../spack_packages/packages/umpire/package.py | 583 +++++++++++++++++ 4 files changed, 1678 insertions(+) create mode 100644 scripts/spack_packages/packages/camp/package.py create mode 100644 scripts/spack_packages/packages/chai/package.py create mode 100644 scripts/spack_packages/packages/raja/package.py create mode 100644 scripts/spack_packages/packages/umpire/package.py diff --git a/scripts/spack_packages/packages/camp/package.py b/scripts/spack_packages/packages/camp/package.py new file mode 100644 index 00000000..b9fc471c --- /dev/null +++ b/scripts/spack_packages/packages/camp/package.py @@ -0,0 +1,158 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# GEOSX_EDIT_START +# Victor: This is adapted from the radius-spack-configs GitHub repo. +# We're using this since the new 2025.09 packages suite has not been upstreamed to spack yet. +# See PR: https://github.com/LLNL/radiuss-spack-configs/pull/143 +# +# TODO: Delete this file after the new 2025.09 suite becomes available on spack_packages +# +# GEOSX_EDIT_END +# yapf: disable + +from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_string +from spack_repo.builtin.build_systems.cmake import CMakePackage +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage + +from spack.package import * + + +class Camp(CMakePackage, CudaPackage, ROCmPackage): + """ + Compiler agnostic metaprogramming library providing concepts, + type operations and tuples for C++ and cuda + """ + + homepage = "https://github.com/LLNL/camp" + git = "https://github.com/LLNL/camp.git" + url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" + + maintainers("adrienbernede", "kab163", "trws") + + license("BSD-3-Clause") + + version("main", branch="main", submodules=False) + version( + "2025.09.2", + tag="v2025.09.2", + commit="4070ce93a802849d61037310a87c50cc24c9e498", + submodules=False, + ) + version( + "2025.09.0", + tag="v2025.09.0", + commit="b642f29b9d0eee9113bea2791958c29243063e5c", + submodules=False, + ) + version( + "2025.03.0", + tag="v2025.03.0", + commit="ee0a3069a7ae72da8bcea63c06260fad34901d43", + submodules=False, + ) + version( + "2024.07.0", + tag="v2024.07.0", + commit="0f07de4240c42e0b38a8d872a20440cb4b33d9f5", + submodules=False, + ) + version( + "2024.02.1", + tag="v2024.02.1", + commit="79c320fa09db987923b56884afdc9f82f4b70fc4", + submodules=False, + ) + version( + "2024.02.0", + tag="v2024.02.0", + commit="03c80a6c6ab4f97e76a52639563daec71435a277", + submodules=False, + ) + version( + "2023.06.0", + tag="v2023.06.0", + commit="ac34c25b722a06b138bc045d38bfa5e8fa3ec9c5", + submodules=False, + ) + version("2022.10.1", sha256="2d12f1a46f5a6d01880fc075cfbd332e2cf296816a7c1aa12d4ee5644d386f02") + version("2022.10.0", sha256="3561c3ef00bbcb61fe3183c53d49b110e54910f47e7fc689ad9ccce57e55d6b8") + version("2022.03.2", sha256="bc4aaeacfe8f2912e28f7a36fc731ab9e481bee15f2c6daf0cb208eed3f201eb") + version("2022.03.0", sha256="e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721") + version("0.3.0", sha256="129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb") + version("0.2.3", sha256="58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7") + version("0.2.2", sha256="194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819") + version("0.1.0", sha256="fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc") + + depends_on("c", type="build") + depends_on("cxx", type="build") + + # TODO: figure out gtest dependency and then set this default True. + variant("tests", default=False, description="Build tests") + variant("openmp", default=False, description="Build with OpenMP support") + variant("omptarget", default=False, description="Build with OpenMP Target support") + variant("sycl", default=False, description="Build with Sycl support") + + with when("+cuda"): + depends_on("cub", when="^cuda@:10") + + depends_on("blt", type="build") + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") + depends_on("blt@0.6.2:", type="build", when="@2024.02.1:") + depends_on("blt@0.6.1", type="build", when="@2024.02.0") + depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2023.06.0") + + #patch("libstdc++-13-missing-header.patch", when="@:2022.10") + #patch("camp-rocm6.patch", when="@0.2.3 +rocm ^hip@6:") + + conflicts("^blt@:0.3.6", when="+rocm") + + conflicts("+omptarget +rocm") + conflicts("+sycl +omptarget") + conflicts("+sycl +rocm") + conflicts( + "+sycl", + when="@:2024.02.99", + msg="Support for SYCL was introduced in RAJA after 2024.02 release, " + "please use a newer release.", + ) + + def cmake_args(self): + spec = self.spec + + options = [] + + options.append("-DBLT_SOURCE_DIR={0}".format(spec["blt"].prefix)) + + options.append(self.define_from_variant("ENABLE_CUDA", "cuda")) + if spec.satisfies("+cuda"): + options.append("-DCUDA_TOOLKIT_ROOT_DIR={0}".format(spec["cuda"].prefix)) + + if not spec.satisfies("cuda_arch=none"): + cuda_arch = spec.variants["cuda_arch"].value + options.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) + options.append("-DCUDA_ARCH=sm_{0}".format(cuda_arch[0])) + flag = "-arch sm_{0}".format(cuda_arch[0]) + options.append("-DCMAKE_CUDA_FLAGS:STRING={0}".format(flag)) + + options.append(self.define_from_variant("ENABLE_HIP", "rocm")) + if spec.satisfies("+rocm"): + options.append("-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)) + + archs = ";".join(self.spec.variants["amdgpu_target"].value) + options.append("-DCMAKE_HIP_ARCHITECTURES={0}".format(archs)) + options.append("-DGPU_TARGETS={0}".format(archs)) + options.append("-DAMDGPU_TARGETS={0}".format(archs)) + + if spec.satisfies("+omptarget"): + options.append(cmake_cache_string("RAJA_DATA_ALIGN", 64)) + + options.append(self.define_from_variant("ENABLE_TESTS", "tests")) + options.append(self.define_from_variant("ENABLE_OPENMP", "openmp")) + options.append(self.define_from_variant("CAMP_ENABLE_TARGET_OPENMP", "omptarget")) + options.append(self.define_from_variant("ENABLE_SYCL", "sycl")) + + return options diff --git a/scripts/spack_packages/packages/chai/package.py b/scripts/spack_packages/packages/chai/package.py new file mode 100644 index 00000000..e25c0314 --- /dev/null +++ b/scripts/spack_packages/packages/chai/package.py @@ -0,0 +1,352 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# GEOSX_EDIT_START +# Victor: This is adapted from the radius-spack-configs GitHub repo. +# We're using this since the new 2025.09 packages suite has not been upstreamed to spack yet. +# See PR: https://github.com/LLNL/radiuss-spack-configs/pull/143 +# +# TODO: Delete this file after the new 2025.09 suite becomes available on spack_packages +# +# GEOSX_EDIT_END +# yapf: disable + +import socket + +from spack_repo.builtin.build_systems.cached_cmake import ( + CachedCMakePackage, + cmake_cache_option, + cmake_cache_path, + cmake_cache_string, +) +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage +from spack_repo.builtin.packages.blt.package import llnl_link_helpers + +from spack.package import * + + +class Chai(CachedCMakePackage, CudaPackage, ROCmPackage): + """ + Copy-hiding array interface for data migration between memory spaces + """ + + homepage = "https://github.com/LLNL/CHAI" + git = "https://github.com/LLNL/CHAI.git" + tags = ["ecp", "e4s", "radiuss"] + + maintainers("adayton1", "adrienbernede", "davidbeckingsale", "kab163") + + license("BSD-3-Clause") + + version("develop", branch="develop", submodules=False) + version( + "2025.09.0", + tag="v2025.09.0", + commit="352ae302535d9dc5ba50b77bf508c89fc7500d30", + submodules=False, + ) + version( + "2025.03.0", + tag="v2025.03.0", + commit="79f6414a00a89070054ac97baed47d21d10c83a4", + submodules=False, + ) + version( + "2024.07.0", + tag="v2024.07.0", + commit="df7741f1dbbdc5fff5f7d626151fdf1904e62b19", + submodules=False, + ) + version( + "2024.02.2", + tag="v2024.02.2", + commit="5ba0944d862513f600432c34b009824875df27e5", + submodules=False, + ) + version( + "2024.02.1", + tag="v2024.02.1", + commit="7597134729bd3a38b45b67b4dfbf7f199d8106f3", + submodules=False, + ) + version( + "2024.02.0", + tag="v2024.02.0", + commit="31773a2f0d30f3f64c82939f60fc4da32cf33261", + submodules=False, + ) + version( + "2023.06.0", + tag="v2023.06.0", + commit="6fe3470ad020303530af2f3dbbfe18826bd3319b", + submodules=False, + ) + version( + "2022.10.0", + tag="v2022.10.0", + commit="9510efd33b06e4443b15447eebb7dad761822654", + submodules=False, + ) + version( + "2022.03.0", + tag="v2022.03.0", + commit="f0b809de1ac194376866b3ac0f5933d4146ec09e", + submodules=False, + ) + version( + "2.4.0", tag="v2.4.0", commit="77d22da28187245a2c5454cf471c0c352bd98ad7", submodules=True + ) + version( + "2.3.0", tag="v2.3.0", commit="42f3fbcc0b966227b40b4467dc919a4c24f07196", submodules=True + ) + version( + "2.2.2", tag="v2.2.2", commit="56e75fc0f805b2746f3992af0c00c474513e3b24", submodules=True + ) + version( + "2.2.1", tag="v2.2.1", commit="c912f583828ea5963850816e3e232cc45608ccf7", submodules=True + ) + version( + "2.2.0", tag="v2.2.0", commit="18536c61a4817db6b3b3025f35e2dd3ae532330c", submodules=True + ) + version( + "2.1.1", tag="v2.1.1", commit="496911e00d15c350560860f7964cd5fb5ab7f515", submodules=True + ) + version( + "2.1.0", tag="v2.1.0", commit="fff02768068a64970b34760a1041585319edee87", submodules=True + ) + version( + "2.0.0", tag="v2.0.0", commit="63139cf45443b1266950826b165e042c7679b557", submodules=True + ) + version( + "1.2.0", tag="v1.2.0", commit="7bb5bc12e4508db45910d8e2b98444687da7ebf6", submodules=True + ) + version( + "1.1.0", tag="v1.1.0", commit="907d5f40d653a73955387067799913397807adf3", submodules=True + ) + version("1.0", tag="v1.0", commit="501a098ad879dc8deb4a74fcfe8c08c283a10627", submodules=True) + + depends_on("c", type="build") + depends_on("cxx", type="build") + + # Patching Umpire for dual BLT targets import changed MPI target name in Umpire link interface + # We propagate the patch here. + #patch("change_mpi_target_name_umpire_patch.patch", when="@2022.10.0:2023.06.0") + + variant("enable_pick", default=False, when="@:2024", description="Enable pick method") + variant( + "separable_compilation", + default=True, + description="Build with CUDA_SEPARABLE_COMPILATION flag on ", + ) + variant("shared", default=True, description="Build Shared Libs") + variant("mpi", default=False, description="Enable MPI support") + variant("raja", default=False, description="Build plugin for RAJA") + variant("examples", default=True, description="Build examples.") + variant("openmp", default=False, description="Build using OpenMP") + # TODO: figure out gtest dependency and then set this default True + # and remove the +tests conflict below. + variant( + "tests", + default="none", + values=("none", "basic", "benchmarks"), + multi=False, + description="Tests to run", + ) + + depends_on("cmake", type="build") + depends_on("cmake@3.23:", type="build", when="@2024.07.0:") + depends_on("cmake@3.14:", type="build", when="@2022.03.0:2024.2") + depends_on("cmake@3.9:", type="build", when="+cuda") + depends_on("cmake@3.8:", type="build") + + depends_on("blt", type="build") + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") + depends_on("blt@0.6.2:", type="build", when="@2024.02.1:") + depends_on("blt@0.6.1", type="build", when="@2024.02.0") + depends_on("blt@0.5.3", type="build", when="@2023.06.0") + depends_on("blt@0.5.2:0.5.3", type="build", when="@2022.10.0") + depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0") + depends_on("blt@0.4.1:0.5.3", type="build", when="@2.4.0") + depends_on("blt@0.4.0:0.5.3", type="build", when="@2.3.0") + depends_on("blt@0.3.6:0.5.3", type="build", when="@:2.2.2") + conflicts("^blt@:0.3.6", when="+rocm") + + depends_on("umpire") + depends_on("umpire@2025.09.0:", when="@2025.09.0:") + depends_on("umpire@2025.03.0:", when="@2025.03.0:") + depends_on("umpire@2024.07.0", when="@2024.07.0") + depends_on("umpire@2024.02.1", when="@2024.02.1") + depends_on("umpire@2024.02.0", when="@2024.02.0") + depends_on("umpire@2023.06.0", when="@2023.06.0") + depends_on("umpire@2022.10.0:2023.06.0", when="@2022.10.0") + depends_on("umpire@2022.03.0:2023.06.0", when="@2022.03.0") + depends_on("umpire@6.0.0", when="@2.4.0") + depends_on("umpire@4.1.2", when="@2.2.0:2.3.0") + + depends_on("umpire+mpi", when="+mpi") + + with when("+cuda"): + depends_on("umpire+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("umpire+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + with when("@2024.02.0:"): + depends_on("umpire~fmt_header_only") + + with when("+rocm"): + depends_on("umpire+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on( + "umpire+rocm amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch) + ) + + with when("+raja"): + depends_on("raja~openmp", when="~openmp") + depends_on("raja+openmp", when="+openmp") + depends_on("raja@2025.09.0:", when="@2025.09.0:") + depends_on("raja@2025.03.2:", when="@2025.03.1:") + depends_on("raja@2025.03.0:", when="@2025.03.0:") + depends_on("raja@2024.07.0", when="@2024.07.0") + depends_on("raja@2024.02.2", when="@2024.02.2") + depends_on("raja@2024.02.1", when="@2024.02.1") + depends_on("raja@2024.02.0", when="@2024.02.0") + depends_on("raja@2023.06.0", when="@2023.06.0") + depends_on("raja@2022.10.0:2023.06.0", when="@2022.10.0") + depends_on("raja@2022.03.0:2023.06.0", when="@2022.03.0") + depends_on("raja@0.14.0", when="@2.4.0") + depends_on("raja@0.13.0", when="@2.3.0") + depends_on("raja@0.12.0", when="@2.2.0:2.2.2") + + with when("+cuda"): + depends_on("raja+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("raja+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + with when("+rocm"): + depends_on("raja+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on( + "raja+rocm amdgpu_target={0}".format(arch), + when="amdgpu_target={0}".format(arch), + ) + + depends_on("mpi", when="+mpi") + + def _get_sys_type(self, spec): + sys_type = spec.architecture + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + return sys_type + + @property + def cache_name(self): + hostname = socket.gethostname() + if "SYS_TYPE" in env: + hostname = hostname.rstrip("1234567890") + return "{0}-{1}-{2}@{3}-{4}.cmake".format( + hostname, + self._get_sys_type(self.spec), + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash(8), + ) + + def initconfig_compiler_entries(self): + spec = self.spec + compiler = self.compiler + # Default entries are already defined in CachedCMakePackage, inherit them: + entries = super().initconfig_compiler_entries() + + if spec.satisfies("+rocm ^blt@:0.6"): + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + + llnl_link_helpers(entries, spec, compiler) + + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + entries.append("#------------------{0}".format("-" * 30)) + entries.append("# Package custom hardware settings") + entries.append("#------------------{0}\n".format("-" * 30)) + + if spec.satisfies("+cuda"): + entries.append(cmake_cache_option("ENABLE_CUDA", True)) + if spec.satisfies("+separable_compilation"): + entries.append(cmake_cache_option("CMAKE_CUDA_SEPARABLE_COMPILATION", True)) + entries.append(cmake_cache_option("CUDA_SEPARABLE_COMPILATION", True)) + else: + entries.append(cmake_cache_option("ENABLE_CUDA", False)) + + if spec.satisfies("+rocm"): + entries.append(cmake_cache_option("ENABLE_HIP", True)) + else: + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + return entries + + def initconfig_mpi_entries(self): + spec = self.spec + + entries = super(Chai, self).initconfig_mpi_entries() + entries.append(cmake_cache_option("ENABLE_MPI", spec.satisfies("+mpi"))) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "CHAI_" if spec.satisfies("@2022.03.0:") else "" + + # TPL locations + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# TPLs") + entries.append("#------------------{0}\n".format("-" * 60)) + + # - BLT + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + + # - RAJA + if spec.satisfies("+raja"): + entries.append(cmake_cache_option("{}ENABLE_RAJA_PLUGIN".format(option_prefix), True)) + entries.append(cmake_cache_path("RAJA_DIR", spec["raja"].prefix)) + else: + entries.append(cmake_cache_option("{}ENABLE_RAJA_PLUGIN".format(option_prefix), False)) + + # - Umpire + entries.append(cmake_cache_path("umpire_DIR", spec["umpire"].prefix)) + + # Build options + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# Build Options") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_string("CMAKE_BUILD_TYPE", spec.variants["build_type"].value)) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", spec.satisfies("+shared"))) + + # Generic options that have a prefixed equivalent in CHAI CMake + entries.append(cmake_cache_option("ENABLE_OPENMP", spec.satisfies("+openmp"))) + entries.append(cmake_cache_option("ENABLE_EXAMPLES", spec.satisfies("+examples"))) + entries.append(cmake_cache_option("ENABLE_DOCS", False)) + if spec.satisfies("tests=benchmarks"): + # BLT requires ENABLE_TESTS=True to enable benchmarks + entries.append(cmake_cache_option("ENABLE_BENCHMARKS", True)) + entries.append(cmake_cache_option("ENABLE_TESTS", True)) + else: + entries.append(cmake_cache_option("ENABLE_TESTS", not spec.satisfies("tests=none"))) + + # Prefixed options that used to be name without one + entries.append( + cmake_cache_option( + "{}ENABLE_PICK".format(option_prefix), spec.satisfies("+enable_pick") + ) + ) + + return entries + + def cmake_args(self): + return [] diff --git a/scripts/spack_packages/packages/raja/package.py b/scripts/spack_packages/packages/raja/package.py new file mode 100644 index 00000000..71186d84 --- /dev/null +++ b/scripts/spack_packages/packages/raja/package.py @@ -0,0 +1,585 @@ +# Copyright 2013-2025 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# GEOSX_EDIT_START +# Victor: This is adapted from the radius-spack-configs GitHub repo. +# We're using this since the new 2025.09 packages suite has not been upstreamed to spack yet. +# See PR: https://github.com/LLNL/radiuss-spack-configs/pull/143 +# +# TODO: Delete this file after the new 2025.09 suite becomes available on spack_packages +# +# GEOSX_EDIT_END +# yapf: disable + +import socket + +from spack_repo.builtin.build_systems.cached_cmake import ( + CachedCMakePackage, + cmake_cache_option, + cmake_cache_path, + cmake_cache_string, +) +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage +from spack_repo.builtin.packages.blt.package import llnl_link_helpers + +from spack.package import * + + +# Starting with 2022.03.0, the only submodule we want to fetch is tpl/desul +# since there is no package for it. Other RAJA submodules are defined as +# dependencies. +def submodules(package): + submodules = [] + submodules.append("tpl/desul") + return submodules + + +class Raja(CachedCMakePackage, CudaPackage, ROCmPackage): + """RAJA Parallel Framework.""" + + homepage = "https://github.com/LLNL/RAJA" + git = "https://github.com/LLNL/RAJA.git" + tags = ["radiuss", "e4s"] + + maintainers("adrienbernede", "davidbeckingsale", "kab163") + + license("BSD-3-Clause") + + version("develop", branch="develop", submodules=submodules) + version("main", branch="main", submodules=submodules) + version( + "2025.09.0", + tag="v2025.09.0", + commit="ca756788dbdd43fec2a3840389126ae94a905d5f", + submodules=submodules, + ) + version( + "2025.03.2", + tag="v2025.03.2", + commit="6e36a94380adbe88fed11a3213fc08461428ece0", + submodules=submodules, + ) + version( + "2025.03.0", + tag="v2025.03.0", + commit="1d70abf171474d331f1409908bdf1b1c3fe19222", + submodules=submodules, + ) + version( + "2024.07.0", + tag="v2024.07.0", + commit="4d7fcba55ebc7cb972b7cc9f6778b48e43792ea1", + submodules=submodules, + ) + version( + "2024.02.2", + tag="v2024.02.2", + commit="593f756b14ac57ded33ee61d8d2292d4beb840e6", + submodules=submodules, + ) + version( + "2024.02.1", + tag="v2024.02.1", + commit="3ada0950b0774ec907d30a9eceaf6af7478b833b", + submodules=submodules, + ) + version( + "2024.02.0", + tag="v2024.02.0", + commit="82d1b926ada0fbb15a4a6e0adadc30c715cfda7b", + submodules=submodules, + ) + version( + "2023.06.1", + tag="v2023.06.1", + commit="9b5f61edf3aa1e6fdbc9a4b30828c81504639963", + submodules=submodules, + ) + version( + "2023.06.0", + tag="v2023.06.0", + commit="e330b2560747d5417cd7bd265fab3fb91d32ecbd", + submodules=submodules, + ) + version( + "2022.10.5", + tag="v2022.10.5", + commit="3774f51339459bbbdb77055aa23f82919b6335b6", + submodules=submodules, + ) + version( + "2022.10.4", + tag="v2022.10.4", + commit="c2a6b1740759ae3ae7c85b35e20dbffbe235355d", + submodules=submodules, + ) + version( + "2022.03.0", + tag="v2022.03.0", + commit="4351fe6a50bd579511a625b017c9e054885e7fd2", + submodules=submodules, + ) + version( + "0.14.0", + tag="v0.14.0", + commit="357933a42842dd91de5c1034204d937fce0a2a44", + submodules="True", + ) + version( + "0.13.0", + tag="v0.13.0", + commit="3047fa720132d19ee143b1fcdacaa72971f5988c", + submodules="True", + ) + version( + "0.12.1", + tag="v0.12.1", + commit="9cb6370bb2868e35ebba23cdce927f5f7f9da530", + submodules="True", + ) + version( + "0.12.0", + tag="v0.12.0", + commit="32d92e38da41cc8d4db25ec79b9884a73a0cb3a1", + submodules="True", + ) + version( + "0.11.0", + tag="v0.11.0", + commit="0502b9b69c4cb60aa0afbdf699b555c76cb18f22", + submodules="True", + ) + version( + "0.10.1", + tag="v0.10.1", + commit="be91e040130678b1350dbda56cc352433db758bd", + submodules="True", + ) + version( + "0.10.0", + tag="v0.10.0", + commit="53cb89cf788d28bc4ed2b4e6f75483fdd26024aa", + submodules="True", + ) + version( + "0.9.0", tag="v0.9.0", commit="df7ca1fa892b6ac4147c614d2d739d5022f63fc7", submodules="True" + ) + version( + "0.8.0", tag="v0.8.0", commit="8d19a8c2cbac611de6f92ad8852b9f3454b27e63", submodules="True" + ) + version( + "0.7.0", tag="v0.7.0", commit="caa33b371b586dfae3d8569caee91c5eddfd7b31", submodules="True" + ) + version( + "0.6.0", tag="v0.6.0", commit="cc7a97e8b4e52c3de820c9dfacd358822a147871", submodules="True" + ) + version( + "0.5.3", tag="v0.5.3", commit="1ca35c0ed2a43a3fa9c6cd70c5d25f16d88ecd8c", submodules="True" + ) + version( + "0.5.2", tag="v0.5.2", commit="4d5c3d5d7f311838855f7010810610349e729f64", submodules="True" + ) + version( + "0.5.1", tag="v0.5.1", commit="bf340abe5199d7e051520913c9a7a5de336b5820", submodules="True" + ) + version( + "0.5.0", tag="v0.5.0", commit="9b539d84fdad049f65caeba836f41031f5baf4cc", submodules="True" + ) + version( + "0.4.1", tag="v0.4.1", commit="3618cfe95d6a442fa50fbe7bfbcf654cf9f800b9", submodules="True" + ) + version( + "0.4.0", tag="v0.4.0", commit="31b2a48192542c2da426885baa5af0ed57606b78", submodules="True" + ) + + depends_on("cxx", type="build") # generated + + # export targets when building pre-2.4.0 release with BLT 0.4.0+ + patch( + "https://github.com/LLNL/RAJA/commit/eca1124ee4af380d6613adc6012c307d1fd4176b.patch?full_index=1", + sha256="12bb78c00b6683ad3e7fd4e3f87f9776bae074b722431b79696bc862816735ef", + when="@:0.13.0 ^blt@0.4:", + ) + + # Backward compatibility is stopped from ROCm 6.0 + # Future relase will have the change from PR https://github.com/LLNL/RAJA/pull/1568 + patch( + "https://github.com/LLNL/RAJA/commit/406eb8dee05a41eb32c421c375688a4863b60642.patch?full_index=1", + sha256="d9ce5ef038555cbccb330a9016b7be77e56ae0660583cba955dab9d0297a4b07", + when="^hip@6.0", + ) + + # Fix compilation issue reported by Intel from their new compiler version + patch( + "https://github.com/LLNL/RAJA/pull/1668.patch?full_index=1", + sha256="c0548fc5220f24082fb2592d5b4e8b7c8c783b87906d5f0950d53953d25161f6", + when="@2024.02.1:2024.02.99 %oneapi@2025:", + ) + + variant("openmp", default=False, description="Build OpenMP backend") + variant("shared", default=False, description="Build shared libs") + variant("desul", default=False, description="Build desul atomics backend") + variant("vectorization", default=True, description="Build SIMD/SIMT intrinsics support") + variant( + "omptask", default=False, description="Build OpenMP task variants of internal algorithms" + ) + variant("omptarget", default=False, description="Build OpenMP on target device support") + variant("sycl", default=False, description="Build sycl backend") + variant("gpu-profiling", default=False, description="Enable GPU profiling") + + variant("plugins", default=False, description="Enable runtime plugins") + variant("examples", default=True, description="Build examples.") + variant("exercises", default=True, description="Build exercises.") + # TODO: figure out gtest dependency and then set this default True + # and remove the +tests conflict below. + variant("tests", default=False, description="Build tests") + + # we don't use variants to express the failing test, we only add a variant to + # define whether we want to run all the tests (including those known to fail) + # or only the passing ones. + variant( + "run-all-tests", + default=False, + description="Run all the tests, including those known to fail.", + ) + + variant( + "lowopttest", + default=False, + description="For developers, lowers optimization level to pass tests with some compilers", + ) + + depends_on("cxx", type="build") # generated + + depends_on("blt", type="build") + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") + depends_on("blt@0.6.2:", type="build", when="@2024.02.1:") + depends_on("blt@0.6.1", type="build", when="@2024.02.0") + depends_on("blt@0.5.3", type="build", when="@2023.06.0:2023.06.1") + depends_on("blt@0.5.2:0.5.3", type="build", when="@2022.10.5") + depends_on("blt@0.5.0:0.5.3", type="build", when="@0.14.1:2022.10.4") + depends_on("blt@0.4.1", type="build", when="@0.14.0") + depends_on("blt@0.4.0:0.4.1", type="build", when="@0.13.0") + depends_on("blt@0.3.6:0.4.1", type="build", when="@:0.12.0") + conflicts("^blt@:0.3.6", when="+rocm") + + depends_on("camp") + depends_on("camp+openmp", when="+openmp") + depends_on("camp+omptarget", when="+omptarget") + depends_on("camp+sycl", when="+sycl") + depends_on("camp@main", when="@develop") + depends_on("camp@2025.09.2:", when="@2025.09.0:") + depends_on("camp@2025.03.0:", when="@2025.03.0:") + depends_on("camp@2024.07.0:", when="@2024.07.0:") + depends_on("camp@2024.02.1:", when="@2024.02.1:") + depends_on("camp@2024.02.0", when="@2024.02.0") + depends_on("camp@2023.06.0", when="@2023.06.0:2023.06.1") + depends_on("camp@2022.10.1:2023.06.0", when="@2022.10.3:2022.10.5") + depends_on("camp@2022.10.0:2023.06.0", when="@2022.10.0:2022.10.2") + depends_on("camp@2022.03.2", when="@2022.03.0:2022.03.1") + depends_on("camp@0.2.2:0.2.3", when="@0.14.0") + depends_on("camp@0.1.0", when="@0.10.0:0.13.0") + + depends_on("cmake@3.23:", when="@2024.07.0:", type="build") + depends_on("cmake@3.23:", when="@2022.10.0:2024.02.2+rocm", type="build") + depends_on("cmake@3.20:", when="@2022.10.0:2024.02.2", type="build") + depends_on("cmake@3.20:", when="@:2022.03+rocm", type="build") + depends_on("cmake@3.14:", when="@:2022.03", type="build") + + depends_on("llvm-openmp", when="+openmp %apple-clang") + + depends_on("rocprim", when="+rocm") + with when("+rocm @0.12.0:"): + depends_on("camp+rocm") + for arch in ROCmPackage.amdgpu_targets: + depends_on( + "camp+rocm amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch) + ) + conflicts("+openmp", when="@:2022.03") + + with when("+cuda @0.12.0:"): + depends_on("camp+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("camp +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + conflicts("+gpu-profiling", when="~cuda~rocm", msg="GPU profiling requires CUDA or ROCm") + conflicts("+gpu-profiling +cuda", when="@:2022.02.99") + conflicts("+gpu-profiling +rocm", when="@:2022.02.99") + + conflicts("+omptarget +rocm") + conflicts("+sycl +omptarget") + conflicts("+sycl +rocm") + conflicts( + "+sycl", + when="@:2024.02.99", + msg="Support for SYCL was introduced in RAJA after 2024.02 release, " + "please use a newer release.", + ) + + def _get_sys_type(self, spec): + sys_type = spec.architecture + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + return sys_type + + @property + def libs(self): + shared = "+shared" in self.spec + return find_libraries("libRAJA", root=self.prefix, shared=shared, recursive=True) + + @property + def cache_name(self): + hostname = socket.gethostname() + if "SYS_TYPE" in env: + hostname = hostname.rstrip("1234567890") + return "{0}-{1}-{2}@{3}-{4}.cmake".format( + hostname, + self._get_sys_type(self.spec), + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash(8), + ) + + def initconfig_compiler_entries(self): + spec = self.spec + compiler = self.compiler + # Default entries are already defined in CachedCMakePackage, inherit them: + entries = super().initconfig_compiler_entries() + + if spec.satisfies("+rocm ^blt@:0.6"): + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + + llnl_link_helpers(entries, spec, compiler) + + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + entries.append("#------------------{0}".format("-" * 30)) + entries.append("# Package custom hardware settings") + entries.append("#------------------{0}\n".format("-" * 30)) + + entries.append(cmake_cache_option("ENABLE_OPENMP", spec.satisfies("+openmp"))) + entries.append(cmake_cache_option("ENABLE_CUDA", spec.satisfies("+cuda"))) + + if spec.satisfies("+rocm"): + entries.append(cmake_cache_option("ENABLE_HIP", True)) + hipcc_flags = [] + if self.spec.satisfies("@2025.09.0:"): + hipcc_flags.append("-std=c++17") + elif self.spec.satisfies("@0.14.0:2025.09.0"): + hipcc_flags.append("-std=c++14") + entries.append(cmake_cache_string("HIP_HIPCC_FLAGS", " ".join(hipcc_flags))) + else: + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "RAJA_" if spec.satisfies("@0.14.0:") else "" + + # TPL locations + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# TPLs") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + if "camp" in self.spec: + entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) + + # Build options + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# Build Options") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_string("CMAKE_BUILD_TYPE", spec.variants["build_type"].value)) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", spec.satisfies("+shared"))) + + entries.append(cmake_cache_option("RAJA_ENABLE_DESUL_ATOMICS", spec.satisfies("+desul"))) + + entries.append( + cmake_cache_option("RAJA_ENABLE_VECTORIZATION", spec.satisfies("+vectorization")) + ) + + entries.append(cmake_cache_option("RAJA_ENABLE_OPENMP_TASK", spec.satisfies("+omptask"))) + + entries.append( + cmake_cache_option("RAJA_ENABLE_TARGET_OPENMP", spec.satisfies("+omptarget")) + ) + + entries.append(cmake_cache_option("RAJA_ENABLE_SYCL", spec.satisfies("+sycl"))) + entries.append( + cmake_cache_option("RAJA_ENABLE_NV_TOOLS_EXT", spec.satisfies("+gpu-profiling +cuda")) + ) + entries.append( + cmake_cache_option("RAJA_ENABLE_ROCTX", spec.satisfies("+gpu-profiling +rocm")) + ) + + if spec.satisfies("+lowopttest"): + entries.append(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", "-O1")) + + # C++17 + if (spec.satisfies("@2025.09.0:") or + (spec.satisfies("@2024.07.0:") and spec.satisfies("+sycl"))): + entries.append(cmake_cache_string("BLT_CXX_STD", "c++17")) + # C++14 + elif spec.satisfies("@0.14.0:2025.09.0"): + entries.append(cmake_cache_string("BLT_CXX_STD", "c++14")) + + if spec.satisfies("+desul"): + if spec.satisfies("+cuda"): + entries.append(cmake_cache_string("CMAKE_CUDA_STANDARD", "14")) + + entries.append( + cmake_cache_option("RAJA_ENABLE_RUNTIME_PLUGINS", spec.satisfies("+plugins")) + ) + + if spec.satisfies("+omptarget"): + entries.append( + cmake_cache_string( + "BLT_OPENMP_COMPILE_FLAGS", "-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" + ) + ) + entries.append( + cmake_cache_string( + "BLT_OPENMP_LINK_FLAGS", "-fopenmp;-fopenmp-targets=nvptx64-nvidia-cuda" + ) + ) + + entries.append( + cmake_cache_option( + "{}ENABLE_EXAMPLES".format(option_prefix), spec.satisfies("+examples") + ) + ) + if spec.satisfies("@0.14.0:"): + entries.append( + cmake_cache_option( + "{}ENABLE_EXERCISES".format(option_prefix), spec.satisfies("+exercises") + ) + ) + else: + entries.append(cmake_cache_option("ENABLE_EXERCISES", spec.satisfies("+exercises"))) + + # TODO: Treat the workaround when building tests with spack wrapper + # For now, removing it to test CI, which builds tests outside of wrapper. + # Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS which + # is used by the spack compiler wrapper. This can go away when BLT + # removes -Werror from GTest flags + # + # if self.spec.satisfies("%clang target=ppc64le:") + # or (not self.run_tests and not spec.satisfies("+tests")): + if not self.run_tests and not spec.satisfies("+tests"): + entries.append(cmake_cache_option("ENABLE_TESTS", False)) + else: + entries.append(cmake_cache_option("ENABLE_TESTS", True)) + if not spec.satisfies("+run-all-tests"): + if spec.satisfies("%clang@12.0.0:13.9.999"): + entries.append( + cmake_cache_string( + "CTEST_CUSTOM_TESTS_IGNORE", + "test-algorithm-sort-OpenMP.exe;test-algorithm-stable-sort-OpenMP.exe", + ) + ) + excluded_tests = [ + "test-algorithm-sort-Cuda.exe", + "test-algorithm-stable-sort-Cuda.exe", + "test-algorithm-sort-OpenMP.exe", + "test-algorithm-stable-sort-OpenMP.exe", + ] + if spec.satisfies("+cuda %clang@12.0.0:13.9.999"): + entries.append( + cmake_cache_string("CTEST_CUSTOM_TESTS_IGNORE", ";".join(excluded_tests)) + ) + if spec.satisfies("+cuda %xl@16.1.1.12"): + entries.append( + cmake_cache_string( + "CTEST_CUSTOM_TESTS_IGNORE", + "test-algorithm-sort-Cuda.exe;test-algorithm-stable-sort-Cuda.exe", + ) + ) + + entries.append(cmake_cache_option("RAJA_HOST_CONFIG_LOADED", True)) + + return entries + + def cmake_args(self): + return [] + + @property + def build_relpath(self): + """Relative path to the cmake build subdirectory.""" + return join_path("..", self.build_dirname) + + @run_after("install") + def setup_build_tests(self): + """Copy the build test files after the package is installed to a + relative install test subdirectory for use during `spack test run`.""" + # Now copy the relative files + cache_extra_test_sources(self, self.build_relpath) + + # Ensure the path exists since relying on a relative path at the + # same level as the normal stage source path. + mkdirp(install_test_root(self)) + + @property + def _extra_tests_path(self): + # TODO: The tests should be converted to re-build and run examples + # TODO: using the installed libraries. + return join_path(install_test_root(self), self.build_relpath, "bin") + + def run_example(self, exe, expected): + """run and check outputs of the example""" + with working_dir(self._extra_tests_path): + example = which(exe) + if example is None: + raise SkipTest(f"{exe} was not built") + + out = example(output=str.split, error=str.split) + check_outputs(expected, out) + + def test_line_of_sight(self): + """check line of sight example""" + self.run_example( + "ex5_line-of-sight_solution", + [r"C-style sequential", r"RAJA sequential", r"result -- PASS"], + ) + + def test_stencil_offset_layout(self): + """check stencil offset layout""" + self.run_example( + "ex6_stencil-offset-layout_solution", [r"RAJA Views \(permuted\)", r"result -- PASS"] + ) + + def test_tiled_matrix(self): + """check tiled matrix transpose""" + self.run_example( + "ex8_tiled-matrix-transpose_solution", + [r"C-version", r"RAJA sequential", r"result -- PASS"], + ) + + def test_dynamic_tile(self): + """check kernel dynamic tile""" + self.run_example("kernel-dynamic-tile", [r"Running index", r"(24,24)"]) + + def test_plugin_example(self): + """check plugin example""" + self.run_example("plugin-example", [r"Launching host kernel for the 10 time"]) + + def test_matrix_multiply(self): + """check batched matrix multiple tutorial""" + self.run_example( + "tut_batched-matrix-multiply", [r"batched matrix multiplication", r"result -- PASS"] + ) + + def test_wave_equation(self): + """check wave equation""" + self.run_example("wave-eqn", [r"Max Error = 2", r"Evolved solution to time"]) diff --git a/scripts/spack_packages/packages/umpire/package.py b/scripts/spack_packages/packages/umpire/package.py new file mode 100644 index 00000000..2c0756ad --- /dev/null +++ b/scripts/spack_packages/packages/umpire/package.py @@ -0,0 +1,583 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# GEOSX_EDIT_START +# Victor: This is adapted from the radius-spack-configs GitHub repo. +# We're using this since the new 2025.09 packages suite has not been upstreamed to spack yet. +# See PR: https://github.com/LLNL/radiuss-spack-configs/pull/143 +# +# TODO: Delete this file after the new 2025.09 suite becomes available on spack_packages +# +# GEOSX_EDIT_END +# yapf: disable + +import os +import socket + +from spack_repo.builtin.build_systems.cached_cmake import ( + CachedCMakePackage, + cmake_cache_option, + cmake_cache_path, + cmake_cache_string, +) +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage +from spack_repo.builtin.packages.blt.package import llnl_link_helpers + +from spack.package import * + + +class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage): + """An application-focused API for memory management on NUMA & GPU + architectures""" + + homepage = "https://github.com/LLNL/Umpire" + git = "https://github.com/LLNL/Umpire.git" + tags = ["radiuss", "e4s"] + + maintainers("adrienbernede", "davidbeckingsale", "kab163") + + license("MIT") + + version("develop", branch="develop", submodules=False) + version( + "2025.09.0", + tag="v2025.09.0", + commit="6b0ea9edbbbc741c8a429768d946549cd3bd7d33", + submodules=False, + ) + version( + "2025.03.0", + tag="v2025.03.0", + commit="1ed0669c57f041baa1f1070693991c3a7a43e7ee", + submodules=False, + ) + version( + "2024.07.0", + tag="v2024.07.0", + commit="abd729f40064175e999a83d11d6b073dac4c01d2", + submodules=False, + ) + version( + "2024.02.1", + tag="v2024.02.1", + commit="3058d562fc707650e904f9321b1ee9bcebad3ae2", + submodules=False, + ) + version( + "2024.02.0", + tag="v2024.02.0", + commit="1db3fef913a70d8882ca510a4830c77c388873e0", + submodules=False, + ) + version( + "2023.06.0", + tag="v2023.06.0", + commit="1e5ef604de88e81bb3b6fc4a5d914be833529da5", + submodules=False, + ) + version( + "2022.10.0", + tag="v2022.10.0", + commit="93b1441aaa258c1dcd211a552b75cff6461a2a8a", + submodules=False, + ) + version( + "2022.03.1", + tag="v2022.03.1", + commit="6bf231bdbbc797df70d60027ddb714ac2ef7c0a1", + submodules=False, + ) + version( + "2022.03.0", + tag="v2022.03.0", + commit="2db6224ae0c3f3e0bbd6722e95c1167b7f79be7b", + submodules=False, + ) + version( + "6.0.0", tag="v6.0.0", commit="5f886b4299496b7cb6f9d62dc1372ce6d3832fbc", submodules=True + ) + version( + "5.0.1", tag="v5.0.1", commit="5201a47a35e3844160dcbecd0916f8c96aa7dd07", submodules=True + ) + version( + "5.0.0", tag="v5.0.0", commit="2196615500057e068f2d93597b4f8da89d582afb", submodules=True + ) + version( + "4.1.2", tag="v4.1.2", commit="447f4640eff7b8f39d3c59404f3b03629b90c021", submodules=True + ) + version( + "4.1.1", tag="v4.1.1", commit="df1830b5ea04185f93fc229ed667da62d1d2d6e3", submodules=True + ) + version( + "4.1.0", tag="v4.1.0", commit="62f146d9c6b291cd79b29386dcb84b30f7b4212e", submodules=True + ) + version( + "4.0.1", tag="v4.0.1", commit="06d8692d084a88b15b0ef2794a1da779197da747", submodules=True + ) + version( + "4.0.0", tag="v4.0.0", commit="bdd598512516bdc4238502f180c8a7e145c6e68f", submodules=True + ) + version( + "3.0.0", tag="v3.0.0", commit="657676087574f61f9d90b996a3bdbf4e1cdfc92e", submodules=True + ) + version( + "2.1.0", tag="v2.1.0", commit="52e10c05cd40dfdfde186c1e63213695f5aeaf65", submodules=True + ) + version( + "2.0.0", tag="v2.0.0", commit="0dc8b4736357645b99632ee7c17a3dc6af771fbb", submodules=True + ) + version( + "1.1.0", tag="v1.1.0", commit="3db26e6a2626ee8c0cfa5c9769cfac6e33587122", submodules=True + ) + version( + "1.0.1", tag="v1.0.1", commit="a6741073431cab3a7a2434f9119a54d18e9978f4", submodules=True + ) + version( + "1.0.0", tag="v1.0.0", commit="82482fd7450ab378db110f06f7e0302112c22c05", submodules=True + ) + version( + "0.3.5", tag="v0.3.5", commit="a283977bb548cbaa0221bdb6c9832f7834f69e74", submodules=True + ) + version( + "0.3.4", tag="v0.3.4", commit="20a77408d8ae467af21d5802d14afe54f1253694", submodules=True + ) + version( + "0.3.3", tag="v0.3.3", commit="715a8bd003eb1d9db1e2ac7ff2c6251cfd445c27", submodules=True + ) + version( + "0.3.2", tag="v0.3.2", commit="06f37f2011fa4d9482f15e04fc206e2e7b7aa9e2", submodules=True + ) + version( + "0.3.1", tag="v0.3.1", commit="aef223065fdfe85d1e46bab95e3874821702891a", submodules=True + ) + version( + "0.3.0", tag="v0.3.0", commit="529004f9e88fbb49ee93a97465ff904be249039c", submodules=True + ) + version( + "0.2.4", tag="v0.2.4", commit="f774afae69b6f2e5c99ea8bf5660ccf68bd5436d", submodules=True + ) + version( + "0.2.3", tag="v0.2.3", commit="af158291f574701aabb6a2b16e6536aefaf4496e", submodules=True + ) + version( + "0.2.2", tag="v0.2.2", commit="68f4b86fd877c9ca00c9438c603e5dbc40d5f219", submodules=True + ) + version( + "0.2.1", tag="v0.2.1", commit="c22df368e2f52398351f49fbe2522bd1150ad171", submodules=True + ) + version( + "0.2.0", tag="v0.2.0", commit="7910b8d4dbfe83faacf65e864304ca916e34b86c", submodules=True + ) + version( + "0.1.4", tag="v0.1.4", commit="c2848289ba9d8c85346610d25af9531b82c50fc3", submodules=True + ) + version( + "0.1.3", tag="v0.1.3", commit="cc347edeb17f5f30f694aa47f395d17369a2e449", submodules=True + ) + + depends_on("c", type="build") # generated + depends_on("cxx", type="build") # generated + depends_on("fortran", type="build") # generated + + # Some projects importing both camp and umpire targets end up with conflicts in BLT targets + # import. This is not addressing the root cause, which will be addressed in BLT@5.4.0 and will + # require adapting umpire build system. + #patch("dual_blt_import_umpire_2022.10_2023.06.patch", when="@2022.10.0:2023.06.0") + #patch("export_includes.patch", when="@2022.10.0") + #patch("std-filesystem-pr784.patch", when="@2022.03.1 +rocm ^blt@0.5.2:") + #patch("camp_target_umpire_3.0.0.patch", when="@3.0.0") + #patch("cmake_version_check.patch", when="@4.1") + #patch("missing_header_for_numeric_limits.patch", when="@4.1:5.0.1") + + # export targets when building pre-6.0.0 release with BLT 0.4.0+ + patch( + "https://github.com/LLNL/Umpire/commit/5773ce9af88952c8d23f9bcdcb2e503ceda40763.patch?full_index=1", + sha256="f3b21335ce5cf9c0fecc852a94dfec90fb5703032ac97f9fee104af9408d8899", + when="@:5.0.1 ^blt@0.4:", + ) + + # https://github.com/LLNL/Umpire/pull/805 + patch( + "https://github.com/LLNL/Umpire/commit/47ff0aa1f7a01a917c3b7ac618e8a9e44a10fd25.patch?full_index=1", + sha256="802f074a05e1cb1f428e13d99c5fcb1435f86bd8f36a1ea2f7b6756e6625e0a0", + when="@2022.10.0", + ) + + # https://github.com/LLNL/Umpire/pull/816 + patch( + "https://github.com/LLNL/Umpire/commit/2292d1d6078f6d9523b7ad0886ffa053644569d5.patch?full_index=1", + sha256="170dbcadb9ae36c7e211119c17a812695f11f4fe1be290b750f7af4fb4896192", + when="@2022.10.0", + ) + + # https://github.com/LLNL/Umpire/pull/853 + patch( + "https://github.com/LLNL/Umpire/commit/4bd9b2ded81d3216b3f62e2aad62d0e34fe2c256.patch?full_index=1", + sha256="c9ddae1f4212cef72e1050b6ac482ce5b795dad4977d2462cff2e884b8d7aff5", + when="@2022.10:2023.06", + ) + + variant("fortran", default=False, description="Build C/Fortran API") + variant("c", default=True, description="Build C API") + variant("mpi", default=False, description="Enable MPI support") + variant("ipc_shmem", default=False, description="Enable POSIX shared memory") + variant("mpi3_shmem", default=False, description="Enable MPI3 shared memory") + variant( + "sqlite_experimental", + default=False, + description="Enable sqlite integration with umpire events (Experimental)", + ) + variant("numa", default=False, description="Enable NUMA support") + variant("shared", default=True, description="Enable Shared libs") + variant("openmp", default=False, description="Build with OpenMP support") + variant("omptarget", default=False, description="Build with OpenMP 4.5 support") + variant("deviceconst", default=False, description="Enables support for constant device memory") + variant("examples", default=False, description="Build Umpire Examples") + variant( + "tests", + default="none", + values=("none", "basic", "benchmarks"), + multi=False, + description="Tests to run", + ) + variant("tools", default=False, description="Enable tools") + variant("backtrace", default=False, description="Enable backtrace tools") + variant("dev_benchmarks", default=False, description="Enable developer benchmarks") + variant("device_alloc", default=False, description="Enable DeviceAllocator") + variant("werror", default=True, description="Enable warnings as errors") + variant("asan", default=False, description="Enable ASAN") + variant("sanitizer_tests", default=False, description="Enable address sanitizer tests") + variant("fmt_header_only", default=True, description="Link to header-only fmt target") + + depends_on("cmake@3.23:", when="@2024.07.0:", type="build") + depends_on("cmake@3.23:", when="@2022.10.0: +rocm", type="build") + depends_on("cmake@3.20:", when="@2022.10.0:2024.02.1", type="build") + depends_on("cmake@:3.20", when="@2022.03.0:2022.03 +rocm", type="build") + depends_on("cmake@3.14:", when="@2022.03.0:", type="build") + depends_on("cmake@3.9:", when="+cuda", type="build") + depends_on("cmake@3.8:", type="build") + + depends_on("blt", type="build") + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") + depends_on("blt@0.6.2:", type="build", when="@2024.02.1:") + depends_on("blt@0.6.1", type="build", when="@2024.02.0") + depends_on("blt@0.5.3", type="build", when="@2023.06.0") + depends_on("blt@0.5.2:0.5.3", type="build", when="@2022.10.0") + depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2022.03.1") + depends_on("blt@0.4.1", type="build", when="@6.0.0") + depends_on("blt@0.4.0:0.4.1", type="build", when="@4.1.3:5.0.1") + depends_on("blt@0.3.6:0.4.1", type="build", when="@:4.1.2") + conflicts("^blt@:0.3.6", when="+rocm") + + depends_on("camp") + depends_on("camp@2025.09.2:", when="@2025.09.0:") + depends_on("camp@2025.03.0:", when="@2025.03.0:") + depends_on("camp+openmp", when="+openmp") + depends_on("camp~cuda", when="~cuda") + depends_on("camp~rocm", when="~rocm") + depends_on("camp@main", when="@develop") + depends_on("camp@2024.07.0:", when="@2024.07.0:") + depends_on("camp@2024.02.1", when="@2024.02.1") + depends_on("camp@2024.02.0", when="@2024.02.0") + depends_on("camp@2023.06.0", when="@2023.06.0") + depends_on("camp@2022.10.0:2023.06.0", when="@2022.10.0") + depends_on("camp@2022.03.2:2023.06.0", when="@2022.03.0:2022.03.1") + depends_on("camp@0.2.2:0.2.3", when="@6.0.0") + depends_on("camp@0.1.0", when="@5.0.0:5.0.1") + + depends_on("sqlite", when="+sqlite_experimental") + depends_on("mpi", when="+mpi") + + depends_on("fmt@9.1:11.0", when="@2024.02.0:") + # For some reason, we need c++ 17 explicitly only with intel + depends_on("fmt@9.1:11.0 cxxstd=17", when="@2024.02.0: %intel@19.1") + + with when("@5.0.0:"): + with when("+cuda"): + depends_on("camp+cuda") + for sm_ in CudaPackage.cuda_arch_values: + depends_on("camp+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) + + with when("+rocm"): + depends_on("camp+rocm") + for arch_ in ROCmPackage.amdgpu_targets: + depends_on( + "camp+rocm amdgpu_target={0}".format(arch_), + when="amdgpu_target={0}".format(arch_), + ) + + conflicts("+numa", when="@:0.3.2") + conflicts("~c", when="+fortran", msg="Fortran API requires C API") + + # device allocator must be used with more current umpire versions, rocm 5.4.0 and greater, + # and with either rocm or cuda enabled + conflicts("+device_alloc", when="@:2022.03.0") + conflicts("+device_alloc", when="^hip@:5.3.99") + conflicts("+device_alloc", when="~rocm~cuda") + + conflicts("+deviceconst", when="~rocm~cuda") + conflicts("~openmp", when="+omptarget", msg="OpenMP target requires OpenMP") + conflicts("+cuda", when="+rocm") + conflicts("+tools", when="+rocm") + conflicts( + "+rocm", when="+omptarget", msg="Cant support both rocm and openmp device backends at once" + ) + conflicts("+ipc_shmem", when="@:5.0.1") + conflicts("+mpi3_shmem", when="@:2024.07.0") + conflicts("+mpi3_shmem", when="~mpi") + conflicts("+ipc_shmem", when="+mpi3_shmem @:2025.03.0") + + conflicts("+sqlite_experimental", when="@:6.0.0") + conflicts("+sanitizer_tests", when="~asan") + + # device allocator exports device code, which requires static libs + # currently only available for cuda. + conflicts("+shared", when="+cuda") + + def _get_sys_type(self, spec): + sys_type = spec.architecture + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + return sys_type + + @property + def cache_name(self): + hostname = socket.gethostname() + if "SYS_TYPE" in env: + hostname = hostname.rstrip("1234567890") + return "{0}-{1}-{2}@{3}-{4}.cmake".format( + hostname, + self._get_sys_type(self.spec), + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash(8), + ) + + def initconfig_compiler_entries(self): + spec = self.spec + compiler = self.compiler + # Default entries are already defined in CachedCMakePackage, inherit them: + entries = super().initconfig_compiler_entries() + + if spec.satisfies("+rocm ^blt@:0.6"): + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + if spec.satisfies("+fortran") and compiler.fc is not None: + entries.append(cmake_cache_option("ENABLE_FORTRAN", True)) + else: + entries.append(cmake_cache_option("ENABLE_FORTRAN", False)) + + entries.append( + cmake_cache_option("{}ENABLE_C".format(option_prefix), spec.satisfies("+c")) + ) + + llnl_link_helpers(entries, spec, compiler) + + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + entries.append("#------------------{0}".format("-" * 30)) + entries.append("# Package custom hardware settings") + entries.append("#------------------{0}\n".format("-" * 30)) + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + if spec.satisfies("+cuda"): + entries.append(cmake_cache_option("ENABLE_CUDA", True)) + # Umpire used to pick only the first architecture in the list. The shared logic in + # CachedCMakePackage keeps the list of architectures. + else: + entries.append(cmake_cache_option("ENABLE_CUDA", False)) + + if spec.satisfies("+rocm"): + entries.append(cmake_cache_option("ENABLE_HIP", True)) + else: + entries.append(cmake_cache_option("ENABLE_HIP", False)) + + entries.append( + cmake_cache_option( + "{}ENABLE_DEVICE_CONST".format(option_prefix), spec.satisfies("+deviceconst") + ) + ) + + entries.append( + cmake_cache_option( + "{}ENABLE_OPENMP_TARGET".format(option_prefix), spec.satisfies("+omptarget") + ) + ) + + if spec.satisfies("+omptarget") and spec.satisfies("%xl"): + entries.append(cmake_cache_string("OpenMP_CXX_FLAGS", "-qsmp;-qoffload")) + + return entries + + def initconfig_mpi_entries(self): + spec = self.spec + + entries = super().initconfig_mpi_entries() + entries.append(cmake_cache_option("ENABLE_MPI", spec.satisfies("+mpi"))) + entries.append( + cmake_cache_option("UMPIRE_ENABLE_MPI3_SHARED_MEMORY", spec.satisfies("+mpi3_shmem")) + ) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" + + # TPL locations + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# TPLs") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) + if spec.satisfies("@5.0.0:"): + entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) + + if spec.satisfies("@2024.02.0:"): + entries.append(cmake_cache_path("fmt_DIR", spec["fmt"].prefix)) + + # Build options + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# Build Options") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_string("CMAKE_BUILD_TYPE", spec.variants["build_type"].value)) + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", spec.satisfies("+shared"))) + entries.append(cmake_cache_option("ENABLE_WARNINGS_AS_ERRORS", spec.satisfies("+werror"))) + + # Generic options that have a prefixed equivalent in Umpire CMake + entries.append(cmake_cache_option("ENABLE_OPENMP", spec.satisfies("+openmp"))) + entries.append(cmake_cache_option("ENABLE_EXAMPLES", spec.satisfies("+examples"))) + entries.append(cmake_cache_option("ENABLE_DOCS", False)) + if spec.satisfies("tests=benchmarks") or spec.satisfies("+dev_benchmarks"): + # BLT requires ENABLE_TESTS=True to enable benchmarks + entries.append(cmake_cache_option("ENABLE_BENCHMARKS", True)) + entries.append(cmake_cache_option("ENABLE_TESTS", True)) + else: + entries.append(cmake_cache_option("ENABLE_BENCHMARKS", False)) + entries.append(cmake_cache_option("ENABLE_TESTS", not spec.satisfies("tests=none"))) + + # Prefixed options that used to be name without one + entries.append( + cmake_cache_option("{}ENABLE_NUMA".format(option_prefix), spec.satisfies("+numa")) + ) + entries.append( + cmake_cache_option( + "{}ENABLE_DEVELOPER_BENCHMARKS".format(option_prefix), + spec.satisfies("+dev_benchmarks"), + ) + ) + entries.append( + cmake_cache_option("{}ENABLE_TOOLS".format(option_prefix), spec.satisfies("+tools")) + ) + entries.append( + cmake_cache_option( + "{}ENABLE_BACKTRACE".format(option_prefix), spec.satisfies("+backtrace") + ) + ) + entries.append( + cmake_cache_option("{}ENABLE_ASAN".format(option_prefix), spec.satisfies("+asan")) + ) + entries.append( + cmake_cache_option( + "{}ENABLE_SANITIZER_TESTS".format(option_prefix), + spec.satisfies("+sanitizer_tests"), + ) + ) + + # Recent options, were never name without prefix + entries.append( + cmake_cache_option("UMPIRE_ENABLE_DEVICE_ALLOCATOR", spec.satisfies("+device_alloc")) + ) + entries.append( + cmake_cache_option( + "UMPIRE_ENABLE_SQLITE_EXPERIMENTAL", spec.satisfies("+sqlite_experimental") + ) + ) + if spec.satisfies("+sqlite_experimental"): + entries.append(cmake_cache_path("SQLite3_ROOT", spec["sqlite"].prefix)) + + # This option was renamed later than the others + if spec.satisfies("@2022.10.0:"): + entries.append( + cmake_cache_option("UMPIRE_ENABLE_IPC_SHARED_MEMORY", spec.satisfies("+ipc_shmem")) + ) + else: + entries.append( + cmake_cache_option("ENABLE_IPC_SHARED_MEMORY", spec.satisfies("+ipc_shmem")) + ) + + if spec.satisfies("~fmt_header_only"): + entries.append(cmake_cache_string("UMPIRE_FMT_TARGET", "fmt::fmt")) + + return entries + + def cmake_args(self): + return [] + + def setup_run_environment(self, env): + for library in ["lib", "lib64"]: + lib_path = join_path(self.prefix, library) + if os.path.exists(lib_path): + env.append_path("LD_LIBRARY_PATH", lib_path) + + def run_example(self, exe, expected): + """Perform stand-alone checks on the installed package.""" + + exe_run = which(join_path(self.prefix.bin, exe)) + if exe_run is None: + raise SkipTest(f"{exe} is not installed for version {self.version}") + out = exe_run(output=str.split, error=str.split) + check_outputs(expected, out) + + def test_malloc(self): + """Run Malloc""" + self.run_example("malloc", ["99 should be 99"]) + + def test_recipe_dynamic_pool_heuristic(self): + """Multiple use allocator test""" + self.run_example("recipe_dynamic_pool_heuristic", ["in the pool", "releas"]) + + def test_recipe_no_introspection(self): + """Test without introspection""" + self.run_example("recipe_no_introspection", ["has allocated", "used"]) + + def test_strategy_example(self): + """Memory allocation strategy test""" + self.run_example("strategy_example", ["Available allocators", "HOST"]) + + def test_tut_copy(self): + """Copy data test""" + self.run_example("tut_copy", ["Copied source data"]) + + def test_tut_introspection(self): + """Keep track of pointer allocation test""" + self.run_example("tut_introspection", ["Allocator used is HOST", "size of the allocation"]) + + def test_tut_memset(self): + """Set entire block of memory to one value test""" + self.run_example("tut_memset", ["Set data from HOST"]) + + def test_tut_move(self): + """Move memory test""" + self.run_example("tut_move", ["Moved source data", "HOST"]) + + def test_tut_reallocate(self): + """Reallocate memory test""" + self.run_example("tut_reallocate", ["Reallocated data"]) + + def test_vector_allocator(self): + """Allocate vector memory test""" + self.run_example("vector_allocator", [""]) From 58b8963cd39ddb8d07d8194b183c5399ab7b7b26 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 4 Oct 2025 14:35:39 -0400 Subject: [PATCH 11/55] Add Ubuntu 24.04, gcc 13.3.0 + ROCm 6.4.3 Docker image --- .github/workflows/docker_build_tpls.yml | 6 + docker/spack.yaml | 213 ++++++++++++++++-- docker/tpl-ubuntu-gcc-hip.Dockerfile | 162 +++++++++++++ .../spack_packages/packages/geosx/package.py | 175 +++++++------- 4 files changed, 441 insertions(+), 115 deletions(-) create mode 100644 docker/tpl-ubuntu-gcc-hip.Dockerfile diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index b2a03985..986d1420 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -81,6 +81,12 @@ jobs: # SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU # RUNS_ON: ubuntu-latest + - name: Ubuntu (24.04, gcc 13.3.0 + ROCm 6.4.3) + DOCKER_REPOSITORY: geosx/ubuntu24.04-gcc13.3.0-rocm6.4.3 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc-hip.Dockerfile + DOCKER_ROOT_IMAGE: rocm/dev-ubuntu-24.04:6.4.3 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=13" + RUNS_ON: ubuntu-latest steps: - name: Checkout diff --git a/docker/spack.yaml b/docker/spack.yaml index 1a977a5c..f90c882f 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -18,6 +18,15 @@ spack: - ../versions.yaml toolchains: + amdclang-18: + - spec: '%[virtuals=c]llvm-amdgpu@6.4.3' + when: '%c' + - spec: '%[virtuals=cxx]llvm-amdgpu@6.4.3' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' clang-15: - spec: '%[virtuals=c]llvm@15.0.7+clang~flang~lld~lldb' when: '%c' @@ -80,15 +89,26 @@ spack: when: '%fortran' - spec: '%openmpi@4.1.2' when: '%mpi' + gcc-13: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@13.3.0' + when: '%c' + - spec: '%cxx=gcc@13.3.0' + when: '%cxx' + - spec: '%fortran=gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' packages: all: target: [x86_64] + # Require mpi: require: - openmpi - blas: require: - "netlib-lapack" @@ -96,6 +116,7 @@ spack: require: - "netlib-lapack" + # Compiler packages llvm: externals: - spec: llvm@10.0.0+clang~flang~lld~lldb @@ -110,7 +131,15 @@ spack: compilers: c: /usr/bin/clang-15 cxx: /usr/bin/clang++-15 - + llvm-amdgpu: + buildable: false + externals: + - spec: llvm-amdgpu@6.4.3 + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/amdclang + cxx: /usr/bin/amdclang++ gcc: externals: - spec: gcc@9.5.0 languages:='c,c++,fortran' @@ -141,7 +170,118 @@ spack: c: /usr/bin/gcc-12 cxx: /usr/bin/g++-12 fortran: /usr/bin/gfortran-12 + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + fortran: /usr/bin/gfortran-13 + + # MPI + openmpi: + externals: + - spec: openmpi@4.1.2 %llvm + prefix: /usr + modules: [mpi] + - spec: openmpi@4.1.2 %llvm-amdgpu@6.4.3 + prefix: /usr + - spec: openmpi@4.1.2 %gcc@9.5.0 + prefix: /usr + - spec: openmpi@4.1.2 %gcc@10.5.0 + prefix: /usr + - spec: openmpi@4.1.2 %gcc@11.4.0 + prefix: /usr + - spec: openmpi@4.1.2 %gcc@12.3.0 + prefix: /usr + - spec: openmpi@4.1.2 %gcc@13.3.0 + prefix: /usr + + # CUDA packages + cuda: + buildable: False + externals: + - spec: cuda@11.8.0 +allow-unsupported-compilers + prefix: /usr/local/cuda + + # ROCm packages + hip: + buildable: false + externals: + - spec: hip@6.4.3 + prefix: /opt/rocm-6.4.3 + rocprim: + buildable: false + externals: + - spec: rocprim@6.4.3 + prefix: /opt/rocm-6.4.3 + rocsparse: + buildable: false + externals: + - spec: rocsparse@6.4.3 + prefix: /opt/rocm-6.4.3 + roctracer: + buildable: false + externals: + - spec: roctracer@6.4.3 + prefix: /opt/rocm-6.4.3 + rocblas: + buildable: false + externals: + - spec: rocblas@6.4.3 + prefix: /opt/rocm-6.4.3 + rocrand: + buildable: false + externals: + - spec: rocrand@6.4.3 + prefix: /opt/rocm-6.4.3 + rocsolver: + buildable: false + externals: + - spec: rocsolver@6.4.3 + prefix: /opt/rocm-6.4.3 + rocthrust: + buildable: false + externals: + - spec: rocthrust@6.4.3 + prefix: /opt/rocm-6.4.3 + hipblas: + buildable: false + externals: + - spec: hipblas@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + hipsparse: + buildable: false + externals: + - spec: hipsparse@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + hipfft: + buildable: false + externals: + - spec: hipfft@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + hipsolver: + buildable: false + externals: + - spec: hipsolver@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + hiprand: + buildable: false + externals: + - spec: hiprand@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + rocm-device-libs: + buildable: false + externals: + - spec: rocm-device-libs@6.4.3 + prefix: /opt/rocm-6.4.3 + hsa-rocr-dev: + buildable: false + externals: + - spec: hsa-rocr-dev@6.4.3 + prefix: /opt/rocm-6.4.3 + # System packages autoconf: version: [2.71] buildable: false @@ -160,11 +300,6 @@ spack: externals: - spec: cmake@3.28.3 prefix: /usr/local - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/local/cuda findutils: version: [4.7.0] buildable: false @@ -181,7 +316,6 @@ spack: externals: - spec: mpfr@6.0.2 prefix: /usr - netlib-lapack: buildable: false externals: @@ -192,19 +326,6 @@ spack: externals: - spec: netlib-blas@3.10.0 prefix: /usr - openmpi: - externals: - - spec: openmpi@4.1.2 %llvm - prefix: /usr - modules: [mpi] - - spec: openmpi@4.1.2 %gcc@9.5.0 - prefix: /usr - - spec: openmpi@4.1.2 %gcc@10.5.0 - prefix: /usr - - spec: openmpi@4.1.2 %gcc@11.4.0 - prefix: /usr - - spec: openmpi@4.1.2 %gcc@12.3.0 - prefix: /usr perl: buildable: false externals: @@ -250,3 +371,53 @@ spack: externals: - spec: zlib@1.2.11 prefix: /usr + z3: + buildable: false + externals: + - spec: z3@4.8.12 + prefix: /usr + gettext: + buildable: false + externals: + - spec: gettext@0.21 + prefix: /usr + bison: + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /usr + flex: + buildable: false + externals: + - spec: flex@2.6.4 + prefix: /usr + bzip2: + buildable: false + externals: + - spec: bzip2@1.0.8 + prefix: /usr + help2man: + buildable: false + externals: + - spec: help2man@1.49.3 + prefix: /usr + libtool: + buildable: false + externals: + - spec: libtool@2.4.7 + prefix: /usr + gmp: + buildable: false + externals: + - spec: gmp@6.3.0 + prefix: /usr + pugixml: + buildable: false + externals: + - spec: pugixml@1.13.0 + prefix: /usr + gmake: + buildable: false + externals: + - spec: gmake@4.4.1 + prefix: /usr diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-gcc-hip.Dockerfile new file mode 100644 index 00000000..c10b3bec --- /dev/null +++ b/docker/tpl-ubuntu-gcc-hip.Dockerfile @@ -0,0 +1,162 @@ +# NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build + +# Base image is set by workflow via DOCKER_ROOT_IMAGE +ARG DOCKER_ROOT_IMAGE=rocm/dev-ubuntu-24.04:6.4.3 + +FROM ${DOCKER_ROOT_IMAGE} AS tpl_toolchain_intersect_geosx_toolchain +ARG SRC_DIR + +ARG INSTALL_DIR +ENV GEOSX_TPL_DIR=$INSTALL_DIR + +# Parameters +ARG GCC_MAJOR_VERSION=13 +ARG AMDGPU_TARGET=gfx942 +ARG ROCM_VERSION=6.4.3 + +# Allow changing the number of cores used for building code via spack +ARG SPACK_BUILD_JOBS=4 +ENV SPACK_BUILD_JOBS=${SPACK_BUILD_JOBS} + +RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \ + apt-get update + +# Install system packages +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget \ + gnupg \ + ca-certificates \ + libtbb12 \ + libtbbmalloc2 \ + libblas-dev \ + libz3-dev \ + zlib1g-dev \ + libmpich-dev \ + mpich \ + python3 \ + python3-dev \ + python3-pip \ + python3-sphinx \ + doxygen \ + pkg-config \ + xz-utils \ + gettext \ + bison \ + flex \ + bzip2 \ + help2man \ + libtool \ + libgmp-dev \ + unzip \ + libmpfr-dev \ + lbzip2 \ + bzip2 \ + gnupg \ + virtualenv \ + libpugixml-dev \ + roctracer-dev \ + rocsparse-dev \ + rocsolver-dev \ + rocblas-dev \ + rocprim-dev \ + rocrand-dev \ + rocthrust-dev \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Install clingo for Spack (use pip without upgrading pip to avoid Debian conflict) +RUN python3 -m pip install clingo --break-system-packages + +# Install CMake +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh + +# Hack for spack to see Ubuntu's mpich in a standard prefix layout +RUN mkdir -p /opt/mpich-system/bin /opt/mpich-system/include /opt/mpich-system/lib && \ + ln -s /usr/bin/mpicc /opt/mpich-system/bin/mpicc && \ + ln -s /usr/bin/mpicxx /opt/mpich-system/bin/mpicxx && \ + ln -s /usr/bin/mpif90 /opt/mpich-system/bin/mpif90 && \ + ln -s /usr/bin/mpifort /opt/mpich-system/bin/mpifort && \ + ln -s /usr/bin/mpirun /opt/mpich-system/bin/mpirun && \ + ln -s /usr/lib/x86_64-linux-gnu/mpich/include/* /opt/mpich-system/include/ && \ + ln -s /usr/lib/x86_64-linux-gnu/mpich/lib/* /opt/mpich-system/lib/ + +# Installing TPLs +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +ARG SRC_DIR +ARG BLD_DIR + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libtbb-dev \ + make \ + bc \ + file \ + patch \ + ca-certificates \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +# --spack-debug to debug build +RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ + mkdir -p ${GEOSX_TPL_DIR} && \ + ./scripts/uberenv/uberenv.py \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %gcc-13 ^mpich@4.2.0 ^caliper~papi" \ + --spack-env-file=${SRC_DIR}/docker/spack-rocm.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k && \ +# Remove host-config generated for LvArray + rm lvarray* && \ +# Rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated.cmake && \ +# Remove extraneous spack files + cd ${GEOSX_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + +# Extract only TPLs from previous stage +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +ARG SRC_DIR + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR + +# Extract the generated host-config +COPY --from=tpl_toolchain /spack-generated.cmake / + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get install -y --no-install-recommends \ + openssh-client \ + ca-certificates \ + curl \ + python3 \ + texlive \ + texlive-latex-extra \ + graphviz \ + libxml2-utils \ + git \ + ghostscript \ + ninja-build \ + python3-dev \ + python3-sphinx \ + python3-mpi4py \ + python3-scipy \ + python3-virtualenv \ + python3-matplotlib \ + python3-venv \ + python3-pytest && \ + rm -rf /var/lib/apt/lists/* + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache + +# Helpful environment defaults for HIP +ENV ROCM_PATH=/opt/rocm-${ROCM_VERSION} +ENV HIP_PATH=${ROCM_PATH}/hip +ENV PATH=${ROCM_PATH}/bin:${ROCM_PATH}/llvm/bin:${PATH} +ENV LD_LIBRARY_PATH=${ROCM_PATH}/lib:${ROCM_PATH}/lib64:${ROCM_PATH}/llvm/lib +ENV CMAKE_HIP_ARCHITECTURES=${AMDGPU_TARGET} \ No newline at end of file diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index bcfdc70a..3c6ca058 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -12,22 +12,18 @@ from os import environ as env from os.path import join as pjoin -from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_path - from spack_repo.builtin.build_systems.cmake import CMakePackage from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.cached_cmake import ( + CachedCMakePackage, + cmake_cache_option, + cmake_cache_path, + cmake_cache_string, +) # Tested specs are located at scripts/spack_configs/<$SYS_TYPE>/spack.yaml (e.g. %clang@10.0.1) - # WARNING: +petsc variant is yet to be tested. - -def cmake_cache_entry(name, value, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE PATH "%s")\n\n' % (name, value, comment) - - def cmake_cache_list(name, value, comment=""): """Generate a list for a cmake cache variable""" @@ -36,19 +32,6 @@ def cmake_cache_list(name, value, comment=""): return 'set(%s %s CACHE STRING "%s")\n\n' % (name, join_str.join(value), comment) -def cmake_cache_string(name, string, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE STRING "%s")\n\n' % (name, string, comment) - - -def cmake_cache_option(name, boolean_value, comment=""): - """Generate a string for a cmake configuration option""" - - value = "ON" if boolean_value else "OFF" - return 'set(%s %s CACHE BOOL "%s")\n\n' % (name, value, comment) - - class Geosx(CMakePackage, CudaPackage, ROCmPackage): """GEOSX simulation framework.""" @@ -75,6 +58,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): values=('trilinos', 'hypre', 'petsc'), multi=False) variant('pygeosx', default=True, description='Enable pygeosx.') + variant('cxxstd', default='17', description='CXX standard.') # SPHINX_END_VARIANTS @@ -109,15 +93,15 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # # Performance portability # - depends_on('raja ~examples~exercises~shared') + depends_on("raja ~examples~exercises~shared") depends_on("raja~openmp", when="~openmp") depends_on("raja+openmp", when="+openmp") - depends_on('umpire +c~examples+fortran~device_alloc~shared') + depends_on("umpire +c~examples+fortran~device_alloc~shared") depends_on("umpire~openmp", when="~openmp") depends_on("umpire+openmp", when="+openmp") - depends_on('chai +raja~examples~shared') + depends_on("chai +raja~examples~shared") depends_on("chai~openmp", when="~openmp") depends_on("chai+openmp", when="+openmp") @@ -155,7 +139,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on('pugixml@1.13 ~shared') - depends_on('fmt@10.0.0 cxxstd=14') + depends_on('fmt@11') depends_on('vtk@9.4.2', when='+vtk') # @@ -226,6 +210,12 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): phases = ['geos_hostconfig', 'lvarray_hostconfig'] #phases = ['hostconfig', 'cmake', 'build', 'install'] + # This method is called to finalize dependencies + def dependencies(self): + super().dependencies() + # Add a conditional dependency on fmt with the cxxstd variant + self.depends_on(f"fmt@11 cxxstd=c++{self.spec.variants['cxxstd'].value}") + @run_after('build') @on_package_attributes(run_tests=True) def check(self): @@ -250,25 +240,22 @@ def _get_sys_type(self, spec): return sys_type def _get_host_config_path(self, spec, lvarray=False): - var = '' - - if 'no-avx' in str(spec.compiler_flags): - var += "-noAVX" - - if '+cuda' in spec: - var = '-'.join([var, 'cuda']) - var += "@" + str(spec['cuda'].version) - elif '+rocm' in spec: - var = '-'.join([var, 'rocm']) - var += "@" + str(spec['hip'].version) - + gpu_backend = "" + if "+cuda" in spec: + gpu_backend = f"cuda@{spec['cuda'].version}" + elif "+rocm" in spec: + gpu_backend = f"rocm@{spec['hip'].version}" hostname = socket.gethostname().rstrip('1234567890') if lvarray: hostname = "lvarray-" + hostname - host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, self._get_sys_type(spec), (str(spec.compiler.name)), str(spec.compiler.version),var) + host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, + self._get_sys_type(spec), + str(spec.compiler.name), + str(spec.compiler.version), + gpu_backend) dest_dir = self.stage.source_path host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) @@ -330,15 +317,15 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n".format("-" * 80)) cfg.write("# Compilers\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write(cmake_cache_path("CMAKE_C_COMPILER", c_compiler)) cflags = ' '.join(spec.compiler_flags['cflags']) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + cfg.write(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + cfg.write(cmake_cache_path("CMAKE_CXX_COMPILER", cpp_compiler)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) + cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) release_flags = "-O3 -DNDEBUG" if "clang" in self.compiler.cxx: @@ -350,21 +337,21 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) + cfg.write(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_string("BLT_CXX_STD", "c++17")) + cfg.write(cmake_cache_string("BLT_CXX_STD", f"c++{spec.variants['cxxstd'].value}")) cfg.write("#{0}\n".format("-" * 80)) cfg.write("# MPI\n") cfg.write("#{0}\n\n".format("-" * 80)) cfg.write(cmake_cache_option('ENABLE_MPI', True)) - cfg.write(cmake_cache_entry('MPI_C_COMPILER', spec['mpi'].mpicc)) - cfg.write(cmake_cache_entry('MPI_CXX_COMPILER', spec['mpi'].mpicxx)) + cfg.write(cmake_cache_path('MPI_C_COMPILER', spec['mpi'].mpicc)) + cfg.write(cmake_cache_path('MPI_CXX_COMPILER', spec['mpi'].mpicxx)) hostname = socket.gethostname().rstrip('1234567890') @@ -372,14 +359,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): and hostname != 'p3dev': cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) + cfg.write(cmake_cache_string('MPIEXEC', 'lrun')) + cfg.write(cmake_cache_string('MPIEXEC_NUMPROC_FLAG', '-n')) else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) + cfg.write(cmake_cache_string('MPIEXEC', 'jsrun')) cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) elif sys_type in ('toss_4_x86_64_ib_cray'): - cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) + cfg.write(cmake_cache_string('MPIEXEC', 'srun')) + cfg.write(cmake_cache_string('MPIEXEC_NUMPROC_FLAG', '-n')) else: # Taken from cached_cmake class: # https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/cached_cmake.py#L180-234 @@ -440,12 +427,12 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+cuda' in spec: cfg.write(cmake_cache_option('ENABLE_CUDA', True)) - cfg.write(cmake_cache_entry('CMAKE_CUDA_STANDARD', 17)) + cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value}) cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) + cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' - cfg.write(cmake_cache_entry('CMAKE_CUDA_COMPILER', cudacompiler)) + cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' 'cross-execution-space-call,reorder,' @@ -475,7 +462,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cuda_stack_size = int(spec.variants['cuda_stack_size'].value) if 0 < cuda_stack_size: cfg.write(cmake_cache_option('ENABLE_CUDA_STACK_SIZE', True, "Adjust the CUDA stack size limit")) - cfg.write(cmake_cache_entry('CUDA_STACK_SIZE', cuda_stack_size, "CUDA stack size in KB")) + cfg.write(cmake_cache_string('CUDA_STACK_SIZE', cuda_stack_size, "CUDA stack size in KB")) else: cfg.write(cmake_cache_option('ENABLE_CUDA', False)) @@ -485,14 +472,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+rocm' in spec: cfg.write(cmake_cache_option('ENABLE_HIP', True)) - cfg.write(cmake_cache_string('CMAKE_HIP_STANDARD', 17)) - cfg.write(cmake_cache_entry('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) + cfg.write(cmake_cache_string('CMAKE_HIP_STANDARD', spec.variants['cxxstd'].value)) + cfg.write(cmake_cache_path('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) if not spec.satisfies('amdgpu_target=none'): cmake_hip_archs = ";".join(spec.variants["amdgpu_target"].value) cfg.write(cmake_cache_string('CMAKE_HIP_ARCHITECTURES', cmake_hip_archs)) - cfg.write(cmake_cache_entry('ROCM_PATH', spec['hip'].prefix)) + cfg.write(cmake_cache_path('ROCM_PATH', spec['hip'].prefix)) else: cfg.write(cmake_cache_option('ENABLE_HIP', False)) @@ -501,14 +488,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_CHAI', True)) - cfg.write(cmake_cache_entry('CHAI_DIR', spec['chai'].prefix)) + cfg.write(cmake_cache_path('CHAI_DIR', spec['chai'].prefix)) - cfg.write(cmake_cache_entry('RAJA_DIR', spec['raja'].prefix)) + cfg.write(cmake_cache_path('RAJA_DIR', spec['raja'].prefix)) cfg.write(cmake_cache_option('ENABLE_UMPIRE', True)) - cfg.write(cmake_cache_entry('UMPIRE_DIR', spec['umpire'].prefix)) + cfg.write(cmake_cache_path('UMPIRE_DIR', spec['umpire'].prefix)) - cfg.write(cmake_cache_entry('CAMP_DIR', spec['camp'].prefix)) + cfg.write(cmake_cache_path('CAMP_DIR', spec['camp'].prefix)) # yapf: disable io_tpls = ( @@ -528,12 +515,12 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+caliper' in spec: cfg.write(cmake_cache_option('ENABLE_CALIPER', True)) - cfg.write(cmake_cache_entry('CALIPER_DIR', spec['caliper'].prefix)) - cfg.write(cmake_cache_entry('ADIAK_DIR', spec['adiak'].prefix)) + cfg.write(cmake_cache_path('CALIPER_DIR', spec['caliper'].prefix)) + cfg.write(cmake_cache_path('ADIAK_DIR', spec['adiak'].prefix)) for tpl, cmake_name, enable in io_tpls: if enable: - cfg.write(cmake_cache_entry('{}_DIR'.format(cmake_name), spec[tpl].prefix)) + cfg.write(cmake_cache_path('{}_DIR'.format(cmake_name), spec[tpl].prefix)) else: cfg.write(cmake_cache_option('ENABLE_{}'.format(cmake_name), False)) @@ -543,15 +530,15 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if spec["blas"].name == "intel-oneapi-mkl": cfg.write(cmake_cache_option('ENABLE_MKL', True)) - cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-oneapi-mkl'].prefix.include)) + cfg.write(cmake_cache_path('MKL_INCLUDE_DIRS', spec['intel-oneapi-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-oneapi-mkl'].libs)) elif spec["blas"].name == "mkl": cfg.write(cmake_cache_option('ENABLE_MKL', True)) - cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) + cfg.write(cmake_cache_path('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) elif spec["blas"].name == "essl": cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) + cfg.write(cmake_cache_path('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) @@ -577,7 +564,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) for tpl, cmake_name, enable in math_tpls: if enable: - cfg.write(cmake_cache_entry('{}_DIR'.format(cmake_name), spec[tpl].prefix)) + cfg.write(cmake_cache_path('{}_DIR'.format(cmake_name), spec[tpl].prefix)) if tpl == 'hypre' and '+cuda' in spec: cfg.write(cmake_cache_string('ENABLE_HYPRE_DEVICE', "CUDA")) @@ -600,8 +587,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('# Python\n') cfg.write('#{0}\n\n'.format('-' * 80)) - cfg.write(cmake_cache_entry('Python3_ROOT_DIR', os.path.join(spec['python'].prefix))) - cfg.write(cmake_cache_entry('Python3_EXECUTABLE', os.path.join(spec['python'].prefix.bin, 'python3'))) + cfg.write(cmake_cache_path('Python3_ROOT_DIR', os.path.join(spec['python'].prefix))) + cfg.write(cmake_cache_path('Python3_EXECUTABLE', os.path.join(spec['python'].prefix.bin, 'python3'))) if '+pygeosx' in spec: cfg.write(cmake_cache_option('ENABLE_PYGEOSX', True)) @@ -613,10 +600,10 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+docs' in spec: sphinx_bin_dir = spec['py-sphinx'].prefix.bin - cfg.write(cmake_cache_entry('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) + cfg.write(cmake_cache_path('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) doxygen_bin_dir = spec['doxygen'].prefix.bin - cfg.write(cmake_cache_entry('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) + cfg.write(cmake_cache_path('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) else: cfg.write(cmake_cache_option('ENABLE_DOCS', False)) cfg.write(cmake_cache_option('ENABLE_DOXYGEN', False)) @@ -629,14 +616,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_UNCRUSTIFY', '+uncrustify' in spec)) if '+uncrustify' in spec: cfg.write( - cmake_cache_entry('UNCRUSTIFY_EXECUTABLE', os.path.join(spec['uncrustify'].prefix.bin, 'uncrustify'))) + cmake_cache_path('UNCRUSTIFY_EXECUTABLE', os.path.join(spec['uncrustify'].prefix.bin, 'uncrustify'))) if '+addr2line' in spec: cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# addr2line\n') cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_ADDR2LINE', True)) - cfg.write(cmake_cache_entry('ADDR2LINE_EXEC ', '/usr/bin/addr2line')) + cfg.write(cmake_cache_path('ADDR2LINE_EXEC ', '/usr/bin/addr2line')) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Other\n') @@ -644,7 +631,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+mathpresso' in spec: cfg.write(cmake_cache_option('ENABLE_MATHPRESSO', True)) - cfg.write(cmake_cache_entry('MATHPRESSO_DIR', spec['mathpresso'].prefix)) + cfg.write(cmake_cache_path('MATHPRESSO_DIR', spec['mathpresso'].prefix)) cfg.write(cmake_cache_option('ENABLE_XML_UPDATES', True)) else: cfg.write(cmake_cache_option('ENABLE_MATHPRESSO', False)) @@ -719,15 +706,15 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n".format("-" * 80)) cfg.write("# Compilers\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write(cmake_cache_path("CMAKE_C_COMPILER", c_compiler)) cflags = ' '.join(spec.compiler_flags['cflags']) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + cfg.write(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + cfg.write(cmake_cache_path("CMAKE_CXX_COMPILER", cpp_compiler)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) + cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) release_flags = "-O3 -DNDEBUG" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", release_flags)) @@ -737,19 +724,19 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) + cfg.write(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') cfg.write('#{0}\n\n'.format('-' * 80)) if '+cuda' in spec: cfg.write(cmake_cache_option('ENABLE_CUDA', True)) - cfg.write(cmake_cache_entry('CMAKE_CUDA_STANDARD', 17)) + cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value)) cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) + cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' - cfg.write(cmake_cache_entry('CMAKE_CUDA_COMPILER', cudacompiler)) + cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' 'cross-execution-space-call,reorder,' @@ -780,14 +767,14 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_CHAI', True)) - cfg.write(cmake_cache_entry('CHAI_DIR', spec['chai'].prefix)) + cfg.write(cmake_cache_path('CHAI_DIR', spec['chai'].prefix)) - cfg.write(cmake_cache_entry('RAJA_DIR', spec['raja'].prefix)) + cfg.write(cmake_cache_path('RAJA_DIR', spec['raja'].prefix)) cfg.write(cmake_cache_option('ENABLE_UMPIRE', True)) - cfg.write(cmake_cache_entry('UMPIRE_DIR', spec['umpire'].prefix)) + cfg.write(cmake_cache_path('UMPIRE_DIR', spec['umpire'].prefix)) - cfg.write(cmake_cache_entry('CAMP_DIR', spec['camp'].prefix)) + cfg.write(cmake_cache_path('CAMP_DIR', spec['camp'].prefix)) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# IO TPLs\n') @@ -795,18 +782,18 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+caliper' in spec: cfg.write(cmake_cache_option('ENABLE_CALIPER', True)) - cfg.write(cmake_cache_entry('CALIPER_DIR', spec['caliper'].prefix)) - cfg.write(cmake_cache_entry('adiak_DIR', spec['adiak'].prefix + '/lib/cmake/adiak')) + cfg.write(cmake_cache_path('CALIPER_DIR', spec['caliper'].prefix)) + cfg.write(cmake_cache_path('adiak_DIR', spec['adiak'].prefix + '/lib/cmake/adiak')) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Documentation\n') cfg.write('#{0}\n\n'.format('-' * 80)) if '+docs' in spec: sphinx_bin_dir = spec['py-sphinx'].prefix.bin - cfg.write(cmake_cache_entry('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) + cfg.write(cmake_cache_path('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) doxygen_bin_dir = spec['doxygen'].prefix.bin - cfg.write(cmake_cache_entry('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) + cfg.write(cmake_cache_path('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) else: cfg.write(cmake_cache_option('ENABLE_DOXYGEN', False)) cfg.write(cmake_cache_option('ENABLE_SPHINX', False)) From 171b7655734ea23b1e7d17e39090a09dfc719c08 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 4 Oct 2025 14:41:43 -0400 Subject: [PATCH 12/55] Fix Typo --- scripts/spack_packages/packages/geosx/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 3c6ca058..fe85131d 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -427,7 +427,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+cuda' in spec: cfg.write(cmake_cache_option('ENABLE_CUDA', True)) - cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value}) + cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value) cudatoolkitdir = spec['cuda'].prefix cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) From dab3d7164c1f44823122c0b3dbf4099dc4c5c59b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 4 Oct 2025 15:11:08 -0400 Subject: [PATCH 13/55] Fix Typo --- scripts/spack_packages/packages/geosx/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index fe85131d..b75fde95 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -427,7 +427,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+cuda' in spec: cfg.write(cmake_cache_option('ENABLE_CUDA', True)) - cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value) + cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value)) cudatoolkitdir = spec['cuda'].prefix cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) From e328178456019f40d1273ad1c9ce9012c0b75f4d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 4 Oct 2025 15:34:56 -0400 Subject: [PATCH 14/55] Minor issues --- docker/tpl-ubuntu-gcc-hip.Dockerfile | 4 ++-- scripts/spack_packages/packages/geosx/package.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-gcc-hip.Dockerfile index c10b3bec..040b8c46 100644 --- a/docker/tpl-ubuntu-gcc-hip.Dockerfile +++ b/docker/tpl-ubuntu-gcc-hip.Dockerfile @@ -106,8 +106,8 @@ RUN apt-get update && \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %gcc-13 ^mpich@4.2.0 ^caliper~papi" \ - --spack-env-file=${SRC_DIR}/docker/spack-rocm.yaml \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %gcc-13 ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ -k && \ diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index b75fde95..12fc074a 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -171,6 +171,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on("hypre +rocm+superlu-dist+mixedint+mpi+umpire~shared cflags='-fPIC' cxxflags='-fPIC'", when='+rocm') depends_on("hypre ~openmp", when="~openmp") depends_on("hypre +openmp", when="+openmp") + depends_on("hypre +caliper", when="+caliper") depends_on('petsc@3.19.4~hdf5~hypre+int64', when='+petsc') depends_on('petsc+ptscotch', when='+petsc+scotch') From edf3113f5aa7b8e2b3b8d6cd2b822f25de7f775d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 6 Oct 2025 16:39:47 -0400 Subject: [PATCH 15/55] CI fixes --- docker/spack.yaml | 5 ----- docker/tpl-centos-gcc-cuda.Dockerfile | 12 ++++++++---- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 6 +++++- docker/tpl-rockylinux-gcc-cuda-12.Dockerfile | 4 ++++ docker/tpl-ubuntu-clang-cuda.Dockerfile | 4 ++++ docker/tpl-ubuntu-clang.Dockerfile | 4 ++++ docker/tpl-ubuntu-gcc.Dockerfile | 4 ++++ 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/docker/spack.yaml b/docker/spack.yaml index f90c882f..952629b0 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -321,11 +321,6 @@ spack: externals: - spec: netlib-lapack@3.10.0 prefix: /usr - netlib-blas: - buildable: false - externals: - - spec: netlib-blas@3.10.0 - prefix: /usr perl: buildable: false externals: diff --git a/docker/tpl-centos-gcc-cuda.Dockerfile b/docker/tpl-centos-gcc-cuda.Dockerfile index f99c9f19..dee69adf 100644 --- a/docker/tpl-centos-gcc-cuda.Dockerfile +++ b/docker/tpl-centos-gcc-cuda.Dockerfile @@ -1,5 +1,5 @@ # NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments -ARG TMP_DIR=/tmp +ARG TMP_DIR=/tmp ARG SRC_DIR=$TMP_DIR/thirdPartyLibs ARG BLD_DIR=$TMP_DIR/build @@ -11,12 +11,12 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo + sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo # Using gcc 8.3.1 provided by the Software Collections (SCL). RUN yum install -y \ centos-release-scl - + # Modify the SCLo repository configuration RUN sed -i 's|^mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo && \ sed -i 's|^baseurl=http://mirror.centos.org/centos/\$releasever/sclo/\$basearch/rh|baseurl=http://vault.centos.org/7.9.2009/sclo/x86_64/rh|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo && \ @@ -25,7 +25,7 @@ RUN sed -i 's|^mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo # Install necessary tools and update the system RUN yum -y update && \ - yum -y install yum-utils + yum -y install yum-utils RUN yum install -y \ devtoolset-8-gcc \ @@ -48,6 +48,10 @@ RUN yum -y install \ # xz \ unzip \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ gnupg \ && pip3 install virtualenv diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index f761368c..ec6a9ce0 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -13,7 +13,7 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR RUN dnf clean all && \ dnf -y update && \ dnf -y install \ - which \ + which \ clang-17.0.6 \ gcc-toolset-13 \ python3 \ @@ -28,6 +28,10 @@ RUN dnf clean all && \ unzip \ mpfr-devel \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ gnupg \ xz \ python3-virtualenv diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index d73e87ff..af7a969d 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -26,6 +26,10 @@ RUN dnf clean all && \ unzip \ mpfr-devel \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ gnupg \ xz \ python3-virtualenv diff --git a/docker/tpl-ubuntu-clang-cuda.Dockerfile b/docker/tpl-ubuntu-clang-cuda.Dockerfile index 1a1b3628..44e45fa9 100644 --- a/docker/tpl-ubuntu-clang-cuda.Dockerfile +++ b/docker/tpl-ubuntu-clang-cuda.Dockerfile @@ -34,6 +34,10 @@ RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \ libmpfr-dev \ lbzip2 \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ gnupg \ virtualenv diff --git a/docker/tpl-ubuntu-clang.Dockerfile b/docker/tpl-ubuntu-clang.Dockerfile index 0ef16273..b992f9dd 100644 --- a/docker/tpl-ubuntu-clang.Dockerfile +++ b/docker/tpl-ubuntu-clang.Dockerfile @@ -39,6 +39,10 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ libmpfr-dev \ lbzip2 \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ gnupg \ virtualenv diff --git a/docker/tpl-ubuntu-gcc.Dockerfile b/docker/tpl-ubuntu-gcc.Dockerfile index 2f74d46c..e7f0f28a 100644 --- a/docker/tpl-ubuntu-gcc.Dockerfile +++ b/docker/tpl-ubuntu-gcc.Dockerfile @@ -64,6 +64,10 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ libmpfr-dev \ lbzip2 \ bzip2 \ + gettext \ + bison \ + flex \ + help2man \ gnupg \ virtualenv From 5f067ce42020bcc5bb51405258fe07475d9110db Mon Sep 17 00:00:00 2001 From: Brian Han Date: Mon, 6 Oct 2025 16:02:34 -0700 Subject: [PATCH 16/55] use upstream hdf5 recipe --- .../spack_packages/packages/hdf5/package.py | 749 ------------------ 1 file changed, 749 deletions(-) delete mode 100644 scripts/spack_packages/packages/hdf5/package.py diff --git a/scripts/spack_packages/packages/hdf5/package.py b/scripts/spack_packages/packages/hdf5/package.py deleted file mode 100644 index 1a2db178..00000000 --- a/scripts/spack_packages/packages/hdf5/package.py +++ /dev/null @@ -1,749 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# GEOS_EDIT_START -# Older hdf5 spack recipe that works on lassen -# (latest spack release fails with MPI related configuration failure) -# GEOS_EDIT_END - -import os -import shutil - -from spack.package import * -from spack_repo.builtin.build_systems.cmake import CMakePackage - -class Hdf5(CMakePackage): - """HDF5 is a data model, library, and file format for storing and managing - data. It supports an unlimited variety of datatypes, and is designed for - flexible and efficient I/O and for high volume and complex data. - """ - - homepage = "https://portal.hdfgroup.org" - url = "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.8/src/hdf5-1.10.8.tar.gz" - list_url = "https://support.hdfgroup.org/ftp/HDF5/releases" - list_depth = 3 - git = "https://github.com/HDFGroup/hdf5.git" - maintainers = [ - "lrknox", - "brtnfld", - "byrnHDF", - "ChristopherHogan", - "epourmal", - "gheber", - "hyoklee", - "lkurz", - "soumagne", - ] - - tags = ["e4s"] - - test_requires_compiler = True - - # The 'develop' version is renamed so that we could uninstall (or patch) it - # without affecting other develop version. - version("develop-1.13", branch="develop") - version("develop-1.12", branch="hdf5_1_12") - version("develop-1.10", branch="hdf5_1_10") - version("develop-1.8", branch="hdf5_1_8") - - # Odd versions are considered experimental releases - version("1.13.2", sha256="01643fa5b37dba7be7c4db6bbf3c5d07adf5c1fa17dbfaaa632a279b1b2f06da") - - # Even versions are maintenance versions - version( - "1.12.2", - sha256="2a89af03d56ce7502dcae18232c241281ad1773561ec00c0f0e8ee2463910f14", - preferred=True, - ) - version( - "1.12.1", - sha256="79c66ff67e666665369396e9c90b32e238e501f345afd2234186bfb8331081ca", - preferred=True, - ) - version( - "1.12.0", - sha256="a62dcb276658cb78e6795dd29bf926ed7a9bc4edf6e77025cd2c689a8f97c17a", - preferred=True, - ) - version( - "1.10.9", - sha256="f5b77f59b705a755a5a223372d0222c7bc408fe8db6fa8d9d7ecf8bce291b8dd", - preferred=True, - ) - version( - "1.10.8", - sha256="d341b80d380dd763753a0ebe22915e11e87aac4e44a084a850646ff934d19c80", - preferred=True, - ) - version( - "1.10.7", - sha256="7a1a0a54371275ce2dfc5cd093775bb025c365846512961e7e5ceaecb437ef15", - preferred=True, - ) - version( - "1.10.6", - sha256="5f9a3ee85db4ea1d3b1fa9159352aebc2af72732fc2f58c96a3f0768dba0e9aa", - preferred=True, - ) - version( - "1.10.5", - sha256="6d4ce8bf902a97b050f6f491f4268634e252a63dadd6656a1a9be5b7b7726fa8", - preferred=True, - ) - version( - "1.10.4", - sha256="8f60dc4dd6ab5fcd23c750d1dc5bca3d0453bdce5c8cdaf0a4a61a9d1122adb2", - preferred=True, - ) - version( - "1.10.3", - sha256="b600d7c914cfa80ae127cd1a1539981213fee9994ac22ebec9e3845e951d9b39", - preferred=True, - ) - version( - "1.10.2", - sha256="bfec1be8c366965a99812cf02ddc97e4b708c1754fccba5414d4adccdc073866", - preferred=True, - ) - version( - "1.10.1", - sha256="048a9d149fb99aaa1680a712963f5a78e9c43b588d0e79d55e06760ec377c172", - preferred=True, - ) - version( - "1.10.0-patch1", - sha256="6e78cfe32a10e6e0629393cdfddf6cfa536571efdaf85f08e35326e1b4e9eff0", - preferred=True, - ) - version( - "1.10.0", - sha256="81f6201aba5c30dced5dcd62f5d5477a2790fd5850e02ac514ca8bf3e2bb375a", - preferred=True, - ) - version( - "1.8.22", - sha256="8406d96d9355ef8961d2739fb8fd5474ad4cdf52f3cfac657733defd9709bfaa", - preferred=True, - ) - version( - "1.8.21", - sha256="87d8c82eba5cf766d97cd06c054f4639c1049c4adeaa3a79f77f8bd374f80f37", - preferred=True, - ) - version( - "1.8.19", - sha256="a4335849f19fae88c264fd0df046bc321a78c536b2548fc508627a790564dc38", - preferred=True, - ) - version( - "1.8.18", - sha256="cdb195ad8d9e6782acf24b2488061289f615628c2ccda8457b0a0c3fb7a8a063", - preferred=True, - ) - version( - "1.8.17", - sha256="d9cda297ee76ade9881c4208987939250d397bae6252d0ccb66fa7d24d67e263", - preferred=True, - ) - version( - "1.8.16", - sha256="ed17178abd9928a7237f30370189ba767b9e39e0db45917c2ac4665eb9cb4771", - preferred=True, - ) - version( - "1.8.15", - sha256="4e963216b7d32469596bc1321a8c3f6e0c278dcbbdb7be6414c63c081b34c275", - preferred=True, - ) - version( - "1.8.14", - sha256="1dbefeeef7f591897c632b2b090db96bb8d35ad035beaa36bc39cb2bc67e0639", - preferred=True, - ) - version( - "1.8.13", - sha256="82f6b38eec103b4fccfbf14892786e0c27a8135d3252d8601cf5bf20066d38c1", - preferred=True, - ) - version( - "1.8.12", - sha256="b5cccea850096962b5fd9e96f22c4f47d2379224bb41130d9bc038bb6c37dfcb", - preferred=True, - ) - version( - "1.8.10", - sha256="4813b79c5fb8701a625b9924b8203bc7154a77f9b826ad4e034144b4056a160a", - preferred=True, - ) - - variant("shared", default=True, description="Builds a shared version of the library") - - variant("hl", default=False, description="Enable the high-level library") - variant("cxx", default=False, description="Enable C++ support") - variant("fortran", default=False, description="Enable Fortran support") - variant("java", when="@1.10:", default=False, description="Enable Java support") - variant("threadsafe", default=False, description="Enable thread-safe capabilities") - variant("tools", default=True, description="Enable building tools") - variant("mpi", default=True, description="Enable MPI support") - variant("szip", default=False, description="Enable szip support") - # Build HDF5 with API compatibility. - variant( - "api", - default="default", - description="Choose api compatibility for earlier version", - values=("default", "v114", "v112", "v110", "v18", "v16"), - multi=False, - ) - - depends_on("c", type="build") - depends_on("cxx", type="build") - depends_on("fortran", type="build", when="+fortran") - - depends_on("cmake@3.12:", type="build") - - depends_on("mpi", when="+mpi") - depends_on("java", type=("build", "run"), when="+java") - depends_on("szip", when="+szip") - depends_on("zlib@1.1.2:") - - # The compiler wrappers (h5cc, h5fc, etc.) run 'pkg-config'. - depends_on("pkgconfig", type="run") - - conflicts("api=v114", when="@1.6:1.12", msg="v114 is not compatible with this release") - conflicts("api=v112", when="@1.6:1.10", msg="v112 is not compatible with this release") - conflicts("api=v110", when="@1.6:1.8", msg="v110 is not compatible with this release") - conflicts("api=v18", when="@1.6.0:1.6", msg="v18 is not compatible with this release") - - # The Java wrappers cannot be built without shared libs. - conflicts("+java", when="~shared") - # Fortran fails built with shared for old HDF5 versions - conflicts("+fortran", when="+shared@:1.8.15") - # See https://github.com/spack/spack/issues/31085 - conflicts("+fortran+mpi", when="@1.8.22") - - # GEOS_EDIT_START - # Commenting out unused patches - - # There are several officially unsupported combinations of the features: - # 1. Thread safety is not guaranteed via high-level C-API but in some cases - # it works. - # conflicts('+threadsafe+hl') - - # 2. Thread safety is not guaranteed via Fortran (CXX) API, but it's - # possible for a dependency tree to contain a package that uses Fortran - # (CXX) API in a single thread and another one that uses low-level C-API - # in multiple threads. To allow for such scenarios, we don't specify the - # following conflicts. - # conflicts('+threadsafe+cxx') - # conflicts('+threadsafe+fortran') - - # 3. Parallel features are not supported via CXX API, but for the reasons - # described in #2 we allow for such combination. - # conflicts('+mpi+cxx') - - # There are known build failures with intel@18.0.1. This issue is - # discussed and patch is provided at - # https://software.intel.com/en-us/forums/intel-fortran-compiler-for-linux-and-mac-os-x/topic/747951. - #patch("h5f90global-mult-obj-same-equivalence-same-common-block.patch", when="@1.10.1%intel@18") - - # Turn line comments into block comments to conform with pre-C99 language - # standards. Versions of hdf5 after 1.8.10 don't require this patch, - # either because they conform to pre-C99 or neglect to ask for pre-C99 - # language standards from their compiler. The hdf5 build system adds - # the -ansi cflag (run 'man gcc' for info on -ansi) for some versions - # of some compilers (see hdf5-1.8.10/config/gnu-flags). The hdf5 build - # system does not provide an option to disable -ansi, but since the - # pre-C99 code is restricted to just five lines of line comments in - # three src files, this patch accomplishes the simple task of patching the - # three src files and leaves the hdf5 build system alone. - #patch("pre-c99-comments.patch", when="@1.8.10") - - # There are build errors with GCC 8, see - # https://forum.hdfgroup.org/t/1-10-2-h5detect-compile-error-gcc-8-1-0-on-centos-7-2-solved/4441 - # patch( - # "https://salsa.debian.org/debian-gis-team/hdf5/raw/bf94804af5f80f662cad80a5527535b3c6537df6/debian/patches/gcc-8.patch", - # sha256="57cee5ff1992b4098eda079815c36fc2da9b10e00a9056df054f2384c4fc7523", - # when="@1.10.2%gcc@8:", - # ) - - # Disable MPI C++ interface when C++ is disabled, otherwise downstream - # libraries fail to link; see https://github.com/spack/spack/issues/12586 - # patch( - # "h5public-skip-mpicxx.patch", - # when="@1.8.10:1.8.21,1.10.0:1.10.5+mpi~cxx", - # sha256="b61e2f058964ad85be6ee5ecea10080bf79e73f83ff88d1fa4b602d00209da9c", - # ) - - # Fixes BOZ literal constant error when compiled with GCC 10. - # The issue is described here: https://github.com/spack/spack/issues/18625 - # patch( - # "hdf5_1.8_gcc10.patch", - # when="@:1.8.21", - # sha256="0e20187cda3980a4fdff410da92358b63de7ebef2df1d7a425371af78e50f666", - # ) - - # patch("fortran-kinds.patch", when="@1.10.7") - - # This patch may only be needed with GCC11.2 on macOS, but it's valid for - # any of the head HDF5 versions as of 12/2021. Since it's impossible to - # tell what Fortran version is part of a mixed apple-clang toolchain on - # macOS (which is the norm), and this might be an issue for other compilers - # as well, we just apply it to all platforms. - # See https://github.com/HDFGroup/hdf5/issues/1157 - # patch("fortran-kinds-2.patch", when="@1.10.8,1.12.1") - - # The argument 'buf_size' of the C function 'h5fget_file_image_c' is - # declared as intent(in) though it is modified by the invocation. As a - # result, aggressive compilers such as Fujitsu's may do a wrong - # optimization to cause an error. - # GEOS_EDIT_END - - def patch(self): - filter_file( - "INTEGER(SIZE_T), INTENT(IN) :: buf_size", - "INTEGER(SIZE_T), INTENT(OUT) :: buf_size", - "fortran/src/H5Fff.F90", - string=True, - ignore_absent=True, - ) - filter_file( - "INTEGER(SIZE_T), INTENT(IN) :: buf_size", - "INTEGER(SIZE_T), INTENT(OUT) :: buf_size", - "fortran/src/H5Fff_F03.f90", - string=True, - ignore_absent=True, - ) - if self.run_tests: - # hdf5 has ~2200 CPU-intensive tests, some of them have races: - # Often, these loop endless(at least on one Xeon and one EPYC). - # testphdf5 fails indeterministic. This fixes finishing the tests - filter_file( - "REMOVE_ITEM H5P_TESTS", - "REMOVE_ITEM H5P_TESTS t_bigio t_shapesame testphdf5", - "testpar/CMakeTests.cmake", - ) - - # The parallel compiler wrappers (i.e. h5pcc, h5pfc, etc.) reference MPI - # compiler wrappers and do not need to be changed. - filter_compiler_wrappers( - "h5cc", "h5hlcc", "h5fc", "h5hlfc", "h5c++", "h5hlc++", relative_root="bin" - ) - - def url_for_version(self, version): - url = ( - "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-{0}/hdf5-{1}/src/hdf5-{1}.tar.gz" - ) - return url.format(version.up_to(2), version) - - def flag_handler(self, name, flags): - spec = self.spec - cmake_flags = [] - - if name == "cflags": - if spec.compiler.name in ["gcc", "clang", "apple-clang", "oneapi"]: - # Quiet warnings/errors about implicit declaration of functions - # in C99: - cmake_flags.append("-Wno-error=implicit-function-declaration") - # Note that this flag will cause an error if building %nvhpc. - if spec.satisfies("@:1.8.12~shared"): - # More recent versions set CMAKE_POSITION_INDEPENDENT_CODE to - # True and build with PIC flags. - cmake_flags.append(self.compiler.cc_pic_flag) - elif name == "cxxflags": - if spec.satisfies("@:1.8.12+cxx~shared"): - cmake_flags.append(self.compiler.cxx_pic_flag) - elif name == "fflags": - if spec.satisfies("%cce+fortran"): - # Cray compiler generates module files with uppercase names by - # default, which is not handled by the CMake scripts. The - # following flag forces the compiler to produce module files - # with lowercase names. - cmake_flags.append("-ef") - if spec.satisfies("@:1.8.12+fortran~shared"): - cmake_flags.append(self.compiler.fc_pic_flag) - elif name == "ldlibs": - if "+fortran %fj" in spec: - cmake_flags.extend(["-lfj90i", "-lfj90f", "-lfjsrcinfo", "-lelf"]) - - return flags, None, (cmake_flags or None) - - @property - def libs(self): - """HDF5 can be queried for the following parameters: - - - "hl": high-level interface - - "cxx": C++ APIs - - "fortran": Fortran APIs - - "java": Java APIs - - :return: list of matching libraries - """ - query_parameters = self.spec.last_query.extra_parameters - - shared = "+shared" in self.spec - - # This map contains a translation from query_parameters - # to the libraries needed - query2libraries = { - tuple(): ["libhdf5"], - ("cxx", "fortran", "hl", "java"): [ - # When installed with Autotools, the basename of the real - # library file implementing the High-level Fortran interface is - # 'libhdf5hl_fortran'. Starting versions 1.8.22, 1.10.5 and - # 1.12.0, the Autotools installation also produces a symbolic - # link 'libhdf5_hl_fortran.' to - # 'libhdf5hl_fortran.'. Note that in the case of the - # dynamic library, the latter is a symlink to the real sonamed - # file 'libhdf5_fortran.so.'. This means that all - # dynamically linked executables/libraries of the dependent - # packages need 'libhdf5_fortran.so.' with the same - # DT_SONAME entry. However, the CMake installation (at least - # starting version 1.8.10) does not produce it. Instead, the - # basename of the library file is 'libhdf5_hl_fortran'. Which - # means that switching to CMake requires rebuilding of all - # dependant packages that use the High-level Fortran interface. - # Therefore, we do not try to preserve backward compatibility - # with Autotools installations by creating symlinks. The only - # packages that could benefit from it would be those that - # hardcode the library name in their building systems. Such - # packages should simply be patched. - "libhdf5_hl_fortran", - "libhdf5_hl_f90cstub", - "libhdf5_hl_cpp", - "libhdf5_hl", - "libhdf5_fortran", - "libhdf5_f90cstub", - "libhdf5_java", - "libhdf5", - ], - ("cxx", "hl"): [ - "libhdf5_hl_cpp", - "libhdf5_hl", - "libhdf5", - ], - ("fortran", "hl"): [ - "libhdf5_hl_fortran", - "libhdf5_hl_f90cstub", - "libhdf5_hl", - "libhdf5_fortran", - "libhdf5_f90cstub", - "libhdf5", - ], - ("hl",): [ - "libhdf5_hl", - "libhdf5", - ], - ("cxx", "fortran"): [ - "libhdf5_fortran", - "libhdf5_f90cstub", - "libhdf5_cpp", - "libhdf5", - ], - ("cxx",): [ - "libhdf5_cpp", - "libhdf5", - ], - ("fortran",): [ - "libhdf5_fortran", - "libhdf5_f90cstub", - "libhdf5", - ], - ("java",): [ - "libhdf5_java", - "libhdf5", - ], - } - - # Turn the query into the appropriate key - key = tuple(sorted(query_parameters)) - libraries = query2libraries[key] - - return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True) - - @when("@:1.8.21,1.10.0:1.10.5+szip") - def setup_build_environment(self, env): - env.set("SZIP_INSTALL", self.spec["szip"].prefix) - - @run_before("cmake") - def fortran_check(self): - if "+fortran" in self.spec and not self.compiler.fc: - msg = "cannot build a Fortran variant without a Fortran compiler" - raise RuntimeError(msg) - - def cmake_args(self): - spec = self.spec - - if spec.satisfies("@:1.8.15+shared"): - tty.warn("hdf5@:1.8.15+shared does not produce static libraries") - - args = [ - # Always enable this option. This does not actually enable any - # features: it only *allows* the user to specify certain - # combinations of other arguments. - self.define("ALLOW_UNSUPPORTED", True), - # Speed-up the building by skipping the examples: - self.define("HDF5_BUILD_EXAMPLES", False), - self.define( - "BUILD_TESTING", - self.run_tests or - # Version 1.8.22 fails to build the tools when shared libraries - # are enabled but the tests are disabled. - spec.satisfies("@1.8.22+shared+tools"), - ), - self.define("HDF5_ENABLE_Z_LIB_SUPPORT", True), - self.define_from_variant("HDF5_ENABLE_SZIP_SUPPORT", "szip"), - self.define_from_variant("HDF5_ENABLE_SZIP_ENCODING", "szip"), - self.define_from_variant("BUILD_SHARED_LIBS", "shared"), - self.define("ONLY_SHARED_LIBS", False), - self.define_from_variant("HDF5_ENABLE_PARALLEL", "mpi"), - self.define_from_variant("HDF5_ENABLE_THREADSAFE", "threadsafe"), - self.define_from_variant("HDF5_BUILD_HL_LIB", "hl"), - self.define_from_variant("HDF5_BUILD_CPP_LIB", "cxx"), - self.define_from_variant("HDF5_BUILD_FORTRAN", "fortran"), - self.define_from_variant("HDF5_BUILD_JAVA", "java"), - self.define_from_variant("HDF5_BUILD_TOOLS", "tools"), - ] - - api = spec.variants["api"].value - if api != "default": - args.append(self.define("DEFAULT_API_VERSION", api)) - - if "+mpi" in spec: - args.append(self.define("CMAKE_C_COMPILER", spec["mpi"].mpicc)) - - if "+cxx" in self.spec: - args.append(self.define("CMAKE_CXX_COMPILER", spec["mpi"].mpicxx)) - - if "+fortran" in self.spec: - args.append(self.define("CMAKE_Fortran_COMPILER", spec["mpi"].mpifc)) - - # work-around for https://github.com/HDFGroup/hdf5/issues/1320 - if spec.satisfies("@1.10.8,1.13.0"): - args.append(self.define("HDF5_INSTALL_CMAKE_DIR", "share/cmake/hdf5")) - - return args - - @run_after("install") - def ensure_parallel_compiler_wrappers(self): - # When installed with Autotools and starting at least version 1.8.10, - # the package produces C compiler wrapper called either 'h5cc' (when MPI - # support is disabled) or 'h5pcc' (when MPI support is enabled). The - # CMake installation produces the wrapper called 'h5cc' (regardless of - # whether MPI support is enabled) only starting versions 1.8.21, 1.10.2 - # and 1.12.0. The current develop versions also produce 'h5pcc' when MPI - # support is enabled and the file is identical to 'h5cc'. Here, we make - # sure that 'h5pcc' is available when MPI support is enabled (only for - # versions that generate 'h5cc'). - if self.spec.satisfies("@1.8.21:1.8.22,1.10.2:1.10.7,1.12.0+mpi"): - with working_dir(self.prefix.bin): - # No try/except here, fix the condition above instead: - symlink("h5cc", "h5pcc") - - # The same as for 'h5pcc'. However, the CMake installation produces the - # Fortran compiler wrapper called 'h5fc' only starting versions 1.8.22, - # 1.10.6 and 1.12.0. The current develop versions do not produce 'h5pfc' - # at all. Here, we make sure that 'h5pfc' is available when Fortran and - # MPI support are enabled (only for versions that generate 'h5fc'). - if self.spec.satisfies( - "@1.8.22:1.8," "1.10.6:1.10," "1.12.0:1.12," "develop:" "+fortran+mpi" - ): - with working_dir(self.prefix.bin): - # No try/except here, fix the condition above instead: - symlink("h5fc", "h5pfc") - - @run_after("install") - def fix_package_config(self): - # We need to fix the pkg-config files, which are also used by the - # compiler wrappers. The files are created starting versions 1.8.21, - # 1.10.2 and 1.12.0. However, they are broken (except for the version - # 1.8.22): the files are named -.pc but reference - # packages. This was fixed in the develop versions at some point: the - # files started referencing - packages but got broken - # again: the files got names .pc but references had not been - # updated accordingly. Another issue, which we address here, is that - # some Linux distributions install pkg-config files named hdf5.pc and we - # want to override them. Therefore, the following solution makes sure - # that each -.pc file is symlinked by .pc and all - # references to - packages in the original files are - # replaced with references to packages. - pc_files = find(self.prefix.lib.pkgconfig, "hdf5*.pc", recursive=False) - - if not pc_files: - # This also tells us that the pkgconfig directory does not exist. - return - - # Replace versioned references in all pkg-config files: - filter_file( - r"(Requires(?:\.private)?:.*)(hdf5[^\s,]*)(?:-[^\s,]*)(.*)", - r"\1\2\3", - *pc_files, - backup=False - ) - - # Create non-versioned symlinks to the versioned pkg-config files: - with working_dir(self.prefix.lib.pkgconfig): - for f in pc_files: - src_filename = os.path.basename(f) - version_sep_idx = src_filename.find("-") - if version_sep_idx > -1: - tgt_filename = src_filename[:version_sep_idx] + ".pc" - if not os.path.exists(tgt_filename): - symlink(src_filename, tgt_filename) - - @run_after("install") - @on_package_attributes(run_tests=True) - def check_install(self): - self._check_install() - - def _check_install(self): - # Build and run a small program to test the installed HDF5 library - spec = self.spec - print("Checking HDF5 installation...") - checkdir = "spack-check" - with working_dir(checkdir, create=True): - # Because the release number in a develop branch is not fixed, - # only the major and minor version numbers are compared. - # Otherwise all 3 numbers are checked. - if "develop" in str(spec.version.up_to(3)): - source = r""" -#include -#include -#include -int main(int argc, char **argv) { - unsigned majnum, minnum, relnum; - herr_t herr = H5get_libversion(&majnum, &minnum, &relnum); - assert(!herr); - printf("HDF5 version %d.%d %u.%u\n", H5_VERS_MAJOR, H5_VERS_MINOR, - majnum, minnum); - return 0; -} -""" - else: - source = r""" -#include -#include -#include -int main(int argc, char **argv) { - unsigned majnum, minnum, relnum; - herr_t herr = H5get_libversion(&majnum, &minnum, &relnum); - assert(!herr); - printf("HDF5 version %d.%d.%d %u.%u.%u\n", H5_VERS_MAJOR, H5_VERS_MINOR, - H5_VERS_RELEASE, majnum, minnum, relnum); - return 0; -} -""" - expected = """\ -HDF5 version {version} {version} -""".format( - version=str(spec.version.up_to(3)) - ) - if "develop" in expected: - # Remove 'develop-' from the version in spack for checking - # version against the version in the HDF5 code. - expected = """\ -HDF5 version {version} {version} -""".format( - version=str(spec.version.up_to(3)).partition("-")[2] - ) - with open("check.c", "w") as f: - f.write(source) - if "+mpi" in spec: - cc = Executable(spec["mpi"].mpicc) - else: - cc = Executable(self.compiler.cc) - cc(*(["-c", "check.c"] + spec["hdf5"].headers.cpp_flags.split())) - cc(*(["-o", "check", "check.o"] + spec["hdf5"].libs.ld_flags.split())) - try: - check = Executable("./check") - output = check(output=str) - except ProcessError: - output = "" - success = output == expected - if not success: - print("Produced output does not match expected output.") - print("Expected output:") - print("-" * 80) - print(expected) - print("-" * 80) - print("Produced output:") - print("-" * 80) - print(output) - print("-" * 80) - raise RuntimeError("HDF5 install check failed") - shutil.rmtree(checkdir) - - def _test_check_versions(self): - """Perform version checks on selected installed package binaries.""" - spec_vers_str = "Version {0}".format(self.spec.version) - if "develop" in spec_vers_str: - # Remove 'develop-' from the version in spack for checking - # version against the version in the HDF5 code. - spec_vers_str = spec_vers_str.partition("-")[2] - - exes = [ - "h5copy", - "h5diff", - "h5dump", - "h5format_convert", - "h5ls", - "h5mkgrp", - "h5repack", - "h5stat", - "h5unjam", - ] - use_short_opt = ["h52gif", "h5repart", "h5unjam"] - for exe in exes: - reason = "test: ensuring version of {0} is {1}".format(exe, spec_vers_str) - option = "-V" if exe in use_short_opt else "--version" - self.run_test( - exe, option, spec_vers_str, installed=True, purpose=reason, skip_missing=True - ) - - def _test_example(self): - """This test performs copy, dump, and diff on an example hdf5 file.""" - test_data_dir = self.test_suite.current_test_data_dir - - filename = "spack.h5" - h5_file = test_data_dir.join(filename) - - reason = "test: ensuring h5dump produces expected output" - expected = get_escaped_text_output(test_data_dir.join("dump.out")) - self.run_test( - "h5dump", - filename, - expected, - installed=True, - purpose=reason, - skip_missing=True, - work_dir=test_data_dir, - ) - - reason = "test: ensuring h5copy runs" - options = ["-i", h5_file, "-s", "Spack", "-o", "test.h5", "-d", "Spack"] - self.run_test( - "h5copy", options, [], installed=True, purpose=reason, skip_missing=True, work_dir="." - ) - - reason = "test: ensuring h5diff shows no differences between orig and" " copy" - self.run_test( - "h5diff", - [h5_file, "test.h5"], - [], - installed=True, - purpose=reason, - skip_missing=True, - work_dir=".", - ) - - def test(self): - """Perform smoke tests on the installed package.""" - # Simple version check tests on known binaries - self._test_check_versions() - - # Run sequence of commands on an hdf5 file - self._test_example() - - # Run existing install check - self._check_install() \ No newline at end of file From b30e2627201bd9abd265b76eb0a46c7263d00c35 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Mon, 6 Oct 2025 16:03:28 -0700 Subject: [PATCH 17/55] Generalize .gitignore for uberenv testing --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 89c70d36..704d7cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ .vscode/ spack-*.txt __pycache__ -uberenv_libs +uberenv_* From bf0e15630db6e0cf2ff906f3ddcb277024ae6fac Mon Sep 17 00:00:00 2001 From: Brian Han Date: Mon, 6 Oct 2025 16:04:25 -0700 Subject: [PATCH 18/55] Toolchain changes for toss 4 - CUDA is untested --- .../spack_configs/toss_4_x86_64_ib/spack.yaml | 56 ++++++++++++++----- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 02d82253..68f05645 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -10,24 +10,28 @@ # See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. #------------------------------------------------------------------------------------------------------------ -# geosx@develop %clang-14 -# geosx@develop %gcc-12 -# geosx@develop %gcc-13 -# geosx@develop %gcc-12-noAVX -# geosx@develop %gcc-13-noAVX -# # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+docs %clang-14" # # python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-12" # # python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-13" # -# No AVX instructions +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %clang-14" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %clang-19" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %gcc-12 ^cuda@12.6.0+allow-unsupported-compilers" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %clang-14 ^cuda@12.6.0+allow-unsupported-compilers" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %clang-19 ^cuda@12.9.1+allow-unsupported-compilers" +# +# +# No AVX instructions: # python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-12-noAVX" # -# python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-13-noAVX" - spack: config: @@ -49,14 +53,23 @@ spack: - ../versions.yaml toolchains: + clang-19: + - spec: '%c=llvm@19.1.3' + when: '%c' + - spec: '%cxx=llvm@19.1.3' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.clang_19' + when: '%mpi' clang-14: - spec: '%[virtuals=c]llvm@14.0.6+clang~flang~lld~lldb' when: '%c' - spec: '%[virtuals=cxx]llvm@14.0.6+clang~flang~lld~lldb' when: '%cxx' - - spec: '%[virtuals=fortran]gcc@10.3.1' + - spec: '%[virtuals=fortran]gcc@12.1.1' when: '%fortran' - - spec: '%mvapich2@2.3.7.clang' + - spec: '%mvapich2@2.3.7.clang_14' when: '%mpi' gcc-12: - spec: '%c=gcc@12.1.1' @@ -126,6 +139,13 @@ spack: c: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang cxx: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang++ extra_rpaths: [] + - spec: llvm@19.1.3+clang~lld~lldb + prefix: /usr/tce/packages/clang/clang-19.1.3 + extra_attributes: + compilers: + c: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang + cxx: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang++ + flags: {} gcc: externals: - spec: gcc@10.3.1 languages:='c,c++,fortran' @@ -163,8 +183,10 @@ spack: prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic - spec: mvapich2@2.3.7.gcc process_managers=slurm %gcc@13.3.1 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-13.3.1-magic - - spec: mvapich2@2.3.7.clang process_managers=slurm %clang@14.0.6 + - spec: mvapich2@2.3.7.clang_14 process_managers=slurm %llvm@14.0.6 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6-magic + - spec: mvapich2@2.3.7.clang_19 process_managers=slurm %llvm@19.1.3 + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3 # Spack issue prevents compilation: https://github.com/spack/spack/issues/49994 # intel-oneapi-mkl: @@ -173,6 +195,14 @@ spack: # prefix: /usr/tce/packages/mkl/mkl-2022.1.0/ # buildable: False + cuda: + buildable: False + externals: + - spec: cuda@12.6.0 +allow-unsupported-compilers + prefix: /usr/tce/packages/cuda/cuda-12.6.0 + - spec: cuda@12.9.1 +allow-unsupported-compilers + prefix: /usr/tce/packages/cuda/cuda-12.9.1 + # System level packages to not build papi: buildable: False From 93a6e9bea6a0776ceeefef640b57707733625801 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Mon, 6 Oct 2025 16:05:18 -0700 Subject: [PATCH 19/55] toss 4 cray rocm toolchain changes - hypre+magma unexpected concretization failure to work through --- .../toss_4_x86_64_ib_cray/spack.yaml | 184 ++++++++++-------- 1 file changed, 104 insertions(+), 80 deletions(-) diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 814e4e5d..f129e39b 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -13,7 +13,7 @@ # geosx@develop%cce@20.0.0 # # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="%cce@20.0.0 +docs" +# python3 ./scripts/uberenv/uberenv.py --spec="+rocm~pygeosx+hypre~trilinos~petsc~docs amdgpu_target=gfx942 %cce_20 ^hip@6.4.2 ^hipblas@6.4.2 ^hipsparse@6.4.2 ^hsa-rocr-dev@6.4.2 ^rocprim@6.4.2 ^hdf5 cflags=-Wno-int-conversion" # @@ -36,76 +36,28 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers: - - compiler: - spec: cce@=20.0.0 - paths: - cc: /usr/tce/packages/cce-tce/cce-20.0.0/bin/craycc - cxx: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayCC - f77: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayftn - fc: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayftn - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - PrgEnv-cray/8.6.0 - - craype-x86-genoa - - craype/2.7.35 - - craype-network-ofi - - craype-accel-amd-gfx942 - - cray-mpich/9.0.1 - - cray-libsci/25.09.0 - - cce/20.0.0 - - libfabric/2.1 - - flux_wrappers/0.1 - - StdEnv - - perftools-base/25.09.0 - - xpmem/2.6.5 - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: - - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib - - /opt/cray/pe/mpich/9.0.1/gtl/lib - - compiler: - spec: rocmcc@=6.4.2 - paths: - cc: /opt/rocm-6.4.2/llvm/bin/amdclang - cxx: /opt/rocm-6.4.2/llvm/bin/amdclang++ - f77: /opt/rocm-6.4.2/llvm/bin/amdflang - fc: /opt/rocm-6.4.2/llvm/bin/amdflang - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - rocm/6.4.2 - - PrgEnv-cray/8.6.0 - - craype-x86-genoa - - craype/2.7.35 - - craype-network-ofi - - craype-accel-amd-gfx942 - - cray-mpich/9.0.1 - - cray-libsci/25.09.0 - - cce/20.0.0 - - libfabric/2.1 - - flux_wrappers/0.1 - - StdEnv - - perftools-base/25.09.0 - - xpmem/2.6.5 - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: - - /opt/rocm-6.4.2/lib - - /opt/rocm-6.4.2/llvm/lib - - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib - - /opt/cray/pe/mpich/9.0.1/gtl/lib + toolchains: + llvm-amdgpu_6_4_2: + - spec: '%c=llvm-amdgpu@6.4.2' + when: '%c' + - spec: '%cxx=llvm-amdgpu@6.4.2' + when: '%cxx' + - spec: '%fortran=llvm-amdgpu@6.4.2' + when: '%fortran' + - spec: '%cray-mpich@9.0.1_llvm_amdgpu' + when: '%mpi' + cce_20: + - spec: '%c=cce@20.0.0' + when: '%c' + - spec: '%cxx=cce@20.0.0' + when: '%cxx' + - spec: '%fortran=cce@20.0.0' + when: '%fortran' + - spec: '%cray-mpich@9.0.1_cce' + when: '%mpi' packages: all: - require: target=zen4 providers: blas: [netlib-lapack] lapack: [netlib-lapack] @@ -113,14 +65,94 @@ spack: zlib-api: [zlib] pkgconfig: [pkgconf] + cce: + externals: + - spec: cce@=20.0.0 + prefix: /usr/tce/packages/cce-tce/cce-20.0.0 + extra_attributes: + compilers: + c: /usr/tce/packages/cce-tce/cce-20.0.0/bin/craycc + cxx: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayCC + fortran: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayftn + # flags: {} + # environment: {} + # extra_rpaths: [] + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: + - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib + - /opt/cray/pe/mpich/9.0.1/gtl/lib + flags: + cflags: -fPIC + cxxflags: -fPIC + fflags: -fPIC + modules: + - PrgEnv-cray/8.6.0 + - craype-x86-genoa + - craype/2.7.35 + - craype-network-ofi + - craype-accel-amd-gfx942 + - cray-mpich/9.0.1 + - cray-libsci/25.09.0 + - cce/20.0.0 + - libfabric/2.1 + - flux_wrappers/0.1 + - StdEnv + - perftools-base/25.09.0 + - xpmem/2.6.5 + + + llvm-amdgpu: + externals: + - spec: llvm-amdgpu@=6.4.2 + prefix: /opt/rocm-6.4.2/llvm + extra_attributes: + compilers: + c: /opt/rocm-6.4.2/llvm/bin/amdclang + cxx: /opt/rocm-6.4.2/llvm/bin/amdclang++ + fortran: /opt/rocm-6.4.2/llvm/bin/amdflang + # environment: {} + # extra_rpaths: [] + # flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: + - /opt/rocm-6.4.2/lib + - /opt/rocm-6.4.2/llvm/lib + - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib + - /opt/cray/pe/mpich/9.0.1/gtl/lib + modules: + - rocm/6.4.2 + - PrgEnv-cray/8.6.0 + - craype-x86-genoa + - craype/2.7.35 + - craype-network-ofi + - craype-accel-amd-gfx942 + - cray-mpich/9.0.1 + - cray-libsci/25.09.0 + - cce/20.0.0 + - libfabric/2.1 + - flux_wrappers/0.1 + - StdEnv + - perftools-base/25.09.0 + - xpmem/2.6.5 + + + # Lock down which MPI we are using cray-mpich: buildable: false externals: - - spec: cray-mpich@9.0.1%cce@20.0.0 - prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-cce-20.0.0-magic - - spec: cray-mpich@9.0.1%rocmcc@6.4.2 - prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-rocmcc-6.4.2-cce-20.0.0-magic + - spec: cray-mpich@9.0.1_cce %cce@20.0.0 + # prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-cce-20.0.0-magic + prefix: /usr/tce/packages/cray-mpich-tce/cray-mpich-9.0.1-rocmcc-6.4.2-cce-20.0.0/ + - spec: cray-mpich@9.0.1_llvm_amdgpu %llvm-amdgpu@6.4.2 + # prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-rocmcc-6.4.2-cce-20.0.0-magic + prefix: /usr/tce/packages/cray-mpich-tce/cray-mpich-9.0.1-rocmcc-6.4.2/ # ROCm packages hip: @@ -163,14 +195,6 @@ spack: prefix: /opt/rocm-6.4.0 - spec: hipsolver@6.4.2 prefix: /opt/rocm-6.4.2 - llvm-amdgpu: - version: [6.4.0, 6.4.2] - buildable: false - externals: - - spec: llvm-amdgpu@6.4.0 - prefix: /opt/rocm-6.4.0/llvm - - spec: llvm-amdgpu@6.4.2 - prefix: /opt/rocm-6.4.2/llvm hsa-rocr-dev: version: [6.4.0, 6.4.2] buildable: false From 07beddb8b4a84f3b04f71b5b12e440c1c5eaab8a Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 02:59:24 -0400 Subject: [PATCH 20/55] Fix minor issues --- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 1 - docker/tpl-rockylinux-gcc-cuda-12.Dockerfile | 1 - 2 files changed, 2 deletions(-) diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index ec6a9ce0..2c0ca66f 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -31,7 +31,6 @@ RUN dnf clean all && \ flex \ bison \ gettext \ - help2man \ gnupg \ xz \ python3-virtualenv diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index af7a969d..cfa9ca80 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -29,7 +29,6 @@ RUN dnf clean all && \ flex \ bison \ gettext \ - help2man \ gnupg \ xz \ python3-virtualenv From cffc14a3cc87a080849d6a69edd496d5c6ddb9ad Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 02:59:48 -0400 Subject: [PATCH 21/55] Testing out llvm-amdgpu --- docker/tpl-ubuntu-gcc-hip.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-gcc-hip.Dockerfile index 040b8c46..6fa10532 100644 --- a/docker/tpl-ubuntu-gcc-hip.Dockerfile +++ b/docker/tpl-ubuntu-gcc-hip.Dockerfile @@ -106,7 +106,7 @@ RUN apt-get update && \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %gcc-13 ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-18 ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ From 369e808ea567e794f6314daf7b3dac33d5d0f226 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 09:15:31 -0700 Subject: [PATCH 22/55] Fix target arch --- scripts/spack_configs/toss_4_x86_64_ib/spack.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 68f05645..577d6811 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -41,8 +41,8 @@ spack: all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' misc_cache: $spack/../misc_cache test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage + build_stage: $spack/../build_stage + build_jobs: 112 # Regular TPLs do not need views view: false @@ -112,7 +112,7 @@ spack: packages: all: - target: [ivybridge] + target: [sapphirerapids] mpi: require: From 9f17cc942359d867f33661966f2828977a22e097 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 09:15:54 -0700 Subject: [PATCH 23/55] bump up build_jobs --- scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index f129e39b..7c4cc2e6 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -26,7 +26,7 @@ spack: misc_cache: $spack/../misc_cache test_stage: $spack/../test_stage build_stage: $spack/../build_stage - build_jobs: 48 + build_jobs: 96 # Regular TPLs do not need views view: false From ca9dbc8ff66723e9129ec7649d06a1794f6aa175 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 09:16:17 -0700 Subject: [PATCH 24/55] Add --no-permissions flag --- scripts/setupLC-TPL-uberenv-helper.bash | 81 +++++++----- scripts/setupLC-TPL-uberenv.bash | 165 ++++++++++++++++++------ 2 files changed, 174 insertions(+), 72 deletions(-) diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index 8bf5feeb..e62f9e94 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -1,56 +1,75 @@ #!/bin/bash ## Builds the TPLs for a specific system and host config. -## Usage ./setupLC-TPL-uberenv-helper.bash pathToInstallDirectory machine compiler spackSpecToBuild commandToGetANode +## Usage ./setupLC-TPL-uberenv-helper.bash [ExtraArgs...] + +# --- 1. Initialize Control Variable --- +# By default, we will set permissions. +SET_PERMISSIONS=true + +# --- Argument Parsing --- INSTALL_DIR=$1 MACHINE=$2 COMPILER=$3 SPEC=\"${4}\" GET_A_NODE=$5 -## Eat up the command line arguments so the rest can be forwarded to config-build. -shift -shift -shift -shift -shift +# Eat up the primary arguments; the rest in $@ are forwarded. +shift 5 +# --- 2. Check for the --no-permissions Flag --- +# Loop through the forwarded arguments to find our flag. +for arg in "$@"; do + if [[ "$arg" == "--no-permissions" ]]; then + SET_PERMISSIONS=false + echo "Found --no-permissions flag. Will skip permission updates." + break + fi +done + +# --- Main Execution --- CONFIG=$MACHINE-$COMPILER LOG_FILE=$CONFIG.log echo "Building the TPLs on $MACHINE for $COMPILER to be installed at $INSTALL_DIR. Progress will be written to $LOG_FILE." -ssh $MACHINE -t " -. /etc/profile && -cd $PWD && -$GET_A_NODE ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix ${INSTALL_DIR}/${CONFIG}_tpls --spack-env-name ${CONFIG}_env && -exit" > $LOG_FILE 2>&1 +# Note: The ssh command forwards the extra arguments ($@) to uberenv.py +ssh "$MACHINE" -t " +. /etc/profile && +cd \"$PWD\" && +$GET_A_NODE ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" --spack-env-name \"${CONFIG}_env\" \"$@\" && +exit" > "$LOG_FILE" 2>&1 ## Check the last ten lines of the log file. ## A successful install should show up on one of the final lines. -tail -10 $LOG_FILE | grep -E "Successfully installed geos" > /dev/null +tail -10 "$LOG_FILE" | grep -E "Successfully installed geos" > /dev/null if [ $? -eq 0 ]; then - echo "Cleanup extra build files at ${INSTALL_DIR}/${CONFIG}_tpls/ ." - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/${CONFIG}_env - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/.spack-db - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/misc_cache - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/spack - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/build_stage - - echo "Updating file permissions at ${INSTALL_DIR}/${CONFIG}_tpls/ ." + echo "Cleanup extra build files at ${INSTALL_DIR}/${CONFIG}_tpls/." + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/${CONFIG}_env" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/.spack-db" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/misc_cache" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/spack" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/build_stage" + + # --- 3. Conditionally Set Permissions --- + if [ "$SET_PERMISSIONS" = true ]; then + echo "Updating file permissions at ${INSTALL_DIR}/${CONFIG}_tpls/." # Install directory root - chmod g+rx $INSTALL_DIR - chgrp GEOS $INSTALL_DIR + chmod g+rx "$INSTALL_DIR" + chgrp GEOS "$INSTALL_DIR" # Update only executable and library directories to avoid NFS errors - chmod g+rx -R $INSTALL_DIR/${CONFIG}_tpls/bin - chgrp GEOS -R $INSTALL_DIR/${CONFIG}_tpls/bin - chmod g+rx -R $INSTALL_DIR/${CONFIG}_tpls/${COMPILER%%-*}* - chgrp GEOS -R $INSTALL_DIR/${CONFIG}_tpls/${COMPILER%%-*}* + chmod g+rx -R "${INSTALL_DIR}/${CONFIG}_tpls/bin" + chgrp GEOS -R "${INSTALL_DIR}/${CONFIG}_tpls/bin" + chmod g+rx -R "${INSTALL_DIR}/${CONFIG}_tpls/${COMPILER%%-*}"* + chgrp GEOS -R "${INSTALL_DIR}/${CONFIG}_tpls/${COMPILER%%-*}"* + else + echo "Skipping permission updates as requested." + fi - echo "Build of ${CONFIG} completed successfully." - exit 0 + echo "Build of ${CONFIG} completed successfully." + exit 0 else - echo "Build of ${CONFIG} seemed to fail, check $LOG_FILE." - exit 1 + echo "Build of ${CONFIG} seemed to fail, check $LOG_FILE." + exit 1 fi diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index c118b53e..479eebc2 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -1,62 +1,145 @@ #!/bin/bash -## Builds the TPLs on all LC systems. Must be run from the top level TPL directory. -## Usage ./setupLC-TPL-uberenv.bash pathToInstallDirectory +## Builds the TPLs on specified LC systems. +## Usage: ./setupLC-TPL-uberenv.bash [MachineList] [ExtraArgs...] +## +## InstallDir: Absolute path to the installation directory. +## MachineList: (Optional) Comma-separated list of machines to build on +## (e.g., "dane,matrix"). Defaults to all. +## ExtraArgs: (Optional) Additional arguments forwarded to the helper script. +## Use --no-permissions to skip all chmod/chgrp calls. + +# --- Configuration --- +# All known machines. Add new machine names here. +declare -a ALL_MACHINES=("dane" "matrix" "tuolumne") + +# --- Argument Parsing --- INSTALL_DIR=$1 +MACHINE_LIST_STR=${2:-"all"} # Default to "all" if the second argument is not provided -## Eat up the command line arguments so the rest can be forwarded to setupLC-TPL-helper. -shift -shift +# Validate INSTALL_DIR before proceeding +if [[ -z "$INSTALL_DIR" ]]; then + echo "ERROR: No installation directory path was provided." >&2 + exit 1 +fi +if [[ ! -d "$INSTALL_DIR" ]]; then + echo "ERROR: Installation directory '$INSTALL_DIR' does not exist." >&2 + exit 1 +fi +if [[ ! "$INSTALL_DIR" = /* ]]; then + echo "ERROR: Installation directory must be an absolute path." >&2 + exit 1 +fi -## Trap the interupt signal and kill all children. -trap 'killall' INT +# Eat up the first two arguments so the rest can be forwarded. +shift 2 2>/dev/null -killall() { - trap '' INT TERM # ignore INT and TERM while shutting down - echo "**** Shutting down. Killing chid processes ****" # added double quotes - kill -TERM 0 # fixed order, send TERM not INT - wait - echo DONE -} +# --- Initialize Control Variable and Parse Extra Arguments --- +SET_PERMISSIONS=true +declare -a FORWARDED_ARGS=() + +# Loop through remaining args, filter out our flag, and collect the rest. +for arg in "$@"; do + if [[ "$arg" == "--no-permissions" ]]; then + SET_PERMISSIONS=false + else + FORWARDED_ARGS+=("$arg") + fi +done +# --- Setup --- +# Check for uberenv script if [[ ! -e "scripts/uberenv/uberenv.py" ]]; then - echo "uberenv.py script not found. Please initialize uberenv submodule first." - exit + echo "ERROR: uberenv.py script not found. Please initialize uberenv submodule first." >&2 + exit 1 fi -if [[ -z $INSTALL_DIR ]]; then - echo "No installation directory path was provided" - exit +# Determine which machines to run on +declare -a MACHINES_TO_RUN +if [[ "$MACHINE_LIST_STR" == "all" ]]; then + MACHINES_TO_RUN=("${ALL_MACHINES[@]}") +else + # Convert comma-separated string to array + IFS=',' read -r -a MACHINES_TO_RUN <<< "$MACHINE_LIST_STR" fi -if [[ ! -d $INSTALL_DIR ]]; then - echo "Installation directory $INSTALL_DIR does not exist. Please initialize first." - exit -fi +# --- Functions --- +# Trap the interrupt signal and kill all children. +trap 'kill_children' INT -if [[ ! "$INSTALL_DIR" = /* ]]; then - echo "Installation directory $INSTALL_DIR must be an absolute path." - exit -fi +function kill_children() { + trap '' INT TERM # Ignore signals while shutting down + echo -e "\n**** Shutting down. Sending TERM signal to child processes ****" + # Kill the entire process group, which includes all background jobs + kill -TERM 0 + wait + echo "DONE" +} + +# Function to launch jobs for a specific machine +function launch_jobs() { + local machine=$1 + shift # The rest of $@ are the forwarded arguments + local UBERENV_HELPER=./scripts/setupLC-TPL-uberenv-helper.bash + + echo "-----> Launching jobs for [${machine}]..." + + case "$machine" in + dane) + ALLOC_CMD="salloc -N 1 -n 112 --exclusive -t 120 -A vortex" + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %gcc-12" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12noAVX "+docs %gcc-12-noAVX" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %gcc-13" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane clang-14 "+docs %clang-14" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane clang-19 "+docs %clang-19" "${ALLOC_CMD}" "$@" & + ;; + + matrix) + ALLOC_CMD="salloc -N 1 --exclusive -t 120 -A vortex" + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %gcc-12 ^cuda@12.6.0+allow-unsupported-compilers" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix clang-14-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %clang-14 ^cuda@12.6.0+allow-unsupported-compilers" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix clang-19-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %clang-19 ^cuda@12.9.1+allow-unsupported-compilers" "${ALLOC_CMD}" "$@" & + ;; -echo "Building all LC TPLs from $GEOS_BRANCH to be installed at $INSTALL_DIR..." + tuolumne) + ALLOC_CMD="salloc -N 1 --exclusive -t 120 -A vortex" + "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs %cce-20 amdgpu_target=gfx942" "${ALLOC_CMD}" "$@" & + ;; -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane gcc-12 "%gcc@12.1.1 +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane gcc-12noAVX "%gcc@12noAVX +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane gcc-13 "%gcc@13.3.1 +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane clang-14 "%clang@14.0.6 +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane clang-19 "%clang@19.1.3 +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR matrix gcc-12-cuda-12.6 "%gcc@12.1.1+cuda~uncrustify cuda_arch=90 ^cuda@12.6.0+allow-unsupported-compilers" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR matrix gcc-13-cuda-12.9 "%gcc@13.3.1+cuda~uncrustify cuda_arch=90 ^cuda@12.9.1+allow-unsupported-compilers" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR matrix clang-14-cuda-12.6 "%clang@14.0.6+cuda~uncrustify cuda_arch=90 ^cuda@12.6.0+allow-unsupported-compilers" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR matrix clang-19-cuda-12.9 "%clang@19.1.3+cuda~uncrustify cuda_arch=90 ^cuda@12.9.1+allow-unsupported-compilers" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR tuolumne cce-20-rocm-6.4.2 "%cce@20.0.0 +rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & + *) + echo "WARNING: Unknown machine '$machine'. Skipping." >&2 + ;; + esac +} + +# --- Main Execution --- +echo "Building TPLs for machines: ${MACHINES_TO_RUN[*]}" +echo "Installation directory: $INSTALL_DIR" +echo "Forwarded arguments: ${FORWARDED_ARGS[*]}" +if [ "$SET_PERMISSIONS" = false ]; then + echo "Permissions: SKIPPING all chmod/chgrp calls." +fi +echo "---" +for machine in "${MACHINES_TO_RUN[@]}"; do + launch_jobs "$machine" "${FORWARDED_ARGS[@]}" +done + +echo "All jobs launched. Waiting for completion..." # Note: Estimated completion time is ~90 minutes. # Check log files for unreported completion of jobs. wait -chmod -R g+rx $INSTALL_DIR -chgrp -R GEOS $INSTALL_DIR +# --- Conditionally Set Final Permissions --- +if [ "$SET_PERMISSIONS" = true ]; then + echo "---" + echo "Finalizing permissions..." + chmod -R g+rx "$INSTALL_DIR" + chgrp -R GEOS "$INSTALL_DIR" +else + echo "---" + echo "Skipping final permission updates as requested." +fi -echo "Complete" +echo "Complete." From dbd3777c13c6bd7388d8d9d80d355c8c03fff4e9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" <50467563+victorapm@users.noreply.github.com> Date: Tue, 7 Oct 2025 13:50:01 -0400 Subject: [PATCH 25/55] Update scripts/setupLC-TPL-uberenv-helper.bash Co-authored-by: Brian Han --- scripts/setupLC-TPL-uberenv-helper.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index e62f9e94..de90dae8 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -49,6 +49,7 @@ if [ $? -eq 0 ]; then rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/.spack-db" rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/misc_cache" rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/spack" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/builtin_spack_packages_repo" rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/build_stage" # --- 3. Conditionally Set Permissions --- From 6a0bdc20c2bf8e5248ab1f85d3d7ce81b2e2b8e5 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 11:58:24 -0700 Subject: [PATCH 26/55] Fix ATS flags for Dane and Matrix --- scripts/spack_packages/packages/geosx/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 07116776..d6ba8d69 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -661,9 +661,9 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): # Lassen if sys_type in ('blueos_3_ppc64le_ib_p9'): cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--ats jsrun_omp --ats jsrun_bind=packed')) - # Ruby + # Dane/Matrix if sys_type in ('toss_4_x86_64_ib'): - cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm56')) + cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm112')) def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): """ From 1a9d325ea90055478a361ee592bb800d4f204dba Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 13:01:53 -0700 Subject: [PATCH 27/55] Add bison, flex, and gcc-runtime to toss_4_x86_64_ib --- .../spack_configs/toss_4_x86_64_ib/spack.yaml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 577d6811..6478ec55 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -175,6 +175,16 @@ spack: fortran: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran extra_rpaths: [] + gcc-runtime: + buildable: False + externals: + - spec: gcc-runtime@13.3.1 + prefix: /usr/tce/packages/gcc/gcc-13.3.1/lib/gcc/x86_64-redhat-linux/13 + - spec: gcc-runtime@12.1.1 + prefix: /usr/tce/packages/gcc/gcc-12.1.1/lib/gcc/x86_64-redhat-linux/12 + - spec: gcc-runtime@11.2.1 + prefix: /usr/tce/packages/gcc/gcc-11.2.1/lib/gcc/x86_64-redhat-linux/11 + # Lock down which MPI we are using mvapich2: buildable: False @@ -285,3 +295,13 @@ spack: externals: - spec: zlib@1.2.11 prefix: /usr + bison: + buildable: false + externals: + - spec: bison@3.0.4 + prefix: /usr + flex: + buildable: false + externals: + - spec: flex@2.6.1 + prefix: /usr From 0d29f73df785f9f2ef1d27a8e59f179404b54e98 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 21:52:23 -0400 Subject: [PATCH 28/55] Add ninja and fix lapack installation --- docker/spack.yaml | 7 ++++++- docker/tpl-ubuntu-gcc-hip.Dockerfile | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/spack.yaml b/docker/spack.yaml index 952629b0..754d2eda 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -319,7 +319,7 @@ spack: netlib-lapack: buildable: false externals: - - spec: netlib-lapack@3.10.0 + - spec: netlib-lapack@3.12.0 prefix: /usr perl: buildable: false @@ -416,3 +416,8 @@ spack: externals: - spec: gmake@4.4.1 prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.11.1 + prefix: /usr diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-gcc-hip.Dockerfile index 6fa10532..9e950982 100644 --- a/docker/tpl-ubuntu-gcc-hip.Dockerfile +++ b/docker/tpl-ubuntu-gcc-hip.Dockerfile @@ -32,6 +32,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libtbb12 \ libtbbmalloc2 \ libblas-dev \ + liblapack-dev \ libz3-dev \ zlib1g-dev \ libmpich-dev \ @@ -92,6 +93,8 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libtbb-dev \ make \ + ninja-build \ + gmake \ bc \ file \ patch \ @@ -106,7 +109,7 @@ RUN apt-get update && \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-18 ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} generator==ninja %amdclang-18 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ From 0ce88c8d04d95d5cf660edd1ebeabc7d6c317f4b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 21:53:34 -0400 Subject: [PATCH 29/55] Disable Sherlock jobs --- .github/workflows/docker_build_tpls.yml | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 2e733d0a..b2a03985 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -67,20 +67,20 @@ jobs: DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 HOST_ARCH: ppc64le RUNS_ON: streak2 - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - TPL_DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "~pygeosx~openmp %gcc@10.1.0" - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU - RUNS_ON: ubuntu-latest - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 - TPL_DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU - RUNS_ON: ubuntu-latest + # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + # TPL_DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + # SPEC: "~pygeosx~openmp %gcc@10.1.0" + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU + # RUNS_ON: ubuntu-latest + # - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 + # TPL_DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + # SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU + # RUNS_ON: ubuntu-latest steps: - name: Checkout From 9020283e026b41ac5d91f35f28af23735eab7706 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 21:55:48 -0400 Subject: [PATCH 30/55] Leave only ROCm job for now --- .github/workflows/docker_build_tpls.yml | 96 ++++++++++++------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 986d1420..ef30ad9c 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -19,54 +19,54 @@ jobs: matrix: include: - - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_ROOT_IMAGE: ubuntu:20.04 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" - RUNS_ON: ubuntu-latest - - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" - RUNS_ON: ubuntu-latest - - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" - RUNS_ON: ubuntu-latest - - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" - RUNS_ON: ubuntu-latest - - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 - TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" - RUNS_ON: ubuntu-latest - - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) - DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 - TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile - RUNS_ON: ubuntu-latest - - name: Rockylinux (8, gcc 13.3, cuda 12.9.1) - DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1 - TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12.Dockerfile - RUNS_ON: Runner_4core_16GB - - name: Rockylinux (8, clang 17.0.6, cuda 12.9.1) - DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 - TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile - RUNS_ON: Runner_4core_16GB - - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) - DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 - TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile - DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 - HOST_ARCH: ppc64le - RUNS_ON: streak2 + # - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) + # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_ROOT_IMAGE: ubuntu:20.04 + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces + # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 + # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 + # TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" + # RUNS_ON: ubuntu-latest + # - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) + # DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 + # TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile + # RUNS_ON: ubuntu-latest + # - name: Rockylinux (8, gcc 13.3, cuda 12.9.1) + # DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1 + # TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12.Dockerfile + # RUNS_ON: Runner_4core_16GB + # - name: Rockylinux (8, clang 17.0.6, cuda 12.9.1) + # DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 + # TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile + # RUNS_ON: Runner_4core_16GB + # - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) + # DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 + # TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile + # DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 + # HOST_ARCH: ppc64le + # RUNS_ON: streak2 # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 # TPL_DOCKERFILE: docker/Stanford/Dockerfile From c4e5c099c16788648e39e39bd1a6abb1591238d2 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 22:06:20 -0400 Subject: [PATCH 31/55] Minor --- docker/tpl-ubuntu-gcc-hip.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-gcc-hip.Dockerfile index 9e950982..4819e178 100644 --- a/docker/tpl-ubuntu-gcc-hip.Dockerfile +++ b/docker/tpl-ubuntu-gcc-hip.Dockerfile @@ -94,7 +94,6 @@ RUN apt-get update && \ libtbb-dev \ make \ ninja-build \ - gmake \ bc \ file \ patch \ From 09ff5f3387d8d340411841609b51fc5a789f6929 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 7 Oct 2025 23:06:45 -0400 Subject: [PATCH 32/55] Remove Ninja --- docker/tpl-ubuntu-gcc-hip.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-gcc-hip.Dockerfile index 4819e178..d067e7ee 100644 --- a/docker/tpl-ubuntu-gcc-hip.Dockerfile +++ b/docker/tpl-ubuntu-gcc-hip.Dockerfile @@ -108,7 +108,7 @@ RUN apt-get update && \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} generator==ninja %amdclang-18 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-18 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ From b51be36ef8a9b890835f1202cbc308c6d11207cd Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 8 Oct 2025 00:17:51 -0400 Subject: [PATCH 33/55] Experiment with metis/parmetis in shared mode --- scripts/spack_packages/packages/geosx/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 6ab1cf25..3b94daae 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -145,8 +145,8 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # # Math # - depends_on("parmetis@4.0.3+int64~shared cflags='-fPIC' cxxflags='-fPIC'") - depends_on("metis +int64~shared cflags='-fPIC' cxxflags='-fPIC'") + depends_on("parmetis@4.0.3+int64+shared cflags='-fPIC' cxxflags='-fPIC'") + depends_on("metis +int64+shared cflags='-fPIC' cxxflags='-fPIC'") depends_on("superlu-dist +int64 fflags='-fPIC'") depends_on("superlu-dist~openmp", when="~openmp") From a562183d78a080c54393f8a538e1a8596aa92be6 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 8 Oct 2025 01:55:24 -0400 Subject: [PATCH 34/55] Switch to OpenMPI --- docker/tpl-ubuntu-gcc-hip.Dockerfile | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-gcc-hip.Dockerfile index d067e7ee..8ff47772 100644 --- a/docker/tpl-ubuntu-gcc-hip.Dockerfile +++ b/docker/tpl-ubuntu-gcc-hip.Dockerfile @@ -35,8 +35,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ liblapack-dev \ libz3-dev \ zlib1g-dev \ - libmpich-dev \ - mpich \ + openmpi-bin \ + libopenmpi-dev \ python3 \ python3-dev \ python3-pip \ @@ -74,15 +74,14 @@ RUN python3 -m pip install clingo --break-system-packages # Install CMake RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh -# Hack for spack to see Ubuntu's mpich in a standard prefix layout -RUN mkdir -p /opt/mpich-system/bin /opt/mpich-system/include /opt/mpich-system/lib && \ - ln -s /usr/bin/mpicc /opt/mpich-system/bin/mpicc && \ - ln -s /usr/bin/mpicxx /opt/mpich-system/bin/mpicxx && \ - ln -s /usr/bin/mpif90 /opt/mpich-system/bin/mpif90 && \ - ln -s /usr/bin/mpifort /opt/mpich-system/bin/mpifort && \ - ln -s /usr/bin/mpirun /opt/mpich-system/bin/mpirun && \ - ln -s /usr/lib/x86_64-linux-gnu/mpich/include/* /opt/mpich-system/include/ && \ - ln -s /usr/lib/x86_64-linux-gnu/mpich/lib/* /opt/mpich-system/lib/ +ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ + CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ + MPICC=/usr/bin/mpicc \ + MPICXX=/usr/bin/mpicxx \ + MPIEXEC=/usr/bin/mpirun + +ENV OMPI_CC=$CC \ + OMPI_CXX=$CXX # Installing TPLs FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain From c9085088bddf60762c5de22fff445efd1c9bfd44 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 8 Oct 2025 02:44:47 -0400 Subject: [PATCH 35/55] Fix openmpi location --- docker/spack.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/spack.yaml b/docker/spack.yaml index 754d2eda..a6d5834e 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -182,20 +182,20 @@ spack: openmpi: externals: - spec: openmpi@4.1.2 %llvm - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi modules: [mpi] - spec: openmpi@4.1.2 %llvm-amdgpu@6.4.3 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi - spec: openmpi@4.1.2 %gcc@9.5.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi - spec: openmpi@4.1.2 %gcc@10.5.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi - spec: openmpi@4.1.2 %gcc@11.4.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi - spec: openmpi@4.1.2 %gcc@12.3.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi - spec: openmpi@4.1.2 %gcc@13.3.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi # CUDA packages cuda: From cef36c334c353ea43d6703c09bfc4f0edb95902e Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 8 Oct 2025 11:47:32 -0700 Subject: [PATCH 36/55] Add bison, flex, and ninja to toss_4_x86_64_ib_cray --- .../toss_4_x86_64_ib_cray/spack.yaml | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 7c4cc2e6..c23812ad 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -13,7 +13,7 @@ # geosx@develop%cce@20.0.0 # # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+rocm~pygeosx+hypre~trilinos~petsc~docs amdgpu_target=gfx942 %cce_20 ^hip@6.4.2 ^hipblas@6.4.2 ^hipsparse@6.4.2 ^hsa-rocr-dev@6.4.2 ^rocprim@6.4.2 ^hdf5 cflags=-Wno-int-conversion" +# python3 ./scripts/uberenv/uberenv.py --spec="+rocm~pygeosx+hypre~trilinos~petsc~docs amdgpu_target=gfx942 %cce-20 ^hip@6.4.2 ^hipblas@6.4.2 ^hipsparse@6.4.2 ^hsa-rocr-dev@6.4.2 ^rocprim@6.4.2 ^hdf5 cflags=-Wno-int-conversion" # @@ -46,7 +46,7 @@ spack: when: '%fortran' - spec: '%cray-mpich@9.0.1_llvm_amdgpu' when: '%mpi' - cce_20: + cce-20: - spec: '%c=cce@20.0.0' when: '%c' - spec: '%cxx=cce@20.0.0' @@ -77,10 +77,6 @@ spack: # flags: {} # environment: {} # extra_rpaths: [] - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex extra_rpaths: - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib - /opt/cray/pe/mpich/9.0.1/gtl/lib @@ -116,10 +112,6 @@ spack: # environment: {} # extra_rpaths: [] # flags: {} - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex extra_rpaths: - /opt/rocm-6.4.2/lib - /opt/rocm-6.4.2/llvm/lib @@ -445,4 +437,19 @@ spack: buildable: false externals: - spec: zlib@1.2.11 + prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.10.2 + prefix: /usr/tce/packages/ninja/ninja-1.10.2 + flex: + buildable: false + externals: + - spec: flex@2.6.1 + prefix: /usr + bison: + buildable: false + externals: + - spec: bison@3.0.4 prefix: /usr \ No newline at end of file From d7c4bb7a5578c0a6b2fcc7a53e9035e372ec3d2f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 8 Oct 2025 11:49:06 -0700 Subject: [PATCH 37/55] Add --clean option to setupLC-TPL scripts --- scripts/setupLC-TPL-uberenv-helper.bash | 23 +++++++++++++++++++---- scripts/setupLC-TPL-uberenv.bash | 6 +++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index de90dae8..bf91fdce 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -3,9 +3,10 @@ ## Builds the TPLs for a specific system and host config. ## Usage ./setupLC-TPL-uberenv-helper.bash [ExtraArgs...] -# --- 1. Initialize Control Variable --- -# By default, we will set permissions. +# --- 1. Initialize Control Variables --- +# By default, we will set permissions and reuse a previous build if available SET_PERMISSIONS=true +CLEAN=false # --- Argument Parsing --- INSTALL_DIR=$1 @@ -22,8 +23,12 @@ shift 5 for arg in "$@"; do if [[ "$arg" == "--no-permissions" ]]; then SET_PERMISSIONS=false - echo "Found --no-permissions flag. Will skip permission updates." - break + echo "Found --no-permissions flag in uberenv-helper. Will skip permission updates." + shift + elif [[ "$arg" == "--clean" ]]; then + CLEAN=true + echo "Found --clean flag in uberenv-helper. Will clean build first" + shift fi done @@ -31,6 +36,15 @@ done CONFIG=$MACHINE-$COMPILER LOG_FILE=$CONFIG.log +# --- Clean build --- +if [ "$CLEAN" = true ]; then + DEST=${INSTALL_DIR}/${CONFIG}_tpls + echo "Removing ${DEST}" && rm -rf "${DEST}" + + DEST=${LOG_FILE} + echo "Removing ${DEST}" && rm -rf "${DEST}" +fi + echo "Building the TPLs on $MACHINE for $COMPILER to be installed at $INSTALL_DIR. Progress will be written to $LOG_FILE." # Note: The ssh command forwards the extra arguments ($@) to uberenv.py @@ -39,6 +53,7 @@ ssh "$MACHINE" -t " cd \"$PWD\" && $GET_A_NODE ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" --spack-env-name \"${CONFIG}_env\" \"$@\" && exit" > "$LOG_FILE" 2>&1 +#echo "Getting a node: $GET_A_NODE" ## Check the last ten lines of the log file. ## A successful install should show up on one of the final lines. diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index 479eebc2..33ab4e93 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -8,6 +8,7 @@ ## (e.g., "dane,matrix"). Defaults to all. ## ExtraArgs: (Optional) Additional arguments forwarded to the helper script. ## Use --no-permissions to skip all chmod/chgrp calls. +## Use --clean to clean data from previous build. # --- Configuration --- # All known machines. Add new machine names here. @@ -42,9 +43,8 @@ declare -a FORWARDED_ARGS=() for arg in "$@"; do if [[ "$arg" == "--no-permissions" ]]; then SET_PERMISSIONS=false - else - FORWARDED_ARGS+=("$arg") fi + FORWARDED_ARGS+=("$arg") done # --- Setup --- @@ -104,7 +104,7 @@ function launch_jobs() { tuolumne) ALLOC_CMD="salloc -N 1 --exclusive -t 120 -A vortex" - "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs %cce-20 amdgpu_target=gfx942" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs %cce-20 amdgpu_target=gfx942 ^vtk generator=ninja" "${ALLOC_CMD}" "$@" & ;; *) From bc7cde866b734e6e8a08d05a94940a6b59947c81 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Wed, 8 Oct 2025 15:34:12 -0700 Subject: [PATCH 38/55] Add workarounds to get %llvm-amdgpu, simplify spec for rocm to fix magma issue --- scripts/setupLC-TPL-uberenv.bash | 3 +- .../spack_configs/toss_4_x86_64_ib/spack.yaml | 84 +++++++++---------- .../toss_4_x86_64_ib_cray/spack.yaml | 26 +++--- .../spack_packages/packages/chai/package.py | 9 ++ .../packages/llvm-amdgpu/package.py | 21 +++++ 5 files changed, 88 insertions(+), 55 deletions(-) create mode 100644 scripts/spack_packages/packages/chai/package.py create mode 100644 scripts/spack_packages/packages/llvm-amdgpu/package.py diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index 33ab4e93..a5f1c885 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -104,7 +104,8 @@ function launch_jobs() { tuolumne) ALLOC_CMD="salloc -N 1 --exclusive -t 120 -A vortex" - "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs %cce-20 amdgpu_target=gfx942 ^vtk generator=ninja" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %cce-20 ^vtk generator=ninja" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne llvm-amdgpu-6.4.2-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %llvm-amdgpu_6_4_2 ^vtk generator=ninja" "${ALLOC_CMD}" "$@" & ;; *) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 6478ec55..c80c5089 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -63,52 +63,52 @@ spack: - spec: '%mvapich2@2.3.7.clang_19' when: '%mpi' clang-14: - - spec: '%[virtuals=c]llvm@14.0.6+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@14.0.6+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@12.1.1' - when: '%fortran' - - spec: '%mvapich2@2.3.7.clang_14' - when: '%mpi' + - spec: '%c=llvm@14.0.6' + when: '%c' + - spec: '%cxx=llvm@14.0.6' + when: '%cxx' + - spec: '%fortran=gcc@10.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.clang_14' + when: '%mpi' gcc-12: - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%mvapich2@2.3.7.gcc' - when: '%mpi' + - spec: '%c=gcc@12.1.1' + when: '%c' + - spec: '%cxx=gcc@12.1.1' + when: '%cxx' + - spec: '%fortran=gcc@12.1.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' gcc-12-noAVX: - - spec: cxxflags='-march=x86-64-v2 -mno-avx512f' - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%mvapich2@2.3.7.gcc' - when: '%mpi' + - spec: cxxflags='-march=x86-64-v2 -mno-avx512f' + - spec: '%c=gcc@12.1.1' + when: '%c' + - spec: '%cxx=gcc@12.1.1' + when: '%cxx' + - spec: '%fortran=gcc@12.1.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' gcc-13: - - spec: '%c=gcc@13.3.1' - when: '%c' - - spec: '%cxx=gcc@13.3.1' - when: '%cxx' - - spec: '%fortran=gcc@13.3.1' - when: '%fortran' - - spec: '%mvapich2@2.3.7.gcc' - when: '%mpi' + - spec: '%c=gcc@13.3.1' + when: '%c' + - spec: '%cxx=gcc@13.3.1' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' gcc-13-noAVX: - - spec: cxxflags='-march=x86-64-v2 -mno-avx512f' - - spec: '%c=gcc@13.3.1' - when: '%c' - - spec: '%cxx=gcc@13.3.1' - when: '%cxx' - - spec: '%fortran=gcc@13.3.1' - when: '%fortran' - - spec: '%mvapich2@2.3.7.gcc' - when: '%mpi' + - spec: cxxflags='-march=x86-64-v2 -mno-avx512f' + - spec: '%c=gcc@13.3.1' + when: '%c' + - spec: '%cxx=gcc@13.3.1' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' packages: all: diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index c23812ad..01de68cd 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -11,10 +11,12 @@ #------------------------------------------------------------------------------------------------------------ # geosx@develop%cce@20.0.0 +# geosx@develop%llvm-amdgpu_6_4_2 # # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+rocm~pygeosx+hypre~trilinos~petsc~docs amdgpu_target=gfx942 %cce-20 ^hip@6.4.2 ^hipblas@6.4.2 ^hipsparse@6.4.2 ^hsa-rocr-dev@6.4.2 ^rocprim@6.4.2 ^hdf5 cflags=-Wno-int-conversion" +# python3 ./scripts/uberenv/uberenv.py --spec="+rocm~pygeosx+hypre~trilinos~petsc~docs amdgpu_target=gfx942 %cce-20 ^vtk generator=ninja" # +# python3 ./scripts/uberenv/uberenv.py --spec="+rocm~pygeosx+hypre~trilinos~petsc~docs amdgpu_target=gfx942 %llvm-amdgpu_6_4_2 ^vtk generator=ninja" spack: @@ -74,16 +76,14 @@ spack: c: /usr/tce/packages/cce-tce/cce-20.0.0/bin/craycc cxx: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayCC fortran: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayftn - # flags: {} - # environment: {} - # extra_rpaths: [] + flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex extra_rpaths: - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib - /opt/cray/pe/mpich/9.0.1/gtl/lib - flags: - cflags: -fPIC - cxxflags: -fPIC - fflags: -fPIC modules: - PrgEnv-cray/8.6.0 - craype-x86-genoa @@ -109,9 +109,11 @@ spack: c: /opt/rocm-6.4.2/llvm/bin/amdclang cxx: /opt/rocm-6.4.2/llvm/bin/amdclang++ fortran: /opt/rocm-6.4.2/llvm/bin/amdflang - # environment: {} - # extra_rpaths: [] - # flags: {} + flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex extra_rpaths: - /opt/rocm-6.4.2/lib - /opt/rocm-6.4.2/llvm/lib @@ -452,4 +454,4 @@ spack: buildable: false externals: - spec: bison@3.0.4 - prefix: /usr \ No newline at end of file + prefix: /usr diff --git a/scripts/spack_packages/packages/chai/package.py b/scripts/spack_packages/packages/chai/package.py new file mode 100644 index 00000000..16b68a56 --- /dev/null +++ b/scripts/spack_packages/packages/chai/package.py @@ -0,0 +1,9 @@ +import os + +from spack.package import * +from spack_repo.builtin.packages.chai.package import Chai as BuiltinChai + +class Chai(BuiltinChai): + # Bypass llnl_link_helpers failure + depends_on("fortran") + diff --git a/scripts/spack_packages/packages/llvm-amdgpu/package.py b/scripts/spack_packages/packages/llvm-amdgpu/package.py new file mode 100644 index 00000000..93dd170d --- /dev/null +++ b/scripts/spack_packages/packages/llvm-amdgpu/package.py @@ -0,0 +1,21 @@ +# Copyright 2013-2025 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +from spack.package import * +from spack_repo.builtin.packages.llvm_amdgpu.package import LlvmAmdgpu as BuiltinLlvmAmdgpu + +class LlvmAmdgpu(BuiltinLlvmAmdgpu): + + # PR that adds this change is pending: https://github.com/spack/spack-packages/pull/1557 + provides("fortran") + + # Fix from slack: + # https://spackpm.slack.com/archives/C08Q62S7XEX/p1751072888930439?thread_ts=1750704656.170759&cid=C08Q62S7XEX + compiler_wrapper_link_paths = { + "c": "rocmcc/amdclang", + "cxx": "rocmcc/amdclang++", + "fortran": "rocmcc/amdflang" + } From 7fc4acef5d2bf0a7740595b503ce2cfad3152300 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Wed, 8 Oct 2025 16:44:56 -0700 Subject: [PATCH 39/55] Use newer external blas - let spack build flex (scotch compilation failure otherwise) --- scripts/spack_configs/toss_4_x86_64_ib/spack.yaml | 15 ++++++++------- .../toss_4_x86_64_ib_cray/spack.yaml | 15 ++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index c80c5089..36f2328f 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -298,10 +298,11 @@ spack: bison: buildable: false externals: - - spec: bison@3.0.4 - prefix: /usr - flex: - buildable: false - externals: - - spec: flex@2.6.1 - prefix: /usr + - spec: bison@3.8.2 + prefix: /collab/usr/gapps/python/build/spack-toss4.1/opt/spack/linux-rhel8-ivybridge/gcc-10.3.1/bison-3.8.2-ihsbaayrrhbmxufyobmimzq3lhxxdbck/ + # Avoid using external flex - issues compiling scotch + # flex: + # buildable: false + # externals: + # - spec: flex@2.6.1 + # prefix: /usr diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 01de68cd..4bc6d09e 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -445,13 +445,14 @@ spack: externals: - spec: ninja@1.10.2 prefix: /usr/tce/packages/ninja/ninja-1.10.2 - flex: - buildable: false - externals: - - spec: flex@2.6.1 - prefix: /usr + # Avoid using external flex - issues compiling scotch + # flex: + # buildable: false + # externals: + # - spec: flex@2.6.1 + # prefix: /usr bison: buildable: false externals: - - spec: bison@3.0.4 - prefix: /usr + - spec: bison@3.8.2 + prefix: /collab/usr/gapps/python/build/spack-toss4.1/opt/spack/linux-rhel8-ivybridge/gcc-10.3.1/bison-3.8.2-ihsbaayrrhbmxufyobmimzq3lhxxdbck/ From 0579eb64076cb96c6fb26ad2b596b411303e7b49 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 8 Oct 2025 19:33:23 -0700 Subject: [PATCH 40/55] Add gcc-runtime to toss_4_x86_64_ib_cray --- scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 4bc6d09e..59f648a6 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -135,7 +135,15 @@ spack: - perftools-base/25.09.0 - xpmem/2.6.5 - + gcc-runtime: + buildable: False + externals: + - spec: gcc-runtime@13.3.1 + prefix: /usr/tce/packages/gcc/gcc-13.3.1/lib/gcc/x86_64-redhat-linux/13 + - spec: gcc-runtime@12.1.1 + prefix: /usr/tce/packages/gcc/gcc-12.1.1/lib/gcc/x86_64-redhat-linux/12 + - spec: gcc-runtime@11.2.1 + prefix: /usr/tce/packages/gcc/gcc-11.2.1/lib/gcc/x86_64-redhat-linux/11 # Lock down which MPI we are using cray-mpich: From c749320b47e985dc3ac3ea0388857a1454d0351b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 8 Oct 2025 19:33:43 -0700 Subject: [PATCH 41/55] Add ninja to toss_4_x86_64_ib --- scripts/spack_configs/toss_4_x86_64_ib/spack.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 36f2328f..9810870e 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -295,6 +295,11 @@ spack: externals: - spec: zlib@1.2.11 prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.11.1 + prefix: /usr/tce/packages/ninja/ninja-1.11.1 bison: buildable: false externals: From 856789960c2e386738395d77d2b09cc849d73780 Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 9 Oct 2025 08:59:30 -0700 Subject: [PATCH 42/55] vtk recipe correction - depends_on c and cxx --- scripts/spack_packages/packages/vtk/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/spack_packages/packages/vtk/package.py b/scripts/spack_packages/packages/vtk/package.py index aa0ee9d8..c75d6cfe 100644 --- a/scripts/spack_packages/packages/vtk/package.py +++ b/scripts/spack_packages/packages/vtk/package.py @@ -52,7 +52,10 @@ class Vtk(CMakePackage): # We need mpi4py if buidling python wrappers and using MPI depends_on('py-mpi4py', when='+python+mpi', type='run') + depends_on("c", type="build") + depends_on("cxx", type="build") depends_on('mpi', when='+mpi') + patch_dir = os.path.join(os.path.dirname(__file__), '9.4.2-patch') if os.path.isdir(patch_dir): From d2cc951efa5beee22e2697c4a3a62ef74d286e4b Mon Sep 17 00:00:00 2001 From: Brian Han Date: Thu, 9 Oct 2025 09:14:21 -0700 Subject: [PATCH 43/55] Remove noAVX builds --- scripts/setupLC-TPL-uberenv.bash | 1 - .../spack_configs/toss_4_x86_64_ib/spack.yaml | 24 ------------------- .../spack_packages/packages/geosx/package.py | 3 --- 3 files changed, 28 deletions(-) diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index a5f1c885..debd7baf 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -88,7 +88,6 @@ function launch_jobs() { dane) ALLOC_CMD="salloc -N 1 -n 112 --exclusive -t 120 -A vortex" "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %gcc-12" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12noAVX "+docs %gcc-12-noAVX" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %gcc-13" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane clang-14 "+docs %clang-14" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane clang-19 "+docs %clang-19" "${ALLOC_CMD}" "$@" & diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 9810870e..fc593828 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -28,10 +28,6 @@ # # python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %clang-19 ^cuda@12.9.1+allow-unsupported-compilers" # -# -# No AVX instructions: -# python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-12-noAVX" -# spack: config: @@ -72,16 +68,6 @@ spack: - spec: '%mvapich2@2.3.7.clang_14' when: '%mpi' gcc-12: - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%mvapich2@2.3.7.gcc' - when: '%mpi' - gcc-12-noAVX: - - spec: cxxflags='-march=x86-64-v2 -mno-avx512f' - spec: '%c=gcc@12.1.1' when: '%c' - spec: '%cxx=gcc@12.1.1' @@ -99,16 +85,6 @@ spack: when: '%fortran' - spec: '%mvapich2@2.3.7.gcc' when: '%mpi' - gcc-13-noAVX: - - spec: cxxflags='-march=x86-64-v2 -mno-avx512f' - - spec: '%c=gcc@13.3.1' - when: '%c' - - spec: '%cxx=gcc@13.3.1' - when: '%cxx' - - spec: '%fortran=gcc@13.3.1' - when: '%fortran' - - spec: '%mvapich2@2.3.7.gcc' - when: '%mpi' packages: all: diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index d6ba8d69..4e1b9bd2 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -254,9 +254,6 @@ def _get_sys_type(self, spec): def _get_host_config_path(self, spec, lvarray=False): var = '' - if 'no-avx' in str(spec.compiler_flags): - var += "-noAVX" - if '+cuda' in spec: var = '-'.join([var, 'cuda']) var += "@" + str(spec['cuda'].version) From bf0cc94d727d12dac86cefbb56444ba43bbbb09c Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 9 Oct 2025 11:39:17 -0700 Subject: [PATCH 44/55] Minor tweaks --- scripts/setupLC-TPL-uberenv-helper.bash | 8 +++---- scripts/setupLC-TPL-uberenv.bash | 32 +++++++++++++------------ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index bf91fdce..63f2e96b 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -7,6 +7,7 @@ # By default, we will set permissions and reuse a previous build if available SET_PERMISSIONS=true CLEAN=false +: ${USER:=$(whoami)} # --- Argument Parsing --- INSTALL_DIR=$1 @@ -48,12 +49,11 @@ fi echo "Building the TPLs on $MACHINE for $COMPILER to be installed at $INSTALL_DIR. Progress will be written to $LOG_FILE." # Note: The ssh command forwards the extra arguments ($@) to uberenv.py -ssh "$MACHINE" -t " +ssh "${USER}@${MACHINE}.llnl.gov" -t " . /etc/profile && cd \"$PWD\" && -$GET_A_NODE ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" --spack-env-name \"${CONFIG}_env\" \"$@\" && -exit" > "$LOG_FILE" 2>&1 -#echo "Getting a node: $GET_A_NODE" +$GET_A_NODE date && ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" --spack-env-name \"${CONFIG}_env\" \"$@\" && +date && exit" > "$LOG_FILE" 2>&1 ## Check the last ten lines of the log file. ## A successful install should show up on one of the final lines. diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index debd7baf..1c735db7 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -80,31 +80,33 @@ function kill_children() { function launch_jobs() { local machine=$1 shift # The rest of $@ are the forwarded arguments - local UBERENV_HELPER=./scripts/setupLC-TPL-uberenv-helper.bash + local UBERENV_HELPER="./scripts/setupLC-TPL-uberenv-helper.bash" + local COMMON="^vtk generator=ninja" echo "-----> Launching jobs for [${machine}]..." + # Note: The max. time allowed on the debug queue is 1h. If we need more, switch to pbatch case "$machine" in dane) - ALLOC_CMD="salloc -N 1 -n 112 --exclusive -t 120 -A vortex" - "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %gcc-12" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %gcc-13" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" dane clang-14 "+docs %clang-14" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" dane clang-19 "+docs %clang-19" "${ALLOC_CMD}" "$@" & + ALLOC_CMD="salloc -N 1 --exclusive -t 60 -A vortex -ppdebug" + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %gcc-12 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %gcc-13 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane clang-14 "+docs %clang-14 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane clang-19 "+docs %clang-19 ${COMMON}" "${ALLOC_CMD}" "$@" & ;; matrix) - ALLOC_CMD="salloc -N 1 --exclusive -t 120 -A vortex" - "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %gcc-12 ^cuda@12.6.0+allow-unsupported-compilers" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" matrix clang-14-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %clang-14 ^cuda@12.6.0+allow-unsupported-compilers" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" matrix clang-19-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %clang-19 ^cuda@12.9.1+allow-unsupported-compilers" "${ALLOC_CMD}" "$@" & + ALLOC_CMD="salloc -N 1 --exclusive -t 60 -A vortex -ppdebug" + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %gcc-12 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix clang-14-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %clang-14 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix clang-19-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %clang-19 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & ;; - tuolumne) - ALLOC_CMD="salloc -N 1 --exclusive -t 120 -A vortex" - "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %cce-20 ^vtk generator=ninja" "${ALLOC_CMD}" "$@" & - "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne llvm-amdgpu-6.4.2-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %llvm-amdgpu_6_4_2 ^vtk generator=ninja" "${ALLOC_CMD}" "$@" & + tuo|tuolumne) + ALLOC_CMD="salloc -N 1 --exclusive -t 60 -A vortex -ppdebug" + "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %cce-20 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne llvm-amdgpu-6.4.2-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %llvm-amdgpu_6_4_2 ${COMMON}" "${ALLOC_CMD}" "$@" & ;; *) From a0ce8589880c777c195efff14f6561e3a8f0f4dc Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 9 Oct 2025 15:59:26 -0400 Subject: [PATCH 45/55] Simplify packages --- scripts/spack_configs/versions.yaml | 14 +- .../spack_packages/packages/camp/package.py | 151 +---- .../spack_packages/packages/chai/package.py | 2 + .../spack_packages/packages/raja/package.py | 423 +------------ .../spack_packages/packages/umpire/package.py | 569 +----------------- 5 files changed, 22 insertions(+), 1137 deletions(-) diff --git a/scripts/spack_configs/versions.yaml b/scripts/spack_configs/versions.yaml index c68f2161..e798f2be 100644 --- a/scripts/spack_configs/versions.yaml +++ b/scripts/spack_configs/versions.yaml @@ -22,21 +22,21 @@ packages: hypre: require: "@git.907a2d07b64fe47bdde4540c54665c83ced83a2c=master" - # v2025.0.9.1 + # v2025.09.1 chai: require: "@git.51e66b8a5a6caac41aa329c8643180b8e2b33e5f=develop" - # v2025.0.3.0 + # v2025.09.0 umpire: - require: "@git.1ed0669c57f041baa1f1070693991c3a7a43e7ee=develop" + require: "@git.6b0ea9edbbbc741c8a429768d946549cd3bd7d33=develop" - # v2025.0.3.0 + # v2025.09.0 raja: - require: "@git.1d70abf171474d331f1409908bdf1b1c3fe19222=develop" + require: "@git.ca756788dbdd43fec2a3840389126ae94a905d5f=develop" - # v2025.0.3.0 + # v2025.09.2 camp: - require: "@git.ee0a3069a7ae72da8bcea63c06260fad34901d43=main" + require: "@git.4070ce93a802849d61037310a87c50cc24c9e498=main" # v2.12.0 caliper: diff --git a/scripts/spack_packages/packages/camp/package.py b/scripts/spack_packages/packages/camp/package.py index b9fc471c..4fd65fd4 100644 --- a/scripts/spack_packages/packages/camp/package.py +++ b/scripts/spack_packages/packages/camp/package.py @@ -2,157 +2,14 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# GEOSX_EDIT_START -# Victor: This is adapted from the radius-spack-configs GitHub repo. -# We're using this since the new 2025.09 packages suite has not been upstreamed to spack yet. -# See PR: https://github.com/LLNL/radiuss-spack-configs/pull/143 -# -# TODO: Delete this file after the new 2025.09 suite becomes available on spack_packages -# -# GEOSX_EDIT_END -# yapf: disable - -from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_string -from spack_repo.builtin.build_systems.cmake import CMakePackage -from spack_repo.builtin.build_systems.cuda import CudaPackage -from spack_repo.builtin.build_systems.rocm import ROCmPackage +import os from spack.package import * +from spack_repo.builtin.packages.camp.package import Camp as BuiltinCamp - -class Camp(CMakePackage, CudaPackage, ROCmPackage): - """ - Compiler agnostic metaprogramming library providing concepts, - type operations and tuples for C++ and cuda - """ - - homepage = "https://github.com/LLNL/camp" - git = "https://github.com/LLNL/camp.git" - url = "https://github.com/LLNL/camp/archive/v0.1.0.tar.gz" - - maintainers("adrienbernede", "kab163", "trws") - - license("BSD-3-Clause") - - version("main", branch="main", submodules=False) - version( - "2025.09.2", - tag="v2025.09.2", - commit="4070ce93a802849d61037310a87c50cc24c9e498", - submodules=False, - ) - version( - "2025.09.0", - tag="v2025.09.0", - commit="b642f29b9d0eee9113bea2791958c29243063e5c", - submodules=False, - ) - version( - "2025.03.0", - tag="v2025.03.0", - commit="ee0a3069a7ae72da8bcea63c06260fad34901d43", - submodules=False, - ) - version( - "2024.07.0", - tag="v2024.07.0", - commit="0f07de4240c42e0b38a8d872a20440cb4b33d9f5", - submodules=False, - ) - version( - "2024.02.1", - tag="v2024.02.1", - commit="79c320fa09db987923b56884afdc9f82f4b70fc4", - submodules=False, - ) - version( - "2024.02.0", - tag="v2024.02.0", - commit="03c80a6c6ab4f97e76a52639563daec71435a277", - submodules=False, - ) - version( - "2023.06.0", - tag="v2023.06.0", - commit="ac34c25b722a06b138bc045d38bfa5e8fa3ec9c5", - submodules=False, - ) - version("2022.10.1", sha256="2d12f1a46f5a6d01880fc075cfbd332e2cf296816a7c1aa12d4ee5644d386f02") - version("2022.10.0", sha256="3561c3ef00bbcb61fe3183c53d49b110e54910f47e7fc689ad9ccce57e55d6b8") - version("2022.03.2", sha256="bc4aaeacfe8f2912e28f7a36fc731ab9e481bee15f2c6daf0cb208eed3f201eb") - version("2022.03.0", sha256="e9090d5ee191ea3a8e36b47a8fe78f3ac95d51804f1d986d931e85b8f8dad721") - version("0.3.0", sha256="129431a049ca5825443038ad5a37a86ba6d09b2618d5fe65d35f83136575afdb") - version("0.2.3", sha256="58a0f3bd5eadb588d7dc83f3d050aff8c8db639fc89e8d6553f9ce34fc2421a7") - version("0.2.2", sha256="194d38b57e50e3494482a7f94940b27f37a2bee8291f2574d64db342b981d819") - version("0.1.0", sha256="fd4f0f2a60b82a12a1d9f943f8893dc6fe770db493f8fae5ef6f7d0c439bebcc") - - depends_on("c", type="build") +class Camp(BuiltinCamp): depends_on("cxx", type="build") - # TODO: figure out gtest dependency and then set this default True. - variant("tests", default=False, description="Build tests") - variant("openmp", default=False, description="Build with OpenMP support") - variant("omptarget", default=False, description="Build with OpenMP Target support") - variant("sycl", default=False, description="Build with Sycl support") - - with when("+cuda"): - depends_on("cub", when="^cuda@:10") - - depends_on("blt", type="build") + # From radiuss-packages (PR #143) depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") - depends_on("blt@0.6.2:", type="build", when="@2024.02.1:") - depends_on("blt@0.6.1", type="build", when="@2024.02.0") - depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2023.06.0") - - #patch("libstdc++-13-missing-header.patch", when="@:2022.10") - #patch("camp-rocm6.patch", when="@0.2.3 +rocm ^hip@6:") - - conflicts("^blt@:0.3.6", when="+rocm") - - conflicts("+omptarget +rocm") - conflicts("+sycl +omptarget") - conflicts("+sycl +rocm") - conflicts( - "+sycl", - when="@:2024.02.99", - msg="Support for SYCL was introduced in RAJA after 2024.02 release, " - "please use a newer release.", - ) - - def cmake_args(self): - spec = self.spec - - options = [] - - options.append("-DBLT_SOURCE_DIR={0}".format(spec["blt"].prefix)) - - options.append(self.define_from_variant("ENABLE_CUDA", "cuda")) - if spec.satisfies("+cuda"): - options.append("-DCUDA_TOOLKIT_ROOT_DIR={0}".format(spec["cuda"].prefix)) - - if not spec.satisfies("cuda_arch=none"): - cuda_arch = spec.variants["cuda_arch"].value - options.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0])) - options.append("-DCUDA_ARCH=sm_{0}".format(cuda_arch[0])) - flag = "-arch sm_{0}".format(cuda_arch[0]) - options.append("-DCMAKE_CUDA_FLAGS:STRING={0}".format(flag)) - - options.append(self.define_from_variant("ENABLE_HIP", "rocm")) - if spec.satisfies("+rocm"): - options.append("-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix)) - - archs = ";".join(self.spec.variants["amdgpu_target"].value) - options.append("-DCMAKE_HIP_ARCHITECTURES={0}".format(archs)) - options.append("-DGPU_TARGETS={0}".format(archs)) - options.append("-DAMDGPU_TARGETS={0}".format(archs)) - - if spec.satisfies("+omptarget"): - options.append(cmake_cache_string("RAJA_DATA_ALIGN", 64)) - - options.append(self.define_from_variant("ENABLE_TESTS", "tests")) - options.append(self.define_from_variant("ENABLE_OPENMP", "openmp")) - options.append(self.define_from_variant("CAMP_ENABLE_TARGET_OPENMP", "omptarget")) - options.append(self.define_from_variant("ENABLE_SYCL", "sycl")) - - return options diff --git a/scripts/spack_packages/packages/chai/package.py b/scripts/spack_packages/packages/chai/package.py index cabbdc96..bdc3f94b 100644 --- a/scripts/spack_packages/packages/chai/package.py +++ b/scripts/spack_packages/packages/chai/package.py @@ -10,6 +10,8 @@ class Chai(BuiltinChai): # Bypass llnl_link_helpers failure depends_on("fortran") + depends_on("c", type="build") + depends_on("cxx", type="build") # From radiuss-packages (PR #143) depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") diff --git a/scripts/spack_packages/packages/raja/package.py b/scripts/spack_packages/packages/raja/package.py index 71186d84..3ef41bba 100644 --- a/scripts/spack_packages/packages/raja/package.py +++ b/scripts/spack_packages/packages/raja/package.py @@ -1,361 +1,23 @@ -# Copyright 2013-2025 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# GEOSX_EDIT_START -# Victor: This is adapted from the radius-spack-configs GitHub repo. -# We're using this since the new 2025.09 packages suite has not been upstreamed to spack yet. -# See PR: https://github.com/LLNL/radiuss-spack-configs/pull/143 -# -# TODO: Delete this file after the new 2025.09 suite becomes available on spack_packages -# -# GEOSX_EDIT_END -# yapf: disable - -import socket - -from spack_repo.builtin.build_systems.cached_cmake import ( - CachedCMakePackage, - cmake_cache_option, - cmake_cache_path, - cmake_cache_string, -) -from spack_repo.builtin.build_systems.cuda import CudaPackage -from spack_repo.builtin.build_systems.rocm import ROCmPackage -from spack_repo.builtin.packages.blt.package import llnl_link_helpers +import os from spack.package import * +from spack_repo.builtin.packages.raja.package import Raja as BuiltinRaja +class Raja(BuiltinRaja): + depends_on("cxx", type="build") -# Starting with 2022.03.0, the only submodule we want to fetch is tpl/desul -# since there is no package for it. Other RAJA submodules are defined as -# dependencies. -def submodules(package): - submodules = [] - submodules.append("tpl/desul") - return submodules - - -class Raja(CachedCMakePackage, CudaPackage, ROCmPackage): - """RAJA Parallel Framework.""" - - homepage = "https://github.com/LLNL/RAJA" - git = "https://github.com/LLNL/RAJA.git" - tags = ["radiuss", "e4s"] - - maintainers("adrienbernede", "davidbeckingsale", "kab163") - - license("BSD-3-Clause") - - version("develop", branch="develop", submodules=submodules) - version("main", branch="main", submodules=submodules) - version( - "2025.09.0", - tag="v2025.09.0", - commit="ca756788dbdd43fec2a3840389126ae94a905d5f", - submodules=submodules, - ) - version( - "2025.03.2", - tag="v2025.03.2", - commit="6e36a94380adbe88fed11a3213fc08461428ece0", - submodules=submodules, - ) - version( - "2025.03.0", - tag="v2025.03.0", - commit="1d70abf171474d331f1409908bdf1b1c3fe19222", - submodules=submodules, - ) - version( - "2024.07.0", - tag="v2024.07.0", - commit="4d7fcba55ebc7cb972b7cc9f6778b48e43792ea1", - submodules=submodules, - ) - version( - "2024.02.2", - tag="v2024.02.2", - commit="593f756b14ac57ded33ee61d8d2292d4beb840e6", - submodules=submodules, - ) - version( - "2024.02.1", - tag="v2024.02.1", - commit="3ada0950b0774ec907d30a9eceaf6af7478b833b", - submodules=submodules, - ) - version( - "2024.02.0", - tag="v2024.02.0", - commit="82d1b926ada0fbb15a4a6e0adadc30c715cfda7b", - submodules=submodules, - ) - version( - "2023.06.1", - tag="v2023.06.1", - commit="9b5f61edf3aa1e6fdbc9a4b30828c81504639963", - submodules=submodules, - ) - version( - "2023.06.0", - tag="v2023.06.0", - commit="e330b2560747d5417cd7bd265fab3fb91d32ecbd", - submodules=submodules, - ) - version( - "2022.10.5", - tag="v2022.10.5", - commit="3774f51339459bbbdb77055aa23f82919b6335b6", - submodules=submodules, - ) - version( - "2022.10.4", - tag="v2022.10.4", - commit="c2a6b1740759ae3ae7c85b35e20dbffbe235355d", - submodules=submodules, - ) - version( - "2022.03.0", - tag="v2022.03.0", - commit="4351fe6a50bd579511a625b017c9e054885e7fd2", - submodules=submodules, - ) - version( - "0.14.0", - tag="v0.14.0", - commit="357933a42842dd91de5c1034204d937fce0a2a44", - submodules="True", - ) - version( - "0.13.0", - tag="v0.13.0", - commit="3047fa720132d19ee143b1fcdacaa72971f5988c", - submodules="True", - ) - version( - "0.12.1", - tag="v0.12.1", - commit="9cb6370bb2868e35ebba23cdce927f5f7f9da530", - submodules="True", - ) - version( - "0.12.0", - tag="v0.12.0", - commit="32d92e38da41cc8d4db25ec79b9884a73a0cb3a1", - submodules="True", - ) - version( - "0.11.0", - tag="v0.11.0", - commit="0502b9b69c4cb60aa0afbdf699b555c76cb18f22", - submodules="True", - ) - version( - "0.10.1", - tag="v0.10.1", - commit="be91e040130678b1350dbda56cc352433db758bd", - submodules="True", - ) - version( - "0.10.0", - tag="v0.10.0", - commit="53cb89cf788d28bc4ed2b4e6f75483fdd26024aa", - submodules="True", - ) - version( - "0.9.0", tag="v0.9.0", commit="df7ca1fa892b6ac4147c614d2d739d5022f63fc7", submodules="True" - ) - version( - "0.8.0", tag="v0.8.0", commit="8d19a8c2cbac611de6f92ad8852b9f3454b27e63", submodules="True" - ) - version( - "0.7.0", tag="v0.7.0", commit="caa33b371b586dfae3d8569caee91c5eddfd7b31", submodules="True" - ) - version( - "0.6.0", tag="v0.6.0", commit="cc7a97e8b4e52c3de820c9dfacd358822a147871", submodules="True" - ) - version( - "0.5.3", tag="v0.5.3", commit="1ca35c0ed2a43a3fa9c6cd70c5d25f16d88ecd8c", submodules="True" - ) - version( - "0.5.2", tag="v0.5.2", commit="4d5c3d5d7f311838855f7010810610349e729f64", submodules="True" - ) - version( - "0.5.1", tag="v0.5.1", commit="bf340abe5199d7e051520913c9a7a5de336b5820", submodules="True" - ) - version( - "0.5.0", tag="v0.5.0", commit="9b539d84fdad049f65caeba836f41031f5baf4cc", submodules="True" - ) - version( - "0.4.1", tag="v0.4.1", commit="3618cfe95d6a442fa50fbe7bfbcf654cf9f800b9", submodules="True" - ) - version( - "0.4.0", tag="v0.4.0", commit="31b2a48192542c2da426885baa5af0ed57606b78", submodules="True" - ) - - depends_on("cxx", type="build") # generated - - # export targets when building pre-2.4.0 release with BLT 0.4.0+ - patch( - "https://github.com/LLNL/RAJA/commit/eca1124ee4af380d6613adc6012c307d1fd4176b.patch?full_index=1", - sha256="12bb78c00b6683ad3e7fd4e3f87f9776bae074b722431b79696bc862816735ef", - when="@:0.13.0 ^blt@0.4:", - ) - - # Backward compatibility is stopped from ROCm 6.0 - # Future relase will have the change from PR https://github.com/LLNL/RAJA/pull/1568 - patch( - "https://github.com/LLNL/RAJA/commit/406eb8dee05a41eb32c421c375688a4863b60642.patch?full_index=1", - sha256="d9ce5ef038555cbccb330a9016b7be77e56ae0660583cba955dab9d0297a4b07", - when="^hip@6.0", - ) - - # Fix compilation issue reported by Intel from their new compiler version - patch( - "https://github.com/LLNL/RAJA/pull/1668.patch?full_index=1", - sha256="c0548fc5220f24082fb2592d5b4e8b7c8c783b87906d5f0950d53953d25161f6", - when="@2024.02.1:2024.02.99 %oneapi@2025:", - ) - - variant("openmp", default=False, description="Build OpenMP backend") - variant("shared", default=False, description="Build shared libs") - variant("desul", default=False, description="Build desul atomics backend") - variant("vectorization", default=True, description="Build SIMD/SIMT intrinsics support") - variant( - "omptask", default=False, description="Build OpenMP task variants of internal algorithms" - ) - variant("omptarget", default=False, description="Build OpenMP on target device support") - variant("sycl", default=False, description="Build sycl backend") - variant("gpu-profiling", default=False, description="Enable GPU profiling") - - variant("plugins", default=False, description="Enable runtime plugins") - variant("examples", default=True, description="Build examples.") - variant("exercises", default=True, description="Build exercises.") - # TODO: figure out gtest dependency and then set this default True - # and remove the +tests conflict below. - variant("tests", default=False, description="Build tests") - - # we don't use variants to express the failing test, we only add a variant to - # define whether we want to run all the tests (including those known to fail) - # or only the passing ones. - variant( - "run-all-tests", - default=False, - description="Run all the tests, including those known to fail.", - ) - - variant( - "lowopttest", - default=False, - description="For developers, lowers optimization level to pass tests with some compilers", - ) - - depends_on("cxx", type="build") # generated - - depends_on("blt", type="build") + # From radiuss-packages (PR #143) depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") - depends_on("blt@0.6.2:", type="build", when="@2024.02.1:") - depends_on("blt@0.6.1", type="build", when="@2024.02.0") - depends_on("blt@0.5.3", type="build", when="@2023.06.0:2023.06.1") - depends_on("blt@0.5.2:0.5.3", type="build", when="@2022.10.5") - depends_on("blt@0.5.0:0.5.3", type="build", when="@0.14.1:2022.10.4") - depends_on("blt@0.4.1", type="build", when="@0.14.0") - depends_on("blt@0.4.0:0.4.1", type="build", when="@0.13.0") - depends_on("blt@0.3.6:0.4.1", type="build", when="@:0.12.0") - conflicts("^blt@:0.3.6", when="+rocm") - - depends_on("camp") - depends_on("camp+openmp", when="+openmp") - depends_on("camp+omptarget", when="+omptarget") - depends_on("camp+sycl", when="+sycl") depends_on("camp@main", when="@develop") depends_on("camp@2025.09.2:", when="@2025.09.0:") depends_on("camp@2025.03.0:", when="@2025.03.0:") depends_on("camp@2024.07.0:", when="@2024.07.0:") depends_on("camp@2024.02.1:", when="@2024.02.1:") - depends_on("camp@2024.02.0", when="@2024.02.0") - depends_on("camp@2023.06.0", when="@2023.06.0:2023.06.1") - depends_on("camp@2022.10.1:2023.06.0", when="@2022.10.3:2022.10.5") - depends_on("camp@2022.10.0:2023.06.0", when="@2022.10.0:2022.10.2") - depends_on("camp@2022.03.2", when="@2022.03.0:2022.03.1") - depends_on("camp@0.2.2:0.2.3", when="@0.14.0") - depends_on("camp@0.1.0", when="@0.10.0:0.13.0") - - depends_on("cmake@3.23:", when="@2024.07.0:", type="build") - depends_on("cmake@3.23:", when="@2022.10.0:2024.02.2+rocm", type="build") - depends_on("cmake@3.20:", when="@2022.10.0:2024.02.2", type="build") - depends_on("cmake@3.20:", when="@:2022.03+rocm", type="build") - depends_on("cmake@3.14:", when="@:2022.03", type="build") - - depends_on("llvm-openmp", when="+openmp %apple-clang") - - depends_on("rocprim", when="+rocm") - with when("+rocm @0.12.0:"): - depends_on("camp+rocm") - for arch in ROCmPackage.amdgpu_targets: - depends_on( - "camp+rocm amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch) - ) - conflicts("+openmp", when="@:2022.03") - - with when("+cuda @0.12.0:"): - depends_on("camp+cuda") - for sm_ in CudaPackage.cuda_arch_values: - depends_on("camp +cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) - - conflicts("+gpu-profiling", when="~cuda~rocm", msg="GPU profiling requires CUDA or ROCm") - conflicts("+gpu-profiling +cuda", when="@:2022.02.99") - conflicts("+gpu-profiling +rocm", when="@:2022.02.99") - - conflicts("+omptarget +rocm") - conflicts("+sycl +omptarget") - conflicts("+sycl +rocm") - conflicts( - "+sycl", - when="@:2024.02.99", - msg="Support for SYCL was introduced in RAJA after 2024.02 release, " - "please use a newer release.", - ) - - def _get_sys_type(self, spec): - sys_type = spec.architecture - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - @property - def libs(self): - shared = "+shared" in self.spec - return find_libraries("libRAJA", root=self.prefix, shared=shared, recursive=True) - - @property - def cache_name(self): - hostname = socket.gethostname() - if "SYS_TYPE" in env: - hostname = hostname.rstrip("1234567890") - return "{0}-{1}-{2}@{3}-{4}.cmake".format( - hostname, - self._get_sys_type(self.spec), - self.spec.compiler.name, - self.spec.compiler.version, - self.spec.dag_hash(8), - ) - - def initconfig_compiler_entries(self): - spec = self.spec - compiler = self.compiler - # Default entries are already defined in CachedCMakePackage, inherit them: - entries = super().initconfig_compiler_entries() - - if spec.satisfies("+rocm ^blt@:0.6"): - entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) - - llnl_link_helpers(entries, spec, compiler) - - return entries def initconfig_hardware_entries(self): spec = self.spec @@ -510,76 +172,3 @@ def initconfig_package_entries(self): entries.append(cmake_cache_option("RAJA_HOST_CONFIG_LOADED", True)) return entries - - def cmake_args(self): - return [] - - @property - def build_relpath(self): - """Relative path to the cmake build subdirectory.""" - return join_path("..", self.build_dirname) - - @run_after("install") - def setup_build_tests(self): - """Copy the build test files after the package is installed to a - relative install test subdirectory for use during `spack test run`.""" - # Now copy the relative files - cache_extra_test_sources(self, self.build_relpath) - - # Ensure the path exists since relying on a relative path at the - # same level as the normal stage source path. - mkdirp(install_test_root(self)) - - @property - def _extra_tests_path(self): - # TODO: The tests should be converted to re-build and run examples - # TODO: using the installed libraries. - return join_path(install_test_root(self), self.build_relpath, "bin") - - def run_example(self, exe, expected): - """run and check outputs of the example""" - with working_dir(self._extra_tests_path): - example = which(exe) - if example is None: - raise SkipTest(f"{exe} was not built") - - out = example(output=str.split, error=str.split) - check_outputs(expected, out) - - def test_line_of_sight(self): - """check line of sight example""" - self.run_example( - "ex5_line-of-sight_solution", - [r"C-style sequential", r"RAJA sequential", r"result -- PASS"], - ) - - def test_stencil_offset_layout(self): - """check stencil offset layout""" - self.run_example( - "ex6_stencil-offset-layout_solution", [r"RAJA Views \(permuted\)", r"result -- PASS"] - ) - - def test_tiled_matrix(self): - """check tiled matrix transpose""" - self.run_example( - "ex8_tiled-matrix-transpose_solution", - [r"C-version", r"RAJA sequential", r"result -- PASS"], - ) - - def test_dynamic_tile(self): - """check kernel dynamic tile""" - self.run_example("kernel-dynamic-tile", [r"Running index", r"(24,24)"]) - - def test_plugin_example(self): - """check plugin example""" - self.run_example("plugin-example", [r"Launching host kernel for the 10 time"]) - - def test_matrix_multiply(self): - """check batched matrix multiple tutorial""" - self.run_example( - "tut_batched-matrix-multiply", [r"batched matrix multiplication", r"result -- PASS"] - ) - - def test_wave_equation(self): - """check wave equation""" - self.run_example("wave-eqn", [r"Max Error = 2", r"Evolved solution to time"]) diff --git a/scripts/spack_packages/packages/umpire/package.py b/scripts/spack_packages/packages/umpire/package.py index 2c0756ad..4037e59d 100644 --- a/scripts/spack_packages/packages/umpire/package.py +++ b/scripts/spack_packages/packages/umpire/package.py @@ -2,582 +2,19 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# GEOSX_EDIT_START -# Victor: This is adapted from the radius-spack-configs GitHub repo. -# We're using this since the new 2025.09 packages suite has not been upstreamed to spack yet. -# See PR: https://github.com/LLNL/radiuss-spack-configs/pull/143 -# -# TODO: Delete this file after the new 2025.09 suite becomes available on spack_packages -# -# GEOSX_EDIT_END -# yapf: disable - import os -import socket - -from spack_repo.builtin.build_systems.cached_cmake import ( - CachedCMakePackage, - cmake_cache_option, - cmake_cache_path, - cmake_cache_string, -) -from spack_repo.builtin.build_systems.cuda import CudaPackage -from spack_repo.builtin.build_systems.rocm import ROCmPackage -from spack_repo.builtin.packages.blt.package import llnl_link_helpers from spack.package import * +from spack_repo.builtin.packages.umpire.package import Umpire as BuiltinUmpire - -class Umpire(CachedCMakePackage, CudaPackage, ROCmPackage): - """An application-focused API for memory management on NUMA & GPU - architectures""" - - homepage = "https://github.com/LLNL/Umpire" - git = "https://github.com/LLNL/Umpire.git" - tags = ["radiuss", "e4s"] - - maintainers("adrienbernede", "davidbeckingsale", "kab163") - - license("MIT") - - version("develop", branch="develop", submodules=False) - version( - "2025.09.0", - tag="v2025.09.0", - commit="6b0ea9edbbbc741c8a429768d946549cd3bd7d33", - submodules=False, - ) - version( - "2025.03.0", - tag="v2025.03.0", - commit="1ed0669c57f041baa1f1070693991c3a7a43e7ee", - submodules=False, - ) - version( - "2024.07.0", - tag="v2024.07.0", - commit="abd729f40064175e999a83d11d6b073dac4c01d2", - submodules=False, - ) - version( - "2024.02.1", - tag="v2024.02.1", - commit="3058d562fc707650e904f9321b1ee9bcebad3ae2", - submodules=False, - ) - version( - "2024.02.0", - tag="v2024.02.0", - commit="1db3fef913a70d8882ca510a4830c77c388873e0", - submodules=False, - ) - version( - "2023.06.0", - tag="v2023.06.0", - commit="1e5ef604de88e81bb3b6fc4a5d914be833529da5", - submodules=False, - ) - version( - "2022.10.0", - tag="v2022.10.0", - commit="93b1441aaa258c1dcd211a552b75cff6461a2a8a", - submodules=False, - ) - version( - "2022.03.1", - tag="v2022.03.1", - commit="6bf231bdbbc797df70d60027ddb714ac2ef7c0a1", - submodules=False, - ) - version( - "2022.03.0", - tag="v2022.03.0", - commit="2db6224ae0c3f3e0bbd6722e95c1167b7f79be7b", - submodules=False, - ) - version( - "6.0.0", tag="v6.0.0", commit="5f886b4299496b7cb6f9d62dc1372ce6d3832fbc", submodules=True - ) - version( - "5.0.1", tag="v5.0.1", commit="5201a47a35e3844160dcbecd0916f8c96aa7dd07", submodules=True - ) - version( - "5.0.0", tag="v5.0.0", commit="2196615500057e068f2d93597b4f8da89d582afb", submodules=True - ) - version( - "4.1.2", tag="v4.1.2", commit="447f4640eff7b8f39d3c59404f3b03629b90c021", submodules=True - ) - version( - "4.1.1", tag="v4.1.1", commit="df1830b5ea04185f93fc229ed667da62d1d2d6e3", submodules=True - ) - version( - "4.1.0", tag="v4.1.0", commit="62f146d9c6b291cd79b29386dcb84b30f7b4212e", submodules=True - ) - version( - "4.0.1", tag="v4.0.1", commit="06d8692d084a88b15b0ef2794a1da779197da747", submodules=True - ) - version( - "4.0.0", tag="v4.0.0", commit="bdd598512516bdc4238502f180c8a7e145c6e68f", submodules=True - ) - version( - "3.0.0", tag="v3.0.0", commit="657676087574f61f9d90b996a3bdbf4e1cdfc92e", submodules=True - ) - version( - "2.1.0", tag="v2.1.0", commit="52e10c05cd40dfdfde186c1e63213695f5aeaf65", submodules=True - ) - version( - "2.0.0", tag="v2.0.0", commit="0dc8b4736357645b99632ee7c17a3dc6af771fbb", submodules=True - ) - version( - "1.1.0", tag="v1.1.0", commit="3db26e6a2626ee8c0cfa5c9769cfac6e33587122", submodules=True - ) - version( - "1.0.1", tag="v1.0.1", commit="a6741073431cab3a7a2434f9119a54d18e9978f4", submodules=True - ) - version( - "1.0.0", tag="v1.0.0", commit="82482fd7450ab378db110f06f7e0302112c22c05", submodules=True - ) - version( - "0.3.5", tag="v0.3.5", commit="a283977bb548cbaa0221bdb6c9832f7834f69e74", submodules=True - ) - version( - "0.3.4", tag="v0.3.4", commit="20a77408d8ae467af21d5802d14afe54f1253694", submodules=True - ) - version( - "0.3.3", tag="v0.3.3", commit="715a8bd003eb1d9db1e2ac7ff2c6251cfd445c27", submodules=True - ) - version( - "0.3.2", tag="v0.3.2", commit="06f37f2011fa4d9482f15e04fc206e2e7b7aa9e2", submodules=True - ) - version( - "0.3.1", tag="v0.3.1", commit="aef223065fdfe85d1e46bab95e3874821702891a", submodules=True - ) - version( - "0.3.0", tag="v0.3.0", commit="529004f9e88fbb49ee93a97465ff904be249039c", submodules=True - ) - version( - "0.2.4", tag="v0.2.4", commit="f774afae69b6f2e5c99ea8bf5660ccf68bd5436d", submodules=True - ) - version( - "0.2.3", tag="v0.2.3", commit="af158291f574701aabb6a2b16e6536aefaf4496e", submodules=True - ) - version( - "0.2.2", tag="v0.2.2", commit="68f4b86fd877c9ca00c9438c603e5dbc40d5f219", submodules=True - ) - version( - "0.2.1", tag="v0.2.1", commit="c22df368e2f52398351f49fbe2522bd1150ad171", submodules=True - ) - version( - "0.2.0", tag="v0.2.0", commit="7910b8d4dbfe83faacf65e864304ca916e34b86c", submodules=True - ) - version( - "0.1.4", tag="v0.1.4", commit="c2848289ba9d8c85346610d25af9531b82c50fc3", submodules=True - ) - version( - "0.1.3", tag="v0.1.3", commit="cc347edeb17f5f30f694aa47f395d17369a2e449", submodules=True - ) - +class Umpire(BuiltinUmpire): depends_on("c", type="build") # generated depends_on("cxx", type="build") # generated depends_on("fortran", type="build") # generated - # Some projects importing both camp and umpire targets end up with conflicts in BLT targets - # import. This is not addressing the root cause, which will be addressed in BLT@5.4.0 and will - # require adapting umpire build system. - #patch("dual_blt_import_umpire_2022.10_2023.06.patch", when="@2022.10.0:2023.06.0") - #patch("export_includes.patch", when="@2022.10.0") - #patch("std-filesystem-pr784.patch", when="@2022.03.1 +rocm ^blt@0.5.2:") - #patch("camp_target_umpire_3.0.0.patch", when="@3.0.0") - #patch("cmake_version_check.patch", when="@4.1") - #patch("missing_header_for_numeric_limits.patch", when="@4.1:5.0.1") - - # export targets when building pre-6.0.0 release with BLT 0.4.0+ - patch( - "https://github.com/LLNL/Umpire/commit/5773ce9af88952c8d23f9bcdcb2e503ceda40763.patch?full_index=1", - sha256="f3b21335ce5cf9c0fecc852a94dfec90fb5703032ac97f9fee104af9408d8899", - when="@:5.0.1 ^blt@0.4:", - ) - - # https://github.com/LLNL/Umpire/pull/805 - patch( - "https://github.com/LLNL/Umpire/commit/47ff0aa1f7a01a917c3b7ac618e8a9e44a10fd25.patch?full_index=1", - sha256="802f074a05e1cb1f428e13d99c5fcb1435f86bd8f36a1ea2f7b6756e6625e0a0", - when="@2022.10.0", - ) - - # https://github.com/LLNL/Umpire/pull/816 - patch( - "https://github.com/LLNL/Umpire/commit/2292d1d6078f6d9523b7ad0886ffa053644569d5.patch?full_index=1", - sha256="170dbcadb9ae36c7e211119c17a812695f11f4fe1be290b750f7af4fb4896192", - when="@2022.10.0", - ) - - # https://github.com/LLNL/Umpire/pull/853 - patch( - "https://github.com/LLNL/Umpire/commit/4bd9b2ded81d3216b3f62e2aad62d0e34fe2c256.patch?full_index=1", - sha256="c9ddae1f4212cef72e1050b6ac482ce5b795dad4977d2462cff2e884b8d7aff5", - when="@2022.10:2023.06", - ) - - variant("fortran", default=False, description="Build C/Fortran API") - variant("c", default=True, description="Build C API") - variant("mpi", default=False, description="Enable MPI support") - variant("ipc_shmem", default=False, description="Enable POSIX shared memory") - variant("mpi3_shmem", default=False, description="Enable MPI3 shared memory") - variant( - "sqlite_experimental", - default=False, - description="Enable sqlite integration with umpire events (Experimental)", - ) - variant("numa", default=False, description="Enable NUMA support") - variant("shared", default=True, description="Enable Shared libs") - variant("openmp", default=False, description="Build with OpenMP support") - variant("omptarget", default=False, description="Build with OpenMP 4.5 support") - variant("deviceconst", default=False, description="Enables support for constant device memory") - variant("examples", default=False, description="Build Umpire Examples") - variant( - "tests", - default="none", - values=("none", "basic", "benchmarks"), - multi=False, - description="Tests to run", - ) - variant("tools", default=False, description="Enable tools") - variant("backtrace", default=False, description="Enable backtrace tools") - variant("dev_benchmarks", default=False, description="Enable developer benchmarks") - variant("device_alloc", default=False, description="Enable DeviceAllocator") - variant("werror", default=True, description="Enable warnings as errors") - variant("asan", default=False, description="Enable ASAN") - variant("sanitizer_tests", default=False, description="Enable address sanitizer tests") - variant("fmt_header_only", default=True, description="Link to header-only fmt target") - - depends_on("cmake@3.23:", when="@2024.07.0:", type="build") - depends_on("cmake@3.23:", when="@2022.10.0: +rocm", type="build") - depends_on("cmake@3.20:", when="@2022.10.0:2024.02.1", type="build") - depends_on("cmake@:3.20", when="@2022.03.0:2022.03 +rocm", type="build") - depends_on("cmake@3.14:", when="@2022.03.0:", type="build") - depends_on("cmake@3.9:", when="+cuda", type="build") - depends_on("cmake@3.8:", type="build") - - depends_on("blt", type="build") + # From radiuss-packages (PR #143) depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") - depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") - depends_on("blt@0.6.2:", type="build", when="@2024.02.1:") - depends_on("blt@0.6.1", type="build", when="@2024.02.0") - depends_on("blt@0.5.3", type="build", when="@2023.06.0") - depends_on("blt@0.5.2:0.5.3", type="build", when="@2022.10.0") - depends_on("blt@0.5.0:0.5.3", type="build", when="@2022.03.0:2022.03.1") - depends_on("blt@0.4.1", type="build", when="@6.0.0") - depends_on("blt@0.4.0:0.4.1", type="build", when="@4.1.3:5.0.1") - depends_on("blt@0.3.6:0.4.1", type="build", when="@:4.1.2") - conflicts("^blt@:0.3.6", when="+rocm") - - depends_on("camp") depends_on("camp@2025.09.2:", when="@2025.09.0:") depends_on("camp@2025.03.0:", when="@2025.03.0:") - depends_on("camp+openmp", when="+openmp") - depends_on("camp~cuda", when="~cuda") - depends_on("camp~rocm", when="~rocm") - depends_on("camp@main", when="@develop") - depends_on("camp@2024.07.0:", when="@2024.07.0:") - depends_on("camp@2024.02.1", when="@2024.02.1") - depends_on("camp@2024.02.0", when="@2024.02.0") - depends_on("camp@2023.06.0", when="@2023.06.0") - depends_on("camp@2022.10.0:2023.06.0", when="@2022.10.0") - depends_on("camp@2022.03.2:2023.06.0", when="@2022.03.0:2022.03.1") - depends_on("camp@0.2.2:0.2.3", when="@6.0.0") - depends_on("camp@0.1.0", when="@5.0.0:5.0.1") - - depends_on("sqlite", when="+sqlite_experimental") - depends_on("mpi", when="+mpi") - depends_on("fmt@9.1:11.0", when="@2024.02.0:") - # For some reason, we need c++ 17 explicitly only with intel - depends_on("fmt@9.1:11.0 cxxstd=17", when="@2024.02.0: %intel@19.1") - - with when("@5.0.0:"): - with when("+cuda"): - depends_on("camp+cuda") - for sm_ in CudaPackage.cuda_arch_values: - depends_on("camp+cuda cuda_arch={0}".format(sm_), when="cuda_arch={0}".format(sm_)) - - with when("+rocm"): - depends_on("camp+rocm") - for arch_ in ROCmPackage.amdgpu_targets: - depends_on( - "camp+rocm amdgpu_target={0}".format(arch_), - when="amdgpu_target={0}".format(arch_), - ) - - conflicts("+numa", when="@:0.3.2") - conflicts("~c", when="+fortran", msg="Fortran API requires C API") - - # device allocator must be used with more current umpire versions, rocm 5.4.0 and greater, - # and with either rocm or cuda enabled - conflicts("+device_alloc", when="@:2022.03.0") - conflicts("+device_alloc", when="^hip@:5.3.99") - conflicts("+device_alloc", when="~rocm~cuda") - - conflicts("+deviceconst", when="~rocm~cuda") - conflicts("~openmp", when="+omptarget", msg="OpenMP target requires OpenMP") - conflicts("+cuda", when="+rocm") - conflicts("+tools", when="+rocm") - conflicts( - "+rocm", when="+omptarget", msg="Cant support both rocm and openmp device backends at once" - ) - conflicts("+ipc_shmem", when="@:5.0.1") - conflicts("+mpi3_shmem", when="@:2024.07.0") - conflicts("+mpi3_shmem", when="~mpi") conflicts("+ipc_shmem", when="+mpi3_shmem @:2025.03.0") - - conflicts("+sqlite_experimental", when="@:6.0.0") - conflicts("+sanitizer_tests", when="~asan") - - # device allocator exports device code, which requires static libs - # currently only available for cuda. - conflicts("+shared", when="+cuda") - - def _get_sys_type(self, spec): - sys_type = spec.architecture - if "SYS_TYPE" in env: - sys_type = env["SYS_TYPE"] - return sys_type - - @property - def cache_name(self): - hostname = socket.gethostname() - if "SYS_TYPE" in env: - hostname = hostname.rstrip("1234567890") - return "{0}-{1}-{2}@{3}-{4}.cmake".format( - hostname, - self._get_sys_type(self.spec), - self.spec.compiler.name, - self.spec.compiler.version, - self.spec.dag_hash(8), - ) - - def initconfig_compiler_entries(self): - spec = self.spec - compiler = self.compiler - # Default entries are already defined in CachedCMakePackage, inherit them: - entries = super().initconfig_compiler_entries() - - if spec.satisfies("+rocm ^blt@:0.6"): - entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) - - option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" - - if spec.satisfies("+fortran") and compiler.fc is not None: - entries.append(cmake_cache_option("ENABLE_FORTRAN", True)) - else: - entries.append(cmake_cache_option("ENABLE_FORTRAN", False)) - - entries.append( - cmake_cache_option("{}ENABLE_C".format(option_prefix), spec.satisfies("+c")) - ) - - llnl_link_helpers(entries, spec, compiler) - - return entries - - def initconfig_hardware_entries(self): - spec = self.spec - entries = super().initconfig_hardware_entries() - - entries.append("#------------------{0}".format("-" * 30)) - entries.append("# Package custom hardware settings") - entries.append("#------------------{0}\n".format("-" * 30)) - - option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" - - if spec.satisfies("+cuda"): - entries.append(cmake_cache_option("ENABLE_CUDA", True)) - # Umpire used to pick only the first architecture in the list. The shared logic in - # CachedCMakePackage keeps the list of architectures. - else: - entries.append(cmake_cache_option("ENABLE_CUDA", False)) - - if spec.satisfies("+rocm"): - entries.append(cmake_cache_option("ENABLE_HIP", True)) - else: - entries.append(cmake_cache_option("ENABLE_HIP", False)) - - entries.append( - cmake_cache_option( - "{}ENABLE_DEVICE_CONST".format(option_prefix), spec.satisfies("+deviceconst") - ) - ) - - entries.append( - cmake_cache_option( - "{}ENABLE_OPENMP_TARGET".format(option_prefix), spec.satisfies("+omptarget") - ) - ) - - if spec.satisfies("+omptarget") and spec.satisfies("%xl"): - entries.append(cmake_cache_string("OpenMP_CXX_FLAGS", "-qsmp;-qoffload")) - - return entries - - def initconfig_mpi_entries(self): - spec = self.spec - - entries = super().initconfig_mpi_entries() - entries.append(cmake_cache_option("ENABLE_MPI", spec.satisfies("+mpi"))) - entries.append( - cmake_cache_option("UMPIRE_ENABLE_MPI3_SHARED_MEMORY", spec.satisfies("+mpi3_shmem")) - ) - - return entries - - def initconfig_package_entries(self): - spec = self.spec - entries = [] - - option_prefix = "UMPIRE_" if spec.satisfies("@2022.03.0:") else "" - - # TPL locations - entries.append("#------------------{0}".format("-" * 60)) - entries.append("# TPLs") - entries.append("#------------------{0}\n".format("-" * 60)) - - entries.append(cmake_cache_path("BLT_SOURCE_DIR", spec["blt"].prefix)) - if spec.satisfies("@5.0.0:"): - entries.append(cmake_cache_path("camp_DIR", spec["camp"].prefix)) - - if spec.satisfies("@2024.02.0:"): - entries.append(cmake_cache_path("fmt_DIR", spec["fmt"].prefix)) - - # Build options - entries.append("#------------------{0}".format("-" * 60)) - entries.append("# Build Options") - entries.append("#------------------{0}\n".format("-" * 60)) - - entries.append(cmake_cache_string("CMAKE_BUILD_TYPE", spec.variants["build_type"].value)) - entries.append(cmake_cache_option("BUILD_SHARED_LIBS", spec.satisfies("+shared"))) - entries.append(cmake_cache_option("ENABLE_WARNINGS_AS_ERRORS", spec.satisfies("+werror"))) - - # Generic options that have a prefixed equivalent in Umpire CMake - entries.append(cmake_cache_option("ENABLE_OPENMP", spec.satisfies("+openmp"))) - entries.append(cmake_cache_option("ENABLE_EXAMPLES", spec.satisfies("+examples"))) - entries.append(cmake_cache_option("ENABLE_DOCS", False)) - if spec.satisfies("tests=benchmarks") or spec.satisfies("+dev_benchmarks"): - # BLT requires ENABLE_TESTS=True to enable benchmarks - entries.append(cmake_cache_option("ENABLE_BENCHMARKS", True)) - entries.append(cmake_cache_option("ENABLE_TESTS", True)) - else: - entries.append(cmake_cache_option("ENABLE_BENCHMARKS", False)) - entries.append(cmake_cache_option("ENABLE_TESTS", not spec.satisfies("tests=none"))) - - # Prefixed options that used to be name without one - entries.append( - cmake_cache_option("{}ENABLE_NUMA".format(option_prefix), spec.satisfies("+numa")) - ) - entries.append( - cmake_cache_option( - "{}ENABLE_DEVELOPER_BENCHMARKS".format(option_prefix), - spec.satisfies("+dev_benchmarks"), - ) - ) - entries.append( - cmake_cache_option("{}ENABLE_TOOLS".format(option_prefix), spec.satisfies("+tools")) - ) - entries.append( - cmake_cache_option( - "{}ENABLE_BACKTRACE".format(option_prefix), spec.satisfies("+backtrace") - ) - ) - entries.append( - cmake_cache_option("{}ENABLE_ASAN".format(option_prefix), spec.satisfies("+asan")) - ) - entries.append( - cmake_cache_option( - "{}ENABLE_SANITIZER_TESTS".format(option_prefix), - spec.satisfies("+sanitizer_tests"), - ) - ) - - # Recent options, were never name without prefix - entries.append( - cmake_cache_option("UMPIRE_ENABLE_DEVICE_ALLOCATOR", spec.satisfies("+device_alloc")) - ) - entries.append( - cmake_cache_option( - "UMPIRE_ENABLE_SQLITE_EXPERIMENTAL", spec.satisfies("+sqlite_experimental") - ) - ) - if spec.satisfies("+sqlite_experimental"): - entries.append(cmake_cache_path("SQLite3_ROOT", spec["sqlite"].prefix)) - - # This option was renamed later than the others - if spec.satisfies("@2022.10.0:"): - entries.append( - cmake_cache_option("UMPIRE_ENABLE_IPC_SHARED_MEMORY", spec.satisfies("+ipc_shmem")) - ) - else: - entries.append( - cmake_cache_option("ENABLE_IPC_SHARED_MEMORY", spec.satisfies("+ipc_shmem")) - ) - - if spec.satisfies("~fmt_header_only"): - entries.append(cmake_cache_string("UMPIRE_FMT_TARGET", "fmt::fmt")) - - return entries - - def cmake_args(self): - return [] - - def setup_run_environment(self, env): - for library in ["lib", "lib64"]: - lib_path = join_path(self.prefix, library) - if os.path.exists(lib_path): - env.append_path("LD_LIBRARY_PATH", lib_path) - - def run_example(self, exe, expected): - """Perform stand-alone checks on the installed package.""" - - exe_run = which(join_path(self.prefix.bin, exe)) - if exe_run is None: - raise SkipTest(f"{exe} is not installed for version {self.version}") - out = exe_run(output=str.split, error=str.split) - check_outputs(expected, out) - - def test_malloc(self): - """Run Malloc""" - self.run_example("malloc", ["99 should be 99"]) - - def test_recipe_dynamic_pool_heuristic(self): - """Multiple use allocator test""" - self.run_example("recipe_dynamic_pool_heuristic", ["in the pool", "releas"]) - - def test_recipe_no_introspection(self): - """Test without introspection""" - self.run_example("recipe_no_introspection", ["has allocated", "used"]) - - def test_strategy_example(self): - """Memory allocation strategy test""" - self.run_example("strategy_example", ["Available allocators", "HOST"]) - - def test_tut_copy(self): - """Copy data test""" - self.run_example("tut_copy", ["Copied source data"]) - - def test_tut_introspection(self): - """Keep track of pointer allocation test""" - self.run_example("tut_introspection", ["Allocator used is HOST", "size of the allocation"]) - - def test_tut_memset(self): - """Set entire block of memory to one value test""" - self.run_example("tut_memset", ["Set data from HOST"]) - - def test_tut_move(self): - """Move memory test""" - self.run_example("tut_move", ["Moved source data", "HOST"]) - - def test_tut_reallocate(self): - """Reallocate memory test""" - self.run_example("tut_reallocate", ["Reallocated data"]) - - def test_vector_allocator(self): - """Allocate vector memory test""" - self.run_example("vector_allocator", [""]) From 8ef9df9101dec087c6f5f73113096e076256c59c Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 9 Oct 2025 14:22:06 -0700 Subject: [PATCH 46/55] Bug fix for running on compute node --- scripts/setupLC-TPL-uberenv-helper.bash | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index 63f2e96b..7fda59ab 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -48,12 +48,11 @@ fi echo "Building the TPLs on $MACHINE for $COMPILER to be installed at $INSTALL_DIR. Progress will be written to $LOG_FILE." -# Note: The ssh command forwards the extra arguments ($@) to uberenv.py -ssh "${USER}@${MACHINE}.llnl.gov" -t " -. /etc/profile && -cd \"$PWD\" && -$GET_A_NODE date && ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" --spack-env-name \"${CONFIG}_env\" \"$@\" && -date && exit" > "$LOG_FILE" 2>&1 +# Define the command to be run on the compute node +REMOTE_CMD="date && ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix \\\"${INSTALL_DIR}/${CONFIG}_tpls\\\" --spack-env-name \\\"${CONFIG}_env\\\" \\\"$@\\\" && date" + +# Execute it via ssh +ssh "${USER}@${MACHINE}.llnl.gov" -t ". /etc/profile && cd \"$PWD\" && $GET_A_NODE bash -c \"$REMOTE_CMD\" && exit" > "$LOG_FILE" 2>&1 ## Check the last ten lines of the log file. ## A successful install should show up on one of the final lines. From 636cb64b48ea52ac6c83c3c00289a1a6e455664c Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 9 Oct 2025 17:25:48 -0400 Subject: [PATCH 47/55] OpenMPI fixes --- docker/spack.yaml | 16 ++++++++-------- docker/tpl-ubuntu-gcc-hip.Dockerfile | 18 ++++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/docker/spack.yaml b/docker/spack.yaml index a6d5834e..40da6744 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -6,8 +6,7 @@ spack: all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' misc_cache: $spack/../misc_cache test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage + build_stage: $spack/../build_stage # Regular TPLs do not need views view: false @@ -18,7 +17,7 @@ spack: - ../versions.yaml toolchains: - amdclang-18: + amdclang-19: - spec: '%[virtuals=c]llvm-amdgpu@6.4.3' when: '%c' - spec: '%[virtuals=cxx]llvm-amdgpu@6.4.3' @@ -381,11 +380,12 @@ spack: externals: - spec: bison@3.8.2 prefix: /usr - flex: - buildable: false - externals: - - spec: flex@2.6.4 - prefix: /usr + # Rely on spack to build a newer flex for scotch + # flex: + # buildable: false + # externals: + # - spec: flex@2.6.4 + # prefix: /usr bzip2: buildable: false externals: diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-gcc-hip.Dockerfile index 8ff47772..d9aa5bb2 100644 --- a/docker/tpl-ubuntu-gcc-hip.Dockerfile +++ b/docker/tpl-ubuntu-gcc-hip.Dockerfile @@ -13,7 +13,6 @@ ARG INSTALL_DIR ENV GEOSX_TPL_DIR=$INSTALL_DIR # Parameters -ARG GCC_MAJOR_VERSION=13 ARG AMDGPU_TARGET=gfx942 ARG ROCM_VERSION=6.4.3 @@ -74,14 +73,17 @@ RUN python3 -m pip install clingo --break-system-packages # Install CMake RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh -ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ - CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables +ENV CC=/usr/bin/amdclang \ + CXX=/usr/bin/amdclang++ \ MPICC=/usr/bin/mpicc \ MPICXX=/usr/bin/mpicxx \ - MPIEXEC=/usr/bin/mpirun - -ENV OMPI_CC=$CC \ - OMPI_CXX=$CXX + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/amdclang \ + OMPI_CXX=/usr/bin/amdclang++ # Installing TPLs FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain @@ -107,7 +109,7 @@ RUN apt-get update && \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-18 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ From 6eed25e8c7c4996dc790dd0a8aac6e33a1227328 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 9 Oct 2025 17:27:34 -0400 Subject: [PATCH 48/55] Fix image name --- .github/workflows/docker_build_tpls.yml | 7 +++---- ...ubuntu-gcc-hip.Dockerfile => tpl-ubuntu-hip.Dockerfile} | 0 2 files changed, 3 insertions(+), 4 deletions(-) rename docker/{tpl-ubuntu-gcc-hip.Dockerfile => tpl-ubuntu-hip.Dockerfile} (100%) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index ef30ad9c..35926786 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -81,11 +81,10 @@ jobs: # SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU # RUNS_ON: ubuntu-latest - - name: Ubuntu (24.04, gcc 13.3.0 + ROCm 6.4.3) - DOCKER_REPOSITORY: geosx/ubuntu24.04-gcc13.3.0-rocm6.4.3 - TPL_DOCKERFILE: docker/tpl-ubuntu-gcc-hip.Dockerfile + - name: Ubuntu (24.04, amdclang 19 + ROCm 6.4.3) + DOCKER_REPOSITORY: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3 + TPL_DOCKERFILE: docker/tpl-ubuntu-hip.Dockerfile DOCKER_ROOT_IMAGE: rocm/dev-ubuntu-24.04:6.4.3 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=13" RUNS_ON: ubuntu-latest steps: diff --git a/docker/tpl-ubuntu-gcc-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile similarity index 100% rename from docker/tpl-ubuntu-gcc-hip.Dockerfile rename to docker/tpl-ubuntu-hip.Dockerfile From 25553a205407809200ab813fcc9ba5b8097bc93e Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 9 Oct 2025 21:19:33 -0400 Subject: [PATCH 49/55] Add autotools --- docker/tpl-ubuntu-hip.Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index d9aa5bb2..ee104b67 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -94,6 +94,9 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libtbb-dev \ make \ + autopoint \ + autotools-dev \ + automake \ ninja-build \ bc \ file \ From 8704f302c9088c9ecf97aa83ecd740f052e59d4f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 9 Oct 2025 21:31:10 -0700 Subject: [PATCH 50/55] Bug fix --- scripts/setupLC-TPL-uberenv-helper.bash | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index 7fda59ab..6ee12ca2 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -48,11 +48,20 @@ fi echo "Building the TPLs on $MACHINE for $COMPILER to be installed at $INSTALL_DIR. Progress will be written to $LOG_FILE." -# Define the command to be run on the compute node -REMOTE_CMD="date && ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix \\\"${INSTALL_DIR}/${CONFIG}_tpls\\\" --spack-env-name \\\"${CONFIG}_env\\\" \\\"$@\\\" && date" - -# Execute it via ssh -ssh "${USER}@${MACHINE}.llnl.gov" -t ". /etc/profile && cd \"$PWD\" && $GET_A_NODE bash -c \"$REMOTE_CMD\" && exit" > "$LOG_FILE" 2>&1 +# Execute the command to be run on the compute node via ssh +ssh -t "${USER}@${MACHINE}.llnl.gov" " + source /etc/profile && + cd '${PWD}' && + ${GET_A_NODE} bash -c ' + echo \"Start time: \$(date)\" + ./scripts/uberenv/uberenv.py \ + --spec ${SPEC} \ + --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" \ + --spack-env-name \"${CONFIG}_env\" \ + \"\$@\" + echo \"End time: \$(date)\" + ' +" > "${LOG_FILE}" 2>&1 ## Check the last ten lines of the log file. ## A successful install should show up on one of the final lines. From 0086a752c29fb60e01f4089edb8f7404d5bcbebf Mon Sep 17 00:00:00 2001 From: Brian Han Date: Fri, 10 Oct 2025 07:48:48 -0700 Subject: [PATCH 51/55] Small fix - Use toolchain for Ubuntu clang images --- docker/tpl-ubuntu-clang.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/tpl-ubuntu-clang.Dockerfile b/docker/tpl-ubuntu-clang.Dockerfile index 0ef16273..4e0058ad 100644 --- a/docker/tpl-ubuntu-clang.Dockerfile +++ b/docker/tpl-ubuntu-clang.Dockerfile @@ -88,7 +88,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ # Create symlinks to g++ libraries ln -s /usr/bin/g++-${GCC_MAJOR_VERSION} /usr/bin/g++ && \ ./scripts/uberenv/uberenv.py \ - --spec "~shared~openmp+docs %clang@${CLANG_MAJOR_VERSION} ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "~shared~openmp+docs %clang-${CLANG_MAJOR_VERSION} ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ From f7a265863e6d7efe335f3d573a59f29a118a8bd3 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 10 Oct 2025 11:33:28 -0400 Subject: [PATCH 52/55] Re-enable jobs --- .github/workflows/docker_build_tpls.yml | 96 ++++++++++++------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 35926786..402cff84 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -19,54 +19,54 @@ jobs: matrix: include: - # - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) - # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 - # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - # DOCKER_ROOT_IMAGE: ubuntu:20.04 - # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces - # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 - # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - # DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 - # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) - # DOCKER_ROOT_IMAGE: ubuntu:22.04 - # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) - # DOCKER_ROOT_IMAGE: ubuntu:22.04 - # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 - # TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) - # DOCKER_ROOT_IMAGE: ubuntu:22.04 - # DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 - # TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile - # DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" - # RUNS_ON: ubuntu-latest - # - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) - # DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 - # TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile - # RUNS_ON: ubuntu-latest - # - name: Rockylinux (8, gcc 13.3, cuda 12.9.1) - # DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1 - # TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12.Dockerfile - # RUNS_ON: Runner_4core_16GB - # - name: Rockylinux (8, clang 17.0.6, cuda 12.9.1) - # DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 - # TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile - # RUNS_ON: Runner_4core_16GB - # - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) - # DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 - # TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile - # DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 - # HOST_ARCH: ppc64le - # RUNS_ON: streak2 + - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + DOCKER_ROOT_IMAGE: ubuntu:20.04 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" + RUNS_ON: ubuntu-latest + - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces + DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" + RUNS_ON: ubuntu-latest + - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) + DOCKER_ROOT_IMAGE: ubuntu:22.04 + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" + RUNS_ON: ubuntu-latest + - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) + DOCKER_ROOT_IMAGE: ubuntu:22.04 + DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 + TPL_DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" + RUNS_ON: ubuntu-latest + - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) + DOCKER_ROOT_IMAGE: ubuntu:22.04 + DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 + TPL_DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile + DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" + RUNS_ON: ubuntu-latest + - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) + DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 + TPL_DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile + RUNS_ON: ubuntu-latest + - name: Rockylinux (8, gcc 13.3, cuda 12.9.1) + DOCKER_REPOSITORY: geosx/rockylinux8-gcc13-cuda12.9.1 + TPL_DOCKERFILE: docker/tpl-rockylinux-gcc-cuda-12.Dockerfile + RUNS_ON: Runner_4core_16GB + - name: Rockylinux (8, clang 17.0.6, cuda 12.9.1) + DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.9.1 + TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile + RUNS_ON: Runner_4core_16GB + - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) + DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 + TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile + DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 + HOST_ARCH: ppc64le + RUNS_ON: streak2 # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 # TPL_DOCKERFILE: docker/Stanford/Dockerfile From 3ae8f6fd211e41e3e6b0dd5c055e5f1d9874bec9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 13 Oct 2025 01:25:38 -0400 Subject: [PATCH 53/55] CI fixes --- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 2 ++ docker/tpl-rockylinux-gcc-cuda-12.Dockerfile | 2 ++ docker/tpl-ubuntu-clang-cuda.Dockerfile | 13 +++++++++++++ docker/tpl-ubuntu-clang.Dockerfile | 18 ++++++++++-------- docker/tpl-ubuntu-gcc.Dockerfile | 13 ++++++++----- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index 2c0ca66f..60eb5e87 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -32,6 +32,8 @@ RUN dnf clean all && \ bison \ gettext \ gnupg \ + help2man \ + libtool \ xz \ python3-virtualenv diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index cfa9ca80..dd0582dd 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -30,6 +30,8 @@ RUN dnf clean all && \ bison \ gettext \ gnupg \ + help2man \ + libtool \ xz \ python3-virtualenv diff --git a/docker/tpl-ubuntu-clang-cuda.Dockerfile b/docker/tpl-ubuntu-clang-cuda.Dockerfile index 44e45fa9..dd669c90 100644 --- a/docker/tpl-ubuntu-clang-cuda.Dockerfile +++ b/docker/tpl-ubuntu-clang-cuda.Dockerfile @@ -38,6 +38,7 @@ RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \ bison \ gettext \ help2man \ + libtool \ gnupg \ virtualenv @@ -60,6 +61,18 @@ RUN apt-get install -y --no-install-recommends \ ca-certificates \ git +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables +ENV CC=/usr/bin/clang \ + CXX=/usr/bin/clang++ \ + MPICC=/usr/bin/mpicc \ + MPICXX=/usr/bin/mpicxx \ + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/clang \ + OMPI_CXX=/usr/bin/clang++ + # Run uberenv # Have to create install directory first for uberenv # -k flag is to ignore SSL errors diff --git a/docker/tpl-ubuntu-clang.Dockerfile b/docker/tpl-ubuntu-clang.Dockerfile index 77269fc5..04ae8da2 100644 --- a/docker/tpl-ubuntu-clang.Dockerfile +++ b/docker/tpl-ubuntu-clang.Dockerfile @@ -43,6 +43,7 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ bison \ gettext \ help2man \ + libtool \ gnupg \ virtualenv @@ -73,16 +74,17 @@ RUN apt-get install -y --no-install-recommends \ ca-certificates \ git -# Add MPI environment path info -ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ - CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables +ENV CC=/usr/bin/clang-$CLANG_MAJOR_VERSION \ + CXX=/usr/bin/clang++-$CLANG_MAJOR_VERSION \ MPICC=/usr/bin/mpicc \ MPICXX=/usr/bin/mpicxx \ - MPIEXEC=/usr/bin/mpirun -# The multi-line definition of arguments does not seem happy -# when a variable uses the value of another variable previously defined on the same line. -ENV OMPI_CC=$CC \ - OMPI_CXX=$CXX + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/clang-$CLANG_MAJOR_VERSION \ + OMPI_CXX=/usr/bin/clang++-$CLANG_MAJOR_VERSION # Run uberenv # Have to create install directory first for uberenv diff --git a/docker/tpl-ubuntu-gcc.Dockerfile b/docker/tpl-ubuntu-gcc.Dockerfile index e7f0f28a..a9843dc2 100644 --- a/docker/tpl-ubuntu-gcc.Dockerfile +++ b/docker/tpl-ubuntu-gcc.Dockerfile @@ -68,6 +68,7 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ bison \ flex \ help2man \ + libtool \ gnupg \ virtualenv @@ -77,15 +78,17 @@ RUN python3 -m pip install --upgrade pip && \ RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ MPICC=/usr/bin/mpicc \ MPICXX=/usr/bin/mpicxx \ - MPIEXEC=/usr/bin/mpirun -# The multi-line definition of arguments does not seem happy -# when a variable uses the value of another variable previously defined on the same line. -ENV OMPI_CC=$CC \ - OMPI_CXX=$CXX + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ + OMPI_CXX=/usr/bin/g++-$GCC_MAJOR_VERSION # This stage is dedicated to TPLs uniquely. # A multi-stage build patern will allow to extract what we need for the GEOSX build. From 88906c1c596a8be6b979660b35142f3cfd1e2256 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 13 Oct 2025 11:29:08 -0400 Subject: [PATCH 54/55] CI fixes --- docker/tpl-centos-gcc-cuda.Dockerfile | 2 ++ docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 2 +- docker/tpl-rockylinux-gcc-cuda-12.Dockerfile | 2 +- docker/tpl-ubuntu-clang-cuda.Dockerfile | 3 +++ docker/tpl-ubuntu-clang.Dockerfile | 3 +++ docker/tpl-ubuntu-gcc.Dockerfile | 3 +++ 6 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docker/tpl-centos-gcc-cuda.Dockerfile b/docker/tpl-centos-gcc-cuda.Dockerfile index dee69adf..dcfbbe0e 100644 --- a/docker/tpl-centos-gcc-cuda.Dockerfile +++ b/docker/tpl-centos-gcc-cuda.Dockerfile @@ -69,6 +69,8 @@ ARG BLD_DIR RUN yum install -y \ tbb-devel \ make \ + autopoint \ + autotools-dev \ bc \ file \ patch \ diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index 60eb5e87..205a7f58 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -32,7 +32,6 @@ RUN dnf clean all && \ bison \ gettext \ gnupg \ - help2man \ libtool \ xz \ python3-virtualenv @@ -61,6 +60,7 @@ RUN dnf clean all && \ ca-certificates \ autoconf \ automake \ + autopoint \ m4 \ git diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index dd0582dd..6ef22cd8 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -30,7 +30,6 @@ RUN dnf clean all && \ bison \ gettext \ gnupg \ - help2man \ libtool \ xz \ python3-virtualenv @@ -58,6 +57,7 @@ RUN dnf clean all && \ ca-certificates \ autoconf \ automake \ + autopoint \ m4 \ git diff --git a/docker/tpl-ubuntu-clang-cuda.Dockerfile b/docker/tpl-ubuntu-clang-cuda.Dockerfile index dd669c90..37c3a98b 100644 --- a/docker/tpl-ubuntu-clang-cuda.Dockerfile +++ b/docker/tpl-ubuntu-clang-cuda.Dockerfile @@ -57,6 +57,9 @@ RUN apt-get install -y --no-install-recommends \ libtbb-dev \ bc \ file \ + autopoint \ + autotools-dev \ + automake \ patch \ ca-certificates \ git diff --git a/docker/tpl-ubuntu-clang.Dockerfile b/docker/tpl-ubuntu-clang.Dockerfile index 04ae8da2..574353ea 100644 --- a/docker/tpl-ubuntu-clang.Dockerfile +++ b/docker/tpl-ubuntu-clang.Dockerfile @@ -44,6 +44,9 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ gettext \ help2man \ libtool \ + autopoint \ + autotools-dev \ + automake \ gnupg \ virtualenv diff --git a/docker/tpl-ubuntu-gcc.Dockerfile b/docker/tpl-ubuntu-gcc.Dockerfile index a9843dc2..68fbd86e 100644 --- a/docker/tpl-ubuntu-gcc.Dockerfile +++ b/docker/tpl-ubuntu-gcc.Dockerfile @@ -103,6 +103,9 @@ ARG GCC_MAJOR_VERSION RUN apt-get install -y --no-install-recommends \ libtbb-dev \ make \ + autopoint \ + autotools-dev \ + automake \ bc \ file \ # GEOS patches some tpl. Remove when it's not the case anymore. From 8ee2d9e052ea2d1c5463563efd6324e6f0f47f8b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 13 Oct 2025 19:36:55 -0400 Subject: [PATCH 55/55] Remove autopoint --- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 1 - docker/tpl-rockylinux-gcc-cuda-12.Dockerfile | 1 - 2 files changed, 2 deletions(-) diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index 205a7f58..ede0c379 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -60,7 +60,6 @@ RUN dnf clean all && \ ca-certificates \ autoconf \ automake \ - autopoint \ m4 \ git diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index 6ef22cd8..b5a6127b 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -57,7 +57,6 @@ RUN dnf clean all && \ ca-certificates \ autoconf \ automake \ - autopoint \ m4 \ git