Merge OWP Master Changes with Development Branch (NGWPC-9797)

.github/actions/ngen-build/action.yaml

@@ -52,6 +52,10 @@ inputs:
       required: false
       description: 'Enable mpi support, only available for Linux runners'
       default: 'OFF'
+    build_extern:
+      required: false
+      description: 'Use external dependencies where possible'
+      default: 'OFF'
 outputs:
   build-dir:
     description: "Directory build was performed in"
@@ -110,64 +114,56 @@ runs:
           id: cache-boost-dep
           uses: actions/cache@v4
           with:
-            path: boost_1_79_0
+            path: boost_1_86_0
             key: unix-boost-dep
 
         - name: Get Boost Dependency
           if: steps.cache-boost-dep.outputs.cache-hit != 'true'
           run: |
-            curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download
-            tar xjf boost_1_79_0.tar.bz2
+            curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download
+            tar xjf boost_1_86_0.tar.bz2
           shell: bash
 
         - name: Set Pip Constraints
           run: |
             echo "numpy<2.0" > $GITHUB_WORKSPACE/constraints.txt
             echo "PIP_CONSTRAINT=$GITHUB_WORKSPACE/constraints.txt" >> $GITHUB_ENV
+            echo "UV_CONSTRAINT=$GITHUB_WORKSPACE/constraints.txt" >> $GITHUB_ENV
           shell: bash
 
-        - name: Cache Python Dependencies
-          id: cache-py3-dependencies
-          uses: actions/cache@v3
-          with:
-            path: .venv
-            key: ${{ runner.os }}-python-deps
+        - name: Add uv and Native Python Tooling Translations
+          if: |
+            inputs.use_python != 'OFF'
+          run: |
+            echo 'ACTIVATE_VENV_IF_USE_PYTHON=source .venv/bin/activate' >> $GITHUB_ENV
+            if command -v uv &>/dev/null; then
+              echo 'CREATE_VENV=uv venv .venv' >> $GITHUB_ENV
+              echo 'PIP_INSTALL=uv pip install' >> $GITHUB_ENV
+            else
+              echo 'CREATE_VENV=python3 -m venv .venv' >> $GITHUB_ENV
+              echo 'PIP_INSTALL=pip install' >> $GITHUB_ENV
+            fi
+          shell: bash
 
         - name: Get Numpy Python Dependency
-          # Tried conditioning the cache/install of python with an extra check:
-          # inputs.use_python != 'OFF' &&
-          # but what happens is that a runner not requiring python will create an empty cache
-          # and future runners will pull that and then fail...
-          # What we could do is try to create a master `requirements.txt`
-          # and/or a `test_requirements.txt` file that we can build the hash key from
-          # and read it from the repo...but we still have to always initialize the cache
-          # regardless of whether a given runner uses it, to avoid another runner failing to 
-          # find it.  Or just initialize this minimum requirement of numpy, and let the venv
-          # grow based on other runners needs, effectively building the cache with each new addition
           if: |
-            steps.cache-py3-dependencies.outputs.cache-hit != 'true'
+            inputs.use_python != 'OFF'
           run: |
-            python3 -m venv .venv
-            . .venv/bin/activate
-            pip install pip
-            pip install numpy
+            $CREATE_VENV
+            $ACTIVATE_VENV_IF_USE_PYTHON
+            echo $(which python3)
+            $PIP_INSTALL pip
+            $PIP_INSTALL numpy
             deactivate
           shell: bash
 
         - name: Init Additional Python Dependencies
-          # Don't condition additonal installs on a cache hit
-          # What will happen, however, is that the venv will get updated
-          # and thus the cache will get updated
-          # so any pip install will find modules already installed...
-          # if: |
-          #   inputs.additional_python_requirements != '' &&
-          #   steps.cache-py3-dependencies.outputs.cache-hit != 'true'
           if: |
+            inputs.use_python != 'OFF' &&
             inputs.additional_python_requirements != ''
           run: |
-            python3 -m venv .venv
-            . .venv/bin/activate
-            pip install -r ${{ inputs.additional_python_requirements }}
+            $ACTIVATE_VENV_IF_USE_PYTHON
+            $PIP_INSTALL -r ${{ inputs.additional_python_requirements }}
             deactivate
           shell: bash
 
@@ -185,21 +181,40 @@ runs:
 
         - name: Cmake Initialization
           id: cmake_init
+          # NOTE: -DCMAKE_POLICY_VERSION_MINIMUM=3.5 is required to use cmake version 4
+          # and with older pybind11 versions, the minimum cmake version is set to 3.4
+          # which causes cmake configuration to fail.
           run: |
-            export BOOST_ROOT="$(pwd)/boost_1_79_0"
-            export CFLAGS="-fsanitize=address -O1 -g -fno-omit-frame-pointer -Werror"
-            export CXXFLAGS="-fsanitize=address -O1 -g -fno-omit-frame-pointer -pedantic-errors -Werror -Wpessimizing-move -Wparentheses -Wrange-loop-construct -Wsuggest-override"
-            . .venv/bin/activate
+            export BOOST_ROOT="$(pwd)/boost_1_86_0"
+            export CFLAGS="-fsanitize=address -g -fno-omit-frame-pointer -Werror"
+            export CXXFLAGS="-fsanitize=address -g -fno-omit-frame-pointer -pedantic-errors -Werror -Wpessimizing-move -Wparentheses -Wrange-loop-construct -Wsuggest-override"
+            if [ ${{ runner.os }} == 'macOS' ]
+            then
+              echo "fun:PyType_FromMetaclass" > /tmp/asan_ignore.txt
+              export CFLAGS="$CFLAGS -O0 -fsanitize-ignorelist=/tmp/asan_ignore.txt -fno-common"
+              export CXXFLAGS="$CXXFLAGS -O0 -fsanitize-ignorelist=/tmp/asan_ignore.txt -fno-common"
+            else
+              export CFLAGS="$CFLAGS -O1"
+              export CXXFLAGS="$CXXFLAGS -O1"
+            fi
+            # NOTE: this is not defined if inputs.use_python != 'ON'
+            $ACTIVATE_VENV_IF_USE_PYTHON
             [ ! -d "$BOOST_ROOT" ] && echo "Error: no Boost root found at $BOOST_ROOT" && exit 1
+            echo "Cmake Version:"
+            which cmake
+            cmake --version
             cmake -B ${{ inputs.build-dir }} \
+              -DNGEN_WITH_EXTERN_ALL:BOOL=${{ inputs.build_extern }} \
               -DNGEN_WITH_BMI_C:BOOL=${{ inputs.bmi_c }} \
               -DNGEN_WITH_PYTHON:BOOL=${{ inputs.use_python }} \
               -DNGEN_WITH_UDUNITS:BOOL=${{ inputs.use_udunits }} \
               -DNGEN_WITH_BMI_FORTRAN:BOOL=${{ inputs.bmi_fortran }} \
               -DNGEN_WITH_ROUTING:BOOL=${{ inputs.use_troute }} \
               -DNGEN_WITH_NETCDF:BOOL=${{ inputs.use_netcdf }} \
               -DNGEN_WITH_SQLITE:BOOL=${{ inputs.use_sqlite }} \
-              -DNGEN_WITH_MPI:BOOL=${{ inputs.use_mpi }} -S .
+              -DNGEN_WITH_MPI:BOOL=${{ inputs.use_mpi }} \
+              -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -S .
+
             echo "build-dir=$(echo ${{ inputs.build-dir }})" >> $GITHUB_OUTPUT
           shell: bash
 
@@ -209,8 +224,8 @@ runs:
             # Build Targets
             # Disable leak detection during test enumeration
             export ASAN_OPTIONS=detect_leaks=false
-            # Activate venv so that test discovery run during build works
-            . .venv/bin/activate
+            # NOTE: this is not defined if inputs.use_python != 'ON'
+            $ACTIVATE_VENV_IF_USE_PYTHON
             cmake --build ${{ inputs.build-dir }} --target ${{ inputs.targets }} -- -j  ${{ inputs.build-cores }}
           shell: bash
 

.github/actions/ngen-submod-build/action.yaml

@@ -50,6 +50,14 @@ runs:
         - name: Cmake Initialization
           id: cmake_init
           run: |
+            if [ ${{ runner.os }} == 'macOS' ]
+            then
+                export OPT_LEVEL_FLAG="-O0"
+            else
+                export OPT_LEVEL_FLAG="-O1"
+            fi
+            echo CFLAGS="-fsanitize=address ${OPT_LEVEL_FLAG:?Optimization flag var not set} -g -fno-omit-frame-pointer -Werror" >> $GITHUB_ENV
+            echo CXXFLAGS="-fsanitize=address ${OPT_LEVEL_FLAG:?Optimization flag var not set} -g -fno-omit-frame-pointer -pedantic-errors -Werror -Wpessimizing-move -Wparentheses -Wrange-loop-construct -Wsuggest-override" >> $GITHUB_ENV
             cmake -B ${{ inputs.mod-dir}}/${{ inputs.build-dir }} -S ${{ inputs.mod-dir }} ${{ inputs.cmake-flags }}
             echo "build-dir=$(echo ${{ inputs.mod-dir}}/${{ inputs.build-dir }})" >> $GITHUB_OUTPUT
           shell: bash

.github/workflows/module_integration.yml

@@ -22,7 +22,7 @@ jobs:
     # The type of runner that the job will run on
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-15]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 

.github/workflows/test_and_validate.yml

@@ -24,7 +24,7 @@ jobs:
     # The type of runner that the job will run on
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-15]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
@@ -59,7 +59,7 @@ jobs:
     # The type of runner that the job will run on
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-15]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
@@ -76,7 +76,9 @@ jobs:
           build-cores: ${{ env.LINUX_NUM_PROC_CORES }}
 
       - name: Run Tests
-        run: ./cmake_build/test/compare_pet
+        run: |
+          export ASAN_OPTIONS=${ASAN_OPTIONS}:detect_odr_violation=0
+          ./cmake_build/test/compare_pet
         timeout-minutes: 15
 
       - name: Clean Up
@@ -104,7 +106,7 @@ jobs:
           use_mpi: 'ON'
 
       - name: run_tests
-        run: mpirun --allow-run-as-root -np 2 ./cmake_build/test/test_remote_nexus
+        run: mpirun --allow-run-as-root --oversubscribe -np 4 ./cmake_build/test/test_remote_nexus
         timeout-minutes: 15
 
       - name: Clean Up
@@ -116,7 +118,7 @@ jobs:
     # The type of runner that the job will run on
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-15]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
@@ -139,6 +141,7 @@ jobs:
       - name: Run Tests
         run: |
           cd ./cmake_build/test/
+          export ASAN_OPTIONS=${ASAN_OPTIONS}:detect_odr_violation=0
           ./test_bmi_cpp
           cd ../../
         timeout-minutes: 15
@@ -151,7 +154,7 @@ jobs:
     # The type of runner that the job will run on
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-15]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
@@ -182,7 +185,7 @@ jobs:
     # The type of runner that the job will run on
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-15]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
@@ -215,7 +218,7 @@ jobs:
     # The type of runner that the job will run on
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-15]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
@@ -250,7 +253,7 @@ jobs:
     # The type of runner that the job will run on
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-22.04, macos-15]
       fail-fast: false
     runs-on: ${{ matrix.os }}
 
@@ -272,12 +275,43 @@ jobs:
       - name: Run Unit Tests
         run: |
           . .venv/bin/activate
+          export ASAN_OPTIONS=${ASAN_OPTIONS}:detect_odr_violation=0
           ./cmake_build/test/test_bmi_multi
         timeout-minutes: 15
 
       - name: Clean Up Unit Test Build
         uses: ./.github/actions/clean-build
 
+  # Run BMI protocol tests in linux/unix environment
+  test_bmi_protocols:
+    # The type of runner that the job will run on
+    strategy:
+      matrix:
+        os: [ubuntu-22.04, macos-15]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v4
+
+      - name: Build BMI Protocol Unit Tests
+        uses: ./.github/actions/ngen-build
+        with:
+          targets: "test_bmi_protocols"
+          build-cores: ${{ env.LINUX_NUM_PROC_CORES }}
+
+      - name: run_bmi_protocol_tests
+        run: |
+          cd ./cmake_build/test/
+          export ASAN_OPTIONS=${ASAN_OPTIONS}:detect_odr_violation=0
+          ./test_bmi_protocols
+          cd ../../
+        timeout-minutes: 15
+
+      - name: Clean Up BMI Protocol Unit Test Build
+        uses: ./.github/actions/clean-build
 
   # TODO: fails due to compilation error, at least in large part due to use of POSIX functions not supported on Windows.
   # TODO: Need to determine whether Windows support (in particular, development environment support) is necessary.

CMakeLists.txt

@@ -165,7 +165,7 @@ add_compile_definitions(NGEN_SHARED_LIB_EXTENSION)
 set(Boost_USE_STATIC_LIBS OFF)
 set(Boost_USE_MULTITHREADED ON)
 set(Boost_USE_STATIC_RUNTIME OFF)
-find_package(Boost 1.79.0 REQUIRED)
+find_package(Boost 1.86.0 REQUIRED)
 
 # -----------------------------------------------------------------------------
 if(NGEN_WITH_SQLITE)
@@ -323,6 +323,7 @@ add_subdirectory("src/utilities/mdarray")
 add_subdirectory("src/utilities/mdframe")
 add_subdirectory("src/utilities/logging")
 add_subdirectory("src/utilities/python")
+add_subdirectory("src/utilities/bmi")
 
 target_link_libraries(ngen
     PUBLIC
@@ -336,6 +337,7 @@ target_link_libraries(ngen
         NGen::core_mediator
         NGen::logging
         NGen::parallel
+        NGen::bmi_protocols
 )
 
 if(NGEN_WITH_SQLITE)
@@ -493,3 +495,7 @@ ngen_dependent_multiline_message(NGEN_WITH_PYTHON
 message(STATUS "---------------------------------------------------------------------")
 
 configure_file("${NGEN_INC_DIR}/NGenConfig.h.in" "${CMAKE_CURRENT_BINARY_DIR}/include/NGenConfig.h")
+
+include(GNUInstallDirs)
+install(TARGETS ngen OPTIONAL)
+install(TARGETS partitionGenerator OPTIONAL)

Dockerfile

@@ -42,7 +42,7 @@ ENV LANG="C.UTF-8" \
     HDF5_VERSION="1.10.11" \
     NETCDF_C_VERSION="4.7.4" \
     NETCDF_FORTRAN_VERSION="4.5.4" \
-    BOOST_VERSION="1.83.0"
+    BOOST_VERSION="1.86.0"
 
 # runtime dependencies
 RUN set -eux && \

INSTALL.md

@@ -35,15 +35,15 @@ cd ngen
 **Download the Boost Libraries:**
 
 ```shell
-curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download \
-    && tar -xjf boost_1_79_0.tar.bz2 \
-    && rm boost_1_79_0.tar.bz2
+curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download \
+    && tar -xjf boost_1_86_0.tar.bz2 \
+    && rm boost_1_86_0.tar.bz2
 ```
 
 **Set the ENV for Boost and C compiler:**
 
 ```shell
-set BOOST_ROOT="/boost_1_79_0"
+set BOOST_ROOT="/boost_1_86_0"
 set CXX=/usr/bin/g++
 ```
 
@@ -79,7 +79,7 @@ The following CMake command will configure the build:
 
 ```shell
 cmake -DCMAKE_CXX_COMPILER=/usr/bin/g++ \
-      -DBOOST_ROOT=boost_1_79_0 \
+      -DBOOST_ROOT=boost_1_86_0 \
       -B /build \
       -S .
 ```

doc/BUILDS_AND_CMAKE.md

@@ -104,7 +104,7 @@ In some cases - in particular **Google Test** - the build system will need to be
 
 ## Boost ENV Variable
 
-The Boost libraries must be available for the project to compile.  The details are discussed more in the [Dependencies](DEPENDENCIES.md) doc, but as a helpful hint, the **BOOST_ROOT** environmental variable can be set to the path of the applicable [Boost root directory](https://www.boost.org/doc/libs/1_79_0/more/getting_started/unix-variants.html#the-boost-distribution).  The project's [CMakeLists.txt](../CMakeLists.txt) is written to check for this env variable and use it to set the Boost include directory.
+The Boost libraries must be available for the project to compile.  The details are discussed more in the [Dependencies](DEPENDENCIES.md) doc, but as a helpful hint, the **BOOST_ROOT** environmental variable can be set to the path of the applicable [Boost root directory](https://www.boost.org/doc/libs/1_86_0/more/getting_started/unix-variants.html#the-boost-distribution).  The project's [CMakeLists.txt](../CMakeLists.txt) is written to check for this env variable and use it to set the Boost include directory.
 
 Note that if the variable is not set, it may still be possible for CMake to find Boost, although a *status* message will be printed by CMake indicating **BOOST_ROOT** was not set.