From 3ee3c7bdfe4afff2e5df3594d1525760daf6d83a Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Fri, 26 Dec 2025 09:53:50 -0800 Subject: [PATCH 01/93] Restore support for older-format NetCDF forcings files that have Time variables with dimensions (catchment, time) --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index ad936ca5d0..ef1a8a6cd1 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -147,8 +147,19 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat std::vector raw_time(num_times); try { - time_var.getVar(raw_time.data()); - } catch(const netCDF::exceptions::NcException& e) { + auto dim_count = time_var.getDimCount(); + // Old-format files have dimensions (catchment, time), new-format + // files generated by the forcings engine have just (time) + if (dim_count == 2) { + time_var.getVar({0ul, 0ul}, {1ul, num_times}, raw_time.data()); + } else if (dim_count == 1) { + time_var.getVar({0ul}, {num_times}, raw_time.data()); + } else { + throw std::runtime_error("Unexpected " + std::to_string(dim_count) + + " dimensions on Time variable in NetCDF file '" + + input_path + "'"); + } + } catch(const std::exception& e) { netcdf_ss << "Error reading time variable: " << e.what() << std::endl; LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); throw; From 407ed3e151fb96621109e90fb101b7cde13bf256 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Fri, 26 Dec 2025 09:56:35 -0800 Subject: [PATCH 02/93] Clean up comments and formatting --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index ef1a8a6cd1..03d18bfd7a 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -135,8 +135,6 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat // correct string release nc_free_string(num_ids,&string_buffers[0]); -// Modified code to handle units, epoch start, and reading all time values correctly - KSL - // Get the time variable - getVar collects all values at once and stores in memory // Extremely large timespans could be problematic, but for ngen use cases, this should not be a problem auto time_var = nc_file->getVar("Time"); @@ -168,7 +166,6 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat std::string time_units; try { time_var.getAtt("units").getValues(time_units); - } catch(const netCDF::exceptions::NcException& e) { netcdf_ss << "Error reading time units: " << e.what() << std::endl; LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); @@ -180,9 +177,9 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat double time_scale_factor = 1.0; std::time_t epoch_start_time = 0; - //The following makes some assumptions that NetCDF output from the forcing engine will be relatively uniform - //Specifically, it assumes time values are in units since the Unix Epoch. - //If the forcings engine outputs additional unit formats, this will need to be expanded + // The following makes some assumptions that NetCDF output from the forcing engine will be relatively uniform + // Specifically, it assumes time values are in units since the Unix Epoch. + // If the forcings engine outputs additional unit formats, this will need to be expanded if (time_units.find("minutes since") != std::string::npos) { time_unit = TIME_MINUTES; time_scale_factor = 60.0; @@ -193,14 +190,13 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat time_unit = TIME_SECONDS; time_scale_factor = 1.0; } - //This is also based on the NetCDF from the forcings engine, and may not be super flexible + // This is also based on the NetCDF from the forcings engine, and may not be super flexible std::string datetime_str = time_units.substr(time_units.find("since") + 6); std::tm tm = {}; std::istringstream ss(datetime_str); - ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); //This may be particularly inflexible - epoch_start_time = timegm(&tm); //timegm may not be available in all environments/OSes ie: Windows + ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); // This may be particularly inflexible + epoch_start_time = timegm(&tm); // timegm may not be available in all environments/OSes ie: Windows time_vals.resize(raw_time.size()); -// End modification - KSL std::transform(raw_time.begin(), raw_time.end(), time_vals.begin(), [&](const auto& n) { From e90c79c7f0745df8189ab6d408ac44620a8404f9 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Fri, 26 Dec 2025 11:45:31 -0800 Subject: [PATCH 03/93] Handle more units, with and without a specified reference epoch --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 50 +++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index 03d18bfd7a..ccecc2e0b2 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -177,25 +177,42 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat double time_scale_factor = 1.0; std::time_t epoch_start_time = 0; - // The following makes some assumptions that NetCDF output from the forcing engine will be relatively uniform - // Specifically, it assumes time values are in units since the Unix Epoch. - // If the forcings engine outputs additional unit formats, this will need to be expanded - if (time_units.find("minutes since") != std::string::npos) { + std::string time_base_unit; + auto since_index = time_units.find("since"); + if (since_index != std::string::npos) { + time_base_unit = time_units.substr(0, since_index - 1); + + std::string datetime_str = time_units.substr(since_index + 6); + std::tm tm = {}; + std::istringstream ss(datetime_str); + ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); // This may be particularly inflexible + epoch_start_time = timegm(&tm); // timegm may not be available in all environments/OSes ie: Windows + } else { + time_base_unit = time_units; + } + + if (time_base_unit == "minutes") { time_unit = TIME_MINUTES; time_scale_factor = 60.0; - } else if (time_units.find("hours since") != std::string::npos) { + } else if (time_base_unit == "hours") { time_unit = TIME_HOURS; time_scale_factor = 3600.0; - } else { + } else if (time_base_unit == "seconds" || time_base_unit == "s") { time_unit = TIME_SECONDS; time_scale_factor = 1.0; + } else if (time_base_unit == "milliseconds" || time_base_unit == "ms") { + time_unit = TIME_MILLISECONDS; + time_scale_factor = 1.0e-3; + } else if (time_base_unit == "microseconds" || time_base_unit == "us") { + time_unit = TIME_MICROSECONDS; + time_scale_factor = 1.0e-6; + } else if (time_base_unit == "nanoseconds" || time_base_unit == "ns") { + time_unit = TIME_NANOSECONDS; + time_scale_factor = 1.0e-9; + } else { + Logger::logMsgAndThrowError("In NetCDF file '" + input_path + "', time unit '" + time_base_unit + "' could not be converted"); } - // This is also based on the NetCDF from the forcings engine, and may not be super flexible - std::string datetime_str = time_units.substr(time_units.find("since") + 6); - std::tm tm = {}; - std::istringstream ss(datetime_str); - ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); // This may be particularly inflexible - epoch_start_time = timegm(&tm); // timegm may not be available in all environments/OSes ie: Windows + time_vals.resize(raw_time.size()); std::transform(raw_time.begin(), raw_time.end(), time_vals.begin(), @@ -221,13 +238,20 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat #endif netcdf_ss << "All time intervals are constant within tolerance." << std::endl; - LOG(netcdf_ss.str(), LogLevel::SEVERE); netcdf_ss.str(""); + LOG(netcdf_ss.str(), LogLevel::DEBUG); netcdf_ss.str(""); // determine start_time and stop_time; start_time = time_vals[0]; stop_time = time_vals.back() + time_stride; sim_to_data_time_offset = sim_start_date_time_epoch - start_time; + + netcdf_ss << "NetCDF Forcing from file '" << input_path << "'" + << "Start time " << (time_t)start_time + << ", Stop time " << (time_t)stop_time + << ", sim_start_date_time_epoch " << sim_start_date_time_epoch + ; + LOG(netcdf_ss.str(), LogLevel::DEBUG); netcdf_ss.str(""); } NetCDFPerFeatureDataProvider::~NetCDFPerFeatureDataProvider() = default; From 3002d5404003eaa741e62614fb058c3703d05087 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Fri, 26 Dec 2025 13:39:31 -0800 Subject: [PATCH 04/93] Handle different orderings of variable dimensions --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 51 ++++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index ccecc2e0b2..bb8507145d 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -149,6 +149,9 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat // Old-format files have dimensions (catchment, time), new-format // files generated by the forcings engine have just (time) if (dim_count == 2) { + if (time_var.getDim(0).getName() != "catchment-id" || time_var.getDim(1).getName() != "time") { + Logger::logMsgAndThrowError("In NetCDF file '" + input_path + "', 'Time' variable dimensions don't match expectations"); + } time_var.getVar({0ul, 0ul}, {1ul, num_times}, raw_time.data()); } else if (dim_count == 1) { time_var.getVar({0ul}, {num_times}, raw_time.data()); @@ -335,7 +338,8 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& auto stride = idx2 - idx1; - std::vector start, count; + std::vector start(2), count(2); + std::vector var_index_map(2); auto cat_pos = id_pos[selector.get_id()]; @@ -356,16 +360,29 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& //TODO: Currently assuming a whole variable cache slice across all catchments for a single timestep...but some stuff here to support otherwise. // For reference: https://stackoverflow.com/a/72030286 -//Modified to work with NetCDF dimension shapes and fix errors - KSL size_t cache_slices_t_n = (read_len + cache_slice_t_size - 1) / cache_slice_t_size; // Ceiling division to ensure remainders have a slice - - //Explicitly setting dimension shapes auto dims = ncvar.getDims(); - size_t catchment_dim_size = dims[1].getSize(); - size_t time_dim_size = dims[0].getSize(); - //Cache slicing - modified to work with dimensions structure + int dim_time, dim_catchment; + if (dims.size() != 2) { + Logger::logMsgAndThrowError("Variable dimension count isn't 2"); + } + if (dims[0].getName() == "time" && dims[1].getName() == "catchment-id") { + // Forcings Engine NetCDF output case + dim_time = 0; + dim_catchment = 1; + } else if (dims[1].getName() == "time" && dims[0].getName() == "catchment-id") { + // Classic NetCDF file case + dim_time = 1; + dim_catchment = 0; + } else { + Logger::logMsgAndThrowError("Variable dimensions aren't 'time' and 'catchment-id'"); + } + + size_t time_dim_size = dims[dim_time].getSize(); + size_t catchment_dim_size = dims[dim_catchment].getSize(); + for( size_t i = 0; i < cache_slices_t_n; i++ ) { std::shared_ptr> cached; size_t cache_t_idx = idx1 + i * cache_slice_t_size; @@ -376,14 +393,18 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& cached = value_cache.get(key).get(); } else { cached = std::make_shared>(catchment_dim_size * slice_size); - start.clear(); - start.push_back(cache_t_idx); // start from correct time index - start.push_back(0); // Start from the first catchment - count.clear(); - count.push_back(slice_size); // Read the calculated slice size for time - count.push_back(catchment_dim_size); // Read all catchments + start[dim_time] = cache_t_idx; // start from correct time index + start[dim_catchment] = 0; // Start from the first catchment + count[dim_time] = slice_size; // Read the calculated slice size for time + count[dim_catchment] = catchment_dim_size; // Read all catchments + // Whichever order the file stores the data in, the + // resulting array should have all catchments for a given + // time step contiguous + var_index_map[dim_time] = catchment_dim_size; + var_index_map[dim_catchment] = 1; + try { - ncvar.getVar(start,count,&(*cached)[0]); + ncvar.getVar(start,count, {1l, 1l}, var_index_map, cached->data()); value_cache.insert(key, cached); } catch (netCDF::exceptions::NcException& e) { netcdf_ss << "NetCDF exception: " << e.what() << std::endl; @@ -408,7 +429,7 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& } } } -// End modification + rvalue = 0.0; double a , b = 0.0; From 9f61af0cc07a70c225329535c53a19ccb1cd609f Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Tue, 25 Nov 2025 15:55:21 -0800 Subject: [PATCH 05/93] Flesh out API for state saving and restoring, with adjusted use in Bmi_Module_Formulation --- .../catchment/Bmi_Formulation.hpp | 10 ++ .../catchment/Bmi_Module_Formulation.hpp | 9 +- .../catchment/Bmi_Multi_Formulation.hpp | 2 + .../state_save_restore/State_Save_Restore.hpp | 125 ++++++++++++++++++ .../catchment/Bmi_Module_Formulation.cpp | 8 +- 5 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 include/state_save_restore/State_Save_Restore.hpp diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index f2a13074e8..40cb1df399 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -41,6 +41,8 @@ class Bmi_Formulation_Test; class Bmi_C_Formulation_Test; class Bmi_C_Pet_IT; +class State_Snapshot_Saver; + namespace realization { /** @@ -68,6 +70,14 @@ namespace realization { virtual ~Bmi_Formulation() {}; + /** + * Passes a serialized representation of the model's state to ``saver`` + * + * Asks the model to serialize its state, queries the pointer + * and length, passes that to saver, and then releases it + */ + virtual void save_state(std::shared_ptr saver) const = 0; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 139ee1412f..0910a166b4 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -7,7 +7,6 @@ #include "Bmi_Adapter.hpp" #include #include "bmi_utilities.hpp" -#include #include @@ -47,13 +46,7 @@ namespace realization { void create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global = nullptr) override; void create_formulation(geojson::PropertyMap properties) override; - /** - * Passes a serialized representation of the model's state to ``saver`` - * - * Asks the model to serialize its state, queries the pointer - * and length, passes that to saver, and then releases it - */ - void save_state(std::shared_ptr saver) const; + void save_state(std::shared_ptr saver) const override; /** * Get the collection of forcing output property names this instance can provide. diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 2ecaac5867..0c56cf9049 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -47,6 +47,8 @@ namespace realization { virtual ~Bmi_Multi_Formulation() {}; + void save_state(std::shared_ptr saver) const override; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp new file mode 100644 index 0000000000..2871865dc9 --- /dev/null +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -0,0 +1,125 @@ +#ifndef NGEN_STATE_SAVE_RESTORE_HPP +#define NGEN_STATE_SAVE_RESTORE_HPP + +#include + +#include +#include +#include + +class State_Snapshot_Saver; + +class State_Saver +{ +public: + using snapshot_time_t = std::chrono::time_point; + + // Flag type to indicate whether state saving needs to ensure + // stability of saved data wherever it is stored before returning + // success + enum class State_Durability { relaxed, strict }; + + State_Saver() = default; + virtual ~State_Saver() = default; + + /** + * Return an object suitable for saving a simulation state as of a + * particular moment in time, @param epoch + * + * @param durability indicates whether callers expect all + * potential errors to be checked and reported before finalize() + * and/or State_Snapshot_Saver::finish_saving() return + */ + virtual std::shared_ptr initialize_snapshot(snapshot_time_t epoch, State_Durability durability) = 0; + + /** + * Execute any logic necessary to cleanly finish usage, and + * potentially report errors, before destructors would + * execute. E.g. closing files opened in parallel across MPI + * ranks. + */ + virtual void finalize() = 0; +}; + +class State_Snapshot_Saver +{ +public: + State_Snapshot_Saver() = delete; + State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability); + virtual ~State_Snapshot_Saver() = default; + + /** + * Capture the data from a single unit of the simulation + */ + virtual void save_unit(std::string const& unit_name, boost::span data) = 0; + + /** + * Execute logic to complete the saving process + * + * Data may be flushed here, and delayed errors may be detected + * and reported here. With relaxed durability, error reports may + * not come until the parent State_Saver::finalize() call is made, + * or ever. + */ + virtual void finish_saving() = 0; + +protected: + State_Saver::snapshot_time_t epoch_; + State_Saver::State_Durability durability_; +}; + +class State_Snapshot_Loader; + +class State_Loader +{ +public: + State_Loader() = default; + virtual ~State_Loader() = default; + + /** + * Return an object suitable for loading a simulation state as of + * a particular moment in time, @param epoch + */ + virtual std::shared_ptr initialize_snapshot(State_Saver::snapshot_time_t epoch) = 0; + + /** + * Execute any logic necessary to cleanly finish usage, and + * potentially report errors, before destructors would + * execute. E.g. closing files opened in parallel across MPI + * ranks. + */ + virtual void finalize() = 0; +}; + +class State_Unit_Loader; + +class State_Snapshot_Loader +{ +public: + State_Snapshot_Loader() = default; + virtual ~State_Snapshot_Loader() = default; + + /** + * Load data from whatever source, and pass it to @param unit_loader->load() + */ + virtual void load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) = 0; + + /** + * Execute logic to complete the saving process + * + * Data may be flushed here, and delayed errors may be detected + * and reported here. With relaxed durability, error reports may + * not come until the parent State_Saver::finalize() call is made, + * or ever. + */ + virtual void finish_saving() = 0; +}; + +class State_Unit_Loader +{ + State_Unit_Loader() = default; + virtual ~State_Unit_Loader() = default; + virtual void load(boost::span data) = 0; +}; + +#endif // NGEN_STATE_SAVE_RESTORE_HPP diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 7c541e2d50..52536f8427 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -2,6 +2,7 @@ #include "utilities/logging_utils.h" #include #include "Logger.hpp" +#include std::stringstream bmiform_ss; @@ -15,7 +16,7 @@ namespace realization { inner_create_formulation(properties, true); } - void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { + void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { auto model = get_bmi_model(); size_t size = 1; @@ -25,7 +26,10 @@ namespace realization { auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); boost::span data(serialization_state, size); - saver->save(data); + // Rely on Formulation_Manager also using this->get_id() + // as a unique key for the individual catchment + // formulations + saver->save_unit(this->get_id(), data); model->SetValue("serialization_free", &size); } From bb6011df9a994ed56df1dc6ec1cc2ba85f344390 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Tue, 25 Nov 2025 17:06:24 -0800 Subject: [PATCH 06/93] WIP Wiring all the pieces together --- include/core/Layer.hpp | 4 ++++ include/core/NgenSimulation.hpp | 4 ++++ src/core/Layer.cpp | 12 ++++++++++ src/core/NgenSimulation.cpp | 10 +++++++++ .../catchment/Bmi_Multi_Formulation.cpp | 22 +++++++++++++++++++ 5 files changed, 52 insertions(+) diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 5c3c4481fa..6e5fe707e8 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -16,6 +16,8 @@ namespace hy_features class HY_Features_MPI; } +class State_Snapshot_Saver; + namespace ngen { @@ -110,6 +112,8 @@ namespace ngen std::unordered_map &nexus_indexes, int current_step); + virtual void save_state_snapshot(std::shared_ptr snapshot_saver); + protected: const LayerDescription description; diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 00e5ef49eb..0e164b664a 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -12,6 +12,8 @@ namespace hy_features class HY_Features_MPI; } +class State_Snapshot_Saver; + #include #include #include @@ -62,6 +64,8 @@ class NgenSimulation private: void advance_models_one_output_step(); + void save_state_snapshot(std::shared_ptr snapshot_saver); + int simulation_step_; std::shared_ptr sim_time_; diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 56d6506be6..6aaa0313b9 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -1,5 +1,6 @@ #include #include +#include #if NGEN_WITH_MPI #include "HY_Features_MPI.hpp" @@ -83,3 +84,14 @@ void ngen::Layer::update_models(boost::span catchment_outflows, simulation_time.advance_timestep(); } } + +void ngen::Layer::save_state_snapshot(std::shared_ptr snapshot_saver) +{ + // XXX Handle any of this class's own state as a meta-data unit + + for (auto const& id : processing_units) { + auto r = features.catchment_at(id); + auto r_c = std::dynamic_pointer_cast(r); + r_c->save_state(snapshot_saver); + } +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c5cce69103..c2f9bdf04a 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -108,6 +108,16 @@ void NgenSimulation::advance_models_one_output_step() } +void NgenSimulation::save_state_snapshot(std::shared_ptr snapshot_saver) +{ + + // XXX Handle self, then recursively pass responsibility to Layers + for (auto& layer : layers_) { + layer->save_state_snapshot(snapshot_saver); + } +} + + int NgenSimulation::get_nexus_index(std::string const& nexus_id) const { auto iter = nexus_indexes_.find(nexus_id); diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 145b7dfb73..b934166068 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -13,8 +13,30 @@ #include "Bmi_Py_Formulation.hpp" #include "Logger.hpp" +#include + using namespace realization; +void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { +#if 0 + auto model = get_bmi_model(); + + size_t size = 1; + model->SetValue("serialization_create", &size); + model->GetValue("serialization_size", &size); + + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + boost::span data(serialization_state, size); + + // Rely on Formulation_Manager also using this->get_id() + // as a unique key for the individual catchment + // formulations + saver->save_unit(this->get_id(), data); + + model->SetValue("serialization_free", &size); +#endif +} + void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap properties, bool needs_param_validation) { if (needs_param_validation) { validate_parameters(properties); From 1f8001ba02d4cd7fb8905b6159982e7425423484 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Mon, 8 Dec 2025 17:53:19 -0800 Subject: [PATCH 07/93] Add logic and structures for parsing state saving configuration from realization config --- CMakeLists.txt | 2 + data/example_state_saving_config.json | 109 ++++++++++++++++++ .../state_save_restore/State_Save_Restore.hpp | 28 +++++ src/NGen.cpp | 5 +- src/state_save_restore/CMakeLists.txt | 17 +++ src/state_save_restore/State_Save_Restore.cpp | 64 ++++++++++ 6 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 data/example_state_saving_config.json create mode 100644 src/state_save_restore/CMakeLists.txt create mode 100644 src/state_save_restore/State_Save_Restore.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ecf309e715..8b6f8d9fda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,6 +318,7 @@ add_subdirectory("src/geojson") add_subdirectory("src/bmi") add_subdirectory("src/realizations/catchment") add_subdirectory("src/forcing") +add_subdirectory("src/state_save_restore") add_subdirectory("src/utilities") add_subdirectory("src/utilities/mdarray") add_subdirectory("src/utilities/mdframe") @@ -336,6 +337,7 @@ target_link_libraries(ngen NGen::core_mediator NGen::logging NGen::parallel + NGen::state_save_restore ) if(NGEN_WITH_SQLITE) diff --git a/data/example_state_saving_config.json b/data/example_state_saving_config.json new file mode 100644 index 0000000000..cba48afcdb --- /dev/null +++ b/data/example_state_saving_config.json @@ -0,0 +1,109 @@ +{ + "global": { + "formulations": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "test_bmi_cpp", + "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so", + "init_config": "./data/bmi/c/test/test_bmi_c_config.ini", + "main_output_variable": "OUTPUT_VAR_2", + "variables_names_map" : { + "INPUT_VAR_2": "TMP_2maboveground", + "INPUT_VAR_1": "precip_rate" + }, + "create_function": "bmi_model_create", + "destroy_function": "bmi_model_destroy", + "uses_forcing_file": false + } + } + ], + "forcing": { + "file_pattern": ".*{{id}}.*.csv", + "path": "./data/forcing/" + } + }, + "state_saving": { + "label": "end", + "path": "state_end", + "type": "FilePerUnit", + "when": "EndOfRun" + }, + "time": { + "start_time": "2015-12-01 00:00:00", + "end_time": "2015-12-30 23:00:00", + "output_interval": 3600 + }, + "output_root": "./output_dir/", + "catchments": { + "cat-27": { + "formulations": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "test_bmi_cpp", + "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so", + "init_config": "./data/bmi/c/test/test_bmi_c_config.ini", + "main_output_variable": "OUTPUT_VAR_2", + "variables_names_map" : { + "INPUT_VAR_2": "TMP_2maboveground", + "INPUT_VAR_1": "precip_rate" + }, + "create_function": "bmi_model_create", + "destroy_function": "bmi_model_destroy", + "uses_forcing_file": false + } + } + ], + "forcing": { + "path": "./data/forcing/cat-27_2015-12-01 00_00_00_2015-12-30 23_00_00.csv" + } + }, + "cat-52": { + "formulations": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "test_bmi_cpp", + "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so", + "init_config": "./data/bmi/c/test/test_bmi_c_config.ini", + "main_output_variable": "OUTPUT_VAR_2", + "variables_names_map" : { + "INPUT_VAR_2": "TMP_2maboveground", + "INPUT_VAR_1": "precip_rate" + }, + "create_function": "bmi_model_create", + "destroy_function": "bmi_model_destroy", + "uses_forcing_file": false + } + } + ], + "forcing": { + "path": "./data/forcing/cat-52_2015-12-01 00_00_00_2015-12-30 23_00_00.csv" + } + }, + "cat-67": { + "formulations": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "test_bmi_cpp", + "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so", + "init_config": "./data/bmi/c/test/test_bmi_c_config.ini", + "main_output_variable": "OUTPUT_VAR_2", + "variables_names_map" : { + "INPUT_VAR_2": "TMP_2maboveground", + "INPUT_VAR_1": "precip_rate" + }, + "create_function": "bmi_model_create", + "destroy_function": "bmi_model_destroy", + "uses_forcing_file": false + } + } + ], + "forcing": { + "path": "./data/forcing/cat-67_2015-12-01 00_00_00_2015-12-30 23_00_00.csv" + } + } + } +} diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 2871865dc9..a7226a5df8 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -3,9 +3,37 @@ #include +#include +#include + #include #include #include +#include +#include + +class State_Save_Config +{ +public: + /** + * Expects the tree @param config that potentially contains a "state_saving" key + * + * + */ + State_Save_Config(boost::property_tree::ptree const& config); + + struct instance + { + instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); + + std::string label_; + std::string path_; + std::string mechanism_; + std::string timing_; + }; + + std::vector instances_; +}; class State_Snapshot_Saver; diff --git a/src/NGen.cpp b/src/NGen.cpp index c554ef751a..925b9df0a8 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -54,6 +54,8 @@ #include #include +#include + void ngen::exec_info::runtime_summary(std::ostream& stream) noexcept { stream << "Runtime configuration summary:\n"; @@ -535,9 +537,10 @@ int main(int argc, char* argv[]) { } auto simulation_time_config = realization::config::Time(*possible_simulation_time).make_params(); - sim_time = std::make_shared(simulation_time_config); + auto state_saving_config = State_Save_Config(realization_config); + ss << "Initializing formulations" << std::endl; LOG(ss.str(), LogLevel::INFO); ss.str(""); diff --git a/src/state_save_restore/CMakeLists.txt b/src/state_save_restore/CMakeLists.txt new file mode 100644 index 0000000000..7a3db83e44 --- /dev/null +++ b/src/state_save_restore/CMakeLists.txt @@ -0,0 +1,17 @@ +include(${PROJECT_SOURCE_DIR}/cmake/dynamic_sourced_library.cmake) +dynamic_sourced_cxx_library(state_save_restore "${CMAKE_CURRENT_SOURCE_DIR}") + +add_library(NGen::state_save_restore ALIAS state_save_restore) +target_link_libraries(state_save_restore PUBLIC + NGen::config_header + Boost::boost # Headers-only Boost + ) + +#target_link_libraries(core +# PRIVATE +# ) + +target_include_directories(state_save_restore PUBLIC + ${PROJECT_SOURCE_DIR}/include + ) + diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp new file mode 100644 index 0000000000..1b53fb79fe --- /dev/null +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -0,0 +1,64 @@ +#include + +#include + +#include +#include + +#include + +State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) +{ + auto maybe = tree.get_child_optional("state_saving"); + + // Default initialization will represent the "not enabled" case + if (!maybe) { + LOG("State saving not configured", LogLevel::INFO); + return; + } + + auto subtree = *maybe; + + try { + auto single_what = subtree.get("label"); + auto single_where = subtree.get("path"); + auto single_how = subtree.get("type"); + auto single_when = subtree.get("when"); + + instance i{single_what, single_where, single_how, single_when}; + instances_.push_back(i); + } catch (...) { + LOG("Bad state saving config", LogLevel::WARNING); + throw; + } + + LOG("State saving configured", LogLevel::INFO); +} + +State_Save_Config::instance::instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) + : label_(label) + , path_(path) + , mechanism_(mechanism) + , timing_(timing) +{ + if (mechanism_ == "FilePerUnit") { + // nothing to do here + } else { + Logger::logMsgAndThrowError("Unrecognized state saving mechanism '" + mechanism_ + "'"); + } + + if (timing_ == "EndOfRun") { + // nothing to do here + } else if (timing_ == "FirstOfMonth") { + // nothing to do here + } else { + Logger::logMsgAndThrowError("Unrecognized state saving timing '" + timing_ + "'"); + } +} + +State_Snapshot_Saver::State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) + : epoch_(epoch) + , durability_(durability) +{ + +} From 6b77086fa66b0bc24f2d83b1cb15fb07566fdc21 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Wed, 10 Dec 2025 08:51:22 -0800 Subject: [PATCH 08/93] CMakeLists.txt: Remove commented out bits --- src/state_save_restore/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/state_save_restore/CMakeLists.txt b/src/state_save_restore/CMakeLists.txt index 7a3db83e44..8c09fc7cbc 100644 --- a/src/state_save_restore/CMakeLists.txt +++ b/src/state_save_restore/CMakeLists.txt @@ -7,10 +7,6 @@ target_link_libraries(state_save_restore PUBLIC Boost::boost # Headers-only Boost ) -#target_link_libraries(core -# PRIVATE -# ) - target_include_directories(state_save_restore PUBLIC ${PROJECT_SOURCE_DIR}/include ) From d97e4a760e50931544e946b7eb5374648e0985e1 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Mon, 15 Dec 2025 21:30:25 -0800 Subject: [PATCH 09/93] Add File_Per_Unit state saving mechanism --- include/state_save_restore/File_Per_Unit.hpp | 20 ++++ src/state_save_restore/File_Per_Unit.cpp | 101 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 include/state_save_restore/File_Per_Unit.hpp create mode 100644 src/state_save_restore/File_Per_Unit.cpp diff --git a/include/state_save_restore/File_Per_Unit.hpp b/include/state_save_restore/File_Per_Unit.hpp new file mode 100644 index 0000000000..cb6af8e0ac --- /dev/null +++ b/include/state_save_restore/File_Per_Unit.hpp @@ -0,0 +1,20 @@ +#ifndef NGEN_FILE_PER_UNIT_HPP +#define NGEN_FILE_PER_UNIT_HPP + +#include + +class File_Per_Unit_Saver : public State_Saver +{ +public: + File_Per_Unit_Saver(std::string base_path); + ~File_Per_Unit_Saver(); + + std::shared_ptr initialize_snapshot(snapshot_time_t epoch, State_Durability durability) override; + + void finalize() override; + +private: + std::string base_path_; +}; + +#endif // NGEN_FILE_PER_UNIT_HPP diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp new file mode 100644 index 0000000000..d7c7ddf2b9 --- /dev/null +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -0,0 +1,101 @@ +#include +#include + +#if __has_include() && __cpp_lib_filesystem >= 201703L + #include + using namespace std::filesystem; + #warning "Using STD Filesystem" +#elif __has_include() && defined(__cpp_lib_filesystem) + #include + using namespace std::experimental::filesystem; + #warning "Using Filesystem TS" +#elif __has_include() + #include + using namespace boost::filesystem; + #warning "Using Boost.Filesystem" +#else + #error "No Filesystem library implementation available" +#endif + +#include +#include + +// This class is only declared and defined here, in the .cpp file, +// because it is strictly an implementation detail of the top-level +// File_Per_Unit_Saver class +class File_Per_Unit_Snapshot_Saver : public State_Snapshot_Saver +{ + friend class File_Per_Unit_Saver; + + public: + File_Per_Unit_Snapshot_Saver() = delete; + File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability); + ~File_Per_Unit_Snapshot_Saver(); + +public: + void save_unit(std::string const& unit_name, boost::span data) override; + void finish_saving() override; + +private: + std::string format_epoch(State_Saver::snapshot_time_t epoch); + + path dir_path_; +}; + +File_Per_Unit_Saver::File_Per_Unit_Saver(std::string base_path) + : base_path_(std::move(base_path)) +{ + auto dir_path = path(base_path_); + create_directories(dir_path); +} + +File_Per_Unit_Saver::~File_Per_Unit_Saver() = default; + +std::shared_ptr File_Per_Unit_Saver::initialize_snapshot(snapshot_time_t epoch, State_Durability durability) +{ + return std::make_shared(base_path_, epoch, durability); +} + +void File_Per_Unit_Saver::finalize() +{ + // nothing to be done +} + +File_Per_Unit_Snapshot_Saver::File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) + : State_Snapshot_Saver(epoch, durability) + , dir_path_(base_path / format_epoch(epoch)) +{ + create_directory(dir_path_); +} + +File_Per_Unit_Snapshot_Saver::~File_Per_Unit_Snapshot_Saver() = default; + +std::string File_Per_Unit_Snapshot_Saver::format_epoch(State_Saver::snapshot_time_t epoch) +{ + time_t t = std::chrono::system_clock::to_time_t(epoch); + std::tm tm = *std::gmtime(&t); + + std::stringstream tss; + tss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S"); + return tss.str(); +} + +void File_Per_Unit_Snapshot_Saver::save_unit(std::string const& unit_name, boost::span data) +{ + auto file_path = dir_path_ / unit_name; + try { + std::ofstream stream(file_path.string(), std::ios_base::out | std::ios_base::binary); + stream.write(data.data(), data.size()); + stream.close(); + } catch (std::exception &e) { + LOG("Failed to write state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); + throw; + } +} + +void File_Per_Unit_Snapshot_Saver::finish_saving() +{ + if (durability_ == State_Saver::State_Durability::strict) { + // fsync() or whatever + } +} From 99803e063ee2e105f6fe2109d9861c51a07f97d7 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 16 Jan 2026 14:06:13 -0500 Subject: [PATCH 10/93] State saving for multi-BMI --- CMakeLists.txt | 7 +- data/example_state_saving_config_multi.json | 116 ++++++++++++++++++ include/core/NgenSimulation.hpp | 4 +- .../catchment/Bmi_Fortran_Formulation.hpp | 9 ++ .../catchment/Bmi_Module_Formulation.hpp | 17 +++ .../state_save_restore/State_Save_Restore.hpp | 12 +- src/NGen.cpp | 10 ++ .../catchment/Bmi_Fortran_Formulation.cpp | 10 ++ .../catchment/Bmi_Module_Formulation.cpp | 25 ++-- .../catchment/Bmi_Multi_Formulation.cpp | 69 ++++++++--- src/state_save_restore/CMakeLists.txt | 2 + src/state_save_restore/State_Save_Restore.cpp | 30 +++++ 12 files changed, 283 insertions(+), 28 deletions(-) create mode 100644 data/example_state_saving_config_multi.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b6f8d9fda..86bfa3dab5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,7 +165,12 @@ add_compile_definitions(NGEN_SHARED_LIB_EXTENSION) set(Boost_USE_STATIC_LIBS OFF) set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_RUNTIME OFF) -find_package(Boost 1.79.0 REQUIRED) +if(CMAKE_CXX_STANDARD LESS 17) + # requires non-header filesystem for state saving if C++ 11 or lower + find_package(Boost 1.79.0 REQUIRED COMPONENTS system filesystem) +else() + find_package(Boost 1.79.0 REQUIRED) +endif() # ----------------------------------------------------------------------------- if(NGEN_WITH_SQLITE) diff --git a/data/example_state_saving_config_multi.json b/data/example_state_saving_config_multi.json new file mode 100644 index 0000000000..48ba4d8b58 --- /dev/null +++ b/data/example_state_saving_config_multi.json @@ -0,0 +1,116 @@ +{ + "global": { + "formulations": [ + { + "name": "bmi_multi", + "params": { + "model_type_name": "bmi_multi_noahowp_cfe", + "forcing_file": "", + "init_config": "", + "allow_exceed_end_time": true, + "main_output_variable": "Q_OUT", + "modules": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "bmi_c++_sloth", + "library_file": "./extern/sloth/cmake_build/libslothmodel.so", + "init_config": "/dev/null", + "allow_exceed_end_time": true, + "main_output_variable": "z", + "uses_forcing_file": false, + "model_params": { + "sloth_ice_fraction_schaake(1,double,m,node)": 0.0, + "sloth_ice_fraction_xinanjiang(1,double,1,node)": 0.0, + "sloth_smp(1,double,1,node)": 0.0 + } + } + }, + { + "name": "bmi_fortran", + "params": { + "model_type_name": "bmi_fortran_noahowp", + "library_file": "./extern/noah-owp-modular/cmake_build/libsurfacebmi", + "forcing_file": "", + "init_config": "./data/bmi/fortran/noah-owp-modular-init-{{id}}.namelist.input", + "allow_exceed_end_time": true, + "main_output_variable": "QINSUR", + "variables_names_map": { + "PRCPNONC": "atmosphere_water__liquid_equivalent_precipitation_rate", + "Q2": "atmosphere_air_water~vapor__relative_saturation", + "SFCTMP": "land_surface_air__temperature", + "UU": "land_surface_wind__x_component_of_velocity", + "VV": "land_surface_wind__y_component_of_velocity", + "LWDN": "land_surface_radiation~incoming~longwave__energy_flux", + "SOLDN": "land_surface_radiation~incoming~shortwave__energy_flux", + "SFCPRS": "land_surface_air__pressure" + }, + "uses_forcing_file": false + } + }, + { + "name": "bmi_c", + "params": { + "model_type_name": "bmi_c_pet", + "library_file": "./extern/evapotranspiration/evapotranspiration/cmake_build/libpetbmi", + "forcing_file": "", + "init_config": "./data/bmi/c/pet/{{id}}_bmi_config.ini", + "allow_exceed_end_time": true, + "main_output_variable": "water_potential_evaporation_flux", + "registration_function": "register_bmi_pet", + "variables_names_map": { + "water_potential_evaporation_flux": "potential_evapotranspiration" + }, + "uses_forcing_file": false + } + }, + { + "name": "bmi_c", + "params": { + "model_type_name": "bmi_c_cfe", + "library_file": "./extern/cfe/cmake_build/libcfebmi", + "forcing_file": "", + "init_config": "./data/bmi/c/cfe/{{id}}_bmi_config.ini", + "allow_exceed_end_time": true, + "main_output_variable": "Q_OUT", + "registration_function": "register_bmi_cfe", + "variables_names_map": { + "water_potential_evaporation_flux": "potential_evapotranspiration", + "atmosphere_air_water~vapor__relative_saturation": "SPFH_2maboveground", + "land_surface_air__temperature": "TMP_2maboveground", + "land_surface_wind__x_component_of_velocity": "UGRD_10maboveground", + "land_surface_wind__y_component_of_velocity": "VGRD_10maboveground", + "land_surface_radiation~incoming~longwave__energy_flux": "DLWRF_surface", + "land_surface_radiation~incoming~shortwave__energy_flux": "DSWRF_surface", + "land_surface_air__pressure": "PRES_surface", + "ice_fraction_schaake" : "sloth_ice_fraction_schaake", + "ice_fraction_xinanjiang" : "sloth_ice_fraction_xinanjiang", + "soil_moisture_profile" : "sloth_smp" + }, + "uses_forcing_file": false + } + } + ], + "uses_forcing_file": false + } + } + ], + "forcing": { + "file_pattern": ".*{{id}}.*..csv", + "path": "./data/forcing/", + "provider": "CsvPerFeature" + } + }, + "state_saving": { + "label": "end", + "path": "state_end", + "type": "FilePerUnit", + "when": "EndOfRun" + }, + "time": { + "start_time": "2015-12-01 00:00:00", + "end_time": "2015-12-30 23:00:00", + "output_interval": 3600 + }, + "output_root": "./output_dir/" +} diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 0e164b664a..d60dd165b4 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -61,11 +61,11 @@ class NgenSimulation size_t get_num_output_times() const; std::string get_timestamp_for_step(int step) const; + void save_state_snapshot(std::shared_ptr snapshot_saver); + private: void advance_models_one_output_step(); - void save_state_snapshot(std::shared_ptr snapshot_saver); - int simulation_step_; std::shared_ptr sim_time_; diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 4f6863b883..71225d4885 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -23,6 +23,15 @@ namespace realization { std::string get_formulation_type() const override; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * Because the Fortran BMI has no messaging for 64-bit integers, this overload will use the 32-bit integer interface and copy the results to `size`. + * + * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. + * @return Pointer to the beginning of the serialized data. + */ + const char* create_save_state(uint64_t *size) const override; + protected: /** diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 0910a166b4..b7575e0576 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -46,8 +46,25 @@ namespace realization { void create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global = nullptr) override; void create_formulation(geojson::PropertyMap properties) override; + /** + * Create a save state, save it using the `State_Snapshot_Saver`, then clear the save state from memory. + * `this->get_id()` will be used as the unique ID for the saver. + */ void save_state(std::shared_ptr saver) const override; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * + * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. + * @return Pointer to the beginning of the serialized data. + */ + virtual const char* create_save_state(uint64_t *size) const; + + /** + * Clears any serialized data stored by the BMI from memory. + */ + virtual void free_save_state() const; + /** * Get the collection of forcing output property names this instance can provide. * diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index a7226a5df8..fea1555d21 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -12,6 +12,9 @@ #include #include +class State_Saver; +class State_Snapshot_Saver; + class State_Save_Config { public: @@ -22,6 +25,10 @@ class State_Save_Config */ State_Save_Config(boost::property_tree::ptree const& config); + bool has_end_of_run() const; + + std::shared_ptr end_of_run_saver() const; + struct instance { instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); @@ -32,11 +39,10 @@ class State_Save_Config std::string timing_; }; +private: std::vector instances_; }; -class State_Snapshot_Saver; - class State_Saver { public: @@ -50,6 +56,8 @@ class State_Saver State_Saver() = default; virtual ~State_Saver() = default; + static snapshot_time_t snapshot_time_now(); + /** * Return an object suitable for saving a simulation state as of a * particular moment in time, @param epoch diff --git a/src/NGen.cpp b/src/NGen.cpp index 925b9df0a8..f6cd7855af 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -722,6 +722,16 @@ int main(int argc, char* argv[]) { simulation->run_catchments(); + if (state_saving_config.has_end_of_run()) { + LOG("Saving end-of-run state.", LogLevel::INFO); + std::shared_ptr saver = state_saving_config.end_of_run_saver(); + std::shared_ptr snapshot = saver->initialize_snapshot( + State_Saver::snapshot_time_now(), + State_Saver::State_Durability::strict + ); + simulation->save_state_snapshot(snapshot); + } + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 390a6fb4e7..ae43929ac6 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -93,4 +93,14 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const return 1.0; } +const char* Bmi_Fortran_Formulation::create_save_state(uint64_t *size) const { + auto model = get_bmi_model(); + int size_int = 1; + model->SetValue("serialization_create", &size_int); + model->GetValue("serialization_size", &size_int); + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + *size = static_cast(size_int); + return serialization_state; +} + #endif // NGEN_WITH_BMI_FORTRAN diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 52536f8427..c28317ec6b 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -17,13 +17,8 @@ namespace realization { } void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { - auto model = get_bmi_model(); - - size_t size = 1; - model->SetValue("serialization_create", &size); - model->GetValue("serialization_size", &size); - - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + uint64_t size = 1; + const char* serialization_state = this->create_save_state(&size); boost::span data(serialization_state, size); // Rely on Formulation_Manager also using this->get_id() @@ -31,7 +26,21 @@ namespace realization { // formulations saver->save_unit(this->get_id(), data); - model->SetValue("serialization_free", &size); + this->free_save_state(); + } + + const char* Bmi_Module_Formulation::create_save_state(uint64_t *size) const { + auto model = get_bmi_model(); + model->SetValue("serialization_create", size); + model->GetValue("serialization_size", size); + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + return serialization_state; + } + + void Bmi_Module_Formulation::free_save_state() const { + auto model = get_bmi_model(); + int _; + model->SetValue("serialization_free", &_); } boost::span Bmi_Module_Formulation::get_available_variable_names() const { diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index b934166068..9de11a02d7 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -15,26 +15,65 @@ #include +#if (__cplusplus >= 202002L) +#include +#endif + using namespace realization; void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { -#if 0 - auto model = get_bmi_model(); - - size_t size = 1; - model->SetValue("serialization_create", &size); - model->GetValue("serialization_size", &size); - - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); - boost::span data(serialization_state, size); - - // Rely on Formulation_Manager also using this->get_id() - // as a unique key for the individual catchment - // formulations - saver->save_unit(this->get_id(), data); +#if (__cplusplus < 202002L) + // get system endianness + uint16_t endian_bytes = 0xFF00; + uint8_t *endian_bits = reinterpret_cast(&endian_bytes); + bool is_little_endian = endian_bits[0] == 0; +#endif - model->SetValue("serialization_free", &size); + std::vector> bmi_data; + size_t data_size = 0; + uint64_t ser_size; + // TODO: something more elegant than just skipping sloth + for (const nested_module_ptr &m : modules) { + auto bmi = static_cast(m.get()); + if (bmi->get_model_type_name() != "bmi_c++_sloth") { + ser_size = 1; + const char* serialized = bmi->create_save_state(&ser_size); + bmi_data.push_back(std::make_pair(serialized, ser_size)); + data_size += sizeof(uint64_t) + ser_size; + LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), ser_size); + } + } + char *data = new char[data_size]; + size_t index = 0; + for (const auto &bmi : bmi_data) { + // write the size of the data +#if (__cplusplus < 202002L) + if (is_little_endian) { +#else + if constexpr (std::endian::native == std::endian::little) { #endif + std::memcpy(&data[index], &bmi.second, sizeof(uint64_t)); + } else { + // store the size bytes in reverse order to ensure saved data is always little endian + const char *bytes = reinterpret_cast(&bmi.second); + size_t endian_index = index + sizeof(uint64_t); + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + data[--endian_index] = bytes[i]; + } + } + // write the serialized data + std::memcpy(data + index + sizeof(uint64_t), &bmi.first, bmi.second); + index += sizeof(uint64_t) + bmi.second; + } + boost::span span(data, data_size); + saver->save_unit(this->get_id(), span); + + delete[] data; + for (const nested_module_ptr &m : modules) { + auto bmi = static_cast(m.get()); + bmi->free_save_state(); + } } void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap properties, bool needs_param_validation) { diff --git a/src/state_save_restore/CMakeLists.txt b/src/state_save_restore/CMakeLists.txt index 8c09fc7cbc..cde58c6073 100644 --- a/src/state_save_restore/CMakeLists.txt +++ b/src/state_save_restore/CMakeLists.txt @@ -5,6 +5,8 @@ add_library(NGen::state_save_restore ALIAS state_save_restore) target_link_libraries(state_save_restore PUBLIC NGen::config_header Boost::boost # Headers-only Boost + Boost::system + Boost::filesystem ) target_include_directories(state_save_restore PUBLIC diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 1b53fb79fe..2df94b9515 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -35,6 +36,26 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) LOG("State saving configured", LogLevel::INFO); } +bool State_Save_Config::has_end_of_run() const { + for (const auto& i : instances_) + if (i.timing_ == "EndOfRun") + return true; + return false; +} + +std::shared_ptr State_Save_Config::end_of_run_saver() const { + for (const auto& i : instances_) { + if (i.timing_ == "EndOfRun") { + if (i.mechanism_ == "FilePerUnit") { + return std::make_shared(i.path_); + } else { + Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_ + " is not supported for end of run saving."); + } + } + } + Logger::logMsgAndThrowError("State_Save_Config: No end of run was defined in the realization config."); +} + State_Save_Config::instance::instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) : label_(label) , path_(path) @@ -62,3 +83,12 @@ State_Snapshot_Saver::State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, S { } + +State_Saver::snapshot_time_t State_Saver::snapshot_time_now() { +#if __cplusplus < 201703L // C++ < 17 + auto now = std::chrono::system_clock::now(); + return std::chrono::time_point_cast(now); +#else + return std::chrono::floor(std::chrono::system_clock::now()); +#endif +} From 27256aa4c3e8e1b4b5205194cf76979265196f05 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 19 Jan 2026 13:23:42 -0500 Subject: [PATCH 11/93] Cold start config structure --- include/state_save_restore/File_Per_Unit.hpp | 14 ++ .../state_save_restore/State_Save_Restore.hpp | 35 ++++- src/state_save_restore/File_Per_Unit.cpp | 102 ++++++++++++-- src/state_save_restore/State_Save_Restore.cpp | 127 +++++++++++++----- 4 files changed, 227 insertions(+), 51 deletions(-) diff --git a/include/state_save_restore/File_Per_Unit.hpp b/include/state_save_restore/File_Per_Unit.hpp index cb6af8e0ac..3fdefe71b8 100644 --- a/include/state_save_restore/File_Per_Unit.hpp +++ b/include/state_save_restore/File_Per_Unit.hpp @@ -17,4 +17,18 @@ class File_Per_Unit_Saver : public State_Saver std::string base_path_; }; + +class File_Per_Unit_Loader : public State_Loader +{ +public: + File_Per_Unit_Loader(std::string dir_path); + ~File_Per_Unit_Loader() = default; + + void finalize() override { }; + + std::shared_ptr initialize_snapshot(State_Saver::snapshot_time_t epoch) override; +private: + std::string dir_path_; +}; + #endif // NGEN_FILE_PER_UNIT_HPP diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index fea1555d21..85c48d5d21 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -12,7 +12,26 @@ #include #include +enum class State_Save_Direction { + None = 0, + Save, + Load +}; + +enum class State_Save_Mechanism { + None = 0, + FilePerUnit +}; + +enum class State_Save_When { + None = 0, + EndOfRun, + FirstOfMonth, + StartOfRun +}; + class State_Saver; +class State_Loader; class State_Snapshot_Saver; class State_Save_Config @@ -27,20 +46,29 @@ class State_Save_Config bool has_end_of_run() const; + bool has_cold_start() const; + std::shared_ptr end_of_run_saver() const; + std::shared_ptr cold_start_saver() const; + struct instance { - instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); + instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); + State_Save_Direction direction_; + State_Save_Mechanism mechanism_; + State_Save_When timing_; std::string label_; std::string path_; - std::string mechanism_; - std::string timing_; + + std::string mechanism_string() const; }; private: std::vector instances_; + int end_of_run() const; + int cold_start() const; }; class State_Saver @@ -153,6 +181,7 @@ class State_Snapshot_Loader class State_Unit_Loader { +public: State_Unit_Loader() = default; virtual ~State_Unit_Loader() = default; virtual void load(boost::span data) = 0; diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index d7c7ddf2b9..9ab1993bb5 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -20,6 +20,18 @@ #include #include +namespace unit_saving_utils { + std::string format_epoch(State_Saver::snapshot_time_t epoch) + { + time_t t = std::chrono::system_clock::to_time_t(epoch); + std::tm tm = *std::gmtime(&t); + + std::stringstream tss; + tss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S"); + return tss.str(); + } +} + // This class is only declared and defined here, in the .cpp file, // because it is strictly an implementation detail of the top-level // File_Per_Unit_Saver class @@ -37,8 +49,6 @@ class File_Per_Unit_Snapshot_Saver : public State_Snapshot_Saver void finish_saving() override; private: - std::string format_epoch(State_Saver::snapshot_time_t epoch); - path dir_path_; }; @@ -63,23 +73,13 @@ void File_Per_Unit_Saver::finalize() File_Per_Unit_Snapshot_Saver::File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) : State_Snapshot_Saver(epoch, durability) - , dir_path_(base_path / format_epoch(epoch)) + , dir_path_(base_path / unit_saving_utils::format_epoch(epoch)) { create_directory(dir_path_); } File_Per_Unit_Snapshot_Saver::~File_Per_Unit_Snapshot_Saver() = default; -std::string File_Per_Unit_Snapshot_Saver::format_epoch(State_Saver::snapshot_time_t epoch) -{ - time_t t = std::chrono::system_clock::to_time_t(epoch); - std::tm tm = *std::gmtime(&t); - - std::stringstream tss; - tss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S"); - return tss.str(); -} - void File_Per_Unit_Snapshot_Saver::save_unit(std::string const& unit_name, boost::span data) { auto file_path = dir_path_ / unit_name; @@ -99,3 +99,79 @@ void File_Per_Unit_Snapshot_Saver::finish_saving() // fsync() or whatever } } + + +// This class is only declared and defined here, in the .cpp file, +// because it is strictly an implementation detail of the top-level +// File_Per_Unit_Saver class +class File_Per_Unit_Snapshot_Loader : public State_Snapshot_Loader +{ + friend class State_Snapshot_Loader; +public: + File_Per_Unit_Snapshot_Loader() = default; + File_Per_Unit_Snapshot_Loader(path dir_path); + ~File_Per_Unit_Snapshot_Loader() override = default; + + /** + * Load data from whatever source, and pass it to @param unit_loader->load() + */ + void load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) override; + + /** + * Execute logic to complete the saving process + * + * Data may be flushed here, and delayed errors may be detected + * and reported here. With relaxed durability, error reports may + * not come until the parent State_Saver::finalize() call is made, + * or ever. + */ + void finish_saving() override { }; + +private: + path dir_path_; + std::vector data_; +}; + +File_Per_Unit_Snapshot_Loader::File_Per_Unit_Snapshot_Loader(path dir_path) + : dir_path_(dir_path) +{ + +} + +void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) { + auto file_path = dir_path_ / unit_name; + std::uintmax_t size; + try { + size = file_size(file_path.string()); + } catch (std::exception &e) { + LOG("Failed to read state save data size for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); + throw; + } + std::ifstream stream(file_path.string(), std::ios_base::ate | std::ios_base::binary); + if (!stream) { + LOG("Failed to open state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); + throw; + } + try { + std::vector buffer(size); + stream.read(buffer.data(), size); + boost::span data(buffer.data(), size); + unit_loader->load(data); + } catch (std::exception &e) { + LOG("Failed to read state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); + throw; + } +} + +File_Per_Unit_Loader::File_Per_Unit_Loader(std::string dir_path) + : dir_path_(dir_path) +{ + +} + +std::shared_ptr File_Per_Unit_Loader::initialize_snapshot(State_Saver::snapshot_time_t epoch) +{ + path dir_path(dir_path_); + return std::make_shared(dir_path); +} + diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 2df94b9515..12ddf1e74e 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -18,62 +18,119 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) return; } - auto subtree = *maybe; - - try { - auto single_what = subtree.get("label"); - auto single_where = subtree.get("path"); - auto single_how = subtree.get("type"); - auto single_when = subtree.get("when"); - - instance i{single_what, single_where, single_how, single_when}; - instances_.push_back(i); - } catch (...) { - LOG("Bad state saving config", LogLevel::WARNING); - throw; + auto saving_config = *maybe; + size_t length = saving_config.size(); + for (size_t i = 0; i < length; ++i) { + try { + auto subtree = tree.get_child(std::to_string(i)); + auto direction = subtree.get("direction"); + auto what = subtree.get("label"); + auto where = subtree.get("path"); + auto how = subtree.get("type"); + auto when = subtree.get("when"); + + instance i{direction, what, where, how, when}; + instances_.push_back(i); + } catch (...) { + LOG("Bad state saving config", LogLevel::WARNING); + throw; + } } LOG("State saving configured", LogLevel::INFO); } +int State_Save_Config::end_of_run() const { + for (size_t i = 0; i < instances_.size(); ++i) { + auto &instance = instances_[i]; + if (instance.timing_ == State_Save_When::EndOfRun + && instance.direction_ == State_Save_Direction::Save) + return i; + } + return -1; +} + bool State_Save_Config::has_end_of_run() const { - for (const auto& i : instances_) - if (i.timing_ == "EndOfRun") - return true; - return false; + this->end_of_run() >= 0; } std::shared_ptr State_Save_Config::end_of_run_saver() const { - for (const auto& i : instances_) { - if (i.timing_ == "EndOfRun") { - if (i.mechanism_ == "FilePerUnit") { - return std::make_shared(i.path_); - } else { - Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_ + " is not supported for end of run saving."); - } + int index = this->end_of_run(); + if (index >= 0) { + const auto& i = instances_[index]; + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + return std::make_shared(i.path_); + } else { + Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); } } Logger::logMsgAndThrowError("State_Save_Config: No end of run was defined in the realization config."); } -State_Save_Config::instance::instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) +int State_Save_Config::cold_start() const { + for (size_t i = 0; i < instances_.size(); ++i) { + const auto& instance = instances_[i]; + if (instance.timing_ == State_Save_When::StartOfRun + && instance.direction_ == State_Save_Direction::Load) { + return i; + } + } + return -1; +} + +bool State_Save_Config::has_cold_start() const { + return this->cold_start() >= 0; +} + +std::shared_ptr State_Save_Config::cold_start_saver() const { + int index = this->cold_start(); + if (index >= 0) { + const auto& i = instances_[index]; + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + return std::make_shared(i.path_); + } else { + Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); + } + } +} + +State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) : label_(label) , path_(path) - , mechanism_(mechanism) - , timing_(timing) { - if (mechanism_ == "FilePerUnit") { - // nothing to do here + if (direction == "save") { + direction_ = State_Save_Direction::Save; + } else if (direction == "load") { + direction_ = State_Save_Direction::Load; + } else { + Logger::logMsgAndThrowError("Unrecognized state saving direction '" + direction + "'"); + } + + if (mechanism == "FilePerUnit") { + mechanism_ = State_Save_Mechanism::FilePerUnit; } else { - Logger::logMsgAndThrowError("Unrecognized state saving mechanism '" + mechanism_ + "'"); + Logger::logMsgAndThrowError("Unrecognized state saving mechanism '" + mechanism + "'"); } - if (timing_ == "EndOfRun") { - // nothing to do here - } else if (timing_ == "FirstOfMonth") { - // nothing to do here + if (timing == "EndOfRun") { + timing_ = State_Save_When::EndOfRun; + } else if (timing == "FirstOfMonth") { + timing_ = State_Save_When::FirstOfMonth; + } else if (timing == "StartOfRun") { + timing_ = State_Save_When::StartOfRun; } else { - Logger::logMsgAndThrowError("Unrecognized state saving timing '" + timing_ + "'"); + Logger::logMsgAndThrowError("Unrecognized state saving timing '" + timing + "'"); + } +} + +std::string State_Save_Config::instance::instance::mechanism_string() const { + switch (mechanism_) { + case State_Save_Mechanism::None: + return "None"; + case State_Save_Mechanism::FilePerUnit: + return "FilePerUnit"; + default: + return "Other"; } } From 3588beb06f72faa5364195bc3c7156ca02c8b32f Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 20 Jan 2026 11:22:00 -0500 Subject: [PATCH 12/93] Cold start loading --- data/example_state_saving_config_multi.json | 5 +- include/core/Layer.hpp | 2 + include/core/NgenSimulation.hpp | 2 + .../catchment/Bmi_Formulation.hpp | 8 +++ .../catchment/Bmi_Fortran_Formulation.hpp | 7 +-- .../catchment/Bmi_Module_Formulation.hpp | 27 ++++----- .../catchment/Bmi_Multi_Formulation.hpp | 2 + .../state_save_restore/State_Save_Restore.hpp | 14 +---- src/NGen.cpp | 7 +++ src/core/Layer.cpp | 11 ++++ src/core/NgenSimulation.cpp | 6 ++ .../catchment/Bmi_Fortran_Formulation.cpp | 10 ++-- .../catchment/Bmi_Module_Formulation.cpp | 34 +++++------- .../catchment/Bmi_Multi_Formulation.cpp | 55 ++++++++++++++++--- src/state_save_restore/File_Per_Unit.cpp | 14 ++--- src/state_save_restore/State_Save_Restore.cpp | 3 +- 16 files changed, 133 insertions(+), 74 deletions(-) diff --git a/data/example_state_saving_config_multi.json b/data/example_state_saving_config_multi.json index 48ba4d8b58..ecfaa272fd 100644 --- a/data/example_state_saving_config_multi.json +++ b/data/example_state_saving_config_multi.json @@ -101,12 +101,13 @@ "provider": "CsvPerFeature" } }, - "state_saving": { + "state_saving": [{ + "direction": "save", "label": "end", "path": "state_end", "type": "FilePerUnit", "when": "EndOfRun" - }, + }], "time": { "start_time": "2015-12-01 00:00:00", "end_time": "2015-12-30 23:00:00", diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 6e5fe707e8..ab4a0e4268 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -17,6 +17,7 @@ namespace hy_features } class State_Snapshot_Saver; +class State_Snapshot_Loader; namespace ngen { @@ -113,6 +114,7 @@ namespace ngen int current_step); virtual void save_state_snapshot(std::shared_ptr snapshot_saver); + virtual void load_state_snapshot(std::shared_ptr snapshot_loader); protected: diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index d60dd165b4..c29280d719 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -13,6 +13,7 @@ namespace hy_features } class State_Snapshot_Saver; +class State_Snapshot_Loader; #include #include @@ -62,6 +63,7 @@ class NgenSimulation std::string get_timestamp_for_step(int step) const; void save_state_snapshot(std::shared_ptr snapshot_saver); + void load_state_snapshot(std::shared_ptr snapshot_loader); private: void advance_models_one_output_step(); diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index 40cb1df399..cc1cec0de6 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -42,6 +42,7 @@ class Bmi_C_Formulation_Test; class Bmi_C_Pet_IT; class State_Snapshot_Saver; +class State_Snapshot_Loader; namespace realization { @@ -78,6 +79,13 @@ namespace realization { */ virtual void save_state(std::shared_ptr saver) const = 0; + /** + * Passes a serialized representation of the model's state to ``loader`` + * + * Asks saver to find data for the BMI and passes that data to the BMI for loading. + */ + virtual void load_state(std::shared_ptr loader) const = 0; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 71225d4885..6cc8618bc4 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -24,13 +24,12 @@ namespace realization { std::string get_formulation_type() const override; /** - * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_serialization_state` is called. * Because the Fortran BMI has no messaging for 64-bit integers, this overload will use the 32-bit integer interface and copy the results to `size`. * - * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. - * @return Pointer to the beginning of the serialized data. + * @return Span of the serialized data. */ - const char* create_save_state(uint64_t *size) const override; + const boost::span get_serialization_state() const override; protected: diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index b7575e0576..7653aae8b6 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -52,18 +52,7 @@ namespace realization { */ void save_state(std::shared_ptr saver) const override; - /** - * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. - * - * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. - * @return Pointer to the beginning of the serialized data. - */ - virtual const char* create_save_state(uint64_t *size) const; - - /** - * Clears any serialized data stored by the BMI from memory. - */ - virtual void free_save_state() const; + void load_state(std::shared_ptr loader) const override; /** * Get the collection of forcing output property names this instance can provide. @@ -299,8 +288,20 @@ namespace realization { const std::vector get_bmi_input_variables() const override; const std::vector get_bmi_output_variables() const override; - const boost::span get_serialization_state() const; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_serialization_state` is called. + * + * @return Span of the serialized data. + */ + virtual const boost::span get_serialization_state() const; + /** + * Requests the BMI to load data from a previously saved state. This has a side effect of freeing a current state if it currently exists. + */ void load_serialization_state(const boost::span state) const; + /** + * Requests the BMI to clear a currently saved state from memory. + * Existing state pointers should not be used as the stored data may be freed depending on implementation. + */ void free_serialization_state() const; void set_realization_file_format(bool is_legacy_format); diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 0c56cf9049..cb7aace479 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -49,6 +49,8 @@ namespace realization { void save_state(std::shared_ptr saver) const override; + void load_state(std::shared_ptr loader) const override; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 85c48d5d21..bbf54a3dbe 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -50,7 +50,7 @@ class State_Save_Config std::shared_ptr end_of_run_saver() const; - std::shared_ptr cold_start_saver() const; + std::shared_ptr cold_start_loader() const; struct instance { @@ -155,8 +155,6 @@ class State_Loader virtual void finalize() = 0; }; -class State_Unit_Loader; - class State_Snapshot_Loader { public: @@ -166,7 +164,7 @@ class State_Snapshot_Loader /** * Load data from whatever source, and pass it to @param unit_loader->load() */ - virtual void load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) = 0; + virtual void load_unit(std::string const& unit_name, std::vector &data) = 0; /** * Execute logic to complete the saving process @@ -179,12 +177,4 @@ class State_Snapshot_Loader virtual void finish_saving() = 0; }; -class State_Unit_Loader -{ -public: - State_Unit_Loader() = default; - virtual ~State_Unit_Loader() = default; - virtual void load(boost::span data) = 0; -}; - #endif // NGEN_STATE_SAVE_RESTORE_HPP diff --git a/src/NGen.cpp b/src/NGen.cpp index f6cd7855af..3bddde6ff5 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -720,6 +720,13 @@ int main(int argc, char* argv[]) { std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); + if (state_saving_config.has_cold_start()) { + LOG(LogLevel::INFO, "Loading simulation data from cold start."); + std::shared_ptr cold_loader = state_saving_config.cold_start_loader(); + std::shared_ptr cold_snapshot_loader = cold_loader->initialize_snapshot(State_Saver::snapshot_time_now()); + simulation->load_state_snapshot(cold_snapshot_loader); + } + simulation->run_catchments(); if (state_saving_config.has_end_of_run()) { diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 6aaa0313b9..4d66dc069b 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -95,3 +95,14 @@ void ngen::Layer::save_state_snapshot(std::shared_ptr snap r_c->save_state(snapshot_saver); } } + +void ngen::Layer::load_state_snapshot(std::shared_ptr snapshot_loader) +{ + // XXX Handle any of this class's own state as a meta-data unit + + for (auto const& id : processing_units) { + auto r = features.catchment_at(id); + auto r_c = std::dynamic_pointer_cast(r); + r_c->load_state(snapshot_loader); + } +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c2f9bdf04a..7742e3a057 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -117,6 +117,12 @@ void NgenSimulation::save_state_snapshot(std::shared_ptr s } } +void NgenSimulation::load_state_snapshot(std::shared_ptr snapshot_loader) { + for (auto& layer : layers_) { + layer->load_state_snapshot(snapshot_loader); + } +} + int NgenSimulation::get_nexus_index(std::string const& nexus_id) const { diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index ae43929ac6..b8804eab79 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -93,14 +93,14 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const return 1.0; } -const char* Bmi_Fortran_Formulation::create_save_state(uint64_t *size) const { +const boost::span Bmi_Fortran_Formulation::get_serialization_state() const { auto model = get_bmi_model(); - int size_int = 1; + int size_int = 0; model->SetValue("serialization_create", &size_int); model->GetValue("serialization_size", &size_int); - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); - *size = static_cast(size_int); - return serialization_state; + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + const boost::span span(serialization_state, size_int); + return span; } #endif // NGEN_WITH_BMI_FORTRAN diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index c28317ec6b..b9dc6f2452 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -18,29 +18,21 @@ namespace realization { void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { uint64_t size = 1; - const char* serialization_state = this->create_save_state(&size); - boost::span data(serialization_state, size); + boost::span data = this->get_serialization_state(); // Rely on Formulation_Manager also using this->get_id() // as a unique key for the individual catchment // formulations saver->save_unit(this->get_id(), data); - this->free_save_state(); + this->free_serialization_state(); } - const char* Bmi_Module_Formulation::create_save_state(uint64_t *size) const { - auto model = get_bmi_model(); - model->SetValue("serialization_create", size); - model->GetValue("serialization_size", size); - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); - return serialization_state; - } - - void Bmi_Module_Formulation::free_save_state() const { - auto model = get_bmi_model(); - int _; - model->SetValue("serialization_free", &_); + void Bmi_Module_Formulation::load_state(std::shared_ptr loader) const { + std::vector buffer; + loader->load_unit(this->get_id(), buffer); + boost::span data(buffer.data(), buffer.size()); + this->load_serialization_state(data); } boost::span Bmi_Module_Formulation::get_available_variable_names() const { @@ -1093,12 +1085,12 @@ namespace realization { } const boost::span Bmi_Module_Formulation::get_serialization_state() const { - auto bmi = this->bmi_model; - // create a new serialized state, getting the amount of data that was saved - uint64_t* size = (uint64_t*)bmi->GetValuePtr("serialization_create"); - // get the pointer of the new state - char* serialized = (char*)bmi->GetValuePtr("serialization_state"); - const boost::span span(serialized, *size); + auto model = get_bmi_model(); + uint64_t size = 0; + model->SetValue("serialization_create", &size); + model->GetValue("serialization_size", &size); + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + const boost::span span(serialization_state, size); return span; } diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 9de11a02d7..b44715a093 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -31,17 +31,15 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav std::vector> bmi_data; size_t data_size = 0; - uint64_t ser_size; // TODO: something more elegant than just skipping sloth for (const nested_module_ptr &m : modules) { - auto bmi = static_cast(m.get()); + auto bmi = dynamic_cast(m.get()); if (bmi->get_model_type_name() != "bmi_c++_sloth") { - ser_size = 1; - const char* serialized = bmi->create_save_state(&ser_size); - bmi_data.push_back(std::make_pair(serialized, ser_size)); - data_size += sizeof(uint64_t) + ser_size; - LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), ser_size); + boost::span span = bmi->get_serialization_state(); + bmi_data.push_back(std::make_pair(span.data(), span.size())); + data_size += sizeof(uint64_t) + span.size(); + LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); } } char *data = new char[data_size]; @@ -72,7 +70,46 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav delete[] data; for (const nested_module_ptr &m : modules) { auto bmi = static_cast(m.get()); - bmi->free_save_state(); + if (bmi->get_model_type_name() != "bmi_c++_sloth") { + bmi->free_serialization_state(); + } + } +} + +void Bmi_Multi_Formulation::load_state(std::shared_ptr loader) const { +#if (__cplusplus < 202002L) + // get system endianness + uint16_t endian_bytes = 0xFF00; + uint8_t *endian_bits = reinterpret_cast(&endian_bytes); + bool is_little_endian = endian_bits[0] == 0; +#endif + std::vector data; + loader->load_unit(this->get_id(), data); + size_t index = 0; + for (const nested_module_ptr &m : modules) { + auto bmi = dynamic_cast(m.get()); + if (bmi->get_model_type_name() != "bmi_c++_sloth") { + uint64_t size; +#if (__cplusplus < 202002L) + if (is_little_endian) { +#else + if constexpr (std::endian::native == std::endian::little) { +#endif + memcpy(&size, data.data() + index, sizeof(uint64_t)); + } else { + // read size bytes in reverse order to interpret from little endian + char *size_bytes = reinterpret_cast(&size); + size_t endian_index = sizeof(uint64_t); + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + size_bytes[--endian_index] = data[index + i]; + } + } + boost::span span(data.data() + index + sizeof(uint64_t), size); + bmi->load_serialization_state(span); + index += sizeof(uint64_t) + size; + LOG(LogLevel::DEBUG, "Loading of multi-BMI %s %s completed with a size of %d bytes.", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); + } } } diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index 9ab1993bb5..103b7d6a7a 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -113,9 +113,11 @@ class File_Per_Unit_Snapshot_Loader : public State_Snapshot_Loader ~File_Per_Unit_Snapshot_Loader() override = default; /** - * Load data from whatever source, and pass it to @param unit_loader->load() + * Load data from whatever source and store it in the `data` vector. + * + * @param data The location where the loaded data will be stored. This will be resized to the amount of data loaded. */ - void load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) override; + void load_unit(std::string const& unit_name, std::vector &data) override; /** * Execute logic to complete the saving process @@ -138,7 +140,7 @@ File_Per_Unit_Snapshot_Loader::File_Per_Unit_Snapshot_Loader(path dir_path) } -void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) { +void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, std::vector &data) { auto file_path = dir_path_ / unit_name; std::uintmax_t size; try { @@ -153,10 +155,8 @@ void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, Stat throw; } try { - std::vector buffer(size); - stream.read(buffer.data(), size); - boost::span data(buffer.data(), size); - unit_loader->load(data); + data.resize(size); + stream.read(data.data(), size); } catch (std::exception &e) { LOG("Failed to read state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); throw; diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 12ddf1e74e..ebcf273d60 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -82,7 +82,7 @@ bool State_Save_Config::has_cold_start() const { return this->cold_start() >= 0; } -std::shared_ptr State_Save_Config::cold_start_saver() const { +std::shared_ptr State_Save_Config::cold_start_loader() const { int index = this->cold_start(); if (index >= 0) { const auto& i = instances_[index]; @@ -92,6 +92,7 @@ std::shared_ptr State_Save_Config::cold_start_saver() const { Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); } } + Logger::logMsgAndThrowError("State_Save_Config: No configuration was found for loading a cold start."); } State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) From e27f10ebec04d6c4c7e18c440926cccda744148e Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 20 Jan 2026 11:45:04 -0500 Subject: [PATCH 13/93] Fix config JSON array parsing --- src/state_save_restore/State_Save_Restore.cpp | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index ebcf273d60..f082d729a1 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -18,11 +18,10 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) return; } - auto saving_config = *maybe; - size_t length = saving_config.size(); - for (size_t i = 0; i < length; ++i) { + //auto saving_config = *maybe; + for (const auto& saving_config : *maybe) { try { - auto subtree = tree.get_child(std::to_string(i)); + auto& subtree = saving_config.second; auto direction = subtree.get("direction"); auto what = subtree.get("label"); auto where = subtree.get("path"); @@ -43,15 +42,15 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) int State_Save_Config::end_of_run() const { for (size_t i = 0; i < instances_.size(); ++i) { auto &instance = instances_[i]; - if (instance.timing_ == State_Save_When::EndOfRun - && instance.direction_ == State_Save_Direction::Save) + if (instance.timing_ == State_Save_When::EndOfRun && instance.direction_ == State_Save_Direction::Save) { return i; + } } return -1; } bool State_Save_Config::has_end_of_run() const { - this->end_of_run() >= 0; + return this->end_of_run() >= 0; } std::shared_ptr State_Save_Config::end_of_run_saver() const { @@ -64,14 +63,15 @@ std::shared_ptr State_Save_Config::end_of_run_saver() const { Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); } } - Logger::logMsgAndThrowError("State_Save_Config: No end of run was defined in the realization config."); + auto error = "State_Save_Config: No end of run was defined in the realization config."; + LOG(LogLevel::SEVERE, error); + throw std::runtime_error(error); } int State_Save_Config::cold_start() const { for (size_t i = 0; i < instances_.size(); ++i) { const auto& instance = instances_[i]; - if (instance.timing_ == State_Save_When::StartOfRun - && instance.direction_ == State_Save_Direction::Load) { + if (instance.timing_ == State_Save_When::StartOfRun && instance.direction_ == State_Save_Direction::Load) { return i; } } @@ -92,7 +92,9 @@ std::shared_ptr State_Save_Config::cold_start_loader() const { Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); } } - Logger::logMsgAndThrowError("State_Save_Config: No configuration was found for loading a cold start."); + auto error = "State_Save_Config: No configuration was found for loading a cold start."; + LOG(LogLevel::SEVERE, error); + throw std::runtime_error(error); } State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) From d37625e027afb996b8bdee3c35b48dbcf6a22a8a Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 21 Jan 2026 09:36:35 -0500 Subject: [PATCH 14/93] Less restrictive start and end of run save state --- .../state_save_restore/State_Save_Restore.hpp | 10 +-- src/NGen.cpp | 16 ++--- src/state_save_restore/State_Save_Restore.cpp | 72 ++++++------------- 3 files changed, 31 insertions(+), 67 deletions(-) diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index bbf54a3dbe..7bff90e474 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -44,13 +44,9 @@ class State_Save_Config */ State_Save_Config(boost::property_tree::ptree const& config); - bool has_end_of_run() const; + std::unordered_map> start_of_run_loaders() const; - bool has_cold_start() const; - - std::shared_ptr end_of_run_saver() const; - - std::shared_ptr cold_start_loader() const; + std::unordered_map> end_of_run_savers() const; struct instance { @@ -67,8 +63,6 @@ class State_Save_Config private: std::vector instances_; - int end_of_run() const; - int cold_start() const; }; class State_Saver diff --git a/src/NGen.cpp b/src/NGen.cpp index 3bddde6ff5..8720d1f971 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -720,19 +720,17 @@ int main(int argc, char* argv[]) { std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); - if (state_saving_config.has_cold_start()) { - LOG(LogLevel::INFO, "Loading simulation data from cold start."); - std::shared_ptr cold_loader = state_saving_config.cold_start_loader(); - std::shared_ptr cold_snapshot_loader = cold_loader->initialize_snapshot(State_Saver::snapshot_time_now()); - simulation->load_state_snapshot(cold_snapshot_loader); + for (const auto& start_loader : state_saving_config.start_of_run_loaders()) { + LOG(LogLevel::INFO, "Loading start of run simulation data from state saving config " + start_loader.first); + std::shared_ptr snapshot_loader = start_loader.second->initialize_snapshot(State_Saver::snapshot_time_now()); + simulation->load_state_snapshot(snapshot_loader); } simulation->run_catchments(); - if (state_saving_config.has_end_of_run()) { - LOG("Saving end-of-run state.", LogLevel::INFO); - std::shared_ptr saver = state_saving_config.end_of_run_saver(); - std::shared_ptr snapshot = saver->initialize_snapshot( + for (const auto& end_saver : state_saving_config.end_of_run_savers()) { + LOG(LogLevel::INFO, "Saving end of run simulation data for state saving config " + end_saver.first); + std::shared_ptr snapshot = end_saver.second->initialize_snapshot( State_Saver::snapshot_time_now(), State_Saver::State_Durability::strict ); diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index f082d729a1..fe0fc6a82d 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -39,62 +39,34 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) LOG("State saving configured", LogLevel::INFO); } -int State_Save_Config::end_of_run() const { - for (size_t i = 0; i < instances_.size(); ++i) { - auto &instance = instances_[i]; - if (instance.timing_ == State_Save_When::EndOfRun && instance.direction_ == State_Save_Direction::Save) { - return i; +std::unordered_map> State_Save_Config::start_of_run_loaders() const { + std::unordered_map> loaders; + for (const auto &i : this->instances_) { + if (i.timing_ == State_Save_When::StartOfRun && i.direction_ == State_Save_Direction::Load) { + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + auto loader = std::make_shared(i.path_); + loaders[i.label_] = loader; + } else { + LOG(LogLevel::WARNING, "State_Save_Config: Loading mechanism " + i.mechanism_string() + " is not supported for start of run loading."); + } } } - return -1; + return loaders; } -bool State_Save_Config::has_end_of_run() const { - return this->end_of_run() >= 0; -} - -std::shared_ptr State_Save_Config::end_of_run_saver() const { - int index = this->end_of_run(); - if (index >= 0) { - const auto& i = instances_[index]; - if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { - return std::make_shared(i.path_); - } else { - Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); - } - } - auto error = "State_Save_Config: No end of run was defined in the realization config."; - LOG(LogLevel::SEVERE, error); - throw std::runtime_error(error); -} - -int State_Save_Config::cold_start() const { - for (size_t i = 0; i < instances_.size(); ++i) { - const auto& instance = instances_[i]; - if (instance.timing_ == State_Save_When::StartOfRun && instance.direction_ == State_Save_Direction::Load) { - return i; - } - } - return -1; -} - -bool State_Save_Config::has_cold_start() const { - return this->cold_start() >= 0; -} - -std::shared_ptr State_Save_Config::cold_start_loader() const { - int index = this->cold_start(); - if (index >= 0) { - const auto& i = instances_[index]; - if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { - return std::make_shared(i.path_); - } else { - Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); +std::unordered_map> State_Save_Config::end_of_run_savers() const { + std::unordered_map> savers; + for (const auto &i : this->instances_) { + if (i.timing_ == State_Save_When::EndOfRun && i.direction_ == State_Save_Direction::Save) { + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + auto loader = std::make_shared(i.path_); + savers[i.label_] = loader; + } else { + LOG(LogLevel::WARNING, "State_Save_Config: Loading mechanism " + i.mechanism_string() + " is not supported for start of run loading."); + } } } - auto error = "State_Save_Config: No configuration was found for loading a cold start."; - LOG(LogLevel::SEVERE, error); - throw std::runtime_error(error); + return savers; } State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) From 96d20d3b7802199e1796178de9e21bb53781fa2e Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 21 Jan 2026 10:03:39 -0500 Subject: [PATCH 15/93] Use parent classes for start and end of run states --- .../state_save_restore/State_Save_Restore.hpp | 14 ++++++++++++-- src/state_save_restore/State_Save_Restore.cpp | 18 ++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 7bff90e474..8f4b1b77e5 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -44,9 +44,19 @@ class State_Save_Config */ State_Save_Config(boost::property_tree::ptree const& config); - std::unordered_map> start_of_run_loaders() const; + /** + * Get state loaders that perform before the catchments are run. + * + * @return `std::pair`s of the label from the config and an instance of the loader. + */ + std::vector>> start_of_run_loaders() const; - std::unordered_map> end_of_run_savers() const; + /** + * Get state savers that perform after the catchments have run to completion. + * + * @return `std::pair`s of the label from the config and an instance of the saver. + */ + std::vector>> end_of_run_savers() const; struct instance { diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index fe0fc6a82d..84e2226e95 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -39,13 +39,14 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) LOG("State saving configured", LogLevel::INFO); } -std::unordered_map> State_Save_Config::start_of_run_loaders() const { - std::unordered_map> loaders; +std::vector>> State_Save_Config::start_of_run_loaders() const { + std::vector>> loaders; for (const auto &i : this->instances_) { if (i.timing_ == State_Save_When::StartOfRun && i.direction_ == State_Save_Direction::Load) { if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { auto loader = std::make_shared(i.path_); - loaders[i.label_] = loader; + auto pair = std::make_pair(i.label_, loader); + loaders.push_back(pair); } else { LOG(LogLevel::WARNING, "State_Save_Config: Loading mechanism " + i.mechanism_string() + " is not supported for start of run loading."); } @@ -54,15 +55,16 @@ std::unordered_map> State_Sav return loaders; } -std::unordered_map> State_Save_Config::end_of_run_savers() const { - std::unordered_map> savers; +std::vector>> State_Save_Config::end_of_run_savers() const { + std::vector>> savers; for (const auto &i : this->instances_) { if (i.timing_ == State_Save_When::EndOfRun && i.direction_ == State_Save_Direction::Save) { if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { - auto loader = std::make_shared(i.path_); - savers[i.label_] = loader; + auto saver = std::make_shared(i.path_); + auto pair = std::make_pair(i.label_, saver); + savers.push_back(pair); } else { - LOG(LogLevel::WARNING, "State_Save_Config: Loading mechanism " + i.mechanism_string() + " is not supported for start of run loading."); + LOG(LogLevel::WARNING, "State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for start of run saving."); } } } From 1d7d0b7371c7178c35680a0f258d9143d7ce9cfe Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 27 Jan 2026 08:34:47 -0500 Subject: [PATCH 16/93] Dynamically sized set value option for python BMI adapter --- include/bmi/Bmi_Py_Adapter.hpp | 10 ++++++++++ .../realizations/catchment/Bmi_Module_Formulation.hpp | 2 +- include/realizations/catchment/Bmi_Py_Formulation.hpp | 7 +++++++ src/bmi/Bmi_Py_Adapter.cpp | 6 ++++++ src/realizations/catchment/Bmi_Py_Formulation.cpp | 6 ++++++ 5 files changed, 30 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index cd3d38f268..a8a00ec162 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -590,6 +590,16 @@ namespace models { } } + /** + * Set the value of a variable. This version of setting a variable will send an array with the `size` specified instead of checking the BMI for its current size of the variable. + * + * @param name The name of the BMI variable. + * @param src Pointer to the data that will be sent to the BMI. + * @param size The number of items represented by the pointer. + */ + template + void set_value_unchecked(const std::string &name, T *src, size_t size); + /** * Set values for a model's BMI variable at specified indices. * diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 7653aae8b6..adb3423a06 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -297,7 +297,7 @@ namespace realization { /** * Requests the BMI to load data from a previously saved state. This has a side effect of freeing a current state if it currently exists. */ - void load_serialization_state(const boost::span state) const; + virtual void load_serialization_state(const boost::span state) const; /** * Requests the BMI to clear a currently saved state from memory. * Existing state pointers should not be used as the stored data may be freed depending on implementation. diff --git a/include/realizations/catchment/Bmi_Py_Formulation.hpp b/include/realizations/catchment/Bmi_Py_Formulation.hpp index 76904165c9..19c6a0428e 100644 --- a/include/realizations/catchment/Bmi_Py_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Py_Formulation.hpp @@ -35,6 +35,13 @@ namespace realization { bool is_bmi_output_variable(const std::string &var_name) const override; + /** + * Requests the BMI to load data from a previously saved state. This has a side effect of freeing a current state if it currently exists. + * + * The python BMI requires additional messaging for pre-allocating memory for load + */ + void load_serialization_state(const boost::span state) const override; + protected: std::shared_ptr construct_model(const geojson::PropertyMap &properties) override; diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index 7436e2b48c..ee79de4e14 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -223,4 +223,10 @@ void Bmi_Py_Adapter::UpdateUntil(double time) { bmi_model->attr("update_until")(time); } +template +void Bmi_Py_Adapter::set_value_unchecked(const std::string &name, T *src, size_t size) { + py::array_t src_array(size, src); + bmi_model->attr("set_value")(name, src_array); +} + #endif //NGEN_WITH_PYTHON diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index 7d266db9f8..e804aa5882 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -117,4 +117,10 @@ bool Bmi_Py_Formulation::is_model_initialized() const { return get_bmi_model()->is_model_initialized(); } +void Bmi_Py_Formulation::load_serialization_state(const boost::span state) const { + auto bmi = std::dynamic_pointer_cast(get_bmi_model()); + // load the state through the set value function that does not enforce the input size is the same as the current BMI's size + bmi->set_value_unchecked("serialization_state", state.data(), state.size()); +} + #endif //NGEN_WITH_PYTHON From 200c2b62ed6a21690f42ad0963a08d785e14ea3a Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 30 Jan 2026 11:11:38 -0500 Subject: [PATCH 17/93] Add output suppression flag for UEB --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d0a43acffe..9745760902 100644 --- a/Dockerfile +++ b/Dockerfile @@ -294,7 +294,8 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-soilfreezethaw \ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ueb-bmi \ set -eux && \ - cmake -B extern/ueb-bmi/cmake_build -S extern/ueb-bmi/ -DBMICXX_INCLUDE_DIRS=/ngen-app/ngen/extern/bmi-cxx/ -DBOOST_ROOT=/opt/boost && \ + cmake -B extern/ueb-bmi/cmake_build -S extern/ueb-bmi/ \ + -DUEB_SUPPRESS_OUTPUTS=ON -DBMICXX_INCLUDE_DIRS=/ngen-app/ngen/extern/bmi-cxx/ -DBOOST_ROOT=/opt/boost && \ cmake --build extern/ueb-bmi/cmake_build/ && \ find /ngen-app/ngen/extern/ueb-bmi/ -name '*.o' -exec rm -f {} + From a9fb47b80bd0db653486e6d44f0241a3d4e1e0b9 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Mon, 5 Jan 2026 11:16:21 -0800 Subject: [PATCH 18/93] update t-route submodule ref for named logger --- extern/t-route | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/t-route b/extern/t-route index b2b15a5efc..371f524f20 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit b2b15a5efc3ff2a826a750c73790bdcf8efa3c72 +Subproject commit 371f524f2085c81b1925c8e3a0fe3022b7736c31 From fb23619e577b7057d846e59ce75d91ab3a5d4069 Mon Sep 17 00:00:00 2001 From: Ian Todd <48330440+idtodd@users.noreply.github.com> Date: Tue, 6 Jan 2026 12:37:23 -0500 Subject: [PATCH 19/93] Dynamic Forcing Engine CAT-ID Data Type (NGWPC-9255) (#109) --- include/bmi/Bmi_Py_Adapter.hpp | 31 ++++++---- .../ForcingsEngineLumpedDataProvider.hpp | 4 ++ .../ForcingsEngineLumpedDataProvider.cpp | 62 +++++++++++++++---- 3 files changed, 73 insertions(+), 24 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index a8a00ec162..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -236,24 +236,31 @@ namespace models { * this might cause trouble on certain systems, since (depending on the particular sizes of types) that * could produce duplicate "case" values. */ - //TODO: include other numpy type strings, https://numpy.org/doc/stable/user/basics.types.html - if ( (py_type_name == "int" || py_type_name == "int16") && item_size == sizeof(short)) { + if (item_size == sizeof(signed char) && (py_type_name == "int8" || py_type_name == "numpy.int8")) { + return "signed char"; + } else if (item_size == sizeof(unsigned char) && (py_type_name == "uint8" || py_type_name == "numpy.uint8")) { + return "unsigned char"; + } else if (item_size == sizeof(short) && (py_type_name == "int" || py_type_name == "int16" || py_type_name == "numpy.int16")) { return "short"; - } else if ( (py_type_name == "int" || py_type_name == "int32" )&& item_size == sizeof(int)) { + } else if (item_size == sizeof(unsigned short) && (py_type_name == "uint16" || py_type_name == "numpy.uint16" || py_type_name == "ushort" || py_type_name == "numpy.ushort")) { + return "unsigned short"; + } else if (item_size == sizeof(int) && (py_type_name == "int" || py_type_name == "int32" || py_type_name == "numpy.int32" || py_type_name == "intc" || py_type_name == "numpy.intc")) { return "int"; - } else if (py_type_name == "int" && item_size == sizeof(long)) { + } else if (item_size == sizeof(unsigned int) && (py_type_name == "uint32" || py_type_name == "numpy.uint32" || py_type_name == "uintc" || py_type_name == "numpy.uintc")) { + return "unsigned int"; + } else if (item_size == sizeof(long) && (py_type_name == "int" || py_type_name == "int64" || py_type_name == "numpy.int64" || py_type_name == "long" || py_type_name == "numpy.long")) { return "long"; - } else if ( (py_type_name == "int" || py_type_name == "int64") && item_size == sizeof(long long)) { + } else if (item_size == sizeof(unsigned long) && (py_type_name == "uint64" || py_type_name == "numpy.uint64" || py_type_name == "uint32" || py_type_name == "numpy.uint32" || py_type_name == "ulong" || py_type_name == "numpy.ulong")) { + return "unsigned long"; + } else if (item_size == sizeof(long long) && (py_type_name == "int" || py_type_name == "int64" || py_type_name == "numpy.int64" || py_type_name == "longlong")) { return "long long"; - } else if (py_type_name == "longlong" && item_size == sizeof(long long)) { - return "long long"; //numpy type - } else if ( (py_type_name == "float" || py_type_name == "float32" || py_type_name == "np.float32" || - py_type_name == "numpy.float32" || py_type_name == "np.single" || py_type_name == "numpy.single") && item_size == sizeof(float)) { + } else if (item_size == sizeof(unsigned long long) && (py_type_name == "ulonglong" || py_type_name == "numpy.ulonglong")) { + return "unsigned long long"; + } else if (item_size == sizeof(float) && (py_type_name == "float" || py_type_name == "float32" || py_type_name == "numpy.float32" || py_type_name == "single" || py_type_name == "numpy.single")) { return "float"; - } else if ((py_type_name == "float" || py_type_name == "float64" || py_type_name == "np.float64" || - py_type_name == "numpy.float64") && item_size == sizeof(double)) { + } else if (item_size == sizeof(double) && (py_type_name == "float" || py_type_name == "float64" || py_type_name == "numpy.float64" || py_type_name == "double" || py_type_name == "numpy.double")) { return "double"; - } else if (py_type_name == "float" && item_size == sizeof(long double)) { + } else if (item_size == sizeof(long double) && (py_type_name == "float" || py_type_name == "float128" || py_type_name == "numpy.float128" || py_type_name == "longdouble" || py_type_name == "numpy.longdouble")) { return "long double"; } else { std::string throw_msg; throw_msg.assign( diff --git a/include/forcing/ForcingsEngineLumpedDataProvider.hpp b/include/forcing/ForcingsEngineLumpedDataProvider.hpp index 2b9db04a4a..189f9554ef 100644 --- a/include/forcing/ForcingsEngineLumpedDataProvider.hpp +++ b/include/forcing/ForcingsEngineLumpedDataProvider.hpp @@ -42,6 +42,10 @@ struct ForcingsEngineLumpedDataProvider final : private: std::size_t divide_id_; std::size_t divide_idx_; + + // Search an array of numbers for the first instance of the current `divide_id_` and set the `divide_idx_` to that index if found + template + void find_divide_id(const void *cat_id_ptr, const std::size_t size_id_dimension); }; } // namespace data_access diff --git a/src/forcing/ForcingsEngineLumpedDataProvider.cpp b/src/forcing/ForcingsEngineLumpedDataProvider.cpp index 68d746b4ce..2865072352 100644 --- a/src/forcing/ForcingsEngineLumpedDataProvider.cpp +++ b/src/forcing/ForcingsEngineLumpedDataProvider.cpp @@ -79,31 +79,56 @@ Provider::ForcingsEngineLumpedDataProvider( var_output_names_.erase(cat_id_pos); + const std::size_t cat_id_item_size = static_cast(bmi_->GetVarItemsize("CAT-ID")); const auto size_id_dimension = static_cast( - bmi_->GetVarNbytes("CAT-ID") / bmi_->GetVarItemsize("CAT-ID") + bmi_->GetVarNbytes("CAT-ID") / cat_id_item_size ); ss.str(""); ss << " CAT-ID size: " << size_id_dimension << std::endl; LOG(ss.str(), LogLevel::DEBUG); - // Copy CAT-ID values into instance vector - const auto cat_id_span = boost::span( - static_cast(bmi_->GetValuePtr("CAT-ID")), - size_id_dimension - ); + const std::string cat_id_type = bmi_->GetVarType("CAT-ID"); + const std::string cat_id_cpp_type = bmi_->get_analogous_cxx_type(cat_id_type, cat_id_item_size); + void *cat_id_ptr = bmi_->GetValuePtr("CAT-ID"); + + // locate the index of the CAT-ID in the provider + // max value of divide_idx_ assumed as an error state + divide_idx_ = std::numeric_limits::max(); + if (cat_id_cpp_type == "short") { + this->find_divide_id(cat_id_ptr, size_id_dimension); + } else if (cat_id_cpp_type == "unsigned short") { + this->find_divide_id(cat_id_ptr, size_id_dimension); + } else if (cat_id_cpp_type == "int") { + this->find_divide_id(cat_id_ptr, size_id_dimension); + } else if (cat_id_cpp_type == "unsigned int") { + this->find_divide_id(cat_id_ptr, size_id_dimension); + } else if (cat_id_cpp_type == "long") { + this->find_divide_id(cat_id_ptr, size_id_dimension); + } else if (cat_id_cpp_type == "unsigned long") { + this->find_divide_id(cat_id_ptr, size_id_dimension); + } else if (cat_id_cpp_type == "float") { + this->find_divide_id(cat_id_ptr, size_id_dimension); + } else if (cat_id_cpp_type == "double") { + this->find_divide_id(cat_id_ptr, size_id_dimension); + } else { + ss.str(""); + ss << "(ForcingEngineLumpedDataProvider) Unable to interpret CAT-ID type of C++ type '" + << cat_id_cpp_type << "' (python type '" + << cat_id_type << "')"; + std::string error = ss.str(); + LOG(error, LogLevel::FATAL); + throw std::runtime_error(error); + } - auto divide_id_pos = std::find(cat_id_span.begin(), cat_id_span.end(), divide_id_); - if (divide_id_pos == cat_id_span.end()) { + if (divide_idx_ == std::numeric_limits::max()) { ss.str(""); ss << "Unable to find divide ID `" << divide_id << "` in the given Forcings Engine domain" << std::endl; LOG(ss.str(), LogLevel::SEVERE); - divide_idx_ = static_cast(-1); } else { - divide_idx_ = std::distance(cat_id_span.begin(), divide_id_pos); ss.str(""); - ss << " Divide ID found at index: " << divide_idx_ << std::endl; - LOG(ss.str(), LogLevel::INFO); + ss << " Divide ID " << divide_id << " found at index: " << divide_idx_; + LOG(ss.str(), LogLevel::DEBUG); } ss.str(""); @@ -111,6 +136,19 @@ Provider::ForcingsEngineLumpedDataProvider( LOG(LogLevel::DEBUG, ss.str()); } +template +inline void Provider::find_divide_id(const void *cat_id_ptr, const std::size_t size_id_dimension) { + // create span for viewing that data of type T + auto cat_id_span = boost::span( + static_cast(cat_id_ptr), + size_id_dimension + ); + auto loc = std::find(cat_id_span.begin(), cat_id_span.end(), divide_id_); + if (loc != cat_id_span.end()) { + divide_idx_ = std::distance(cat_id_span.begin(), loc); + } +} + std::size_t Provider::divide() const noexcept { return divide_id_; From c9ee3f492bdaf193019e24955bb7c88691f2a902 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 8 Jan 2026 09:56:18 -0800 Subject: [PATCH 20/93] Docker updates for python ewts packages --- Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9745760902..a20732bdc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -195,7 +195,8 @@ RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ pip3 install 'pandas' && \ pip3 install 'pyyml' && \ pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - pip install /ngen-app/ngen-forcing/ + pip install /ngen-app/ngen-forcing/ && \ + pip install /ngen-app/ngen-forcing/nextgen_forcings_ewts/ WORKDIR /ngen-app/ @@ -299,6 +300,11 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ueb-bmi \ cmake --build extern/ueb-bmi/cmake_build/ && \ find /ngen-app/ngen/extern/ueb-bmi/ -name '*.o' -exec rm -f {} + +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ + set -eux; \ + cd extern/lstm; \ + pip install . ./lstm_ewts + RUN set -eux && \ mkdir --parents /ngencerf/data/ngen-run-logs/ && \ mkdir --parents /ngen-app/bin/ && \ @@ -387,11 +393,9 @@ RUN set -eux && \ mv /ngen-app/merged_git_info.json $GIT_INFO_PATH && \ rm -rf /ngen-app/submodules-json - # Extend PYTHONPATH for LSTM models (preserve venv path from ngen-bmi-forcing) +# Extend PYTHONPATH for LSTM models (preserve venv path from ngen-bmi-forcing) ENV PYTHONPATH="${PYTHONPATH}:/ngen-app/ngen/extern/lstm:/ngen-app/ngen/extern/lstm/lstm" - - WORKDIR / SHELL ["/bin/bash", "-c"] From b732ae69c3b36122e8369e44624458d90f3cdd5c Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 8 Jan 2026 11:04:01 -0800 Subject: [PATCH 21/93] Update lstm and t-route submodules for ewts package --- extern/lstm | 2 +- extern/t-route | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/lstm b/extern/lstm index 43b1e8f320..85a3301dae 160000 --- a/extern/lstm +++ b/extern/lstm @@ -1 +1 @@ -Subproject commit 43b1e8f320f6832f27a73d4b2ae03a338956684b +Subproject commit 85a3301daeff761a54b6ebda6fee7aac977a62ce diff --git a/extern/t-route b/extern/t-route index 371f524f20..917368754d 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit 371f524f2085c81b1925c8e3a0fe3022b7736c31 +Subproject commit 917368754d686137d88e0eba16d7f48d9c6fc0aa From 098c9566aab8d41940bca8482edcea8d43e4f59b Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Mon, 12 Jan 2026 00:00:23 -0800 Subject: [PATCH 22/93] update cicd file to use new branches for releases --- .github/workflows/ngwpc-cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index b879c5228d..4d0b8d2d99 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -2,9 +2,9 @@ name: CI/CD Pipeline on: pull_request: - branches: [main, nwm-main, development, release-candidate] + branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate] push: - branches: [main, nwm-main, development, release-candidate] + branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate] release: types: [published] From f0add01d646a4e8b850c743bf689f8e3a2ee8451 Mon Sep 17 00:00:00 2001 From: Ian Todd <48330440+idtodd@users.noreply.github.com> Date: Tue, 20 Jan 2026 14:33:50 -0500 Subject: [PATCH 23/93] Update submod ref for SMP --- extern/SoilMoistureProfiles/SoilMoistureProfiles | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/SoilMoistureProfiles/SoilMoistureProfiles b/extern/SoilMoistureProfiles/SoilMoistureProfiles index 86ad6f4e79..41c802cb48 160000 --- a/extern/SoilMoistureProfiles/SoilMoistureProfiles +++ b/extern/SoilMoistureProfiles/SoilMoistureProfiles @@ -1 +1 @@ -Subproject commit 86ad6f4e793a8ff83cc26ab21643fda3f9a106c3 +Subproject commit 41c802cb4862e7bd01f6081816a093135bd51282 From f4fcc78f3bfe8b6debb16acb98c10a035af8002b Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 09:59:17 -0500 Subject: [PATCH 24/93] Fix use-after-free error --- include/realizations/catchment/Formulation_Manager.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index df6599f2a5..6b8b2c0422 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -724,12 +724,14 @@ namespace realization { // Iterate over directory entries if (directory != nullptr) { + // handle closing the directory regardless of how the function returns + auto closer = [](DIR *dir){ closedir(dir); }; + std::unique_ptr _(directory, closer); while ((entry = readdir(directory))) { if (std::regex_match(entry->d_name, pattern)) { // Check for regular files and symlinks #ifdef _DIRENT_HAVE_D_TYPE if (entry->d_type == DT_REG || entry->d_type == DT_LNK) { - closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -749,7 +751,6 @@ namespace realization { } if (S_ISREG(st.st_mode)) { - closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -768,7 +769,6 @@ namespace realization { #endif } } - closedir(directory); } else { // The directory wasn't found or otherwise couldn't be opened; forcing data cannot be retrieved std::string throw_msg = "Error opening forcing data dir '" + path + "' after " + std::to_string(attemptCount) + " attempts: " + errMsg; From 193c876cab3e2c24618f82eb9aadabfdc08175d3 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:14:29 -0500 Subject: [PATCH 25/93] Remove explicit interpreter release --- src/NGen.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 8720d1f971..e7e3fe1b7b 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -800,8 +800,6 @@ int main(int argc, char* argv[]) { LOG("[TIMING]: Coastal: " + std::to_string(time_elapsed_coastal.count()), LogLevel::INFO); #endif - _interp.reset(); - auto time_done_total = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_total = time_done_total - time_start; From 03fab9e31bdcfbffda2f79a80e9b2dd84e238437 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:15:24 -0500 Subject: [PATCH 26/93] Ensure bmi_model has not been destroyed --- include/bmi/Bmi_Py_Adapter.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..8ee9014c64 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,7 +47,8 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - Finalize(); + if (bmi_model != NULL) + Finalize(); } /** From b597be4312401ff7d70e5074fa73d8b06a5cfe79 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 11 Dec 2025 12:30:47 -0500 Subject: [PATCH 27/93] Remove null check since that should be an error state if null --- include/bmi/Bmi_Py_Adapter.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 8ee9014c64..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,8 +47,7 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - if (bmi_model != NULL) - Finalize(); + Finalize(); } /** From f01bf47fdeff4d78d03c87fc8070272c3c7ac5ff Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 19 Dec 2025 09:18:15 -0500 Subject: [PATCH 28/93] Finalize forcing engine providers on instances clear --- include/forcing/ForcingsEngineDataProvider.hpp | 15 ++++++--------- .../catchment/Formulation_Manager.hpp | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index f1477f9eb5..d4ce1f957d 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -78,11 +78,14 @@ struct ForcingsEngineStorage { data_[key] = value; } - //! Clear all references to Forcings Engine instances. + //! Clear all references to Forcings Engine instances and run the Finalize methods on each BMI instance. //! @note This will not necessarily destroy the Forcings Engine instances. Since they //! are reference counted, it will only decrement their instance by one. - void clear() + void finalize() { + for (auto &provider : data_) { + provider.second->Finalize(); + } data_.clear(); } @@ -105,13 +108,7 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override - { - if (bmi_ != nullptr) { - bmi_->Finalize(); - bmi_ = nullptr; - } - } + void finalize() override = default; boost::span get_available_variable_names() const override { diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 6b8b2c0422..4006409231 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -318,7 +318,7 @@ namespace realization { data_access::NetCDFPerFeatureDataProvider::cleanup_shared_providers(); #endif #if NGEN_WITH_PYTHON - data_access::detail::ForcingsEngineStorage::instances.clear(); + data_access::detail::ForcingsEngineStorage::instances.finalize(); #endif ss.str(""); ss << "Formulation_Manager finalized" << std::endl; LOG(ss.str(), LogLevel::DEBUG); From dda7b6385108aa3801985264233e48bfacca120c Mon Sep 17 00:00:00 2001 From: Ian Todd <48330440+idtodd@users.noreply.github.com> Date: Mon, 22 Dec 2025 08:22:39 -0500 Subject: [PATCH 29/93] Release BMI reference Co-authored-by: Phil Miller - NOAA --- include/forcing/ForcingsEngineDataProvider.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index d4ce1f957d..37df90d59b 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -108,7 +108,10 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override = default; + void finalize() override + { + bmi_.reset(); + } boost::span get_available_variable_names() const override { From 82462763c2fc6e5ed1549c7b618f007489d9efc6 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 30/93] Revert "Release BMI reference" This reverts commit 7acb111f14ce6f0adeee0f4d69a0dbfbc7127055. --- include/forcing/ForcingsEngineDataProvider.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index 37df90d59b..d4ce1f957d 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -108,10 +108,7 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override - { - bmi_.reset(); - } + void finalize() override = default; boost::span get_available_variable_names() const override { From 155712a3f5a34bf8caae8a628c72d6d9a8faccd9 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 31/93] Revert "Finalize forcing engine providers on instances clear" This reverts commit db3ba59587837d26b6c9dde6a38c3ff2f4740ad0. --- include/forcing/ForcingsEngineDataProvider.hpp | 15 +++++++++------ .../catchment/Formulation_Manager.hpp | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index d4ce1f957d..f1477f9eb5 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -78,14 +78,11 @@ struct ForcingsEngineStorage { data_[key] = value; } - //! Clear all references to Forcings Engine instances and run the Finalize methods on each BMI instance. + //! Clear all references to Forcings Engine instances. //! @note This will not necessarily destroy the Forcings Engine instances. Since they //! are reference counted, it will only decrement their instance by one. - void finalize() + void clear() { - for (auto &provider : data_) { - provider.second->Finalize(); - } data_.clear(); } @@ -108,7 +105,13 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override = default; + void finalize() override + { + if (bmi_ != nullptr) { + bmi_->Finalize(); + bmi_ = nullptr; + } + } boost::span get_available_variable_names() const override { diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 4006409231..6b8b2c0422 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -318,7 +318,7 @@ namespace realization { data_access::NetCDFPerFeatureDataProvider::cleanup_shared_providers(); #endif #if NGEN_WITH_PYTHON - data_access::detail::ForcingsEngineStorage::instances.finalize(); + data_access::detail::ForcingsEngineStorage::instances.clear(); #endif ss.str(""); ss << "Formulation_Manager finalized" << std::endl; LOG(ss.str(), LogLevel::DEBUG); From 8a9b8635a28670eeb8afa6081eda5f32460e6348 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 32/93] Revert "Remove null check since that should be an error state if null" This reverts commit f4b1ce8520be147a9484a3dfe9c2276effd109a7. --- include/bmi/Bmi_Py_Adapter.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..8ee9014c64 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,7 +47,8 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - Finalize(); + if (bmi_model != NULL) + Finalize(); } /** From fb86c40c428375af7be1c8483ac144e9d8b2bd84 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 33/93] Revert "Ensure bmi_model has not been destroyed" This reverts commit c77b3e38869304a51a813f7e06e1f07011423ed0. --- include/bmi/Bmi_Py_Adapter.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 8ee9014c64..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,8 +47,7 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - if (bmi_model != NULL) - Finalize(); + Finalize(); } /** From cd5a61b7d3199a441d62f624a1be914850afa549 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 34/93] Revert "Remove explicit interpreter release" This reverts commit 82ce9bcd746465b92dae87f70cb4436d9dcc803b. --- src/NGen.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/NGen.cpp b/src/NGen.cpp index e7e3fe1b7b..8720d1f971 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -800,6 +800,8 @@ int main(int argc, char* argv[]) { LOG("[TIMING]: Coastal: " + std::to_string(time_elapsed_coastal.count()), LogLevel::INFO); #endif + _interp.reset(); + auto time_done_total = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_total = time_done_total - time_start; From 05df3537cf34b079ec350e5a4d222b94c7ab0010 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 35/93] Revert "Fix use-after-free error" This reverts commit 8869534d3820e1e962266f59fe0f20962966fcf8. --- include/realizations/catchment/Formulation_Manager.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 6b8b2c0422..df6599f2a5 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -724,14 +724,12 @@ namespace realization { // Iterate over directory entries if (directory != nullptr) { - // handle closing the directory regardless of how the function returns - auto closer = [](DIR *dir){ closedir(dir); }; - std::unique_ptr _(directory, closer); while ((entry = readdir(directory))) { if (std::regex_match(entry->d_name, pattern)) { // Check for regular files and symlinks #ifdef _DIRENT_HAVE_D_TYPE if (entry->d_type == DT_REG || entry->d_type == DT_LNK) { + closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -751,6 +749,7 @@ namespace realization { } if (S_ISREG(st.st_mode)) { + closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -769,6 +768,7 @@ namespace realization { #endif } } + closedir(directory); } else { // The directory wasn't found or otherwise couldn't be opened; forcing data cannot be retrieved std::string throw_msg = "Error opening forcing data dir '" + path + "' after " + std::to_string(attemptCount) + " attempts: " + errMsg; From 56617406b93d5248041c0bbe067b66188a3b4b80 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 09:59:17 -0500 Subject: [PATCH 36/93] Fix use-after-free error --- include/realizations/catchment/Formulation_Manager.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index df6599f2a5..6b8b2c0422 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -724,12 +724,14 @@ namespace realization { // Iterate over directory entries if (directory != nullptr) { + // handle closing the directory regardless of how the function returns + auto closer = [](DIR *dir){ closedir(dir); }; + std::unique_ptr _(directory, closer); while ((entry = readdir(directory))) { if (std::regex_match(entry->d_name, pattern)) { // Check for regular files and symlinks #ifdef _DIRENT_HAVE_D_TYPE if (entry->d_type == DT_REG || entry->d_type == DT_LNK) { - closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -749,7 +751,6 @@ namespace realization { } if (S_ISREG(st.st_mode)) { - closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -768,7 +769,6 @@ namespace realization { #endif } } - closedir(directory); } else { // The directory wasn't found or otherwise couldn't be opened; forcing data cannot be retrieved std::string throw_msg = "Error opening forcing data dir '" + path + "' after " + std::to_string(attemptCount) + " attempts: " + errMsg; From 98e3fb0acbea999c9838c6c5daaeef6e7b4c1a32 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:14:29 -0500 Subject: [PATCH 37/93] Remove explicit interpreter release --- src/NGen.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 8720d1f971..e7e3fe1b7b 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -800,8 +800,6 @@ int main(int argc, char* argv[]) { LOG("[TIMING]: Coastal: " + std::to_string(time_elapsed_coastal.count()), LogLevel::INFO); #endif - _interp.reset(); - auto time_done_total = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_total = time_done_total - time_start; From 7a75de6d097e14078c852b7792130a4c069563db Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:15:24 -0500 Subject: [PATCH 38/93] Ensure bmi_model has not been destroyed --- include/bmi/Bmi_Py_Adapter.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..8ee9014c64 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,7 +47,8 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - Finalize(); + if (bmi_model != NULL) + Finalize(); } /** From 8aa2ccd57ef6fd2dae953cc06e22ddaad9e67a91 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 11 Dec 2025 12:30:47 -0500 Subject: [PATCH 39/93] Remove null check since that should be an error state if null --- include/bmi/Bmi_Py_Adapter.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 8ee9014c64..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,8 +47,7 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - if (bmi_model != NULL) - Finalize(); + Finalize(); } /** From e173a4e362090b736d67fca741d782c8b3fd0e52 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 19 Dec 2025 09:18:15 -0500 Subject: [PATCH 40/93] Finalize forcing engine providers on instances clear --- include/forcing/ForcingsEngineDataProvider.hpp | 15 ++++++--------- .../catchment/Formulation_Manager.hpp | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index f1477f9eb5..d4ce1f957d 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -78,11 +78,14 @@ struct ForcingsEngineStorage { data_[key] = value; } - //! Clear all references to Forcings Engine instances. + //! Clear all references to Forcings Engine instances and run the Finalize methods on each BMI instance. //! @note This will not necessarily destroy the Forcings Engine instances. Since they //! are reference counted, it will only decrement their instance by one. - void clear() + void finalize() { + for (auto &provider : data_) { + provider.second->Finalize(); + } data_.clear(); } @@ -105,13 +108,7 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override - { - if (bmi_ != nullptr) { - bmi_->Finalize(); - bmi_ = nullptr; - } - } + void finalize() override = default; boost::span get_available_variable_names() const override { diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 6b8b2c0422..4006409231 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -318,7 +318,7 @@ namespace realization { data_access::NetCDFPerFeatureDataProvider::cleanup_shared_providers(); #endif #if NGEN_WITH_PYTHON - data_access::detail::ForcingsEngineStorage::instances.clear(); + data_access::detail::ForcingsEngineStorage::instances.finalize(); #endif ss.str(""); ss << "Formulation_Manager finalized" << std::endl; LOG(ss.str(), LogLevel::DEBUG); From 38a484441c37d5264863096782d5f49fd6dbe9f8 Mon Sep 17 00:00:00 2001 From: Ian Todd <48330440+idtodd@users.noreply.github.com> Date: Mon, 22 Dec 2025 08:22:39 -0500 Subject: [PATCH 41/93] Release BMI reference Co-authored-by: Phil Miller - NOAA --- include/forcing/ForcingsEngineDataProvider.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index d4ce1f957d..37df90d59b 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -108,7 +108,10 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override = default; + void finalize() override + { + bmi_.reset(); + } boost::span get_available_variable_names() const override { From dcc24cb79125b19aac0b2b6f6b54d9d377d56243 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 21 Jan 2026 10:47:37 -0500 Subject: [PATCH 42/93] Update tear down method name --- test/forcing/ForcingsEngineLumpedDataProvider_Test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/forcing/ForcingsEngineLumpedDataProvider_Test.cpp b/test/forcing/ForcingsEngineLumpedDataProvider_Test.cpp index 00863354dc..b5ba3fe3d1 100644 --- a/test/forcing/ForcingsEngineLumpedDataProvider_Test.cpp +++ b/test/forcing/ForcingsEngineLumpedDataProvider_Test.cpp @@ -71,7 +71,7 @@ void TestFixture::SetUpTestSuite() void TestFixture::TearDownTestSuite() { - data_access::detail::ForcingsEngineStorage::instances.clear(); + data_access::detail::ForcingsEngineStorage::instances.finalize(); gil_.reset(); #if NGEN_WITH_MPI From f67e220923583dce52475c869cbf5e2a50701865 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 3 Dec 2025 17:39:36 -0800 Subject: [PATCH 43/93] Catch csv file errors and log exceptions initializing formulation --- include/forcing/CsvPerFeatureForcingProvider.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/forcing/CsvPerFeatureForcingProvider.hpp b/include/forcing/CsvPerFeatureForcingProvider.hpp index 1570c67b2e..25368b8d9e 100644 --- a/include/forcing/CsvPerFeatureForcingProvider.hpp +++ b/include/forcing/CsvPerFeatureForcingProvider.hpp @@ -423,7 +423,7 @@ class CsvPerFeatureForcingProvider : public data_access::GenericDataProvider available_forcings_units[var_name] = units; } else { - std::string msg = "Forcing file " + file_name + " is missing a column header name"; + std::string msg = "Forcing file " + file_name + " is missing column header names"; LOG(msg, LogLevel::FATAL); throw std::runtime_error(msg); } From f5795bd2652a1963d1aaa2d33d9e8498763926ab Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 4 Dec 2025 09:32:55 -0800 Subject: [PATCH 44/93] Add check for empty CSV forcing file. --- include/forcing/CsvPerFeatureForcingProvider.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/forcing/CsvPerFeatureForcingProvider.hpp b/include/forcing/CsvPerFeatureForcingProvider.hpp index 25368b8d9e..1570c67b2e 100644 --- a/include/forcing/CsvPerFeatureForcingProvider.hpp +++ b/include/forcing/CsvPerFeatureForcingProvider.hpp @@ -423,7 +423,7 @@ class CsvPerFeatureForcingProvider : public data_access::GenericDataProvider available_forcings_units[var_name] = units; } else { - std::string msg = "Forcing file " + file_name + " is missing column header names"; + std::string msg = "Forcing file " + file_name + " is missing a column header name"; LOG(msg, LogLevel::FATAL); throw std::runtime_error(msg); } From e4d8303738b1b5fc17ad617e3a6928bba0fbbe43 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Tue, 20 Jan 2026 18:06:38 -0800 Subject: [PATCH 45/93] Update submodule references for LASAM, SFT, SMP, CFE, LSTM, Snow17, T-Route, ueb_bmi --- extern/LASAM | 2 +- extern/SoilFreezeThaw/SoilFreezeThaw | 2 +- extern/SoilMoistureProfiles/SoilMoistureProfiles | 2 +- extern/cfe/cfe | 2 +- extern/lstm | 2 +- extern/snow17 | 2 +- extern/t-route | 2 +- extern/ueb-bmi | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/extern/LASAM b/extern/LASAM index 88f18a5adc..664f5fbd38 160000 --- a/extern/LASAM +++ b/extern/LASAM @@ -1 +1 @@ -Subproject commit 88f18a5adc0b34fc04e749f26227da6a42aaeb95 +Subproject commit 664f5fbd3869e51a405afa9aaf6cb4dcc96e09b9 diff --git a/extern/SoilFreezeThaw/SoilFreezeThaw b/extern/SoilFreezeThaw/SoilFreezeThaw index ab641a8209..e7fd675f07 160000 --- a/extern/SoilFreezeThaw/SoilFreezeThaw +++ b/extern/SoilFreezeThaw/SoilFreezeThaw @@ -1 +1 @@ -Subproject commit ab641a820920acb788dc47513a1e0ccbf31483c2 +Subproject commit e7fd675f074e4755dc9953a7ae68668064cd7de6 diff --git a/extern/SoilMoistureProfiles/SoilMoistureProfiles b/extern/SoilMoistureProfiles/SoilMoistureProfiles index 41c802cb48..705798948d 160000 --- a/extern/SoilMoistureProfiles/SoilMoistureProfiles +++ b/extern/SoilMoistureProfiles/SoilMoistureProfiles @@ -1 +1 @@ -Subproject commit 41c802cb4862e7bd01f6081816a093135bd51282 +Subproject commit 705798948d899f7a05a083141cc04c189684aa1c diff --git a/extern/cfe/cfe b/extern/cfe/cfe index 33571ec2ef..4dfd64f43b 160000 --- a/extern/cfe/cfe +++ b/extern/cfe/cfe @@ -1 +1 @@ -Subproject commit 33571ec2eff06f6c5bc0f8ee7b6d9d02f33be147 +Subproject commit 4dfd64f43bdd851affff540b6e2a9032ef301be9 diff --git a/extern/lstm b/extern/lstm index 85a3301dae..ce43783660 160000 --- a/extern/lstm +++ b/extern/lstm @@ -1 +1 @@ -Subproject commit 85a3301daeff761a54b6ebda6fee7aac977a62ce +Subproject commit ce43783660642f9952147a76b81b4cbd4e5c3ad4 diff --git a/extern/snow17 b/extern/snow17 index 10c2510bfa..dbcfd09da7 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit 10c2510bfa45743a3828ea0fc890f79974b48390 +Subproject commit dbcfd09da7602bfdb860a2f9ccb770af95fb4b36 diff --git a/extern/t-route b/extern/t-route index 917368754d..ec6051374e 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit 917368754d686137d88e0eba16d7f48d9c6fc0aa +Subproject commit ec6051374ef5a8090571a20dffdcecdd2a427db3 diff --git a/extern/ueb-bmi b/extern/ueb-bmi index 299976367a..627787f3d9 160000 --- a/extern/ueb-bmi +++ b/extern/ueb-bmi @@ -1 +1 @@ -Subproject commit 299976367a5329602fc1443f932e9cbf6de4ace6 +Subproject commit 627787f3d97b789f65f27fc415476bd1e0fb60af From fce61886d5d00b696075ca4072e5a6e34bec8beb Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:15:24 -0500 Subject: [PATCH 46/93] Ensure bmi_model has not been destroyed --- include/bmi/Bmi_Py_Adapter.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..8ee9014c64 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,7 +47,8 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - Finalize(); + if (bmi_model != NULL) + Finalize(); } /** From 63556dcf6c92cd2a19b5aac98916c9dfa9530ab3 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 11 Dec 2025 12:30:47 -0500 Subject: [PATCH 47/93] Remove null check since that should be an error state if null --- include/bmi/Bmi_Py_Adapter.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 8ee9014c64..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,8 +47,7 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - if (bmi_model != NULL) - Finalize(); + Finalize(); } /** From adba13c0b40aa2c4c09e58d9d068318f88c5a7b3 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 19 Dec 2025 09:18:15 -0500 Subject: [PATCH 48/93] Finalize forcing engine providers on instances clear --- extern/t-route | 2 +- extern/ueb-bmi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/t-route b/extern/t-route index ec6051374e..bae205ff72 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit ec6051374ef5a8090571a20dffdcecdd2a427db3 +Subproject commit bae205ff723b669bba43c7fa353451c5fbe1eb0d diff --git a/extern/ueb-bmi b/extern/ueb-bmi index 627787f3d9..e661f7afe4 160000 --- a/extern/ueb-bmi +++ b/extern/ueb-bmi @@ -1 +1 @@ -Subproject commit 627787f3d97b789f65f27fc415476bd1e0fb60af +Subproject commit e661f7afe456bb380853abdc941d0cfa21df5c5e From b744ee10ddf7dc57fb33bcdb5e0672f904913e4c Mon Sep 17 00:00:00 2001 From: Ian Todd <48330440+idtodd@users.noreply.github.com> Date: Fri, 23 Jan 2026 08:59:39 -0500 Subject: [PATCH 49/93] Ensure BMIs Destroyed Before Calling MPI_Finalize (#123) --- src/NGen.cpp | 78 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index e7e3fe1b7b..bafd5d6389 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -168,7 +168,7 @@ void write_nexus_outflow_csv_files(std::string const& output_root, } } -int main(int argc, char* argv[]) { +int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::string catchmentDataFile = ""; std::string nexusDataFile = ""; std::string REALIZATION_CONFIG_PATH = ""; @@ -176,18 +176,7 @@ int main(int argc, char* argv[]) { std::string PARTITION_PATH = ""; std::stringstream ss(""); - // This default value should lead to behavior matching the single-process case in the standalone or non-MPI case - int mpi_num_procs = 1; - // Define in the non-MPI case so that we don't need to conditionally compile `if (mpi_rank == 0)` - int mpi_rank = 0; - if (argc > 1 && std::string{argv[1]} == "--info") { -#if NGEN_WITH_MPI - MPI_Init(nullptr, nullptr); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_num_procs); -#endif - if (mpi_rank == 0) { std::ostringstream output; output << ngen::exec_info::build_summary; @@ -203,11 +192,7 @@ int main(int argc, char* argv[]) { ss.str(""); } // if (mpi_rank == 0) -#if NGEN_WITH_MPI - MPI_Finalize(); -#endif - - exit(1); + return 1; } auto time_start = std::chrono::steady_clock::now(); @@ -284,8 +269,8 @@ int main(int argc, char* argv[]) { ss << " VIRTUAL_ENV environment variable: " << (std::getenv("VIRTUAL_ENV") == nullptr ? "(not set)" : std::getenv("VIRTUAL_ENV")) << std::endl; - ss << " Discovered venv: " << _interp->getDiscoveredVenvPath() << std::endl; - auto paths = _interp->getSystemPath(); + ss << " Discovered venv: " << utils::ngenPy::InterpreterUtil::getInstance()->getDiscoveredVenvPath() << std::endl; + auto paths = utils::ngenPy::InterpreterUtil::getInstance()->getSystemPath(); ss << " System paths:" << std::endl; for (std::string& path : std::get<1>(paths)) { if (!path.empty()) { @@ -297,7 +282,7 @@ int main(int argc, char* argv[]) { std::cout << ss.str() << std::endl; LOG(ss.str(), LogLevel::INFO); ss.str(""); - exit(0); // Unsure if this path should have a non-zero exit code? + return 0; // Unsure if this path should have a non-zero exit code? } else if (argc < 6) { ss << "Missing required args:" << std::endl; @@ -316,19 +301,13 @@ int main(int argc, char* argv[]) { ss.str(""); } - exit(-1); + return -1; } else { catchmentDataFile = argv[1]; nexusDataFile = argv[3]; REALIZATION_CONFIG_PATH = argv[5]; #if NGEN_WITH_MPI - - // Initalize MPI - MPI_Init(NULL, NULL); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_num_procs); - if (argc >= 7) { LOG("argc >= 7", LogLevel::INFO); ss.str(""); @@ -337,7 +316,7 @@ int main(int argc, char* argv[]) { ss << "Missing required argument for partition file path." << std::endl; LOG(ss.str(), LogLevel::WARNING); ss.str(""); - exit(-1); + return -1; } if (argc >= 8) { @@ -350,7 +329,7 @@ int main(int argc, char* argv[]) { << std::endl; LOG(ss.str(), LogLevel::WARNING); ss.str(""); - exit(-1); + return -1; } } #endif // NGEN_WITH_MPI @@ -410,7 +389,7 @@ int main(int argc, char* argv[]) { #endif // NGEN_WITH_MPI if (error) - exit(-1); + return -1; // split the subset strings into vectors boost::split(catchment_subset_ids, argv[2], [](char c) { return c == ','; }); @@ -822,9 +801,46 @@ int main(int argc, char* argv[]) { ss.str(""); } + return 0; +} + +/** + * This function acts as a wrapper around the main executing body of NGEN. + * Currently, the end of the `run_ngen` triggers the destruction of the catchment BMIs. + * A future improvement would be to turn `run_ngen` into `main` and have a cleaner ownership model of the BMIs + * so they can be finalized explicitly instead of when their `shared_ptr` reference count goes to zero. + */ +int main(int argc, char* argv[]) { + // This default value should lead to behavior matching the single-process case in the standalone or non-MPI case + int mpi_num_procs = 1; + // Define in the non-MPI case so that we don't need to conditionally compile `if (mpi_rank == 0)` + int mpi_rank = 0; + +#if NGEN_WITH_MPI + // initialize MPI if needed + MPI_Init(nullptr, nullptr); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_num_procs); +#endif + +#if NGEN_WITH_PYTHON + // Start Python interpreter via the manager singleton + // Need to bind to a variable so that the underlying reference count + // is incremented, this essentially becomes the global reference to keep + // the interpreter alive till the end of `main` + auto _interp = utils::ngenPy::InterpreterUtil::getInstance(); +#endif // NGEN_WITH_PYTHON + + int result = run_ngen(argc, argv, mpi_num_procs, mpi_rank); + +#if NGEN_WITH_PYTHON + // explicitly destroy the interpreter before calling MPI_Finalize + _interp.reset(); +#endif + #if NGEN_WITH_MPI MPI_Finalize(); #endif - return 0; + return result; } From 3fc428f09401459e75067e8212167bd1a4df98ca Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 30 Jan 2026 15:46:29 -0500 Subject: [PATCH 50/93] Ensure data ownership not passed to python --- include/bmi/Bmi_Py_Adapter.hpp | 10 +++++++++- src/bmi/Bmi_Py_Adapter.cpp | 6 ------ src/realizations/catchment/Bmi_Py_Formulation.cpp | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..3e956a41d8 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -599,13 +599,21 @@ namespace models { /** * Set the value of a variable. This version of setting a variable will send an array with the `size` specified instead of checking the BMI for its current size of the variable. + * Ownership of the pointer will remain in C++, so the consuming BMI should not maintain a reference to the values beyond the scope of its `set_value` method. * * @param name The name of the BMI variable. * @param src Pointer to the data that will be sent to the BMI. * @param size The number of items represented by the pointer. */ template - void set_value_unchecked(const std::string &name, T *src, size_t size); + void set_value_unchecked(const std::string &name, T *src, size_t size) { + // declare readonly array info with the pointer and size + py::buffer_info info(src, static_cast(size), true); + // create the array with the info and NULL handler so python doesn't take ownership + py::array_t src_array(info, nullptr); + // pass the array to python to read; the BMI should not attempt to maintain a reference beyond the scope of this function to prevent trying to use freed memory + bmi_model->attr("set_value")(name, src_array); + } /** * Set values for a model's BMI variable at specified indices. diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index ee79de4e14..7436e2b48c 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -223,10 +223,4 @@ void Bmi_Py_Adapter::UpdateUntil(double time) { bmi_model->attr("update_until")(time); } -template -void Bmi_Py_Adapter::set_value_unchecked(const std::string &name, T *src, size_t size) { - py::array_t src_array(size, src); - bmi_model->attr("set_value")(name, src_array); -} - #endif //NGEN_WITH_PYTHON diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index e804aa5882..a3dc65df1e 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -120,7 +120,7 @@ bool Bmi_Py_Formulation::is_model_initialized() const { void Bmi_Py_Formulation::load_serialization_state(const boost::span state) const { auto bmi = std::dynamic_pointer_cast(get_bmi_model()); // load the state through the set value function that does not enforce the input size is the same as the current BMI's size - bmi->set_value_unchecked("serialization_state", state.data(), state.size()); + bmi->set_value_unchecked("serialization_state", state.data(), state.size()); } #endif //NGEN_WITH_PYTHON From a5d2428aef01655745407505ba72f0d02be5d266 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 4 Feb 2026 13:02:51 -0500 Subject: [PATCH 51/93] T-Route save stating for hot start --- include/bmi/Bmi_Py_Adapter.hpp | 33 +++-- include/core/Layer.hpp | 1 + include/core/NgenSimulation.hpp | 21 ++++ .../catchment/Bmi_Formulation.hpp | 9 ++ .../catchment/Bmi_Module_Formulation.hpp | 2 + .../catchment/Bmi_Multi_Formulation.hpp | 2 + .../state_save_restore/State_Save_Restore.hpp | 14 ++- src/NGen.cpp | 35 +++--- src/bmi/Bmi_Py_Adapter.cpp | 30 +++-- src/core/Layer.cpp | 11 ++ src/core/NgenSimulation.cpp | 115 +++++++++++++++--- .../catchment/Bmi_Module_Formulation.cpp | 6 + .../catchment/Bmi_Multi_Formulation.cpp | 91 +++++++------- src/state_save_restore/File_Per_Unit.cpp | 9 +- src/state_save_restore/State_Save_Restore.cpp | 13 ++ 15 files changed, 286 insertions(+), 106 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 3e956a41d8..60d97be394 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -542,24 +542,32 @@ namespace models { std::string py_type = GetVarType(name); std::string cxx_type = get_analogous_cxx_type(py_type, (size_t) itemSize); - if (cxx_type == "short") { - set_value(name, (short *) src); + if (cxx_type == "signed char") { + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned char") { + this->set_value(name, static_cast(src)); + } else if (cxx_type == "short") { + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned short") { + this->set_value(name, static_cast(src)); } else if (cxx_type == "int") { - set_value(name, (int *) src); + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned int") { + this->set_value(name, static_cast(src)); } else if (cxx_type == "long") { - set_value(name, (long *) src); + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned long") { + this->set_value(name, static_cast(src)); } else if (cxx_type == "long long") { - //FIXME this gets dicey -- if a python numpy array is of type np.int64 (long long), - //but a c++ int* is passed to this function as src, it will fail in undefined ways... - //the template type overload may be perferred for doing SetValue from framework components - //such as forcing providers... - set_value(name, (long long *) src); + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned long long") { + this->set_value(name, static_cast(src)); } else if (cxx_type == "float") { - set_value(name, (float *) src); + this->set_value(name, static_cast(src)); } else if (cxx_type == "double") { - set_value(name, (double *) src); + this->set_value(name, static_cast(src)); } else if (cxx_type == "long double") { - set_value(name, (long double *) src); + this->set_value(name, static_cast(src)); } else { std::string throw_msg; throw_msg.assign("Bmi_Py_Adapter cannot set values for variable '" + name + "' that has unrecognized C++ type '" + cxx_type + "'"); @@ -567,7 +575,6 @@ namespace models { throw std::runtime_error(throw_msg); } } - /** * Set the values of the given BMI variable for the model, sourcing new data from the provided vector. * diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index ab4a0e4268..9a21401fb2 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -115,6 +115,7 @@ namespace ngen virtual void save_state_snapshot(std::shared_ptr snapshot_saver); virtual void load_state_snapshot(std::shared_ptr snapshot_loader); + virtual void load_hot_start(std::shared_ptr snapshot_loader); protected: diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index c29280d719..3189045edc 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -15,6 +15,10 @@ namespace hy_features class State_Snapshot_Saver; class State_Snapshot_Loader; +#if NGEN_WITH_ROUTING +#include "bmi/Bmi_Py_Adapter.hpp" +#endif // NGEN_WITH_ROUTING + #include #include #include @@ -51,6 +55,9 @@ class NgenSimulation */ void run_catchments(); + // Tear down of any items stored on the NgenSimulation object that could throw errors and, thus, should be kept separate from the deconstructor. + void finalize(); + /** * Run t-route on the stored nexus outflow values for the full configured duration of the simulation */ @@ -64,6 +71,14 @@ class NgenSimulation void save_state_snapshot(std::shared_ptr snapshot_saver); void load_state_snapshot(std::shared_ptr snapshot_loader); + /** + * Saves a snapshot state that's intended to be run at the end of a simulation. + * + * This version of saving will include T-Route BMI data and exclude the nexus outflow data stored during the catchment processing. + */ + void save_end_of_run(std::shared_ptr snapshot_saver); + // Load a snapshot of the end of a previous run. This will create a T-Route python adapter if the loader finds a unit for it and the config path is not empty. + void load_hot_start(std::shared_ptr snapshot_loader, const std::string &t_route_config_file_with_path); private: void advance_models_one_output_step(); @@ -80,6 +95,12 @@ class NgenSimulation std::vector catchment_outflows_; std::unordered_map nexus_indexes_; std::vector nexus_downstream_flows_; +#if NGEN_WITH_ROUTING + std::unique_ptr py_troute_; +#endif // NGEN_WITH_ROUTING + void make_troute(const std::string &t_route_config_file_with_path); + + std::string unit_name() const; int mpi_rank_; int mpi_num_procs_; diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index cc1cec0de6..ad2009f150 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -86,6 +86,15 @@ namespace realization { */ virtual void load_state(std::shared_ptr loader) const = 0; + /** + * Passes a serialized representation of the model's state to ``loader`` + * + * Asks saver to find data for the BMI and passes that data to the BMI for loading. + * + * Differes from `load_state` by also restting the internal time value to its initial state. + */ + virtual void load_hot_start(std::shared_ptr loader) const = 0; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index adb3423a06..aa454d12b9 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -54,6 +54,8 @@ namespace realization { void load_state(std::shared_ptr loader) const override; + void load_hot_start(std::shared_ptr loader) const override; + /** * Get the collection of forcing output property names this instance can provide. * diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index cb7aace479..3b7e09fc04 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -51,6 +51,8 @@ namespace realization { void load_state(std::shared_ptr loader) const override; + void load_hot_start(std::shared_ptr loader) const override; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 8f4b1b77e5..f311a5cfd4 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -58,6 +58,13 @@ class State_Save_Config */ std::vector>> end_of_run_savers() const; + /** + * Get state loader that is intended to be performed before catchment processing starts. + * + * The returned pointer may be NULL if no configuration was made for existing data. + */ + std::unique_ptr hot_start() const; + struct instance { instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); @@ -165,10 +172,15 @@ class State_Snapshot_Loader State_Snapshot_Loader() = default; virtual ~State_Snapshot_Loader() = default; + /** + * Check if data of a unit name exists. + */ + virtual bool has_unit(const std::string &unit_name) = 0; + /** * Load data from whatever source, and pass it to @param unit_loader->load() */ - virtual void load_unit(std::string const& unit_name, std::vector &data) = 0; + virtual void load_unit(const std::string &unit_name, std::vector &data) = 0; /** * Execute logic to complete the saving process diff --git a/src/NGen.cpp b/src/NGen.cpp index bafd5d6389..398adc1041 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -699,23 +699,17 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); - for (const auto& start_loader : state_saving_config.start_of_run_loaders()) { - LOG(LogLevel::INFO, "Loading start of run simulation data from state saving config " + start_loader.first); - std::shared_ptr snapshot_loader = start_loader.second->initialize_snapshot(State_Saver::snapshot_time_now()); - simulation->load_state_snapshot(snapshot_loader); + { // optionally run hot start loader if set in state saving config + auto hot_start_loader = state_saving_config.hot_start(); + if (hot_start_loader) { + LOG(LogLevel::INFO, "Loading hot start data from prior snapshot."); + std::shared_ptr snapshot_loader = hot_start_loader->initialize_snapshot(State_Saver::snapshot_time_now()); + simulation->load_hot_start(snapshot_loader, manager->get_t_route_config_file_with_path()); + } } simulation->run_catchments(); - for (const auto& end_saver : state_saving_config.end_of_run_savers()) { - LOG(LogLevel::INFO, "Saving end of run simulation data for state saving config " + end_saver.first); - std::shared_ptr snapshot = end_saver.second->initialize_snapshot( - State_Saver::snapshot_time_now(), - State_Saver::State_Durability::strict - ); - simulation->save_state_snapshot(snapshot); - } - #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif @@ -751,8 +745,6 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::chrono::duration time_elapsed_nexus_output = time_done_nexus_output - time_done_simulation; LOG("[TIMING]: Nexus outflow file writing: " + std::to_string(time_elapsed_nexus_output.count()), LogLevel::INFO); - manager->finalize(); - #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif @@ -779,6 +771,19 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { LOG("[TIMING]: Coastal: " + std::to_string(time_elapsed_coastal.count()), LogLevel::INFO); #endif + // run any end-of-run state saving after T-Route has finished but before starting to tear down data structures + for (const auto& end_saver : state_saving_config.end_of_run_savers()) { + LOG(LogLevel::INFO, "Saving end of run simulation data for state saving config " + end_saver.first); + std::shared_ptr snapshot = end_saver.second->initialize_snapshot( + State_Saver::snapshot_time_now(), + State_Saver::State_Durability::strict + ); + simulation->save_end_of_run(snapshot); + } + + simulation->finalize(); + manager->finalize(); + auto time_done_total = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_total = time_done_total - time_start; diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index 7436e2b48c..d1a1c6f681 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -104,25 +104,35 @@ void Bmi_Py_Adapter::GetValue(std::string name, void *dest) { msg += e.what(); Logger::logMsgAndThrowError(msg); } - - if (cxx_type == "short") { - copy_to_array(name, (short *) dest); + if (cxx_type == "signed char") { + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned char") { + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "short") { + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned short") { + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "int") { - copy_to_array(name, (int *) dest); + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned int") { + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "long") { - copy_to_array(name, (long *) dest); + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned long") { + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "long long") { - copy_to_array(name, (long long *) dest); + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned long long") { + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "float") { - copy_to_array(name, (float *) dest); + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "double") { - copy_to_array(name, (double *) dest); + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "long double") { - copy_to_array(name, (long double *) dest); + this->copy_to_array(name, static_cast(dest)); } else { Logger::logMsgAndThrowError("Bmi_Py_Adapter can't get value of unsupported type: " + cxx_type); } - } void Bmi_Py_Adapter::GetValueAtIndices(std::string name, void *dest, int *inds, int count) { diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 4d66dc069b..4f927d6523 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -106,3 +106,14 @@ void ngen::Layer::load_state_snapshot(std::shared_ptr sna r_c->load_state(snapshot_loader); } } + +void ngen::Layer::load_hot_start(std::shared_ptr snapshot_loader) +{ + // XXX Handle any of this class's own state as a meta-data unit + + for (auto const& id : processing_units) { + auto r = features.catchment_at(id); + auto r_c = std::dynamic_pointer_cast(r); + r_c->load_hot_start(snapshot_loader); + } +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 7742e3a057..3000599634 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -8,12 +8,14 @@ #include "HY_Features.hpp" #endif -#if NGEN_WITH_ROUTING -#include "bmi/Bmi_Py_Adapter.hpp" -#endif // NGEN_WITH_ROUTING - +#include "state_save_restore/State_Save_Restore.hpp" #include "parallel_utils.h" +namespace { + const auto NGEN_UNIT_NAME = "ngen"; + const auto TROUTE_UNIT_NAME = "troute"; +} + NgenSimulation::NgenSimulation( Simulation_Time const& sim_time, std::vector> layers, @@ -54,6 +56,15 @@ void NgenSimulation::run_catchments() } } +void NgenSimulation::finalize() { +#if NGEN_WITH_ROUTING + if (this->py_troute_) { + this->py_troute_->Finalize(); + this->py_troute_.reset(); + } +#endif // NGEN_WITH_ROUTING +} + void NgenSimulation::advance_models_one_output_step() { // The Inner loop will advance all layers unless doing so will break one of two constraints @@ -110,19 +121,90 @@ void NgenSimulation::advance_models_one_output_step() void NgenSimulation::save_state_snapshot(std::shared_ptr snapshot_saver) { - + // TODO: save the current nexus data + auto unit_name = this->unit_name(); // XXX Handle self, then recursively pass responsibility to Layers for (auto& layer : layers_) { layer->save_state_snapshot(snapshot_saver); } } +void NgenSimulation::save_end_of_run(std::shared_ptr snapshot_saver) +{ + for (auto& layer : layers_) { + layer->save_state_snapshot(snapshot_saver); + } +#if NGEN_WITH_ROUTING + if (this->mpi_rank_ == 0 && this->py_troute_) { + uint64_t serialization_size; + this->py_troute_->SetValue("serialization_create", &serialization_size); + this->py_troute_->GetValue("serialization_size", &serialization_size); + void *troute_state = this->py_troute_->GetValuePtr("serialization_state"); + boost::span span(static_cast(troute_state), serialization_size); + snapshot_saver->save_unit(TROUTE_UNIT_NAME, span); + this->py_troute_->SetValue("serialization_free", &serialization_size); + } +#endif // NGEN_WITH_ROUTING +} + void NgenSimulation::load_state_snapshot(std::shared_ptr snapshot_loader) { + // TODO: load the state data related to nexus outflows + auto unit_name = this->unit_name(); for (auto& layer : layers_) { layer->load_state_snapshot(snapshot_loader); } } +void NgenSimulation::load_hot_start(std::shared_ptr snapshot_loader, const std::string &t_route_config_file_with_path) { + for (auto& layer : layers_) { + layer->load_hot_start(snapshot_loader); + } +#if NGEN_WITH_ROUTING + if (this->mpi_rank_ == 0) { + bool config_file_set = !t_route_config_file_with_path.empty(); + bool snapshot_exists = snapshot_loader->has_unit(TROUTE_UNIT_NAME); + if (config_file_set && snapshot_exists) { + LOG(LogLevel::DEBUG, "Loading T-Route data from snapshot."); + std::vector troute_data; + snapshot_loader->load_unit(TROUTE_UNIT_NAME, troute_data); + if (py_troute_ == NULL) { + this->make_troute(t_route_config_file_with_path); + } + py_troute_->set_value_unchecked("serialization_state", troute_data.data(), troute_data.size()); + double rt; // unused by the BMI but needed for messaging + py_troute_->SetValue("reset_time", &rt); + } else if (!config_file_set && !snapshot_exists) { + LOG(LogLevel::DEBUG, "No data set for loading T-Route."); + } else if (config_file_set && !snapshot_exists) { + LOG(LogLevel::WARNING, "A T-Route config file was provided but the load data does not contain T-Route data. T-Route will be run as a cold start."); + } else if (!config_file_set && snapshot_exists) { + LOG(LogLevel::WARNING, "A T-Route hot start snapshot exists but no config file was provided. T-Route will not be loaded or run,"); + } + } +#endif // NGEN_WITH_ROUTING +} + + +void NgenSimulation::make_troute(const std::string &t_route_config_file_with_path) { +#if NGEN_WITH_ROUTING + this->py_troute_ = std::make_unique( + "T-Route", + t_route_config_file_with_path, + "troute_nwm_bmi.troute_bmi.BmiTroute", + true + ); +#endif // NGEN_WITH_ROUTING +} + + +std::string NgenSimulation::unit_name() const { +#if NGEN_WITH_MPI + return "ngen_" + std::to_string(this->mpi_rank_); +#else + return "ngen_0"; +#endif // NGEN_WITH_MPI +} + int NgenSimulation::get_nexus_index(std::string const& nexus_id) const { @@ -219,14 +301,12 @@ void NgenSimulation::run_routing(NgenSimulation::hy_features_t &features, std::s int delta_time = sim_time_->get_output_interval_seconds(); // model for routing - models::bmi::Bmi_Py_Adapter py_troute("T-Route", t_route_config_file_with_path, "troute_nwm_bmi.troute_bmi.BmiTroute", true); + if (this->py_troute_ == NULL) { + this->make_troute(t_route_config_file_with_path); + } - // tell BMI to resize nexus containers - int64_t nexus_count = routing_nexus_indexes->size(); - py_troute.SetValue("land_surface_water_source__volume_flow_rate__count", &nexus_count); - py_troute.SetValue("land_surface_water_source__id__count", &nexus_count); // set up nexus id indexes - std::vector nexus_df_index(nexus_count); + std::vector nexus_df_index(routing_nexus_indexes->size()); for (const auto& key_value : *routing_nexus_indexes) { int id_index = key_value.second; @@ -244,14 +324,11 @@ void NgenSimulation::run_routing(NgenSimulation::hy_features_t &features, std::s } nexus_df_index[id_index] = id_as_int; } - py_troute.SetValue("land_surface_water_source__id", nexus_df_index.data()); - for (int i = 0; i < number_of_timesteps; ++i) { - py_troute.SetValue("land_surface_water_source__volume_flow_rate", - routing_nexus_downflows->data() + (i * nexus_count)); - py_troute.Update(); - } - // Finalize will write the output file - py_troute.Finalize(); + // use unchecked messaging to allow the BMI to change its container size + py_troute_->set_value_unchecked("land_surface_water_source__id", nexus_df_index.data(), nexus_df_index.size()); + py_troute_->set_value_unchecked("land_surface_water_source__volume_flow_rate", routing_nexus_downflows->data(), routing_nexus_downflows->size()); + // run the T-Route model and create outputs through Update + py_troute_->Update(); } #endif // NGEN_WITH_ROUTING } diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index b9dc6f2452..c0c29ce418 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -35,6 +35,12 @@ namespace realization { this->load_serialization_state(data); } + void Bmi_Module_Formulation::load_hot_start(std::shared_ptr loader) const { + this->load_state(loader); + double rt; + this->get_bmi_model()->SetValue("reset_time", &rt); + } + boost::span Bmi_Module_Formulation::get_available_variable_names() const { return available_forcings; } diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index b44715a093..b6f91b8b09 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -21,36 +21,38 @@ using namespace realization; -void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { +namespace { + // Check if the system's byte order is little endianness + constexpr bool is_little_endian() { #if (__cplusplus < 202002L) - // get system endianness - uint16_t endian_bytes = 0xFF00; - uint8_t *endian_bits = reinterpret_cast(&endian_bytes); - bool is_little_endian = endian_bits[0] == 0; + // C++ less than 2020 requires making a two byte object and checking which byte is 0 + uint16_t endian_bytes = 0xFF00; + uint8_t *endian_bits = reinterpret_cast(&endian_bytes); + return endian_bits[0] == 0; +#else + // C++ 2020+ has as simpler method + return std::endian::native == std::endian::little; #endif + } +} + +void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { std::vector> bmi_data; size_t data_size = 0; - // TODO: something more elegant than just skipping sloth for (const nested_module_ptr &m : modules) { auto bmi = dynamic_cast(m.get()); - if (bmi->get_model_type_name() != "bmi_c++_sloth") { - boost::span span = bmi->get_serialization_state(); - bmi_data.push_back(std::make_pair(span.data(), span.size())); - data_size += sizeof(uint64_t) + span.size(); - LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); - } + boost::span span = bmi->get_serialization_state(); + bmi_data.push_back(std::make_pair(span.data(), span.size())); + data_size += sizeof(uint64_t) + span.size(); + LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); } char *data = new char[data_size]; size_t index = 0; for (const auto &bmi : bmi_data) { // write the size of the data -#if (__cplusplus < 202002L) - if (is_little_endian) { -#else - if constexpr (std::endian::native == std::endian::little) { -#endif + if (is_little_endian()) { std::memcpy(&data[index], &bmi.second, sizeof(uint64_t)); } else { // store the size bytes in reverse order to ensure saved data is always little endian @@ -70,46 +72,41 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav delete[] data; for (const nested_module_ptr &m : modules) { auto bmi = static_cast(m.get()); - if (bmi->get_model_type_name() != "bmi_c++_sloth") { - bmi->free_serialization_state(); - } + bmi->free_serialization_state(); } } void Bmi_Multi_Formulation::load_state(std::shared_ptr loader) const { -#if (__cplusplus < 202002L) - // get system endianness - uint16_t endian_bytes = 0xFF00; - uint8_t *endian_bits = reinterpret_cast(&endian_bytes); - bool is_little_endian = endian_bits[0] == 0; -#endif std::vector data; loader->load_unit(this->get_id(), data); size_t index = 0; for (const nested_module_ptr &m : modules) { auto bmi = dynamic_cast(m.get()); - if (bmi->get_model_type_name() != "bmi_c++_sloth") { - uint64_t size; -#if (__cplusplus < 202002L) - if (is_little_endian) { -#else - if constexpr (std::endian::native == std::endian::little) { -#endif - memcpy(&size, data.data() + index, sizeof(uint64_t)); - } else { - // read size bytes in reverse order to interpret from little endian - char *size_bytes = reinterpret_cast(&size); - size_t endian_index = sizeof(uint64_t); - for (size_t i = 0; i < sizeof(uint64_t); ++i) { - size_bytes[--endian_index] = data[index + i]; - } + uint64_t size; + if (is_little_endian()) { + memcpy(&size, data.data() + index, sizeof(uint64_t)); + } else { + // read size bytes in reverse order to interpret from little endian + char *size_bytes = reinterpret_cast(&size); + size_t endian_index = sizeof(uint64_t); + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + size_bytes[--endian_index] = data[index + i]; } - boost::span span(data.data() + index + sizeof(uint64_t), size); - bmi->load_serialization_state(span); - index += sizeof(uint64_t) + size; - LOG(LogLevel::DEBUG, "Loading of multi-BMI %s %s completed with a size of %d bytes.", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); } + boost::span span(data.data() + index + sizeof(uint64_t), size); + bmi->load_serialization_state(span); + index += sizeof(uint64_t) + size; + LOG(LogLevel::DEBUG, "Loading of multi-BMI %s %s completed with a size of %d bytes.", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); + } +} + +void Bmi_Multi_Formulation::load_hot_start(std::shared_ptr loader) const { + this->load_state(loader); + double rt; + for (const nested_module_ptr &m : modules) { + auto bmi = dynamic_cast(m.get()); + bmi->get_bmi_model()->SetValue("reset_time", &rt); } } diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index 103b7d6a7a..cfa152a92e 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -112,12 +112,14 @@ class File_Per_Unit_Snapshot_Loader : public State_Snapshot_Loader File_Per_Unit_Snapshot_Loader(path dir_path); ~File_Per_Unit_Snapshot_Loader() override = default; + bool has_unit(const std::string &unit_name) override; + /** * Load data from whatever source and store it in the `data` vector. * * @param data The location where the loaded data will be stored. This will be resized to the amount of data loaded. */ - void load_unit(std::string const& unit_name, std::vector &data) override; + void load_unit(const std::string &unit_name, std::vector &data) override; /** * Execute logic to complete the saving process @@ -140,6 +142,11 @@ File_Per_Unit_Snapshot_Loader::File_Per_Unit_Snapshot_Loader(path dir_path) } +bool File_Per_Unit_Snapshot_Loader::has_unit(const std::string &unit_name) { + auto file_path = dir_path_ / unit_name; + return exists(file_path.string()); +} + void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, std::vector &data) { auto file_path = dir_path_ / unit_name; std::uintmax_t size; diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 84e2226e95..3f5838e195 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -71,6 +71,19 @@ std::vector>> State_Save_Con return savers; } +std::unique_ptr State_Save_Config::hot_start() const { + for (const auto &i : this->instances_) { + if (i.direction_ == State_Save_Direction::Load && i.timing_ == State_Save_When::StartOfRun) { + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + return std::make_unique(i.path_); + } else { + LOG(LogLevel::WARNING, "State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for start of run saving."); + } + } + } + return std::unique_ptr(); +} + State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) : label_(label) , path_(path) From 6e9a5f01aa4a309b629ff24def006d7c3dd34dbb Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 4 Feb 2026 13:53:05 -0500 Subject: [PATCH 52/93] Global source for serialization message names --- .../catchment/Bmi_Fortran_Formulation.hpp | 2 +- include/state_save_restore/State_Save_Utils.hpp | 12 ++++++++++++ src/core/NgenSimulation.cpp | 13 +++++++------ .../catchment/Bmi_Fortran_Formulation.cpp | 7 ++++--- .../catchment/Bmi_Module_Formulation.cpp | 13 +++++++------ .../catchment/Bmi_Multi_Formulation.cpp | 13 +++++++------ src/realizations/catchment/Bmi_Py_Formulation.cpp | 3 ++- 7 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 include/state_save_restore/State_Save_Utils.hpp diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 6cc8618bc4..4da087be52 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -25,7 +25,7 @@ namespace realization { /** * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_serialization_state` is called. - * Because the Fortran BMI has no messaging for 64-bit integers, this overload will use the 32-bit integer interface and copy the results to `size`. + * Because the Fortran BMI has no messaging for 64-bit integers, this overload will use the 32-bit integer interface. * * @return Span of the serialized data. */ diff --git a/include/state_save_restore/State_Save_Utils.hpp b/include/state_save_restore/State_Save_Utils.hpp new file mode 100644 index 0000000000..8c0504f7a8 --- /dev/null +++ b/include/state_save_restore/State_Save_Utils.hpp @@ -0,0 +1,12 @@ +#ifndef NGEN_STATE_SAVE_UTILS_HPP +#define NGEN_STATE_SAVE_UTILS_HPP + +namespace StateSaveNames { + inline constexpr auto CREATE = "serialization_create"; + inline constexpr auto STATE = "serialization_state"; + inline constexpr auto FREE = "serialization_free"; + inline constexpr auto SIZE = "serialization_size"; + inline constexpr auto RESET = "reset_time"; +} + +#endif diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 3000599634..86e0112f86 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -8,6 +8,7 @@ #include "HY_Features.hpp" #endif +#include "state_save_restore/State_Save_Utils.hpp" #include "state_save_restore/State_Save_Restore.hpp" #include "parallel_utils.h" @@ -137,12 +138,12 @@ void NgenSimulation::save_end_of_run(std::shared_ptr snaps #if NGEN_WITH_ROUTING if (this->mpi_rank_ == 0 && this->py_troute_) { uint64_t serialization_size; - this->py_troute_->SetValue("serialization_create", &serialization_size); - this->py_troute_->GetValue("serialization_size", &serialization_size); - void *troute_state = this->py_troute_->GetValuePtr("serialization_state"); + this->py_troute_->SetValue(StateSaveNames::CREATE, &serialization_size); + this->py_troute_->GetValue(StateSaveNames::SIZE, &serialization_size); + void *troute_state = this->py_troute_->GetValuePtr(StateSaveNames::STATE); boost::span span(static_cast(troute_state), serialization_size); snapshot_saver->save_unit(TROUTE_UNIT_NAME, span); - this->py_troute_->SetValue("serialization_free", &serialization_size); + this->py_troute_->SetValue(StateSaveNames::FREE, &serialization_size); } #endif // NGEN_WITH_ROUTING } @@ -170,9 +171,9 @@ void NgenSimulation::load_hot_start(std::shared_ptr snaps if (py_troute_ == NULL) { this->make_troute(t_route_config_file_with_path); } - py_troute_->set_value_unchecked("serialization_state", troute_data.data(), troute_data.size()); + py_troute_->set_value_unchecked(StateSaveNames::STATE, troute_data.data(), troute_data.size()); double rt; // unused by the BMI but needed for messaging - py_troute_->SetValue("reset_time", &rt); + py_troute_->SetValue(StateSaveNames::RESET, &rt); } else if (!config_file_set && !snapshot_exists) { LOG(LogLevel::DEBUG, "No data set for loading T-Route."); } else if (config_file_set && !snapshot_exists) { diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index b8804eab79..ee9d3179ef 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -6,6 +6,7 @@ #include "Bmi_Fortran_Formulation.hpp" #include "Bmi_Fortran_Adapter.hpp" #include "Constants.h" +#include "state_save_restore/State_Save_Utils.hpp" using namespace realization; using namespace models::bmi; @@ -96,9 +97,9 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const const boost::span Bmi_Fortran_Formulation::get_serialization_state() const { auto model = get_bmi_model(); int size_int = 0; - model->SetValue("serialization_create", &size_int); - model->GetValue("serialization_size", &size_int); - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + model->SetValue(StateSaveNames::CREATE, &size_int); + model->GetValue(StateSaveNames::SIZE, &size_int); + auto serialization_state = static_cast(model->GetValuePtr(StateSaveNames::STATE)); const boost::span span(serialization_state, size_int); return span; } diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index c0c29ce418..09c4b64512 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -2,6 +2,7 @@ #include "utilities/logging_utils.h" #include #include "Logger.hpp" +#include "state_save_restore/State_Save_Utils.hpp" #include std::stringstream bmiform_ss; @@ -38,7 +39,7 @@ namespace realization { void Bmi_Module_Formulation::load_hot_start(std::shared_ptr loader) const { this->load_state(loader); double rt; - this->get_bmi_model()->SetValue("reset_time", &rt); + this->get_bmi_model()->SetValue(StateSaveNames::FREE, &rt); } boost::span Bmi_Module_Formulation::get_available_variable_names() const { @@ -1093,9 +1094,9 @@ namespace realization { const boost::span Bmi_Module_Formulation::get_serialization_state() const { auto model = get_bmi_model(); uint64_t size = 0; - model->SetValue("serialization_create", &size); - model->GetValue("serialization_size", &size); - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + model->SetValue(StateSaveNames::CREATE, &size); + model->GetValue(StateSaveNames::SIZE, &size); + auto serialization_state = static_cast(model->GetValuePtr(StateSaveNames::STATE)); const boost::span span(serialization_state, size); return span; } @@ -1105,13 +1106,13 @@ namespace realization { // grab the pointer to the underlying state data void* data = (void*)state.data(); // load the state through SetValue - bmi->SetValue("serialization_state", data); + bmi->SetValue(StateSaveNames::STATE, data); } void Bmi_Module_Formulation::free_serialization_state() const { auto bmi = this->bmi_model; // send message to clear memory associated with serialized data void* _; // this pointer will be unused by SetValue - bmi->SetValue("serialization_free", _); + bmi->SetValue(StateSaveNames::FREE, _); } } diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index b6f91b8b09..e2b0ea8106 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -13,6 +13,7 @@ #include "Bmi_Py_Formulation.hpp" #include "Logger.hpp" +#include "state_save_restore/State_Save_Utils.hpp" #include #if (__cplusplus >= 202002L) @@ -48,12 +49,13 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); } - char *data = new char[data_size]; + std::vector data; + data.reserve(data_size); size_t index = 0; for (const auto &bmi : bmi_data) { // write the size of the data if (is_little_endian()) { - std::memcpy(&data[index], &bmi.second, sizeof(uint64_t)); + std::memcpy(data.data() + index, &bmi.second, sizeof(uint64_t)); } else { // store the size bytes in reverse order to ensure saved data is always little endian const char *bytes = reinterpret_cast(&bmi.second); @@ -63,13 +65,12 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav } } // write the serialized data - std::memcpy(data + index + sizeof(uint64_t), &bmi.first, bmi.second); + std::memcpy(data.data() + index + sizeof(uint64_t), &bmi.first, bmi.second); index += sizeof(uint64_t) + bmi.second; } - boost::span span(data, data_size); + boost::span span(data.data(), data_size); saver->save_unit(this->get_id(), span); - delete[] data; for (const nested_module_ptr &m : modules) { auto bmi = static_cast(m.get()); bmi->free_serialization_state(); @@ -106,7 +107,7 @@ void Bmi_Multi_Formulation::load_hot_start(std::shared_ptr(m.get()); - bmi->get_bmi_model()->SetValue("reset_time", &rt); + bmi->get_bmi_model()->SetValue(StateSaveNames::RESET, &rt); } } diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index a3dc65df1e..15e66677d0 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -1,5 +1,6 @@ #include #include "Logger.hpp" +#include "state_save_restore/State_Save_Utils.hpp" #if NGEN_WITH_PYTHON @@ -120,7 +121,7 @@ bool Bmi_Py_Formulation::is_model_initialized() const { void Bmi_Py_Formulation::load_serialization_state(const boost::span state) const { auto bmi = std::dynamic_pointer_cast(get_bmi_model()); // load the state through the set value function that does not enforce the input size is the same as the current BMI's size - bmi->set_value_unchecked("serialization_state", state.data(), state.size()); + bmi->set_value_unchecked(StateSaveNames::STATE, state.data(), state.size()); } #endif //NGEN_WITH_PYTHON From 9e8a3caa13492702e196d17e25114f9a6e511e5f Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 4 Feb 2026 14:20:23 -0500 Subject: [PATCH 53/93] Expand applicable types for get value as double --- .../catchment/Bmi_Py_Formulation.cpp | 60 +++++++------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index 15e66677d0..f0c912fa44 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -54,50 +54,30 @@ double Bmi_Py_Formulation::get_var_value_as_double(const int &index, const std:: std::string val_type = model->GetVarType(var_name); size_t val_item_size = (size_t)model->GetVarItemsize(var_name); + std::string cxx_type = model->get_analogous_cxx_type(val_type, val_item_size); //void *dest; int indices[1]; indices[0] = index; - - // The available types and how they are handled here should match what is in SetValueAtIndices - if (val_type == "int" && val_item_size == sizeof(short)) { - short dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double)dest; - } - if (val_type == "int" && val_item_size == sizeof(int)) { - int dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double)dest; - } - if (val_type == "int" && val_item_size == sizeof(long)) { - long dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double)dest; - } - if (val_type == "int" && val_item_size == sizeof(long long)) { - long long dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double)dest; - } - if (val_type == "float" || val_type == "float16" || val_type == "float32" || val_type == "float64") { - if (val_item_size == sizeof(float)) { - float dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double) dest; - } - if (val_item_size == sizeof(double)) { - double dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return dest; - } - if (val_item_size == sizeof(long double)) { - long double dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double) dest; - } - } - + // macro for both checking and converting based on type from get_analogous_cxx_type +#define GET_DOUBLE(type) if (cxx_type == #type) {\ + type dest;\ + model->get_value_at_indices(var_name, &dest, indices, 1, false);\ + return static_cast(dest);} + GET_DOUBLE(signed char) + else GET_DOUBLE(unsigned char) + else GET_DOUBLE(short) + else GET_DOUBLE(unsigned short) + else GET_DOUBLE(int) + else GET_DOUBLE(unsigned int) + else GET_DOUBLE(long) + else GET_DOUBLE(unsigned long) + else GET_DOUBLE(long long) + else GET_DOUBLE(unsigned long long) + else GET_DOUBLE(float) + else GET_DOUBLE(double) + else GET_DOUBLE(long double) +#undef GET_DOUBLE Logger::logMsgAndThrowError("Unable to get value of variable " + var_name + " from " + get_model_type_name() + " as double: no logic for converting variable type " + val_type); From 92bebce9e74a0b2bc9810ede1d0b394d9c8e7540 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 4 Feb 2026 14:29:51 -0500 Subject: [PATCH 54/93] C++14 consts --- .../state_save_restore/State_Save_Utils.hpp | 10 +++--- .../catchment/Bmi_Py_Formulation.cpp | 36 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/include/state_save_restore/State_Save_Utils.hpp b/include/state_save_restore/State_Save_Utils.hpp index 8c0504f7a8..8e6c127f6a 100644 --- a/include/state_save_restore/State_Save_Utils.hpp +++ b/include/state_save_restore/State_Save_Utils.hpp @@ -2,11 +2,11 @@ #define NGEN_STATE_SAVE_UTILS_HPP namespace StateSaveNames { - inline constexpr auto CREATE = "serialization_create"; - inline constexpr auto STATE = "serialization_state"; - inline constexpr auto FREE = "serialization_free"; - inline constexpr auto SIZE = "serialization_size"; - inline constexpr auto RESET = "reset_time"; + const auto CREATE = "serialization_create"; + const auto STATE = "serialization_state"; + const auto FREE = "serialization_free"; + const auto SIZE = "serialization_size"; + const auto RESET = "reset_time"; } #endif diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index f0c912fa44..046bd68207 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -60,24 +60,24 @@ double Bmi_Py_Formulation::get_var_value_as_double(const int &index, const std:: int indices[1]; indices[0] = index; // macro for both checking and converting based on type from get_analogous_cxx_type -#define GET_DOUBLE(type) if (cxx_type == #type) {\ - type dest;\ - model->get_value_at_indices(var_name, &dest, indices, 1, false);\ - return static_cast(dest);} - GET_DOUBLE(signed char) - else GET_DOUBLE(unsigned char) - else GET_DOUBLE(short) - else GET_DOUBLE(unsigned short) - else GET_DOUBLE(int) - else GET_DOUBLE(unsigned int) - else GET_DOUBLE(long) - else GET_DOUBLE(unsigned long) - else GET_DOUBLE(long long) - else GET_DOUBLE(unsigned long long) - else GET_DOUBLE(float) - else GET_DOUBLE(double) - else GET_DOUBLE(long double) -#undef GET_DOUBLE +#define PY_BMI_DOUBLE_AT_INDEX(type) if (cxx_type == #type) {\ + type dest;\ + model->get_value_at_indices(var_name, &dest, indices, 1, false);\ + return static_cast(dest);} + PY_BMI_DOUBLE_AT_INDEX(signed char) + else PY_BMI_DOUBLE_AT_INDEX(unsigned char) + else PY_BMI_DOUBLE_AT_INDEX(short) + else PY_BMI_DOUBLE_AT_INDEX(unsigned short) + else PY_BMI_DOUBLE_AT_INDEX(int) + else PY_BMI_DOUBLE_AT_INDEX(unsigned int) + else PY_BMI_DOUBLE_AT_INDEX(long) + else PY_BMI_DOUBLE_AT_INDEX(unsigned long) + else PY_BMI_DOUBLE_AT_INDEX(long long) + else PY_BMI_DOUBLE_AT_INDEX(unsigned long long) + else PY_BMI_DOUBLE_AT_INDEX(float) + else PY_BMI_DOUBLE_AT_INDEX(double) + else PY_BMI_DOUBLE_AT_INDEX(long double) +#undef PY_BMI_DOUBLE_AT_INDEX Logger::logMsgAndThrowError("Unable to get value of variable " + var_name + " from " + get_model_type_name() + " as double: no logic for converting variable type " + val_type); From d05953a2f2a8216c8ea4ceda239a51fb2f71db81 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 5 Feb 2026 07:53:02 -0500 Subject: [PATCH 55/93] Expand python typing interface --- include/bmi/Bmi_Py_Adapter.hpp | 46 +++++++++++++--------------------- src/bmi/Bmi_Py_Adapter.cpp | 38 ++++++++++++++-------------- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 60d97be394..2e48dc7845 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -541,39 +541,29 @@ namespace models { int itemSize = GetVarItemsize(name); std::string py_type = GetVarType(name); std::string cxx_type = get_analogous_cxx_type(py_type, (size_t) itemSize); - - if (cxx_type == "signed char") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned char") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "short") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned short") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "int") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned int") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "long") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned long") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "long long") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned long long") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "float") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "double") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "long double") { - this->set_value(name, static_cast(src)); - } else { + // macro for checking type and setting value + #define BMI_PY_SET_VALUE(type) if (cxx_type == #type) {\ + this->set_value(name, static_cast(src)); } + BMI_PY_SET_VALUE(signed char) + else BMI_PY_SET_VALUE(unsigned char) + else BMI_PY_SET_VALUE(short) + else BMI_PY_SET_VALUE(unsigned short) + else BMI_PY_SET_VALUE(int) + else BMI_PY_SET_VALUE(unsigned int) + else BMI_PY_SET_VALUE(long) + else BMI_PY_SET_VALUE(unsigned long) + else BMI_PY_SET_VALUE(long long) + else BMI_PY_SET_VALUE(unsigned long long) + else BMI_PY_SET_VALUE(float) + else BMI_PY_SET_VALUE(double) + else BMI_PY_SET_VALUE(long double) + else { std::string throw_msg; throw_msg.assign("Bmi_Py_Adapter cannot set values for variable '" + name + "' that has unrecognized C++ type '" + cxx_type + "'"); LOG(throw_msg, LogLevel::WARNING); throw std::runtime_error(throw_msg); } + #undef BMI_PY_SET_VALUE } /** * Set the values of the given BMI variable for the model, sourcing new data from the provided vector. diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index d1a1c6f681..5fdc3eb8ec 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -199,30 +199,30 @@ std::string Bmi_Py_Adapter::get_bmi_type_simple_name() const { void Bmi_Py_Adapter::SetValueAtIndices(std::string name, int *inds, int count, void *src) { std::string val_type = GetVarType(name); size_t val_item_size = (size_t)GetVarItemsize(name); - - // The available types and how they are handled here should match what is in get_value_at_indices - if (val_type == "int" && val_item_size == sizeof(short)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "int" && val_item_size == sizeof(int)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "int" && val_item_size == sizeof(long)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "int" && val_item_size == sizeof(long long)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "float" && val_item_size == sizeof(float)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "float" && val_item_size == sizeof(double)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "float64" && val_item_size == sizeof(double)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "float" && val_item_size == sizeof(long double)) { - set_value_at_indices(name, inds, count, src, val_type); - } else { + std::string cxx_type = this->get_analogous_cxx_type(val_type, val_item_size); + + // macro for checking type and calling `set_value_at_indices` with that type + #define BMI_PY_SET_VALUE_INDEX(type) if (cxx_type == #type) { this->set_value_at_indices(name, inds, count, src, val_type); } + BMI_PY_SET_VALUE_INDEX(signed char) + else BMI_PY_SET_VALUE_INDEX(unsigned char) + else BMI_PY_SET_VALUE_INDEX(short) + else BMI_PY_SET_VALUE_INDEX(unsigned short) + else BMI_PY_SET_VALUE_INDEX(int) + else BMI_PY_SET_VALUE_INDEX(unsigned int) + else BMI_PY_SET_VALUE_INDEX(long) + else BMI_PY_SET_VALUE_INDEX(unsigned long) + else BMI_PY_SET_VALUE_INDEX(long long) + else BMI_PY_SET_VALUE_INDEX(unsigned long long) + else BMI_PY_SET_VALUE_INDEX(float) + else BMI_PY_SET_VALUE_INDEX(double) + else BMI_PY_SET_VALUE_INDEX(long double) + else { Logger::logMsgAndThrowError( "(Bmi_Py_Adapter) Failed attempt to SET values of BMI variable '" + name + "' from '" + model_name + "' model: model advertises unsupported combination of type (" + val_type + ") and size (" + std::to_string(val_item_size) + ")."); } + #undef BMI_PY_SET_VALUE_INDEX } void Bmi_Py_Adapter::Update() { From 6d5ceeffc676526e26fea2f8b1e5ff8313fb933a Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 5 Feb 2026 07:53:26 -0500 Subject: [PATCH 56/93] Prevent multiple hot start config definitions --- src/state_save_restore/State_Save_Restore.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 3f5838e195..58fe75fad1 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -18,7 +18,7 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) return; } - //auto saving_config = *maybe; + bool hot_start = false; for (const auto& saving_config : *maybe) { try { auto& subtree = saving_config.second; @@ -29,9 +29,14 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) auto when = subtree.get("when"); instance i{direction, what, where, how, when}; + if (i.timing_ == State_Save_When::StartOfRun && i.direction_ == State_Save_Direction::Load) { + if (hot_start) + throw std::runtime_error("Only one hot start state saving configuration is allowed."); + hot_start = true; + } instances_.push_back(i); - } catch (...) { - LOG("Bad state saving config", LogLevel::WARNING); + } catch (std::exception &e) { + LOG("Bad state saving config: " + std::string(e.what()), LogLevel::WARNING); throw; } } From bf027f6796d2fa8011191247994677d70c254497 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 6 Feb 2026 10:26:57 -0500 Subject: [PATCH 57/93] Use Boost for serializing Multi-BMI --- .../catchment/Bmi_Formulation.hpp | 6 +- .../catchment/Bmi_Fortran_Formulation.hpp | 2 +- .../catchment/Bmi_Module_Formulation.hpp | 12 +- .../catchment/Bmi_Multi_Formulation.hpp | 10 +- .../catchment/Bmi_Py_Formulation.hpp | 2 +- include/state_save_restore/File_Per_Unit.hpp | 8 +- .../state_save_restore/State_Save_Restore.hpp | 11 +- include/state_save_restore/vecbuf.hpp | 125 ++++++++++++++++++ src/NGen.cpp | 7 +- .../catchment/Bmi_Fortran_Formulation.cpp | 2 +- .../catchment/Bmi_Module_Formulation.cpp | 12 +- .../catchment/Bmi_Multi_Formulation.cpp | 121 +++++++---------- .../catchment/Bmi_Py_Formulation.cpp | 2 +- src/realizations/catchment/CMakeLists.txt | 9 ++ src/state_save_restore/File_Per_Unit.cpp | 32 +++-- src/state_save_restore/State_Save_Restore.cpp | 5 +- 16 files changed, 250 insertions(+), 116 deletions(-) create mode 100644 include/state_save_restore/vecbuf.hpp diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index ad2009f150..91236fea42 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -77,14 +77,14 @@ namespace realization { * Asks the model to serialize its state, queries the pointer * and length, passes that to saver, and then releases it */ - virtual void save_state(std::shared_ptr saver) const = 0; + virtual void save_state(std::shared_ptr saver) = 0; /** * Passes a serialized representation of the model's state to ``loader`` * * Asks saver to find data for the BMI and passes that data to the BMI for loading. */ - virtual void load_state(std::shared_ptr loader) const = 0; + virtual void load_state(std::shared_ptr loader) = 0; /** * Passes a serialized representation of the model's state to ``loader`` @@ -93,7 +93,7 @@ namespace realization { * * Differes from `load_state` by also restting the internal time value to its initial state. */ - virtual void load_hot_start(std::shared_ptr loader) const = 0; + virtual void load_hot_start(std::shared_ptr loader) = 0; /** * Convert a time value from the model to an epoch time in seconds. diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 4da087be52..205303f9d5 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -29,7 +29,7 @@ namespace realization { * * @return Span of the serialized data. */ - const boost::span get_serialization_state() const override; + const boost::span get_serialization_state() override; protected: diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index aa454d12b9..ab57526a15 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -50,11 +50,11 @@ namespace realization { * Create a save state, save it using the `State_Snapshot_Saver`, then clear the save state from memory. * `this->get_id()` will be used as the unique ID for the saver. */ - void save_state(std::shared_ptr saver) const override; + void save_state(std::shared_ptr saver) override; - void load_state(std::shared_ptr loader) const override; + void load_state(std::shared_ptr loader) override; - void load_hot_start(std::shared_ptr loader) const override; + void load_hot_start(std::shared_ptr loader) override; /** * Get the collection of forcing output property names this instance can provide. @@ -295,16 +295,16 @@ namespace realization { * * @return Span of the serialized data. */ - virtual const boost::span get_serialization_state() const; + virtual const boost::span get_serialization_state(); /** * Requests the BMI to load data from a previously saved state. This has a side effect of freeing a current state if it currently exists. */ - virtual void load_serialization_state(const boost::span state) const; + virtual void load_serialization_state(const boost::span state); /** * Requests the BMI to clear a currently saved state from memory. * Existing state pointers should not be used as the stored data may be freed depending on implementation. */ - void free_serialization_state() const; + void free_serialization_state(); void set_realization_file_format(bool is_legacy_format); protected: diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 3b7e09fc04..bd64b063af 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -15,6 +15,7 @@ #include "ConfigurationException.hpp" #include "ExternalIntegrationException.hpp" +#include #define BMI_REALIZATION_CFG_PARAM_REQ__MODULES "modules" #define BMI_REALIZATION_CFG_PARAM_OPT__DEFAULT_OUT_VALS "default_output_values" @@ -47,11 +48,11 @@ namespace realization { virtual ~Bmi_Multi_Formulation() {}; - void save_state(std::shared_ptr saver) const override; + void save_state(std::shared_ptr saver) override; - void load_state(std::shared_ptr loader) const override; + void load_state(std::shared_ptr loader) override; - void load_hot_start(std::shared_ptr loader) const override; + void load_hot_start(std::shared_ptr loader) override; /** * Convert a time value from the model to an epoch time in seconds. @@ -671,6 +672,7 @@ namespace realization { bool is_realization_legacy_format() const; private: + friend class boost::serialization::access; /** * Setup a deferred provider for a nested module, tracking the class as needed. @@ -770,6 +772,8 @@ namespace realization { friend Bmi_Multi_Formulation_Test; friend class ::Bmi_Cpp_Multi_Array_Test; + template + void serialize(Archive& ar, const unsigned int version); }; } diff --git a/include/realizations/catchment/Bmi_Py_Formulation.hpp b/include/realizations/catchment/Bmi_Py_Formulation.hpp index 19c6a0428e..d3830d6282 100644 --- a/include/realizations/catchment/Bmi_Py_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Py_Formulation.hpp @@ -40,7 +40,7 @@ namespace realization { * * The python BMI requires additional messaging for pre-allocating memory for load */ - void load_serialization_state(const boost::span state) const override; + void load_serialization_state(const boost::span state) override; protected: diff --git a/include/state_save_restore/File_Per_Unit.hpp b/include/state_save_restore/File_Per_Unit.hpp index 3fdefe71b8..faec8d966a 100644 --- a/include/state_save_restore/File_Per_Unit.hpp +++ b/include/state_save_restore/File_Per_Unit.hpp @@ -9,7 +9,9 @@ class File_Per_Unit_Saver : public State_Saver File_Per_Unit_Saver(std::string base_path); ~File_Per_Unit_Saver(); - std::shared_ptr initialize_snapshot(snapshot_time_t epoch, State_Durability durability) override; + std::shared_ptr initialize_snapshot(State_Durability durability) override; + + std::shared_ptr initialize_checkpoint_snapshot(snapshot_time_t epoch, State_Durability durability) override; void finalize() override; @@ -26,7 +28,9 @@ class File_Per_Unit_Loader : public State_Loader void finalize() override { }; - std::shared_ptr initialize_snapshot(State_Saver::snapshot_time_t epoch) override; + std::shared_ptr initialize_snapshot() override; + + std::shared_ptr initialize_checkpoint_snapshot(State_Saver::snapshot_time_t epoch) override; private: std::string dir_path_; }; diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index f311a5cfd4..3d3a3bf692 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -105,7 +105,9 @@ class State_Saver * potential errors to be checked and reported before finalize() * and/or State_Snapshot_Saver::finish_saving() return */ - virtual std::shared_ptr initialize_snapshot(snapshot_time_t epoch, State_Durability durability) = 0; + virtual std::shared_ptr initialize_snapshot(State_Durability durability) = 0; + + virtual std::shared_ptr initialize_checkpoint_snapshot(snapshot_time_t epoch, State_Durability durability) = 0; /** * Execute any logic necessary to cleanly finish usage, and @@ -120,7 +122,7 @@ class State_Snapshot_Saver { public: State_Snapshot_Saver() = delete; - State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability); + State_Snapshot_Saver(State_Saver::State_Durability durability); virtual ~State_Snapshot_Saver() = default; /** @@ -139,7 +141,6 @@ class State_Snapshot_Saver virtual void finish_saving() = 0; protected: - State_Saver::snapshot_time_t epoch_; State_Saver::State_Durability durability_; }; @@ -155,7 +156,9 @@ class State_Loader * Return an object suitable for loading a simulation state as of * a particular moment in time, @param epoch */ - virtual std::shared_ptr initialize_snapshot(State_Saver::snapshot_time_t epoch) = 0; + virtual std::shared_ptr initialize_snapshot() = 0; + + virtual std::shared_ptr initialize_checkpoint_snapshot(State_Saver::snapshot_time_t epoch) = 0; /** * Execute any logic necessary to cleanly finish usage, and diff --git a/include/state_save_restore/vecbuf.hpp b/include/state_save_restore/vecbuf.hpp new file mode 100644 index 0000000000..a20bc70e73 --- /dev/null +++ b/include/state_save_restore/vecbuf.hpp @@ -0,0 +1,125 @@ +#ifndef HPP_STRING_VECBUF +#define HPP_STRING_VECBUF +// https://gist.github.com/stephanlachnit/4a06f8475afd144e73235e2a2584b000 +// SPDX-FileCopyrightText: 2023 Stephan Lachnit +// SPDX-License-Identifier: MIT + +#include +#include +#include + +template> +class vecbuf : public std::basic_streambuf { +public: + using streambuf = std::basic_streambuf; + using char_type = typename streambuf::char_type; + using int_type = typename streambuf::int_type; + using traits_type = typename streambuf::traits_type; + using vector = std::vector; + using value_type = typename vector::value_type; + using size_type = typename vector::size_type; + + // Constructor for vecbuf with optional initial capacity + vecbuf(size_type capacity = 0) : vector_() { reserve(capacity); } + + // Forwarder for std::vector::shrink_to_fit() + constexpr void shrink_to_fit() { vector_.shrink_to_fit(); } + + // Forwarder for std::vector::clear() + constexpr void clear() { vector_.clear(); } + + // Forwarder for std::vector::resize(size) + constexpr void resize(size_type size) { vector_.resize(size); } + + // Forwarder for std::vector::reserve + constexpr void reserve(size_type capacity) { vector_.reserve(capacity); setp_from_vector(); } + + // Increase the capacity of the buffer by reserving the current_size + additional_capacity + constexpr void reserve_additional(size_type additional_capacity) { reserve(size() + additional_capacity); } + + // Forwarder for std::vector::data + constexpr value_type* data() { return vector_.data(); } + + // Forwarder for std::vector::size + constexpr size_type size() const { return vector_.size(); } + + // Forwarder for std::vector::capacity + constexpr size_type capacity() const { return vector_.capacity(); } + + // Implements std::basic_streambuf::xsputn + std::streamsize xsputn(const char_type* s, std::streamsize count) override { + try { + reserve_additional(count); + } + catch (const std::bad_alloc& error) { + // reserve did not work, use slow algorithm + return xsputn_slow(s, count); + } + // reserve worked, use fast algorithm + return xsputn_fast(s, count); + } + +protected: + // Calculates value to std::basic_streambuf::pbase from vector + constexpr value_type* pbase_from_vector() const { return const_cast(vector_.data()); } + + // Calculates value to std::basic_streambuf::pptr from vector + constexpr value_type* pptr_from_vector() const { return const_cast(vector_.data() + vector_.size()); } + + // Calculates value to std::basic_streambuf::epptr from vector + constexpr value_type* epptr_from_vector() const { return const_cast(vector_.data()) + vector_.capacity(); } + + // Sets the values for std::basic_streambuf::pbase, std::basic_streambuf::pptr and std::basic_streambuf::epptr from vector + constexpr void setp_from_vector() { streambuf::setp(pbase_from_vector(), epptr_from_vector()); streambuf::pbump(size()); } + +private: + // std::vector containing the data + vector vector_; + + // Fast implementation of std::basic_streambuf::xsputn if reserve_additional(count) succeeded + std::streamsize xsputn_fast(const char_type* s, std::streamsize count) { + // store current pptr (end of vector location) + auto* old_pptr = pptr_from_vector(); + // resize the vector, does not move since space already reserved + vector_.resize(vector_.size() + count); + // directly memcpy new content to old pptr (end of vector before it was resized) + traits_type::copy(old_pptr, s, count); + // reserve() already calls setp_from_vector(), only adjust pptr to new epptr + streambuf::pbump(count); + + return count; + } + + // Slow implementation of std::basic_streambuf::xsputn if reserve_additional(count) did not succeed, might calls std::basic_streambuf::overflow() + std::streamsize xsputn_slow(const char_type* s, std::streamsize count) { + // reserving entire vector failed, emplace char for char + std::streamsize written = 0; + while (written < count) { + try { + // copy one char, should throw eventually std::bad_alloc + vector_.emplace_back(s[written]); + } + catch (const std::bad_alloc& error) { + // try overflow(), if eof return, else continue + int_type c = this->overflow(traits_type::to_int_type(s[written])); + if (traits_type::eq_int_type(c, traits_type::eof())) { + return written; + } + } + // update pbase, pptr and epptr + setp_from_vector(); + written++; + } + return written; + } + +}; + +class membuf : public std::streambuf { +public: + membuf(char *begin, size_t size) { + this->setg(begin, begin, begin + size); + } +}; + +#endif diff --git a/src/NGen.cpp b/src/NGen.cpp index 398adc1041..79946fd64f 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -703,7 +703,7 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { auto hot_start_loader = state_saving_config.hot_start(); if (hot_start_loader) { LOG(LogLevel::INFO, "Loading hot start data from prior snapshot."); - std::shared_ptr snapshot_loader = hot_start_loader->initialize_snapshot(State_Saver::snapshot_time_now()); + std::shared_ptr snapshot_loader = hot_start_loader->initialize_snapshot(); simulation->load_hot_start(snapshot_loader, manager->get_t_route_config_file_with_path()); } } @@ -774,10 +774,7 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { // run any end-of-run state saving after T-Route has finished but before starting to tear down data structures for (const auto& end_saver : state_saving_config.end_of_run_savers()) { LOG(LogLevel::INFO, "Saving end of run simulation data for state saving config " + end_saver.first); - std::shared_ptr snapshot = end_saver.second->initialize_snapshot( - State_Saver::snapshot_time_now(), - State_Saver::State_Durability::strict - ); + std::shared_ptr snapshot = end_saver.second->initialize_snapshot(State_Saver::State_Durability::strict); simulation->save_end_of_run(snapshot); } diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index ee9d3179ef..3f1e338813 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -94,7 +94,7 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const return 1.0; } -const boost::span Bmi_Fortran_Formulation::get_serialization_state() const { +const boost::span Bmi_Fortran_Formulation::get_serialization_state() { auto model = get_bmi_model(); int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 09c4b64512..b4b233718c 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -17,7 +17,7 @@ namespace realization { inner_create_formulation(properties, true); } - void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { + void Bmi_Module_Formulation::save_state(std::shared_ptr saver) { uint64_t size = 1; boost::span data = this->get_serialization_state(); @@ -29,14 +29,14 @@ namespace realization { this->free_serialization_state(); } - void Bmi_Module_Formulation::load_state(std::shared_ptr loader) const { + void Bmi_Module_Formulation::load_state(std::shared_ptr loader) { std::vector buffer; loader->load_unit(this->get_id(), buffer); boost::span data(buffer.data(), buffer.size()); this->load_serialization_state(data); } - void Bmi_Module_Formulation::load_hot_start(std::shared_ptr loader) const { + void Bmi_Module_Formulation::load_hot_start(std::shared_ptr loader) { this->load_state(loader); double rt; this->get_bmi_model()->SetValue(StateSaveNames::FREE, &rt); @@ -1091,7 +1091,7 @@ namespace realization { } - const boost::span Bmi_Module_Formulation::get_serialization_state() const { + const boost::span Bmi_Module_Formulation::get_serialization_state() { auto model = get_bmi_model(); uint64_t size = 0; model->SetValue(StateSaveNames::CREATE, &size); @@ -1101,7 +1101,7 @@ namespace realization { return span; } - void Bmi_Module_Formulation::load_serialization_state(const boost::span state) const { + void Bmi_Module_Formulation::load_serialization_state(const boost::span state) { auto bmi = this->bmi_model; // grab the pointer to the underlying state data void* data = (void*)state.data(); @@ -1109,7 +1109,7 @@ namespace realization { bmi->SetValue(StateSaveNames::STATE, data); } - void Bmi_Module_Formulation::free_serialization_state() const { + void Bmi_Module_Formulation::free_serialization_state() { auto bmi = this->bmi_model; // send message to clear memory associated with serialized data void* _; // this pointer will be unused by SetValue diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index e2b0ea8106..290c475cba 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -13,98 +13,51 @@ #include "Bmi_Py_Formulation.hpp" #include "Logger.hpp" +#include "state_save_restore/vecbuf.hpp" #include "state_save_restore/State_Save_Utils.hpp" #include +#include +#include +#include + #if (__cplusplus >= 202002L) #include #endif using namespace realization; -namespace { - // Check if the system's byte order is little endianness - constexpr bool is_little_endian() { -#if (__cplusplus < 202002L) - // C++ less than 2020 requires making a two byte object and checking which byte is 0 - uint16_t endian_bytes = 0xFF00; - uint8_t *endian_bits = reinterpret_cast(&endian_bytes); - return endian_bits[0] == 0; -#else - // C++ 2020+ has as simpler method - return std::endian::native == std::endian::little; -#endif - } -} - -void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { - std::vector> bmi_data; - size_t data_size = 0; +void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) { + LOG(LogLevel::DEBUG, "Saving state for Multi-BMI %s", this->get_id()); + vecbuf data; + boost::archive::binary_oarchive archive(data); + // serialization function handles freeing the sub-BMI states after archiving them + archive << (*this); + // it's recommended to keep data pointers around until serialization completes, + // so freeing the BMI states is done after the data buffer has been completely written to for (const nested_module_ptr &m : modules) { auto bmi = dynamic_cast(m.get()); - boost::span span = bmi->get_serialization_state(); - bmi_data.push_back(std::make_pair(span.data(), span.size())); - data_size += sizeof(uint64_t) + span.size(); - LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); - } - std::vector data; - data.reserve(data_size); - size_t index = 0; - for (const auto &bmi : bmi_data) { - // write the size of the data - if (is_little_endian()) { - std::memcpy(data.data() + index, &bmi.second, sizeof(uint64_t)); - } else { - // store the size bytes in reverse order to ensure saved data is always little endian - const char *bytes = reinterpret_cast(&bmi.second); - size_t endian_index = index + sizeof(uint64_t); - for (size_t i = 0; i < sizeof(uint64_t); ++i) { - data[--endian_index] = bytes[i]; - } - } - // write the serialized data - std::memcpy(data.data() + index + sizeof(uint64_t), &bmi.first, bmi.second); - index += sizeof(uint64_t) + bmi.second; - } - boost::span span(data.data(), data_size); - saver->save_unit(this->get_id(), span); - - for (const nested_module_ptr &m : modules) { - auto bmi = static_cast(m.get()); bmi->free_serialization_state(); } + boost::span span(data.data(), data.size()); + saver->save_unit(this->get_id(), span); } -void Bmi_Multi_Formulation::load_state(std::shared_ptr loader) const { +void Bmi_Multi_Formulation::load_state(std::shared_ptr loader) { + LOG(LogLevel::DEBUG, "Loading save state for Multi-BMI %s", this->get_id()); std::vector data; loader->load_unit(this->get_id(), data); - size_t index = 0; - for (const nested_module_ptr &m : modules) { - auto bmi = dynamic_cast(m.get()); - uint64_t size; - if (is_little_endian()) { - memcpy(&size, data.data() + index, sizeof(uint64_t)); - } else { - // read size bytes in reverse order to interpret from little endian - char *size_bytes = reinterpret_cast(&size); - size_t endian_index = sizeof(uint64_t); - for (size_t i = 0; i < sizeof(uint64_t); ++i) { - size_bytes[--endian_index] = data[index + i]; - } - } - boost::span span(data.data() + index + sizeof(uint64_t), size); - bmi->load_serialization_state(span); - index += sizeof(uint64_t) + size; - LOG(LogLevel::DEBUG, "Loading of multi-BMI %s %s completed with a size of %d bytes.", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); - } + membuf stream(data.data(), data.size()); + boost::archive::binary_iarchive archive(stream); + archive >> (*this); } -void Bmi_Multi_Formulation::load_hot_start(std::shared_ptr loader) const { +void Bmi_Multi_Formulation::load_hot_start(std::shared_ptr loader) { this->load_state(loader); double rt; + LOG(LogLevel::DEBUG, "Resetting time for sub-BMIs"); + // Multi-BMI's current forwards its primary BMI's current time, so no additional action needed for the formulation's reset time for (const nested_module_ptr &m : modules) { auto bmi = dynamic_cast(m.get()); bmi->get_bmi_model()->SetValue(StateSaveNames::RESET, &rt); @@ -705,6 +658,34 @@ void Bmi_Multi_Formulation::set_realization_file_format(bool is_legacy_format){ legacy_json_format = is_legacy_format; } +template +void Bmi_Multi_Formulation::serialize(Archive &ar, const unsigned int version) { + uint64_t data_size; + std::vector buffer; + for (const nested_module_ptr &m : modules) { + auto bmi = dynamic_cast(m.get()); + // if saving, make the BMI's state and record its size and data + if (Archive::is_saving::value) { + LOG(LogLevel::DEBUG, "Saving state from sub-BMI " + bmi->get_model_type_name()); + boost::span span = bmi->get_serialization_state(); + data_size = span.size(); + ar & data_size; + ar & boost::serialization::make_array(span.data(), data_size); + // it's recommended to keep raw pointers alive throughout the entire seiralization process, + // so responsibility for freeing the BMIs' state is left to the caller of this function + } + // if loading, get the current data size stored at the front, then load that much data as a char blob passed to the BMI + else { + LOG(LogLevel::DEBUG, "Loading state from sub-BMI " + bmi->get_model_type_name()); + ar & data_size; + buffer.resize(data_size); + ar & boost::serialization::make_array(buffer.data(), data_size); + boost::span span(buffer.data(), data_size); + bmi->load_serialization_state(span); + } + } +} + //Function to find whether any item in the string vector is empty or blank int find_empty_string_index(const std::vector& str_vector) { for (int i = 0; i < str_vector.size(); ++i) { diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index 046bd68207..1712ec9cc7 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -98,7 +98,7 @@ bool Bmi_Py_Formulation::is_model_initialized() const { return get_bmi_model()->is_model_initialized(); } -void Bmi_Py_Formulation::load_serialization_state(const boost::span state) const { +void Bmi_Py_Formulation::load_serialization_state(const boost::span state) { auto bmi = std::dynamic_pointer_cast(get_bmi_model()); // load the state through the set value function that does not enforce the input size is the same as the current BMI's size bmi->set_value_unchecked(StateSaveNames::STATE, state.data(), state.size()); diff --git a/src/realizations/catchment/CMakeLists.txt b/src/realizations/catchment/CMakeLists.txt index db9bc0f04d..a82c797110 100644 --- a/src/realizations/catchment/CMakeLists.txt +++ b/src/realizations/catchment/CMakeLists.txt @@ -3,6 +3,15 @@ dynamic_sourced_cxx_library(realizations_catchment "${CMAKE_CURRENT_SOURCE_DIR}" add_library(NGen::realizations_catchment ALIAS realizations_catchment) +# ----------------------------------------------------------------------------- +# Find the Boost library and configure usage +set(Boost_USE_STATIC_LIBS OFF) +set(Boost_USE_MULTITHREADED ON) +set(Boost_USE_STATIC_RUNTIME OFF) +find_package(Boost 1.79.0 REQUIRED COMPONENTS serialization) + +target_link_libraries(realizations_catchment PRIVATE Boost::serialization) + target_include_directories(realizations_catchment PUBLIC ${PROJECT_SOURCE_DIR}/include/core ${PROJECT_SOURCE_DIR}/include/core/catchment diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index cfa152a92e..80bce91418 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -41,7 +41,7 @@ class File_Per_Unit_Snapshot_Saver : public State_Snapshot_Saver public: File_Per_Unit_Snapshot_Saver() = delete; - File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability); + File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::State_Durability durability); ~File_Per_Unit_Snapshot_Saver(); public: @@ -61,9 +61,16 @@ File_Per_Unit_Saver::File_Per_Unit_Saver(std::string base_path) File_Per_Unit_Saver::~File_Per_Unit_Saver() = default; -std::shared_ptr File_Per_Unit_Saver::initialize_snapshot(snapshot_time_t epoch, State_Durability durability) +std::shared_ptr File_Per_Unit_Saver::initialize_snapshot(State_Durability durability) { + // TODO + return std::make_shared(path(this->base_path_), durability); +} + +std::shared_ptr File_Per_Unit_Saver::initialize_checkpoint_snapshot(snapshot_time_t epoch, State_Durability durability) { - return std::make_shared(base_path_, epoch, durability); + path checkpoint_path = path(this->base_path_) / unit_saving_utils::format_epoch(epoch); + create_directory(checkpoint_path); + return std::make_shared(checkpoint_path, durability); } void File_Per_Unit_Saver::finalize() @@ -71,9 +78,9 @@ void File_Per_Unit_Saver::finalize() // nothing to be done } -File_Per_Unit_Snapshot_Saver::File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) - : State_Snapshot_Saver(epoch, durability) - , dir_path_(base_path / unit_saving_utils::format_epoch(epoch)) +File_Per_Unit_Snapshot_Saver::File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::State_Durability durability) + : State_Snapshot_Saver(durability) + , dir_path_(base_path) { create_directory(dir_path_); } @@ -156,7 +163,7 @@ void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, std: LOG("Failed to read state save data size for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); throw; } - std::ifstream stream(file_path.string(), std::ios_base::ate | std::ios_base::binary); + std::ifstream stream(file_path.string(), std::ios_base::binary); if (!stream) { LOG("Failed to open state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); throw; @@ -176,9 +183,14 @@ File_Per_Unit_Loader::File_Per_Unit_Loader(std::string dir_path) } -std::shared_ptr File_Per_Unit_Loader::initialize_snapshot(State_Saver::snapshot_time_t epoch) +std::shared_ptr File_Per_Unit_Loader::initialize_snapshot() +{ + return std::make_shared(path(dir_path_)); +} + +std::shared_ptr File_Per_Unit_Loader::initialize_checkpoint_snapshot(State_Saver::snapshot_time_t epoch) { - path dir_path(dir_path_); - return std::make_shared(dir_path); + path checkpoint_path = path(dir_path_) / unit_saving_utils::format_epoch(epoch);; + return std::make_shared(checkpoint_path); } diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 58fe75fad1..ee3f5ae3c9 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -129,9 +129,8 @@ std::string State_Save_Config::instance::instance::mechanism_string() const { } } -State_Snapshot_Saver::State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) - : epoch_(epoch) - , durability_(durability) +State_Snapshot_Saver::State_Snapshot_Saver(State_Saver::State_Durability durability) + : durability_(durability) { } From a1614e097b70ac6dbb99f78758a4f2db92c96f8a Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 10:51:35 -0500 Subject: [PATCH 58/93] Add GetValuePtrInt for Fortran Adapter --- include/bmi/Bmi_Fortran_Adapter.hpp | 16 ++++++++++++++++ .../catchment/Bmi_Fortran_Formulation.cpp | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Fortran_Adapter.hpp b/include/bmi/Bmi_Fortran_Adapter.hpp index f3818c96e8..21f416bacc 100644 --- a/include/bmi/Bmi_Fortran_Adapter.hpp +++ b/include/bmi/Bmi_Fortran_Adapter.hpp @@ -231,6 +231,22 @@ namespace models { return ptr; } + int* GetValuePtrInt(const std::string &name) { + int nbytes; + if (get_var_nbytes(&bmi_model->handle, name.c_str(), &nbytes) != BMI_SUCCESS) { + std::string throw_msg; throw_msg.assign(model_name + " failed to get int pointer for BMI variable " + name + "."); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); + } + int *dest; + if (get_value_ptr_int(&bmi_model->handle, name.c_str(), dest) != BMI_SUCCESS) { + std::string throw_msg; throw_msg.assign(model_name + " failed to get int pointer for BMI variable " + name + "."); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); + } + return dest; + } + /** * Get the size (in bytes) of one item of a variable. * diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 3f1e338813..0ceb36bc93 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -95,11 +95,12 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const } const boost::span Bmi_Fortran_Formulation::get_serialization_state() { - auto model = get_bmi_model(); + auto model = dynamic_cast(get_bmi_model().get()); int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); model->GetValue(StateSaveNames::SIZE, &size_int); - auto serialization_state = static_cast(model->GetValuePtr(StateSaveNames::STATE)); + int *serialization_ptr = model->GetValuePtrInt(StateSaveNames::STATE); + char *serialization_state = reinterpret_cast(serialization_ptr); const boost::span span(serialization_state, size_int); return span; } From 8c56622f36777c80a0315a8ecba943e54992a935 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 17 Feb 2026 09:21:39 -0500 Subject: [PATCH 59/93] Fortran state hackery --- include/bmi/Bmi_Fortran_Adapter.hpp | 16 ---------- .../catchment/Bmi_Fortran_Formulation.hpp | 9 ++++++ .../catchment/Bmi_Module_Formulation.hpp | 2 +- .../state_save_restore/State_Save_Restore.hpp | 18 +---------- .../state_save_restore/State_Save_Utils.hpp | 18 +++++++++++ .../catchment/Bmi_Fortran_Formulation.cpp | 30 ++++++++++++++++--- 6 files changed, 55 insertions(+), 38 deletions(-) diff --git a/include/bmi/Bmi_Fortran_Adapter.hpp b/include/bmi/Bmi_Fortran_Adapter.hpp index 21f416bacc..f3818c96e8 100644 --- a/include/bmi/Bmi_Fortran_Adapter.hpp +++ b/include/bmi/Bmi_Fortran_Adapter.hpp @@ -231,22 +231,6 @@ namespace models { return ptr; } - int* GetValuePtrInt(const std::string &name) { - int nbytes; - if (get_var_nbytes(&bmi_model->handle, name.c_str(), &nbytes) != BMI_SUCCESS) { - std::string throw_msg; throw_msg.assign(model_name + " failed to get int pointer for BMI variable " + name + "."); - LOG(throw_msg, LogLevel::WARNING); - throw std::runtime_error(throw_msg); - } - int *dest; - if (get_value_ptr_int(&bmi_model->handle, name.c_str(), dest) != BMI_SUCCESS) { - std::string throw_msg; throw_msg.assign(model_name + " failed to get int pointer for BMI variable " + name + "."); - LOG(throw_msg, LogLevel::WARNING); - throw std::runtime_error(throw_msg); - } - return dest; - } - /** * Get the size (in bytes) of one item of a variable. * diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 205303f9d5..e7736b04a1 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -5,6 +5,7 @@ #if NGEN_WITH_BMI_FORTRAN +#include #include "Bmi_Module_Formulation.hpp" #include @@ -31,6 +32,10 @@ namespace realization { */ const boost::span get_serialization_state() override; + void load_serialization_state(boost::span state) override; + + void free_serialization_state() override; + protected: /** @@ -57,6 +62,10 @@ namespace realization { friend class ::Bmi_Multi_Formulation_Test; friend class ::Bmi_Formulation_Test; friend class ::Bmi_Fortran_Formulation_Test; + + private: + // location to store serialized state from the BMI because pointer interfaces are not available for Fotran + std::vector serialized_state; }; } diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index ab57526a15..775c9394d1 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -304,7 +304,7 @@ namespace realization { * Requests the BMI to clear a currently saved state from memory. * Existing state pointers should not be used as the stored data may be freed depending on implementation. */ - void free_serialization_state(); + virtual void free_serialization_state(); void set_realization_file_format(bool is_legacy_format); protected: diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 3d3a3bf692..57a88b5f3e 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -12,23 +12,7 @@ #include #include -enum class State_Save_Direction { - None = 0, - Save, - Load -}; - -enum class State_Save_Mechanism { - None = 0, - FilePerUnit -}; - -enum class State_Save_When { - None = 0, - EndOfRun, - FirstOfMonth, - StartOfRun -}; +#include "State_Save_Utils.hpp" class State_Saver; class State_Loader; diff --git a/include/state_save_restore/State_Save_Utils.hpp b/include/state_save_restore/State_Save_Utils.hpp index 8e6c127f6a..9713b660af 100644 --- a/include/state_save_restore/State_Save_Utils.hpp +++ b/include/state_save_restore/State_Save_Utils.hpp @@ -9,4 +9,22 @@ namespace StateSaveNames { const auto RESET = "reset_time"; } +enum class State_Save_Direction { + None = 0, + Save, + Load +}; + +enum class State_Save_Mechanism { + None = 0, + FilePerUnit +}; + +enum class State_Save_When { + None = 0, + EndOfRun, + FirstOfMonth, + StartOfRun +}; + #endif diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 0ceb36bc93..62b327ac32 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -95,14 +95,36 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const } const boost::span Bmi_Fortran_Formulation::get_serialization_state() { - auto model = dynamic_cast(get_bmi_model().get()); + auto model = this->get_bmi_model(); + // create the serialized state on the Fortran BMI int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); model->GetValue(StateSaveNames::SIZE, &size_int); - int *serialization_ptr = model->GetValuePtrInt(StateSaveNames::STATE); - char *serialization_state = reinterpret_cast(serialization_ptr); - const boost::span span(serialization_state, size_int); + // since GetValuePtr on the Fortran BMI does not work currently, store the data on the formulation + this->serialized_state.resize(size_int); + model->GetValue(StateSaveNames::STATE, this->serialized_state.data()); + // the BMI can have its state freed immediately since the data is now stored on the formulation + model->SetValue(StateSaveNames::FREE, &size_int); + // return a span of the data stored on the formulation + const boost::span(this->serialized_state.data(), this->serialized_state.size()); return span; } +void Bmi_Fortran_Formulation::load_serialization_state(boost::span state) { + auto model = this->get_bmi_model(); + int int_array_size = std::ceil(state.size() / static_cast(sizeof(int))); + // setting size is a workaround for loading the state. + // The BMI Fortran interface shapes the incoming pointer to the same size as the data currently backing the BMI's variable. + // By setting the size, the BMI can lie about the size of its state variable to that interface. + model->SetValue(StateSaveNames::SIZE, &int_array_size); + model->SetValue(StateSaveNames::STATE, state.data()); +} + +void Bmi_Fortran_Formulation::free_serialization_state() { + // The serialized data needs to be stored on the formluation since GetValuePtr is not available on Fortran BMIs. + // The backing BMI's serialization data should already be freed during `get_serialization_state`, so clearing the formulation's data is all that is needed. + this->serialized_state.clear(); + this->serialized_state.shrink_to_fit(); +} + #endif // NGEN_WITH_BMI_FORTRAN From 3dff58c3d954293f8050d16f2ff2933279697603 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 17 Feb 2026 09:25:14 -0500 Subject: [PATCH 60/93] Add variable name --- src/realizations/catchment/Bmi_Fortran_Formulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 62b327ac32..7e7407e2b3 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -106,7 +106,7 @@ const boost::span Bmi_Fortran_Formulation::get_serialization_state() { // the BMI can have its state freed immediately since the data is now stored on the formulation model->SetValue(StateSaveNames::FREE, &size_int); // return a span of the data stored on the formulation - const boost::span(this->serialized_state.data(), this->serialized_state.size()); + const boost::span span(this->serialized_state.data(), this->serialized_state.size()); return span; } From 57c382223ed77b79abd871c99c377d22b681d584 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 09:51:18 -0500 Subject: [PATCH 61/93] Update submodules for testing --- extern/LASAM | 2 +- extern/SoilFreezeThaw/SoilFreezeThaw | 2 +- extern/SoilMoistureProfiles/SoilMoistureProfiles | 2 +- extern/cfe/cfe | 2 +- extern/evapotranspiration/evapotranspiration | 2 +- extern/lstm | 2 +- extern/noah-owp-modular/noah-owp-modular | 2 +- extern/sac-sma/sac-sma | 2 +- extern/sloth | 2 +- extern/snow17 | 2 +- extern/t-route | 2 +- extern/topmodel/topmodel | 2 +- extern/ueb-bmi | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/extern/LASAM b/extern/LASAM index 664f5fbd38..f31e9fbb54 160000 --- a/extern/LASAM +++ b/extern/LASAM @@ -1 +1 @@ -Subproject commit 664f5fbd3869e51a405afa9aaf6cb4dcc96e09b9 +Subproject commit f31e9fbb54307bc918f13831e1810f533ea942cd diff --git a/extern/SoilFreezeThaw/SoilFreezeThaw b/extern/SoilFreezeThaw/SoilFreezeThaw index e7fd675f07..1ac738e816 160000 --- a/extern/SoilFreezeThaw/SoilFreezeThaw +++ b/extern/SoilFreezeThaw/SoilFreezeThaw @@ -1 +1 @@ -Subproject commit e7fd675f074e4755dc9953a7ae68668064cd7de6 +Subproject commit 1ac738e816b31aac190e8f6e19ad1226b07b1632 diff --git a/extern/SoilMoistureProfiles/SoilMoistureProfiles b/extern/SoilMoistureProfiles/SoilMoistureProfiles index 705798948d..1a55a16a33 160000 --- a/extern/SoilMoistureProfiles/SoilMoistureProfiles +++ b/extern/SoilMoistureProfiles/SoilMoistureProfiles @@ -1 +1 @@ -Subproject commit 705798948d899f7a05a083141cc04c189684aa1c +Subproject commit 1a55a16a337aceb14043d7b2e7e751b7c7bee0b6 diff --git a/extern/cfe/cfe b/extern/cfe/cfe index 4dfd64f43b..855c58665b 160000 --- a/extern/cfe/cfe +++ b/extern/cfe/cfe @@ -1 +1 @@ -Subproject commit 4dfd64f43bdd851affff540b6e2a9032ef301be9 +Subproject commit 855c58665bad8c1668604c110f9f74d6456fe724 diff --git a/extern/evapotranspiration/evapotranspiration b/extern/evapotranspiration/evapotranspiration index 096208ad62..836c146cbe 160000 --- a/extern/evapotranspiration/evapotranspiration +++ b/extern/evapotranspiration/evapotranspiration @@ -1 +1 @@ -Subproject commit 096208ad624e07216617f770a3447eb829266112 +Subproject commit 836c146cbeef10740af0dd2e570a7764bf4dadd2 diff --git a/extern/lstm b/extern/lstm index ce43783660..72c2ab73d8 160000 --- a/extern/lstm +++ b/extern/lstm @@ -1 +1 @@ -Subproject commit ce43783660642f9952147a76b81b4cbd4e5c3ad4 +Subproject commit 72c2ab73d80652aac18a3d35b2b5af212eb88122 diff --git a/extern/noah-owp-modular/noah-owp-modular b/extern/noah-owp-modular/noah-owp-modular index 25579b4948..dd9260175d 160000 --- a/extern/noah-owp-modular/noah-owp-modular +++ b/extern/noah-owp-modular/noah-owp-modular @@ -1 +1 @@ -Subproject commit 25579b4948e28e5afd0bed3e99e08a806fd9fc7c +Subproject commit dd9260175d2783abe50f6c409885b8e1fd097f5e diff --git a/extern/sac-sma/sac-sma b/extern/sac-sma/sac-sma index b40f61ca9e..eef00b537d 160000 --- a/extern/sac-sma/sac-sma +++ b/extern/sac-sma/sac-sma @@ -1 +1 @@ -Subproject commit b40f61ca9e82d4c4d0fc6171b314714af0160ab3 +Subproject commit eef00b537d556933c7bd645fc5aadcea6b2452c1 diff --git a/extern/sloth b/extern/sloth index ee0d982ccc..2745e1b0f9 160000 --- a/extern/sloth +++ b/extern/sloth @@ -1 +1 @@ -Subproject commit ee0d982ccc07663cfea7bf0ac4d645841e19ccc1 +Subproject commit 2745e1b0f954f5a98afa00f844e96bb436827996 diff --git a/extern/snow17 b/extern/snow17 index dbcfd09da7..043c625659 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit dbcfd09da7602bfdb860a2f9ccb770af95fb4b36 +Subproject commit 043c625659e4235887cb27acf1f1d34e5f87fc9e diff --git a/extern/t-route b/extern/t-route index bae205ff72..c7f2953b42 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit bae205ff723b669bba43c7fa353451c5fbe1eb0d +Subproject commit c7f2953b42cbc264f7315d81c9175af0f244bade diff --git a/extern/topmodel/topmodel b/extern/topmodel/topmodel index fa4f7e56db..b35249c195 160000 --- a/extern/topmodel/topmodel +++ b/extern/topmodel/topmodel @@ -1 +1 @@ -Subproject commit fa4f7e56dbe46df8cc0d7ca9095102290170b866 +Subproject commit b35249c1954abee61885bbd75d8b1765b5a311a5 diff --git a/extern/ueb-bmi b/extern/ueb-bmi index e661f7afe4..8e1e8dd0ce 160000 --- a/extern/ueb-bmi +++ b/extern/ueb-bmi @@ -1 +1 @@ -Subproject commit e661f7afe456bb380853abdc941d0cfa21df5c5e +Subproject commit 8e1e8dd0ce941fb385b046a9ab94dabc2000caa6 From 082383d6dd42b0a373f45ce9990d659a57d0ac84 Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Sun, 25 Jan 2026 11:54:12 -0500 Subject: [PATCH 62/93] Remove explicit pip install call for package nextgen_forcings_ewts --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a20732bdc3..fef1bf71c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -195,8 +195,7 @@ RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ pip3 install 'pandas' && \ pip3 install 'pyyml' && \ pip3 install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - pip install /ngen-app/ngen-forcing/ && \ - pip install /ngen-app/ngen-forcing/nextgen_forcings_ewts/ + pip install /ngen-app/ngen-forcing WORKDIR /ngen-app/ From eefea2d49c913205fdc116b0a39432b222059c1b Mon Sep 17 00:00:00 2001 From: Max Kipp Date: Sat, 31 Jan 2026 23:40:32 -0500 Subject: [PATCH 63/93] Add new Dockerfile arg to specify the full name of the ngen-forcing base image --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fef1bf71c9..ab073b41f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,9 @@ # Stage: Base – Common Setup ############################## ARG NGEN_FORCING_IMAGE_TAG=latest -FROM ghcr.io/ngwpc/ngen-bmi-forcing:${NGEN_FORCING_IMAGE_TAG} AS base +ARG NGEN_FORCING_IMAGE=ghcr.io/ngwpc/ngen-bmi-forcing:${NGEN_FORCING_IMAGE_TAG} + +FROM ${NGEN_FORCING_IMAGE} AS base # Uncomment when building locally #FROM ngen-bmi-forcing AS base From 4cfa923a8daa62eb266296117ffdad3dedcf3270 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Wed, 4 Feb 2026 20:25:43 -0800 Subject: [PATCH 64/93] updates to cicd and dockerfile --- .github/workflows/ngwpc-cicd.yml | 421 ++++++++++++++++++++++--------- Dockerfile | 23 +- Dockerfile.test | 35 +++ 3 files changed, 353 insertions(+), 126 deletions(-) create mode 100644 Dockerfile.test diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 4d0b8d2d99..7e1e8f04fa 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -7,6 +7,14 @@ on: branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate] release: types: [published] + repository_dispatch: + types: [base_image_published] + workflow_dispatch: + inputs: + trigger_downstream: + description: 'PENDING APPROVAL: Trigger downstream repositories pipeline after successful run' + required: true + type: boolean permissions: contents: read @@ -17,48 +25,163 @@ env: REGISTRY: ghcr.io jobs: + # set variables for use in other jobs setup: name: setup runs-on: ubuntu-latest outputs: + org: ${{ steps.vars.outputs.org }} image_base: ${{ steps.vars.outputs.image_base }} pr_tag: ${{ steps.vars.outputs.pr_tag }} commit_sha: ${{ steps.vars.outputs.commit_sha }} commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} test_image_tag: ${{ steps.vars.outputs.test_image_tag }} + alias_tag: ${{ steps.vars.outputs.alias_tag }} + forcing_tag: ${{ steps.vars.outputs.forcing_tag }} + clean_ref: ${{ steps.vars.outputs.clean_ref }} steps: - name: Compute image vars id: vars shell: bash run: | set -euo pipefail + + # one datetime for all time variables + NOW=$(date -u +'%Y-%m-%d %H:%M:%S') + + # for OCI labels + BUILD_DATE=$(date -u -d "$NOW" +'%Y-%m-%dT%H:%M:%SZ') + + # for Docker image tags + TIMESTAMP=$(date -u -d "$NOW" +'%Y%m%d%H%M%SZ') + + # set forcing tag from payload or default to 'latest' + if [ -n "${{ github.event.client_payload.forcing_tag || '' }}" ]; then + FORCING_TAG="${{ github.event.client_payload.forcing_tag }}" + else + FORCING_TAG="latest" + fi + + # logic to get the real branch name even on pull requests + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + REAL_REF="${{ github.head_ref }}" + else + REAL_REF="${{ github.ref_name }}" + fi + + # clean ref name and short commit sha + CLEAN_REF=$(echo "$REAL_REF" | tr '[:upper:]' '[:lower:]' | sed 's/\//-/g') + SHORT_SHA="${GITHUB_SHA:0:12}" + + # set variables to use with Docker images ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" REPO="$(basename "${GITHUB_REPOSITORY}")" IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" - echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" - if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then - PR_NUM="${{ github.event.pull_request.number }}" - PR_TAG="pr-${PR_NUM}-build" - echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" - echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" - fi + # logic for the tags: + # test_image_tag: used for the initial build and test + # alias_tag: used for final tagging on successful tests + # for pull requests, use pr--build + # for workflow_dispatch and pushes, use - + + # test tag is always commit short sha + TEST_TAG="${SHORT_SHA}" - if [ "${GITHUB_EVENT_NAME}" = "push" ]; then - COMMIT_SHA="${GITHUB_SHA}" - SHORT_SHA="${COMMIT_SHA:0:12}" - echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" - echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" - echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + ALIAS="pr-${{ github.event.pull_request.number }}-build" + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] || [ "$GITHUB_EVENT_NAME" = "push" ]; then + ALIAS="${TIMESTAMP}-${CLEAN_REF}" + else + ALIAS="${SHORT_SHA}" # fallback fi + # save outputs + echo "org=${ORG}" >> "$GITHUB_OUTPUT" + echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" + echo "build_date=${BUILD_DATE}" >> "$GITHUB_OUTPUT" + echo "forcing_tag=${FORCING_TAG}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${TEST_TAG}" >> "$GITHUB_OUTPUT" + echo "alias_tag=${ALIAS}" >> "$GITHUB_OUTPUT" + echo "commit_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "clean_ref=${CLEAN_REF}" >> "$GITHUB_OUTPUT" + + # CodeQL scan + codeql-scan: + name: codeql-scan + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'repository_dispatch') || + (github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + needs: [setup] + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + fetch-depth: 0 + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake g++ mpi-default-bin libopenmpi-dev libboost-all-dev libudunits2-dev libnetcdf-dev libnetcdf-c++4-dev + python3 -m pip install numpy==1.26.4 netcdf4 bmipy pandas torch pyyaml pyarrow + #python3 -m pip install -r extern/test_bmi_py/requirements.txt + #python3 -m pip install -r extern/t-route/requirements.txt + + # Initialize CodeQL (C++ selected) + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: cpp + + # Build (replicate your CMake build commands from Dockerfile or local builds) + - name: Build C++ code + env: + PYTHONPATH: ${{ env.PYTHONPATH }} + run: | + cmake -B cmake_build -S . \ + -DPYTHON_EXECUTABLE=$(which python3) \ + -DNGEN_WITH_MPI=ON \ + -DNGEN_WITH_NETCDF=ON \ + -DNGEN_WITH_SQLITE=ON \ + -DNGEN_WITH_UDUNITS=ON \ + -DNGEN_WITH_BMI_FORTRAN=ON \ + -DNGEN_WITH_BMI_C=ON \ + -DNGEN_WITH_PYTHON=ON \ + -DNGEN_WITH_TESTS=OFF \ + -DNGEN_WITH_ROUTING=ON \ + -DNGEN_QUIET=ON \ + -DNGEN_UPDATE_GIT_SUBMODULES=OFF \ + # Using boost from apt for simplicity and code scanning + #-DBOOST_ROOT=/opt/boost + cmake --build cmake_build --target all + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + +# build ngen Docker image build: name: build - if: github.event_name == 'pull_request' || github.event_name == 'push' + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'repository_dispatch') || + (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: setup + needs: + [ + setup, + codeql-scan + ] + outputs: + digest: ${{ steps.build_image.outputs.digest }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: recursive fetch-depth: 0 @@ -84,17 +207,42 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build & push image + id: build_image uses: docker/build-push-action@v6 + env: + BUILD_DATE: ${{ env.BUILD_DATE }} with: context: . + # file: Dockerfile.test # comment out when done testing push: true tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} + build-args: | + ORG=${{ needs.setup.outputs.org }} + NGEN_FORCING_IMAGE_TAG=${{ needs.setup.outputs.forcing_tag }} + BASE_IMAGE_DIGEST=${{ github.event.client_payload.digest || 'unknown' }} + BASE_IMAGE_REVISION=${{ github.event.client_payload.source_sha || 'unknown' }} + IMAGE_SOURCE=https://github.com/${{ github.repository }} + IMAGE_VENDOR=${{ github.repository_owner }} + IMAGE_VERSION=${{ github.ref_name }} + IMAGE_REVISION=${{ github.sha }} + IMAGE_CREATED=${{ env.BUILD_DATE }} + CI_COMMIT_REF_NAME=${{ needs.setup.outputs.clean_ref }} +# run unit tests inside the built Docker image unit-test: name: unit-test - if: github.event_name == 'pull_request' || github.event_name == 'push' + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'repository_dispatch') || + (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: [setup, build] + needs: + [ + setup, + codeql-scan, + build + ] container: image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} steps: @@ -109,123 +257,94 @@ jobs: exit 1; } - # SonarQube scan (only runs on internal NGWPC self-hosted runners) - sonarqube-internal: - name: sonarqube-internal - if: (github.event_name == 'pull_request' || github.event_name == 'push') && github.repository_owner == 'NGWPC' - runs-on: self-hosted - needs: [setup, build, unit-test] - #TODO: Configure SonarQube Scans - continue-on-error: true - container: - image: sonarsource/sonar-scanner-cli - options: --entrypoint="" --user 0 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: SonarQube Scan - env: - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: sonar-scanner -X -Dsonar.verbose=true - - # CodeQL scan - # TODO: Update to scan as desired. - # Added as a minimal MVP for Static Code Analysis during GitHub migration - codeql-scan: - name: codeql-scan - if: github.event_name == 'pull_request' || github.event_name == 'push' +# run container security scan using Trivy + container-scanning: + name: container-scanning + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'repository_dispatch') || + (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: [setup, build] - permissions: - actions: read - contents: read - security-events: write + needs: + [ + setup, + codeql-scan, + build + ] steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - name: Install build dependencies - run: | - sudo apt-get update - sudo apt-get install -y cmake g++ mpi-default-bin libopenmpi-dev libboost-all-dev libudunits2-dev libnetcdf-dev libnetcdf-c++4-dev - python3 -m pip install numpy==1.26.4 netcdf4 bmipy pandas torch pyyaml pyarrow - #python3 -m pip install -r extern/test_bmi_py/requirements.txt - #python3 -m pip install -r extern/t-route/requirements.txt - # Initialize CodeQL (C++ selected) - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + - name: Install Trivy + uses: aquasecurity/setup-trivy@v0.2.2 with: - languages: cpp - # Build (replicate your CMake build commands from Dockerfile or local builds) - - name: Build C++ code + cache: true + version: v0.68.2 + + - name: Trivy scan env: - PYTHONPATH: ${{ env.PYTHONPATH }} + TMPDIR: /mnt/trivy-temp run: | - cmake -B cmake_build -S . \ - -DPYTHON_EXECUTABLE=$(which python3) \ - -DNGEN_WITH_MPI=ON \ - -DNGEN_WITH_NETCDF=ON \ - -DNGEN_WITH_SQLITE=ON \ - -DNGEN_WITH_UDUNITS=ON \ - -DNGEN_WITH_BMI_FORTRAN=ON \ - -DNGEN_WITH_BMI_C=ON \ - -DNGEN_WITH_PYTHON=ON \ - -DNGEN_WITH_TESTS=OFF \ - -DNGEN_WITH_ROUTING=ON \ - -DNGEN_QUIET=ON \ - -DNGEN_UPDATE_GIT_SUBMODULES=OFF \ - # Using boost from apt for simplicity and code scanning - #-DBOOST_ROOT=/opt/boost - cmake --build cmake_build --target all - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + sudo mkdir -p $TMPDIR + sudo chown -R $USER:$USER $TMPDIR - container-scanning: - name: container-scanning - if: github.event_name == 'pull_request' || github.event_name == 'push' - runs-on: ubuntu-latest - needs: [setup, build] - steps: - - name: Scan container with Trivy - uses: aquasecurity/trivy-action@0.20.0 - with: - image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} - format: 'template' - template: '@/contrib/sarif.tpl' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' - - name: Upload Trivy results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: 'trivy-results.sarif' + trivy image \ + --format sarif \ + --output trivy-results.sarif \ + --severity CRITICAL,HIGH \ + --scanners vuln \ + --ignore-unfixed \ + --timeout 45m \ + ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} - deploy-latest-on-development: - name: deploy-latest-on-development - if: github.event_name == 'push' && github.ref_name == 'development' +# promote Docker image tags after successful tests + promote-tags: + name: Promote Tags + if: | + (github.event_name == 'pull_request') || + (github.event_name == 'push') || + (github.event_name == 'repository_dispatch') || + (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: [setup, build, unit-test, sonarqube-internal, codeql-scan, container-scanning] + needs: + [ + setup, + codeql-scan, + build, + unit-test, + container-scanning, + ] steps: - - name: Tag image with 'latest' + - name: Tag image with alias and latest shell: bash run: | set -euo pipefail - IMAGE_BASE="${{ needs.setup.outputs.image_base }}" - SHORT_SHA="${{ needs.setup.outputs.commit_sha_short }}" - # ensure skopeo is available + # ensure skopeo is available for promotion if ! command -v skopeo >/dev/null 2>&1; then sudo apt-get update -y sudo apt-get install -y --no-install-recommends skopeo fi + IMAGE_BASE="${{ needs.setup.outputs.image_base }}" + TEST_TAG="${{ needs.setup.outputs.test_image_tag }}" + ALIAS_TAG="${{ needs.setup.outputs.alias_tag }}" + + # apply the primary alias (pr tag or timestamp-branch) skopeo copy \ --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:latest" + --all \ + "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}" + # additionally tag with 'latest' if on development branch push + if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF_NAME" = "development" ]; then + skopeo copy \ + --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ + --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ + --all \ + "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:latest" + fi + +# tag official release Docker image with release tag release: name: release if: github.event_name == 'release' && github.event.action == 'published' @@ -239,21 +358,10 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail + TAG="${{ github.event.release.tag_name }}" REPO="${{ github.repository }}" - # ensure jq is available - if ! command -v jq >/dev/null 2>&1; then - sudo apt-get update -y - sudo apt-get install -y --no-install-recommends jq - fi - - # ensure gh cli is available - if ! command -v gh >/dev/null 2>&1; then - sudo apt-get update -y - sudo apt-get install -y --no-install-recommends gh - fi - REF_JSON="$(gh api "repos/${REPO}/git/refs/tags/${TAG}")" OBJ_SHA="$(jq -r '.object.sha' <<<"$REF_JSON")" OBJ_TYPE="$(jq -r '.object.type' <<<"$REF_JSON")" @@ -272,6 +380,7 @@ jobs: shell: bash run: | set -euo pipefail + IMAGE_BASE="${{ needs.setup.outputs.image_base }}" SHORT_SHA="${{ steps.rev.outputs.short_sha }}" RELEASE_TAG="${{ github.event.release.tag_name }}" @@ -286,3 +395,65 @@ jobs: --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:${RELEASE_TAG}" + + # trigger repos downstream of ngen to build on new base image + # notify-downstream: + # name: notify-downstream + # if: | + # (github.event_name == 'push') || + # (github.event_name == 'repository_dispatch') || + # (github.event_name == 'workflow_dispatch' && inputs.trigger_downstream) + # runs-on: ubuntu-latest + # needs: + # [ + # setup, + # codeql-scan, + # build, + # unit-test, + # container-scanning, + # promote-tags + # ] + # env: + # # list of downstream repo names to notify + # DOWNSTREAM_REPOS: '["nwm-cal-mgr", "nwm-fcst-mgr"]' + # steps: + # - name: Trigger Dispatch + # uses: actions/github-script@v8 + # with: + # github-token: ${{ secrets.NWM_CICD_PAT }} + # script: | + # // set variables + # const repos = JSON.parse(process.env.DOWNSTREAM_REPOS); + # const owner = context.repo.owner; + # const repo = context.repo.repo; + # const dockerImageTag = "${{ needs.setup.outputs.alias_tag }}"; + + # // construct payload + # const payload = { + # event_type: 'base_image_published', + # client_payload: { + # owner: owner, + # repository: repo, + # docker_image: '${{ needs.setup.outputs.image_base }}', + # docker_tag: dockerImageTag, + # digest: '${{ needs.build.outputs.digest }}', + # source_sha: '${{ needs.setup.outputs.commit_sha }}', + # } + # }; + + # // iterate downstream repos and dispatch + # for (const downstreamRepoName of repos) { + # console.log(`Dispatching to ${owner}/${downstreamRepoName} with tag '${dockerImageTag}'...`); + # try { + # await github.rest.repos.createDispatchEvent({ + # owner: owner, + # repo: downstreamRepoName, + # event_type: payload.event_type, + # client_payload: payload.client_payload + # }); + # console.log(`✅ Successfully triggered ${downstreamRepoName}`); + # } catch (error) { + # console.error(`❌ Failed to trigger ${downstreamRepoName}:`, error.message); + # core.setFailed(`Failed to dispatch to ${downstreamRepoName}`); + # } + # } diff --git a/Dockerfile b/Dockerfile index ab073b41f7..01d1851ab1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ ############################## # Stage: Base – Common Setup ############################## +ARG ORG=ngwpc ARG NGEN_FORCING_IMAGE_TAG=latest ARG NGEN_FORCING_IMAGE=ghcr.io/ngwpc/ngen-bmi-forcing:${NGEN_FORCING_IMAGE_TAG} @@ -11,6 +12,26 @@ FROM ${NGEN_FORCING_IMAGE} AS base # Uncomment when building locally #FROM ngen-bmi-forcing AS base +# OCI Metadata Arguments +ARG NGEN_FORCING_IMAGE +ARG BASE_IMAGE_DIGEST="unknown" +ARG BASE_IMAGE_REVISION="unknown" +ARG IMAGE_SOURCE="unknown" +ARG IMAGE_VENDOR="unknown" +ARG IMAGE_VERSION="unknown" +ARG IMAGE_REVISION="unknown" +ARG IMAGE_CREATED="unknown" + +# OCI Standard Labels +LABEL org.opencontainers.image.base.name="${NGEN_FORCING_IMAGE}" \ + org.opencontainers.image.base.digest="${BASE_IMAGE_DIGEST}" \ + io.ngwpc.image.base.revision="${BASE_IMAGE_REVISION}" \ + org.opencontainers.image.source="${IMAGE_SOURCE}" \ + org.opencontainers.image.vendor="${IMAGE_VENDOR}" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.revision="${IMAGE_REVISION}" \ + org.opencontainers.image.created="${IMAGE_CREATED}" + # cannot remove LANG even though https://bugs.python.org/issue19846 is fixed # last attempted removal of LANG broke many users: # https://github.com/docker-library/python/pull/570 @@ -279,7 +300,7 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-snow17 \ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-sac-sma \ set -eux && \ cmake -B extern/sac-sma/cmake_build -S extern/sac-sma/ -DBOOST_ROOT=/opt/boost && \ - cmake --build extern/sac-sma/cmake_build/ && \ + cmake --build extern/sac-sma/cmake_build/ && \ find /ngen-app/ngen/extern/sac-sma -name '*.o' -exec rm -f {} + RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-soilmoistureprofiles \ diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000000..d636fa8adb --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,35 @@ +# Dockerfile.test +ARG ORG=ngwpc +ARG NGEN_FORCING_IMAGE_TAG=latest +ARG BASE_IMAGE_NAME="alpine:latest" + +FROM ${BASE_IMAGE_NAME} + +# OCI Metadata Arguments +ARG BASE_IMAGE_NAME +ARG BASE_IMAGE_DIGEST="unknown" +ARG BASE_IMAGE_REVISION="unknown" +ARG IMAGE_SOURCE="unknown" +ARG IMAGE_VENDOR="unknown" +ARG IMAGE_VERSION="unknown" +ARG IMAGE_REVISION="unknown" +ARG IMAGE_CREATED="unknown" + +# OCI Standard Labels +LABEL org.opencontainers.image.base.name="${BASE_IMAGE_NAME}" \ + org.opencontainers.image.base.digest="${BASE_IMAGE_DIGEST}" \ + io.ngwpc.image.base.revision="${BASE_IMAGE_REVISION}" \ + org.opencontainers.image.source="${IMAGE_SOURCE}" \ + org.opencontainers.image.vendor="${IMAGE_VENDOR}" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.revision="${IMAGE_REVISION}" \ + org.opencontainers.image.created="${IMAGE_CREATED}" + +# Create a dummy file just to prove we did something +RUN echo "This is a lightweight test build for pipeline verification." > /build_info.txt + +# Add a timestamp so every build layer looks slightly different (optional, forces fresh hash) +ARG BUILD_DATE +RUN echo "Built on $BUILD_DATE" >> /build_info.txt + +CMD ["cat", "/build_info.txt"] From 18e0f2c6e60bdc66aa38aa8530bf9f7077d0c50a Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Wed, 4 Feb 2026 21:56:39 -0800 Subject: [PATCH 65/93] updated cicd file --- .github/workflows/ngwpc-cicd.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 7e1e8f04fa..ec615094f1 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -173,11 +173,7 @@ jobs: (github.event_name == 'repository_dispatch') || (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: - [ - setup, - codeql-scan - ] + needs: [setup] outputs: digest: ${{ steps.build_image.outputs.digest }} steps: @@ -240,7 +236,6 @@ jobs: needs: [ setup, - codeql-scan, build ] container: @@ -269,7 +264,6 @@ jobs: needs: [ setup, - codeql-scan, build ] steps: From f0a392ccaff7ac104d3841fc8f9c09232bb5ca8d Mon Sep 17 00:00:00 2001 From: "siva.selvanathan" Date: Wed, 4 Feb 2026 16:39:10 -0500 Subject: [PATCH 66/93] Added unit conversion code to ForcingsEngineLumpedDataProvider which resolves the SWE bug. --- .../ForcingsEngineLumpedDataProvider.cpp | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/forcing/ForcingsEngineLumpedDataProvider.cpp b/src/forcing/ForcingsEngineLumpedDataProvider.cpp index 2865072352..6a4b5075a4 100644 --- a/src/forcing/ForcingsEngineLumpedDataProvider.cpp +++ b/src/forcing/ForcingsEngineLumpedDataProvider.cpp @@ -3,6 +3,7 @@ #include #include // for std::put_time #include +#include #include namespace data_access { @@ -175,7 +176,8 @@ Provider::data_type Provider::get_value( } auto variable = ensure_variable(selector.get_variable_name()); - + auto output_units = selector.get_output_units(); + if (m == ReSampleMethod::SUM || m == ReSampleMethod::MEAN) { double acc = 0.0; const auto start = clock_type::from_time_t(selector.get_init_time()); @@ -226,14 +228,23 @@ Provider::data_type Provider::get_value( bmi_->UpdateUntil(std::chrono::duration_cast(current - time_begin_).count()); acc += static_cast(bmi_->GetValuePtr(variable))[divide_idx_]; } - if (m == ReSampleMethod::MEAN) { auto duration = std::chrono::duration_cast(current - start).count(); auto num_time_steps = duration / time_step_.count(); acc /= num_time_steps; } - - return acc; + // Convert units + try { + return UnitsHelper::get_converted_value(bmi_->GetVarUnits(variable), acc, output_units); + } + catch (const std::runtime_error& e) { + data_access::unit_conversion_exception uce(e.what()); + uce.provider_model_name = "ForcedEngineLumpedDataProvider " + selector.get_id(); + uce.provider_bmi_var_name = variable; + uce.provider_units = bmi_->GetVarUnits(variable); + uce.unconverted_values.push_back(acc); + throw uce; + } } ss.str(""); @@ -258,6 +269,7 @@ std::vector Provider::get_values( } auto variable = ensure_variable(selector.get_variable_name()); + auto output_units = selector.get_output_units(); const auto start = clock_type::from_time_t(selector.get_init_time()); const auto end = std::chrono::seconds{selector.get_duration_secs()} + start; @@ -307,7 +319,19 @@ std::vector Provider::get_values( while (current < end) { current += time_step_; bmi_->UpdateUntil(std::chrono::duration_cast(current - time_begin_).count()); - values.push_back(static_cast(bmi_->GetValuePtr(variable))[divide_idx_]); + double var_value = static_cast(bmi_->GetValuePtr(variable))[divide_idx_]; + // Convert units + try { + values.push_back(UnitsHelper::get_converted_value(bmi_->GetVarUnits(variable), var_value, output_units)); + } + catch (const std::runtime_error& e) { + data_access::unit_conversion_exception uce(e.what()); + uce.provider_model_name = "ForcedEngineLumpedDataProvider " + selector.get_id(); + uce.provider_bmi_var_name = variable; + uce.provider_units = bmi_->GetVarUnits(variable); + uce.unconverted_values.push_back(var_value); + throw uce; + } } return values; From 65a113b5c8396f79d0cd5bbca3905c4c8ac1a037 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 10 Feb 2026 20:39:25 -0800 Subject: [PATCH 67/93] updated cicd file --- .github/workflows/ngwpc-cicd.yml | 66 ++++++++++++++++---------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index ec615094f1..e4e15ea957 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -2,19 +2,27 @@ name: CI/CD Pipeline on: pull_request: - branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate] + branches: + - ngwpc-candidate + - ngwpc-release + - main + - nwm-main + - development + - release-candidate push: - branches: [ngwpc-candidate, ngwpc-release, main, nwm-main, development, release-candidate] - release: - types: [published] - repository_dispatch: - types: [base_image_published] + branches: + - ngwpc-candidate + - ngwpc-release + - main + - nwm-main + - development + - release-candidate workflow_dispatch: inputs: - trigger_downstream: - description: 'PENDING APPROVAL: Trigger downstream repositories pipeline after successful run' - required: true - type: boolean + NGEN_FORCING_IMAGE_TAG: + description: 'NGEN_FORCING_IMAGE_TAG' + required: false + type: string permissions: contents: read @@ -37,7 +45,7 @@ jobs: commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} test_image_tag: ${{ steps.vars.outputs.test_image_tag }} alias_tag: ${{ steps.vars.outputs.alias_tag }} - forcing_tag: ${{ steps.vars.outputs.forcing_tag }} + build_date: ${{ steps.vars.outputs.build_date }} clean_ref: ${{ steps.vars.outputs.clean_ref }} steps: - name: Compute image vars @@ -46,6 +54,11 @@ jobs: run: | set -euo pipefail + # set variables to use with Docker images + ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" + REPO="$(basename "${GITHUB_REPOSITORY}")" + IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" + # one datetime for all time variables NOW=$(date -u +'%Y-%m-%d %H:%M:%S') @@ -55,44 +68,35 @@ jobs: # for Docker image tags TIMESTAMP=$(date -u -d "$NOW" +'%Y%m%d%H%M%SZ') - # set forcing tag from payload or default to 'latest' - if [ -n "${{ github.event.client_payload.forcing_tag || '' }}" ]; then - FORCING_TAG="${{ github.event.client_payload.forcing_tag }}" - else - FORCING_TAG="latest" - fi - - # logic to get the real branch name even on pull requests + # logic to get the real branch name and commit SHA on pull requests if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then REAL_REF="${{ github.head_ref }}" + REAL_SHA="${{ github.event.pull_request.head.sha }}" else REAL_REF="${{ github.ref_name }}" + REAL_SHA="${GITHUB_SHA}" fi # clean ref name and short commit sha CLEAN_REF=$(echo "$REAL_REF" | tr '[:upper:]' '[:lower:]' | sed 's/\//-/g') SHORT_SHA="${GITHUB_SHA:0:12}" - # set variables to use with Docker images - ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" - REPO="$(basename "${GITHUB_REPOSITORY}")" - IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" - # logic for the tags: - # test_image_tag: used for the initial build and test + # test_image_tag (commit short sha): used for the initial build and test # alias_tag: used for final tagging on successful tests - # for pull requests, use pr--build - # for workflow_dispatch and pushes, use - # test tag is always commit short sha TEST_TAG="${SHORT_SHA}" if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + # for pull requests, use pr--build ALIAS="pr-${{ github.event.pull_request.number }}-build" - elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] || [ "$GITHUB_EVENT_NAME" = "push" ]; then - ALIAS="${TIMESTAMP}-${CLEAN_REF}" + elif [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "${{ github.ref_type }}" = "tag" ]; then + # for manual workflow dispatch on tags, use the git tag + ALIAS="${CLEAN_REF}" else - ALIAS="${SHORT_SHA}" # fallback + # for pushes to branches, use timestamp-branchname + ALIAS="${TIMESTAMP}-${CLEAN_REF}" fi # save outputs @@ -215,8 +219,6 @@ jobs: build-args: | ORG=${{ needs.setup.outputs.org }} NGEN_FORCING_IMAGE_TAG=${{ needs.setup.outputs.forcing_tag }} - BASE_IMAGE_DIGEST=${{ github.event.client_payload.digest || 'unknown' }} - BASE_IMAGE_REVISION=${{ github.event.client_payload.source_sha || 'unknown' }} IMAGE_SOURCE=https://github.com/${{ github.repository }} IMAGE_VENDOR=${{ github.repository_owner }} IMAGE_VERSION=${{ github.ref_name }} From bdf27f483f02245a2e041762197ab26d6b087dd4 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 10 Feb 2026 20:56:44 -0800 Subject: [PATCH 68/93] updated cicd file --- .github/workflows/ngwpc-cicd.yml | 168 ++++--------------------------- 1 file changed, 17 insertions(+), 151 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index e4e15ea957..248114acab 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -103,10 +103,9 @@ jobs: echo "org=${ORG}" >> "$GITHUB_OUTPUT" echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" echo "build_date=${BUILD_DATE}" >> "$GITHUB_OUTPUT" - echo "forcing_tag=${FORCING_TAG}" >> "$GITHUB_OUTPUT" echo "test_image_tag=${TEST_TAG}" >> "$GITHUB_OUTPUT" echo "alias_tag=${ALIAS}" >> "$GITHUB_OUTPUT" - echo "commit_sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT" + echo "commit_sha=${REAL_SHA}" >> "$GITHUB_OUTPUT" echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" echo "clean_ref=${CLEAN_REF}" >> "$GITHUB_OUTPUT" @@ -116,10 +115,9 @@ jobs: if: | (github.event_name == 'pull_request') || (github.event_name == 'push') || - (github.event_name == 'repository_dispatch') || (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: [setup] + needs: setup permissions: actions: read contents: read @@ -136,13 +134,11 @@ jobs: python3 -m pip install numpy==1.26.4 netcdf4 bmipy pandas torch pyyaml pyarrow #python3 -m pip install -r extern/test_bmi_py/requirements.txt #python3 -m pip install -r extern/t-route/requirements.txt - # Initialize CodeQL (C++ selected) - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: languages: cpp - # Build (replicate your CMake build commands from Dockerfile or local builds) - name: Build C++ code env: @@ -164,7 +160,6 @@ jobs: # Using boost from apt for simplicity and code scanning #-DBOOST_ROOT=/opt/boost cmake --build cmake_build --target all - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 @@ -174,10 +169,9 @@ jobs: if: | (github.event_name == 'pull_request') || (github.event_name == 'push') || - (github.event_name == 'repository_dispatch') || (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - needs: [setup] + needs: setup outputs: digest: ${{ steps.build_image.outputs.digest }} steps: @@ -185,7 +179,6 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Verify submodules recursively run: | echo "Checking all submodules recursively..." @@ -198,19 +191,15 @@ jobs: pwd ls -l ' - - name: Log in to registry uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build & push image id: build_image uses: docker/build-push-action@v6 - env: - BUILD_DATE: ${{ env.BUILD_DATE }} with: context: . # file: Dockerfile.test # comment out when done testing @@ -218,12 +207,12 @@ jobs: tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} build-args: | ORG=${{ needs.setup.outputs.org }} - NGEN_FORCING_IMAGE_TAG=${{ needs.setup.outputs.forcing_tag }} + NGEN_FORCING_IMAGE_TAG=${{ github.event.inputs.NGEN_FORCING_IMAGE_TAG || 'latest' }} IMAGE_SOURCE=https://github.com/${{ github.repository }} IMAGE_VENDOR=${{ github.repository_owner }} - IMAGE_VERSION=${{ github.ref_name }} - IMAGE_REVISION=${{ github.sha }} - IMAGE_CREATED=${{ env.BUILD_DATE }} + IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }} + IMAGE_REVISION=${{ needs.setup.outputs.commit_sha }} + IMAGE_CREATED=${{ needs.setup.outputs.build_date }} CI_COMMIT_REF_NAME=${{ needs.setup.outputs.clean_ref }} # run unit tests inside the built Docker image @@ -232,14 +221,11 @@ jobs: if: | (github.event_name == 'pull_request') || (github.event_name == 'push') || - (github.event_name == 'repository_dispatch') || (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest needs: - [ - setup, - build - ] + - setup + - build container: image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} steps: @@ -260,14 +246,11 @@ jobs: if: | (github.event_name == 'pull_request') || (github.event_name == 'push') || - (github.event_name == 'repository_dispatch') || (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest needs: - [ - setup, - build - ] + - setup + - build steps: - name: Install Trivy uses: aquasecurity/setup-trivy@v0.2.2 @@ -297,17 +280,14 @@ jobs: if: | (github.event_name == 'pull_request') || (github.event_name == 'push') || - (github.event_name == 'repository_dispatch') || (github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest needs: - [ - setup, - codeql-scan, - build, - unit-test, - container-scanning, - ] + - setup + - codeql-scan + - build + - unit-test + - container-scanning steps: - name: Tag image with alias and latest shell: bash @@ -331,7 +311,7 @@ jobs: --all \ "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}" - # additionally tag with 'latest' if on development branch push + # tag with 'latest' on development branch push if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF_NAME" = "development" ]; then skopeo copy \ --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ @@ -339,117 +319,3 @@ jobs: --all \ "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:latest" fi - -# tag official release Docker image with release tag - release: - name: release - if: github.event_name == 'release' && github.event.action == 'published' - runs-on: ubuntu-latest - needs: setup - steps: - - name: Get commit sha for the tag - id: rev - shell: bash - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - - TAG="${{ github.event.release.tag_name }}" - REPO="${{ github.repository }}" - - REF_JSON="$(gh api "repos/${REPO}/git/refs/tags/${TAG}")" - OBJ_SHA="$(jq -r '.object.sha' <<<"$REF_JSON")" - OBJ_TYPE="$(jq -r '.object.type' <<<"$REF_JSON")" - - if [ "$OBJ_TYPE" = "tag" ]; then - TAG_OBJ="$(gh api "repos/${REPO}/git/tags/${OBJ_SHA}")" - COMMIT_SHA="$(jq -r '.object.sha' <<<"$TAG_OBJ")" - else - COMMIT_SHA="$OBJ_SHA" - fi - - SHORT_SHA="${COMMIT_SHA:0:12}" - echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" - - - name: Tag image with release tag - shell: bash - run: | - set -euo pipefail - - IMAGE_BASE="${{ needs.setup.outputs.image_base }}" - SHORT_SHA="${{ steps.rev.outputs.short_sha }}" - RELEASE_TAG="${{ github.event.release.tag_name }}" - - # ensure skopeo is available - if ! command -v skopeo >/dev/null 2>&1; then - sudo apt-get update -y - sudo apt-get install -y --no-install-recommends skopeo - fi - - skopeo copy \ - --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ - docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:${RELEASE_TAG}" - - # trigger repos downstream of ngen to build on new base image - # notify-downstream: - # name: notify-downstream - # if: | - # (github.event_name == 'push') || - # (github.event_name == 'repository_dispatch') || - # (github.event_name == 'workflow_dispatch' && inputs.trigger_downstream) - # runs-on: ubuntu-latest - # needs: - # [ - # setup, - # codeql-scan, - # build, - # unit-test, - # container-scanning, - # promote-tags - # ] - # env: - # # list of downstream repo names to notify - # DOWNSTREAM_REPOS: '["nwm-cal-mgr", "nwm-fcst-mgr"]' - # steps: - # - name: Trigger Dispatch - # uses: actions/github-script@v8 - # with: - # github-token: ${{ secrets.NWM_CICD_PAT }} - # script: | - # // set variables - # const repos = JSON.parse(process.env.DOWNSTREAM_REPOS); - # const owner = context.repo.owner; - # const repo = context.repo.repo; - # const dockerImageTag = "${{ needs.setup.outputs.alias_tag }}"; - - # // construct payload - # const payload = { - # event_type: 'base_image_published', - # client_payload: { - # owner: owner, - # repository: repo, - # docker_image: '${{ needs.setup.outputs.image_base }}', - # docker_tag: dockerImageTag, - # digest: '${{ needs.build.outputs.digest }}', - # source_sha: '${{ needs.setup.outputs.commit_sha }}', - # } - # }; - - # // iterate downstream repos and dispatch - # for (const downstreamRepoName of repos) { - # console.log(`Dispatching to ${owner}/${downstreamRepoName} with tag '${dockerImageTag}'...`); - # try { - # await github.rest.repos.createDispatchEvent({ - # owner: owner, - # repo: downstreamRepoName, - # event_type: payload.event_type, - # client_payload: payload.client_payload - # }); - # console.log(`✅ Successfully triggered ${downstreamRepoName}`); - # } catch (error) { - # console.error(`❌ Failed to trigger ${downstreamRepoName}:`, error.message); - # core.setFailed(`Failed to dispatch to ${downstreamRepoName}`); - # } - # } From 17643111e732b917d05bdd102b64b993f1d9ecc6 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 10 Feb 2026 21:36:32 -0800 Subject: [PATCH 69/93] updated cicd file --- .github/workflows/ngwpc-cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 248114acab..0a63d13ddb 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -79,7 +79,7 @@ jobs: # clean ref name and short commit sha CLEAN_REF=$(echo "$REAL_REF" | tr '[:upper:]' '[:lower:]' | sed 's/\//-/g') - SHORT_SHA="${GITHUB_SHA:0:12}" + SHORT_SHA="${GITHUB_SHA:0:7}" # logic for the tags: # test_image_tag (commit short sha): used for the initial build and test From 2121a51285cee6f2fece301b93eadec13a6f6582 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 10 Feb 2026 22:13:40 -0800 Subject: [PATCH 70/93] updated cicd file --- .github/workflows/ngwpc-cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 0a63d13ddb..f0f195946c 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -207,7 +207,7 @@ jobs: tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} build-args: | ORG=${{ needs.setup.outputs.org }} - NGEN_FORCING_IMAGE_TAG=${{ github.event.inputs.NGEN_FORCING_IMAGE_TAG || 'latest' }} + NGEN_FORCING_IMAGE_TAG=${{ inputs.NGEN_FORCING_IMAGE_TAG || 'latest' }} IMAGE_SOURCE=https://github.com/${{ github.repository }} IMAGE_VENDOR=${{ github.repository_owner }} IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }} From c470849a098df65c200c353a1e9ef6b2d1c5167c Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 10 Feb 2026 23:38:33 -0800 Subject: [PATCH 71/93] updated cicd file --- .github/workflows/ngwpc-cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index f0f195946c..a02ad7048a 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -79,7 +79,7 @@ jobs: # clean ref name and short commit sha CLEAN_REF=$(echo "$REAL_REF" | tr '[:upper:]' '[:lower:]' | sed 's/\//-/g') - SHORT_SHA="${GITHUB_SHA:0:7}" + SHORT_SHA="${REAL_SHA:0:7}" # logic for the tags: # test_image_tag (commit short sha): used for the initial build and test From 8dae76d8d5b8ff3577523f26085811d4d65f66b2 Mon Sep 17 00:00:00 2001 From: "peter.a.kronenberg" Date: Mon, 9 Feb 2026 16:59:06 -0500 Subject: [PATCH 72/93] Ignore git info --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index ff47c11f5f..12d641e297 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,7 @@ docker .gitpod.Dockerfile .gitpod.yml sonar-project.properties +*_git_info.json # exclude artifacts from native build cmake_build From 45eb0f9bd084de6a2756553556f9f7cb4b52b473 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 09:47:23 -0500 Subject: [PATCH 73/93] Merge remote-tracking branch 'NOAA-OWP/master' into development --- .github/actions/ngen-build/action.yaml | 97 +- .github/actions/ngen-submod-build/action.yaml | 8 + .github/workflows/module_integration.yml | 2 +- .github/workflows/test_and_validate.yml | 52 +- CMakeLists.txt | 10 +- INSTALL.md | 10 +- doc/BUILDS_AND_CMAKE.md | 2 +- doc/DEPENDENCIES.md | 10 +- docker/CENTOS_4.8.5_NGEN_RUN.dockerfile | 6 +- docker/CENTOS_NGEN_RUN.dockerfile | 8 +- docker/CENTOS_TEST.dockerfile | 6 +- docker/CENTOS_latest_NGEN_RUN.dockerfile | 6 +- docker/RHEL_TEST.dockerfile | 6 +- docker/ngen.dockerfile | 2 +- extern/iso_c_fortran_bmi/CMakeLists.txt | 2 +- extern/test_bmi_c/include/bmi_test_bmi_c.h | 5 + extern/test_bmi_c/include/test_bmi_c.h | 3 + extern/test_bmi_c/src/bmi_test_bmi_c.c | 50 + extern/test_bmi_c/src/test_bmi_c.c | 2 + extern/test_bmi_cpp/include/test_bmi_cpp.hpp | 13 + extern/test_bmi_cpp/src/test_bmi_cpp.cpp | 27 + .../core/nexus/HY_PointHydroNexusRemote.hpp | 1 + .../catchment/Bmi_Module_Formulation.hpp | 8 + .../catchment/Bmi_Multi_Formulation.hpp | 9 + .../realizations/catchment/Formulation.hpp | 1 + include/utilities/bmi/mass_balance.hpp | 155 + include/utilities/bmi/nonstd/LICENSE.txt | 23 + include/utilities/bmi/nonstd/expected.hpp | 3637 +++++++++++++++++ .../utilities/bmi/nonstd/expected.tweak.hpp | 4 + include/utilities/bmi/protocol.hpp | 199 + include/utilities/bmi/protocols.hpp | 100 + src/core/Layer.cpp | 9 + src/core/nexus/HY_PointHydroNexusRemote.cpp | 126 +- src/geopackage/CMakeLists.txt | 2 +- .../catchment/Bmi_Module_Formulation.cpp | 3 + src/realizations/catchment/CMakeLists.txt | 1 + src/utilities/bmi/CMakeLists.txt | 36 + src/utilities/bmi/mass_balance.cpp | 195 + src/utilities/bmi/protocols.cpp | 69 + test/CMakeLists.txt | 14 + test/core/nexus/NexusRemoteTests.cpp | 434 ++ .../test_bmi_python_config_2.yml | 2 + .../catchments/Bmi_C_Formulation_Test.cpp | 238 +- .../catchments/Bmi_Multi_Formulation_Test.cpp | 33 +- test/utils/bmi/MockConfig.hpp | 70 + test/utils/bmi/mass_balance_Test.cpp | 435 ++ 46 files changed, 5996 insertions(+), 135 deletions(-) create mode 100644 include/utilities/bmi/mass_balance.hpp create mode 100644 include/utilities/bmi/nonstd/LICENSE.txt create mode 100644 include/utilities/bmi/nonstd/expected.hpp create mode 100644 include/utilities/bmi/nonstd/expected.tweak.hpp create mode 100644 include/utilities/bmi/protocol.hpp create mode 100644 include/utilities/bmi/protocols.hpp create mode 100644 src/utilities/bmi/CMakeLists.txt create mode 100644 src/utilities/bmi/mass_balance.cpp create mode 100644 src/utilities/bmi/protocols.cpp create mode 100644 test/data/bmi/test_bmi_python/test_bmi_python_config_2.yml create mode 100644 test/utils/bmi/MockConfig.hpp create mode 100644 test/utils/bmi/mass_balance_Test.cpp diff --git a/.github/actions/ngen-build/action.yaml b/.github/actions/ngen-build/action.yaml index 8c0f79730b..c4558969b0 100644 --- a/.github/actions/ngen-build/action.yaml +++ b/.github/actions/ngen-build/action.yaml @@ -52,6 +52,10 @@ inputs: required: false description: 'Enable mpi support, only available for Linux runners' default: 'OFF' + build_extern: + required: false + description: 'Use external dependencies where possible' + default: 'OFF' outputs: build-dir: description: "Directory build was performed in" @@ -110,64 +114,56 @@ runs: id: cache-boost-dep uses: actions/cache@v4 with: - path: boost_1_79_0 + path: boost_1_86_0 key: unix-boost-dep - name: Get Boost Dependency if: steps.cache-boost-dep.outputs.cache-hit != 'true' run: | - curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download - tar xjf boost_1_79_0.tar.bz2 + curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download + tar xjf boost_1_86_0.tar.bz2 shell: bash - name: Set Pip Constraints run: | echo "numpy<2.0" > $GITHUB_WORKSPACE/constraints.txt echo "PIP_CONSTRAINT=$GITHUB_WORKSPACE/constraints.txt" >> $GITHUB_ENV + echo "UV_CONSTRAINT=$GITHUB_WORKSPACE/constraints.txt" >> $GITHUB_ENV shell: bash - - name: Cache Python Dependencies - id: cache-py3-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: ${{ runner.os }}-python-deps + - name: Add uv and Native Python Tooling Translations + if: | + inputs.use_python != 'OFF' + run: | + echo 'ACTIVATE_VENV_IF_USE_PYTHON=source .venv/bin/activate' >> $GITHUB_ENV + if command -v uv &>/dev/null; then + echo 'CREATE_VENV=uv venv .venv' >> $GITHUB_ENV + echo 'PIP_INSTALL=uv pip install' >> $GITHUB_ENV + else + echo 'CREATE_VENV=python3 -m venv .venv' >> $GITHUB_ENV + echo 'PIP_INSTALL=pip install' >> $GITHUB_ENV + fi + shell: bash - name: Get Numpy Python Dependency - # Tried conditioning the cache/install of python with an extra check: - # inputs.use_python != 'OFF' && - # but what happens is that a runner not requiring python will create an empty cache - # and future runners will pull that and then fail... - # What we could do is try to create a master `requirements.txt` - # and/or a `test_requirements.txt` file that we can build the hash key from - # and read it from the repo...but we still have to always initialize the cache - # regardless of whether a given runner uses it, to avoid another runner failing to - # find it. Or just initialize this minimum requirement of numpy, and let the venv - # grow based on other runners needs, effectively building the cache with each new addition if: | - steps.cache-py3-dependencies.outputs.cache-hit != 'true' + inputs.use_python != 'OFF' run: | - python3 -m venv .venv - . .venv/bin/activate - pip install pip - pip install numpy + $CREATE_VENV + $ACTIVATE_VENV_IF_USE_PYTHON + echo $(which python3) + $PIP_INSTALL pip + $PIP_INSTALL numpy deactivate shell: bash - name: Init Additional Python Dependencies - # Don't condition additonal installs on a cache hit - # What will happen, however, is that the venv will get updated - # and thus the cache will get updated - # so any pip install will find modules already installed... - # if: | - # inputs.additional_python_requirements != '' && - # steps.cache-py3-dependencies.outputs.cache-hit != 'true' if: | + inputs.use_python != 'OFF' && inputs.additional_python_requirements != '' run: | - python3 -m venv .venv - . .venv/bin/activate - pip install -r ${{ inputs.additional_python_requirements }} + $ACTIVATE_VENV_IF_USE_PYTHON + $PIP_INSTALL -r ${{ inputs.additional_python_requirements }} deactivate shell: bash @@ -185,13 +181,30 @@ runs: - name: Cmake Initialization id: cmake_init + # NOTE: -DCMAKE_POLICY_VERSION_MINIMUM=3.5 is required to use cmake version 4 + # and with older pybind11 versions, the minimum cmake version is set to 3.4 + # which causes cmake configuration to fail. run: | - export BOOST_ROOT="$(pwd)/boost_1_79_0" - export CFLAGS="-fsanitize=address -O1 -g -fno-omit-frame-pointer -Werror" - export CXXFLAGS="-fsanitize=address -O1 -g -fno-omit-frame-pointer -pedantic-errors -Werror -Wpessimizing-move -Wparentheses -Wrange-loop-construct -Wsuggest-override" - . .venv/bin/activate + export BOOST_ROOT="$(pwd)/boost_1_86_0" + export CFLAGS="-fsanitize=address -g -fno-omit-frame-pointer -Werror" + export CXXFLAGS="-fsanitize=address -g -fno-omit-frame-pointer -pedantic-errors -Werror -Wpessimizing-move -Wparentheses -Wrange-loop-construct -Wsuggest-override" + if [ ${{ runner.os }} == 'macOS' ] + then + echo "fun:PyType_FromMetaclass" > /tmp/asan_ignore.txt + export CFLAGS="$CFLAGS -O0 -fsanitize-ignorelist=/tmp/asan_ignore.txt -fno-common" + export CXXFLAGS="$CXXFLAGS -O0 -fsanitize-ignorelist=/tmp/asan_ignore.txt -fno-common" + else + export CFLAGS="$CFLAGS -O1" + export CXXFLAGS="$CXXFLAGS -O1" + fi + # NOTE: this is not defined if inputs.use_python != 'ON' + $ACTIVATE_VENV_IF_USE_PYTHON [ ! -d "$BOOST_ROOT" ] && echo "Error: no Boost root found at $BOOST_ROOT" && exit 1 + echo "Cmake Version:" + which cmake + cmake --version cmake -B ${{ inputs.build-dir }} \ + -DNGEN_WITH_EXTERN_ALL:BOOL=${{ inputs.build_extern }} \ -DNGEN_WITH_BMI_C:BOOL=${{ inputs.bmi_c }} \ -DNGEN_WITH_PYTHON:BOOL=${{ inputs.use_python }} \ -DNGEN_WITH_UDUNITS:BOOL=${{ inputs.use_udunits }} \ @@ -199,7 +212,9 @@ runs: -DNGEN_WITH_ROUTING:BOOL=${{ inputs.use_troute }} \ -DNGEN_WITH_NETCDF:BOOL=${{ inputs.use_netcdf }} \ -DNGEN_WITH_SQLITE:BOOL=${{ inputs.use_sqlite }} \ - -DNGEN_WITH_MPI:BOOL=${{ inputs.use_mpi }} -S . + -DNGEN_WITH_MPI:BOOL=${{ inputs.use_mpi }} \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -S . + echo "build-dir=$(echo ${{ inputs.build-dir }})" >> $GITHUB_OUTPUT shell: bash @@ -209,8 +224,8 @@ runs: # Build Targets # Disable leak detection during test enumeration export ASAN_OPTIONS=detect_leaks=false - # Activate venv so that test discovery run during build works - . .venv/bin/activate + # NOTE: this is not defined if inputs.use_python != 'ON' + $ACTIVATE_VENV_IF_USE_PYTHON cmake --build ${{ inputs.build-dir }} --target ${{ inputs.targets }} -- -j ${{ inputs.build-cores }} shell: bash diff --git a/.github/actions/ngen-submod-build/action.yaml b/.github/actions/ngen-submod-build/action.yaml index 349434fa45..f24fc5ba93 100644 --- a/.github/actions/ngen-submod-build/action.yaml +++ b/.github/actions/ngen-submod-build/action.yaml @@ -50,6 +50,14 @@ runs: - name: Cmake Initialization id: cmake_init run: | + if [ ${{ runner.os }} == 'macOS' ] + then + export OPT_LEVEL_FLAG="-O0" + else + export OPT_LEVEL_FLAG="-O1" + fi + echo CFLAGS="-fsanitize=address ${OPT_LEVEL_FLAG:?Optimization flag var not set} -g -fno-omit-frame-pointer -Werror" >> $GITHUB_ENV + echo CXXFLAGS="-fsanitize=address ${OPT_LEVEL_FLAG:?Optimization flag var not set} -g -fno-omit-frame-pointer -pedantic-errors -Werror -Wpessimizing-move -Wparentheses -Wrange-loop-construct -Wsuggest-override" >> $GITHUB_ENV cmake -B ${{ inputs.mod-dir}}/${{ inputs.build-dir }} -S ${{ inputs.mod-dir }} ${{ inputs.cmake-flags }} echo "build-dir=$(echo ${{ inputs.mod-dir}}/${{ inputs.build-dir }})" >> $GITHUB_OUTPUT shell: bash diff --git a/.github/workflows/module_integration.yml b/.github/workflows/module_integration.yml index b89c629079..1e757032ee 100644 --- a/.github/workflows/module_integration.yml +++ b/.github/workflows/module_integration.yml @@ -22,7 +22,7 @@ jobs: # The type of runner that the job will run on strategy: matrix: - os: [ubuntu-22.04, macos-12] + os: [ubuntu-22.04, macos-15] fail-fast: false runs-on: ${{ matrix.os }} diff --git a/.github/workflows/test_and_validate.yml b/.github/workflows/test_and_validate.yml index 00279ae8d8..2a4757c4f1 100644 --- a/.github/workflows/test_and_validate.yml +++ b/.github/workflows/test_and_validate.yml @@ -24,7 +24,7 @@ jobs: # The type of runner that the job will run on strategy: matrix: - os: [ubuntu-22.04, macos-12] + os: [ubuntu-22.04, macos-15] fail-fast: false runs-on: ${{ matrix.os }} @@ -59,7 +59,7 @@ jobs: # The type of runner that the job will run on strategy: matrix: - os: [ubuntu-22.04, macos-12] + os: [ubuntu-22.04, macos-15] fail-fast: false runs-on: ${{ matrix.os }} @@ -76,7 +76,9 @@ jobs: build-cores: ${{ env.LINUX_NUM_PROC_CORES }} - name: Run Tests - run: ./cmake_build/test/compare_pet + run: | + export ASAN_OPTIONS=${ASAN_OPTIONS}:detect_odr_violation=0 + ./cmake_build/test/compare_pet timeout-minutes: 15 - name: Clean Up @@ -104,7 +106,7 @@ jobs: use_mpi: 'ON' - name: run_tests - run: mpirun --allow-run-as-root -np 2 ./cmake_build/test/test_remote_nexus + run: mpirun --allow-run-as-root --oversubscribe -np 4 ./cmake_build/test/test_remote_nexus timeout-minutes: 15 - name: Clean Up @@ -116,7 +118,7 @@ jobs: # The type of runner that the job will run on strategy: matrix: - os: [ubuntu-22.04, macos-12] + os: [ubuntu-22.04, macos-15] fail-fast: false runs-on: ${{ matrix.os }} @@ -139,6 +141,7 @@ jobs: - name: Run Tests run: | cd ./cmake_build/test/ + export ASAN_OPTIONS=${ASAN_OPTIONS}:detect_odr_violation=0 ./test_bmi_cpp cd ../../ timeout-minutes: 15 @@ -151,7 +154,7 @@ jobs: # The type of runner that the job will run on strategy: matrix: - os: [ubuntu-22.04, macos-12] + os: [ubuntu-22.04, macos-15] fail-fast: false runs-on: ${{ matrix.os }} @@ -182,7 +185,7 @@ jobs: # The type of runner that the job will run on strategy: matrix: - os: [ubuntu-22.04, macos-12] + os: [ubuntu-22.04, macos-15] fail-fast: false runs-on: ${{ matrix.os }} @@ -215,7 +218,7 @@ jobs: # The type of runner that the job will run on strategy: matrix: - os: [ubuntu-22.04, macos-12] + os: [ubuntu-22.04, macos-15] fail-fast: false runs-on: ${{ matrix.os }} @@ -250,7 +253,7 @@ jobs: # The type of runner that the job will run on strategy: matrix: - os: [ubuntu-22.04, macos-12] + os: [ubuntu-22.04, macos-15] fail-fast: false runs-on: ${{ matrix.os }} @@ -272,12 +275,43 @@ jobs: - name: Run Unit Tests run: | . .venv/bin/activate + export ASAN_OPTIONS=${ASAN_OPTIONS}:detect_odr_violation=0 ./cmake_build/test/test_bmi_multi timeout-minutes: 15 - name: Clean Up Unit Test Build uses: ./.github/actions/clean-build + # Run BMI protocol tests in linux/unix environment + test_bmi_protocols: + # The type of runner that the job will run on + strategy: + matrix: + os: [ubuntu-22.04, macos-15] + fail-fast: false + runs-on: ${{ matrix.os }} + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v4 + + - name: Build BMI Protocol Unit Tests + uses: ./.github/actions/ngen-build + with: + targets: "test_bmi_protocols" + build-cores: ${{ env.LINUX_NUM_PROC_CORES }} + + - name: run_bmi_protocol_tests + run: | + cd ./cmake_build/test/ + export ASAN_OPTIONS=${ASAN_OPTIONS}:detect_odr_violation=0 + ./test_bmi_protocols + cd ../../ + timeout-minutes: 15 + + - name: Clean Up BMI Protocol Unit Test Build + uses: ./.github/actions/clean-build # TODO: fails due to compilation error, at least in large part due to use of POSIX functions not supported on Windows. # TODO: Need to determine whether Windows support (in particular, development environment support) is necessary. diff --git a/CMakeLists.txt b/CMakeLists.txt index 86bfa3dab5..19d24f30d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,9 +167,9 @@ set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_RUNTIME OFF) if(CMAKE_CXX_STANDARD LESS 17) # requires non-header filesystem for state saving if C++ 11 or lower - find_package(Boost 1.79.0 REQUIRED COMPONENTS system filesystem) + find_package(Boost 1.86.0 REQUIRED COMPONENTS system filesystem) else() - find_package(Boost 1.79.0 REQUIRED) + find_package(Boost 1.86.0 REQUIRED) endif() # ----------------------------------------------------------------------------- @@ -329,6 +329,7 @@ add_subdirectory("src/utilities/mdarray") add_subdirectory("src/utilities/mdframe") add_subdirectory("src/utilities/logging") add_subdirectory("src/utilities/python") +add_subdirectory("src/utilities/bmi") target_link_libraries(ngen PUBLIC @@ -343,6 +344,7 @@ target_link_libraries(ngen NGen::logging NGen::parallel NGen::state_save_restore + NGen::bmi_protocols ) if(NGEN_WITH_SQLITE) @@ -500,3 +502,7 @@ ngen_dependent_multiline_message(NGEN_WITH_PYTHON message(STATUS "---------------------------------------------------------------------") configure_file("${NGEN_INC_DIR}/NGenConfig.h.in" "${CMAKE_CURRENT_BINARY_DIR}/include/NGenConfig.h") + +include(GNUInstallDirs) +install(TARGETS ngen OPTIONAL) +install(TARGETS partitionGenerator OPTIONAL) diff --git a/INSTALL.md b/INSTALL.md index 8b73cb5f97..ad4fad92a2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -35,15 +35,15 @@ cd ngen **Download the Boost Libraries:** ```shell -curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download \ - && tar -xjf boost_1_79_0.tar.bz2 \ - && rm boost_1_79_0.tar.bz2 +curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download \ + && tar -xjf boost_1_86_0.tar.bz2 \ + && rm boost_1_86_0.tar.bz2 ``` **Set the ENV for Boost and C compiler:** ```shell -set BOOST_ROOT="/boost_1_79_0" +set BOOST_ROOT="/boost_1_86_0" set CXX=/usr/bin/g++ ``` @@ -79,7 +79,7 @@ The following CMake command will configure the build: ```shell cmake -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ - -DBOOST_ROOT=boost_1_79_0 \ + -DBOOST_ROOT=boost_1_86_0 \ -B /build \ -S . ``` diff --git a/doc/BUILDS_AND_CMAKE.md b/doc/BUILDS_AND_CMAKE.md index 1f1dd944b0..f17613262c 100644 --- a/doc/BUILDS_AND_CMAKE.md +++ b/doc/BUILDS_AND_CMAKE.md @@ -104,7 +104,7 @@ In some cases - in particular **Google Test** - the build system will need to be ## Boost ENV Variable -The Boost libraries must be available for the project to compile. The details are discussed more in the [Dependencies](DEPENDENCIES.md) doc, but as a helpful hint, the **BOOST_ROOT** environmental variable can be set to the path of the applicable [Boost root directory](https://www.boost.org/doc/libs/1_79_0/more/getting_started/unix-variants.html#the-boost-distribution). The project's [CMakeLists.txt](../CMakeLists.txt) is written to check for this env variable and use it to set the Boost include directory. +The Boost libraries must be available for the project to compile. The details are discussed more in the [Dependencies](DEPENDENCIES.md) doc, but as a helpful hint, the **BOOST_ROOT** environmental variable can be set to the path of the applicable [Boost root directory](https://www.boost.org/doc/libs/1_86_0/more/getting_started/unix-variants.html#the-boost-distribution). The project's [CMakeLists.txt](../CMakeLists.txt) is written to check for this env variable and use it to set the Boost include directory. Note that if the variable is not set, it may still be possible for CMake to find Boost, although a *status* message will be printed by CMake indicating **BOOST_ROOT** was not set. diff --git a/doc/DEPENDENCIES.md b/doc/DEPENDENCIES.md index 6c48b38f94..e5e16c1d33 100644 --- a/doc/DEPENDENCIES.md +++ b/doc/DEPENDENCIES.md @@ -7,7 +7,7 @@ | [Google Test](#google-test) | submodule | `release-1.10.0` | | | [C/C++ Compiler](#c-and-c-compiler) | external | see below | | | [CMake](#cmake) | external | \>= `3.17` | | -| [Boost (Headers Only)](#boost-headers-only) | external | `1.79.0` | headers only library | +| [Boost (Headers Only)](#boost-headers-only) | external | `1.86.0` | headers only library | | [Udunits libraries](https://www.unidata.ucar.edu/software/udunits) | external | >= 2.0 | Can be installed via package manager or from source | | [MPI](https://www.mpi-forum.org) | external | No current implementation or version requirements | Required for [multi-process distributed execution](DISTRIBUTED_PROCESSING.md) | | [Python 3 Libraries](#python-3-libraries) | external | \>= `3.8.0` | Can be [excluded](#overriding-python-dependency). | @@ -78,7 +78,7 @@ Currently, a version of CMake >= `3.14.0` is required. ## Boost (Headers Only) -Boost libraries are used by this project. In particular, [Boost.Geometry](https://www.boost.org/doc/libs/1_79_0/libs/geometry/doc/html/geometry/compilation.html) is used, but others are also. +Boost libraries are used by this project. In particular, [Boost.Geometry](https://www.boost.org/doc/libs/1_86_0/libs/geometry/doc/html/geometry/compilation.html) is used, but others are also. Currently, only headers-only Boost libraries are utilized. As such, they are not exhaustively listed here since getting one essentially gets them all. @@ -88,7 +88,7 @@ Since only headers-only libraries are needed, the Boost headers simply need to b There are a variety of different ways to get the Boost headers locally. Various OS may have packages specifically to install them, though one should take note of whether such packages provide a version of Boost that meets this project's requirements. -Alternatively, the Boost distribution itself can be manually downloaded and unpacked, as described for both [Unix-variants](https://www.boost.org/doc/libs/1_79_0/more/getting_started/unix-variants.html) and [Windows](https://www.boost.org/doc/libs/1_79_0/more/getting_started/windows.html) on the Boost website. +Alternatively, the Boost distribution itself can be manually downloaded and unpacked, as described for both [Unix-variants](https://www.boost.org/doc/libs/1_86_0/more/getting_started/unix-variants.html) and [Windows](https://www.boost.org/doc/libs/1_86_0/more/getting_started/windows.html) on the Boost website. #### Setting **BOOST_ROOT** @@ -96,11 +96,11 @@ If necessary, the project's CMake config is able to use the value of the **BOOST However, it will often be necessary to set **BOOST_ROOT** if Boost was manually set up by downloading the distribution. -The variable should be set to the value of the **boost root directory**, which is something like `/boost_1_79_0`. +The variable should be set to the value of the **boost root directory**, which is something like `/boost_1_86_0`. ### Version Requirements -At present, a version >= `1.79.0` is required. +At present, a version >= `1.86.0` is required. ## Udunits diff --git a/docker/CENTOS_4.8.5_NGEN_RUN.dockerfile b/docker/CENTOS_4.8.5_NGEN_RUN.dockerfile index 9569428d20..794d40827d 100644 --- a/docker/CENTOS_4.8.5_NGEN_RUN.dockerfile +++ b/docker/CENTOS_4.8.5_NGEN_RUN.dockerfile @@ -19,11 +19,11 @@ ENV CXX=/usr/bin/g++ RUN git submodule update --init --recursive -- test/googletest -RUN curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download +RUN curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download -RUN tar -xjf boost_1_79_0.tar.bz2 +RUN tar -xjf boost_1_86_0.tar.bz2 -ENV BOOST_ROOT="boost_1_79_0" +ENV BOOST_ROOT="boost_1_86_0" RUN cmake -B /ngen -S . diff --git a/docker/CENTOS_NGEN_RUN.dockerfile b/docker/CENTOS_NGEN_RUN.dockerfile index 8aee5b9a4b..afaa442d34 100644 --- a/docker/CENTOS_NGEN_RUN.dockerfile +++ b/docker/CENTOS_NGEN_RUN.dockerfile @@ -9,11 +9,11 @@ RUN yum update -y \ && dnf clean all \ && rm -rf /var/cache/yum -RUN curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download \ - && tar -xjf boost_1_79_0.tar.bz2 \ - && rm boost_1_79_0.tar.bz2 +RUN curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download \ + && tar -xjf boost_1_86_0.tar.bz2 \ + && rm boost_1_86_0.tar.bz2 -ENV BOOST_ROOT="/boost_1_79_0" +ENV BOOST_ROOT="/boost_1_86_0" ENV CXX=/usr/bin/g++ diff --git a/docker/CENTOS_TEST.dockerfile b/docker/CENTOS_TEST.dockerfile index 55271d2755..334ac5d5d7 100644 --- a/docker/CENTOS_TEST.dockerfile +++ b/docker/CENTOS_TEST.dockerfile @@ -11,11 +11,11 @@ ENV CXX=/usr/bin/g++ RUN git submodule update --init --recursive -- test/googletest -RUN curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download +RUN curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_86_0.tar.bz2/download -RUN tar -xjf boost_1_79_0.tar.bz2 +RUN tar -xjf boost_1_86_0.tar.bz2 -ENV BOOST_ROOT="boost_1_79_0" +ENV BOOST_ROOT="boost_1_86_0" WORKDIR /ngen diff --git a/docker/CENTOS_latest_NGEN_RUN.dockerfile b/docker/CENTOS_latest_NGEN_RUN.dockerfile index c53a46572e..11963f5209 100644 --- a/docker/CENTOS_latest_NGEN_RUN.dockerfile +++ b/docker/CENTOS_latest_NGEN_RUN.dockerfile @@ -13,11 +13,11 @@ ENV CXX=/usr/bin/g++ RUN git submodule update --init --recursive -- test/googletest -RUN curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download +RUN curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download -RUN tar -xjf boost_1_79_0.tar.bz2 +RUN tar -xjf boost_1_86_0.tar.bz2 -ENV BOOST_ROOT="boost_1_79_0" +ENV BOOST_ROOT="boost_1_86_0" WORKDIR /ngen diff --git a/docker/RHEL_TEST.dockerfile b/docker/RHEL_TEST.dockerfile index 9fe3550639..3103bd8464 100644 --- a/docker/RHEL_TEST.dockerfile +++ b/docker/RHEL_TEST.dockerfile @@ -10,11 +10,11 @@ ENV CXX=/usr/bin/g++ RUN git submodule update --init --recursive -- test/googletest -RUN curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download +RUN curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download -RUN tar -xjf boost_1_79_0.tar.bz2 +RUN tar -xjf boost_1_86_0.tar.bz2 -ENV BOOST_ROOT="boost_1_79_0" +ENV BOOST_ROOT="boost_1_86_0" WORKDIR /ngen diff --git a/docker/ngen.dockerfile b/docker/ngen.dockerfile index 88d880b98f..2d50a7c1a9 100644 --- a/docker/ngen.dockerfile +++ b/docker/ngen.dockerfile @@ -7,7 +7,7 @@ RUN dnf update -y \ && dnf install -y --allowerasing tar git gcc-c++ gcc make cmake udunits2-devel coreutils \ && dnf clean all -ARG BOOST_VERSION="1.79.0" +ARG BOOST_VERSION="1.86.0" RUN export BOOST_ARCHIVE="boost_$(echo ${BOOST_VERSION} | tr '\.' '_').tar.gz" \ && export BOOST_URL="https://sourceforge.net/projects/boost/files/boost/${BOOST_VERSION}/${BOOST_ARCHIVE}/download" \ && cd / \ diff --git a/extern/iso_c_fortran_bmi/CMakeLists.txt b/extern/iso_c_fortran_bmi/CMakeLists.txt index b55642e83f..9c1a17b2a8 100644 --- a/extern/iso_c_fortran_bmi/CMakeLists.txt +++ b/extern/iso_c_fortran_bmi/CMakeLists.txt @@ -33,5 +33,5 @@ install(TARGETS iso_c_bmi install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) configure_file(iso_c_bmi.pc.in iso_c_bmi.pc @ONLY) -install(FILES ${CMAKE_BINARY_DIR}/iso_c_bmi.pc +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iso_c_bmi.pc DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig) diff --git a/extern/test_bmi_c/include/bmi_test_bmi_c.h b/extern/test_bmi_c/include/bmi_test_bmi_c.h index 66a24a1572..3d4f40b4f1 100644 --- a/extern/test_bmi_c/include/bmi_test_bmi_c.h +++ b/extern/test_bmi_c/include/bmi_test_bmi_c.h @@ -1,6 +1,11 @@ #ifndef BMI_TEST_BMI_C_H #define BMI_TEST_BMI_C_H +#define NGEN_MASS_IN "ngen::mass_in" +#define NGEN_MASS_OUT "ngen::mass_out" +#define NGEN_MASS_STORED "ngen::mass_stored" +#define NGEN_MASS_LEAKED "ngen::mass_leaked" + #if defined(__cplusplus) extern "C" { #endif diff --git a/extern/test_bmi_c/include/test_bmi_c.h b/extern/test_bmi_c/include/test_bmi_c.h index fa990d1ebb..1e438259eb 100644 --- a/extern/test_bmi_c/include/test_bmi_c.h +++ b/extern/test_bmi_c/include/test_bmi_c.h @@ -31,6 +31,9 @@ struct test_bmi_c_model { int param_var_1; double param_var_2; double* param_var_3; + + double mass_stored; // Mass balance variable, for testing purposes + double mass_leaked; //Mass balance variable, for testing purposes }; typedef struct test_bmi_c_model test_bmi_c_model; diff --git a/extern/test_bmi_c/src/bmi_test_bmi_c.c b/extern/test_bmi_c/src/bmi_test_bmi_c.c index 187ca666cd..dc9499647e 100644 --- a/extern/test_bmi_c/src/bmi_test_bmi_c.c +++ b/extern/test_bmi_c/src/bmi_test_bmi_c.c @@ -9,6 +9,7 @@ #define INPUT_VAR_NAME_COUNT 2 #define OUTPUT_VAR_NAME_COUNT 2 #define PARAM_VAR_NAME_COUNT 3 +#define MASS_BALANCE_VAR_NAME_COUNT 4 // Don't forget to update Get_value/Get_value_at_indices (and setter) implementation if these are adjusted static const char *output_var_names[OUTPUT_VAR_NAME_COUNT] = { "OUTPUT_VAR_1", "OUTPUT_VAR_2" }; @@ -34,6 +35,13 @@ static const int param_var_item_count[PARAM_VAR_NAME_COUNT] = { 1, 1, 2 }; static const char *param_var_grids[PARAM_VAR_NAME_COUNT] = { 0, 0, 0 }; static const char *param_var_locations[PARAM_VAR_NAME_COUNT] = { "node", "node", "node" }; +static const char *mass_balance_var_names[MASS_BALANCE_VAR_NAME_COUNT] = { NGEN_MASS_IN, NGEN_MASS_OUT, NGEN_MASS_STORED, NGEN_MASS_LEAKED}; +static const char *mass_balance_var_types[MASS_BALANCE_VAR_NAME_COUNT] = { "double", "double", "double", "double"}; +static const char *mass_balance_var_units[MASS_BALANCE_VAR_NAME_COUNT] = { "m", "m", "m", "m" }; +static const int mass_balance_var_item_count[MASS_BALANCE_VAR_NAME_COUNT] = { 1, 1, 1, 1}; +static const char *mass_balance_var_grids[MASS_BALANCE_VAR_NAME_COUNT] = { 0, 0, 0, 0 }; +static const char *mass_balance_var_locations[MASS_BALANCE_VAR_NAME_COUNT] = { "node", "node", "node", "node" }; + static int Finalize (Bmi *self) { // Function assumes everything that is needed is retrieved from the model before Finalize is called. @@ -387,6 +395,23 @@ static int Get_value_ptr (Bmi *self, const char *name, void **dest) *dest = ((test_bmi_c_model *)(self->data))->param_var_3; return BMI_SUCCESS; } + + if (strcmp (name, NGEN_MASS_IN) == 0) { + *dest = ((test_bmi_c_model *)(self->data))->input_var_1; + return BMI_SUCCESS; + } + if (strcmp (name, NGEN_MASS_OUT) == 0) { + *dest = ((test_bmi_c_model *)(self->data))->output_var_1; + return BMI_SUCCESS; + } + if (strcmp (name, NGEN_MASS_STORED) == 0) { + *dest = &((test_bmi_c_model *)(self->data))->mass_stored; + return BMI_SUCCESS; + } + if (strcmp (name, NGEN_MASS_LEAKED) == 0) { + *dest = &((test_bmi_c_model *)(self->data))->mass_leaked; + return BMI_SUCCESS; + } return BMI_FAILURE; } @@ -483,6 +508,14 @@ static int Get_var_nbytes (Bmi *self, const char *name, int * nbytes) } } } + if (item_count < 1) { + for (i = 0; i < MASS_BALANCE_VAR_NAME_COUNT; i++) { + if (strcmp(name, mass_balance_var_names[i]) == 0) { + item_count = mass_balance_var_item_count[i]; + break; + } + } + } if (item_count < 1) item_count = ((test_bmi_c_model *) self->data)->num_time_steps; @@ -515,6 +548,13 @@ static int Get_var_type (Bmi *self, const char *name, char * type) return BMI_SUCCESS; } } + // Finally check to see if in mass balance array + for (i = 0; i < MASS_BALANCE_VAR_NAME_COUNT; i++) { + if (strcmp(name, mass_balance_var_names[i]) == 0) { + snprintf(type, BMI_MAX_TYPE_NAME, "%s", mass_balance_var_types[i]); + return BMI_SUCCESS; + } + } // If we get here, it means the variable name wasn't recognized type[0] = '\0'; return BMI_FAILURE; @@ -538,6 +578,13 @@ static int Get_var_units (Bmi *self, const char *name, char * units) return BMI_SUCCESS; } } + //Check for mass balance + for (i = 0; i < MASS_BALANCE_VAR_NAME_COUNT; i++) { + if (strcmp(name, mass_balance_var_names[i]) == 0) { + snprintf(units, BMI_MAX_UNITS_NAME, "%s", mass_balance_var_units[i]); + return BMI_SUCCESS; + } + } // If we get here, it means the variable name wasn't recognized units[0] = '\0'; return BMI_FAILURE; @@ -560,6 +607,9 @@ static int Initialize (Bmi *self, const char *file) else model = (test_bmi_c_model *) self->data; + model->mass_stored = 0.0; + model->mass_leaked = 0.0; + if (read_init_config(file, model) == BMI_FAILURE) return BMI_FAILURE; diff --git a/extern/test_bmi_c/src/test_bmi_c.c b/extern/test_bmi_c/src/test_bmi_c.c index c81d6b4406..d05dc50b3f 100644 --- a/extern/test_bmi_c/src/test_bmi_c.c +++ b/extern/test_bmi_c/src/test_bmi_c.c @@ -19,5 +19,7 @@ extern int run(test_bmi_c_model* model, long dt) } model->current_model_time += (double)dt; + model->mass_stored = *model->output_var_1 - *model->input_var_1; + model->mass_leaked = 0; return 0; } \ No newline at end of file diff --git a/extern/test_bmi_cpp/include/test_bmi_cpp.hpp b/extern/test_bmi_cpp/include/test_bmi_cpp.hpp index 4b2c31a429..2200eaa20c 100644 --- a/extern/test_bmi_cpp/include/test_bmi_cpp.hpp +++ b/extern/test_bmi_cpp/include/test_bmi_cpp.hpp @@ -25,6 +25,11 @@ #define BMI_TYPE_NAME_SHORT "short" #define BMI_TYPE_NAME_LONG "long" +#define NGEN_MASS_IN "ngen::mass_in" +#define NGEN_MASS_OUT "ngen::mass_out" +#define NGEN_MASS_STORED "ngen::mass_stored" +#define NGEN_MASS_LEAKED "ngen::mass_leaked" + class TestBmiCpp : public bmi::Bmi { public: /** @@ -179,6 +184,11 @@ class TestBmiCpp : public bmi::Bmi { std::vector output_var_locations = { "node", "node" }; std::vector model_var_locations = {}; + std::vector mass_balance_var_names = { NGEN_MASS_IN, NGEN_MASS_OUT, NGEN_MASS_STORED, NGEN_MASS_LEAKED}; + std::vector mass_balance_var_types = { "double", "double", "double", "double"}; + std::vector mass_balance_var_units = { "m", "m", "m", "m" }; + std::vector mass_balance_var_locations = { "node", "node", "node", "node"}; + std::vector input_var_item_count = { 1, 1 }; std::vector output_var_item_count = { 1, 1 }; std::vector model_var_item_count = {}; @@ -223,6 +233,9 @@ class TestBmiCpp : public bmi::Bmi { std::unique_ptr model_var_1 = nullptr; std::unique_ptr model_var_2 = nullptr; + double mass_stored = 0.0; + double mass_leaked = 0.0; + /** * Read the BMI initialization config file and use its contents to set the state of the model. * diff --git a/extern/test_bmi_cpp/src/test_bmi_cpp.cpp b/extern/test_bmi_cpp/src/test_bmi_cpp.cpp index 32536e0c75..57019839fb 100644 --- a/extern/test_bmi_cpp/src/test_bmi_cpp.cpp +++ b/extern/test_bmi_cpp/src/test_bmi_cpp.cpp @@ -165,6 +165,19 @@ void* TestBmiCpp::GetValuePtr(std::string name){ } } + if (name == NGEN_MASS_STORED) { + return &this->mass_stored; + } + if (name == NGEN_MASS_LEAKED) { + return &this->mass_leaked; + } + if (name == NGEN_MASS_IN) { + return this->input_var_1.get(); + } + if (name == NGEN_MASS_OUT) { + return this->output_var_1.get(); + } + throw std::runtime_error("GetValuePtr called for unknown variable: "+name); } @@ -212,6 +225,10 @@ int TestBmiCpp::GetVarNbytes(std::string name){ if(iter != this->model_var_names.end()){ item_count = this->model_var_item_count[iter - this->model_var_names.begin()]; } + iter = std::find(this->mass_balance_var_names.begin(), this->mass_balance_var_names.end(), name); + if(iter != this->mass_balance_var_names.end()){ + item_count = 1; + } if(item_count == -1){ // This is probably impossible to reach--the same conditions above failing will cause a throw // in GetVarItemSize --> GetVarType (called earlier) instead. @@ -233,6 +250,10 @@ std::string TestBmiCpp::GetVarType(std::string name){ if(iter != this->model_var_names.end()){ return this->model_var_types[iter - this->model_var_names.begin()]; } + iter = std::find(this->mass_balance_var_names.begin(), this->mass_balance_var_names.end(), name); + if(iter != this->mass_balance_var_names.end()){ + return this->mass_balance_var_types[iter - this->mass_balance_var_names.begin()]; + } throw std::runtime_error("GetVarType called for non-existent variable: "+name+"" SOURCE_LOC ); } @@ -249,6 +270,10 @@ std::string TestBmiCpp::GetVarUnits(std::string name){ if(iter != this->model_var_names.end()){ return this->model_var_types[iter - this->model_var_names.begin()]; } + iter = std::find(this->mass_balance_var_names.begin(), this->mass_balance_var_names.end(), name); + if(iter != this->mass_balance_var_names.end()){ + return this->mass_balance_var_units[iter - this->mass_balance_var_names.begin()]; + } throw std::runtime_error("GetVarUnits called for non-existent variable: "+name+"" SOURCE_LOC); } @@ -517,4 +542,6 @@ void TestBmiCpp::run(long dt) *this->output_var_5 = *this->model_var_2 * 1.0; } this->current_model_time += (double)dt; + this->mass_stored = *this->output_var_1 - *this->input_var_1; + this->mass_leaked = 0; } diff --git a/include/core/nexus/HY_PointHydroNexusRemote.hpp b/include/core/nexus/HY_PointHydroNexusRemote.hpp index 34d98f9a96..ebac2e9ace 100644 --- a/include/core/nexus/HY_PointHydroNexusRemote.hpp +++ b/include/core/nexus/HY_PointHydroNexusRemote.hpp @@ -72,6 +72,7 @@ class HY_PointHydroNexusRemote : public HY_PointHydroNexus communication_type get_communicator_type() { return type; } private: + void post_receives(); void process_communications(); int world_rank; diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 775c9394d1..150bd2ac38 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -9,6 +9,7 @@ #include "bmi_utilities.hpp" #include +#include "bmi/protocols.hpp" using data_access::MEAN; using data_access::SUM; @@ -307,6 +308,12 @@ namespace realization { virtual void free_serialization_state(); void set_realization_file_format(bool is_legacy_format); + virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const override { + //Create the protocol context, each member is const, and cannot change during the check + models::bmi::protocols::Context ctx{iteration, total_steps, timestamp, id}; + bmi_protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, ctx); + } + protected: /** @@ -520,6 +527,7 @@ namespace realization { bool is_realization_legacy_format() const; private: + models::bmi::protocols::NgenBmiProtocols bmi_protocols; /** * Whether model ``Update`` calls are allowed and handled in some way by the backing model for time steps after * the model's ``end_time``. diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index bd64b063af..9877d253a2 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -53,6 +53,15 @@ namespace realization { void load_state(std::shared_ptr loader) override; void load_hot_start(std::shared_ptr loader) override; + + virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const final { + for( const auto &module : modules ) { + // TODO may need to check on outputs form each module indepdently??? + // Right now, the assumption is that if each component is mass balanced + // then the entire formulation is mass balanced + module->check_mass_balance(iteration, total_steps, timestamp); + } + }; /** * Convert a time value from the model to an epoch time in seconds. diff --git a/include/realizations/catchment/Formulation.hpp b/include/realizations/catchment/Formulation.hpp index 44ff6b8af8..ad8c6f097c 100644 --- a/include/realizations/catchment/Formulation.hpp +++ b/include/realizations/catchment/Formulation.hpp @@ -45,6 +45,7 @@ namespace realization { virtual void create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global = nullptr) = 0; virtual void create_formulation(geojson::PropertyMap properties) = 0; + virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const = 0; protected: virtual const std::vector& get_required_parameters() const = 0; diff --git a/include/utilities/bmi/mass_balance.hpp b/include/utilities/bmi/mass_balance.hpp new file mode 100644 index 0000000000..e8883232ff --- /dev/null +++ b/include/utilities/bmi/mass_balance.hpp @@ -0,0 +1,155 @@ +/* +Author: Nels Frazier +Copyright (C) 2025 Lynker +------------------------------------------------------------------------ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +------------------------------------------------------------------------ +Version 0.3 +Implement is_supported() +Re-align members for more better memory layout/padding +Update docstrings + +Version 0.2 +Conform to updated protocol interface +Removed integration and error exceptions in favor of ProtocolError + +Version 0.1 +Interface of the BMI mass balance protocol +*/ +#pragma once + + +#include +#include +#include +#include + +namespace models{ namespace bmi{ namespace protocols{ + using nonstd::expected; + /** Mass balance variable names **/ + constexpr const char* const INPUT_MASS_NAME = "ngen::mass_in"; + constexpr const char* const OUTPUT_MASS_NAME = "ngen::mass_out"; + constexpr const char* const STORED_MASS_NAME = "ngen::mass_stored"; + constexpr const char* const LEAKED_MASS_NAME = "ngen::mass_leaked"; + + /** Configuration keys for defining configurable properties of the protocol */ + //The top level object key which will contain the map of configuration options + constexpr const char* const CONFIGURATION_KEY = "mass_balance"; + //Configuration option keys + constexpr const char* const TOLERANCE_KEY = "tolerance"; + constexpr const char* const FATAL_KEY = "fatal"; + constexpr const char* const CHECK_KEY = "check"; + constexpr const char* const FREQUENCY_KEY = "frequency"; + + class NgenMassBalance : public NgenBmiProtocol { + /** @brief Mass Balance protocol + * + * This protocol `run()`s a simple mass balance calculation by querying the model for a + * set of mass balance state variables and computing the basic mass balance as + * balance = mass_in - mass_out - mass_stored - mass_leaked. It is then checked against + * a tolerance value to determine if the mass balance is acceptable. + */ + public: + + /** @brief Constructor for the NgenMassBalance protocol + * + * This constructor initializes the mass balance protocol with the given model and properties. + * + * @param model A shared pointer to a Bmi_Adapter object which should be + * initialized before being passed to this constructor. + */ + NgenMassBalance(const ModelPtr& model, const Properties& properties); + + /** + * @brief Construct a new, default Ngen Mass Balance object + * + * By default, the protocol is considered unsupported and won't be checked + */ + NgenMassBalance(); + + virtual ~NgenMassBalance() override; + + private: + + /** + * @brief Run the mass balance protocol + * + * If the configured frequency is -1, the mass balance will only be checked at the end + * of the simulation. If the frequency is greater than 0, the mass balance will be checked + * at the specified frequency based on the current_time_step and the total_steps provided + * in the Context. + * + * Warns or errors at each check if total mass balance is not within the configured + * acceptable tolerance. + * + * @return expected May contain a ProtocolError if + * the protocol fails for any reason. Errors of ProtocolError::PROTOCOL_WARNING + * severity should be logged as warnings, but not cause the simulation to fail. + */ + auto run(const ModelPtr& model, const Context& ctx) const -> expected override; + + /** + * @brief Check if the mass balance protocol is supported by the model + * + * @return expected May contain a ProtocolError if + * the protocol is not supported by the model. + */ + nsel_NODISCARD auto check_support(const ModelPtr& model) -> expected override; + + /** + * @brief Check the model for support and initialize the mass balance protocol from the given properties. + * + * If the model does not support the mass balance protocol, an exception will be thrown, and no mass balance + * will performed when `run()` is called. + * + * A private initialize call is used since it only makes sense to check/run the protocol + * once the model adapter is fully constructed. This should be called by the owner of the + * NgenMassBalance instance once the model is ready. + * + * @param properties Configurable key/value properties for the mass balance protocol. + * If the map contains "mass_balance" object, then the following properties + * are used to configure the protocol: + * tolerance: double, default 1.0E-16. + * check: bool, default true. Whether to perform mass balance check. + * frequency: int, default 1. How often (in time steps) to check mass balance. + * fatal: bool, default false. Whether to treat mass balance errors as fatal. + * Otherwise, mass balance checking will be disabled (check will be false) + * + * @return expected May contain a ProtocolError if + * initialization fails for any reason, since the protocol must + * be effectively "optional", failed initialization results in + * the protocol being disabled for the duration of the simulation. + */ + auto initialize(const ModelPtr& model, const Properties& properties) -> expected override; + + /** + * @brief Whether the protocol is supported by the model + * + * @return true the model exposes the required mass balance variables + * @return false the model does not support mass balance checking via this protocol + */ + bool is_supported() const override final; + + private: + double tolerance; + // How often (in time steps) to check mass balance + int frequency; + // Whether the protocol is supported by the model, false by default + bool supported = false; + // Configurable options/values + bool check; + bool is_fatal; + }; + +}}} + diff --git a/include/utilities/bmi/nonstd/LICENSE.txt b/include/utilities/bmi/nonstd/LICENSE.txt new file mode 100644 index 0000000000..36b7cd93cd --- /dev/null +++ b/include/utilities/bmi/nonstd/LICENSE.txt @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/include/utilities/bmi/nonstd/expected.hpp b/include/utilities/bmi/nonstd/expected.hpp new file mode 100644 index 0000000000..ae1790d131 --- /dev/null +++ b/include/utilities/bmi/nonstd/expected.hpp @@ -0,0 +1,3637 @@ +// Vendored from https://github.com/martinmoene/expected-lite/commit/a7510b213a668306fb038c934e27e53cc01141d4 +// This version targets C++11 and later. +// +// Copyright (C) 2016-2025 Martin Moene. +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// expected lite is based on: +// A proposal to add a utility class to represent expected monad +// by Vicente J. Botet Escriba and Pierre Talbot. http:://wg21.link/p0323 + +#ifndef NONSTD_EXPECTED_LITE_HPP +#define NONSTD_EXPECTED_LITE_HPP + +#define expected_lite_MAJOR 0 +#define expected_lite_MINOR 9 +#define expected_lite_PATCH 0 + +#define expected_lite_VERSION expected_STRINGIFY(expected_lite_MAJOR) "." expected_STRINGIFY(expected_lite_MINOR) "." expected_STRINGIFY(expected_lite_PATCH) + +#define expected_STRINGIFY( x ) expected_STRINGIFY_( x ) +#define expected_STRINGIFY_( x ) #x + +// expected-lite configuration: + +#define nsel_EXPECTED_DEFAULT 0 +#define nsel_EXPECTED_NONSTD 1 +#define nsel_EXPECTED_STD 2 + +// tweak header support: + +#ifdef __has_include +# if __has_include() +# include +# endif +#define expected_HAVE_TWEAK_HEADER 1 +#else +#define expected_HAVE_TWEAK_HEADER 0 +//# pragma message("expected.hpp: Note: Tweak header not supported.") +#endif + +// expected selection and configuration: + +#if !defined( nsel_CONFIG_SELECT_EXPECTED ) +# define nsel_CONFIG_SELECT_EXPECTED ( nsel_HAVE_STD_EXPECTED ? nsel_EXPECTED_STD : nsel_EXPECTED_NONSTD ) +#endif + +// Proposal revisions: +// +// DXXXXR0: -- +// N4015 : -2 (2014-05-26) +// N4109 : -1 (2014-06-29) +// P0323R0: 0 (2016-05-28) +// P0323R1: 1 (2016-10-12) +// -------: +// P0323R2: 2 (2017-06-15) +// P0323R3: 3 (2017-10-15) +// P0323R4: 4 (2017-11-26) +// P0323R5: 5 (2018-02-08) +// P0323R6: 6 (2018-04-02) +// P0323R7: 7 (2018-06-22) * +// +// expected-lite uses 2 and higher + +#ifndef nsel_P0323R +# define nsel_P0323R 7 +#endif + +// Monadic operations proposal revisions: +// +// P2505R0: 0 (2021-12-12) +// P2505R1: 1 (2022-02-10) +// P2505R2: 2 (2022-04-15) +// P2505R3: 3 (2022-06-05) +// P2505R4: 4 (2022-06-15) +// P2505R5: 5 (2022-09-20) * +// +// expected-lite uses 5 + +#ifndef nsel_P2505R +# define nsel_P2505R 5 +#endif + +// Lean and mean inclusion of Windows.h, if applicable; default on for MSVC: + +#if !defined(nsel_CONFIG_WIN32_LEAN_AND_MEAN) && defined(_MSC_VER) +# define nsel_CONFIG_WIN32_LEAN_AND_MEAN 1 +#else +# define nsel_CONFIG_WIN32_LEAN_AND_MEAN 0 +#endif + +// Control marking class expected with [[nodiscard]]]: + +#if !defined(nsel_CONFIG_NO_NODISCARD) +# define nsel_CONFIG_NO_NODISCARD 0 +#else +# define nsel_CONFIG_NO_NODISCARD 1 +#endif + +// Control presence of C++ exception handling (try and auto discover): + +#ifndef nsel_CONFIG_NO_EXCEPTIONS +# if defined(_MSC_VER) +# include // for _HAS_EXCEPTIONS +# endif +# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || (_HAS_EXCEPTIONS) +# define nsel_CONFIG_NO_EXCEPTIONS 0 +# else +# define nsel_CONFIG_NO_EXCEPTIONS 1 +# endif +#endif + +// at default use SEH with MSVC for no C++ exceptions + +#if !defined(nsel_CONFIG_NO_EXCEPTIONS_SEH) && defined(_MSC_VER) +# define nsel_CONFIG_NO_EXCEPTIONS_SEH nsel_CONFIG_NO_EXCEPTIONS +#else +# define nsel_CONFIG_NO_EXCEPTIONS_SEH 0 +#endif + +// C++ language version detection (C++23 is speculative): +// Note: VC14.0/1900 (VS2015) lacks too much from C++14. + +#ifndef nsel_CPLUSPLUS +# if defined(_MSVC_LANG ) && !defined(__clang__) +# define nsel_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG ) +# else +# define nsel_CPLUSPLUS __cplusplus +# endif +#endif + +#define nsel_CPP98_OR_GREATER ( nsel_CPLUSPLUS >= 199711L ) +#define nsel_CPP11_OR_GREATER ( nsel_CPLUSPLUS >= 201103L ) +#define nsel_CPP14_OR_GREATER ( nsel_CPLUSPLUS >= 201402L ) +#define nsel_CPP17_OR_GREATER ( nsel_CPLUSPLUS >= 201703L ) +#define nsel_CPP20_OR_GREATER ( nsel_CPLUSPLUS >= 202002L ) +#define nsel_CPP23_OR_GREATER ( nsel_CPLUSPLUS >= 202300L ) + +// Use C++23 std::expected if available and requested: + +#if nsel_CPP23_OR_GREATER && defined(__has_include ) +# if __has_include( ) +# define nsel_HAVE_STD_EXPECTED 1 +# else +# define nsel_HAVE_STD_EXPECTED 0 +# endif +#else +# define nsel_HAVE_STD_EXPECTED 0 +#endif + +#define nsel_USES_STD_EXPECTED ( (nsel_CONFIG_SELECT_EXPECTED == nsel_EXPECTED_STD) || ((nsel_CONFIG_SELECT_EXPECTED == nsel_EXPECTED_DEFAULT) && nsel_HAVE_STD_EXPECTED) ) + +// +// in_place: code duplicated in any-lite, expected-lite, expected-lite, value-ptr-lite, variant-lite: +// + +#ifndef nonstd_lite_HAVE_IN_PLACE_TYPES +#define nonstd_lite_HAVE_IN_PLACE_TYPES 1 + +// C++17 std::in_place in : + +#if nsel_CPP17_OR_GREATER + +#include + +namespace nonstd { + +using std::in_place; +using std::in_place_type; +using std::in_place_index; +using std::in_place_t; +using std::in_place_type_t; +using std::in_place_index_t; + +#define nonstd_lite_in_place_t( T) std::in_place_t +#define nonstd_lite_in_place_type_t( T) std::in_place_type_t +#define nonstd_lite_in_place_index_t(K) std::in_place_index_t + +#define nonstd_lite_in_place( T) std::in_place_t{} +#define nonstd_lite_in_place_type( T) std::in_place_type_t{} +#define nonstd_lite_in_place_index(K) std::in_place_index_t{} + +} // namespace nonstd + +#else // nsel_CPP17_OR_GREATER + +#include + +namespace nonstd { +namespace detail { + +template< class T > +struct in_place_type_tag {}; + +template< std::size_t K > +struct in_place_index_tag {}; + +} // namespace detail + +struct in_place_t {}; + +template< class T > +inline in_place_t in_place( detail::in_place_type_tag = detail::in_place_type_tag() ) +{ + return in_place_t(); +} + +template< std::size_t K > +inline in_place_t in_place( detail::in_place_index_tag = detail::in_place_index_tag() ) +{ + return in_place_t(); +} + +template< class T > +inline in_place_t in_place_type( detail::in_place_type_tag = detail::in_place_type_tag() ) +{ + return in_place_t(); +} + +template< std::size_t K > +inline in_place_t in_place_index( detail::in_place_index_tag = detail::in_place_index_tag() ) +{ + return in_place_t(); +} + +// mimic templated typedef: + +#define nonstd_lite_in_place_t( T) nonstd::in_place_t(&)( nonstd::detail::in_place_type_tag ) +#define nonstd_lite_in_place_type_t( T) nonstd::in_place_t(&)( nonstd::detail::in_place_type_tag ) +#define nonstd_lite_in_place_index_t(K) nonstd::in_place_t(&)( nonstd::detail::in_place_index_tag ) + +#define nonstd_lite_in_place( T) nonstd::in_place_type +#define nonstd_lite_in_place_type( T) nonstd::in_place_type +#define nonstd_lite_in_place_index(K) nonstd::in_place_index + +} // namespace nonstd + +#endif // nsel_CPP17_OR_GREATER +#endif // nonstd_lite_HAVE_IN_PLACE_TYPES + +// +// Using std::expected: +// + +#if nsel_USES_STD_EXPECTED + +#include + +namespace nonstd { + + using std::expected; + using std::unexpected; + using std::bad_expected_access; + using std::unexpect_t; + using std::unexpect; + + //[[deprecated("replace unexpected_type with unexpected")]] + + template< typename E > + using unexpected_type = unexpected; + + // Unconditionally provide make_unexpected(): + + template< typename E > + constexpr auto make_unexpected( E && value ) -> unexpected< typename std::decay::type > + { + return unexpected< typename std::decay::type >( std::forward(value) ); + } + + template + < + typename E, typename... Args, + typename = std::enable_if< + std::is_constructible::value + > + > + constexpr auto + make_unexpected( std::in_place_t inplace, Args &&... args ) -> unexpected_type< typename std::decay::type > + { + return unexpected_type< typename std::decay::type >( inplace, std::forward(args)...); + } +} // namespace nonstd + +#else // nsel_USES_STD_EXPECTED + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// additional includes: + +#if nsel_CONFIG_WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +#endif + +#if nsel_CONFIG_NO_EXCEPTIONS +# if nsel_CONFIG_NO_EXCEPTIONS_SEH +# include // for ExceptionCodes +# else +// already included: +# endif +#else +# include +#endif + +// C++ feature usage: + +#if nsel_CPP11_OR_GREATER +# define nsel_constexpr constexpr +#else +# define nsel_constexpr /*constexpr*/ +#endif + +#if nsel_CPP14_OR_GREATER +# define nsel_constexpr14 constexpr +#else +# define nsel_constexpr14 /*constexpr*/ +#endif + +#if nsel_CPP17_OR_GREATER +# define nsel_inline17 inline +#else +# define nsel_inline17 /*inline*/ +#endif + +// Compiler versions: +// +// MSVC++ 6.0 _MSC_VER == 1200 nsel_COMPILER_MSVC_VERSION == 60 (Visual Studio 6.0) +// MSVC++ 7.0 _MSC_VER == 1300 nsel_COMPILER_MSVC_VERSION == 70 (Visual Studio .NET 2002) +// MSVC++ 7.1 _MSC_VER == 1310 nsel_COMPILER_MSVC_VERSION == 71 (Visual Studio .NET 2003) +// MSVC++ 8.0 _MSC_VER == 1400 nsel_COMPILER_MSVC_VERSION == 80 (Visual Studio 2005) +// MSVC++ 9.0 _MSC_VER == 1500 nsel_COMPILER_MSVC_VERSION == 90 (Visual Studio 2008) +// MSVC++ 10.0 _MSC_VER == 1600 nsel_COMPILER_MSVC_VERSION == 100 (Visual Studio 2010) +// MSVC++ 11.0 _MSC_VER == 1700 nsel_COMPILER_MSVC_VERSION == 110 (Visual Studio 2012) +// MSVC++ 12.0 _MSC_VER == 1800 nsel_COMPILER_MSVC_VERSION == 120 (Visual Studio 2013) +// MSVC++ 14.0 _MSC_VER == 1900 nsel_COMPILER_MSVC_VERSION == 140 (Visual Studio 2015) +// MSVC++ 14.1 _MSC_VER >= 1910 nsel_COMPILER_MSVC_VERSION == 141 (Visual Studio 2017) +// MSVC++ 14.2 _MSC_VER >= 1920 nsel_COMPILER_MSVC_VERSION == 142 (Visual Studio 2019) + +#if defined(_MSC_VER) && !defined(__clang__) +# define nsel_COMPILER_MSVC_VER (_MSC_VER ) +# define nsel_COMPILER_MSVC_VERSION (_MSC_VER / 10 - 10 * ( 5 + (_MSC_VER < 1900)) ) +#else +# define nsel_COMPILER_MSVC_VER 0 +# define nsel_COMPILER_MSVC_VERSION 0 +#endif + +#define nsel_COMPILER_VERSION( major, minor, patch ) ( 10 * ( 10 * (major) + (minor) ) + (patch) ) + +#if defined(__clang__) +# define nsel_COMPILER_CLANG_VERSION nsel_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__) +#else +# define nsel_COMPILER_CLANG_VERSION 0 +#endif + +#if defined(__GNUC__) && !defined(__clang__) +# define nsel_COMPILER_GNUC_VERSION nsel_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#else +# define nsel_COMPILER_GNUC_VERSION 0 +#endif + +// half-open range [lo..hi): +//#define nsel_BETWEEN( v, lo, hi ) ( (lo) <= (v) && (v) < (hi) ) + +// Method enabling + +#define nsel_REQUIRES_0(...) \ + template< bool B = (__VA_ARGS__), typename std::enable_if::type = 0 > + +#define nsel_REQUIRES_T(...) \ + , typename std::enable_if< (__VA_ARGS__), int >::type = 0 + +#define nsel_REQUIRES_R(R, ...) \ + typename std::enable_if< (__VA_ARGS__), R>::type + +#define nsel_REQUIRES_A(...) \ + , typename std::enable_if< (__VA_ARGS__), void*>::type = nullptr + +// Clang, GNUC, MSVC warning suppression macros: + +#ifdef __clang__ +# pragma clang diagnostic push +#elif defined __GNUC__ +# pragma GCC diagnostic push +#endif // __clang__ + +#if nsel_COMPILER_MSVC_VERSION >= 140 +# define nsel_DISABLE_MSVC_WARNINGS(codes) __pragma( warning(push) ) __pragma( warning(disable: codes) ) +#else +# define nsel_DISABLE_MSVC_WARNINGS(codes) +#endif + +#ifdef __clang__ +# define nsel_RESTORE_WARNINGS() _Pragma("clang diagnostic pop") +# define nsel_RESTORE_MSVC_WARNINGS() +#elif defined __GNUC__ +# define nsel_RESTORE_WARNINGS() _Pragma("GCC diagnostic pop") +# define nsel_RESTORE_MSVC_WARNINGS() +#elif nsel_COMPILER_MSVC_VERSION >= 140 +# define nsel_RESTORE_WARNINGS() __pragma( warning( pop ) ) +# define nsel_RESTORE_MSVC_WARNINGS() nsel_RESTORE_WARNINGS() +#else +# define nsel_RESTORE_WARNINGS() +# define nsel_RESTORE_MSVC_WARNINGS() +#endif + +// Suppress the following MSVC (GSL) warnings: +// - C26409: Avoid calling new and delete explicitly, use std::make_unique instead (r.11) + +nsel_DISABLE_MSVC_WARNINGS( 26409 ) + +// Presence of language and library features: + +#ifdef _HAS_CPP0X +# define nsel_HAS_CPP0X _HAS_CPP0X +#else +# define nsel_HAS_CPP0X 0 +#endif + +// Presence of language and library features: + +#define nsel_CPP11_000 (nsel_CPP11_OR_GREATER) +#define nsel_CPP17_000 (nsel_CPP17_OR_GREATER) + +// Presence of C++11 library features: + +#define nsel_HAVE_ADDRESSOF nsel_CPP11_000 + +// Presence of C++17 language features: + +#define nsel_HAVE_DEPRECATED nsel_CPP17_000 +#define nsel_HAVE_NODISCARD nsel_CPP17_000 + +// C++ feature usage: + +#if nsel_HAVE_DEPRECATED +# define nsel_deprecated(msg) [[deprecated(msg)]] +#else +# define nsel_deprecated(msg) /*[[deprecated]]*/ +#endif + +#if nsel_HAVE_NODISCARD && !nsel_CONFIG_NO_NODISCARD +# define nsel_NODISCARD [[nodiscard]] +#else +# define nsel_NODISCARD /*[[nodiscard]]*/ +#endif + +// +// expected: +// + +namespace nonstd { namespace expected_lite { + +// library features C++11: + +namespace std11 { + +// #if 0 && nsel_HAVE_ADDRESSOF +#if nsel_HAVE_ADDRESSOF + using std::addressof; +#else + template< class T > + T * addressof( T & arg ) noexcept + { + return &arg; + } + + template< class T > + const T * addressof( const T && ) = delete; +#endif +} // namespace std11 + +// type traits C++17: + +namespace std17 { + +#if nsel_CPP17_OR_GREATER + +using std::conjunction; +using std::is_swappable; +using std::is_nothrow_swappable; + +#else // nsel_CPP17_OR_GREATER + +namespace detail { + +using std::swap; + +struct is_swappable +{ + template< typename T, typename = decltype( swap( std::declval(), std::declval() ) ) > + static std::true_type test( int /* unused */); + + template< typename > + static std::false_type test(...); +}; + +struct is_nothrow_swappable +{ + // wrap noexcept(expr) in separate function as work-around for VC140 (VS2015): + + template< typename T > + static constexpr bool satisfies() + { + return noexcept( swap( std::declval(), std::declval() ) ); + } + + template< typename T > + static auto test( int ) -> std::integral_constant()>{} + + template< typename > + static auto test(...) -> std::false_type; +}; +} // namespace detail + +// is [nothrow] swappable: + +template< typename T > +struct is_swappable : decltype( detail::is_swappable::test(0) ){}; + +template< typename T > +struct is_nothrow_swappable : decltype( detail::is_nothrow_swappable::test(0) ){}; + +// conjunction: + +template< typename... > struct conjunction : std::true_type{}; +template< typename B1 > struct conjunction : B1{}; + +template< typename B1, typename... Bn > +struct conjunction : std::conditional, B1>::type{}; + +#endif // nsel_CPP17_OR_GREATER + +} // namespace std17 + +// type traits C++20: + +namespace std20 { + +#if defined(__cpp_lib_remove_cvref) + +using std::remove_cvref; + +#else + +template< typename T > +struct remove_cvref +{ + typedef typename std::remove_cv< typename std::remove_reference::type >::type type; +}; + +#endif + +} // namespace std20 + +// forward declaration: + +template< typename T, typename E > +class expected; + +namespace detail { + +#if nsel_P2505R >= 3 +template< typename T > +struct is_expected : std::false_type {}; + +template< typename T, typename E > +struct is_expected< expected< T, E > > : std::true_type {}; +#endif // nsel_P2505R >= 3 + +/// discriminated union to hold value or 'error'. + +template< typename T, typename E > +class storage_t_noncopy_nonmove_impl +{ + template< typename, typename > friend class nonstd::expected_lite::expected; + +public: + using value_type = T; + using error_type = E; + + // no-op construction + storage_t_noncopy_nonmove_impl() {} + ~storage_t_noncopy_nonmove_impl() {} + + explicit storage_t_noncopy_nonmove_impl( bool has_value ) + : m_has_value( has_value ) + {} + + void construct_value() + { + new( std11::addressof(m_value) ) value_type(); + } + + // void construct_value( value_type const & e ) + // { + // new( std11::addressof(m_value) ) value_type( e ); + // } + + // void construct_value( value_type && e ) + // { + // new( std11::addressof(m_value) ) value_type( std::move( e ) ); + // } + + template< class... Args > + void emplace_value( Args&&... args ) + { + new( std11::addressof(m_value) ) value_type( std::forward(args)...); + } + + template< class U, class... Args > + void emplace_value( std::initializer_list il, Args&&... args ) + { + new( std11::addressof(m_value) ) value_type( il, std::forward(args)... ); + } + + void destruct_value() + { + m_value.~value_type(); + } + + // void construct_error( error_type const & e ) + // { + // // new( std11::addressof(m_error) ) error_type( e ); + // } + + // void construct_error( error_type && e ) + // { + // // new( std11::addressof(m_error) ) error_type( std::move( e ) ); + // } + + template< class... Args > + void emplace_error( Args&&... args ) + { + new( std11::addressof(m_error) ) error_type( std::forward(args)...); + } + + template< class U, class... Args > + void emplace_error( std::initializer_list il, Args&&... args ) + { + new( std11::addressof(m_error) ) error_type( il, std::forward(args)... ); + } + + void destruct_error() + { + m_error.~error_type(); + } + + constexpr value_type const & value() const & + { + return m_value; + } + + value_type & value() & + { + return m_value; + } + + constexpr value_type const && value() const && + { + return std::move( m_value ); + } + + nsel_constexpr14 value_type && value() && + { + return std::move( m_value ); + } + + value_type const * value_ptr() const + { + return std11::addressof(m_value); + } + + value_type * value_ptr() + { + return std11::addressof(m_value); + } + + error_type const & error() const & + { + return m_error; + } + + error_type & error() & + { + return m_error; + } + + constexpr error_type const && error() const && + { + return std::move( m_error ); + } + + nsel_constexpr14 error_type && error() && + { + return std::move( m_error ); + } + + bool has_value() const + { + return m_has_value; + } + + void set_has_value( bool v ) + { + m_has_value = v; + } + +private: + union + { + value_type m_value; + error_type m_error; + }; + + bool m_has_value = false; +}; + +template< typename T, typename E > +class storage_t_impl +{ + template< typename, typename > friend class nonstd::expected_lite::expected; + +public: + using value_type = T; + using error_type = E; + + // no-op construction + storage_t_impl() {} + ~storage_t_impl() {} + + explicit storage_t_impl( bool has_value ) + : m_has_value( has_value ) + {} + + void construct_value() + { + new( std11::addressof(m_value) ) value_type(); + } + + void construct_value( value_type const & e ) + { + new( std11::addressof(m_value) ) value_type( e ); + } + + void construct_value( value_type && e ) + { + new( std11::addressof(m_value) ) value_type( std::move( e ) ); + } + + template< class... Args > + void emplace_value( Args&&... args ) + { + new( std11::addressof(m_value) ) value_type( std::forward(args)...); + } + + template< class U, class... Args > + void emplace_value( std::initializer_list il, Args&&... args ) + { + new( std11::addressof(m_value) ) value_type( il, std::forward(args)... ); + } + + void destruct_value() + { + m_value.~value_type(); + } + + void construct_error( error_type const & e ) + { + new( std11::addressof(m_error) ) error_type( e ); + } + + void construct_error( error_type && e ) + { + new( std11::addressof(m_error) ) error_type( std::move( e ) ); + } + + template< class... Args > + void emplace_error( Args&&... args ) + { + new( std11::addressof(m_error) ) error_type( std::forward(args)...); + } + + template< class U, class... Args > + void emplace_error( std::initializer_list il, Args&&... args ) + { + new( std11::addressof(m_error) ) error_type( il, std::forward(args)... ); + } + + void destruct_error() + { + m_error.~error_type(); + } + + constexpr value_type const & value() const & + { + return m_value; + } + + value_type & value() & + { + return m_value; + } + + constexpr value_type const && value() const && + { + return std::move( m_value ); + } + + nsel_constexpr14 value_type && value() && + { + return std::move( m_value ); + } + + value_type const * value_ptr() const + { + return std11::addressof(m_value); + } + + value_type * value_ptr() + { + return std11::addressof(m_value); + } + + error_type const & error() const & + { + return m_error; + } + + error_type & error() & + { + return m_error; + } + + constexpr error_type const && error() const && + { + return std::move( m_error ); + } + + nsel_constexpr14 error_type && error() && + { + return std::move( m_error ); + } + + bool has_value() const + { + return m_has_value; + } + + void set_has_value( bool v ) + { + m_has_value = v; + } + +private: + union + { + value_type m_value; + error_type m_error; + }; + + bool m_has_value = false; +}; + +/// discriminated union to hold only 'error'. + +template< typename E > +struct storage_t_impl< void, E > +{ + template< typename, typename > friend class nonstd::expected_lite::expected; + +public: + using value_type = void; + using error_type = E; + + // no-op construction + storage_t_impl() {} + ~storage_t_impl() {} + + explicit storage_t_impl( bool has_value ) + : m_has_value( has_value ) + {} + + void construct_error( error_type const & e ) + { + new( std11::addressof(m_error) ) error_type( e ); + } + + void construct_error( error_type && e ) + { + new( std11::addressof(m_error) ) error_type( std::move( e ) ); + } + + template< class... Args > + void emplace_error( Args&&... args ) + { + new( std11::addressof(m_error) ) error_type( std::forward(args)...); + } + + template< class U, class... Args > + void emplace_error( std::initializer_list il, Args&&... args ) + { + new( std11::addressof(m_error) ) error_type( il, std::forward(args)... ); + } + + void destruct_error() + { + m_error.~error_type(); + } + + error_type const & error() const & + { + return m_error; + } + + error_type & error() & + { + return m_error; + } + + constexpr error_type const && error() const && + { + return std::move( m_error ); + } + + nsel_constexpr14 error_type && error() && + { + return std::move( m_error ); + } + + bool has_value() const + { + return m_has_value; + } + + void set_has_value( bool v ) + { + m_has_value = v; + } + +private: + union + { + char m_dummy; + error_type m_error; + }; + + bool m_has_value = false; +}; + +template< typename T, typename E, bool isConstructable, bool isMoveable > +class storage_t +{ +public: +}; + +template< typename T, typename E > +class storage_t : public storage_t_noncopy_nonmove_impl +{ +public: + storage_t() = default; + ~storage_t() = default; + + explicit storage_t( bool has_value ) + : storage_t_noncopy_nonmove_impl( has_value ) + {} + + storage_t( storage_t const & other ) = delete; + storage_t( storage_t && other ) = delete; + +}; + +template< typename T, typename E > +class storage_t : public storage_t_impl +{ +public: + storage_t() = default; + ~storage_t() = default; + + explicit storage_t( bool has_value ) + : storage_t_impl( has_value ) + {} + + storage_t( storage_t const & other ) + : storage_t_impl( other.has_value() ) + { + if ( this->has_value() ) this->construct_value( other.value() ); + else this->construct_error( other.error() ); + } + + storage_t(storage_t && other ) + : storage_t_impl( other.has_value() ) + { + if ( this->has_value() ) this->construct_value( std::move( other.value() ) ); + else this->construct_error( std::move( other.error() ) ); + } +}; + +template< typename E > +class storage_t : public storage_t_impl +{ +public: + storage_t() = default; + ~storage_t() = default; + + explicit storage_t( bool has_value ) + : storage_t_impl( has_value ) + {} + + storage_t( storage_t const & other ) + : storage_t_impl( other.has_value() ) + { + if ( this->has_value() ) ; + else this->construct_error( other.error() ); + } + + storage_t(storage_t && other ) + : storage_t_impl( other.has_value() ) + { + if ( this->has_value() ) ; + else this->construct_error( std::move( other.error() ) ); + } +}; + +template< typename T, typename E > +class storage_t : public storage_t_impl +{ +public: + storage_t() = default; + ~storage_t() = default; + + explicit storage_t( bool has_value ) + : storage_t_impl( has_value ) + {} + + storage_t( storage_t const & other ) + : storage_t_impl(other.has_value()) + { + if ( this->has_value() ) this->construct_value( other.value() ); + else this->construct_error( other.error() ); + } + + storage_t( storage_t && other ) = delete; +}; + +template< typename E > +class storage_t : public storage_t_impl +{ +public: + storage_t() = default; + ~storage_t() = default; + + explicit storage_t( bool has_value ) + : storage_t_impl( has_value ) + {} + + storage_t( storage_t const & other ) + : storage_t_impl(other.has_value()) + { + if ( this->has_value() ) ; + else this->construct_error( other.error() ); + } + + storage_t( storage_t && other ) = delete; +}; + +template< typename T, typename E > +class storage_t : public storage_t_impl +{ +public: + storage_t() = default; + ~storage_t() = default; + + explicit storage_t( bool has_value ) + : storage_t_impl( has_value ) + {} + + storage_t( storage_t const & other ) = delete; + + storage_t( storage_t && other ) + : storage_t_impl( other.has_value() ) + { + if ( this->has_value() ) this->construct_value( std::move( other.value() ) ); + else this->construct_error( std::move( other.error() ) ); + } +}; + +template< typename E > +class storage_t : public storage_t_impl +{ +public: + storage_t() = default; + ~storage_t() = default; + + explicit storage_t( bool has_value ) + : storage_t_impl( has_value ) + {} + + storage_t( storage_t const & other ) = delete; + + storage_t( storage_t && other ) + : storage_t_impl( other.has_value() ) + { + if ( this->has_value() ) ; + else this->construct_error( std::move( other.error() ) ); + } +}; + +#if nsel_P2505R >= 3 +// C++11 invoke implementation +template< typename > +struct is_reference_wrapper : std::false_type {}; +template< typename T > +struct is_reference_wrapper< std::reference_wrapper< T > > : std::true_type {}; + +template< typename FnT, typename ClassT, typename ObjectT, typename... Args + nsel_REQUIRES_T( + std::is_function::value + && ( std::is_same< ClassT, typename std20::remove_cvref< ObjectT >::type >::value + || std::is_base_of< ClassT, typename std20::remove_cvref< ObjectT >::type >::value ) + ) +> +nsel_constexpr auto invoke_member_function_impl( FnT ClassT::* memfnptr, ObjectT && obj, Args && ... args ) + noexcept( noexcept( (std::forward< ObjectT >( obj ).*memfnptr)( std::forward< Args >( args )... ) ) ) + -> decltype( (std::forward< ObjectT >( obj ).*memfnptr)( std::forward< Args >( args )...) ) +{ + return (std::forward< ObjectT >( obj ).*memfnptr)( std::forward< Args >( args )... ); +} + +template< typename FnT, typename ClassT, typename ObjectT, typename... Args + nsel_REQUIRES_T( + std::is_function::value + && is_reference_wrapper< typename std20::remove_cvref< ObjectT >::type >::value + ) +> +nsel_constexpr auto invoke_member_function_impl( FnT ClassT::* memfnptr, ObjectT && obj, Args && ... args ) + noexcept( noexcept( (obj.get().*memfnptr)( std::forward< Args >( args ) ... ) ) ) + -> decltype( (obj.get().*memfnptr)( std::forward< Args >( args ) ... ) ) +{ + return (obj.get().*memfnptr)( std::forward< Args >( args ) ... ); +} + +template< typename FnT, typename ClassT, typename ObjectT, typename... Args + nsel_REQUIRES_T( + std::is_function::value + && !std::is_same< ClassT, typename std20::remove_cvref< ObjectT >::type >::value + && !std::is_base_of< ClassT, typename std20::remove_cvref< ObjectT >::type >::value + && !is_reference_wrapper< typename std20::remove_cvref< ObjectT >::type >::value + ) +> +nsel_constexpr auto invoke_member_function_impl( FnT ClassT::* memfnptr, ObjectT && obj, Args && ... args ) + noexcept( noexcept( ((*std::forward< ObjectT >( obj )).*memfnptr)( std::forward< Args >( args ) ... ) ) ) + -> decltype( ((*std::forward< ObjectT >( obj )).*memfnptr)( std::forward< Args >( args ) ... ) ) +{ + return ((*std::forward(obj)).*memfnptr)( std::forward< Args >( args ) ... ); +} + +template< typename MemberT, typename ClassT, typename ObjectT + nsel_REQUIRES_T( + std::is_same< ClassT, typename std20::remove_cvref< ObjectT >::type >::value + || std::is_base_of< ClassT, typename std20::remove_cvref< ObjectT >::type >::value + ) +> +nsel_constexpr auto invoke_member_object_impl( MemberT ClassT::* memobjptr, ObjectT && obj ) + noexcept( noexcept( std::forward< ObjectT >( obj ).*memobjptr ) ) + -> decltype( std::forward< ObjectT >( obj ).*memobjptr ) +{ + return std::forward< ObjectT >( obj ).*memobjptr; +} + +template< typename MemberT, typename ClassT, typename ObjectT + nsel_REQUIRES_T( + is_reference_wrapper< typename std20::remove_cvref< ObjectT >::type >::value + ) +> +nsel_constexpr auto invoke_member_object_impl( MemberT ClassT::* memobjptr, ObjectT && obj ) + noexcept( noexcept( obj.get().*memobjptr ) ) + -> decltype( obj.get().*memobjptr ) +{ + return obj.get().*memobjptr; +} + +template< typename MemberT, typename ClassT, typename ObjectT + nsel_REQUIRES_T( + !std::is_same< ClassT, typename std20::remove_cvref< ObjectT >::type >::value + && !std::is_base_of< ClassT, typename std20::remove_cvref< ObjectT >::type >::value + && !is_reference_wrapper< typename std20::remove_cvref< ObjectT >::type >::value + ) +> +nsel_constexpr auto invoke_member_object_impl( MemberT ClassT::* memobjptr, ObjectT && obj ) + noexcept( noexcept( (*std::forward< ObjectT >( obj )).*memobjptr ) ) + -> decltype( (*std::forward< ObjectT >( obj )).*memobjptr ) +{ + return (*std::forward< ObjectT >( obj )).*memobjptr; +} + +template< typename F, typename... Args + nsel_REQUIRES_T( + std::is_member_function_pointer< typename std20::remove_cvref< F >::type >::value + ) +> +nsel_constexpr auto invoke( F && f, Args && ... args ) + noexcept( noexcept( invoke_member_function_impl( std::forward< F >( f ), std::forward< Args >( args ) ... ) ) ) + -> decltype( invoke_member_function_impl( std::forward< F >( f ), std::forward< Args >( args ) ... ) ) +{ + return invoke_member_function_impl( std::forward< F >( f ), std::forward< Args >( args ) ... ); +} + +template< typename F, typename... Args + nsel_REQUIRES_T( + std::is_member_object_pointer< typename std20::remove_cvref< F >::type >::value + ) +> +nsel_constexpr auto invoke( F && f, Args && ... args ) + noexcept( noexcept( invoke_member_object_impl( std::forward< F >( f ), std::forward< Args >( args ) ... ) ) ) + -> decltype( invoke_member_object_impl( std::forward< F >( f ), std::forward< Args >( args ) ... ) ) +{ + return invoke_member_object_impl( std::forward< F >( f ), std::forward< Args >( args ) ... ); +} + +template< typename F, typename... Args + nsel_REQUIRES_T( + !std::is_member_function_pointer< typename std20::remove_cvref< F >::type >::value + && !std::is_member_object_pointer< typename std20::remove_cvref< F >::type >::value + ) +> +nsel_constexpr auto invoke( F && f, Args && ... args ) + noexcept( noexcept( std::forward< F >( f )( std::forward< Args >( args ) ... ) ) ) + -> decltype( std::forward< F >( f )( std::forward< Args >( args ) ... ) ) +{ + return std::forward< F >( f )( std::forward< Args >( args ) ... ); +} + +template< typename F, typename ... Args > +using invoke_result_nocvref_t = typename std20::remove_cvref< decltype( ::nonstd::expected_lite::detail::invoke( std::declval< F >(), std::declval< Args >()... ) ) >::type; + +#if nsel_P2505R >= 5 +template< typename F, typename ... Args > +using transform_invoke_result_t = typename std::remove_cv< decltype( ::nonstd::expected_lite::detail::invoke( std::declval< F >(), std::declval< Args >()... ) ) >::type; +#else +template< typename F, typename ... Args > +using transform_invoke_result_t = invoke_result_nocvref_t +#endif // nsel_P2505R >= 5 + +template< typename T > +struct valid_expected_value_type : std::integral_constant< bool, std::is_destructible< T >::value && !std::is_reference< T >::value && !std::is_array< T >::value > {}; + +#endif // nsel_P2505R >= 3 +} // namespace detail + +/// x.x.5 Unexpected object type; unexpected_type; C++17 and later can also use aliased type unexpected. + +#if nsel_P0323R <= 2 +template< typename E = std::exception_ptr > +class unexpected_type +#else +template< typename E > +class unexpected_type +#endif // nsel_P0323R +{ +public: + using error_type = E; + + // x.x.5.2.1 Constructors + +// unexpected_type() = delete; + + constexpr unexpected_type( unexpected_type const & ) = default; + constexpr unexpected_type( unexpected_type && ) = default; + + template< typename... Args + nsel_REQUIRES_T( + std::is_constructible::value + ) + > + constexpr explicit unexpected_type( nonstd_lite_in_place_t(E), Args &&... args ) + : m_error( std::forward( args )...) + {} + + template< typename U, typename... Args + nsel_REQUIRES_T( + std::is_constructible, Args&&...>::value + ) + > + constexpr explicit unexpected_type( nonstd_lite_in_place_t(E), std::initializer_list il, Args &&... args ) + : m_error( il, std::forward( args )...) + {} + + template< typename E2 + nsel_REQUIRES_T( + std::is_constructible::value + && !std::is_same< typename std20::remove_cvref::type, nonstd_lite_in_place_t(E2) >::value + && !std::is_same< typename std20::remove_cvref::type, unexpected_type >::value + ) + > + constexpr explicit unexpected_type( E2 && error ) + : m_error( std::forward( error ) ) + {} + + template< typename E2 + nsel_REQUIRES_T( + std::is_constructible< E, E2>::value + && !std::is_constructible & >::value + && !std::is_constructible >::value + && !std::is_constructible const & >::value + && !std::is_constructible const >::value + && !std::is_convertible< unexpected_type &, E>::value + && !std::is_convertible< unexpected_type , E>::value + && !std::is_convertible< unexpected_type const &, E>::value + && !std::is_convertible< unexpected_type const , E>::value + && !std::is_convertible< E2 const &, E>::value /*=> explicit */ + ) + > + constexpr explicit unexpected_type( unexpected_type const & error ) + : m_error( E{ error.error() } ) + {} + + template< typename E2 + nsel_REQUIRES_T( + std::is_constructible< E, E2>::value + && !std::is_constructible & >::value + && !std::is_constructible >::value + && !std::is_constructible const & >::value + && !std::is_constructible const >::value + && !std::is_convertible< unexpected_type &, E>::value + && !std::is_convertible< unexpected_type , E>::value + && !std::is_convertible< unexpected_type const &, E>::value + && !std::is_convertible< unexpected_type const , E>::value + && std::is_convertible< E2 const &, E>::value /*=> explicit */ + ) + > + constexpr /*non-explicit*/ unexpected_type( unexpected_type const & error ) + : m_error( error.error() ) + {} + + template< typename E2 + nsel_REQUIRES_T( + std::is_constructible< E, E2>::value + && !std::is_constructible & >::value + && !std::is_constructible >::value + && !std::is_constructible const & >::value + && !std::is_constructible const >::value + && !std::is_convertible< unexpected_type &, E>::value + && !std::is_convertible< unexpected_type , E>::value + && !std::is_convertible< unexpected_type const &, E>::value + && !std::is_convertible< unexpected_type const , E>::value + && !std::is_convertible< E2 const &, E>::value /*=> explicit */ + ) + > + constexpr explicit unexpected_type( unexpected_type && error ) + : m_error( E{ std::move( error.error() ) } ) + {} + + template< typename E2 + nsel_REQUIRES_T( + std::is_constructible< E, E2>::value + && !std::is_constructible & >::value + && !std::is_constructible >::value + && !std::is_constructible const & >::value + && !std::is_constructible const >::value + && !std::is_convertible< unexpected_type &, E>::value + && !std::is_convertible< unexpected_type , E>::value + && !std::is_convertible< unexpected_type const &, E>::value + && !std::is_convertible< unexpected_type const , E>::value + && std::is_convertible< E2 const &, E>::value /*=> non-explicit */ + ) + > + constexpr /*non-explicit*/ unexpected_type( unexpected_type && error ) + : m_error( std::move( error.error() ) ) + {} + + // x.x.5.2.2 Assignment + + nsel_constexpr14 unexpected_type& operator=( unexpected_type const & ) = default; + nsel_constexpr14 unexpected_type& operator=( unexpected_type && ) = default; + + template< typename E2 = E > + nsel_constexpr14 unexpected_type & operator=( unexpected_type const & other ) + { + unexpected_type{ other.error() }.swap( *this ); + return *this; + } + + template< typename E2 = E > + nsel_constexpr14 unexpected_type & operator=( unexpected_type && other ) + { + unexpected_type{ std::move( other.error() ) }.swap( *this ); + return *this; + } + + // x.x.5.2.3 Observers + + nsel_constexpr14 E & error() & noexcept + { + return m_error; + } + + constexpr E const & error() const & noexcept + { + return m_error; + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + + nsel_constexpr14 E && error() && noexcept + { + return std::move( m_error ); + } + + constexpr E const && error() const && noexcept + { + return std::move( m_error ); + } + +#endif + + // x.x.5.2.3 Observers - deprecated + + nsel_deprecated("replace value() with error()") + + nsel_constexpr14 E & value() & noexcept + { + return m_error; + } + + nsel_deprecated("replace value() with error()") + + constexpr E const & value() const & noexcept + { + return m_error; + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + + nsel_deprecated("replace value() with error()") + + nsel_constexpr14 E && value() && noexcept + { + return std::move( m_error ); + } + + nsel_deprecated("replace value() with error()") + + constexpr E const && value() const && noexcept + { + return std::move( m_error ); + } + +#endif + + // x.x.5.2.4 Swap + + template< typename U=E > + nsel_REQUIRES_R( void, + std17::is_swappable::value + ) + swap( unexpected_type & other ) noexcept ( + std17::is_nothrow_swappable::value + ) + { + using std::swap; + swap( m_error, other.m_error ); + } + + // TODO: ??? unexpected_type: in-class friend operator==, != + +private: + error_type m_error; +}; + +#if nsel_CPP17_OR_GREATER + +/// template deduction guide: + +template< typename E > +unexpected_type( E ) -> unexpected_type< E >; + +#endif + +/// class unexpected_type, std::exception_ptr specialization (P0323R2) + +#if !nsel_CONFIG_NO_EXCEPTIONS +#if nsel_P0323R <= 2 + +// TODO: Should expected be specialized for particular E types such as exception_ptr and how? +// See p0323r7 2.1. Ergonomics, http://wg21.link/p0323 +template<> +class unexpected_type< std::exception_ptr > +{ +public: + using error_type = std::exception_ptr; + + unexpected_type() = delete; + + ~unexpected_type(){} + + explicit unexpected_type( std::exception_ptr const & error ) + : m_error( error ) + {} + + explicit unexpected_type(std::exception_ptr && error ) + : m_error( std::move( error ) ) + {} + + template< typename E > + explicit unexpected_type( E error ) + : m_error( std::make_exception_ptr( error ) ) + {} + + std::exception_ptr const & value() const + { + return m_error; + } + + std::exception_ptr & value() + { + return m_error; + } + +private: + std::exception_ptr m_error; +}; + +#endif // nsel_P0323R +#endif // !nsel_CONFIG_NO_EXCEPTIONS + +/// x.x.4, Unexpected equality operators + +template< typename E1, typename E2 > +constexpr bool operator==( unexpected_type const & x, unexpected_type const & y ) +{ + return x.error() == y.error(); +} + +template< typename E1, typename E2 > +constexpr bool operator!=( unexpected_type const & x, unexpected_type const & y ) +{ + return ! ( x == y ); +} + +#if nsel_P0323R <= 2 + +template< typename E > +constexpr bool operator<( unexpected_type const & x, unexpected_type const & y ) +{ + return x.error() < y.error(); +} + +template< typename E > +constexpr bool operator>( unexpected_type const & x, unexpected_type const & y ) +{ + return ( y < x ); +} + +template< typename E > +constexpr bool operator<=( unexpected_type const & x, unexpected_type const & y ) +{ + return ! ( y < x ); +} + +template< typename E > +constexpr bool operator>=( unexpected_type const & x, unexpected_type const & y ) +{ + return ! ( x < y ); +} + +#endif // nsel_P0323R + +/// x.x.5 Specialized algorithms + +template< typename E + nsel_REQUIRES_T( + std17::is_swappable::value + ) +> +void swap( unexpected_type & x, unexpected_type & y) noexcept ( noexcept ( x.swap(y) ) ) +{ + x.swap( y ); +} + +#if nsel_P0323R <= 2 + +// unexpected: relational operators for std::exception_ptr: + +inline constexpr bool operator<( unexpected_type const & /*x*/, unexpected_type const & /*y*/ ) +{ + return false; +} + +inline constexpr bool operator>( unexpected_type const & /*x*/, unexpected_type const & /*y*/ ) +{ + return false; +} + +inline constexpr bool operator<=( unexpected_type const & x, unexpected_type const & y ) +{ + return ( x == y ); +} + +inline constexpr bool operator>=( unexpected_type const & x, unexpected_type const & y ) +{ + return ( x == y ); +} + +#endif // nsel_P0323R + +// unexpected: traits + +#if nsel_P0323R <= 3 + +template< typename E > +struct is_unexpected : std::false_type {}; + +template< typename E > +struct is_unexpected< unexpected_type > : std::true_type {}; + +#endif // nsel_P0323R + +// unexpected: factory + +// keep make_unexpected() removed in p0323r2 for pre-C++17: + +template< typename E > +nsel_constexpr14 auto +make_unexpected( E && value ) -> unexpected_type< typename std::decay::type > +{ + return unexpected_type< typename std::decay::type >( std::forward(value) ); +} + +template +< + typename E, typename... Args, + typename = std::enable_if< + std::is_constructible::value + > +> +nsel_constexpr14 auto +make_unexpected( nonstd_lite_in_place_t(E), Args &&... args ) -> unexpected_type< typename std::decay::type > +{ + return std::move( unexpected_type< typename std::decay::type >( nonstd_lite_in_place(E), std::forward(args)...) ); +} + +#if nsel_P0323R <= 3 + +/*nsel_constexpr14*/ auto inline +make_unexpected_from_current_exception() -> unexpected_type< std::exception_ptr > +{ + return unexpected_type< std::exception_ptr >( std::current_exception() ); +} + +#endif // nsel_P0323R + +/// x.x.6, x.x.7 expected access error + +template< typename E > +class nsel_NODISCARD bad_expected_access; + +/// x.x.7 bad_expected_access: expected access error + +template <> +class nsel_NODISCARD bad_expected_access< void > : public std::exception +{ +public: + explicit bad_expected_access() + : std::exception() + {} +}; + +/// x.x.6 bad_expected_access: expected access error + +#if !nsel_CONFIG_NO_EXCEPTIONS + +template< typename E > +class nsel_NODISCARD bad_expected_access : public bad_expected_access< void > +{ +public: + using error_type = E; + + explicit bad_expected_access( error_type error ) + : m_error( error ) + {} + + virtual char const * what() const noexcept override + { + return "bad_expected_access"; + } + + nsel_constexpr14 error_type & error() & + { + return m_error; + } + + constexpr error_type const & error() const & + { + return m_error; + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + + nsel_constexpr14 error_type && error() && + { + return std::move( m_error ); + } + + constexpr error_type const && error() const && + { + return std::move( m_error ); + } + +#endif + +private: + error_type m_error; +}; + +#endif // nsel_CONFIG_NO_EXCEPTIONS + +/// x.x.8 unexpect tag, in_place_unexpected tag: construct an error + +struct unexpect_t{}; +using in_place_unexpected_t = unexpect_t; + +nsel_inline17 constexpr unexpect_t unexpect{}; +nsel_inline17 constexpr unexpect_t in_place_unexpected{}; + +/// class error_traits + +#if nsel_CONFIG_NO_EXCEPTIONS + +namespace detail { + inline bool text( char const * /*text*/ ) { return true; } +} + +template< typename Error > +struct error_traits +{ + static void rethrow( Error const & /*e*/ ) + { +#if nsel_CONFIG_NO_EXCEPTIONS_SEH + RaiseException( EXCEPTION_ACCESS_VIOLATION, EXCEPTION_NONCONTINUABLE, 0, NULL ); +#else + assert( false && detail::text("throw bad_expected_access{ e };") ); +#endif + } +}; + +template<> +struct error_traits< std::exception_ptr > +{ + static void rethrow( std::exception_ptr const & /*e*/ ) + { +#if nsel_CONFIG_NO_EXCEPTIONS_SEH + RaiseException( EXCEPTION_ACCESS_VIOLATION, EXCEPTION_NONCONTINUABLE, 0, NULL ); +#else + assert( false && detail::text("throw bad_expected_access{ e };") ); +#endif + } +}; + +template<> +struct error_traits< std::error_code > +{ + static void rethrow( std::error_code const & /*e*/ ) + { +#if nsel_CONFIG_NO_EXCEPTIONS_SEH + RaiseException( EXCEPTION_ACCESS_VIOLATION, EXCEPTION_NONCONTINUABLE, 0, NULL ); +#else + assert( false && detail::text("throw std::system_error( e );") ); +#endif + } +}; + +#else // nsel_CONFIG_NO_EXCEPTIONS + +template< typename Error > +struct error_traits +{ + static void rethrow( Error const & e ) + { + throw bad_expected_access{ e }; + } +}; + +template<> +struct error_traits< std::exception_ptr > +{ + static void rethrow( std::exception_ptr const & e ) + { + std::rethrow_exception( e ); + } +}; + +template<> +struct error_traits< std::error_code > +{ + static void rethrow( std::error_code const & e ) + { + throw std::system_error( e ); + } +}; + +#endif // nsel_CONFIG_NO_EXCEPTIONS + +#if nsel_P2505R >= 3 +namespace detail { + +// from https://en.cppreference.com/w/cpp/utility/expected/unexpected: +// "the type of the unexpected value. The type must not be an array type, a non-object type, a specialization of std::unexpected, or a cv-qualified type." +template< typename T > +struct valid_unexpected_type : std::integral_constant< bool, + std::is_same< T, typename std20::remove_cvref< T >::type >::value + && std::is_object< T >::value + && !std::is_array< T >::value +> {}; + +template< typename T > +struct valid_unexpected_type< unexpected_type< T > > : std::false_type {}; + +} // namespace detail +#endif // nsel_P2505R >= 3 + +} // namespace expected_lite + +// provide nonstd::unexpected_type: + +using expected_lite::unexpected_type; + +namespace expected_lite { + +/// class expected + +#if nsel_P0323R <= 2 +template< typename T, typename E = std::exception_ptr > +class nsel_NODISCARD expected +#else +template< typename T, typename E > +class nsel_NODISCARD expected +#endif // nsel_P0323R +{ +private: + template< typename, typename > friend class expected; + +public: + using value_type = T; + using error_type = E; + using unexpected_type = nonstd::unexpected_type; + + template< typename U > + struct rebind + { + using type = expected; + }; + + // x.x.4.1 constructors + + nsel_REQUIRES_0( + std::is_default_constructible::value + ) + nsel_constexpr14 expected() + : contained( true ) + { + contained.construct_value(); + } + + nsel_constexpr14 expected( expected const & ) = default; + nsel_constexpr14 expected( expected && ) = default; + + template< typename U, typename G + nsel_REQUIRES_T( + std::is_constructible< T, U const &>::value + && std::is_constructible::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< expected & , T>::value + && !std::is_convertible< expected &&, T>::value + && !std::is_convertible< expected const & , T>::value + && !std::is_convertible< expected const &&, T>::value + && (!std::is_convertible::value || !std::is_convertible::value ) /*=> explicit */ + ) + > + nsel_constexpr14 explicit expected( expected const & other ) + : contained( other.has_value() ) + { + if ( has_value() ) contained.construct_value( T{ other.contained.value() } ); + else contained.construct_error( E{ other.contained.error() } ); + } + + template< typename U, typename G + nsel_REQUIRES_T( + std::is_constructible< T, U const &>::value + && std::is_constructible::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< expected & , T>::value + && !std::is_convertible< expected &&, T>::value + && !std::is_convertible< expected const &, T>::value + && !std::is_convertible< expected const &&, T>::value + && !(!std::is_convertible::value || !std::is_convertible::value ) /*=> non-explicit */ + ) + > + nsel_constexpr14 /*non-explicit*/ expected( expected const & other ) + : contained( other.has_value() ) + { + if ( has_value() ) contained.construct_value( other.contained.value() ); + else contained.construct_error( other.contained.error() ); + } + + template< typename U, typename G + nsel_REQUIRES_T( + std::is_constructible< T, U>::value + && std::is_constructible::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< expected & , T>::value + && !std::is_convertible< expected &&, T>::value + && !std::is_convertible< expected const & , T>::value + && !std::is_convertible< expected const &&, T>::value + && (!std::is_convertible::value || !std::is_convertible::value ) /*=> explicit */ + ) + > + nsel_constexpr14 explicit expected( expected && other ) + : contained( other.has_value() ) + { + if ( has_value() ) contained.construct_value( T{ std::move( other.contained.value() ) } ); + else contained.construct_error( E{ std::move( other.contained.error() ) } ); + } + + template< typename U, typename G + nsel_REQUIRES_T( + std::is_constructible< T, U>::value + && std::is_constructible::value + && !std::is_constructible & >::value + && !std::is_constructible && >::value + && !std::is_constructible const & >::value + && !std::is_constructible const && >::value + && !std::is_convertible< expected & , T>::value + && !std::is_convertible< expected &&, T>::value + && !std::is_convertible< expected const & , T>::value + && !std::is_convertible< expected const &&, T>::value + && !(!std::is_convertible::value || !std::is_convertible::value ) /*=> non-explicit */ + ) + > + nsel_constexpr14 /*non-explicit*/ expected( expected && other ) + : contained( other.has_value() ) + { + if ( has_value() ) contained.construct_value( std::move( other.contained.value() ) ); + else contained.construct_error( std::move( other.contained.error() ) ); + } + + template< typename U = T + nsel_REQUIRES_T( + std::is_copy_constructible::value + ) + > + nsel_constexpr14 expected( value_type const & value ) + : contained( true ) + { + contained.construct_value( value ); + } + + template< typename U = T + nsel_REQUIRES_T( + std::is_constructible::value + && !std::is_same::type, nonstd_lite_in_place_t(U)>::value + && !std::is_same< expected , typename std20::remove_cvref::type>::value + && !std::is_same, typename std20::remove_cvref::type>::value + && !std::is_convertible::value /*=> explicit */ + ) + > + nsel_constexpr14 explicit expected( U && value ) noexcept + ( + std::is_nothrow_move_constructible::value && + std::is_nothrow_move_constructible::value + ) + : contained( true ) + { + contained.construct_value( T{ std::forward( value ) } ); + } + + template< typename U = T + nsel_REQUIRES_T( + std::is_constructible::value + && !std::is_same::type, nonstd_lite_in_place_t(U)>::value + && !std::is_same< expected , typename std20::remove_cvref::type>::value + && !std::is_same, typename std20::remove_cvref::type>::value + && std::is_convertible::value /*=> non-explicit */ + ) + > + nsel_constexpr14 /*non-explicit*/ expected( U && value ) noexcept + ( + std::is_nothrow_move_constructible::value && + std::is_nothrow_move_constructible::value + ) + : contained( true ) + { + contained.construct_value( std::forward( value ) ); + } + + // construct error: + + template< typename G = E + nsel_REQUIRES_T( + std::is_constructible::value + && !std::is_convertible< G const &, E>::value /*=> explicit */ + ) + > + nsel_constexpr14 explicit expected( nonstd::unexpected_type const & error ) + : contained( false ) + { + contained.construct_error( E{ error.error() } ); + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_constructible::value + && std::is_convertible< G const &, E>::value /*=> non-explicit */ + ) + > + nsel_constexpr14 /*non-explicit*/ expected( nonstd::unexpected_type const & error ) + : contained( false ) + { + contained.construct_error( error.error() ); + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_constructible::value + && !std::is_convertible< G&&, E>::value /*=> explicit */ + ) + > + nsel_constexpr14 explicit expected( nonstd::unexpected_type && error ) + : contained( false ) + { + contained.construct_error( E{ std::move( error.error() ) } ); + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_constructible::value + && std::is_convertible< G&&, E>::value /*=> non-explicit */ + ) + > + nsel_constexpr14 /*non-explicit*/ expected( nonstd::unexpected_type && error ) + : contained( false ) + { + contained.construct_error( std::move( error.error() ) ); + } + + // in-place construction, value + + template< typename... Args + nsel_REQUIRES_T( + std::is_constructible::value + ) + > + nsel_constexpr14 explicit expected( nonstd_lite_in_place_t(T), Args&&... args ) + : contained( true ) + { + contained.emplace_value( std::forward( args )... ); + } + + template< typename U, typename... Args + nsel_REQUIRES_T( + std::is_constructible, Args&&...>::value + ) + > + nsel_constexpr14 explicit expected( nonstd_lite_in_place_t(T), std::initializer_list il, Args&&... args ) + : contained( true ) + { + contained.emplace_value( il, std::forward( args )... ); + } + + // in-place construction, error + + template< typename... Args + nsel_REQUIRES_T( + std::is_constructible::value + ) + > + nsel_constexpr14 explicit expected( unexpect_t, Args&&... args ) + : contained( false ) + { + contained.emplace_error( std::forward( args )... ); + } + + template< typename U, typename... Args + nsel_REQUIRES_T( + std::is_constructible, Args&&...>::value + ) + > + nsel_constexpr14 explicit expected( unexpect_t, std::initializer_list il, Args&&... args ) + : contained( false ) + { + contained.emplace_error( il, std::forward( args )... ); + } + + // x.x.4.2 destructor + + // TODO: ~expected: triviality + // Effects: If T is not cv void and is_trivially_destructible_v is false and bool(*this), calls val.~T(). If is_trivially_destructible_v is false and !bool(*this), calls unexpect.~unexpected(). + // Remarks: If either T is cv void or is_trivially_destructible_v is true, and is_trivially_destructible_v is true, then this destructor shall be a trivial destructor. + + ~expected() + { + if ( has_value() ) contained.destruct_value(); + else contained.destruct_error(); + } + + // x.x.4.3 assignment + + expected & operator=( expected const & other ) + { + expected( other ).swap( *this ); + return *this; + } + + expected & operator=( expected && other ) noexcept + ( + std::is_nothrow_move_constructible< T>::value + && std::is_nothrow_move_assignable< T>::value + && std::is_nothrow_move_constructible::value // added for missing + && std::is_nothrow_move_assignable< E>::value ) // nothrow above + { + expected( std::move( other ) ).swap( *this ); + return *this; + } + + template< typename U + nsel_REQUIRES_T( + !std::is_same, typename std20::remove_cvref::type>::value + && std17::conjunction, std::is_same> >::value + && std::is_constructible::value + && std::is_assignable< T&,U>::value + && std::is_nothrow_move_constructible::value ) + > + expected & operator=( U && value ) + { + expected( std::forward( value ) ).swap( *this ); + return *this; + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_constructible::value && + std::is_copy_constructible::value // TODO: std::is_nothrow_copy_constructible + && std::is_copy_assignable::value + ) + > + expected & operator=( nonstd::unexpected_type const & error ) + { + expected( unexpect, error.error() ).swap( *this ); + return *this; + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_constructible::value && + std::is_move_constructible::value // TODO: std::is_nothrow_move_constructible + && std::is_move_assignable::value + ) + > + expected & operator=( nonstd::unexpected_type && error ) + { + expected( unexpect, std::move( error.error() ) ).swap( *this ); + return *this; + } + + template< typename... Args + nsel_REQUIRES_T( + std::is_nothrow_constructible::value + ) + > + value_type & emplace( Args &&... args ) + { + expected( nonstd_lite_in_place(T), std::forward(args)... ).swap( *this ); + return value(); + } + + template< typename U, typename... Args + nsel_REQUIRES_T( + std::is_nothrow_constructible&, Args&&...>::value + ) + > + value_type & emplace( std::initializer_list il, Args &&... args ) + { + expected( nonstd_lite_in_place(T), il, std::forward(args)... ).swap( *this ); + return value(); + } + + // x.x.4.4 swap + + template< typename U=T, typename G=E > + nsel_REQUIRES_R( void, + std17::is_swappable< U>::value + && std17::is_swappable::value + && ( std::is_move_constructible::value || std::is_move_constructible::value ) + ) + swap( expected & other ) noexcept + ( + std::is_nothrow_move_constructible::value && std17::is_nothrow_swappable::value && + std::is_nothrow_move_constructible::value && std17::is_nothrow_swappable::value + ) + { + using std::swap; + + if ( bool(*this) && bool(other) ) { swap( contained.value(), other.contained.value() ); } + else if ( ! bool(*this) && ! bool(other) ) { swap( contained.error(), other.contained.error() ); } + else if ( bool(*this) && ! bool(other) ) { error_type t( std::move( other.error() ) ); + other.contained.destruct_error(); + other.contained.construct_value( std::move( contained.value() ) ); + contained.destruct_value(); + contained.construct_error( std::move( t ) ); + bool has_value = contained.has_value(); + bool other_has_value = other.has_value(); + other.contained.set_has_value(has_value); + contained.set_has_value(other_has_value); + } + else if ( ! bool(*this) && bool(other) ) { other.swap( *this ); } + } + + // x.x.4.5 observers + + constexpr value_type const * operator ->() const + { + return assert( has_value() ), contained.value_ptr(); + } + + value_type * operator ->() + { + return assert( has_value() ), contained.value_ptr(); + } + + constexpr value_type const & operator *() const & + { + return assert( has_value() ), contained.value(); + } + + value_type & operator *() & + { + return assert( has_value() ), contained.value(); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + + constexpr value_type const && operator *() const && + { + return std::move( ( assert( has_value() ), contained.value() ) ); + } + + nsel_constexpr14 value_type && operator *() && + { + return std::move( ( assert( has_value() ), contained.value() ) ); + } + +#endif + + constexpr explicit operator bool() const noexcept + { + return has_value(); + } + + constexpr bool has_value() const noexcept + { + return contained.has_value(); + } + + nsel_DISABLE_MSVC_WARNINGS( 4702 ) // warning C4702: unreachable code, see issue 65. + + constexpr value_type const & value() const & + { + return has_value() + ? ( contained.value() ) + : ( error_traits::rethrow( contained.error() ), contained.value() ); + } + + value_type & value() & + { + return has_value() + ? ( contained.value() ) + : ( error_traits::rethrow( contained.error() ), contained.value() ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + + constexpr value_type const && value() const && + { + return std::move( has_value() + ? ( contained.value() ) + : ( error_traits::rethrow( contained.error() ), contained.value() ) ); + } + + nsel_constexpr14 value_type && value() && + { + return std::move( has_value() + ? ( contained.value() ) + : ( error_traits::rethrow( contained.error() ), contained.value() ) ); + } + +#endif + nsel_RESTORE_MSVC_WARNINGS() + + constexpr error_type const & error() const & + { + return assert( ! has_value() ), contained.error(); + } + + error_type & error() & + { + return assert( ! has_value() ), contained.error(); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + + constexpr error_type const && error() const && + { + return std::move( ( assert( ! has_value() ), contained.error() ) ); + } + + error_type && error() && + { + return std::move( ( assert( ! has_value() ), contained.error() ) ); + } + +#endif + + constexpr unexpected_type get_unexpected() const + { + return make_unexpected( contained.error() ); + } + + template< typename Ex > + bool has_exception() const + { + using ContainedEx = typename std::remove_reference< decltype( get_unexpected().error() ) >::type; + return ! has_value() && std::is_base_of< Ex, ContainedEx>::value; + } + + template< typename U + nsel_REQUIRES_T( + std::is_copy_constructible< T>::value + && std::is_convertible::value + ) + > + value_type value_or( U && v ) const & + { + return has_value() + ? contained.value() + : static_cast( std::forward( v ) ); + } + + template< typename U + nsel_REQUIRES_T( + std::is_move_constructible< T>::value + && std::is_convertible::value + ) + > + value_type value_or( U && v ) && + { + return has_value() + ? std::move( contained.value() ) + : static_cast( std::forward( v ) ); + } + +#if nsel_P2505R >= 4 + template< typename G = E + nsel_REQUIRES_T( + std::is_copy_constructible< E >::value + && std::is_convertible< G, E >::value + ) + > + nsel_constexpr error_type error_or( G && e ) const & + { + return has_value() + ? static_cast< E >( std::forward< G >( e ) ) + : contained.error(); + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_move_constructible< E >::value + && std::is_convertible< G, E >::value + ) + > + nsel_constexpr14 error_type error_or( G && e ) && + { + return has_value() + ? static_cast< E >( std::forward< G >( e ) ) + : std::move( contained.error() ); + } +#endif // nsel_P2505R >= 4 + +#if nsel_P2505R >= 3 + // Monadic operations (P2505) + template< typename F + nsel_REQUIRES_T( + detail::is_expected < detail::invoke_result_nocvref_t< F, value_type & > > ::value + && std::is_same< typename detail::invoke_result_nocvref_t< F, value_type & >::error_type, error_type >::value + && std::is_constructible< error_type, error_type & >::value + ) + > + nsel_constexpr14 detail::invoke_result_nocvref_t< F, value_type & > and_then( F && f ) & + { + return has_value() + ? detail::invoke_result_nocvref_t< F, value_type & >( detail::invoke( std::forward< F >( f ), value() ) ) + : detail::invoke_result_nocvref_t< F, value_type & >( unexpect, error() ); + } + + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F, const value_type & >::error_type, error_type >::value + && std::is_constructible< error_type, const error_type & >::value + ) + > + nsel_constexpr detail::invoke_result_nocvref_t< F, const value_type & > and_then( F && f ) const & + { + return has_value() + ? detail::invoke_result_nocvref_t< F, const value_type & >( detail::invoke( std::forward< F >( f ), value() ) ) + : detail::invoke_result_nocvref_t< F, const value_type & >( unexpect, error() ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F, value_type && >::error_type, error_type >::value + && std::is_constructible< error_type, error_type && >::value + ) + > + nsel_constexpr14 detail::invoke_result_nocvref_t< F, value_type && > and_then( F && f ) && + { + return has_value() + ? detail::invoke_result_nocvref_t< F, value_type && >( detail::invoke( std::forward< F >( f ), std::move( value() ) ) ) + : detail::invoke_result_nocvref_t< F, value_type && >( unexpect, std::move( error() ) ); + } + + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F, const value_type & >::error_type, error_type >::value + && std::is_constructible< error_type, const error_type && >::value + ) + > + nsel_constexpr detail::invoke_result_nocvref_t< F, const value_type && > and_then( F && f ) const && + { + return has_value() + ? detail::invoke_result_nocvref_t< F, const value_type && >( detail::invoke( std::forward< F >( f ), std::move( value() ) ) ) + : detail::invoke_result_nocvref_t< F, const value_type && >( unexpect, std::move( error() ) ); + } +#endif + + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F, error_type & >::value_type, value_type >::value + && std::is_constructible< value_type, value_type & >::value + ) + > + nsel_constexpr14 detail::invoke_result_nocvref_t< F, error_type & > or_else( F && f ) & + { + return has_value() + ? detail::invoke_result_nocvref_t< F, error_type & >( value() ) + : detail::invoke_result_nocvref_t< F, error_type & >( detail::invoke( std::forward< F >( f ), error() ) ); + } + + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F, const error_type & >::value_type, value_type >::value + && std::is_constructible< value_type, const value_type & >::value + ) + > + nsel_constexpr detail::invoke_result_nocvref_t< F, const error_type & > or_else( F && f ) const & + { + return has_value() + ? detail::invoke_result_nocvref_t< F, const error_type & >( value() ) + : detail::invoke_result_nocvref_t< F, const error_type & >( detail::invoke( std::forward< F >( f ), error() ) ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F, error_type && >::value_type, value_type >::value + && std::is_constructible< value_type, value_type && >::value + ) + > + nsel_constexpr14 detail::invoke_result_nocvref_t< F, error_type && > or_else( F && f ) && + { + return has_value() + ? detail::invoke_result_nocvref_t< F, error_type && >( std::move( value() ) ) + : detail::invoke_result_nocvref_t< F, error_type && >( detail::invoke( std::forward< F >( f ), std::move( error() ) ) ); + } + + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F, const error_type && >::value_type, value_type >::value + && std::is_constructible< value_type, const value_type && >::value + ) + > + nsel_constexpr detail::invoke_result_nocvref_t< F, const error_type && > or_else( F && f ) const && + { + return has_value() + ? detail::invoke_result_nocvref_t< F, const error_type && >( std::move( value() ) ) + : detail::invoke_result_nocvref_t< F, const error_type && >( detail::invoke( std::forward< F >( f ), std::move( error() ) ) ); + } +#endif + + template::value + && !std::is_void< detail::transform_invoke_result_t< F, value_type & > >::value + && detail::valid_expected_value_type< detail::transform_invoke_result_t< F, value_type & > >::value + ) + > + nsel_constexpr14 expected< detail::transform_invoke_result_t< F, value_type & >, error_type > transform( F && f ) & + { + return has_value() + ? expected< detail::transform_invoke_result_t< F, value_type & >, error_type >( detail::invoke( std::forward< F >( f ), **this ) ) + : make_unexpected( error() ); + } + + template::value + && std::is_void< detail::transform_invoke_result_t< F, value_type & > >::value + ) + > + nsel_constexpr14 expected< void, error_type > transform( F && f ) & + { + return has_value() + ? ( detail::invoke( std::forward< F >( f ), **this ), expected< void, error_type >() ) + : make_unexpected( error() ); + } + + template::value + && !std::is_void< detail::transform_invoke_result_t< F, const value_type & > >::value + && detail::valid_expected_value_type< detail::transform_invoke_result_t< F, const value_type & > >::value + ) + > + nsel_constexpr expected< detail::transform_invoke_result_t< F, const value_type & >, error_type > transform( F && f ) const & + { + return has_value() + ? expected< detail::transform_invoke_result_t< F, const value_type & >, error_type >( detail::invoke( std::forward< F >( f ), **this ) ) + : make_unexpected( error() ); + } + + template::value + && std::is_void< detail::transform_invoke_result_t< F, const value_type & > >::value + ) + > + nsel_constexpr expected< void, error_type > transform( F && f ) const & + { + return has_value() + ? ( detail::invoke( std::forward< F >( f ), **this ), expected< void, error_type >() ) + : make_unexpected( error() ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + template::value + && !std::is_void< detail::transform_invoke_result_t< F, value_type && > >::value + && detail::valid_expected_value_type< detail::transform_invoke_result_t< F, value_type && > >::value + ) + > + nsel_constexpr14 expected< detail::transform_invoke_result_t< F, value_type && >, error_type > transform( F && f ) && + { + return has_value() + ? expected< detail::transform_invoke_result_t< F, value_type && >, error_type >( detail::invoke( std::forward< F >( f ), std::move( **this ) ) ) + : make_unexpected( std::move( error() ) ); + } + + template::value + && std::is_void< detail::transform_invoke_result_t< F, value_type && > >::value + ) + > + nsel_constexpr14 expected< void, error_type > transform( F && f ) && + { + return has_value() + ? ( detail::invoke( std::forward< F >( f ), **this ), expected< void, error_type >() ) + : make_unexpected( std::move( error() ) ); + } + + template::value + && !std::is_void< detail::transform_invoke_result_t< F, const value_type && > >::value + && detail::valid_expected_value_type< detail::transform_invoke_result_t< F, const value_type && > >::value + ) + > + nsel_constexpr expected< detail::transform_invoke_result_t< F, const value_type && >, error_type > transform( F && f ) const && + { + return has_value() + ? expected< detail::transform_invoke_result_t< F, const value_type && >, error_type >( detail::invoke( std::forward< F >( f ), std::move( **this ) ) ) + : make_unexpected( std::move( error() ) ); + } + + template::value + && std::is_void< detail::transform_invoke_result_t< F, const value_type && > >::value + ) + > + nsel_constexpr expected< void, error_type > transform( F && f ) const && + { + return has_value() + ? ( detail::invoke( std::forward< F >( f ), **this ), expected< void, error_type >() ) + : make_unexpected( std::move( error() ) ); + } +#endif + + template >::value + && std::is_constructible< value_type, value_type & >::value + ) + > + nsel_constexpr14 expected< value_type, detail::transform_invoke_result_t< F, error_type & > > transform_error( F && f ) & + { + return has_value() + ? expected< value_type, detail::transform_invoke_result_t< F, error_type & > >( in_place, **this ) + : make_unexpected( detail::invoke( std::forward< F >( f ), error() ) ); + } + + template >::value + && std::is_constructible< value_type, const value_type & >::value + ) + > + nsel_constexpr expected< value_type, detail::transform_invoke_result_t< F, const error_type & > > transform_error( F && f ) const & + { + return has_value() + ? expected< value_type, detail::transform_invoke_result_t< F, const error_type & > >( in_place, **this ) + : make_unexpected( detail::invoke( std::forward< F >( f ), error() ) ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + template >::value + && std::is_constructible< value_type, value_type && >::value + ) + > + nsel_constexpr14 expected< value_type, detail::transform_invoke_result_t< F, error_type && > > transform_error( F && f ) && + { + return has_value() + ? expected< value_type, detail::transform_invoke_result_t< F, error_type && > >( in_place, std::move( **this ) ) + : make_unexpected( detail::invoke( std::forward< F >( f ), std::move( error() ) ) ); + } + + template >::value + && std::is_constructible< value_type, const value_type && >::value + ) + > + nsel_constexpr expected< value_type, detail::transform_invoke_result_t< F, const error_type && > > transform_error( F && f ) const && + { + return has_value() + ? expected< value_type, detail::transform_invoke_result_t< F, const error_type && > >( in_place, std::move( **this ) ) + : make_unexpected( detail::invoke( std::forward< F >( f ), std::move( error() ) ) ); + } +#endif +#endif // nsel_P2505R >= 3 + // unwrap() + +// template +// constexpr expected expected,E>::unwrap() const&; + +// template +// constexpr expected expected::unwrap() const&; + +// template +// expected expected, E>::unwrap() &&; + +// template +// template expected expected::unwrap() &&; + + // factories + +// template< typename Ex, typename F> +// expected catch_exception(F&& f); + +// template< typename F> +// expected())),E> map(F&& func) ; + +// template< typename F> +// 'see below' bind(F&& func); + +// template< typename F> +// expected catch_error(F&& f); + +// template< typename F> +// 'see below' then(F&& func); + +private: + detail::storage_t + < + T + ,E + , std::is_copy_constructible::value && std::is_copy_constructible::value + , std::is_move_constructible::value && std::is_move_constructible::value + > + contained; +}; + +/// class expected, void specialization + +template< typename E > +class nsel_NODISCARD expected< void, E > +{ +private: + template< typename, typename > friend class expected; + +public: + using value_type = void; + using error_type = E; + using unexpected_type = nonstd::unexpected_type; + + // x.x.4.1 constructors + + constexpr expected() noexcept + : contained( true ) + {} + + nsel_constexpr14 expected( expected const & other ) = default; + nsel_constexpr14 expected( expected && other ) = default; + + constexpr explicit expected( nonstd_lite_in_place_t(void) ) + : contained( true ) + {} + + template< typename G = E + nsel_REQUIRES_T( + !std::is_convertible::value /*=> explicit */ + ) + > + nsel_constexpr14 explicit expected( nonstd::unexpected_type const & error ) + : contained( false ) + { + contained.construct_error( E{ error.error() } ); + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_convertible::value /*=> non-explicit */ + ) + > + nsel_constexpr14 /*non-explicit*/ expected( nonstd::unexpected_type const & error ) + : contained( false ) + { + contained.construct_error( error.error() ); + } + + template< typename G = E + nsel_REQUIRES_T( + !std::is_convertible::value /*=> explicit */ + ) + > + nsel_constexpr14 explicit expected( nonstd::unexpected_type && error ) + : contained( false ) + { + contained.construct_error( E{ std::move( error.error() ) } ); + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_convertible::value /*=> non-explicit */ + ) + > + nsel_constexpr14 /*non-explicit*/ expected( nonstd::unexpected_type && error ) + : contained( false ) + { + contained.construct_error( std::move( error.error() ) ); + } + + template< typename... Args + nsel_REQUIRES_T( + std::is_constructible::value + ) + > + nsel_constexpr14 explicit expected( unexpect_t, Args&&... args ) + : contained( false ) + { + contained.emplace_error( std::forward( args )... ); + } + + template< typename U, typename... Args + nsel_REQUIRES_T( + std::is_constructible, Args&&...>::value + ) + > + nsel_constexpr14 explicit expected( unexpect_t, std::initializer_list il, Args&&... args ) + : contained( false ) + { + contained.emplace_error( il, std::forward( args )... ); + } + + // destructor + + ~expected() + { + if ( ! has_value() ) + { + contained.destruct_error(); + } + } + + // x.x.4.3 assignment + + expected & operator=( expected const & other ) + { + expected( other ).swap( *this ); + return *this; + } + + expected & operator=( expected && other ) noexcept + ( + std::is_nothrow_move_assignable::value && + std::is_nothrow_move_constructible::value ) + { + expected( std::move( other ) ).swap( *this ); + return *this; + } + + void emplace() + { + expected().swap( *this ); + } + + // x.x.4.4 swap + + template< typename G = E > + nsel_REQUIRES_R( void, + std17::is_swappable::value + && std::is_move_constructible::value + ) + swap( expected & other ) noexcept + ( + std::is_nothrow_move_constructible::value && std17::is_nothrow_swappable::value + ) + { + using std::swap; + + if ( ! bool(*this) && ! bool(other) ) { swap( contained.error(), other.contained.error() ); } + else if ( bool(*this) && ! bool(other) ) { contained.construct_error( std::move( other.error() ) ); + bool has_value = contained.has_value(); + bool other_has_value = other.has_value(); + other.contained.set_has_value(has_value); + contained.set_has_value(other_has_value); + } + else if ( ! bool(*this) && bool(other) ) { other.swap( *this ); } + } + + // x.x.4.5 observers + + constexpr explicit operator bool() const noexcept + { + return has_value(); + } + + constexpr bool has_value() const noexcept + { + return contained.has_value(); + } + + void value() const + { + if ( ! has_value() ) + { + error_traits::rethrow( contained.error() ); + } + } + + constexpr error_type const & error() const & + { + return assert( ! has_value() ), contained.error(); + } + + error_type & error() & + { + return assert( ! has_value() ), contained.error(); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + + constexpr error_type const && error() const && + { + return std::move( ( assert( ! has_value() ), contained.error() ) ); + } + + error_type && error() && + { + return std::move( ( assert( ! has_value() ), contained.error() ) ); + } + +#endif + + constexpr unexpected_type get_unexpected() const + { + return make_unexpected( contained.error() ); + } + + template< typename Ex > + bool has_exception() const + { + using ContainedEx = typename std::remove_reference< decltype( get_unexpected().error() ) >::type; + return ! has_value() && std::is_base_of< Ex, ContainedEx>::value; + } + +#if nsel_P2505R >= 4 + template< typename G = E + nsel_REQUIRES_T( + std::is_copy_constructible< E >::value + && std::is_convertible< G, E >::value + ) + > + nsel_constexpr error_type error_or( G && e ) const & + { + return has_value() + ? static_cast< E >( std::forward< G >( e ) ) + : contained.error(); + } + + template< typename G = E + nsel_REQUIRES_T( + std::is_move_constructible< E >::value + && std::is_convertible< G, E >::value + ) + > + nsel_constexpr14 error_type error_or( G && e ) && + { + return has_value() + ? static_cast< E >( std::forward< G >( e ) ) + : std::move( contained.error() ); + } +#endif // nsel_P2505R >= 4 + +#if nsel_P2505R >= 3 + // Monadic operations (P2505) + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F >::error_type, error_type >::value + && std::is_constructible< error_type, error_type & >::value + ) + > + nsel_constexpr14 detail::invoke_result_nocvref_t< F > and_then( F && f ) & + { + return has_value() + ? detail::invoke_result_nocvref_t< F >( detail::invoke( std::forward< F >( f ) ) ) + : detail::invoke_result_nocvref_t< F >( unexpect, error() ); + } + + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F >::error_type, error_type >::value + && std::is_constructible< error_type, const error_type & >::value + ) + > + nsel_constexpr detail::invoke_result_nocvref_t< F > and_then( F && f ) const & + { + return has_value() + ? detail::invoke_result_nocvref_t< F >( detail::invoke( std::forward< F >( f ) ) ) + : detail::invoke_result_nocvref_t< F >( unexpect, error() ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F >::error_type, error_type >::value + && std::is_constructible< error_type, error_type && >::value + ) + > + nsel_constexpr14 detail::invoke_result_nocvref_t< F > and_then( F && f ) && + { + return has_value() + ? detail::invoke_result_nocvref_t< F >( detail::invoke( std::forward< F >( f ) ) ) + : detail::invoke_result_nocvref_t< F >( unexpect, std::move( error() ) ); + } + + template >::value + && std::is_same< typename detail::invoke_result_nocvref_t< F >::error_type, error_type >::value + && std::is_constructible< error_type, const error_type && >::value + ) + > + nsel_constexpr detail::invoke_result_nocvref_t< F > and_then( F && f ) const && + { + return has_value() + ? detail::invoke_result_nocvref_t< F >( detail::invoke( std::forward< F >( f ) ) ) + : detail::invoke_result_nocvref_t< F >( unexpect, std::move( error() ) ); + } +#endif + + template >::value + && std::is_void< typename detail::invoke_result_nocvref_t< F, error_type & >::value_type >::value + ) + > + nsel_constexpr14 detail::invoke_result_nocvref_t< F, error_type & > or_else( F && f ) & + { + return has_value() + ? detail::invoke_result_nocvref_t< F, error_type & >() + : detail::invoke_result_nocvref_t< F, error_type & >( detail::invoke( std::forward< F >( f ), error() ) ); + } + + template >::value + && std::is_void< typename detail::invoke_result_nocvref_t< F, const error_type & >::value_type >::value + ) + > + nsel_constexpr detail::invoke_result_nocvref_t< F, const error_type & > or_else( F && f ) const & + { + return has_value() + ? detail::invoke_result_nocvref_t< F, const error_type & >() + : detail::invoke_result_nocvref_t< F, const error_type & >( detail::invoke( std::forward< F >( f ), error() ) ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + template >::value + && std::is_void< typename detail::invoke_result_nocvref_t< F, error_type && >::value_type >::value + ) + > + nsel_constexpr14 detail::invoke_result_nocvref_t< F, error_type && > or_else( F && f ) && + { + return has_value() + ? detail::invoke_result_nocvref_t< F, error_type && >() + : detail::invoke_result_nocvref_t< F, error_type && >( detail::invoke( std::forward< F >( f ), std::move( error() ) ) ); + } + + template >::value + && std::is_void< typename detail::invoke_result_nocvref_t< F, const error_type && >::value_type >::value + ) + > + nsel_constexpr detail::invoke_result_nocvref_t< F, const error_type && > or_else( F && f ) const && + { + return has_value() + ? detail::invoke_result_nocvref_t< F, const error_type && >() + : detail::invoke_result_nocvref_t< F, const error_type && >( detail::invoke( std::forward< F >( f ), std::move( error() ) ) ); + } +#endif + + template::value + && !std::is_void< detail::transform_invoke_result_t< F > >::value + ) + > + nsel_constexpr14 expected< detail::transform_invoke_result_t< F >, error_type > transform( F && f ) & + { + return has_value() + ? expected< detail::transform_invoke_result_t< F >, error_type >( detail::invoke( std::forward< F >( f ) ) ) + : make_unexpected( error() ); + } + + template::value + && std::is_void< detail::transform_invoke_result_t< F > >::value + ) + > + nsel_constexpr14 expected< void, error_type > transform( F && f ) & + { + return has_value() + ? ( detail::invoke( std::forward< F >( f ) ), expected< void, error_type >() ) + : make_unexpected( error() ); + } + + template::value + && !std::is_void< detail::transform_invoke_result_t< F > >::value + ) + > + nsel_constexpr expected< detail::transform_invoke_result_t< F >, error_type > transform( F && f ) const & + { + return has_value() + ? expected< detail::transform_invoke_result_t< F >, error_type >( detail::invoke( std::forward< F >( f ) ) ) + : make_unexpected( error() ); + } + + template::value + && std::is_void< detail::transform_invoke_result_t< F > >::value + ) + > + nsel_constexpr expected< void, error_type > transform( F && f ) const & + { + return has_value() + ? ( detail::invoke( std::forward< F >( f ) ), expected< void, error_type >() ) + : make_unexpected( error() ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + template::value + && !std::is_void< detail::transform_invoke_result_t< F > >::value + ) + > + nsel_constexpr14 expected< detail::transform_invoke_result_t< F >, error_type > transform( F && f ) && + { + return has_value() + ? expected< detail::transform_invoke_result_t< F >, error_type >( detail::invoke( std::forward< F >( f ) ) ) + : make_unexpected( error() ); + } + + template::value + && std::is_void< detail::transform_invoke_result_t< F > >::value + ) + > + nsel_constexpr14 expected< void, error_type > transform( F && f ) && + { + return has_value() + ? ( detail::invoke( std::forward< F >( f ) ), expected< void, error_type >() ) + : make_unexpected( error() ); + } + + template::value + && !std::is_void< detail::transform_invoke_result_t< F > >::value + ) + > + nsel_constexpr expected< detail::transform_invoke_result_t< F >, error_type > transform( F && f ) const && + { + return has_value() + ? expected< detail::transform_invoke_result_t< F >, error_type >( detail::invoke( std::forward< F >( f ) ) ) + : make_unexpected( error() ); + } + + template::value + && std::is_void< detail::transform_invoke_result_t< F > >::value + ) + > + nsel_constexpr expected< void, error_type > transform( F && f ) const && + { + return has_value() + ? ( detail::invoke( std::forward< F >( f ) ), expected< void, error_type >() ) + : make_unexpected( error() ); + } +#endif + + template >::value + ) + > + nsel_constexpr14 expected< void, detail::transform_invoke_result_t< F, error_type & > > transform_error( F && f ) & + { + return has_value() + ? expected< void, detail::transform_invoke_result_t< F, error_type & > >() + : make_unexpected( detail::invoke( std::forward< F >( f ), error() ) ); + } + + template >::value + ) + > + nsel_constexpr expected< void, detail::transform_invoke_result_t< F, const error_type & > > transform_error( F && f ) const & + { + return has_value() + ? expected< void, detail::transform_invoke_result_t< F, const error_type & > >() + : make_unexpected( detail::invoke( std::forward< F >( f ), error() ) ); + } + +#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490 + template >::value + ) + > + nsel_constexpr14 expected< void, detail::transform_invoke_result_t< F, error_type && > > transform_error( F && f ) && + { + return has_value() + ? expected< void, detail::transform_invoke_result_t< F, error_type && > >() + : make_unexpected( detail::invoke( std::forward< F >( f ), std::move( error() ) ) ); + } + + template >::value + ) + > + nsel_constexpr expected< void, detail::transform_invoke_result_t< F, const error_type && > > transform_error( F && f ) const && + { + return has_value() + ? expected< void, detail::transform_invoke_result_t< F, const error_type && > >() + : make_unexpected( detail::invoke( std::forward< F >( f ), std::move( error() ) ) ); + } +#endif +#endif // nsel_P2505R >= 3 + +// template constexpr 'see below' unwrap() const&; +// +// template 'see below' unwrap() &&; + + // factories + +// template< typename Ex, typename F> +// expected catch_exception(F&& f); +// +// template< typename F> +// expected map(F&& func) ; +// +// template< typename F> +// 'see below' bind(F&& func) ; +// +// template< typename F> +// expected catch_error(F&& f); +// +// template< typename F> +// 'see below' then(F&& func); + +private: + detail::storage_t + < + void + , E + , std::is_copy_constructible::value + , std::is_move_constructible::value + > + contained; +}; + +// x.x.4.6 expected<>: comparison operators + +template< typename T1, typename E1, typename T2, typename E2 + nsel_REQUIRES_T( + !std::is_void::value && !std::is_void::value + ) +> +constexpr bool operator==( expected const & x, expected const & y ) +{ + return bool(x) != bool(y) ? false : bool(x) ? *x == *y : x.error() == y.error(); +} + +template< typename T1, typename E1, typename T2, typename E2 + nsel_REQUIRES_T( + std::is_void::value && std::is_void::value + ) +> +constexpr bool operator==( expected const & x, expected const & y ) +{ + return bool(x) != bool(y) ? false : bool(x) || static_cast( x.error() == y.error() ); +} + +template< typename T1, typename E1, typename T2, typename E2 > +constexpr bool operator!=( expected const & x, expected const & y ) +{ + return !(x == y); +} + +#if nsel_P0323R <= 2 + +template< typename T, typename E > +constexpr bool operator<( expected const & x, expected const & y ) +{ + return (!y) ? false : (!x) ? true : *x < *y; +} + +template< typename T, typename E > +constexpr bool operator>( expected const & x, expected const & y ) +{ + return (y < x); +} + +template< typename T, typename E > +constexpr bool operator<=( expected const & x, expected const & y ) +{ + return !(y < x); +} + +template< typename T, typename E > +constexpr bool operator>=( expected const & x, expected const & y ) +{ + return !(x < y); +} + +#endif + +// x.x.4.7 expected: comparison with T + +template< typename T1, typename E1, typename T2 + nsel_REQUIRES_T( + !std::is_void::value + ) +> +constexpr bool operator==( expected const & x, T2 const & v ) +{ + return bool(x) ? *x == v : false; +} + +template< typename T1, typename E1, typename T2 + nsel_REQUIRES_T( + !std::is_void::value + ) +> +constexpr bool operator==(T2 const & v, expected const & x ) +{ + return bool(x) ? v == *x : false; +} + +template< typename T1, typename E1, typename T2 > +constexpr bool operator!=( expected const & x, T2 const & v ) +{ + return bool(x) ? *x != v : true; +} + +template< typename T1, typename E1, typename T2 > +constexpr bool operator!=( T2 const & v, expected const & x ) +{ + return bool(x) ? v != *x : true; +} + +#if nsel_P0323R <= 2 + +template< typename T, typename E > +constexpr bool operator<( expected const & x, T const & v ) +{ + return bool(x) ? *x < v : true; +} + +template< typename T, typename E > +constexpr bool operator<( T const & v, expected const & x ) +{ + return bool(x) ? v < *x : false; +} + +template< typename T, typename E > +constexpr bool operator>( T const & v, expected const & x ) +{ + return bool(x) ? *x < v : false; +} + +template< typename T, typename E > +constexpr bool operator>( expected const & x, T const & v ) +{ + return bool(x) ? v < *x : false; +} + +template< typename T, typename E > +constexpr bool operator<=( T const & v, expected const & x ) +{ + return bool(x) ? ! ( *x < v ) : false; +} + +template< typename T, typename E > +constexpr bool operator<=( expected const & x, T const & v ) +{ + return bool(x) ? ! ( v < *x ) : true; +} + +template< typename T, typename E > +constexpr bool operator>=( expected const & x, T const & v ) +{ + return bool(x) ? ! ( *x < v ) : false; +} + +template< typename T, typename E > +constexpr bool operator>=( T const & v, expected const & x ) +{ + return bool(x) ? ! ( v < *x ) : true; +} + +#endif // nsel_P0323R + +// x.x.4.8 expected: comparison with unexpected_type + +template< typename T1, typename E1 , typename E2 > +constexpr bool operator==( expected const & x, unexpected_type const & u ) +{ + return (!x) ? x.get_unexpected() == u : false; +} + +template< typename T1, typename E1 , typename E2 > +constexpr bool operator==( unexpected_type const & u, expected const & x ) +{ + return ( x == u ); +} + +template< typename T1, typename E1 , typename E2 > +constexpr bool operator!=( expected const & x, unexpected_type const & u ) +{ + return ! ( x == u ); +} + +template< typename T1, typename E1 , typename E2 > +constexpr bool operator!=( unexpected_type const & u, expected const & x ) +{ + return ! ( x == u ); +} + +#if nsel_P0323R <= 2 + +template< typename T, typename E > +constexpr bool operator<( expected const & x, unexpected_type const & u ) +{ + return (!x) ? ( x.get_unexpected() < u ) : false; +} + +template< typename T, typename E > +constexpr bool operator<( unexpected_type const & u, expected const & x ) +{ + return (!x) ? ( u < x.get_unexpected() ) : true ; +} + +template< typename T, typename E > +constexpr bool operator>( expected const & x, unexpected_type const & u ) +{ + return ( u < x ); +} + +template< typename T, typename E > +constexpr bool operator>( unexpected_type const & u, expected const & x ) +{ + return ( x < u ); +} + +template< typename T, typename E > +constexpr bool operator<=( expected const & x, unexpected_type const & u ) +{ + return ! ( u < x ); +} + +template< typename T, typename E > +constexpr bool operator<=( unexpected_type const & u, expected const & x) +{ + return ! ( x < u ); +} + +template< typename T, typename E > +constexpr bool operator>=( expected const & x, unexpected_type const & u ) +{ + return ! ( u > x ); +} + +template< typename T, typename E > +constexpr bool operator>=( unexpected_type const & u, expected const & x ) +{ + return ! ( x > u ); +} + +#endif // nsel_P0323R + +/// x.x.x Specialized algorithms + +template< typename T, typename E + nsel_REQUIRES_T( + ( std::is_void::value || std::is_move_constructible::value ) + && std::is_move_constructible::value + && std17::is_swappable::value + && std17::is_swappable::value ) +> +void swap( expected & x, expected & y ) noexcept ( noexcept ( x.swap(y) ) ) +{ + x.swap( y ); +} + +#if nsel_P0323R <= 3 + +template< typename T > +constexpr auto make_expected( T && v ) -> expected< typename std::decay::type > +{ + return expected< typename std::decay::type >( std::forward( v ) ); +} + +// expected specialization: + +auto inline make_expected() -> expected +{ + return expected( in_place ); +} + +template< typename T > +constexpr auto make_expected_from_current_exception() -> expected +{ + return expected( make_unexpected_from_current_exception() ); +} + +template< typename T > +auto make_expected_from_exception( std::exception_ptr v ) -> expected +{ + return expected( unexpected_type( std::forward( v ) ) ); +} + +template< typename T, typename E > +constexpr auto make_expected_from_error( E e ) -> expected::type> +{ + return expected::type>( make_unexpected( e ) ); +} + +template< typename F + nsel_REQUIRES_T( ! std::is_same::type, void>::value ) +> +/*nsel_constexpr14*/ +auto make_expected_from_call( F f ) -> expected< typename std::result_of::type > +{ + try + { + return make_expected( f() ); + } + catch (...) + { + return make_unexpected_from_current_exception(); + } +} + +template< typename F + nsel_REQUIRES_T( std::is_same::type, void>::value ) +> +/*nsel_constexpr14*/ +auto make_expected_from_call( F f ) -> expected +{ + try + { + f(); + return make_expected(); + } + catch (...) + { + return make_unexpected_from_current_exception(); + } +} + +#endif // nsel_P0323R + +} // namespace expected_lite + +using namespace expected_lite; + +// using expected_lite::expected; +// using ... + +} // namespace nonstd + +namespace std { + +// expected: hash support + +template< typename T, typename E > +struct hash< nonstd::expected > +{ + using result_type = std::size_t; + using argument_type = nonstd::expected; + + constexpr result_type operator()(argument_type const & arg) const + { + return arg ? std::hash{}(*arg) : result_type{}; + } +}; + +// TBD - ?? remove? see spec. +template< typename T, typename E > +struct hash< nonstd::expected > +{ + using result_type = std::size_t; + using argument_type = nonstd::expected; + + constexpr result_type operator()(argument_type const & arg) const + { + return arg ? std::hash{}(*arg) : result_type{}; + } +}; + +// TBD - implement +// bool(e), hash>()(e) shall evaluate to the hashing true; +// otherwise it evaluates to an unspecified value if E is exception_ptr or +// a combination of hashing false and hash()(e.error()). + +template< typename E > +struct hash< nonstd::expected > +{ +}; + +} // namespace std + +namespace nonstd { + +// void unexpected() is deprecated && removed in C++17 + +#if nsel_CPP17_OR_GREATER || nsel_COMPILER_MSVC_VERSION > 141 +template< typename E > +using unexpected = unexpected_type; +#endif + +} // namespace nonstd + +#undef nsel_REQUIRES +#undef nsel_REQUIRES_0 +#undef nsel_REQUIRES_T + +nsel_RESTORE_WARNINGS() + +#endif // nsel_USES_STD_EXPECTED + +#endif // NONSTD_EXPECTED_LITE_HPP diff --git a/include/utilities/bmi/nonstd/expected.tweak.hpp b/include/utilities/bmi/nonstd/expected.tweak.hpp new file mode 100644 index 0000000000..89a8e1a4ad --- /dev/null +++ b/include/utilities/bmi/nonstd/expected.tweak.hpp @@ -0,0 +1,4 @@ +//tweaks for the expected library configuration/build +// see https://github.com/martinmoene/expected-lite/tree/master?tab=readme-ov-file#configuration +// for documentation on configuration +#define nsel_CONFIG_WIN32_LEAN_AND_MEAN 0 diff --git a/include/utilities/bmi/protocol.hpp b/include/utilities/bmi/protocol.hpp new file mode 100644 index 0000000000..d08dd7681c --- /dev/null +++ b/include/utilities/bmi/protocol.hpp @@ -0,0 +1,199 @@ +/* +Author: Nels Frazier +Copyright (C) 2025 Lynker +------------------------------------------------------------------------ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +------------------------------------------------------------------------ +Version 0.3 +Remove "supported" member variable and added is_supported() method to protocol interface + +Version 0.2 +Enumerate protocol error types and add ProtocolError exception class +Implement error handling via expected and error_or_warning +Removed model member and required model reference in run(), check_support(), and initialize() +Minor refactoring and style changes + +Version 0.1 +Virtual interface for BMI protocols +*/ + +#pragma once + +#include "Bmi_Adapter.hpp" +#include "JSONProperty.hpp" +#include + +namespace models{ namespace bmi{ namespace protocols{ +using nonstd::expected; +using nonstd::make_unexpected; + +enum class Error{ + UNITIALIZED_MODEL, + UNSUPPORTED_PROTOCOL, + INTEGRATION_ERROR, + PROTOCOL_ERROR, + PROTOCOL_WARNING +}; + +class ProtocolError: public std::exception { + public: + ProtocolError () = delete; + ProtocolError(Error err, const std::string& message="") : err(std::move(err)), message(std::move(message)) {} + ProtocolError(const ProtocolError& other) = default; + ProtocolError(ProtocolError&& other) noexcept = default; + ProtocolError& operator=(const ProtocolError& other) = default; + ProtocolError& operator=(ProtocolError&& other) noexcept = default; + ~ProtocolError() = default; + + auto to_string() const -> std::string { + switch (err) { + case Error::UNITIALIZED_MODEL: return "Error(Uninitialized Model)::" + message; + case Error::UNSUPPORTED_PROTOCOL: return "Warning(Unsupported Protocol)::" + message; + case Error::INTEGRATION_ERROR: return "Error(Integration)::" + message; + case Error::PROTOCOL_ERROR: return "Error(Protocol)::" + message; + case Error::PROTOCOL_WARNING: return "Warning(Protocol)::" + message; + default: return "Unknown Error: " + message; + } + } + + auto error_code() const -> const Error& { return err; } + auto get_message() const -> const std::string& { return message; } + + char const *what() const noexcept override { + message = to_string(); + return message.c_str(); + } + + private: + Error err; + mutable std::string message; +}; + +struct Context{ + const int current_time_step; + const int total_steps; + const std::string& timestamp; + const std::string& id; +}; + +using ModelPtr = std::shared_ptr; +using Properties = geojson::PropertyMap; + +class NgenBmiProtocol{ + /** + * @brief Abstract interface for a generic BMI protocol + * + */ + + public: + + virtual ~NgenBmiProtocol() = default; + + protected: + /** + * @brief Handle a ProtocolError by either throwing it or logging it as a warning + * + * @param err The ProtocolError to handle + * @return expected Returns an empty expected if the error was logged as a warning, + * otherwise throws the ProtocolError. + * + * @throws ProtocolError if the error is of type PROTOCOL_ERROR + */ + static auto error_or_warning(const ProtocolError& err) -> expected { + // Log warnings, but throw errors + switch(err.error_code()){ + case Error::PROTOCOL_ERROR: + throw err; + break; + case Error::INTEGRATION_ERROR: + case Error::UNITIALIZED_MODEL: + case Error::UNSUPPORTED_PROTOCOL: + case Error::PROTOCOL_WARNING: + std::cerr << err.to_string() << std::endl; + return make_unexpected( ProtocolError(std::move(err) ) ); + default: + throw err; + } + assert (false && "Unreachable code reached in error_or_warning"); + } + + /** + * @brief Run the BMI protocol against the given model + * + * Execute the logic of the protocol with the provided context and model. + * It is the caller's responsibility to ensure that the model provided is + * consistent with the model provided to the object's initialize() and + * check_support() methods, hence the protected nature of this function. + * + * @param ctx Contextual information for the protocol run + * @param model A shared pointer to a Bmi_Adapter object which should be + * initialized before being passed to this method. + * + * @return expected May contain a ProtocolError if + * the protocol fails for any reason. Errors of ProtocolError::PROTOCOL_WARNING + * severity should be logged as warnings, but not cause the simulation to fail. + */ + nsel_NODISCARD virtual auto run(const ModelPtr& model, const Context& ctx) const -> expected = 0; + + /** + * @brief Check if the BMI protocol is supported by the model + * + * It is the caller's responsibility to ensure that the model provided is + * consistent with the model provided to the object's initialize() and + * run() methods, hence the protected nature of this function. + * + * @param model A shared pointer to a Bmi_Adapter object which should be + * initialized before being passed to this method. + * + * @return expected May contain a ProtocolError if + * the protocol is not supported by the model. + */ + nsel_NODISCARD virtual expected check_support(const ModelPtr& model) = 0; + + /** + * @brief Initialize the BMI protocol from a set of key/value properties + * + * It is the caller's responsibility to ensure that the model provided is + * consistent with the model provided to the object's run() and + * check_support() methods, hence the protected nature of this function. + * + * @param properties key/value pairs for initializing the protocol + * @param model A shared pointer to a Bmi_Adapter object which should be + * initialized before being passed to this method. + * + * @return expected May contain a ProtocolError if + * initialization fails for any reason, since the protocol must + * be effectively "optional", failed initialization results in + * the protocol being disabled for the duration of the simulation. + */ + virtual auto initialize(const ModelPtr& model, const Properties& properties) -> expected = 0; + + /** + * @brief Whether the protocol is supported by the model + * + */ + virtual bool is_supported() const = 0; + + /** + * @brief Friend class for managing one or more protocols + * + * This allows the NgenBmiProtocols container class to access the protected `run()` + * method. This allows the container to ensure consistent application of the + * protocol with a particular bmi model instance throughout the lifecycle of a given + * protocol. + * + */ + friend class NgenBmiProtocols; +}; + +}}} diff --git a/include/utilities/bmi/protocols.hpp b/include/utilities/bmi/protocols.hpp new file mode 100644 index 0000000000..723fcf1364 --- /dev/null +++ b/include/utilities/bmi/protocols.hpp @@ -0,0 +1,100 @@ +/* +Author: Nels Frazier +Copyright (C) 2025 Lynker +------------------------------------------------------------------------ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +------------------------------------------------------------------------ +Version 0.2.1 +Fix NgenBmiProtocols constructor to initialize protocols map when model is null +Fix run() to return the expected returned by the wrapped call + +Version 0.2 +Enumerate protocol types/names +The container now holds a single model pointer and passes it to each protocol +per the updated (v0.2) protocol interface +Keep protocols in a map for dynamic access by enumeration name +add operator<< for Protocol enum + +Version 0.1 +Container and management for abstract BMI protocols +*/ +#pragma once + +#include +#include +#include +#include +#include "Bmi_Adapter.hpp" +#include "JSONProperty.hpp" + +#include "mass_balance.hpp" + + +namespace models{ namespace bmi{ namespace protocols{ + +enum class Protocol { + MASS_BALANCE +}; + +auto operator<<(std::ostream& os, Protocol p) -> std::ostream&; + +class NgenBmiProtocols { + /** + * @brief Container and management interface for BMI protocols for use in ngen + * + */ + + public: + /** + * @brief Construct a new Ngen Bmi Protocols object with a null model + * + */ + NgenBmiProtocols(); + + /** + * @brief Construct a new Ngen Bmi Protocols object for use with a known model + * + * @param model An initialized BMI model + * @param properties Properties for each protocol being initialized + */ + NgenBmiProtocols(std::shared_ptr model, const geojson::PropertyMap& properties); + + /** + * @brief Run a specific BMI protocol by name with a given context + * + * @param protocol_name The name of the protocol to run + * @param ctx The context of the current protocol run + * + * @return expected May contain a ProtocolError if + * the protocol fails for any reason. + */ + auto run(const Protocol& protocol_name, const Context& ctx) const -> expected; + + private: + + /** + * @brief All protocols managed by this container will utilize the same model + * + * This reduces the amount of pointer copying and references across a large simulation + * and it ensures that all protocols see the same model state. + * + */ + std::shared_ptr model; + /** + * @brief Map of protocol name to protocol instance + * + */ + std::unordered_map> protocols; + }; + +}}} diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 4f927d6523..432b918aa3 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -29,6 +29,8 @@ void ngen::Layer::update_models(boost::span catchment_outflows, double response(0.0); try { response = r_c->get_response(output_time_index, simulation_time.get_output_interval_seconds()); + // Check mass balance if able + r_c->check_mass_balance(output_time_index, simulation_time.get_total_output_times(), current_timestamp); } catch(models::external::State_Exception& e) { std::string msg = e.what(); @@ -37,6 +39,13 @@ void ngen::Layer::update_models(boost::span catchment_outflows, +" at feature id "+id; throw models::external::State_Exception(msg); } + catch(std::exception& e){ + std::string msg = e.what(); + msg = msg+" at timestep "+std::to_string(output_time_index) + +" ("+current_timestamp+")" + +" at feature id "+id; + throw std::runtime_error(msg); + } #if NGEN_WITH_ROUTING int results_index = catchment_indexes[id]; catchment_outflows[results_index] += response; diff --git a/src/core/nexus/HY_PointHydroNexusRemote.cpp b/src/core/nexus/HY_PointHydroNexusRemote.cpp index c0e3fa3c16..3d5c97a703 100644 --- a/src/core/nexus/HY_PointHydroNexusRemote.cpp +++ b/src/core/nexus/HY_PointHydroNexusRemote.cpp @@ -18,7 +18,7 @@ void MPI_Handle_Error(int status) } else { - MPI_Abort(MPI_COMM_WORLD,1); + MPI_Abort(MPI_COMM_WORLD, status); } } @@ -92,9 +92,9 @@ HY_PointHydroNexusRemote::HY_PointHydroNexusRemote(std::string nexus_id, Catchme HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() { - long wait_time = 0; - - // This destructore might be called after MPI_Finalize so do not attempt communication if + const unsigned int timeout = 120000; // timeout threshold in milliseconds + unsigned int wait_time = 0; + // This destructor might be called after MPI_Finalize so do not attempt communication if // this has occured int mpi_finalized; MPI_Finalized(&mpi_finalized); @@ -105,14 +105,46 @@ HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() process_communications(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - + if( wait_time < timeout && wait_time > 0 ){ // don't sleep if first call clears comms! + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + else + { + std::cerr << "HY_PointHydroNexusRemote: "<< id + << " destructor timed out after " << timeout/1000 + << " seconds waiting on pending MPI communications\n"; + // The return is is probably best, logging the error. + // There is no good way to recover from this. + // Throwing an exception from destructors is generally not a good idea + // as it can lead to undefined behavior. + // and using std::exit forces the program to terminate immediately, + // even if this situation is recoverable/acceptable in some cases. + return; + } wait_time += 1; + MPI_Finalized(&mpi_finalized); + } +} - if ( wait_time > 120000 ) +void HY_PointHydroNexusRemote::post_receives() +{ + // Post receives if not already posted (for pure receiver nexuses) + if (stored_receives.empty()) + { + for (int rank : upstream_ranks) { - // TODO log warning message that some comunications could not complete - + stored_receives.push_back({}); + stored_receives.back().buffer = std::make_shared(); + int tag = extract(id); + + MPI_Handle_Error(MPI_Irecv( + stored_receives.back().buffer.get(), + 1, + time_step_and_flow_type, + rank, + tag, + MPI_COMM_WORLD, + &stored_receives.back().mpi_request)); } } } @@ -130,31 +162,15 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t } else if ( type == receiver || type == sender_receiver ) { - for ( int rank : upstream_ranks ) - { - int status; - - stored_receives.resize(stored_receives.size() + 1); - stored_receives.back().buffer = std::make_shared(); - - int tag = extract(id); - - //Receive downstream_flow from Upstream Remote Nexus to this Downstream Remote Nexus - status = MPI_Irecv( - stored_receives.back().buffer.get(), - 1, - time_step_and_flow_type, - rank, - tag, - MPI_COMM_WORLD, - &stored_receives.back().mpi_request); - - MPI_Handle_Error(status); - - //std::cerr << "Creating receive with target_rank=" << rank << " on tag=" << tag << "\n"; - } - - //std::cerr << "Waiting on receives\n"; + post_receives(); + // Wait for receives to complete + // This ensures all upstream flows are received before returning + // and that we have matched all sends with receives for a given time step. + // As long as the functions are called appropriately, e.g. one call to + // `add_upstream_flow` per upstream catchment per time step, followed + // by a call to `get_downstream_flow` for each downstream catchment per time step, + // this loop will terminate and ensures the synchronization of flows between + // ranks. while ( stored_receives.size() > 0 ) { process_communications(); @@ -167,6 +183,28 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchment_id, time_step_t t) { + // Process any completed communications to free resources + // If no communications are pending, this call will do nothing. + process_communications(); + // NOTE: It is possible for a partition to get "too far" ahead since the sends are now + // truely asynchronous. For pure receivers and sender_receivers, this isn't a problem + // because the get_downstream_flow function will block until all receives are processed. + // However, for pure senders, this could be a problem. + // We can use this spinlock here to limit how far ahead a partition can get. + // in this case, approximately 100 time steps per downstream catchment... + while( stored_sends.size() > downstream_ranks.size()*100 ) + { + process_communications(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + // Post receives before sending to prevent deadlock + // When stored_receives is empty, we need to post for incoming messages + if ((type == receiver || type == sender_receiver) && stored_receives.empty()) + { + post_receives(); + } + // first add flow to local copy HY_PointHydroNexus::add_upstream_flow(val, catchment_id, t); @@ -205,23 +243,25 @@ void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchme int tag = extract(id); //Send downstream_flow from this Upstream Remote Nexus to the Downstream Remote Nexus - MPI_Isend( + MPI_Handle_Error( + MPI_Isend( stored_sends.back().buffer.get(), 1, time_step_and_flow_type, *downstream_ranks.begin(), //TODO currently only support a SINGLE downstream message pairing tag, MPI_COMM_WORLD, - &stored_sends.back().mpi_request); - - //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; + &stored_sends.back().mpi_request) + ); + //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; - while ( stored_sends.size() > 0 ) - { - process_communications(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } + // Send is async, the next call to add_upstream_flow will test and ensure the send has completed + // and free the memory associated with the send. + // This prevents a potential deadlock situation where a send isn't able to complete + // because the remote receiver is also trying to send and the underlying mpi buffers/protocol + // are forced into a rendevous protocol. So we ensure that we always post receives before sends. + // and that we always test for completed sends before freeing the memory associated with the send. } } } diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index f0dee4aa70..0e544b800b 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -9,4 +9,4 @@ add_library(geopackage proj.cpp add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/utilities) -target_link_libraries(geopackage PUBLIC NGen::geojson Boost::boost sqlite3 NGen::logging) +target_link_libraries(geopackage PUBLIC NGen::geojson Boost::boost SQLite::SQLite3 NGen::logging) diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index b4b233718c..3fd0b3dcd7 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -598,6 +598,9 @@ namespace realization { available_forcing_units[bmi_var_names_map[output_var_name]] = get_bmi_model()->GetVarUnits(output_var_name); //units come from the model output variable. } } + //Initialize all NgenBmiProtocols with the valid adapter pointer and any properties + //provided in the read configuration. + bmi_protocols = models::bmi::protocols::NgenBmiProtocols(get_bmi_model(), properties); } //check if units have not been specified. If not, default to native units. diff --git a/src/realizations/catchment/CMakeLists.txt b/src/realizations/catchment/CMakeLists.txt index a82c797110..6df221d6af 100644 --- a/src/realizations/catchment/CMakeLists.txt +++ b/src/realizations/catchment/CMakeLists.txt @@ -30,5 +30,6 @@ target_link_libraries(realizations_catchment PUBLIC NGen::geojson NGen::logging NGen::ngen_bmi + NGen::bmi_protocols ) diff --git a/src/utilities/bmi/CMakeLists.txt b/src/utilities/bmi/CMakeLists.txt new file mode 100644 index 0000000000..67a62eaf8d --- /dev/null +++ b/src/utilities/bmi/CMakeLists.txt @@ -0,0 +1,36 @@ +# Author: Nels Frazier +# Copyright (C) 2025 Lynker +# ------------------------------------------------------------------------ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------ +# Library Version 0.1 +# BMI mass balance checking protocol + +add_library(ngen_bmi_protocols protocols.cpp mass_balance.cpp) +add_library(NGen::bmi_protocols ALIAS ngen_bmi_protocols) + +target_include_directories(ngen_bmi_protocols PUBLIC + ${PROJECT_SOURCE_DIR}/include/bmi + ${PROJECT_SOURCE_DIR}/include/utilities/bmi + ${PROJECT_SOURCE_DIR}/include/geojson + ${NGEN_INC_DIR} +) + +target_link_libraries(ngen_bmi_protocols + PUBLIC + ${CMAKE_DL_LIBS} + Boost::boost # Headers-only Boost +) + +target_sources(ngen_bmi_protocols + PRIVATE + "${PROJECT_SOURCE_DIR}/src/bmi/Bmi_Adapter.cpp" +) diff --git a/src/utilities/bmi/mass_balance.cpp b/src/utilities/bmi/mass_balance.cpp new file mode 100644 index 0000000000..f542a682a0 --- /dev/null +++ b/src/utilities/bmi/mass_balance.cpp @@ -0,0 +1,195 @@ +/* +Author: Nels Frazier +Copyright (C) 2025 Lynker +------------------------------------------------------------------------ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +------------------------------------------------------------------------ +Version 0.3 (see mass_balance.hpp for details) + +Version 0.2 +Implement error handling via expected and error_or_warning + +Version 0.1 +Implementation the BMI mass balance checking protocol +*/ + +#include "mass_balance.hpp" + +namespace models { namespace bmi { namespace protocols { + +NgenMassBalance::NgenMassBalance(const ModelPtr& model, const Properties& properties) : + check(false), is_fatal(false), tolerance(1.0E-16), frequency(1){ + initialize(model, properties); +} + +NgenMassBalance::NgenMassBalance() : check(false) {} + +NgenMassBalance::~NgenMassBalance() = default; + +auto NgenMassBalance::run(const ModelPtr& model, const Context& ctx) const -> expected { + if( model == nullptr ) { + return make_unexpected( ProtocolError( + Error::UNITIALIZED_MODEL, + "Cannot run mass balance protocol with null model." + ) + ); + } else if( !check || !supported ) { + return {}; + } + bool check_step = false; + //if frequency was set to -1 (or any negative), only check at the end + //use <= to avoid a potential divide by zero should frequency be 0 + //(though frequency 0 should have been caught during initialization and check disabled) + if( frequency > 0 ){ + check_step = (ctx.current_time_step % frequency) == 0; + } + else if(ctx.current_time_step == ctx.total_steps){ + check_step = true; + } + + if(check_step) { + double mass_in, mass_out, mass_stored, mass_leaked, mass_balance; + model->GetValue(INPUT_MASS_NAME, &mass_in); + model->GetValue(OUTPUT_MASS_NAME, &mass_out); + model->GetValue(STORED_MASS_NAME, &mass_stored); + model->GetValue(LEAKED_MASS_NAME, &mass_leaked); + // TODO consider unit conversion if/when it becomes necessary + mass_balance = mass_in - mass_out - mass_stored - mass_leaked; + if ( std::abs(mass_balance) > tolerance || std::isnan(mass_balance)) { + std::stringstream ss; + ss << "mass_balance: " + << "at timestep " << std::to_string(ctx.current_time_step) + << " ("+ctx.timestamp+")" + << " at feature id " << ctx.id <GetComponentName() << "\n\t" << + INPUT_MASS_NAME << "(" << mass_in << ") - " << + OUTPUT_MASS_NAME << " (" << mass_out << ") - " << + STORED_MASS_NAME << " (" << mass_stored << ") - " << + LEAKED_MASS_NAME << " (" << mass_leaked << ") = " << + mass_balance << "\n\t" << "tolerance: " << tolerance << std::endl; + return make_unexpected( ProtocolError( + is_fatal ? Error::PROTOCOL_ERROR : Error::PROTOCOL_WARNING, + ss.str() + ) + ); + } + + } + return {}; +} + +auto NgenMassBalance::check_support(const ModelPtr& model) -> expected { + if (model != nullptr && model->is_model_initialized()) { + double mass_var; + std::vector units; + units.reserve(4); + try{ + for(const auto& name : + {INPUT_MASS_NAME, OUTPUT_MASS_NAME, STORED_MASS_NAME, LEAKED_MASS_NAME} + ){ + model->GetValue(name, &mass_var); + units.push_back( model->GetVarUnits(name) ); + } + //Compare all other units to the first one (+1) + if( std::equal( units.begin()+1, units.end(), units.begin() ) ) { + this->supported = true; + return {}; + } + else{ + // It may be possible to do unit conversion and still do meaninful mass balance + // this could be added as an extended feature, but for now, I don't think this is + // worth the complexity. It is, however, worth the sanity check performed here + // to ensure the units are consistent. + return make_unexpected( ProtocolError( + Error::INTEGRATION_ERROR, + "mass_balance: variables have incosistent units, cannot perform mass balance." + ) + ); + } + } catch (const std::exception &e) { + std::stringstream ss; + ss << "mass_balance: Error getting mass balance values for module '" << model->GetComponentName() << "': " << e.what() << std::endl; + return make_unexpected( ProtocolError( + Error::INTEGRATION_ERROR, + ss.str() + ) + ); + } + } else { + return make_unexpected( ProtocolError( + Error::UNITIALIZED_MODEL, + "Cannot check mass balance for uninitialized model. Disabling mass balance protocol." + ) + ); + } + return {}; +} + +auto NgenMassBalance::initialize(const ModelPtr& model, const Properties& properties) -> expected +{ + //now check if the user has requested to use mass balance + auto protocol_it = properties.find(CONFIGURATION_KEY); + if ( protocol_it != properties.end() ) { + geojson::PropertyMap mass_bal = protocol_it->second.get_values(); + + auto _it = mass_bal.find(TOLERANCE_KEY); + if( _it != mass_bal.end() ) tolerance = _it->second.as_real_number(); + //as_real_number() *should* return a floating point NaN representation + //if the input value were presented as "NaN" -- non numberic values + //non numberic values will throw an exception (not handled here) + //it is expected that the user/configuration is responsible for providing + //a valid numeric value for tolerance + if( std::isnan(tolerance) ) { + check = false; //disable mass balance checking + return error_or_warning( ProtocolError( + Error::PROTOCOL_WARNING, + "mass_balance: tolerance value 'NaN' provided, disabling mass balance check." + ) + ); + } + _it = mass_bal.find(FATAL_KEY); + if( _it != mass_bal.end() ) is_fatal = _it->second.as_boolean(); + + _it = mass_bal.find(CHECK_KEY); + if( _it != mass_bal.end() ) { + check = _it->second.as_boolean(); + } else { + //default to true if not specified + check = true; + } + + _it = mass_bal.find(FREQUENCY_KEY); + if( _it != mass_bal.end() ){ + frequency = _it->second.as_natural_number(); + } else { + frequency = 1; //default, check every timestep + } + if ( frequency == 0 ) { + check = false; // can't check at frequency 0, disable mass balance checking + } + } else{ + //no mass balance requested, or not supported, so don't check it + check = false; + } + if ( check ) { + //Ensure the model is capable of mass balance using the protocol + check_support(model).or_else( error_or_warning ); + } + return {}; // important to return for the expected to be properly created! +} + +bool NgenMassBalance::is_supported() const { + return this->supported; +} + +}}} // end namespace models::bmi::protocols diff --git a/src/utilities/bmi/protocols.cpp b/src/utilities/bmi/protocols.cpp new file mode 100644 index 0000000000..c78651a192 --- /dev/null +++ b/src/utilities/bmi/protocols.cpp @@ -0,0 +1,69 @@ +/* +Author: Nels Frazier +Copyright (C) 2025 Lynker +------------------------------------------------------------------------ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +------------------------------------------------------------------------ +Version 0.2.1 (See bmi/protocols.hpp for details) + +Version 0.2 +Implement error handling via expected and error_or_warning + +Version 0.1 +Container and management for abstract BMI protocols +*/ + +#include "protocols.hpp" + +namespace models{ namespace bmi{ namespace protocols{ + +auto operator<<(std::ostream& os, Protocol p) -> std::ostream& { + switch(p) { + case Protocol::MASS_BALANCE: os << "MASS_BALANCE"; break; + default: os << "UNKNOWN_PROTOCOL"; break; + } + return os; +} + +NgenBmiProtocols::NgenBmiProtocols() + : model(nullptr) { + protocols[Protocol::MASS_BALANCE] = std::make_unique(); +} + +NgenBmiProtocols::NgenBmiProtocols(ModelPtr model, const geojson::PropertyMap& properties) + : model(model) { + //Create and initialize mass balance configurable properties + protocols[Protocol::MASS_BALANCE] = std::make_unique(model, properties); +} + +auto NgenBmiProtocols::run(const Protocol& protocol_name, const Context& ctx) const -> expected { + // Consider using find() vs switch, especially if the number of protocols grows + expected result_or_err; + switch(protocol_name){ + case Protocol::MASS_BALANCE: + return protocols.at(Protocol::MASS_BALANCE)->run(model, ctx) + .or_else( NgenBmiProtocol::error_or_warning ); + break; + default: + std::stringstream ss; + ss << "Error: Request for unsupported protocol: '" << protocol_name << "'."; + return NgenBmiProtocol::error_or_warning( ProtocolError( + Error::UNSUPPORTED_PROTOCOL, + ss.str() + ) + ); + } + return {}; +} + +}}} // end namespace models::bmi::protocols diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 623dd7a481..a6b26ad200 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -207,6 +207,20 @@ if(TARGET test_forcings_engine) add_compile_definitions(NGEN_LUMPED_CONFIG_PATH="${CMAKE_CURRENT_BINARY_DIR}/config_aorc.yml") endif() +########################## BMI Protocol Unit Tests + +ngen_add_test( + test_bmi_protocols + OBJECTS + utils/bmi/mass_balance_Test.cpp + LIBRARIES + NGen::bmi_protocols + NGen::ngen_bmi + gmock + DEPENDS + testbmicppmodel +) + ########################## Series Unit Tests ngen_add_test( test_mdarray diff --git a/test/core/nexus/NexusRemoteTests.cpp b/test/core/nexus/NexusRemoteTests.cpp index 25964759b2..8904602b09 100644 --- a/test/core/nexus/NexusRemoteTests.cpp +++ b/test/core/nexus/NexusRemoteTests.cpp @@ -683,4 +683,438 @@ TEST_F(Nexus_Remote_Test, DISABLED_TestTree1) } + +/****************************************************************************** + * MPI DEADLOCK TESTS + * ================== + * As of commit 9ad6b7bf561fb2f96065511b382bc43a83167f10 + * A potential MPI deadlock scenario was discovered in the HY_PointHydroNexusRemote class. + * + * These tests demonstrate and verify a fix for MPI communication issues + * observed in production when running large domains with many partitions. + * + * ============================================================================= + * What can be shown: + * ============================================================================= + * + * 1. Deadlock prone code: + * The original HY_PointHydroNexusRemote::add_upstream_flow() does: + * a. MPI_Isend() - non-blocking, returns immediately + * b. while(stored_sends.size() > 0) { MPI_Test(); } - BLOCKS until + * the send is confirmed complete by MPI + * + * This pattern is problematic because it blocks progress until the send + * completes, preventing the code from posting receives. + * + * 2. WITH FORCED RENDEZVOUS, THIS PATTERN DEADLOCKS: + * When we force rendezvous protocol using --mca btl_tcp_eager_limit 80, + * the buggy pattern reliably deadlocks. Rendezvous requires a posted + * MPI_Irecv before the send can complete. + * + * 3. PRODUCTION RUNS WERE HANGING: + * Large-scale runs with 384+ partitions and 831K catchments __across multiple nodes__ + * were observed to be hanging (one node was making progress while others were not). + * + * 4. THE FIX IS CORRECT: + * Pre-posting receives (calling MPI_Irecv before MPI_Isend) is the standard + * MPI best practice and eliminates any rendezvous-related deadlock risk. + * + * ============================================================================= + * Assumptions about this fix that are hard to confirm: + * ============================================================================= + * + * We ASSUMED that production hangs occurred because: + * - High connection count (~38,000 connections) exhausted the eager buffer pool + * - This forced MPI to use rendezvous protocol even for small messages + * - Rendezvous + buggy pattern = deadlock + * + * HOWEVER: MPI documentation consistently states that rendezvous protocol is + * triggered by MESSAGE SIZE exceeding eager_limit, NOT by buffer pool exhaustion. + * We cannot find documentation supporting the "pool exhaustion triggers + * rendezvous" theory. + * + * Other possibilities for production hangs (unconfirmed) + * - TCP buffer exhaustion: if the sender's socket buffer fills up before + * the receiver can process messages, subsequent sends can block. + * And if two or more nodes get into this state, they can deadlock. + * - Network fabric issues at scale + * - Something else entirely that our fix happened to address + * + * ============================================================================= + * THE FIX IS STILL CORRECT: + * ============================================================================= + * + * Regardless of the exact production trigger, pre-posting receives is: + * 1. MPI best practice for avoiding deadlock + * 2. Required for correctness under rendezvous protocol + * 3. Harmless under eager protocol (just posts receives earlier) + * + * The fix eliminates a class of potential deadlocks even if we're uncertain + * about the exact mechanism that triggered the production hang. + * + * ============================================================================= + * TRIGGERING THE DEADLOCK IN TESTS: + * ============================================================================= + * + * We use --mca btl_tcp_eager_limit 80 to FORCE rendezvous protocol per-message. + * This is a TEST WORKAROUND that demonstrates the buggy pattern CAN deadlock. + * + * This is NOT necessarily the exact production failure mode - it's a way to + * reliably trigger the deadlock pattern in a controlled test environment. + * + * To reproduce in tests, force tcp communcation and set the eager limit to + * the minimum value (80 bytes -- openmpi_info may show different limits for + * different BTLs/environments). + * + * mpirun --mca btl tcp,self --mca btl_tcp_eager_limit 80 ... + * + * Parameters: + * --mca btl tcp,self : Disable shared memory, use TCP only + * --mca btl_tcp_eager_limit 80 : Force rendezvous per-message (minimum value) + * + * NOTE: Shared memory BTL uses copy-based communication that doesn't require + * a synchronous handshake, so --mca btl_sm_eager_limit 80 will NOT trigger + * the deadlock. + * + ******************************************************************************/ + + +/** + * ============================================================================= + * DISABLED_TestRawMpiDeadlockPattern + * ============================================================================= + * + * PURPOSE: Document and demonstrate the MPI communication pattern that causes + * deadlock when rendezvous protocol is triggered. + * This is the pattern that ngen used up to and including + * commit 9ad6b7bf561fb2f96065511b382bc43a83167f10 + * This test uses RAW MPI calls (not the HY_PointHydroNexusRemote class) to + * clearly illustrate the problematic pattern that existed in the original code. + * + * ============================================================================= + * THE PROBLEMATIC PATTERN (from original add_upstream_flow): + * ============================================================================= + * 1. MPI_Isend (non-blocking send) + * 2. Loop on MPI_Test waiting for send to complete <-- BLOCKS HERE + * 3. MPI_Irecv (never reached if step 2 blocks) + * + * ============================================================================= + * WHY NGEN PARTITIONS CREATE BIDIRECTIONAL COMMUNICATION: + * ============================================================================= + * + * Real hydrological networks are complex. When we partition the domain, the + * partition boundary cuts ACROSS the drainage network, not along it. + * This creates BIDIRECTIONAL communication between partitions. + * + * Analysis of CONUS (384 partitions, 831K catchments) confirms: + * - 182/384 partitions (47%) have BIDIRECTIONAL communication + * + * ============================================================================= + * TEST TOPOLOGY (simplified bidirectional chain): + * ============================================================================= + * + * Rank 0 <────> Rank 1 <────> Rank 2 <────> Rank 3 + * + * Each rank both SENDS to AND RECEIVES from its neighbors. + * + * ============================================================================= + * TO REPRODUCE DEADLOCK: + * ============================================================================= + * timeout 10 mpirun --mca btl tcp,self --mca btl_tcp_eager_limit 80 -n 4 \ + * ./test/test_remote_nexus --gtest_filter="*TestRawMpiDeadlockPattern*" \ + * --gtest_also_run_disabled_tests + * + * EXPECTED: Exit code 124 (timeout killed it) - confirms true deadlock + * + * ============================================================================= + */ +TEST_F(Nexus_Remote_Test, DISABLED_TestRawMpiDeadlockPattern) +{ + if (mpi_num_procs < 4) { + GTEST_SKIP() << "Requires at least 4 MPI ranks to demonstrate bidirectional deadlock"; + } + + // Bidirectional chain topology + std::vector downstream_ranks; + std::vector upstream_ranks; + + if (mpi_rank == 0) { + downstream_ranks.push_back(1); + upstream_ranks.push_back(1); + } else if (mpi_rank == 1) { + downstream_ranks.push_back(0); + downstream_ranks.push_back(2); + upstream_ranks.push_back(0); + upstream_ranks.push_back(2); + } else if (mpi_rank == 2) { + downstream_ranks.push_back(1); + downstream_ranks.push_back(3); + upstream_ranks.push_back(1); + upstream_ranks.push_back(3); + } else if (mpi_rank == 3) { + downstream_ranks.push_back(2); + upstream_ranks.push_back(2); + } + + bool is_sender = !downstream_ranks.empty(); + bool is_receiver = !upstream_ranks.empty(); + + // Create MPI datatype for our message + struct message_t { + long time_step; + long id; + double flow; + }; + + MPI_Datatype msg_type; + int counts[3] = {1, 1, 1}; + MPI_Aint displacements[3] = {0, sizeof(long), 2 * sizeof(long)}; + MPI_Datatype types[3] = {MPI_LONG, MPI_LONG, MPI_DOUBLE}; + MPI_Type_create_struct(3, counts, displacements, types, &msg_type); + MPI_Type_commit(&msg_type); + + const int NUM_MESSAGES = 3500; + const int TAG_BASE = 1000; + + std::cerr << "Rank " << mpi_rank << ": Starting bidirectional deadlock pattern\n"; + + MPI_Barrier(MPI_COMM_WORLD); + + // Storage for async operations + std::map> send_buffers; + std::map> recv_buffers; + std::map> send_requests; + std::map> recv_requests; + + for (int r : downstream_ranks) { + send_buffers[r].resize(NUM_MESSAGES); + send_requests[r].resize(NUM_MESSAGES); + } + for (int r : upstream_ranks) { + recv_buffers[r].resize(NUM_MESSAGES); + recv_requests[r].resize(NUM_MESSAGES); + } + + // Timing asymmetry emulates "work" that causes ranks to run at different "speeds" + if (mpi_rank == 1) { + volatile double dummy = 0.0; + for (int i = 0; i < 100000000; ++i) { + dummy += std::sin(i * 0.0001) * std::cos(i * 0.0002); + } + } + + // THE PROBLEMATIC PATTERN: All ranks try to complete sends BEFORE posting receives + if (is_sender) + { + for (int downstream : downstream_ranks) + { + for (int i = 0; i < NUM_MESSAGES; ++i) + { + send_buffers[downstream][i] = {i, mpi_rank, 100.0 + i}; + int tag = TAG_BASE + mpi_rank * 10000 + downstream * 100 + i; + + MPI_Isend(&send_buffers[downstream][i], 1, msg_type, downstream, tag, + MPI_COMM_WORLD, &send_requests[downstream][i]); + + // BLOCKING WAIT - THIS IS THE "BUG"! + // Under eager protocols, this test returns immediately + // Under rendezvous protocols, this test blocks until + // the receiver posts a matching Irecv. + // Similar logic applies to a full TCP buffer, MPI gets blocked waiting + // for TCP buffer to free up, which requires the receiver to + // read the data. + int flag = 0; + while (!flag) { + MPI_Test(&send_requests[downstream][i], &flag, MPI_STATUS_IGNORE); + } + } + } + } + + // Post receives - NEVER REACHED IN DEADLOCK + if (is_receiver) + { + for (int upstream : upstream_ranks) + { + for (int i = 0; i < NUM_MESSAGES; ++i) + { + int tag = TAG_BASE + upstream * 10000 + mpi_rank * 100 + i; + MPI_Irecv(&recv_buffers[upstream][i], 1, msg_type, upstream, tag, + MPI_COMM_WORLD, &recv_requests[upstream][i]); + } + MPI_Waitall(NUM_MESSAGES, recv_requests[upstream].data(), MPI_STATUSES_IGNORE); + } + } + + MPI_Type_free(&msg_type); + MPI_Barrier(MPI_COMM_WORLD); + std::cerr << "Rank " << mpi_rank << ": Test passed (eager buffer was sufficient)\n"; +} + + +/** + * ============================================================================= + * TestRemoteNexusDeadlockFree + * ============================================================================= + * + * PURPOSE: Verify that HY_PointHydroNexusRemote does NOT deadlock, even with + * extremely small MPI eager buffers that force rendezvous protocol. + * + * This test uses the SAME BIDIRECTIONAL topology as DISABLED_TestRawMpiDeadlockPattern + * but uses the HY_PointHydroNexusRemote class instead of raw MPI calls. + * + * ============================================================================= + * THE FIX: AUTO-POSTED RECEIVES + * ============================================================================= + * + * The HY_PointHydroNexusRemote class now auto-posts MPI_Irecv BEFORE sending. + * This breaks the deadlock cycle because peers can always complete their sends. + * + * ============================================================================= + * TO RUN (with small eager buffer to force rendezvous): + * ============================================================================= + * mpirun --mca btl tcp,self --mca btl_tcp_eager_limit 80 -n 4 \ + * ./test/test_remote_nexus --gtest_filter="*TestRemoteNexusDeadlockFree*" + * + * EXPECTED: Exit code 0 - test passes without deadlock + * + * ============================================================================= + */ +TEST_F(Nexus_Remote_Test, TestRemoteNexusDeadlockFree) +{ + if (mpi_num_procs < 4) { + GTEST_SKIP() << "Requires at least 4 MPI ranks for bidirectional topology"; + } + + // Same bidirectional topology as the deadlock test + std::vector downstream_ranks; + std::vector upstream_ranks; + + if (mpi_rank == 0) { + downstream_ranks.push_back(1); + upstream_ranks.push_back(1); + } else if (mpi_rank == 1) { + downstream_ranks.push_back(0); + downstream_ranks.push_back(2); + upstream_ranks.push_back(0); + upstream_ranks.push_back(2); + } else if (mpi_rank == 2) { + downstream_ranks.push_back(1); + downstream_ranks.push_back(3); + upstream_ranks.push_back(1); + upstream_ranks.push_back(3); + } else if (mpi_rank == 3) { + downstream_ranks.push_back(2); + upstream_ranks.push_back(2); + } + + bool is_sender = !downstream_ranks.empty(); + bool is_receiver = !upstream_ranks.empty(); + + const int NUM_CONNECTIONS = 50; + + std::cerr << "Rank " << mpi_rank << ": Setting up bidirectional topology\n"; + + // Create sender nexuses for each downstream rank + std::map>> senders; + for (int downstream : downstream_ranks) + { + for (int i = 0; i < NUM_CONNECTIONS; ++i) + { + int nex_num = mpi_rank * 100000 + downstream * 1000 + i; + std::string nex_id = "nex-" + std::to_string(nex_num); + std::string my_cat = "cat-send-" + std::to_string(mpi_rank) + "-" + std::to_string(downstream) + "-" + std::to_string(i); + std::string their_cat = "cat-recv-" + std::to_string(downstream) + "-" + std::to_string(mpi_rank) + "-" + std::to_string(i); + + HY_PointHydroNexusRemote::catcment_location_map_t loc_map; + loc_map[their_cat] = downstream; + + std::vector receiving = {their_cat}; + std::vector contributing = {my_cat}; + + senders[downstream].push_back(std::make_shared( + nex_id, receiving, contributing, loc_map)); + } + } + + // Create receiver nexuses for each upstream rank + std::map>> receivers; + for (int upstream : upstream_ranks) + { + for (int i = 0; i < NUM_CONNECTIONS; ++i) + { + int nex_num = upstream * 100000 + mpi_rank * 1000 + i; + std::string nex_id = "nex-" + std::to_string(nex_num); + std::string their_cat = "cat-send-" + std::to_string(upstream) + "-" + std::to_string(mpi_rank) + "-" + std::to_string(i); + std::string my_cat = "cat-recv-" + std::to_string(mpi_rank) + "-" + std::to_string(upstream) + "-" + std::to_string(i); + + HY_PointHydroNexusRemote::catcment_location_map_t loc_map; + loc_map[their_cat] = upstream; + + std::vector receiving = {my_cat}; + std::vector contributing = {their_cat}; + + receivers[upstream].push_back(std::make_shared( + nex_id, receiving, contributing, loc_map)); + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + // Timing asymmetry emulates "work" that causes ranks to run at different "speeds" + if (mpi_rank == 1) { + volatile double dummy = 0.0; + for (int i = 0; i < 100000000; ++i) { + dummy += std::sin(i * 0.0001) * std::cos(i * 0.0002); + } + } + + long ts = 0; + double flow_value = 42.0; + + // Send all flows - with the fix, receives are auto-posted before sending + if (is_sender) + { + std::cerr << "Rank " << mpi_rank << ": Starting sends (receives auto-posted by fix)\n"; + + for (auto& kv : senders) + { + int downstream = kv.first; + auto& nexuses = kv.second; + for (int i = 0; i < NUM_CONNECTIONS; ++i) + { + std::string my_cat = "cat-send-" + std::to_string(mpi_rank) + "-" + std::to_string(downstream) + "-" + std::to_string(i); + nexuses[i]->add_upstream_flow(flow_value, my_cat, ts); + } + } + std::cerr << "Rank " << mpi_rank << ": All sends completed!\n"; + } + + // Receive all flows + if (is_receiver) + { + std::cerr << "Rank " << mpi_rank << ": Receiving from upstream\n"; + for (auto& kv : receivers) + { + int upstream = kv.first; + auto& nexuses = kv.second; + for (int i = 0; i < NUM_CONNECTIONS; ++i) + { + std::string my_cat = "cat-recv-" + std::to_string(mpi_rank) + "-" + std::to_string(upstream) + "-" + std::to_string(i); + double received = nexuses[i]->get_downstream_flow(my_cat, ts, 100.0); + ASSERT_DOUBLE_EQ(flow_value, received); + } + } + } + + senders.clear(); + receivers.clear(); + + MPI_Barrier(MPI_COMM_WORLD); + std::cerr << "Rank " << mpi_rank << ": Test PASSED - no deadlock with remote nexus\n"; +} + + +//#endif // NGEN_MPI_TESTS_ACTIVE + //#endif // NGEN_MPI_TESTS_ACTIVE diff --git a/test/data/bmi/test_bmi_python/test_bmi_python_config_2.yml b/test/data/bmi/test_bmi_python/test_bmi_python_config_2.yml new file mode 100644 index 0000000000..edd5b81e35 --- /dev/null +++ b/test/data/bmi/test_bmi_python/test_bmi_python_config_2.yml @@ -0,0 +1,2 @@ +time_step_seconds: 3600 +initial_time: 0 \ No newline at end of file diff --git a/test/realizations/catchments/Bmi_C_Formulation_Test.cpp b/test/realizations/catchments/Bmi_C_Formulation_Test.cpp index 3b46c648b4..e532f201e9 100644 --- a/test/realizations/catchments/Bmi_C_Formulation_Test.cpp +++ b/test/realizations/catchments/Bmi_C_Formulation_Test.cpp @@ -24,7 +24,7 @@ #include "FileChecker.h" #include "Formulation_Manager.hpp" #include - +#include "../../utils/bmi/MockConfig.hpp" using ::testing::MatchesRegex; using namespace realization; @@ -116,6 +116,7 @@ class Bmi_C_Formulation_Test : public ::testing::Test { std::vector main_output_variable; std::vector registration_functions; std::vector uses_forcing_file; + // std::vector tries_mass_balance; std::vector> forcing_params_examples; std::vector config_properties; std::vector config_prop_ptree; @@ -148,6 +149,7 @@ void Bmi_C_Formulation_Test::SetUp() { main_output_variable = std::vector(EX_COUNT); registration_functions = std::vector(EX_COUNT); uses_forcing_file = std::vector(EX_COUNT); + // tries_mass_balance = std::vector(EX_COUNT); forcing_params_examples = std::vector>(EX_COUNT); config_properties = std::vector(EX_COUNT); config_prop_ptree = std::vector(EX_COUNT); @@ -161,6 +163,7 @@ void Bmi_C_Formulation_Test::SetUp() { main_output_variable[0] = "OUTPUT_VAR_1"; registration_functions[0] = "register_bmi"; uses_forcing_file[0] = false; + // tries_mass_balance[0] = true; catchment_ids[1] = "cat-27"; model_type_name[1] = "test_bmi_c"; @@ -170,6 +173,7 @@ void Bmi_C_Formulation_Test::SetUp() { main_output_variable[1] = "OUTPUT_VAR_1"; registration_functions[1] = "register_bmi"; uses_forcing_file[1] = false; + // tries_mass_balance[1] = false; std::string variables_with_rain_rate = " \"output_variables\": [\"OUTPUT_VAR_2\",\n" " \"OUTPUT_VAR_1\"],\n"; @@ -198,6 +202,9 @@ void Bmi_C_Formulation_Test::SetUp() { + variables_line + " \"uses_forcing_file\": " + (uses_forcing_file[i] ? "true" : "false") + "," " \"model_params\": { \"PARAM_VAR_1\": 42, \"PARAM_VAR_2\": 4.2, \"PARAM_VAR_3\": [4, 2]}" + + // (tries_mass_balance[i] ? \ + // ", \"mass_balance\": {\"tolerance\": 1e-12, \"fatal\":true}" \ + // :"" )+ " }," " \"forcing\": { \"path\": \"" + forcing_file[i] + "\", \"provider\": \"CsvPerFeature\"}" " }" @@ -387,6 +394,235 @@ TEST_F(Bmi_C_Formulation_Test, determine_model_time_offset_0_c) { ASSERT_EQ(get_friend_bmi_model_start_time_forcing_offset_s(formulation), expected_offset); } +using models::bmi::protocols::INPUT_MASS_NAME; +using models::bmi::protocols::OUTPUT_MASS_NAME; +using models::bmi::protocols::STORED_MASS_NAME; +using models::bmi::protocols::LEAKED_MASS_NAME; +using models::bmi::protocols::ProtocolError; + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + // mass balance failure will throw an exception + ptree.put_child("mass_balance", MassBalanceMock(true).get()); + formulation.create_formulation(ptree); + formulation.check_mass_balance(0, 2, "t0"); + formulation.get_response(1, 3600); + formulation.check_mass_balance(1, 2, "t1"); +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_warns) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(false).get()); + formulation.create_formulation(ptree); + //formulation.check_mass_balance(0, 1, "t0"); + formulation.get_response(1, 3600); + double mass_error = 100; + get_friend_bmi_model(formulation)->SetValue(STORED_MASS_NAME, &mass_error); // Force a mass balance error + //ASSERT_THROW(formulation.check_mass_balance(0, 1, "t0"), MassBalanceWarning); + testing::internal::CaptureStderr(); + formulation.check_mass_balance(0, 1, "t0"); + std::string output = testing::internal::GetCapturedStderr(); + std::cerr << output; + EXPECT_THAT(output, testing::HasSubstr("Warning(Protocol)::mass_balance:")); +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_stored_fails) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(true).get()); + formulation.create_formulation(ptree); + //formulation.check_mass_balance(0, 1, "t0"); + formulation.get_response(1, 3600); + double mass_error = 100; + get_friend_bmi_model(formulation)->SetValue(STORED_MASS_NAME, &mass_error); // Force a mass balance error + //formulation.check_mass_balance(0, 1, "t0"); + ASSERT_THROW(formulation.check_mass_balance(0, 1, "t0"), ProtocolError); + try{ + formulation.check_mass_balance(0, 1, "t0"); + } + catch (ProtocolError& e) { + std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance:.*")); + } +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_in_fails_a) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(true, 1e-5).get()); + formulation.create_formulation(ptree); + formulation.get_response(1, 3600); + double mass_error = 2; + get_friend_bmi_model(formulation)->SetValue(INPUT_MASS_NAME, &mass_error); // Force a mass balance error + ASSERT_THROW(formulation.check_mass_balance(0, 1, "t0"), ProtocolError); + try{ + formulation.check_mass_balance(0, 1, "t0"); + } + catch (ProtocolError& e) { + std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance:.*")); + } +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_out_fails) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(true).get()); + formulation.create_formulation(ptree); + formulation.get_response(1, 3600); + double mass_error = 2; + get_friend_bmi_model(formulation)->SetValue(OUTPUT_MASS_NAME, &mass_error); // Force a mass balance error + ASSERT_THROW(formulation.check_mass_balance(0, 1, "t0"), ProtocolError); + try{ + formulation.check_mass_balance(0, 1, "t0"); + } + catch (ProtocolError& e) { + std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance:.*")); + } +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_leaked_fails) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(true).get()); + formulation.create_formulation(ptree); + formulation.get_response(1, 3600); + double mass_error = 2; + get_friend_bmi_model(formulation)->SetValue(LEAKED_MASS_NAME, &mass_error); // Force a mass balance error + ASSERT_THROW(formulation.check_mass_balance(0, 1, "t0"), ProtocolError); + try{ + formulation.check_mass_balance(0, 1, "t0"); + } + catch (ProtocolError& e) { + std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance:.*")); + } +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_tolerance) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(true, 1e-5).get()); + formulation.create_formulation(ptree); + //formulation.check_mass_balance(0, "t0"); + formulation.get_response(1, 3600); + double mass_error; + get_friend_bmi_model(formulation)->GetValue(INPUT_MASS_NAME, &mass_error); + mass_error += 1e-4; // Force a mass balance error not within tolerance + get_friend_bmi_model(formulation)->SetValue(INPUT_MASS_NAME, &mass_error); // Force a mass balance error + ASSERT_THROW(formulation.check_mass_balance(0, 1, "t0"), ProtocolError); + try{ + formulation.check_mass_balance(0, 1, "t0"); + } + catch (ProtocolError& e) { + std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance:.*")); + } +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_tolerance_a) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(true, 1e-5).get()); + formulation.create_formulation(ptree); + //formulation.check_mass_balance(0, 1, "t0"); + formulation.get_response(1, 3600); + double mass_error; + get_friend_bmi_model(formulation)->GetValue(INPUT_MASS_NAME, &mass_error); + mass_error += 1e-6; // Force a mass balance error within tolerance + get_friend_bmi_model(formulation)->SetValue(INPUT_MASS_NAME, &mass_error); // Force a mass balance error + formulation.check_mass_balance(0, 1, "t0"); // Should not throw an error +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_off) { + int ex_index = 1; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + formulation.create_formulation(config_prop_ptree[ex_index]); + formulation.check_mass_balance(0, 2, "t0"); + formulation.get_response(1, 3600); + formulation.check_mass_balance(1, 2, "t1"); +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_missing) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + + std::string catchment_path = "catchments." + catchment_ids[ex_index] + ".bmi_c"; + ptree.erase("mass_balance"); + formulation.create_formulation(ptree); + formulation.check_mass_balance(0, 2, "t0"); + formulation.get_response(1, 3600); + double mass_error = 100; + double more_mass_error = 99; + get_friend_bmi_model(formulation)->SetValue(OUTPUT_MASS_NAME, &mass_error); // Force a mass balance error + get_friend_bmi_model(formulation)->SetValue(INPUT_MASS_NAME, &more_mass_error); // Force a mass balance error + formulation.check_mass_balance(1, 2, "t1"); + +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_frequency) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(true, 1e-5, 2).get()); + formulation.create_formulation(ptree); + double mass_error; + mass_error += 10; // Force a mass balance error above tolerance + get_friend_bmi_model(formulation)->SetValue(OUTPUT_MASS_NAME, &mass_error); // + //Check initial mass balance -- should error which indicates it was propoerly checked + //per frequency setting + ASSERT_THROW(formulation.check_mass_balance(0, 2, "t0"), ProtocolError); + // Call mass balance check again, this should NOT error, since the actual check + // should be skipped due to the frequency setting + formulation.check_mass_balance(1, 2, "t1"); + // Check mass balance again, this SHOULD error since the previous mass balance + // will propagate, and it should now be checked based on the frequency + ASSERT_THROW(formulation.check_mass_balance(2, 2, "t2"), ProtocolError); +} + +TEST_F(Bmi_C_Formulation_Test, check_mass_balance_frequency_1) { + int ex_index = 0; + + Bmi_C_Formulation formulation(catchment_ids[ex_index], std::make_shared(*forcing_params_examples[ex_index]), utils::StreamHandler()); + auto ptree = config_prop_ptree[ex_index]; + ptree.put_child("mass_balance", MassBalanceMock(true, 1e-5, -1).get()); + formulation.create_formulation(ptree); + double mass_error; + mass_error += 10; // Force a mass balance error above tolerance + get_friend_bmi_model(formulation)->SetValue(OUTPUT_MASS_NAME, &mass_error); // + //Check initial mass balance -- should NOT error + formulation.check_mass_balance(0, 2, "t0"); + // Call mass balance check again, this should NOT error, since the actual check + // should be skipped due to the frequency setting + formulation.check_mass_balance(1, 2, "t1"); + // Check mass balance again, this SHOULD error since the this is step 2/2 + // and it will now be checked based on the frequency (-1, check at end) + ASSERT_THROW(formulation.check_mass_balance(2, 2, "t2"), ProtocolError); +} + #endif // NGEN_BMI_C_LIB_TESTS_ACTIVE #endif // NGEN_BMI_C_FORMULATION_TEST_CPP diff --git a/test/realizations/catchments/Bmi_Multi_Formulation_Test.cpp b/test/realizations/catchments/Bmi_Multi_Formulation_Test.cpp index 05c244526e..e0773e6767 100644 --- a/test/realizations/catchments/Bmi_Multi_Formulation_Test.cpp +++ b/test/realizations/catchments/Bmi_Multi_Formulation_Test.cpp @@ -387,7 +387,7 @@ class Bmi_Multi_Formulation_Test : public ::testing::Test { return s; } - inline void buildExampleConfig(const int ex_index) { + inline void buildExampleConfig(const int ex_index, const int nested_count) { std::string outputVariablesSubConfig = (ex_index == 6) ? buildExampleOutputVariablesSubConfig(ex_index, true) : buildExampleOutputVariablesSubConfig(ex_index) + "\n"; std::string config = "{\n" @@ -403,10 +403,12 @@ class Bmi_Multi_Formulation_Test : public ::testing::Test { " \"init_config\": \"\",\n" " \"allow_exceed_end_time\": true,\n" " \"main_output_variable\": \"" + main_output_variables[ex_index] + "\",\n" - " \"modules\": [\n" - + buildExampleNestedModuleSubConfig(ex_index, 0) + ",\n" - + buildExampleNestedModuleSubConfig(ex_index, 1) + "\n" - " ],\n" + " \"modules\": [\n"; + for (int i = 0; i < nested_count - 1; ++i) { + config += buildExampleNestedModuleSubConfig(ex_index, i) + ",\n"; + } + config += buildExampleNestedModuleSubConfig(ex_index, nested_count - 1) + "\n"; + config += " ],\n" " \"uses_forcing_file\": false\n" + outputVariablesSubConfig + " }\n" @@ -462,7 +464,7 @@ class Bmi_Multi_Formulation_Test : public ::testing::Test { main_output_variables[ex_index] = nested_module_main_output_variables[ex_index][example_module_depth[ex_index] - 1]; specified_output_variables[ex_index] = output_variables; - buildExampleConfig(ex_index); + buildExampleConfig(ex_index, nested_module_lists[ex_index].size()); } @@ -490,7 +492,7 @@ void Bmi_Multi_Formulation_Test::SetUp() { // Define this manually to set how many nested modules per example, and implicitly how many examples. // This means example_module_depth.size() example scenarios with example_module_depth[i] nested modules in each scenario. - example_module_depth = {2, 2, 2, 2, 2, 2, 2}; + example_module_depth = {2, 2, 2, 2, 2, 2, 2, 3}; // Initialize the members for holding required input and result test data for individual example scenarios setupExampleDataCollections(); @@ -533,7 +535,12 @@ void Bmi_Multi_Formulation_Test::SetUp() { initializeTestExample(6, "cat-27", {std::string(BMI_CPP_TYPE), std::string(BMI_FORTRAN_TYPE)}, { "OUTPUT_VAR_3","OUTPUT_VAR_3","OUTPUT_VAR_3" }); - + #if NGEN_WITH_BMI_C + initializeTestExample(7, "cat-27", {std::string(BMI_C_TYPE), std::string(BMI_FORTRAN_TYPE), std::string(BMI_PYTHON_TYPE)}, {"OUTPUT_VAR_1__0"}); // Output var from C module... + #else + initializeTestExample(7, "cat-27", {std::string(BMI_FORTRAN_TYPE), std::string(BMI_PYTHON_TYPE)}, {"OUTPUT_VAR_1__0"}); // Output var from Fortran module... + + #endif // NGEN_WITH_PYTHON } /** Simple test to make sure the model config from example 0 initializes. */ @@ -940,6 +947,16 @@ TEST_F(Bmi_Multi_Formulation_Test, GetAvailableVariableNames) { ); } } + +TEST_F(Bmi_Multi_Formulation_Test, MassBalanceCheck) { + int ex_index = 6; + + Bmi_Multi_Formulation formulation(catchment_ids[ex_index], std::make_unique(*forcing_params_examples[ex_index]), utils::StreamHandler()); + formulation.create_formulation(config_prop_ptree[ex_index]); + + formulation.check_mass_balance(0, 1, "t0"); +} + #endif // NGEN_WITH_BMI_C || NGEN_WITH_BMI_FORTRAN || NGEN_WITH_PYTHON #endif // NGEN_BMI_MULTI_FORMULATION_TEST_CPP diff --git a/test/utils/bmi/MockConfig.hpp b/test/utils/bmi/MockConfig.hpp new file mode 100644 index 0000000000..094ce440df --- /dev/null +++ b/test/utils/bmi/MockConfig.hpp @@ -0,0 +1,70 @@ +/* +Author: Nels Frazier +Copyright (C) 2025 Lynker +------------------------------------------------------------------------ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +------------------------------------------------------------------------ +*/ +#pragma once +#include +#include "protocols.hpp" +#include "JSONProperty.hpp" + +static const auto noneConfig = std::map { + {"none", geojson::JSONProperty("none", true)} +}; + +static models::bmi::protocols::Context make_context(int current_time_step, int total_steps, const std::string& timestamp, const std::string& id) { + return models::bmi::protocols::Context{ + current_time_step, + total_steps, + timestamp, + id + }; +} + +class MassBalanceMock { + public: + + MassBalanceMock( bool fatal = false, double tolerance = 1e-12, int frequency = 1, bool check = true) + : properties() { + boost::property_tree::ptree config; + config.put("check", check); + config.put("tolerance", tolerance); + config.put("fatal", fatal); + config.put("frequency", frequency); + properties.add_child("mass_balance", config); + } + + MassBalanceMock( bool fatal, const char* tolerance){ + boost::property_tree::ptree config; + config.put("check", true); + config.put("tolerance", tolerance); + config.put("fatal", fatal); + config.put("frequency", 1); + properties.add_child("mass_balance", config); + } + + const boost::property_tree::ptree& get() const { + return properties.get_child("mass_balance"); + } + + const geojson::PropertyMap as_json_property() const { + auto props = geojson::JSONProperty("mass_balance", properties); + // geojson::JSONProperty::print_property(props, 1); + return props.get_values(); + } + + private: + boost::property_tree::ptree properties; +}; \ No newline at end of file diff --git a/test/utils/bmi/mass_balance_Test.cpp b/test/utils/bmi/mass_balance_Test.cpp new file mode 100644 index 0000000000..9d834c3ce3 --- /dev/null +++ b/test/utils/bmi/mass_balance_Test.cpp @@ -0,0 +1,435 @@ +/* +Author: Nels Frazier +Copyright (C) 2025 Lynker +------------------------------------------------------------------------ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +------------------------------------------------------------------------ +*/ +#ifdef NGEN_BMI_CPP_LIB_TESTS_ACTIVE +//Test utitilities +// #include "bmi.hpp" +#include "gtest/gtest.h" +#include "gmock/gmock.h" +//Mock configuration and context helpers +#include "MockConfig.hpp" +//BMI model/adapter +#include "FileChecker.h" +#include "Bmi_Cpp_Adapter.hpp" +#include +//Interface under test +#include "protocols.hpp" + +using ::testing::MatchesRegex; +//protocol symbols +using models::bmi::protocols::NgenBmiProtocols; +using models::bmi::protocols::INPUT_MASS_NAME; +using models::bmi::protocols::OUTPUT_MASS_NAME; +using models::bmi::protocols::STORED_MASS_NAME; +using models::bmi::protocols::LEAKED_MASS_NAME; +using models::bmi::protocols::ProtocolError; +using nonstd::expected_lite::expected; + +// Use the ngen bmi c++ test library +#ifndef BMI_TEST_CPP_LOCAL_LIB_NAME +#ifdef __APPLE__ +#define BMI_TEST_CPP_LOCAL_LIB_NAME "libtestbmicppmodel.dylib" +#else +#ifdef __GNUC__ + #define BMI_TEST_CPP_LOCAL_LIB_NAME "libtestbmicppmodel.so" + #endif // __GNUC__ +#endif // __APPLE__ +#endif // BMI_TEST_CPP_LOCAL_LIB_NAME + +#define CREATOR_FUNC "bmi_model_create" +#define DESTROYER_FUNC "bmi_model_destroy" + +using namespace models::bmi; + +// Copy of the struct def used within the test_bmi_c test library + +class Bmi_Cpp_Test_Adapter : public ::testing::Test { +protected: + + void SetUp() override; + + void TearDown() override; + + std::string file_search(const std::vector &parent_dir_options, const std::string& file_basename); + + std::string config_file_name_0; + std::string lib_file_name_0; + std::string bmi_module_type_name_0; + std::unique_ptr adapter; +}; + +void Bmi_Cpp_Test_Adapter::SetUp() { + /** + * @brief Set up the test environment for the protocol tests. + * + * The protocol requires a valid BMI model to operate on, so this setup + * function initializes a BMI C++ adapter using the test BMI C++ model + * + */ + // Uses the same config files as the C test model... + std::vector config_path_options = { + "test/data/bmi/test_bmi_c/", + "./test/data/bmi/test_bmi_c/", + "../test/data/bmi/test_bmi_c/", + "../../test/data/bmi/test_bmi_c/", + }; + std::string config_basename_0 = "test_bmi_c_config_0.txt"; + config_file_name_0 = file_search(config_path_options, config_basename_0); + + std::vector lib_dir_opts = { + "./extern/test_bmi_cpp/cmake_build/", + "../extern/test_bmi_cpp/cmake_build/", + "../../extern/test_bmi_cpp/cmake_build/" + }; + lib_file_name_0 = file_search(lib_dir_opts, BMI_TEST_CPP_LOCAL_LIB_NAME); + bmi_module_type_name_0 = "test_bmi_cpp"; + try { + adapter = std::make_unique(bmi_module_type_name_0, lib_file_name_0, config_file_name_0, + true, CREATOR_FUNC, DESTROYER_FUNC); + } + catch (const std::exception &e) { + std::clog << e.what() << std::endl; + throw e; + } + } + +void Bmi_Cpp_Test_Adapter::TearDown() { + +} + +std::string +Bmi_Cpp_Test_Adapter::file_search(const std::vector &parent_dir_options, const std::string& file_basename) { + // Build vector of names by building combinations of the path and basename options + std::vector name_combinations; + + // Build so that all path names are tried for given basename before trying a different basename option + for (auto & path_option : parent_dir_options) + name_combinations.push_back(path_option + file_basename); + + return utils::FileChecker::find_first_readable(name_combinations); +} + +class Bmi_Mass_Balance_Test : public Bmi_Cpp_Test_Adapter { +protected: + void SetUp() override { + Bmi_Cpp_Test_Adapter::SetUp(); + model = std::shared_ptr(adapter.release()); + model_name = model->GetComponentName(); + } + std::string time = "t0"; + std::string model_name; + std::shared_ptr model; +}; + +TEST_F(Bmi_Mass_Balance_Test, bad_model) { + model = nullptr; // simulate uninitialized model + auto properties = MassBalanceMock(true).as_json_property(); + auto context = make_context(0, 2, time, model_name); + testing::internal::CaptureStderr(); + auto protocols = NgenBmiProtocols(model, properties); + std::string output = testing::internal::GetCapturedStderr(); + EXPECT_THAT(output, MatchesRegex("Error\\(Uninitialized Model\\).*Disabling mass balance protocol.\n")); + testing::internal::CaptureStderr(); + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, context); + EXPECT_TRUE( ! result.has_value() ); // should have an error!!! + EXPECT_EQ( result.error().error_code(), models::bmi::protocols::Error::UNITIALIZED_MODEL ); + output = testing::internal::GetCapturedStderr(); + EXPECT_THAT(output, MatchesRegex("Error\\(Uninitialized Model\\).*\n")); +} + +TEST_F(Bmi_Mass_Balance_Test, default_construct) { + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(); + testing::internal::CaptureStderr(); + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, context); + EXPECT_TRUE( ! result.has_value() ); // should have an error!!! + EXPECT_EQ( result.error().error_code(), models::bmi::protocols::Error::UNITIALIZED_MODEL ); + std::string output = testing::internal::GetCapturedStderr(); + EXPECT_THAT(output, MatchesRegex("Error\\(Uninitialized Model\\).*\n")); +} + +TEST_F(Bmi_Mass_Balance_Test, check) { + auto properties = MassBalanceMock(true).as_json_property(); + auto context = make_context(0, 2, time, model_name); + testing::internal::CaptureStderr(); + auto protocols = NgenBmiProtocols(model, properties); + protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, context); + model->Update(); + time = "t1"; + protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + std::string output = testing::internal::GetCapturedStderr(); + // Not warning/errors printed, and no exceptions thrown is success + EXPECT_EQ(output, ""); +} + +TEST_F(Bmi_Mass_Balance_Test, warns) { + auto properties = MassBalanceMock(false).as_json_property(); + auto context = make_context(0, 2, time, model_name); + testing::internal::CaptureStderr(); + auto protocols = NgenBmiProtocols(model, properties); + double mass_error = 100; + model->SetValue(STORED_MASS_NAME, &mass_error); // Force a mass balance error + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + // The expected should be an error not a value (void in this case) + EXPECT_FALSE( result.has_value() ); // should have a warning!!! + EXPECT_THAT( result.error().to_string(), testing::HasSubstr("Warning(Protocol)::mass_balance:") ); + std::string output = testing::internal::GetCapturedStderr(); + // std::cerr << output; + //Warning was sent to stderr + EXPECT_THAT(output, testing::HasSubstr("Warning(Protocol)::mass_balance:")); +} + +TEST_F(Bmi_Mass_Balance_Test, storage_fails) { + auto properties = MassBalanceMock(true, 1e-5).as_json_property(); + auto context = make_context(0, 2, time, model_name); + + auto protocols = NgenBmiProtocols(model, properties); + model->Update(); // advance model + double mass_error = 2; + model->SetValue(STORED_MASS_NAME, &mass_error); // Force a mass balance error + time = "t1"; + + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)), ProtocolError); + try{ + // This should throw, so result won't be defined... + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + } + catch (ProtocolError& e) { + // std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance: at timestep 1 \\(t1\\).*")); + } +} + +TEST_F(Bmi_Mass_Balance_Test, in_fails) { + auto properties = MassBalanceMock(true, 1e-5).as_json_property(); + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + model->Update(); // advance model + time = "t1"; + double mass_error = 2; + model->SetValue(INPUT_MASS_NAME, &mass_error); // Force a mass balance error + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)), ProtocolError); + try{ + // This should throw, so result won't be defined... + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + } + catch (ProtocolError& e) { + // std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance: at timestep 1 \\(t1\\).*")); + } +} + +TEST_F(Bmi_Mass_Balance_Test, out_fails) { + auto properties = MassBalanceMock(true, 1e-5).as_json_property(); + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + model->Update(); // advance model + time = "t1"; + double mass_error = 2; + model->SetValue(OUTPUT_MASS_NAME, &mass_error); // Force a mass balance error + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)), ProtocolError); + try{ + // This should throw, so result won't be defined... + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + } + catch (ProtocolError& e) { + // std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance: at timestep 1 \\(t1\\).*")); + } +} + +TEST_F(Bmi_Mass_Balance_Test, leaked_fails) { + auto properties = MassBalanceMock(true, 1e-5).as_json_property(); + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + model->Update(); // advance model + time = "t1"; + double mass_error = 2; + model->SetValue(LEAKED_MASS_NAME, &mass_error); // Force a mass balance error + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)), ProtocolError); + try{ + // This should throw, so result won't be defined... + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + } + catch (ProtocolError& e) { + // std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance: at timestep 1 \\(t1\\).*")); + } +} + +TEST_F(Bmi_Mass_Balance_Test, tolerance_fails) { + auto properties = MassBalanceMock(true, 1e-5).as_json_property();; + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + model->Update(); // advance model + double mass_error; + model->GetValue(INPUT_MASS_NAME, &mass_error); + mass_error += 1e-4; // Force a mass balance error not within tolerance + model->SetValue(INPUT_MASS_NAME, &mass_error); // Force a mass balance error + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(0, 2, time, model_name)), ProtocolError); + try{ + // This should throw, so result won't be defined... + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(0, 2, time, model_name)); + } + catch (ProtocolError& e) { + // std::cerr << e.to_string() << std::endl; + EXPECT_THAT(e.to_string(), MatchesRegex("Error\\(Protocol\\)::mass_balance: at timestep 0 \\(t0\\).*")); + } +} + +TEST_F(Bmi_Mass_Balance_Test, tolerance_passes) { + auto properties = MassBalanceMock(true, 1e-5).as_json_property(); + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + model->Update(); // advance model + double mass_error; + model->GetValue(INPUT_MASS_NAME, &mass_error); + mass_error += 1e-6; // Force a mass balance error within tolerance + model->SetValue(INPUT_MASS_NAME, &mass_error); // Force a mass balance error + // should not thow! + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(0, 2, time, model_name)); + EXPECT_TRUE( result.has_value() ); // should pass +} + +TEST_F(Bmi_Mass_Balance_Test, disabled) { + auto properties = MassBalanceMock(true, 1e-5, 1, false).as_json_property(); + auto context = make_context(0, 2, time, model_name); + testing::internal::CaptureStderr(); + auto protocols = NgenBmiProtocols(model, properties); + + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, context); + EXPECT_TRUE( result.has_value() ); // should pass, as mass balance is disabled + model->Update(); // advance model + time = "t1"; + double mass_error = 100; + model->SetValue(STORED_MASS_NAME, &mass_error); // Force a mass balance error + result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + // should not throw, as mass balance is disabled + EXPECT_TRUE( result.has_value() ); // should pass, as mass balance is disabled + std::string output = testing::internal::GetCapturedStderr(); + // No warnings/errors printed, and no exceptions thrown is success + EXPECT_EQ(output, ""); +} + +TEST_F(Bmi_Mass_Balance_Test, unconfigured) { + auto properties = noneConfig; + auto context = make_context(0, 2, time, model_name); + testing::internal::CaptureStderr(); + auto protocols = NgenBmiProtocols(model, properties); + + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, context); + EXPECT_TRUE( result.has_value() ); // should pass, as mass balance is disabled + model->Update(); // advance model + time = "t1"; + double mass_error = 100; + model->SetValue(STORED_MASS_NAME, &mass_error); // Force a mass balance error + result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + // should not throw, as mass balance is disabled + EXPECT_TRUE( result.has_value() ); // should pass, as mass balance is disabled + std::string output = testing::internal::GetCapturedStderr(); + // No warnings/errors printed, and no exceptions thrown is success + EXPECT_EQ(output, ""); +} + +TEST_F(Bmi_Mass_Balance_Test, frequency) { + auto properties = MassBalanceMock(true, 1e-5, 2).as_json_property(); + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + double mass_error = 10; // Force a mass balance error above tolerance + model->SetValue(OUTPUT_MASS_NAME, &mass_error); + //Check initial mass balance -- should error which indicates it was propoerly checked + //per frequency setting + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(0, 2, time, model_name)), ProtocolError); + time = "t1"; + model->Update(); // advance model + model->SetValue(OUTPUT_MASS_NAME, &mass_error); + // Call mass balance check again, this should NOT error, since the actual check + // should be skipped due to the frequency setting + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + EXPECT_TRUE( result.has_value() ); // should pass + time = "t2"; + model->Update(); // advance model + model->SetValue(OUTPUT_MASS_NAME, &mass_error); + // Check mass balance again, this SHOULD error since the previous mass balance + // will propagate, and it should now be checked based on the frequency + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(2, 2, time, model_name)), ProtocolError); +} + +TEST_F(Bmi_Mass_Balance_Test, frequency_zero) { + auto properties = MassBalanceMock(true, 1e-5, 0).as_json_property(); + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + double mass_error = 10; // Force a mass balance error above tolerance + model->SetValue(OUTPUT_MASS_NAME, &mass_error); + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(0, 2, time, model_name)); + EXPECT_TRUE( result.has_value() ); // should pass, frequency 0 means never check +} + +TEST_F(Bmi_Mass_Balance_Test, frequency_end) { + auto properties = MassBalanceMock(true, 1e-5, -1).as_json_property(); + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + double mass_error = 10; // Force a mass balance error above tolerance + model->SetValue(OUTPUT_MASS_NAME, &mass_error); + //Check initial mass balance -- should not error due to frequency setting + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(0, 2, time, model_name)); + EXPECT_TRUE( result.has_value() ); // should pass + time = "t1"; + model->Update(); // advance model + model->SetValue(OUTPUT_MASS_NAME, &mass_error); + // Call mass balance check again, this should NOT error, since the actual check + // should be skipped due to the frequency setting + result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)); + EXPECT_TRUE( result.has_value() ); // should pass + time = "t2"; + model->Update(); // advance model + model->SetValue(OUTPUT_MASS_NAME, &mass_error); + // Check mass balance again, this SHOULD error since its the last timestep and the previous mass balance + // will propagate, and it should now be checked based on the frequency + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(2, 2, time, model_name)), ProtocolError); +} + +TEST_F(Bmi_Mass_Balance_Test, nan) { + auto properties = MassBalanceMock(true, "NaN").as_json_property(); + auto context = make_context(0, 2, time, model_name); + testing::internal::CaptureStderr(); + auto protocols = NgenBmiProtocols(model, properties); + std::string output = testing::internal::GetCapturedStderr(); + // std::cerr << output; + EXPECT_THAT(output, testing::HasSubstr("Warning(Protocol)::mass_balance: tolerance value 'NaN'")); + double mass_error = 10; + model->SetValue(OUTPUT_MASS_NAME, &mass_error); // + // Would cause an error if tolerance were a number, should only see warning in the output above + // and this should pass without error... + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(0, 1, time, model_name)); + EXPECT_TRUE( result.has_value() ); // should pass +} + +TEST_F(Bmi_Mass_Balance_Test, model_nan) { + auto properties = MassBalanceMock(true).as_json_property(); + auto context = make_context(0, 2, time, model_name); + auto protocols = NgenBmiProtocols(model, properties); + auto result = protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, context); + EXPECT_TRUE( result.has_value() ); // should pass + double mass_error = std::numeric_limits::quiet_NaN(); + model->SetValue(OUTPUT_MASS_NAME, &mass_error); // Force a NaN into the mass balance computation + time = "t1"; + // should cause an error since mass balance will be NaN using this value in its computation + ASSERT_THROW(protocols.run(models::bmi::protocols::Protocol::MASS_BALANCE, make_context(1, 2, time, model_name)), ProtocolError); +} + +#endif // NGEN_BMI_CPP_LIB_TESTS_ACTIVE \ No newline at end of file From 60f2ebf4509afdaa4503fa3c8122ff616a2abf1b Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 11:50:12 -0500 Subject: [PATCH 74/93] Add logging library for access to Logger.hpp --- src/utilities/bmi/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utilities/bmi/CMakeLists.txt b/src/utilities/bmi/CMakeLists.txt index 67a62eaf8d..5d86e87f70 100644 --- a/src/utilities/bmi/CMakeLists.txt +++ b/src/utilities/bmi/CMakeLists.txt @@ -28,6 +28,7 @@ target_link_libraries(ngen_bmi_protocols PUBLIC ${CMAKE_DL_LIBS} Boost::boost # Headers-only Boost + NGen::logging ) target_sources(ngen_bmi_protocols From ebc42f81dd6c13e8aacafc9ba93bd29d26c34014 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 12:58:33 -0500 Subject: [PATCH 75/93] Update Boost version requirement --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 01d1851ab1..b3ff2e483e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ ENV LANG="C.UTF-8" \ HDF5_VERSION="1.10.11" \ NETCDF_C_VERSION="4.7.4" \ NETCDF_FORTRAN_VERSION="4.5.4" \ - BOOST_VERSION="1.83.0" + BOOST_VERSION="1.86.0" # runtime dependencies RUN set -eux && \ From 2a439e1cdcffc7fd6f8269ddaf6ca81e8d9841c6 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 13:04:36 -0500 Subject: [PATCH 76/93] Fix download URL to Boost 1.86.0 --- docker/CENTOS_TEST.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/CENTOS_TEST.dockerfile b/docker/CENTOS_TEST.dockerfile index 334ac5d5d7..418b28b086 100644 --- a/docker/CENTOS_TEST.dockerfile +++ b/docker/CENTOS_TEST.dockerfile @@ -11,7 +11,7 @@ ENV CXX=/usr/bin/g++ RUN git submodule update --init --recursive -- test/googletest -RUN curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_86_0.tar.bz2/download +RUN curl -L -o boost_1_86_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download RUN tar -xjf boost_1_86_0.tar.bz2 From e2f414cf0884e5652829d600472ca97b4b7a43e3 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 17 Feb 2026 17:45:10 -0800 Subject: [PATCH 77/93] fixing boost version in code scan --- .github/workflows/ngwpc-cicd.yml | 45 +++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index a02ad7048a..86809ac19f 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -130,7 +130,7 @@ jobs: - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y cmake g++ mpi-default-bin libopenmpi-dev libboost-all-dev libudunits2-dev libnetcdf-dev libnetcdf-c++4-dev + sudo apt-get install -y cmake g++ mpi-default-bin libopenmpi-dev libudunits2-dev libnetcdf-dev libnetcdf-c++4-dev python3 -m pip install numpy==1.26.4 netcdf4 bmipy pandas torch pyyaml pyarrow #python3 -m pip install -r extern/test_bmi_py/requirements.txt #python3 -m pip install -r extern/t-route/requirements.txt @@ -139,12 +139,47 @@ jobs: uses: github/codeql-action/init@v4 with: languages: cpp - # Build (replicate your CMake build commands from Dockerfile or local builds) + - name: Install Boost 1.86.0 + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential curl ca-certificates bzip2 python3-dev + + BOOST_VER=1.86.0 + BOOST_UNDERSCORE=1_86_0 + PREFIX=/opt/boost-${BOOST_VER} + + curl -fL --retry 5 --retry-delay 2 \ + -o /tmp/boost.tar.bz2 \ + "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/boost_${BOOST_UNDERSCORE}.tar.bz2" + + tar -xjf /tmp/boost.tar.bz2 -C /tmp + cd "/tmp/boost_${BOOST_UNDERSCORE}" + + ./bootstrap.sh --prefix="${PREFIX}" + + # Build only common compiled libs (adjust if you need more) + ./b2 -j"$(nproc)" install \ + --with-system \ + --with-filesystem \ + --with-program_options \ + --with-thread \ + --with-regex \ + --with-date_time + + echo "BOOST_ROOT=${PREFIX}" >> "$GITHUB_ENV" + echo "CMAKE_PREFIX_PATH=${PREFIX}:${CMAKE_PREFIX_PATH:-}" >> "$GITHUB_ENV" + - name: Build C++ code env: PYTHONPATH: ${{ env.PYTHONPATH }} run: | cmake -B cmake_build -S . \ + -DCMAKE_PREFIX_PATH="${BOOST_ROOT}" \ + -DBoost_NO_SYSTEM_PATHS=ON \ + -DBOOST_ROOT="${BOOST_ROOT}" \ -DPYTHON_EXECUTABLE=$(which python3) \ -DNGEN_WITH_MPI=ON \ -DNGEN_WITH_NETCDF=ON \ @@ -156,9 +191,7 @@ jobs: -DNGEN_WITH_TESTS=OFF \ -DNGEN_WITH_ROUTING=ON \ -DNGEN_QUIET=ON \ - -DNGEN_UPDATE_GIT_SUBMODULES=OFF \ - # Using boost from apt for simplicity and code scanning - #-DBOOST_ROOT=/opt/boost + -DNGEN_UPDATE_GIT_SUBMODULES=OFF cmake --build cmake_build --target all - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 @@ -312,7 +345,7 @@ jobs: "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}" # tag with 'latest' on development branch push - if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF_NAME" = "development" ]; then + if [ "$GITHUB_REF_NAME" = "development" ]; then skopeo copy \ --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ From f05800c2c074fe452fff18d4d48a98f9aa5ac88c Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 17 Feb 2026 18:07:29 -0800 Subject: [PATCH 78/93] fixing boost version in code scan --- .github/workflows/ngwpc-cicd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 86809ac19f..f7e3f05194 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -139,7 +139,7 @@ jobs: uses: github/codeql-action/init@v4 with: languages: cpp - - name: Install Boost 1.86.0 + - name: Install Boost shell: bash run: | set -euo pipefail @@ -151,9 +151,9 @@ jobs: BOOST_UNDERSCORE=1_86_0 PREFIX=/opt/boost-${BOOST_VER} - curl -fL --retry 5 --retry-delay 2 \ + curl -fL --retry 10 --retry-delay 2 --max-time 600 \ -o /tmp/boost.tar.bz2 \ - "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/boost_${BOOST_UNDERSCORE}.tar.bz2" + "https://sourceforge.net/projects/boost/files/boost/${BOOST_VER}/boost_${BOOST_UNDERSCORE}.tar.bz2/download" tar -xjf /tmp/boost.tar.bz2 -C /tmp cd "/tmp/boost_${BOOST_UNDERSCORE}" From 9e342baef694942f773ae0190c6558cf7e18e25a Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 17 Feb 2026 19:17:11 -0800 Subject: [PATCH 79/93] fixing boost version in code scan --- .github/workflows/ngwpc-cicd.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index f7e3f05194..a921cdeeb9 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -167,7 +167,8 @@ jobs: --with-program_options \ --with-thread \ --with-regex \ - --with-date_time + --with-date_time \ + --with-serialization echo "BOOST_ROOT=${PREFIX}" >> "$GITHUB_ENV" echo "CMAKE_PREFIX_PATH=${PREFIX}:${CMAKE_PREFIX_PATH:-}" >> "$GITHUB_ENV" From 163c12a3f6ad7d58f389cfb131330491a0a73200 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 17 Feb 2026 19:28:04 -0800 Subject: [PATCH 80/93] cicd updates --- .github/workflows/ngwpc-cicd.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index a921cdeeb9..ba166cba2c 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -160,7 +160,7 @@ jobs: ./bootstrap.sh --prefix="${PREFIX}" - # Build only common compiled libs (adjust if you need more) + # Build Boost libraries ./b2 -j"$(nproc)" install \ --with-system \ --with-filesystem \ @@ -169,7 +169,6 @@ jobs: --with-regex \ --with-date_time \ --with-serialization - echo "BOOST_ROOT=${PREFIX}" >> "$GITHUB_ENV" echo "CMAKE_PREFIX_PATH=${PREFIX}:${CMAKE_PREFIX_PATH:-}" >> "$GITHUB_ENV" From bf87b7c061b6a631c96626b342d8a0faa7ff73e3 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 17 Feb 2026 20:19:57 -0800 Subject: [PATCH 81/93] cicd updates --- .github/workflows/ngwpc-cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index ba166cba2c..2e293bef98 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -344,7 +344,7 @@ jobs: --all \ "docker://${IMAGE_BASE}:${TEST_TAG}" "docker://${IMAGE_BASE}:${ALIAS_TAG}" - # tag with 'latest' on development branch push + # tag with 'latest' on development branch if [ "$GITHUB_REF_NAME" = "development" ]; then skopeo copy \ --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ From e8f63bbc68e4d9c6f067773e70605e2767891f52 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Tue, 25 Nov 2025 15:55:21 -0800 Subject: [PATCH 82/93] Flesh out API for state saving and restoring, with adjusted use in Bmi_Module_Formulation --- include/realizations/catchment/Bmi_Multi_Formulation.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 9877d253a2..f14cae0b6a 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -63,6 +63,8 @@ namespace realization { } }; + void save_state(std::shared_ptr saver) const override; + /** * Convert a time value from the model to an epoch time in seconds. * From 5b335da07ef5569735b5b075c404d76809f25bf6 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Mon, 8 Dec 2025 17:53:19 -0800 Subject: [PATCH 83/93] Add logic and structures for parsing state saving configuration from realization config --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 19d24f30d4..8ba84add7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -345,6 +345,7 @@ target_link_libraries(ngen NGen::parallel NGen::state_save_restore NGen::bmi_protocols + NGen::state_save_restore ) if(NGEN_WITH_SQLITE) From db2f43275170ea5fadcca93d6df731ec04502e6b Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 16 Jan 2026 14:06:13 -0500 Subject: [PATCH 84/93] State saving for multi-BMI --- .../catchment/Bmi_Module_Formulation.hpp | 13 +++++++++++++ src/NGen.cpp | 10 ++++++++++ .../catchment/Bmi_Multi_Formulation.cpp | 4 ---- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 150bd2ac38..bcc1f251e1 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -57,6 +57,19 @@ namespace realization { void load_hot_start(std::shared_ptr loader) override; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * + * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. + * @return Pointer to the beginning of the serialized data. + */ + virtual const char* create_save_state(uint64_t *size) const; + + /** + * Clears any serialized data stored by the BMI from memory. + */ + virtual void free_save_state() const; + /** * Get the collection of forcing output property names this instance can provide. * diff --git a/src/NGen.cpp b/src/NGen.cpp index 79946fd64f..6e1fc3014e 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -710,6 +710,16 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { simulation->run_catchments(); + if (state_saving_config.has_end_of_run()) { + LOG("Saving end-of-run state.", LogLevel::INFO); + std::shared_ptr saver = state_saving_config.end_of_run_saver(); + std::shared_ptr snapshot = saver->initialize_snapshot( + State_Saver::snapshot_time_now(), + State_Saver::State_Durability::strict + ); + simulation->save_state_snapshot(snapshot); + } + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 290c475cba..1f4fbbf245 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -21,10 +21,6 @@ #include #include -#if (__cplusplus >= 202002L) -#include -#endif - using namespace realization; From 005aa3f0af81a6feb7ba98285e4b7cefab3436f2 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 6 Feb 2026 10:26:57 -0500 Subject: [PATCH 85/93] Use Boost for serializing Multi-BMI --- include/realizations/catchment/Bmi_Multi_Formulation.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index f14cae0b6a..6414b20119 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -63,7 +63,11 @@ namespace realization { } }; - void save_state(std::shared_ptr saver) const override; + void save_state(std::shared_ptr saver) override; + + void load_state(std::shared_ptr loader) override; + + void load_hot_start(std::shared_ptr loader) override; /** * Convert a time value from the model to an epoch time in seconds. From 3189c31867dcea37c319c53a613e99842f421de7 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 14:35:01 -0500 Subject: [PATCH 86/93] Remove duplicate declaration --- include/realizations/catchment/Bmi_Multi_Formulation.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 6414b20119..20d9158b94 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -48,12 +48,6 @@ namespace realization { virtual ~Bmi_Multi_Formulation() {}; - void save_state(std::shared_ptr saver) override; - - void load_state(std::shared_ptr loader) override; - - void load_hot_start(std::shared_ptr loader) override; - virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const final { for( const auto &module : modules ) { // TODO may need to check on outputs form each module indepdently??? From 9a0a3292ae505162f61c2a8af11a6271f2d6d9be Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 14:39:05 -0500 Subject: [PATCH 87/93] Remove old state calls --- src/NGen.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 6e1fc3014e..79946fd64f 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -710,16 +710,6 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { simulation->run_catchments(); - if (state_saving_config.has_end_of_run()) { - LOG("Saving end-of-run state.", LogLevel::INFO); - std::shared_ptr saver = state_saving_config.end_of_run_saver(); - std::shared_ptr snapshot = saver->initialize_snapshot( - State_Saver::snapshot_time_now(), - State_Saver::State_Durability::strict - ); - simulation->save_state_snapshot(snapshot); - } - #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif From 245c1c0867c8cbba8945e03d33937d7ebffa5e5c Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 14:44:03 -0500 Subject: [PATCH 88/93] Remove old state saving definitions --- .../catchment/Bmi_Module_Formulation.hpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index bcc1f251e1..150bd2ac38 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -57,19 +57,6 @@ namespace realization { void load_hot_start(std::shared_ptr loader) override; - /** - * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. - * - * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. - * @return Pointer to the beginning of the serialized data. - */ - virtual const char* create_save_state(uint64_t *size) const; - - /** - * Clears any serialized data stored by the BMI from memory. - */ - virtual void free_save_state() const; - /** * Get the collection of forcing output property names this instance can provide. * From ac0cc3042a832de990757d4c44c7c60b881b3322 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 15:00:47 -0500 Subject: [PATCH 89/93] Update submodule references for bug fixes --- extern/sac-sma/sac-sma | 2 +- extern/topmodel/topmodel | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/sac-sma/sac-sma b/extern/sac-sma/sac-sma index eef00b537d..dbc938593b 160000 --- a/extern/sac-sma/sac-sma +++ b/extern/sac-sma/sac-sma @@ -1 +1 @@ -Subproject commit eef00b537d556933c7bd645fc5aadcea6b2452c1 +Subproject commit dbc938593b935565f2579ae666849284e65f369c diff --git a/extern/topmodel/topmodel b/extern/topmodel/topmodel index b35249c195..a776e34874 160000 --- a/extern/topmodel/topmodel +++ b/extern/topmodel/topmodel @@ -1 +1 @@ -Subproject commit b35249c1954abee61885bbd75d8b1765b5a311a5 +Subproject commit a776e3487499e6163ea8c423abb4dd382c66e480 From 3f1d39672de7e81ee99350eb8381f2b88e6bebbb Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 15:36:39 -0500 Subject: [PATCH 90/93] Update submodules for testing --- extern/lstm | 2 +- extern/snow17 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extern/lstm b/extern/lstm index 72c2ab73d8..650d5ed6a3 160000 --- a/extern/lstm +++ b/extern/lstm @@ -1 +1 @@ -Subproject commit 72c2ab73d80652aac18a3d35b2b5af212eb88122 +Subproject commit 650d5ed6a3986c8fda103a121f6da724cb7220a7 diff --git a/extern/snow17 b/extern/snow17 index 043c625659..369a08a454 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit 043c625659e4235887cb27acf1f1d34e5f87fc9e +Subproject commit 369a08a4544a78063e65c9f00a453d9416c12c52 From 7eee79ff7c7bfce1d8194a358c68fc1f80ec943f Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 19 Feb 2026 08:31:25 -0500 Subject: [PATCH 91/93] Update submodule for snow17 fix --- extern/snow17 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/snow17 b/extern/snow17 index 369a08a454..6d98c4593b 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit 369a08a4544a78063e65c9f00a453d9416c12c52 +Subproject commit 6d98c4593b4d0dc5f14747c77d0149394697d7a2 From a47604ab6b96f30e659d82d2108b4f21628e18be Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 20 Feb 2026 14:45:30 -0500 Subject: [PATCH 92/93] Update submodels for bug fixes --- extern/noah-owp-modular/noah-owp-modular | 2 +- extern/sac-sma/sac-sma | 2 +- extern/snow17 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extern/noah-owp-modular/noah-owp-modular b/extern/noah-owp-modular/noah-owp-modular index dd9260175d..25579b4948 160000 --- a/extern/noah-owp-modular/noah-owp-modular +++ b/extern/noah-owp-modular/noah-owp-modular @@ -1 +1 @@ -Subproject commit dd9260175d2783abe50f6c409885b8e1fd097f5e +Subproject commit 25579b4948e28e5afd0bed3e99e08a806fd9fc7c diff --git a/extern/sac-sma/sac-sma b/extern/sac-sma/sac-sma index dbc938593b..6c7e0fdf31 160000 --- a/extern/sac-sma/sac-sma +++ b/extern/sac-sma/sac-sma @@ -1 +1 @@ -Subproject commit dbc938593b935565f2579ae666849284e65f369c +Subproject commit 6c7e0fdf318dde72a5030c78d76c85c578ab6cac diff --git a/extern/snow17 b/extern/snow17 index 6d98c4593b..b9578bb902 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit 6d98c4593b4d0dc5f14747c77d0149394697d7a2 +Subproject commit b9578bb902cf1e38107824654849a67542295d03 From 7cad804b846fbc581ca69235fb6b2746942f55d5 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 20 Feb 2026 15:05:15 -0500 Subject: [PATCH 93/93] Fortran packages report byte size of data --- extern/noah-owp-modular/noah-owp-modular | 2 +- extern/sac-sma/sac-sma | 2 +- extern/snow17 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extern/noah-owp-modular/noah-owp-modular b/extern/noah-owp-modular/noah-owp-modular index 25579b4948..f2d074b3ad 160000 --- a/extern/noah-owp-modular/noah-owp-modular +++ b/extern/noah-owp-modular/noah-owp-modular @@ -1 +1 @@ -Subproject commit 25579b4948e28e5afd0bed3e99e08a806fd9fc7c +Subproject commit f2d074b3adfe0fabb7784d4d20b06081da233c08 diff --git a/extern/sac-sma/sac-sma b/extern/sac-sma/sac-sma index 6c7e0fdf31..39715f7522 160000 --- a/extern/sac-sma/sac-sma +++ b/extern/sac-sma/sac-sma @@ -1 +1 @@ -Subproject commit 6c7e0fdf318dde72a5030c78d76c85c578ab6cac +Subproject commit 39715f75222abfd3b6ce63606a1ed46aea41e6b9 diff --git a/extern/snow17 b/extern/snow17 index b9578bb902..c591c0c05c 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit b9578bb902cf1e38107824654849a67542295d03 +Subproject commit c591c0c05c042a4592e8cefb7bf9ab8598571f3f