From a23abfca057934d6538f63b9ca01761fc89f3b0f Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 18 Dec 2025 12:50:40 -0800 Subject: [PATCH 1/8] Update logger to use named logger. Convert 'logger.' log statements to 'LOG.' to be consistent across EWTS loggers. --- lstm/bmi_lstm.py | 37 +++++------ lstm/logger.py | 142 +++++++++++++++++++++---------------------- lstm/run_lstm_bmi.py | 23 ++++--- 3 files changed, 102 insertions(+), 100 deletions(-) diff --git a/lstm/bmi_lstm.py b/lstm/bmi_lstm.py index 7e2697f..ae1fd62 100644 --- a/lstm/bmi_lstm.py +++ b/lstm/bmi_lstm.py @@ -62,9 +62,12 @@ from . import nextgen_cuda_lstm from .base import BmiBase -from .logger import configure_logging, logger +from .logger import configure_logging, MODULE_NAME from .model_state import State, StateFacade, Var +import logging +LOG = logging.getLogger(MODULE_NAME) + # -------------- Dynamic Attributes ----------------------------- _dynamic_input_vars = [ ("land_surface_radiation~incoming~longwave__energy_flux", "W m-2"), @@ -282,7 +285,7 @@ def initialize_lstm(cfg: dict[str, typing.Any]) -> nextgen_cuda_lstm.Nextgen_Cud def gather_inputs( state: Valuer, internal_input_names: typing.Iterable[str] ) -> npt.NDArray: - logger.debug("Collecting LSTM inputs ...") + LOG.debug("Collecting LSTM inputs ...") input_list = [] for lstm_name in internal_input_names: @@ -291,29 +294,29 @@ def gather_inputs( assert value.size == 1, "`value` should a single scalar in a 1d array" input_list.append(value[0]) - logger.debug(f" {lstm_name=}") - logger.debug(f" {bmi_name=}") - logger.debug(f" {type(value)=}") - logger.debug(f" {value=}") + LOG.debug(f" {lstm_name=}") + LOG.debug(f" {bmi_name=}") + LOG.debug(f" {type(value)=}") + LOG.debug(f" {value=}") collected = bmi_array(input_list) - logger.debug(f"Collected inputs: {collected}") + LOG.debug(f"Collected inputs: {collected}") return collected def scale_inputs( input: npt.NDArray, mean: npt.NDArray, std: npt.NDArray ) -> npt.NDArray: - logger.debug("Normalizing the tensor...") - logger.debug(" input_mean =", mean) - logger.debug(" input_std =", std) + LOG.debug("Normalizing the tensor...") + LOG.debug(" input_mean =", mean) + LOG.debug(" input_std =", std) # Center and scale the input values for use in torch input_array_scaled = (input - mean) / std - logger.debug(f"### input_array ={input}") - logger.debug(f"### dtype(input_array) ={input.dtype}") - logger.debug(f"### type(input_array_scaled) ={type(input_array_scaled)}") - logger.debug(f"### dtype(input_array_scaled) ={input_array_scaled.dtype}") + LOG.debug(f"### input_array ={input}") + LOG.debug(f"### dtype(input_array) ={input.dtype}") + LOG.debug(f"### type(input_array_scaled) ={type(input_array_scaled)}") + LOG.debug(f"### dtype(input_array_scaled) ={input_array_scaled.dtype}") return input_array_scaled @@ -324,7 +327,7 @@ def scale_outputs( output_std: npt.NDArray, output_scale_factor_cms: float, ): - logger.debug(f"model output: {output[0, 0, 0].numpy().tolist()}") + LOG.debug(f"model output: {output[0, 0, 0].numpy().tolist()}") if cfg["target_variables"][0] in ["qobs_mm_per_hour", "QObs(mm/hr)", "QObs(mm/h)"]: surface_runoff_mm = output[0, 0, 0].numpy() * output_std + output_mean @@ -458,7 +461,7 @@ def update(self) -> None: def update_until(self, time: float) -> None: if time <= self.get_current_time(): current_time = self.get_current_time() - logger.warning(f"no update performed: {time=} <= {current_time=}") + LOG.warning(f"no update performed: {time=} <= {current_time=}") return None n_steps, remainder = divmod( @@ -466,7 +469,7 @@ def update_until(self, time: float) -> None: ) if remainder != 0: - logger.warning( + LOG.warning( f"time is not multiple of time step size. updating until: {time - remainder=} " ) diff --git a/lstm/logger.py b/lstm/logger.py index 75c894e..a5cb8c4 100644 --- a/lstm/logger.py +++ b/lstm/logger.py @@ -1,20 +1,13 @@ -# -# Copyright (C) 2025 Austin Raney, Lynker -# -# Author: Austin Raney -# +"""Logging module to integrate with NGEN""" + from __future__ import annotations +import getpass import logging - -logger = logging.getLogger() -_configured = False - -import sys -from datetime import datetime, timezone import os +import sys import time -import getpass +from datetime import datetime, timezone MODULE_NAME = "LSTM"; LOG_DIR_NGENCERF = "/ngencerf/data"; # ngenCERF log directory string if environement var empty. @@ -29,6 +22,8 @@ EV_MODULE_LOGFILEPATH = "LSTM_LOGFILEPATH"; # This modules log full log filename class CustomFormatter(logging.Formatter): + """A custom formatting class for logging""" + LEVEL_NAME_MAP = { logging.DEBUG: "DEBUG", logging.INFO: "INFO", @@ -36,6 +31,21 @@ class CustomFormatter(logging.Formatter): logging.ERROR: "SEVERE", logging.CRITICAL: "FATAL" } + + # Apply custom formatter (UTC timestamps applied only to this formatter) + def converter(self, timestamp): + """Override time converter to return UTC time tuple""" + return time.gmtime(timestamp) + + def formatTime(self, record, datefmt=None): + """Use our UTC converter""" + ct = self.converter(record.created) + if datefmt: + s = time.strftime(datefmt, ct) + else: + t = time.strftime("%Y-%m-%d %H:%M:%S", ct) + s = f"{t},{int(record.msecs):03d}" + return s def format(self, record): original_levelname = record.levelname @@ -73,6 +83,7 @@ def get_log_file_path(): if ngenEnvVar: logFilePath = ngenEnvVar else: + print(f"Module {MODULE_NAME} Env var {EV_NGEN_LOGFILEPATH} not found. Creating default log name.") appendEntries = False if os.path.isdir(LOG_DIR_NGENCERF): logFileDir = LOG_DIR_NGENCERF + DS + LOG_DIR_DEFAULT @@ -83,12 +94,12 @@ def get_log_file_path(): # Set full log path username = getpass.getuser() if username: - logFieDir = logFieDir + DS + username + logFileDir = logFileDir + DS + username else: logFileDir = logFileDir + DS + create_timestamp(True) # Create directory - with os.makedirs(logFileDir, exist_ok=True): - logFilePath = logFileDir + DS + MODULE_NAME + "_" + create_timestamp() + "." + LOG_FILE_EXT + os.makedirs(logFileDir, exist_ok=True) + logFilePath = logFileDir + DS + MODULE_NAME + "_" + create_timestamp() + "." + LOG_FILE_EXT except Exception as e: logFilePath = "" @@ -105,18 +116,16 @@ def get_log_file_path(): else: raise IOError except: - print(f"Unable to open log file for {MODULE_NAME}: {logFilePath}", flush=True) - print(f"Log entries will be writen to stdout", flush=True) + print(f"Module {MODULE_NAME} Unable to open log file: {logFilePath}", flush=True) + print(f"Module {MODULE_NAME} Log entries will be writen to stdout", flush=True) return logFilePath, appendEntries def get_log_level() -> str: levelEnvVar = os.getenv(EV_MODULE_LOGLEVEL, "") if levelEnvVar: - print(f"{EV_MODULE_LOGLEVEL}={levelEnvVar}", flush=True) return levelEnvVar.strip().upper() else: - print(f"{EV_MODULE_LOGLEVEL} not found. Using INFO log level", flush=True) return "INFO" def translate_ngwpc_log_level(ngwpc_log_level: str) -> str: @@ -126,16 +135,27 @@ def translate_ngwpc_log_level(ngwpc_log_level: str) -> str: elif (ll == "FATAL"): return "CRITICAL" return ll - + +def force_info(handler, logger, msg, *args): + record = logger.makeRecord( + logger.name, + logging.INFO, + __file__, + 0, + msg, + args, + None, + ) + handler.emit(record) + def configure_logging(): ''' Set logging level and specify logger configuration based on environment variables set by ngen Arguments --------- - logging._Level: Log level - ** Not used in NGWPC version. Instead the ngen logger defines environment variables that are read - + none + Returns ------- None @@ -152,83 +172,59 @@ def configure_logging(): it is only opened once for each ngen run (vs for each catchment) See also https://docs.python.org/3/library/logging.html - + ''' - global _configured # Tell Python this refers to the module-level variable - modulePathEnvVarSet = os.getenv(EV_MODULE_LOGFILEPATH, "") - if modulePathEnvVarSet and _configured: - return # Nothing to do — already configured, and env var is set - elif not modulePathEnvVarSet and _configured: - # Need set a log file since the ngen.log was truncated. - logFilePath, appendEntries = get_log_file_path() - if (logFilePath): - # Set the open mode - openMode = 'a' if appendEntries else 'w' - handler = logging.FileHandler(logFilePath, mode=openMode) - else: - handler = logging.StreamHandler(sys.stdout) - return + + # Use a named logger to ensure entries are identified as this + # MODULE_NAME and are not miss-identfied in the ngen log. + logger = logging.getLogger(MODULE_NAME) + if getattr(logger, "_initialized", False): + return # logger already initialized, nothing else to do loggingEnabled = True moduleEnvVar = os.getenv(EV_EWTS_LOGGING, "") if moduleEnvVar: - print(f"{EV_EWTS_LOGGING}={moduleEnvVar}", flush=True) if (moduleEnvVar == "DISABLED"): loggingEnabled = False else: - print(f"{EV_EWTS_LOGGING} not found.", flush=True) - + print(f"Module {MODULE_NAME} Env var {EV_EWTS_LOGGING} not found. Using logging defaults.") + if (loggingEnabled == False): print(f"Module {MODULE_NAME} Logging DISABLED", flush=True) - logging.disable(logging.CRITICAL) # Disables all logs at CRITICAL and below (i.e., everything) + logger.disabled = True # Disables all logs at CRITICAL and below (i.e., everything) else: print(f"Module {MODULE_NAME} Logging ENABLED", flush=True) - - # Get the log file name from env var or a default + + # Get the log file name from env var or a default logFilePath, appendEntries = get_log_file_path() if (logFilePath): # Set the open mode openMode = 'a' if appendEntries else 'w' handler = logging.FileHandler(logFilePath, mode=openMode) else: + print(f"Module {MODULE_NAME} unable to create log file. Using stdout.") handler = logging.StreamHandler(sys.stdout) - + # Get the log level from env var or a default log_level = get_log_level() - + # Format the module name: uppercase, fixed length, left-justify or trimmed formatted_module = MODULE_NAME.upper().ljust(LOG_MODULE_NAME_LEN)[:LOG_MODULE_NAME_LEN] - + # Apply custom formatter - formatted_module = MODULE_NAME.upper().ljust(LOG_MODULE_NAME_LEN)[:LOG_MODULE_NAME_LEN] formatter = CustomFormatter( fmt=f"%(asctime)s.%(msecs)03d {formatted_module} %(levelname_padded)s %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" ) handler.setFormatter(formatter) - - # Setup root logger - logging.getLogger().handlers.clear() # Clear any default handlers - logging.getLogger().setLevel(translate_ngwpc_log_level(log_level)) - logging.getLogger().addHandler(handler) - - # Ensure UTC timestamps - logging.Formatter.converter = time.gmtime - - # Save the current log level - current_level = logging.getLogger().getEffectiveLevel() - - try: - # Temporarily set log level to INFO - logging.getLogger().setLevel(logging.INFO) - - # Log the message at INFO level - logging.info(f"Log level set to {log_level}") - print(f"Module {MODULE_NAME} Log Level set to {log_level}", flush=True) - finally: - # Restore the original log level - logging.getLogger().setLevel(current_level) - - # Set this true so the logger is only configured once - _configured = True + # Setup logger + logger.handlers.clear() # Clear any default handlers + logger.setLevel(translate_ngwpc_log_level(log_level)) + logger.addHandler(handler) + + # Write log level INFO message to log regradless of the actual log level + force_info(handler, logger, "Log level set to %s", log_level) + print(f"Module {MODULE_NAME} Log Level set to {log_level}", flush=True) + + logger._initialized = True diff --git a/lstm/run_lstm_bmi.py b/lstm/run_lstm_bmi.py index 07214d4..0fa7986 100644 --- a/lstm/run_lstm_bmi.py +++ b/lstm/run_lstm_bmi.py @@ -7,7 +7,10 @@ from netCDF4 import Dataset # This is the BMI LSTM that we will be running import bmi_lstm -from .logger import logger +from .logger import MODULE_NAME + +import logging +LOG = logging.getLogger(MODULE_NAME) # Define primary bmi config and input data file paths #bmi_cfg_file=Path('./bmi_config_files/01022500_hourly_all_attributes_forcings.yml') @@ -17,19 +20,19 @@ sample_data_file = run_dir + 'data/usgs-streamflow-nldas_hourly.nc' # creating an instance of an LSTM model -logger.debug('Creating an instance of an BMI_LSTM model object') +LOG.debug('Creating an instance of an BMI_LSTM model object') model = bmi_lstm.bmi_LSTM() # Initializing the BMI -logger.debug('Initializing the BMI') +LOG.debug('Initializing the BMI') model.initialize(bmi_cfg_file) # Get input data that matches the LSTM test runs -logger.debug('Gathering input data') +LOG.debug('Gathering input data') sample_data = Dataset(sample_data_file, 'r') # Now loop through the inputs, set the forcing values, and update the model -logger.debug('Set values & update model for number of timesteps = 100') +LOG.debug('Set values & update model for number of timesteps = 100') for precip, temp in zip(list(sample_data['total_precipitation'][3].data), list(sample_data['temperature'][3].data)): @@ -44,8 +47,8 @@ #model.get_value('atmosphere_water__liquid_equivalent_precipitation_rate', dest_array) #precips = dest_array[0] - #logger.debug(' Temperature and precipitation are set to {:.2f} and {:.2f}'.format(temperature, precip)) - logger.debug(' Temperature and precipitation are set to {:.2f} and {:.2f}'.format(temp, precip)) + #LOG.debug(' Temperature and precipitation are set to {:.2f} and {:.2f}'.format(temperature, precip)) + LOG.debug(' Temperature and precipitation are set to {:.2f} and {:.2f}'.format(temp, precip)) #model.update_until(model.t+model._time_step_size) model.update() @@ -53,12 +56,12 @@ model.get_value('land_surface_water__runoff_volume_flux', dest_array) runoff = dest_array[0] - logger.debug(' Streamflow (cms) at time {} ({}) is {:.2f}'.format(model.get_current_time(), model.get_time_units(), runoff)) + LOG.debug(' Streamflow (cms) at time {} ({}) is {:.2f}'.format(model.get_current_time(), model.get_time_units(), runoff)) if model.t > 100: - #logger.debug('Stopping the loop') + #LOG.debug('Stopping the loop') break # Finalizing the BMI -logger.debug('Finalizing the BMI') +LOG.debug('Finalizing the BMI') model.finalize() From 64fc2bbd7ee8ef6f4234dd392490ae5823287d76 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 18 Dec 2025 13:05:45 -0800 Subject: [PATCH 2/8] Add initializing info log msg --- lstm/bmi_lstm.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lstm/bmi_lstm.py b/lstm/bmi_lstm.py index ae1fd62..d93533f 100644 --- a/lstm/bmi_lstm.py +++ b/lstm/bmi_lstm.py @@ -406,16 +406,17 @@ def __init__(self) -> None: self.ensemble_members: list[EnsembleMember] def initialize(self, config_file: str) -> None: + + # configure the Error Warning and Trapping System logger + configure_logging() + + LOG.info(f"Initializing with {config_file}") + # read and setup main configuration file with open(config_file, "r") as fp: self.cfg_bmi = yaml.safe_load(fp) coerce_config(self.cfg_bmi) - # TODO: aaraney: config logging levels to python logging levels - # setup logging - # self.cfg_bmi["verbose"] - configure_logging() - # ----------- The output is area normalized, this is needed to un-normalize it # mm->m km2 -> m2 hour->s output_factor_cms = ( From 43b1e8f320f6832f27a73d4b2ae03a338956684b Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 18 Dec 2025 13:26:37 -0800 Subject: [PATCH 3/8] Fix debug log statements requiring format specification --- lstm/bmi_lstm.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lstm/bmi_lstm.py b/lstm/bmi_lstm.py index d93533f..51815dd 100644 --- a/lstm/bmi_lstm.py +++ b/lstm/bmi_lstm.py @@ -300,7 +300,7 @@ def gather_inputs( LOG.debug(f" {value=}") collected = bmi_array(input_list) - LOG.debug(f"Collected inputs: {collected}") + LOG.debug(f"Collected inputs: %s", collected) return collected @@ -308,15 +308,15 @@ def scale_inputs( input: npt.NDArray, mean: npt.NDArray, std: npt.NDArray ) -> npt.NDArray: LOG.debug("Normalizing the tensor...") - LOG.debug(" input_mean =", mean) - LOG.debug(" input_std =", std) + LOG.debug(" input_mean = %s", mean) + LOG.debug(" input_std = %s", std) # Center and scale the input values for use in torch input_array_scaled = (input - mean) / std - LOG.debug(f"### input_array ={input}") - LOG.debug(f"### dtype(input_array) ={input.dtype}") - LOG.debug(f"### type(input_array_scaled) ={type(input_array_scaled)}") - LOG.debug(f"### dtype(input_array_scaled) ={input_array_scaled.dtype}") + LOG.debug("### input_array = %s", input) + LOG.debug("### dtype(input_array) = %s", input.dtype) + LOG.debug("### type(input_array_scaled) = %s", type(input_array_scaled)) + LOG.debug("### dtype(input_array_scaled) = %s", input_array_scaled.dtype) return input_array_scaled From 4915189b6c0bdd97ac378b7c002c55dce5271666 Mon Sep 17 00:00:00 2001 From: "jeff.wade" Date: Tue, 6 Jan 2026 13:48:21 -0500 Subject: [PATCH 4/8] Add precipitation as an output variable --- lstm/bmi_lstm.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lstm/bmi_lstm.py b/lstm/bmi_lstm.py index 51815dd..8c85d21 100644 --- a/lstm/bmi_lstm.py +++ b/lstm/bmi_lstm.py @@ -88,6 +88,7 @@ _output_vars = [ ("land_surface_water__runoff_volume_flux", "m3 s-1"), ("land_surface_water__runoff_depth", "m"), + ("precipitation_rate", "mm s-1"), ] # -------------- Name Mappings ----------------------------- @@ -176,6 +177,9 @@ def update(self, state: Valuer) -> typing.Iterable[Var]: with torch.no_grad(): inputs = gather_inputs(state, self.input_names) + # Retrieve precipitation value for output + precipitation_mm_h = state.value("atmosphere_water__liquid_equivalent_precipitation_rate") + scaled = scale_inputs( inputs, self.scalars.input_mean, self.scalars.input_std ) @@ -193,6 +197,7 @@ def update(self, state: Valuer) -> typing.Iterable[Var]: self.scalars.output_mean, self.scalars.output_std, self.output_scaling_factor_cms, + precipitation_mm_h ) @@ -326,6 +331,7 @@ def scale_outputs( output_mean: npt.NDArray, output_std: npt.NDArray, output_scale_factor_cms: float, + precipitation_value: npt.NDArray, ): LOG.debug(f"model output: {output[0, 0, 0].numpy().tolist()}") @@ -350,6 +356,9 @@ def scale_outputs( # (1/1000) * (self.cfg_bmi['area_sqkm'] * 1000*1000) * (1/3600) surface_runoff_volume_m3_s = surface_runoff_mm * output_scale_factor_cms + # Convert precipitation for mm/h to mm/s for output + precip_mms = precipitation_value[0] / 3600.0 + # TODO: aaraney: consider making this into a class or closure to avoid so # many small allocations. yield from ( @@ -363,6 +372,11 @@ def scale_outputs( unit="m3 s-1", value=bmi_array([surface_runoff_volume_m3_s]), ), + Var( + name="precipitation_rate", + unit="mm s-1", + value=bmi_array([precip_mms]) + ), ) From d0b38f05165f4f0009b6110abef9766167574233 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 7 Jan 2026 12:41:55 -0800 Subject: [PATCH 5/8] Add lstm_ewts package and unit tests --- lstm_ewts/pyproject.toml | 13 +++ lstm_ewts/src/lstm_ewts/__init__.py | 34 ++++++++ lstm_ewts/src/lstm_ewts/config.py | 126 +++++++++++++++++++++++++++ lstm_ewts/src/lstm_ewts/constants.py | 40 +++++++++ lstm_ewts/src/lstm_ewts/formatter.py | 60 +++++++++++++ lstm_ewts/src/lstm_ewts/paths.py | 115 ++++++++++++++++++++++++ tests/lstm_ewts/conftest.py | 25 ++++++ tests/lstm_ewts/test_config.py | 81 +++++++++++++++++ tests/lstm_ewts/test_constants.py | 10 +++ tests/lstm_ewts/test_formatter.py | 65 ++++++++++++++ tests/lstm_ewts/test_paths.py | 115 ++++++++++++++++++++++++ 11 files changed, 684 insertions(+) create mode 100644 lstm_ewts/pyproject.toml create mode 100644 lstm_ewts/src/lstm_ewts/__init__.py create mode 100644 lstm_ewts/src/lstm_ewts/config.py create mode 100644 lstm_ewts/src/lstm_ewts/constants.py create mode 100644 lstm_ewts/src/lstm_ewts/formatter.py create mode 100644 lstm_ewts/src/lstm_ewts/paths.py create mode 100644 tests/lstm_ewts/conftest.py create mode 100644 tests/lstm_ewts/test_config.py create mode 100644 tests/lstm_ewts/test_constants.py create mode 100644 tests/lstm_ewts/test_formatter.py create mode 100644 tests/lstm_ewts/test_paths.py diff --git a/lstm_ewts/pyproject.toml b/lstm_ewts/pyproject.toml new file mode 100644 index 0000000..aa30740 --- /dev/null +++ b/lstm_ewts/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools>=70"] +build-backend = "setuptools.build_meta" + +[project] +name = "lstm-ewts" +version = "0.1.0" +description = "EWTS helper package for LSTM" +requires-python = ">=3.8" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["lstm_ewts*"] diff --git a/lstm_ewts/src/lstm_ewts/__init__.py b/lstm_ewts/src/lstm_ewts/__init__.py new file mode 100644 index 0000000..7c8a90b --- /dev/null +++ b/lstm_ewts/src/lstm_ewts/__init__.py @@ -0,0 +1,34 @@ +""" +Error Warning and Trapping System (EWTS) Package API + +This package provides a centralized, named logging configuration for the +Error, Warning, and Trapping System used throughout the codebase. + +EWTS configures a single, shared logger in the Python logging framework, +identified by a fixed module name. All modules that participate in EWTS +logging retrieve this logger by name via the standard logging API. + +Logging configuration should be performed once at application startup by +calling configure_logging(). The configuration function is idempotent: +subsequent calls have no effect and will not reconfigure handlers or levels. + +The logger name is exposed to allow any module to obtain the configured +logger without importing internal implementation details. + +Typical usage: + + At application startup: + from lstm_ewts import configure_logging + configure_logging() + + Within other modules: + import logging + from lstm_ewts import MODULE_NAME + + LOG = logging.getLogger(MODULE_NAME) +""" + +from .constants import MODULE_NAME +from .config import configure_logging + +__all__ = ["MODULE_NAME", "configure_logging"] diff --git a/lstm_ewts/src/lstm_ewts/config.py b/lstm_ewts/src/lstm_ewts/config.py new file mode 100644 index 0000000..f5f0267 --- /dev/null +++ b/lstm_ewts/src/lstm_ewts/config.py @@ -0,0 +1,126 @@ +""" +Logging configuration for the Error Warning and Trapping System (EWTS). + +This module defines the centralized logging configuration used by EWTS. +It is responsible for creating and configuring a single, named logger +within the Python logging framework, based on environment variables +provided by the runtime environment (e.g., ngen). + +Logging configuration is performed via configure_logging(), which applies +handlers, formatters, and log levels to the EWTS logger. The configuration +function is idempotent: once the logger has been initialized, subsequent +calls return immediately without modifying the existing configuration. + +Configuration behavior is controlled by environment variables, whose names +are defined in constants.py: + + - EV_EWTS_LOGGING: + Enables or disables EWTS logging. If set to "DISABLED", logging is + disabled entirely for the EWTS logger. If unset, logging is enabled + by default. + + - EV_MODULE_LOGLEVEL: + Specifies the log level for the EWTS logger. Supported values include + standard Python logging levels as well as ngen-style levels (e.g., + "SEVERE", "FATAL"), which are translated to Python equivalents. + +Log output is directed to a file determined by the path-resolution utilities +in paths.py. If a log file cannot be created, logging falls back to stdout. + +This module does not expose logging APIs directly; callers are expected to +retrieve the configured logger by name using logging.getLogger(MODULE_NAME). +""" + +import logging +import sys +import os + +from .constants import ( + MODULE_NAME, + EV_EWTS_LOGGING, + EV_MODULE_LOGLEVEL, + LOG_MODULE_NAME_LEN, +) +from .formatter import CustomFormatter +from .paths import get_log_file_path + +def translate_ngwpc_log_level(level: str) -> str: + level = level.strip().upper() + return { + "SEVERE": "ERROR", + "FATAL": "CRITICAL", + }.get(level, level) + + +def force_info(handler, logger, msg, *args): + record = logger.makeRecord( + logger.name, + logging.INFO, + __file__, + 0, + msg, + args, + None, + ) + handler.emit(record) + + +def configure_logging(): + ''' + Set logging level and specify logger configuration based on environment variables set by ngen + ''' + logger = logging.getLogger(MODULE_NAME) + + if getattr(logger, "_initialized", False): + return logger # logger already initialized, nothing else to do + + # Default to enabled if flag not set or is set to disabled + raw_value = os.getenv(EV_EWTS_LOGGING) + normalized = (raw_value or "").strip().lower() # convert None or "" to "", lowercase for easy comparison + + # Determine if logging is enabled + enabled = normalized != "disabled" + + # Inform user if logging is enabled by default (env not explicitly set to "enabled") + if enabled and normalized not in ("enabled",): + print(f"{EV_EWTS_LOGGING} not explicitly set to 'ENABLED'; logging ENABLED by default", flush=True) + + if not enabled: + logger.disabled = True + logger._initialized = True + print(f"Module {MODULE_NAME} Logging DISABLED", flush=True) + return logger + + print(f"Module {MODULE_NAME} Logging ENABLED", flush=True) + + logFilePath, appendEntries = get_log_file_path() + + handler = ( + logging.FileHandler(logFilePath, mode="a" if appendEntries else "w") + if logFilePath + else logging.StreamHandler(sys.stdout) + ) + + log_level = translate_ngwpc_log_level( + os.getenv(EV_MODULE_LOGLEVEL, "INFO") + ) + + module_fmt = MODULE_NAME.upper().ljust(LOG_MODULE_NAME_LEN)[:LOG_MODULE_NAME_LEN] + + formatter = CustomFormatter( + fmt=f"%(asctime)s.%(msecs)03d {module_fmt} %(levelname_padded)s %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S", + ) + handler.setFormatter(formatter) + + # Setup logger + logger.handlers.clear() # Clear any default handlers + logger.setLevel(log_level) + logger.addHandler(handler) + + # Write log level INFO message to log regradless of the actual log level + force_info(handler, logger, "Log level set to %s", log_level) + print(f"Module {MODULE_NAME} Log Level set to {log_level}", flush=True) + + logger._initialized = True + return logger diff --git a/lstm_ewts/src/lstm_ewts/constants.py b/lstm_ewts/src/lstm_ewts/constants.py new file mode 100644 index 0000000..de196db --- /dev/null +++ b/lstm_ewts/src/lstm_ewts/constants.py @@ -0,0 +1,40 @@ +""" +Constants and configuration keys for the Error Warning and Trapping System (EWTS). + +This module defines all constant values used by EWTS for logging configuration, +environment variable integration, and log file naming. These values represent +the stable interface between EWTS, ngen, and participating Python modules. + +Constants are grouped into two categories: + + 1) Module-specific constants: + Values that uniquely identify the current ngen module, including the + logger name and module-specific environment variables. + + 2) Common constants: + Values shared across ngen modules that control global logging behavior, + filesystem layout, and integration with the ngen runtime environment. + +These constants are intentionally centralized to ensure consistent behavior +across the codebase and to avoid hard-coded strings in implementation logic. +Callers should treat these values as read-only. +""" + + +# Values unique to each ngen module +MODULE_NAME = "LSTM" +EV_MODULE_LOGLEVEL = "LSTM_LOGLEVEL" # This modules log level +EV_MODULE_LOGFILEPATH = "LSTM_LOGFILEPATH" # This modules log full log filename + +# Values common to all ngen modules +EV_NGEN_LOGFILEPATH = "NGEN_LOG_FILE_PATH" # Environment variable name with the log file location typically set by ngen +EV_EWTS_LOGGING = "NGEN_EWTS_LOGGING" # Environment variable name with the enable/disable state for the Error Warning + # and Trapping System typically set by ngen + +DS = "/" # Directory separator +LOG_DIR_DEFAULT = "run-logs" # Default parent log directory string if env var empty & ngencerf doesn't exist +LOG_DIR_NGENCERF = "/ngencerf/data" # ngenCERF log directory string if environement var empty. +LOG_FILE_EXT = "log" # Log file name extension +LOG_MODULE_NAME_LEN = 8 # Width of module name for log entries + + diff --git a/lstm_ewts/src/lstm_ewts/formatter.py b/lstm_ewts/src/lstm_ewts/formatter.py new file mode 100644 index 0000000..f2531c1 --- /dev/null +++ b/lstm_ewts/src/lstm_ewts/formatter.py @@ -0,0 +1,60 @@ +""" +Custom log record formatting for the Error Warning and Trapping System (EWTS). + +This module defines a custom logging formatter used by EWTS to produce +consistent, ngen-compatible log output across all participating modules. + +The formatter applies the following behaviors: + + - Forces all timestamps to UTC, independent of system locale settings. + - Formats timestamps with millisecond precision. + - Maps Python logging levels to ngen-style severity names + (e.g., ERROR → SEVERE, CRITICAL → FATAL). + - Pads and normalizes level names to fixed width for column alignment. + - Strips trailing whitespace and newline characters from log messages. + +The formatter operates entirely within the Python logging framework and does +not modify logger configuration or handler behavior. It is intended to be used +by the EWTS logging configuration layer and not instantiated directly by +application code. +""" + +import logging +import time + +class CustomFormatter(logging.Formatter): + LEVEL_NAME_MAP = { + logging.DEBUG: "DEBUG", + logging.INFO: "INFO", + logging.WARNING: "WARNING", + logging.ERROR: "SEVERE", + logging.CRITICAL: "FATAL" + } + + # Apply custom formatter (UTC timestamps applied only to this formatter) + def converter(self, timestamp): + """Override time converter to return UTC time tuple""" + return time.gmtime(timestamp) + + def formatTime(self, record, datefmt=None): + """Use our UTC converter""" + ct = self.converter(record.created) + if datefmt: + return time.strftime(datefmt, ct) + t = time.strftime("%Y-%m-%d %H:%M:%S", ct) + return f"{t},{int(record.msecs):03d}" + + def format(self, record): + # Strip trailing whitespace/newlines from the message + if record.msg: + record.msg = str(record.msg).rstrip() + + # Map level names + original_levelname = record.levelname + record.levelname = self.LEVEL_NAME_MAP.get(record.levelno, original_levelname) + record.levelname_padded = record.levelname.ljust(7)[:7] # Exactly 7 chars + formatted = super().format(record) + + # Restore original levelname + record.levelname = original_levelname # Restore original in case it's reused + return formatted diff --git a/lstm_ewts/src/lstm_ewts/paths.py b/lstm_ewts/src/lstm_ewts/paths.py new file mode 100644 index 0000000..f896480 --- /dev/null +++ b/lstm_ewts/src/lstm_ewts/paths.py @@ -0,0 +1,115 @@ +""" +Log file path resolution utilities for the Error Warning and Trapping System (EWTS). + +This module provides helper functions for constructing and validating log file +paths used by the EWTS logging configuration. Log file selection follows a +well-defined precedence based on environment variables and runtime availability. + +Log file path precedence: + + 1. If the NGEN-provided log file path is available via the environment variable + defined in EV_NGEN_LOGFILEPATH, use that path. + + 2. Otherwise, create a default, module-specific log file: + 2.1) Create a base log directory under the ngenCERF data directory if it + exists; otherwise fall back to the user's home directory. + 2.2) Create a child directory using the current username if available, + otherwise use the current UTC date (YYYYMMDD). + 2.3) Construct a log filename using the module name and a UTC timestamp. + +The resolved log file path is validated by attempting to open the file. Upon +successful creation or reuse, the full log file path is stored in the +EV_MODULE_LOGFILEPATH environment variable so subsequent calls reuse the same +file. If log file creation fails, entries will be written to stdout. + +This module does not configure loggers directly; it only resolves filesystem +paths and associated metadata required by the logging configuration layer. +""" + +import getpass +import os +from datetime import datetime, timezone + +from .constants import ( + MODULE_NAME, + EV_NGEN_LOGFILEPATH, + EV_MODULE_LOGFILEPATH, + DS, + LOG_DIR_DEFAULT, + LOG_DIR_NGENCERF, + LOG_FILE_EXT, +) + +def create_timestamp(date_only=False, iso=False, append_ms=False): + now = datetime.now(timezone.utc) + + if date_only: + ts = now.strftime("%Y%m%d") + elif iso: + ts = now.strftime("%Y-%m-%dT%H:%M:%S") + else: + ts = now.strftime("%Y%m%dT%H%M%S") + + if append_ms: + ts += f".{now.microsecond // 1000:03d}" + + return ts + +def get_log_file_path(): + # Determine the log file path using the following precedence: + # 1) Use the ngen-provided log file path if available in the NGEN_LOG_FILE_PATH environment variable + # 2) Otherwise, create a default module-specific log file using the module name and a UTC timestamp. + # 2.1) First create a subdirectory under the ngenCERF data directory if available, otherwise the user home directory. + # 2.2) Next create a subdirectory name using the username, if available, otherwise use the YYYYMMDD. + # 2.3) Attempt to open the log file and upon failure, use stdout. + + appendEntries = True + moduleLogFileExists = False + + # Determine if a log file has laready been opened for this module (either the ngen log or default) + moduleEnvVar = os.getenv(EV_MODULE_LOGFILEPATH, "") + if moduleEnvVar: + logFilePath = moduleEnvVar + moduleLogFileExists = True + else: + ngenEnvVar = os.getenv(EV_NGEN_LOGFILEPATH, "") + if ngenEnvVar: + logFilePath = ngenEnvVar + else: + print(f"Module {MODULE_NAME} Env var {EV_NGEN_LOGFILEPATH} not found. Creating default log name.") + appendEntries = False + baseDir = ( + f"{LOG_DIR_NGENCERF}{DS}{LOG_DIR_DEFAULT}" + if os.path.isdir(LOG_DIR_NGENCERF) + else f"{os.path.expanduser('~')}{DS}{LOG_DIR_DEFAULT}" + ) + try: + os.makedirs(baseDir, exist_ok=True) + + childDir = getpass.getuser() or create_timestamp(True) + logFileDir = f"{baseDir}{DS}{childDir}" + os.makedirs(logFileDir, exist_ok=True) + + logFilePath = ( + f"{logFileDir}{DS}{MODULE_NAME}_{create_timestamp()}.{LOG_FILE_EXT}" + ) + except Exception as e: + print(f"Module {MODULE_NAME} {e}", flush=True) + logFilePath = "" + + # Ensure log file can be opened and set module env var + try: + if (logFilePath): + mode = "a" if appendEntries else "w" + with open(logFilePath, mode): + pass + if not moduleLogFileExists: + os.environ[EV_MODULE_LOGFILEPATH] = logFilePath + print(f"Module {MODULE_NAME} Log File: {logFilePath}", flush=True) + else: + raise IOError + except Exception: + print(f"Module {MODULE_NAME} Unable to open log file: {logFilePath}", flush=True) + print(f"Module {MODULE_NAME} Log entries will be writen to stdout", flush=True) + + return logFilePath, appendEntries diff --git a/tests/lstm_ewts/conftest.py b/tests/lstm_ewts/conftest.py new file mode 100644 index 0000000..2d9f623 --- /dev/null +++ b/tests/lstm_ewts/conftest.py @@ -0,0 +1,25 @@ +import logging +import pytest + + +@pytest.fixture +def clean_ewts_env(monkeypatch): + """ + Ensure EWTS-related environment variables are unset and + logging is reset before each test. + """ + # EWTS / module env vars + monkeypatch.delenv("NGEN_LOG_FILE_PATH", raising=False) + monkeypatch.delenv("LSTM_LOGLEVEL", raising=False) + monkeypatch.delenv("LSTM_LOGFILEPATH", raising=False) + monkeypatch.delenv("NGEN_EWTS_LOGGING", raising=False) + + # Reset logging state (important!) + logging.shutdown() + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + + yield + + # Cleanup after test (defensive) + logging.shutdown() diff --git a/tests/lstm_ewts/test_config.py b/tests/lstm_ewts/test_config.py new file mode 100644 index 0000000..238a03d --- /dev/null +++ b/tests/lstm_ewts/test_config.py @@ -0,0 +1,81 @@ +import pytest + +import logging +from lstm_ewts.config import configure_logging, translate_ngwpc_log_level +from lstm_ewts.constants import MODULE_NAME, EV_EWTS_LOGGING + +# ------------------------------ +def test_configure_logging_default(clean_ewts_env): + logger = configure_logging() + + assert logger.name == MODULE_NAME + assert logger.level == logging.INFO + assert not logger.disabled + +# ------------------------------ +def test_configure_logging_idempotent(clean_ewts_env): + logger1 = configure_logging() + logger2 = configure_logging() + + assert logger1 is logger2 + assert getattr(logger1, "_initialized", False) + +# ------------------------------ +@pytest.mark.parametrize("inp,expected", [ + ("INFO", "INFO"), + ("SeVeRe", "ERROR"), + ("fatal", "CRITICAL"), + (" debug ", "DEBUG"), +]) +def test_translate_ngwpc_log_level(inp, expected): + assert translate_ngwpc_log_level(inp) == expected + +# ------------------------------ +@pytest.mark.parametrize("env_value,expected_enabled", [ + (None, True), # default: enabled + ("DISABLED", False), + ("ENABLED", True), + ("disabled", False), + ("enabled", True), + ("anystring", True), + ("", True), +]) +@pytest.mark.parametrize("level_input,expected_level", [ + ("DEBUG", logging.DEBUG), + ("INFO", logging.INFO), + ("SEVERE", logging.ERROR), + ("FATAL", logging.CRITICAL), +]) +def test_ewts_logger_matrix(clean_ewts_env, monkeypatch, capsys, env_value, expected_enabled, level_input, expected_level): + # Set environment variables + if env_value is None: + monkeypatch.delenv("NGEN_EWTS_LOGGING", raising=False) + else: + monkeypatch.setenv("NGEN_EWTS_LOGGING", env_value) + + monkeypatch.setenv("LSTM_LOGLEVEL", level_input) + + # Force logger re-initialization + logger = logging.getLogger(MODULE_NAME) + logger.handlers.clear() + logger._initialized = False + logger.disabled = False # ensure proper reset + + # Configure logger + logger = configure_logging() + + # Capture stdout + captured = capsys.readouterr() + + # Assertions + assert logger.name == MODULE_NAME + assert (not logger.disabled) == expected_enabled # True if enabled + if expected_enabled: + assert logger.level == expected_level + + # Assertions for default-enabled print + if expected_enabled and (env_value is None or env_value not in ("ENABLED", "enabled")): + assert f"{EV_EWTS_LOGGING} not explicitly set" in captured.out + else: + assert f"{EV_EWTS_LOGGING} not explicitly set" not in captured.out + diff --git a/tests/lstm_ewts/test_constants.py b/tests/lstm_ewts/test_constants.py new file mode 100644 index 0000000..4499bc5 --- /dev/null +++ b/tests/lstm_ewts/test_constants.py @@ -0,0 +1,10 @@ +from lstm_ewts.constants import ( + MODULE_NAME, + LOG_MODULE_NAME_LEN, +) + +def test_module_name_is_string(): + assert isinstance(MODULE_NAME, str) + +def test_module_name_length_fits_field(): + assert len(MODULE_NAME) <= LOG_MODULE_NAME_LEN diff --git a/tests/lstm_ewts/test_formatter.py b/tests/lstm_ewts/test_formatter.py new file mode 100644 index 0000000..3a6af0c --- /dev/null +++ b/tests/lstm_ewts/test_formatter.py @@ -0,0 +1,65 @@ +import logging +import pytest +from lstm_ewts.formatter import CustomFormatter +from lstm_ewts.constants import MODULE_NAME + +@pytest.fixture +def formatter(): + fmt = "%(asctime)s %(levelname_padded)s %(message)s" + return CustomFormatter(fmt=fmt, datefmt="%Y-%m-%dT%H:%M:%S") + +@pytest.mark.parametrize( + "level,expected", + [ + (logging.DEBUG, "DEBUG"), + (logging.INFO, "INFO"), + (logging.WARNING, "WARNING"), + (logging.ERROR, "SEVERE"), + (logging.CRITICAL, "FATAL"), + ] +) +def test_level_name_mapping(formatter, level, expected): + record = logging.LogRecord( + name=MODULE_NAME, + level=level, + pathname="test", + lineno=0, + msg="Test message", + args=None, + exc_info=None + ) + formatted = formatter.format(record) + # Level name should appear in formatted string + assert expected in formatted + +def test_utc_timestamp(formatter): + record = logging.LogRecord( + name=MODULE_NAME, + level=logging.INFO, + pathname="test", + lineno=0, + msg="UTC test", + args=None, + exc_info=None + ) + formatted = formatter.format(record) + # Timestamp should be in UTC format "YYYY-MM-DDTHH:MM:SS" + ts_str = formatted.split()[0] + from datetime import datetime + dt = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S") + # It's enough to check it parses without error + +def test_trailing_whitespace_stripped(formatter): + record = logging.LogRecord( + name=MODULE_NAME, + level=logging.INFO, + pathname="test", + lineno=0, + msg="Message with space \n", + args=None, + exc_info=None + ) + formatted = formatter.format(record) + # Trailing whitespace/newline should be removed + assert " \n" not in formatted + assert formatted.endswith("Message with space") diff --git a/tests/lstm_ewts/test_paths.py b/tests/lstm_ewts/test_paths.py new file mode 100644 index 0000000..f0072e4 --- /dev/null +++ b/tests/lstm_ewts/test_paths.py @@ -0,0 +1,115 @@ +import os +import getpass +from datetime import datetime +import pytest +from lstm_ewts import paths +from lstm_ewts.paths import create_timestamp, get_log_file_path +from lstm_ewts.constants import MODULE_NAME, EV_MODULE_LOGFILEPATH, EV_NGEN_LOGFILEPATH + +# ------------------------------- +# Fixture for a clean log environment +# ------------------------------- +@pytest.fixture +def clean_log_env(tmp_path, monkeypatch): + """Set up a temporary log environment and clean env vars. + + Yields a dict with: + tmp_dir : Path of temporary base directory + monkeypatch : the pytest monkeypatch object for further tweaks + """ + # Clear env vars + monkeypatch.delenv(EV_MODULE_LOGFILEPATH, raising=False) + monkeypatch.delenv(EV_NGEN_LOGFILEPATH, raising=False) + + # Patch constants to use tmp_path + monkeypatch.setattr(paths, "LOG_DIR_NGENCERF", tmp_path) + monkeypatch.setattr(paths, "LOG_DIR_DEFAULT", "run-logs") + + yield {"tmp_dir": tmp_path, "monkeypatch": monkeypatch} + + +# ------------------------------- +# Tests for create_timestamp() +# ------------------------------- +def test_create_timestamp_default(): + ts = create_timestamp() + assert len(ts) >= 15 + assert "T" in ts + +def test_create_timestamp_date_only(): + ts = create_timestamp(date_only=True) + assert len(ts) == 8 + +def test_create_timestamp_iso(): + ts = create_timestamp(iso=True) + assert "T" in ts and "-" in ts and ":" in ts + +def test_create_timestamp_append_ms(): + ts = create_timestamp(append_ms=True) + assert "." in ts + + +# ------------------------------- +# Tests for get_log_file_path() +# ------------------------------- +def test_get_log_file_path_uses_module_env(clean_log_env): + tmp_path = clean_log_env["tmp_dir"] + monkeypatch = clean_log_env["monkeypatch"] + + logfile = tmp_path / "test_module.log" + monkeypatch.setenv(EV_MODULE_LOGFILEPATH, str(logfile)) + + path, append = get_log_file_path() + assert path == str(logfile) + assert append is True + + +def test_get_log_file_path_uses_ngen_env(clean_log_env): + monkeypatch = clean_log_env["monkeypatch"] + tmp_path = clean_log_env["tmp_dir"] + + monkeypatch.delenv(EV_MODULE_LOGFILEPATH, raising=False) + ngen_file = tmp_path / "ngen.log" + monkeypatch.setenv(EV_NGEN_LOGFILEPATH, str(ngen_file)) + + path, append = get_log_file_path() + assert path == str(ngen_file) + assert append is True + + +def test_get_log_file_path_creates_user_subdir(clean_log_env): + tmp_path = clean_log_env["tmp_dir"] + monkeypatch = clean_log_env["monkeypatch"] + + monkeypatch.delenv(EV_MODULE_LOGFILEPATH, raising=False) + monkeypatch.delenv(EV_NGEN_LOGFILEPATH, raising=False) + + # Use real username + monkeypatch.setattr(getpass, "getuser", lambda: "alice") + + path, append = get_log_file_path() + + # Subdirectory should be username + subdir = os.path.basename(os.path.dirname(path)) + assert subdir == "alice" + assert path.endswith(".log") + assert os.path.exists(path) + + +def test_get_log_file_path_fallback_username(clean_log_env): + tmp_path = clean_log_env["tmp_dir"] + monkeypatch = clean_log_env["monkeypatch"] + + monkeypatch.delenv(EV_MODULE_LOGFILEPATH, raising=False) + monkeypatch.delenv(EV_NGEN_LOGFILEPATH, raising=False) + + # Simulate getuser() returning None + monkeypatch.setattr(getpass, "getuser", lambda: None) + + path, append = get_log_file_path() + + subdir = os.path.basename(os.path.dirname(path)) + # Should fall back to YYYYMMDD + assert len(subdir) == 8 and subdir.isdigit() + assert path.endswith(".log") + assert os.path.exists(path) From 0aae6bab4d9d3bac5d87cff893a7c4ed51be1fb4 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 8 Jan 2026 09:38:47 -0800 Subject: [PATCH 6/8] Use lstm_ewts package --- lstm/bmi_lstm.py | 2 +- lstm/logger.py | 230 ----------------------------------------------- 2 files changed, 1 insertion(+), 231 deletions(-) delete mode 100644 lstm/logger.py diff --git a/lstm/bmi_lstm.py b/lstm/bmi_lstm.py index 8c85d21..85422e5 100644 --- a/lstm/bmi_lstm.py +++ b/lstm/bmi_lstm.py @@ -62,7 +62,7 @@ from . import nextgen_cuda_lstm from .base import BmiBase -from .logger import configure_logging, MODULE_NAME +from lstm_ewts import configure_logging, MODULE_NAME from .model_state import State, StateFacade, Var import logging diff --git a/lstm/logger.py b/lstm/logger.py deleted file mode 100644 index a5cb8c4..0000000 --- a/lstm/logger.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Logging module to integrate with NGEN""" - -from __future__ import annotations - -import getpass -import logging -import os -import sys -import time -from datetime import datetime, timezone - -MODULE_NAME = "LSTM"; -LOG_DIR_NGENCERF = "/ngencerf/data"; # ngenCERF log directory string if environement var empty. -LOG_DIR_DEFAULT = "run-logs"; # Default parent log directory string if env var empty & ngencerf dosn't exist -LOG_FILE_EXT = "log"; # Log file name extension -DS = "/"; # Directory separator -LOG_MODULE_NAME_LEN = 8; # Width of module name for log entries - -EV_EWTS_LOGGING = "NGEN_EWTS_LOGGING"; # Enable/disable of Error Warning and Trapping System -EV_NGEN_LOGFILEPATH = "NGEN_LOG_FILE_PATH"; # ngen log file -EV_MODULE_LOGLEVEL = "LSTM_LOGLEVEL"; # This modules log level -EV_MODULE_LOGFILEPATH = "LSTM_LOGFILEPATH"; # This modules log full log filename - -class CustomFormatter(logging.Formatter): - """A custom formatting class for logging""" - - LEVEL_NAME_MAP = { - logging.DEBUG: "DEBUG", - logging.INFO: "INFO", - logging.WARNING: "WARNING", - logging.ERROR: "SEVERE", - logging.CRITICAL: "FATAL" - } - - # Apply custom formatter (UTC timestamps applied only to this formatter) - def converter(self, timestamp): - """Override time converter to return UTC time tuple""" - return time.gmtime(timestamp) - - def formatTime(self, record, datefmt=None): - """Use our UTC converter""" - ct = self.converter(record.created) - if datefmt: - s = time.strftime(datefmt, ct) - else: - t = time.strftime("%Y-%m-%d %H:%M:%S", ct) - s = f"{t},{int(record.msecs):03d}" - return s - - def format(self, record): - original_levelname = record.levelname - record.levelname = self.LEVEL_NAME_MAP.get(record.levelno, original_levelname) - record.levelname_padded = record.levelname.ljust(7)[:7] # Exactly 7 chars - formatted = super().format(record) - record.levelname = original_levelname # Restore original in case it's reused - return formatted - -def create_timestamp(date_only: bool = False, iso: bool = False, append_ms: bool = False) -> str: - now = datetime.now(timezone.utc) - - if date_only: - ts_base = now.strftime("%Y%m%d") - elif iso: - ts_base = now.strftime("%Y-%m-%dT%H:%M:%S") - else: - ts_base = now.strftime("%Y%m%dT%H%M%S") - - if append_ms: - ms_str = f".{now.microsecond // 1000:03d}" - return ts_base + ms_str - else: - return ts_base - -def get_log_file_path(): - appendEntries = True - moduleLogEnvExists = False - moduleEnvVar = os.getenv(EV_MODULE_LOGFILEPATH, "") - if moduleEnvVar: - logFilePath = moduleEnvVar - moduleLogEnvExists = True - else: - ngenEnvVar = os.getenv(EV_NGEN_LOGFILEPATH, "") - if ngenEnvVar: - logFilePath = ngenEnvVar - else: - print(f"Module {MODULE_NAME} Env var {EV_NGEN_LOGFILEPATH} not found. Creating default log name.") - appendEntries = False - if os.path.isdir(LOG_DIR_NGENCERF): - logFileDir = LOG_DIR_NGENCERF + DS + LOG_DIR_DEFAULT - else: - logFileDir = os.path.expanduser("~") + DS + LOG_DIR_DEFAULT - try: - os.makedirs(logFileDir, exist_ok=True) - # Set full log path - username = getpass.getuser() - if username: - logFileDir = logFileDir + DS + username - else: - logFileDir = logFileDir + DS + create_timestamp(True) - # Create directory - os.makedirs(logFileDir, exist_ok=True) - logFilePath = logFileDir + DS + MODULE_NAME + "_" + create_timestamp() + "." + LOG_FILE_EXT - except Exception as e: - logFilePath = "" - - # Ensure log file can be opened and set module env var - try: - if (logFilePath): - if (appendEntries): - logFile = open(logFilePath, "a") - else: - logFile = open(logFilePath, "w") - if (moduleLogEnvExists == False): - os.environ[EV_MODULE_LOGFILEPATH] = logFilePath - print(f"Module {MODULE_NAME} Log File: {logFilePath}", flush=True) - else: - raise IOError - except: - print(f"Module {MODULE_NAME} Unable to open log file: {logFilePath}", flush=True) - print(f"Module {MODULE_NAME} Log entries will be writen to stdout", flush=True) - - return logFilePath, appendEntries - -def get_log_level() -> str: - levelEnvVar = os.getenv(EV_MODULE_LOGLEVEL, "") - if levelEnvVar: - return levelEnvVar.strip().upper() - else: - return "INFO" - -def translate_ngwpc_log_level(ngwpc_log_level: str) -> str: - ll = ngwpc_log_level.strip().upper() - if (ll == "SEVERE"): - return "ERROR" - elif (ll == "FATAL"): - return "CRITICAL" - return ll - -def force_info(handler, logger, msg, *args): - record = logger.makeRecord( - logger.name, - logging.INFO, - __file__, - 0, - msg, - args, - None, - ) - handler.emit(record) - -def configure_logging(): - ''' - Set logging level and specify logger configuration based on environment variables set by ngen - - Arguments - --------- - none - - Returns - ------- - None - - Notes - ----- - In the absense of logging environment variables the log level defaults to INFO and - the pathname is set as follows: - - Use the module log file if available (unset when first run by ngen), otherwise - - Use ngen log file if available, otherwise - - Use /ngencerf/data/run-logs//_ if available, otherwise - - Use ~/run-logs//_ - - Onced opened, save the full log path to the modules log environment variable so - it is only opened once for each ngen run (vs for each catchment) - - See also https://docs.python.org/3/library/logging.html - - ''' - - # Use a named logger to ensure entries are identified as this - # MODULE_NAME and are not miss-identfied in the ngen log. - logger = logging.getLogger(MODULE_NAME) - if getattr(logger, "_initialized", False): - return # logger already initialized, nothing else to do - - loggingEnabled = True - moduleEnvVar = os.getenv(EV_EWTS_LOGGING, "") - if moduleEnvVar: - if (moduleEnvVar == "DISABLED"): - loggingEnabled = False - else: - print(f"Module {MODULE_NAME} Env var {EV_EWTS_LOGGING} not found. Using logging defaults.") - - if (loggingEnabled == False): - print(f"Module {MODULE_NAME} Logging DISABLED", flush=True) - logger.disabled = True # Disables all logs at CRITICAL and below (i.e., everything) - else: - print(f"Module {MODULE_NAME} Logging ENABLED", flush=True) - - # Get the log file name from env var or a default - logFilePath, appendEntries = get_log_file_path() - if (logFilePath): - # Set the open mode - openMode = 'a' if appendEntries else 'w' - handler = logging.FileHandler(logFilePath, mode=openMode) - else: - print(f"Module {MODULE_NAME} unable to create log file. Using stdout.") - handler = logging.StreamHandler(sys.stdout) - - # Get the log level from env var or a default - log_level = get_log_level() - - # Format the module name: uppercase, fixed length, left-justify or trimmed - formatted_module = MODULE_NAME.upper().ljust(LOG_MODULE_NAME_LEN)[:LOG_MODULE_NAME_LEN] - - # Apply custom formatter - formatter = CustomFormatter( - fmt=f"%(asctime)s.%(msecs)03d {formatted_module} %(levelname_padded)s %(message)s", - datefmt="%Y-%m-%dT%H:%M:%S" - ) - handler.setFormatter(formatter) - - # Setup logger - logger.handlers.clear() # Clear any default handlers - logger.setLevel(translate_ngwpc_log_level(log_level)) - logger.addHandler(handler) - - # Write log level INFO message to log regradless of the actual log level - force_info(handler, logger, "Log level set to %s", log_level) - print(f"Module {MODULE_NAME} Log Level set to {log_level}", flush=True) - - logger._initialized = True From ddd60113eaee43ec81e0eab41cc3849a2f65d426 Mon Sep 17 00:00:00 2001 From: Carolyn Maynard Date: Thu, 8 Jan 2026 10:10:12 -0800 Subject: [PATCH 7/8] Update pip install for unit tests workflow Add building the lstm_ewts package --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 318a375..ab5ce33 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -32,7 +32,7 @@ jobs: - name: Install dependencies run: | pip install -U pip # upgrade pip - pip install '.[develop]' + pip install '.[develop] ./lstm_ewts' - name: Echo dependency versions run: | pip freeze From 85a3301daeff761a54b6ebda6fee7aac977a62ce Mon Sep 17 00:00:00 2001 From: Carolyn Maynard Date: Thu, 8 Jan 2026 10:11:16 -0800 Subject: [PATCH 8/8] Fix pip install command in unit_tests.yml --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ab5ce33..2bb9dd3 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -32,7 +32,7 @@ jobs: - name: Install dependencies run: | pip install -U pip # upgrade pip - pip install '.[develop] ./lstm_ewts' + pip install '.[develop]' './lstm_ewts' - name: Echo dependency versions run: | pip freeze