From 6747f73f1ecc05c490f8bad8c441e195ae8bc2c1 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Thu, 20 Nov 2025 10:36:22 -0700 Subject: [PATCH 01/15] Introduce external triggers (xtriggers) as new workflow element --- initialize/applications/DA.py | 1 + initialize/config/Component.py | 1 + initialize/suites/Cycle.py | 4 ++++ initialize/suites/SuiteBase.py | 10 ++++++++++ 4 files changed, 16 insertions(+) diff --git a/initialize/applications/DA.py b/initialize/applications/DA.py index 74f2c27d..a6e2043e 100644 --- a/initialize/applications/DA.py +++ b/initialize/applications/DA.py @@ -176,6 +176,7 @@ def export(self, previousForecast:str, ef:ExtendedForecast): for st in self.__subtasks: self._tasks += st._tasks self._dependencies += st._dependencies + self._xtriggers += st._xtriggers # depends on previous Forecast self.tf.addDependencies([previousForecast]) diff --git a/initialize/config/Component.py b/initialize/config/Component.py index 7b18e0d3..9aa1be08 100644 --- a/initialize/config/Component.py +++ b/initialize/config/Component.py @@ -37,6 +37,7 @@ def __init__(self, config:Config): self._queues = [] self._tasks = [] self._dependencies = [] + self._xtriggers = [] ################### # extract SubConfig diff --git a/initialize/suites/Cycle.py b/initialize/suites/Cycle.py index bd6ca884..a0cd4bb5 100644 --- a/initialize/suites/Cycle.py +++ b/initialize/suites/Cycle.py @@ -115,3 +115,7 @@ def __init__(self, conf:Config): 'initic', 'observations', ] + + self.xtriggerComponents += [ + 'da' + ] diff --git a/initialize/suites/SuiteBase.py b/initialize/suites/SuiteBase.py index ca63c071..2b86ec58 100644 --- a/initialize/suites/SuiteBase.py +++ b/initialize/suites/SuiteBase.py @@ -28,10 +28,12 @@ def __init__(self, conf:Config): self.queueComponents = [] self.dependencyComponents = [] self.taskComponents = [] + self.xtriggerComponents = [] self._queues = [] self._dependencies = [] self._tasks = [] + self._xtriggers = [] self.logPrefix = self.__class__.__name__+': ' host = os.getenv('NCAR_HOST') @@ -122,6 +124,11 @@ def submit(self): self._tasks += [''' # '''+ k] self._tasks += self.c[k]._tasks + + for k in self.xtriggerComponents: + self._xtriggers += [''' + # ''' + k] + self._xtriggers += self.c[k]._xtriggers self.__export() @@ -170,6 +177,9 @@ def __export(self): # default: 3 runahead limit = P'''+str(self.c['workflow']['max active cycle points']-1)+''' + [[xtriggers]] +'''+''.join(self._xtriggers)+''' + [[queues]] '''+''.join(self._queues)+''' From 43a708dc61cff5168e492de9512b066c3be29db6 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Thu, 20 Nov 2025 10:42:28 -0700 Subject: [PATCH 02/15] Introduce option to couple Var DA run to EnKF run --- initialize/applications/Variational.py | 58 +++++++++++++++++++++++- scenarios/3dhybrid_OIE120km_coupled.yaml | 46 +++++++++++++++++++ scenarios/defaults/variational.yaml | 13 ++++++ 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 scenarios/3dhybrid_OIE120km_coupled.yaml diff --git a/initialize/applications/Variational.py b/initialize/applications/Variational.py index 65fb5a5b..61185ca4 100644 --- a/initialize/applications/Variational.py +++ b/initialize/applications/Variational.py @@ -8,6 +8,8 @@ ''' from collections import OrderedDict +from getpass import getuser +import os.path from initialize.applications.Members import Members @@ -40,6 +42,20 @@ class Variational(Component): 'ensembleCovarianceWeight': float, 'staticCovarianceWeight': float, + ## Coupling to an external EnKF run + # Must be specified if coupledToEnKF == True, not used otherwise + # Specify the full name of the workflow, constructed as: + # Experiment['prefix'] + Experiment['name'] + Experiment['suffix'] + Experiment['suite identifier'] + # The default values are: + # Experiment['prefix'] = "${USER}_" + # Experiment['name']: see Cycle.py + # Experiment['suffix'] = "" + # Experiment['suite identifier'] = "" + # It is best practice to specify the experiment prefix, name, and suffix in the coupled yaml files + # to make sure the coupled workflow can be identified correctly. + # Note: the workflow automatically prepends 'MPAS-Workflow' to workflowNameEnKF where needed + # to make the identifier consistent with submit.csh + 'workflowNameEnKF': str, } variablesWithDefaults = { @@ -154,6 +170,11 @@ class Variational(Component): ## IR/VIS land surface coefficients classification # OPTIONS: USGS, IGBP, NPOESS 'IRVISlandCoeff': ['IGBP', str], + + ## Coupling to an external EnKF run + # When coupledToEnKF is true, this variational run uses the forecast ensemble of a + # concurrently running external EnKF run to construct the ensemble B matrix. + 'coupledToEnKF': [False, bool], } def __init__(self, @@ -286,6 +307,22 @@ def __init__(self, self._setOrDie('.'.join(['covariance', r, 'bumpCovVBalDir']), str, None, 'bumpCovVBalDir') self._setOrDie('.'.join(['covariance', r, 'hybridCoefficientsDir']), str, None, 'hybridCoefficientsDir') + # coupling to EnKF + if self['coupledToEnKF']: + # The coupling to EnKF is meaningful if we run a single variational DA, but undefined + # if we run an EDA + if self.NN > 1: + raise NotImplementedError("Behavior of EDA coupled to EnKF is undefined. Set members.n == 1.") + # A coupled variational run requires the name of the EnKF workflow to + # set the external dependency + if self['workflowNameEnKF'] is None: + raise ValueError("workflowNameEnKF has to be specified for a coupled Var DA") + # Overwrite the ensemble directory to make sure we are reading the forecasts from the + # coupled EnKF run + workDirEnKF = os.path.join('/glade', 'derecho', 'scratch', getuser(), + 'pandac', self['workflowNameEnKF']) + self._set('ensPbDir0', f'{os.path.join(workDirEnKF, "CyclingFC", "{{prevDateTime}}")}') + self._cshVars = list(self._vtable.keys()) ######################## @@ -355,7 +392,26 @@ def __init__(self, [[Variationals]] inherit = '''+self.tf.execute+''' '''+vartask.job()+vartask.directives()] - + + if self['coupledToEnKF']: + # The coupled variational run uses the forecast ensemble from the EnKF to generate + # the ensemble B. This introduces a dependency between the variational DA at the current + # cycle point and the EnKF forecast from the previous cycle point. The following external + # trigger defines this dependency. + # Note: submit.csh prepends 'MPAS-Workflow/' to the workflow name to generate the workflow ID + workflowIdEnKF = os.path.join('MPAS-Workflow', self['workflowNameEnKF']) + callIntervalInS = 30 # call interval for external trigger (default is 10) + self._xtriggers +=[('\n' + ' enkf_forecast = workflow_state(' + f'workflow="{workflowIdEnKF}", ' + 'task="ForecastFinished__", point="%(point)s", ' + f'offset="-PT{workflow["CyclingWindowHR"]}H")' + f':PT{callIntervalInS}S')] + + # Add the external dependency to the graph + self._dependencies += [('\n' + ' @enkf_forecast => ' + self.tf.pre)] + if EDASize == 1: # single instance or ensemble of Variational(s) for mm in range(1, self.NN+1, 1): diff --git a/scenarios/3dhybrid_OIE120km_coupled.yaml b/scenarios/3dhybrid_OIE120km_coupled.yaml new file mode 100644 index 00000000..e9a5ea67 --- /dev/null +++ b/scenarios/3dhybrid_OIE120km_coupled.yaml @@ -0,0 +1,46 @@ +experiment: + prefix: "stoedtli_" + name: "3dhybrid_coupled_test_" + suffix: "2" + +firstbackground: + resource: "PANDAC.GFS" + +forecast: + #execute: False # the default is True + post: [] # use this when doing extended forecast + +externalanalyses: + resource: "GFS.PANDAC" + +members: + n: 1 + +model: + outerMesh: 120km + innerMesh: 120km + ensembleMesh: 120km + +observations: + resource: PANDACArchiveForVarBC + +variational: + DAType: 3dhybrid + biasCorrection: True + ensembleCovarianceWeight: 0.75 + staticCovarianceWeight: 0.25 + nInnerIterations: [60,60,] + ensemble: + forecasts: + resource: "PANDAC.GETKF_coupled" + #execute: False # the default is True + post: [] # this turns off verifyobs + coupledToEnKF: True + workflowNameEnKF: "stoedtli_getkf_coupled_test_2" + +workflow: + first cycle point: 20180414T18 + #restart cycle point: 20180415T00 + final cycle point: 20180415T00 + #CyclingWindowHR: 24 # default is 6 for cycling DA + #max active cycle points: 4 # used for independent 'extendedforecast' diff --git a/scenarios/defaults/variational.yaml b/scenarios/defaults/variational.yaml index eddb2e24..4aaf73de 100644 --- a/scenarios/defaults/variational.yaml +++ b/scenarios/defaults/variational.yaml @@ -112,6 +112,19 @@ variational: # zero padding: 3 # nmembers: 80 + GETKF_coupled: + 120km: + directory0: 'setByCoupledWorkflow' + memberPrefix: mem + memberNDigits: 3 + maxMembers: 20 + + 60km: + directory0: 'setByCoupledWorkflow' + memberPrefix: mem + memberNDigits: 3 + maxMembers: 20 + # externally-produced localization files localization: #{{ensembleMesh}}: From f6199a1bbe61224180bcfbb2c3b7c8ae210eca8a Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Fri, 21 Nov 2025 14:06:33 -0700 Subject: [PATCH 03/15] Introduce option to couple EnKF run to var DA run --- initialize/applications/EnKF.py | 70 +++++++++++++++++++++++++++ scenarios/getkf_OIE120km_coupled.yaml | 54 +++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 scenarios/getkf_OIE120km_coupled.yaml diff --git a/initialize/applications/EnKF.py b/initialize/applications/EnKF.py index 3689dcc1..657427e5 100755 --- a/initialize/applications/EnKF.py +++ b/initialize/applications/EnKF.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 from collections import OrderedDict +from getpass import getuser +import os.path from initialize.applications.Members import Members @@ -86,6 +88,7 @@ class EnKF(Component): 'biasCorrection': [False, bool], # directories that stores varBC coefficients that are updated with variational DA + # if coupledToVarDA == True the staticVarBcDir is automatically set to the coupled var DA directory 'staticVarBcDir': ['/glade/campaign/mmm/parc/ivette/pandac/year7Exp/ivette_3dhybrid-allsky-60-60-iter_O30kmI60km_ensB-SE80+RTPP70_VarBC_v3.0.2_newBenchmark_allsky-amsua/CyclingDA', str], ## tropprsMethod @@ -118,6 +121,34 @@ class EnKF(Component): ## IR/VIS land surface coefficients classification # OPTIONS: USGS, IGBP, NPOESS 'IRVISlandCoeff': ['IGBP', str], + + ## Coupling to an external variational DA run + # When coupledToVarDA is true, this EnKF run can be recentered (recenterAnalyses == True) on the variational DA run and/or + # it can use the variational bias correction coefficients (biasCorrection == True) of the variational DA run + 'coupledToVarDA': [False, bool], + + ## Recentering of analysis ensemble + # When recenterAnalyses is true, the analysis ensemble of this EnKF is recentered on the analysis of a + # concurrently running var DA run. Requires coupledToVarDA == True. + # The recentering application is not implemented yet. + 'recenterAnalyses': [False, bool], + } + + optionalVariables = { + ## Coupling to an external variational DA run + # Must be specified if coupledToVarDA == True, not used otherwise + # Specify the full name of the workflow, constructed as: + # Experiment['prefix'] + Experiment['name'] + Experiment['suffix'] + Experiment['suite identifier'] + # The default values are: + # Experiment['prefix'] = "${USER}_" + # Experiment['name']: see Cycle.py + # Experiment['suffix'] = "" + # Experiment['suite identifier'] = "" + # It is best practice to specify the experiment prefix, name, and suffix in the coupled yaml files + # to make sure the coupled workflow can be identified correctly. + # Note: the workflow automatically prepends 'MPAS-Workflow' to workflowNameVarDA where needed + # to make the identifier consistent with submit.csh + 'workflowNameVarDA': str, } def __init__(self, @@ -183,6 +214,28 @@ def __init__(self, # TODO: this needs to be non-zero for EnKF workflows that use IAU, get value from forecast self._set('ensPbOffsetHR', 0) + # coupling to variational DA + if self['coupledToVarDA']: + # A coupled EnKF run should use some information from the variational DA. + # Stop the run if no information is used in the current configuration to make sure + # user can adjust settings. + if not self['biasCorrection'] and not self['recenterAnalyses']: + raise ValueError(("Coupled EnKF does not use any information from var DA." + "biasCorrectionEnKF and/or recenterAnalyses should be set to true.")) + # A coupled EnKF run requires the name of the variational DA workflow to + # set the external dependency + if self['workflowNameVarDA'] is None: + raise ValueError("workflowNameVarDA has to be specified for a coupled EnKF") + # Overwrite the static var BC directory to make sure we are reading the satbias files from + # the coupled var DA run + workDirVarDA = os.path.join('/glade', 'derecho', 'scratch', getuser(), + 'pandac', self['workflowNameVarDA']) + self._set('staticVarBcDir', f'{os.path.join(workDirVarDA, "CyclingDA")}') + else: + # Recentering the analysis ensemble is not defined if the EnKF run is not coupled. + if self['recenterAnalyses']: + raise NotImplementedError("Recentering of the analyses ensemble is undefined if the EnKF run is not coupled.") + self._cshVars = list(self._vtable.keys()) ######################## @@ -262,6 +315,23 @@ def __init__(self, inherit = '''+self.tf.execute+''', BATCH script = $origin/bin/EnKF.csh Solver '''+solvertask.job()+solvertask.directives()] + + if self['coupledToVarDA']: + # The coupled EnKF run uses the analysis file and/or the variational bias correction files + # from the variational DA run. This introduces a dependency between the EnKF and the + # variational DA at the current cycle point. The following external trigger defines this dependency. + # Note: submit.csh prepends 'MPAS-Workflow/' to the workflow name to generate the workflow ID + workflowIdVarDA = os.path.join('MPAS-Workflow', self['workflowNameVarDA']) + callIntervalInS = 30 # call interval for external trigger (default is 10) + self._xtriggers +=[('\n' + ' var_da = workflow_state(' + f'workflow="{workflowIdVarDA}", ' + 'task="DAFinished__", point="%(point)s")' + f':PT{callIntervalInS}S')] + + # Add the dependency to the graph + self._dependencies += [('\n' + ' @var_da => ' + self.tf.pre)] if self['diagEnKFOMA'] and self['retainObsFeedback']: self._tasks += [''' diff --git a/scenarios/getkf_OIE120km_coupled.yaml b/scenarios/getkf_OIE120km_coupled.yaml new file mode 100644 index 00000000..aa08aab7 --- /dev/null +++ b/scenarios/getkf_OIE120km_coupled.yaml @@ -0,0 +1,54 @@ +experiment: + prefix: 'stoedtli_' + name: 'getkf_coupled_test_' + suffix: '3' + +enkf: + solver: GETKF + # localization + horizontal localization lengthscale: 1.2e6 + vertical localization lengthscale: 6000.0 + vertical localization lengthscale units: height + fraction of retained variance: 0.95 + # inflation + rtpp value: 0.5 + rtps value: 0.9 + # ensemble OMA calculation + retainObsFeedback: True + concatenateObsFeedback: True + # coupling to var DA (bias correction, recentering) + coupledToVarDA: True + workflowNameVarDA: 'stoedtli_3dhybrid_coupled_test_3' + recenterAnalyses: False # not implemented yet + biasCorrection: True + #staticVarBcDir: /glade/campaign/mmm/parc/jban/pandac/year7Exp/jban_3dhybrid-60-60-iter_O30kmI60km_mhsRaw_clrsky/CyclingDA + post: [] + +externalanalyses: + resource: "GFS.PANDAC" + +firstbackground: + resource: "PANDAC.EnsPertB" + +forecast: + post: [] + +hofx: + retainObsFeedback: False + +members: + n: 20 + +model: + outerMesh: 120km + # TODO: make inner and ensemble meshes unnecessary + # related to {{PrepareExternalAnalysisInner}} and {{PrepareExternalAnalysisEnsemble}} + innerMesh: 120km + ensembleMesh: 120km + +observations: + resource: PANDACArchiveForVarBC + +workflow: + first cycle point: 20180414T18 + final cycle point: 20180415T00 From 3a6ed66ebfda3658401214ea382777cbc096acd4 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Tue, 2 Dec 2025 11:09:35 -0700 Subject: [PATCH 04/15] Update default resources for 120km getkf --- scenarios/defaults/enkf.yaml | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/scenarios/defaults/enkf.yaml b/scenarios/defaults/enkf.yaml index dabc43f3..44c7742f 100644 --- a/scenarios/defaults/enkf.yaml +++ b/scenarios/defaults/enkf.yaml @@ -127,30 +127,20 @@ enkf: secondsPerMember: 3 GETKF: observer: - # cost for record (20 members, 95% variance retained [13 eig], PBS JOB email) - # 8 x 32 PE : 12.2 min., 118 GB (single precision) - nodes: 2 - PEPerNode: 128 + nodes: 1 + PEPerNode: 64 memory: 235GB - baseSeconds: 800 - secondsPerMember: 50 + baseSeconds: 200 + secondsPerMember: 20 solver: - # cost for record (5 members, PBS JOB email) - # 4 x 32 PE : ?.? min., ?? GB (single precision) - # 8 x 32 PE : ?.? min., ?? GB (single precision) - nodes: 2 + nodes: 1 PEPerNode: 128 memory: 235GB baseSeconds: 200 - secondsPerMember: 300 + secondsPerMember: 20 diagoma: - # cost for record (5 members, PBS JOB email) - # 2 x 32 PE : 1.7 min., 14.6 GB (single precision) - # 2 x 32 PE : 3.5 min., 31 GB (double precision) - # cost for record (20 members, PBS JOB email) - # 2 x 32 PE : 3.0 min., 31.7 GB (single precision) - nodes: 2 - PEPerNode: 128 + nodes: 1 + PEPerNode: 64 memory: 235GB - baseSeconds: 300 - secondsPerMember: 3 + baseSeconds: 200 + secondsPerMember: 10 From 252b89dd599d5ff19e9716af9d9b1268e3c9f9b9 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Fri, 5 Dec 2025 12:37:21 -0700 Subject: [PATCH 05/15] Add basic EnKF recentering task --- initialize/applications/EnKF.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/initialize/applications/EnKF.py b/initialize/applications/EnKF.py index 657427e5..5e94b7dd 100755 --- a/initialize/applications/EnKF.py +++ b/initialize/applications/EnKF.py @@ -226,10 +226,13 @@ def __init__(self, # set the external dependency if self['workflowNameVarDA'] is None: raise ValueError("workflowNameVarDA has to be specified for a coupled EnKF") - # Overwrite the static var BC directory to make sure we are reading the satbias files from - # the coupled var DA run + # Export the var DA run information to the csh file + self._set('workflowNameVarDA', self['workflowNameVarDA']) workDirVarDA = os.path.join('/glade', 'derecho', 'scratch', getuser(), 'pandac', self['workflowNameVarDA']) + self._set('workDirVarDA', workDirVarDA) + # Overwrite the static var BC directory to make sure we are reading the satbias files from + # the coupled var DA run self._set('staticVarBcDir', f'{os.path.join(workDirVarDA, "CyclingDA")}') else: # Recentering the analysis ensemble is not defined if the EnKF run is not coupled. @@ -329,9 +332,25 @@ def __init__(self, 'task="DAFinished__", point="%(point)s")' f':PT{callIntervalInS}S')] - # Add the dependency to the graph + # Add the dependency on the var DA run to the graph self._dependencies += [('\n' ' @var_da => ' + self.tf.pre)] + if self['recenterAnalyses']: + # Add the dependency: the EnKFDiagOMA task is optional. If the task is enabled, the recentering + # task should run after it. If it is not enabled, the recentering task should run after the solver task. + if self['diagEnKFOMA'] and self['retainObsFeedback']: + self._dependencies += [('\n' + ' EnKFDiagOMA => RecenterEnKF')] + else: + self._dependencies += [('\n' + ' EnKFSolver => RecenterEnKF')] + # Add the task + self._tasks += [('\n' + ' [[RecenterEnKF]]' + '\n' + f' inherit = {self.tf.execute}' + '\n' + ' script = $origin/bin/RecenterAnalysisEnsemble.csh')] if self['diagEnKFOMA'] and self['retainObsFeedback']: self._tasks += [''' From 8b31f0db96a8de6d4321b3a18a6d4daf6d34fe1b Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Fri, 5 Dec 2025 12:49:33 -0700 Subject: [PATCH 06/15] Add recenter executable info to workflow --- initialize/framework/Build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/initialize/framework/Build.py b/initialize/framework/Build.py index ff441d01..b2e6e0cc 100644 --- a/initialize/framework/Build.py +++ b/initialize/framework/Build.py @@ -110,6 +110,10 @@ def __init__(self, config:Config, model:Model=None): self._set('SACAEXE', 'mpasjedi_saca.x') self._set('SACABuildDir', self['mpas bundle']+'/bin') + ## Ensemble recentering + self._set('RecenterEXE', 'mpasjedi_ens_recenter.x') + self._set('RecenterBuildDir', '/glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build/bin') + if model is not None: # MPAS-Model From 1ef86e474f315617e83d7793725a34e0e9e7da7f Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Fri, 5 Dec 2025 12:52:50 -0700 Subject: [PATCH 07/15] Add placeholder script for recenter analysis task --- bin/RecenterAnalysisEnsemble.csh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 bin/RecenterAnalysisEnsemble.csh diff --git a/bin/RecenterAnalysisEnsemble.csh b/bin/RecenterAnalysisEnsemble.csh new file mode 100755 index 00000000..64978723 --- /dev/null +++ b/bin/RecenterAnalysisEnsemble.csh @@ -0,0 +1,15 @@ +#!/bin/csh -f + +# (C) Copyright 2025 UCAR +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# Recenters an analysis ensemble on a given center state + +date + +echo "Recentering analysis ensemble." +sleep 10 + +date From 0ad3eb76d7e584a717ad00c806e24e53b6af2d11 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Mon, 8 Dec 2025 08:59:02 -0700 Subject: [PATCH 08/15] Add resources and job runner to recenter task --- initialize/applications/EnKF.py | 25 +++++++++++++++++++++++-- scenarios/defaults/enkf.yaml | 6 ++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/initialize/applications/EnKF.py b/initialize/applications/EnKF.py index 5e94b7dd..5965cea1 100755 --- a/initialize/applications/EnKF.py +++ b/initialize/applications/EnKF.py @@ -345,12 +345,33 @@ def __init__(self, self._dependencies += [('\n' ' EnKFSolver => RecenterEnKF')] # Add the task + keyListRecenter = { + # Notes: + # - the recenter application is currently small enough to run on develop + # - the develop queue does not support a job_priority flag, so removed this for now + 'retry': {'t': str}, + 'baseSeconds': {'t': int}, + 'secondsPerMember': {'t': int}, + 'nodes': {'t': int}, + 'PEPerNode': {'t': int}, + 'memory': {'t': str}, + 'queue': {'def': hpc['SharedQueue']}, + 'account': {'def': hpc['CriticalAccount']}, + 'email': {'def': True, 't': bool}, + } + resourceRecenter = meshes['Outer'].name + '.' + solver + '.recenter' + recenterJob = Resource(self._conf, keyListRecenter, ('job', resourceRecenter)) + recenterJob._set('seconds', recenterJob['baseSeconds'] + recenterJob['secondsPerMember'] * NN) + recenterTask = TaskLookup[hpc.system](recenterJob) self._tasks += [('\n' ' [[RecenterEnKF]]' '\n' - f' inherit = {self.tf.execute}' + f' inherit = {self.tf.execute}, BATCH' '\n' - ' script = $origin/bin/RecenterAnalysisEnsemble.csh')] + ' script = $origin/bin/RecenterAnalysisEnsemble.csh' + '\n' + f'{recenterTask.job() + recenterTask.directives()}' + '\n')] if self['diagEnKFOMA'] and self['retainObsFeedback']: self._tasks += [''' diff --git a/scenarios/defaults/enkf.yaml b/scenarios/defaults/enkf.yaml index 44c7742f..741a71d9 100644 --- a/scenarios/defaults/enkf.yaml +++ b/scenarios/defaults/enkf.yaml @@ -144,3 +144,9 @@ enkf: memory: 235GB baseSeconds: 200 secondsPerMember: 10 + recenter: + nodes: 1 + PEPerNode: 1 + memory: 45GB + baseSeconds: 100 + secondsPerMember: 5 From 53ddc41ce059fe1e888143e26f704808a10c0681 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Mon, 8 Dec 2025 11:58:43 -0700 Subject: [PATCH 09/15] Add draft version of recenter yraml file and shell script from standalone test --- bin/RecenterAnalysisEnsemble.csh | 62 +++++++++++- config/jedi/applications/recenter.yaml | 127 +++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 config/jedi/applications/recenter.yaml diff --git a/bin/RecenterAnalysisEnsemble.csh b/bin/RecenterAnalysisEnsemble.csh index 64978723..9820af0d 100755 --- a/bin/RecenterAnalysisEnsemble.csh +++ b/bin/RecenterAnalysisEnsemble.csh @@ -7,9 +7,63 @@ # Recenters an analysis ensemble on a given center state -date +source config/auto/build.csh # RecenterBuildDir, RecenterExe +source config/auto/enkf.csh # ensPbMemNDigits, ensPbNMembers, ensPbMemPrefix, workDirVarDA +source config/auto/model.csh # outerNamelistFile, outerStreamsFile +source config/auto/naming.csh # analysisSubDir, ANFilePrefix, DAWorkDir + +# Copy configuration file +# todo: change to run directory of cycling run +set yamlFileRecenter = "ens_recenter_test.yaml" +if ( -f "$yamlFileRecenter" ) then + rm "$yamlFileRecenter" +endif +cp "ens_recenter_template.yaml" "$yamlFileRecenter" + +# Date information +set CYLC_TASK_CYCLE_POINT = "20180415T0600Z" # will be set by cylc eventually +set yyyy = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 1-4` +set mm = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 5-6` +set dd = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 7-8` +set hh = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 10-11` +set cycleDate = "${yyyy}${mm}${dd}${hh}" +set cycleDateISO8601 = "${yyyy}-${mm}-${dd}T${hh}:00:00Z" +sed -i "s@{{cycleDateISO8601}}@$cycleDateISO8601@" $yamlFileRecenter + +# Center geometry information +# todo: update so that varDA can have different resolution +set cyclingDADirVar = "${workDirVarDA}/CyclingDA/${cycleDate}" +set namelistFileCenter = "${cyclingDADirVar}/${outerNamelistFile}" +set streamsFileCenter = "${cyclingDADirVar}/${outerStreamsFile}" +sed -i "s@{{namelistFileCenter}}@$namelistFileCenter@" $yamlFileRecenter +sed -i "s@{{streamsFileCenter}}@$streamsFileCenter@" $yamlFileRecenter + +# Center analysis information. This assumes that analysisSubDir and ANFilePrefix in +# the EnKF and var DA run are identical. This is not necessarily true but reasonable +set mpasFileSuffix = '$Y-$M-$D_$h.$m.$s.nc' +set analysisFileCenter = "${cyclingDADirVar}/${analysisSubDir}/${ANFilePrefix}.${mpasFileSuffix}" +sed -i "s@{{analysisFileCenter}}@$analysisFileCenter@" $yamlFileRecenter + +# EnKF ensemble geometry information +set cyclingDADirEnKF = "${DAWorkDir}/${cycleDate}" +set namelistFileEnsemble = "${cyclingDADirEnKF}/${outerNamelistFile}" +set streamsFileEnsemble = "${cyclingDADirEnKF}/${outerStreamsFile}" +sed -i "s@{{namelistFileEnsemble}}@$namelistFileEnsemble@" $yamlFileRecenter +sed -i "s@{{streamsFileEnsemble}}@$streamsFileEnsemble@" $yamlFileRecenter + +# EnKF analysis ensemble information +# todo: rename original analysis files and update this section +set analysisFileEnsemble = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${ANFilePrefix}.${mpasFileSuffix}" +sed -i "s@{{analysisFileEnsemble}}@$analysisFileEnsemble@" $yamlFileRecenter +sed -i "s@{{paddingEnsembleMembers}}@$ensPbMemNDigits@" $yamlFileRecenter +sed -i "s@{{numberEnsembleMembers}}@$ensPbNMembers@" $yamlFileRecenter + +# Recentered output +set analysisFileRecenter = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${ANFilePrefix}.${mpasFileSuffix}" +sed -i "s@{{analysisFileRecenter}}@$analysisFileRecenter@" $yamlFileRecenter + +# Link and run executable +ln -sfv "${RecenterBuildDir}/${RecenterEXE}" ./ +#mpiexec ./${RecenterEXE} -echo "Recentering analysis ensemble." -sleep 10 -date diff --git a/config/jedi/applications/recenter.yaml b/config/jedi/applications/recenter.yaml new file mode 100644 index 00000000..0820cdac --- /dev/null +++ b/config/jedi/applications/recenter.yaml @@ -0,0 +1,127 @@ +_state variables: &daStateVars +- water_vapor_mixing_ratio_wrt_dry_air +- cloud_liquid_water +- rain_water +- cloud_liquid_ice +- snow_water +- graupel +- cloud_ice_number_concentration +- rain_number_concentration +- cloud_droplet_number_concentration +- u +- w +- dry_air_density +- pressure_p +- air_potential_temperature +- relative_humidity +- eastward_wind +- northward_wind +- air_pressure_at_surface +- cldfrac +- re_cloud +- re_ice +- re_snow +- refl10cm +- refl10cm_max +- rainc +- rainnc +- lai +- sfc_albedo +- mavail +- sfc_emiss +- thc +- ust +- z0 +- znt +- skin_temperature_at_surface +- snow +- snowc +- snowh +- sst +- tmn +- vegetation_area_fraction +- seaice +- seaice_fraction +- eastward_wind_at_10m +- northward_wind_at_10m +- water_vapor_mixing_ratio_wrt_moist_air_at_2m +- air_temperature_at_2m +- precipw +- sh2o +- smois +- tslb + +# The following variables from the da_stream are not recentered: +# 1) static variables (terrain, hydrostatic balances): +#- ter +#- xland +#- dzs +#- zs +#- pressure_base +#- rho_base +#- theta_base +# +# 2) real-valued variables that are identical across ensemble members: +#- sfc_albbck +#- sfc_emibck +# +# 3) scalar integer variables identical across ensemble members +# (a real-valued mean would not be meaningful) +#- isice_lu +#- iswater_lu +# +# 4) variables containing information about past states +#- xicem +#- h_oml_initial +# +# 5) redundant variables +#- air_pressure (note: wa already recentered pressure_p) + +_current date: ¤tDate {{cycleDateISO8601}} #2018-04-15T06:00:00Z + +recenter variables: *daStateVars + +center geometry: + nml_file: {{namelistFileCenter}} # ./data/120km/namelist.atmosphere_120km + streams_file: {{streamsFileCenter}} # ./data/120km/streams.atmosphere_120km + deallocate non-da fields: false + +center: + #filename: /glade/u/home/stoedtli/scratch/pandac/stoedtli_3denvar_120km_uncoupled_1/CyclingDA/2018041506/an/an.$Y-$M-$D_$h.$m.$s.nc + filename: {{analysisFileCenter}} # ./data/c5/ens_mean.$Y-$M-$D_$h.$m.$s.nc + date: *currentDate + state variables: *daStateVars + stream name: da_state + transform model to analysis: false + +#center output: + #filename: ./data/c5/center.$Y-$M-$D_$h.$m.$s.nc + #stream name: da_state + +ensemble geometry: + nml_file: {{namelistFileEnsemble}} # ./data/120km/namelist.atmosphere_120km + streams_file: {{streamsFileEnsemble}} # ./data/120km/streams.atmosphere_120km + deallocate non-da fields: false + +ensemble: + members from template: + template: + date: *currentDate + state variables: *daStateVars + #filename: /glade/u/home/stoedtli/scratch/pandac/stoedtli_getkf_120km_20mem_uncoupled_1/CyclingDA/2018041506/an/mem%iMember%/an.$Y-$M-$D_$h.$m.$s.nc + filename: {{analysisFileEnsemble}} # /glade/u/home/stoedtli/scratch/pandac/stoedtli_getkf_120km_20mem_uncoupled_1/CyclingDA/2018041506/an/mem%iMember%/an.$Y-$M-$D_$h.$m.$s.nc + stream name: da_state + transform model to analysis: false + pattern: %iMember% + start: 1 + zero padding: {{paddingEnsembleMembers}} + nmembers: {{numberEnsembleMembers}} + +#ensemble meanoutput: + #filename: ./data/c5/mean.$Y-$M-$D_$h.$m.$s.nc + #stream name: da_state + +recentered output: + filename: {{analysisFileRecenter}} #./data/c5/mem%{member}%_recentered.$Y-$M-$D_$h.$m.$s.nc + stream name: da_state + From e11a7837be9b367a10c8b3516e88ea48da203ec4 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Mon, 8 Dec 2025 12:08:23 -0700 Subject: [PATCH 10/15] Clean up recenter yaml file --- config/jedi/applications/recenter.yaml | 81 ++++++++++++-------------- 1 file changed, 36 insertions(+), 45 deletions(-) diff --git a/config/jedi/applications/recenter.yaml b/config/jedi/applications/recenter.yaml index 0820cdac..a3ab560a 100644 --- a/config/jedi/applications/recenter.yaml +++ b/config/jedi/applications/recenter.yaml @@ -1,3 +1,31 @@ +# Variables to recenter: _state variables +# --------------------------------------- +# All variables from the da_stream I/O stream are recentered, except for the following: +# 1) static variables (terrain, hydrostatic balances): +# - ter +# - xland +# - dzs +# - zs +# - pressure_base +# - rho_base +# - theta_base +# +# 2) real-valued variables that are identical across ensemble members: +# - sfc_albbck +# - sfc_emibck +# +# 3) scalar integer variables identical across ensemble members +# (a real-valued mean would not be meaningful) +# - isice_lu +# - iswater_lu +# +# 4) variables containing information about past states +# - xicem +# - h_oml_initial +# +# 5) redundant variables +# - air_pressure (note: we do recenter pressure_p) + _state variables: &daStateVars - water_vapor_mixing_ratio_wrt_dry_air - cloud_liquid_water @@ -51,56 +79,25 @@ _state variables: &daStateVars - smois - tslb -# The following variables from the da_stream are not recentered: -# 1) static variables (terrain, hydrostatic balances): -#- ter -#- xland -#- dzs -#- zs -#- pressure_base -#- rho_base -#- theta_base -# -# 2) real-valued variables that are identical across ensemble members: -#- sfc_albbck -#- sfc_emibck -# -# 3) scalar integer variables identical across ensemble members -# (a real-valued mean would not be meaningful) -#- isice_lu -#- iswater_lu -# -# 4) variables containing information about past states -#- xicem -#- h_oml_initial -# -# 5) redundant variables -#- air_pressure (note: wa already recentered pressure_p) - -_current date: ¤tDate {{cycleDateISO8601}} #2018-04-15T06:00:00Z +_current date: ¤tDate {{cycleDateISO8601}} recenter variables: *daStateVars center geometry: - nml_file: {{namelistFileCenter}} # ./data/120km/namelist.atmosphere_120km - streams_file: {{streamsFileCenter}} # ./data/120km/streams.atmosphere_120km + nml_file: {{namelistFileCenter}} + streams_file: {{streamsFileCenter}} deallocate non-da fields: false center: - #filename: /glade/u/home/stoedtli/scratch/pandac/stoedtli_3denvar_120km_uncoupled_1/CyclingDA/2018041506/an/an.$Y-$M-$D_$h.$m.$s.nc - filename: {{analysisFileCenter}} # ./data/c5/ens_mean.$Y-$M-$D_$h.$m.$s.nc + filename: {{analysisFileCenter}} date: *currentDate state variables: *daStateVars stream name: da_state transform model to analysis: false -#center output: - #filename: ./data/c5/center.$Y-$M-$D_$h.$m.$s.nc - #stream name: da_state - ensemble geometry: - nml_file: {{namelistFileEnsemble}} # ./data/120km/namelist.atmosphere_120km - streams_file: {{streamsFileEnsemble}} # ./data/120km/streams.atmosphere_120km + nml_file: {{namelistFileEnsemble}} + streams_file: {{streamsFileEnsemble}} deallocate non-da fields: false ensemble: @@ -108,8 +105,7 @@ ensemble: template: date: *currentDate state variables: *daStateVars - #filename: /glade/u/home/stoedtli/scratch/pandac/stoedtli_getkf_120km_20mem_uncoupled_1/CyclingDA/2018041506/an/mem%iMember%/an.$Y-$M-$D_$h.$m.$s.nc - filename: {{analysisFileEnsemble}} # /glade/u/home/stoedtli/scratch/pandac/stoedtli_getkf_120km_20mem_uncoupled_1/CyclingDA/2018041506/an/mem%iMember%/an.$Y-$M-$D_$h.$m.$s.nc + filename: {{analysisFileEnsemble}} stream name: da_state transform model to analysis: false pattern: %iMember% @@ -117,11 +113,6 @@ ensemble: zero padding: {{paddingEnsembleMembers}} nmembers: {{numberEnsembleMembers}} -#ensemble meanoutput: - #filename: ./data/c5/mean.$Y-$M-$D_$h.$m.$s.nc - #stream name: da_state - recentered output: - filename: {{analysisFileRecenter}} #./data/c5/mem%{member}%_recentered.$Y-$M-$D_$h.$m.$s.nc + filename: {{analysisFileRecenter}} stream name: da_state - From 0be745d4500132bcb6cfb47dab2fe066e0f12084 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Mon, 8 Dec 2025 16:50:11 -0700 Subject: [PATCH 11/15] First working version of fully fledged recenter application --- bin/RecenterAnalysisEnsemble.csh | 88 +++++++++++++++++++-------- scenarios/defaults/enkf.yaml | 2 +- scenarios/getkf_OIE120km_coupled.yaml | 14 ++--- 3 files changed, 71 insertions(+), 33 deletions(-) diff --git a/bin/RecenterAnalysisEnsemble.csh b/bin/RecenterAnalysisEnsemble.csh index 9820af0d..6e29c4ec 100755 --- a/bin/RecenterAnalysisEnsemble.csh +++ b/bin/RecenterAnalysisEnsemble.csh @@ -5,32 +5,44 @@ # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# Recenters an analysis ensemble on a given center state +# Recenters an analysis ensemble on a given center state. +# Currently only works for an EnKF analysis ensemble +source config/environmentJEDI.csh # sets up the environment etc. source config/auto/build.csh # RecenterBuildDir, RecenterExe source config/auto/enkf.csh # ensPbMemNDigits, ensPbNMembers, ensPbMemPrefix, workDirVarDA +source config/auto/experiment.csh # ConfigDir source config/auto/model.csh # outerNamelistFile, outerStreamsFile source config/auto/naming.csh # analysisSubDir, ANFilePrefix, DAWorkDir -# Copy configuration file -# todo: change to run directory of cycling run -set yamlFileRecenter = "ens_recenter_test.yaml" -if ( -f "$yamlFileRecenter" ) then - rm "$yamlFileRecenter" -endif -cp "ens_recenter_template.yaml" "$yamlFileRecenter" - -# Date information -set CYLC_TASK_CYCLE_POINT = "20180415T0600Z" # will be set by cylc eventually +# Obtain date information +# set CYLC_TASK_CYCLE_POINT = "20180415T0600Z" # will be set by cylc eventually +echo "Cylc task cycle point: $CYLC_TASK_CYCLE_POINT" # debugging set yyyy = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 1-4` set mm = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 5-6` set dd = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 7-8` set hh = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 10-11` set cycleDate = "${yyyy}${mm}${dd}${hh}" set cycleDateISO8601 = "${yyyy}-${mm}-${dd}T${hh}:00:00Z" +set cycleDateMpas = "${yyyy}-${mm}-${dd}_${hh}.00.00" +echo "Current cycle date: $cycleDate" # debugging + +# Change to work directory and copy configuration yaml file +set cyclingDADirEnKF = "${DAWorkDir}/${cycleDate}" +set yamlFileRecenter = "recenter.yaml" +cd "${cyclingDADirEnKF}/run" +echo "Working in directory: `pwd`" # debugging +cp -v "${ConfigDir}/jedi/applications/${yamlFileRecenter}" "$yamlFileRecenter" +if ( $status != 0 ) then + echo "ERROR: recenter yaml not available --> $yamlFileRecenter" > ./FAIL + exit 1 +endif + +# Populate placeholders in yaml file +# 1) date information sed -i "s@{{cycleDateISO8601}}@$cycleDateISO8601@" $yamlFileRecenter -# Center geometry information +# 2) center geometry information # todo: update so that varDA can have different resolution set cyclingDADirVar = "${workDirVarDA}/CyclingDA/${cycleDate}" set namelistFileCenter = "${cyclingDADirVar}/${outerNamelistFile}" @@ -38,32 +50,58 @@ set streamsFileCenter = "${cyclingDADirVar}/${outerStreamsFile}" sed -i "s@{{namelistFileCenter}}@$namelistFileCenter@" $yamlFileRecenter sed -i "s@{{streamsFileCenter}}@$streamsFileCenter@" $yamlFileRecenter -# Center analysis information. This assumes that analysisSubDir and ANFilePrefix in -# the EnKF and var DA run are identical. This is not necessarily true but reasonable -set mpasFileSuffix = '$Y-$M-$D_$h.$m.$s.nc' +# 3) center analysis information. +# The current assumption is that analysisSubDir and ANFilePrefix in the +# EnKF and var DA run are identical. This is not necessarily true but reasonable +# set mpasFileSuffix = '$Y-$M-$D_$h.$m.$s.nc' +set mpasFileSuffix = "${cycleDateMpas}.nc" set analysisFileCenter = "${cyclingDADirVar}/${analysisSubDir}/${ANFilePrefix}.${mpasFileSuffix}" sed -i "s@{{analysisFileCenter}}@$analysisFileCenter@" $yamlFileRecenter -# EnKF ensemble geometry information -set cyclingDADirEnKF = "${DAWorkDir}/${cycleDate}" +# 4) EnKF ensemble geometry information +# set cyclingDADirEnKF = "${DAWorkDir}/${cycleDate}" set namelistFileEnsemble = "${cyclingDADirEnKF}/${outerNamelistFile}" set streamsFileEnsemble = "${cyclingDADirEnKF}/${outerStreamsFile}" sed -i "s@{{namelistFileEnsemble}}@$namelistFileEnsemble@" $yamlFileRecenter sed -i "s@{{streamsFileEnsemble}}@$streamsFileEnsemble@" $yamlFileRecenter -# EnKF analysis ensemble information -# todo: rename original analysis files and update this section -set analysisFileEnsemble = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${ANFilePrefix}.${mpasFileSuffix}" +# 5) EnKF analysis ensemble information +# Add an _orig suffix to label the original analysis ensemble member +set analysisFileEnsembleBase = "${ANFilePrefix}_orig.${mpasFileSuffix}" +# set analysisFileEnsemble = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${ANFilePrefix}_orig.${mpasFileSuffix}" +set analysisFileEnsemble = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${analysisFileEnsembleBase}" sed -i "s@{{analysisFileEnsemble}}@$analysisFileEnsemble@" $yamlFileRecenter sed -i "s@{{paddingEnsembleMembers}}@$ensPbMemNDigits@" $yamlFileRecenter sed -i "s@{{numberEnsembleMembers}}@$ensPbNMembers@" $yamlFileRecenter -# Recentered output -set analysisFileRecenter = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${ANFilePrefix}.${mpasFileSuffix}" +# 6) Recentered output +# The recentered analysis files obtain the standard analysis file name, so that a subsequent +# forecast can be started from them +set analysisFileRecenterBase = "${ANFilePrefix}.${mpasFileSuffix}" +# set analysisFileRecenter = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${ANFilePrefix}.${mpasFileSuffix}" +set analysisFileRecenter = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%{member}%/${analysisFileRecenterBase}" sed -i "s@{{analysisFileRecenter}}@$analysisFileRecenter@" $yamlFileRecenter -# Link and run executable -ln -sfv "${RecenterBuildDir}/${RecenterEXE}" ./ -#mpiexec ./${RecenterEXE} +# Copy original analysis files (recall that the recentered files have the original name) +@ i = 1 +while ( $i <= $ensPbNMembers ) + set memberDirBase = `printf "${ensPbMemPrefix}%0${ensPbMemNDigits}d" $i` + set memberDir = "${cyclingDADirEnKF}/${analysisSubDir}/${memberDirBase}" + cp -v "${memberDir}/${analysisFileRecenterBase}" "${memberDir}/${analysisFileEnsembleBase}" + @ i++ +end +# Link and run the recentering executable. This overwrites the analysis files +set jediOutputFile = "recenter.log" +ln -sfv "${RecenterBuildDir}/${RecenterEXE}" ./ +mpiexec "./${RecenterEXE}" "$yamlFileRecenter" "./${jediOutputFile}" >& recenter.log.all +# Make sure the application terminated successfully +grep 'Run: Finishing oops.* with status = 0' "$jediOutputFile" +if ( $status != 0 ) then + echo "ERROR: recenter application failed" > ./FAIL + exit 1 +# to do: is a further cleanup along these lines necessary? +# else +# rm ${appName}.log.0* +endif diff --git a/scenarios/defaults/enkf.yaml b/scenarios/defaults/enkf.yaml index 741a71d9..26328d54 100644 --- a/scenarios/defaults/enkf.yaml +++ b/scenarios/defaults/enkf.yaml @@ -149,4 +149,4 @@ enkf: PEPerNode: 1 memory: 45GB baseSeconds: 100 - secondsPerMember: 5 + secondsPerMember: 10 diff --git a/scenarios/getkf_OIE120km_coupled.yaml b/scenarios/getkf_OIE120km_coupled.yaml index aa08aab7..7ff59159 100644 --- a/scenarios/getkf_OIE120km_coupled.yaml +++ b/scenarios/getkf_OIE120km_coupled.yaml @@ -1,7 +1,7 @@ experiment: prefix: 'stoedtli_' name: 'getkf_coupled_test_' - suffix: '3' + suffix: '5' enkf: solver: GETKF @@ -13,13 +13,13 @@ enkf: # inflation rtpp value: 0.5 rtps value: 0.9 - # ensemble OMA calculation + # ensemble observation equivalent (background, analysis) retainObsFeedback: True - concatenateObsFeedback: True + concatenateObsFeedback: False # coupling to var DA (bias correction, recentering) coupledToVarDA: True workflowNameVarDA: 'stoedtli_3dhybrid_coupled_test_3' - recenterAnalyses: False # not implemented yet + recenterAnalyses: True biasCorrection: True #staticVarBcDir: /glade/campaign/mmm/parc/jban/pandac/year7Exp/jban_3dhybrid-60-60-iter_O30kmI60km_mhsRaw_clrsky/CyclingDA post: [] @@ -33,9 +33,6 @@ firstbackground: forecast: post: [] -hofx: - retainObsFeedback: False - members: n: 20 @@ -52,3 +49,6 @@ observations: workflow: first cycle point: 20180414T18 final cycle point: 20180415T00 + +build: + mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build From b37cff0217a331b1248e6ac7e400dfa6a8a66703 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Fri, 12 Dec 2025 13:43:37 -0700 Subject: [PATCH 12/15] Update build directory for recenter application --- initialize/framework/Build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/initialize/framework/Build.py b/initialize/framework/Build.py index b2e6e0cc..42f607bf 100644 --- a/initialize/framework/Build.py +++ b/initialize/framework/Build.py @@ -112,7 +112,7 @@ def __init__(self, config:Config, model:Model=None): ## Ensemble recentering self._set('RecenterEXE', 'mpasjedi_ens_recenter.x') - self._set('RecenterBuildDir', '/glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build/bin') + self._set('RecenterBuildDir', self['mpas bundle']+'/bin') if model is not None: From f24a76959086752c12a4378998c21c398d0ecfb7 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Fri, 12 Dec 2025 13:46:54 -0700 Subject: [PATCH 13/15] Add resource for uncoupled EnKF ensemble --- scenarios/defaults/variational.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scenarios/defaults/variational.yaml b/scenarios/defaults/variational.yaml index 4aaf73de..3d72d32e 100644 --- a/scenarios/defaults/variational.yaml +++ b/scenarios/defaults/variational.yaml @@ -125,6 +125,13 @@ variational: memberNDigits: 3 maxMembers: 20 + GETKF_uncoupled: + 120km: + directory0: '/glade/derecho/scratch/stoedtli/pandac/stoedtli_getkf_120km_20mem_uncoupled_1/CyclingFC/{{prevDateTime}}' + memberPrefix: mem + memberNDigits: 3 + maxMembers: 20 + # externally-produced localization files localization: #{{ensembleMesh}}: From f1efe9e741ce79eaa928ba27e376f2ae40153962 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Fri, 12 Dec 2025 13:51:07 -0700 Subject: [PATCH 14/15] Replace 3dhybrid by 3denvar scenario for coupled runs --- ...pled.yaml => 3denvar_OIE120km_coupled.yaml} | 18 ++++++++++-------- scenarios/getkf_OIE120km_coupled.yaml | 8 ++++---- 2 files changed, 14 insertions(+), 12 deletions(-) rename scenarios/{3dhybrid_OIE120km_coupled.yaml => 3denvar_OIE120km_coupled.yaml} (67%) diff --git a/scenarios/3dhybrid_OIE120km_coupled.yaml b/scenarios/3denvar_OIE120km_coupled.yaml similarity index 67% rename from scenarios/3dhybrid_OIE120km_coupled.yaml rename to scenarios/3denvar_OIE120km_coupled.yaml index e9a5ea67..c28d7740 100644 --- a/scenarios/3dhybrid_OIE120km_coupled.yaml +++ b/scenarios/3denvar_OIE120km_coupled.yaml @@ -1,14 +1,14 @@ experiment: prefix: "stoedtli_" - name: "3dhybrid_coupled_test_" - suffix: "2" + name: '3denvar_120km_coupled_' + suffix: "1" firstbackground: resource: "PANDAC.GFS" forecast: #execute: False # the default is True - post: [] # use this when doing extended forecast + post: ['verifymodel'] # use this when doing extended forecast externalanalyses: resource: "GFS.PANDAC" @@ -25,22 +25,24 @@ observations: resource: PANDACArchiveForVarBC variational: - DAType: 3dhybrid + DAType: 3denvar biasCorrection: True - ensembleCovarianceWeight: 0.75 - staticCovarianceWeight: 0.25 nInnerIterations: [60,60,] ensemble: forecasts: + # resource: "PANDAC.GETKF_uncoupled" resource: "PANDAC.GETKF_coupled" #execute: False # the default is True post: [] # this turns off verifyobs coupledToEnKF: True - workflowNameEnKF: "stoedtli_getkf_coupled_test_2" + workflowNameEnKF: "stoedtli_getkf_120km_20mem_coupled_1" workflow: first cycle point: 20180414T18 #restart cycle point: 20180415T00 - final cycle point: 20180415T00 + final cycle point: 20180422T00 #CyclingWindowHR: 24 # default is 6 for cycling DA #max active cycle points: 4 # used for independent 'extendedforecast' + +build: + mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build diff --git a/scenarios/getkf_OIE120km_coupled.yaml b/scenarios/getkf_OIE120km_coupled.yaml index 7ff59159..f73beec5 100644 --- a/scenarios/getkf_OIE120km_coupled.yaml +++ b/scenarios/getkf_OIE120km_coupled.yaml @@ -1,7 +1,7 @@ experiment: prefix: 'stoedtli_' - name: 'getkf_coupled_test_' - suffix: '5' + name: 'getkf_120km_20mem_coupled_' + suffix: '1' enkf: solver: GETKF @@ -18,7 +18,7 @@ enkf: concatenateObsFeedback: False # coupling to var DA (bias correction, recentering) coupledToVarDA: True - workflowNameVarDA: 'stoedtli_3dhybrid_coupled_test_3' + workflowNameVarDA: 'stoedtli_3denvar_120km_coupled_1' recenterAnalyses: True biasCorrection: True #staticVarBcDir: /glade/campaign/mmm/parc/jban/pandac/year7Exp/jban_3dhybrid-60-60-iter_O30kmI60km_mhsRaw_clrsky/CyclingDA @@ -48,7 +48,7 @@ observations: workflow: first cycle point: 20180414T18 - final cycle point: 20180415T00 + final cycle point: 20180422T00 build: mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build From 7eb14d8e72adc01b52ad63138c212b17a531b5b2 Mon Sep 17 00:00:00 2001 From: Simon Toedtli Date: Fri, 12 Dec 2025 13:55:15 -0700 Subject: [PATCH 15/15] Add uncoupled baseline scenarios --- scenarios/3denvar_OIE120km_uncoupled.yaml | 46 +++++++++++++++++++ scenarios/getkf_OIE120km_uncoupled.yaml | 55 +++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 scenarios/3denvar_OIE120km_uncoupled.yaml create mode 100644 scenarios/getkf_OIE120km_uncoupled.yaml diff --git a/scenarios/3denvar_OIE120km_uncoupled.yaml b/scenarios/3denvar_OIE120km_uncoupled.yaml new file mode 100644 index 00000000..e7767e79 --- /dev/null +++ b/scenarios/3denvar_OIE120km_uncoupled.yaml @@ -0,0 +1,46 @@ +experiment: + prefix: "stoedtli_" + name: '3denvar_120km_uncoupled_' + suffix: "1" + +firstbackground: + resource: "PANDAC.GFS" + +forecast: + execute: False # the default is True + post: ['verifymodel'] # use this when doing extended forecast + +externalanalyses: + resource: "GFS.PANDAC" + +members: + n: 1 + +model: + outerMesh: 120km + innerMesh: 120km + ensembleMesh: 120km + +observations: + resource: PANDACArchiveForVarBC + +variational: + DAType: 3denvar + biasCorrection: True + nInnerIterations: [60,60,] + ensemble: + forecasts: + resource: "PANDAC.GETKF_uncoupled" + execute: False # the default is True + post: [] # this turns off verifyobs + coupledToEnKF: False + +workflow: + first cycle point: 20180414T18 + #restart cycle point: 20180415T00 + final cycle point: 20180422T00 + #CyclingWindowHR: 24 # default is 6 for cycling DA + #max active cycle points: 4 # used for independent 'extendedforecast' + +build: + mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build diff --git a/scenarios/getkf_OIE120km_uncoupled.yaml b/scenarios/getkf_OIE120km_uncoupled.yaml new file mode 100644 index 00000000..2552fcc1 --- /dev/null +++ b/scenarios/getkf_OIE120km_uncoupled.yaml @@ -0,0 +1,55 @@ +experiment: + prefix: 'stoedtli_' + name: 'getkf_120km_20mem_uncoupled_' + suffix: '1' + +enkf: + solver: GETKF + # localization + horizontal localization lengthscale: 1.2e6 + vertical localization lengthscale: 6000.0 + vertical localization lengthscale units: height + fraction of retained variance: 0.95 + # inflation + rtpp value: 0.5 + rtps value: 0.9 + # ensemble observation equivalent (background, analysis) + retainObsFeedback: True + concatenateObsFeedback: False + # coupling to var DA (bias correction, recentering) + coupledToVarDA: False + #workflowNameVarDA: 'stoedtli_3dhybrid_coupled_test_3' + #recenterAnalyses: False # not implemented yet + biasCorrection: True + staticVarBcDir: /glade/campaign/mmm/parc/jban/pandac/year7Exp/jban_3dhybrid-60-60-iter_O30kmI60km_mhsRaw_clrsky/CyclingDA + post: [] + +externalanalyses: + resource: "GFS.PANDAC" + +firstbackground: + resource: "PANDAC.EnsPertB" + +forecast: + post: [] + +members: + n: 20 + +model: + outerMesh: 120km + # TODO: make inner and ensemble meshes unnecessary + # related to {{PrepareExternalAnalysisInner}} and {{PrepareExternalAnalysisEnsemble}} + innerMesh: 120km + ensembleMesh: 120km + +observations: + resource: PANDACArchiveForVarBC + +workflow: + first cycle point: 20180414T18 + #restart cycle point: 20180415T06 + final cycle point: 20180422T00 + +build: + mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build