diff --git a/bin/RecenterAnalysisEnsemble.csh b/bin/RecenterAnalysisEnsemble.csh new file mode 100755 index 00000000..6e29c4ec --- /dev/null +++ b/bin/RecenterAnalysisEnsemble.csh @@ -0,0 +1,107 @@ +#!/bin/csh -f + +# (C) Copyright 2025 UCAR +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# Recenters an analysis ensemble on a given center state. +# Currently only works for an EnKF analysis ensemble + +source config/environmentJEDI.csh # sets up the environment etc. +source config/auto/build.csh # RecenterBuildDir, RecenterExe +source config/auto/enkf.csh # ensPbMemNDigits, ensPbNMembers, ensPbMemPrefix, workDirVarDA +source config/auto/experiment.csh # ConfigDir +source config/auto/model.csh # outerNamelistFile, outerStreamsFile +source config/auto/naming.csh # analysisSubDir, ANFilePrefix, DAWorkDir + +# Obtain date information +# set CYLC_TASK_CYCLE_POINT = "20180415T0600Z" # will be set by cylc eventually +echo "Cylc task cycle point: $CYLC_TASK_CYCLE_POINT" # debugging +set yyyy = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 1-4` +set mm = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 5-6` +set dd = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 7-8` +set hh = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 10-11` +set cycleDate = "${yyyy}${mm}${dd}${hh}" +set cycleDateISO8601 = "${yyyy}-${mm}-${dd}T${hh}:00:00Z" +set cycleDateMpas = "${yyyy}-${mm}-${dd}_${hh}.00.00" +echo "Current cycle date: $cycleDate" # debugging + +# Change to work directory and copy configuration yaml file +set cyclingDADirEnKF = "${DAWorkDir}/${cycleDate}" +set yamlFileRecenter = "recenter.yaml" +cd "${cyclingDADirEnKF}/run" +echo "Working in directory: `pwd`" # debugging +cp -v "${ConfigDir}/jedi/applications/${yamlFileRecenter}" "$yamlFileRecenter" +if ( $status != 0 ) then + echo "ERROR: recenter yaml not available --> $yamlFileRecenter" > ./FAIL + exit 1 +endif + +# Populate placeholders in yaml file +# 1) date information +sed -i "s@{{cycleDateISO8601}}@$cycleDateISO8601@" $yamlFileRecenter + +# 2) center geometry information +# todo: update so that varDA can have different resolution +set cyclingDADirVar = "${workDirVarDA}/CyclingDA/${cycleDate}" +set namelistFileCenter = "${cyclingDADirVar}/${outerNamelistFile}" +set streamsFileCenter = "${cyclingDADirVar}/${outerStreamsFile}" +sed -i "s@{{namelistFileCenter}}@$namelistFileCenter@" $yamlFileRecenter +sed -i "s@{{streamsFileCenter}}@$streamsFileCenter@" $yamlFileRecenter + +# 3) center analysis information. +# The current assumption is that analysisSubDir and ANFilePrefix in the +# EnKF and var DA run are identical. This is not necessarily true but reasonable +# set mpasFileSuffix = '$Y-$M-$D_$h.$m.$s.nc' +set mpasFileSuffix = "${cycleDateMpas}.nc" +set analysisFileCenter = "${cyclingDADirVar}/${analysisSubDir}/${ANFilePrefix}.${mpasFileSuffix}" +sed -i "s@{{analysisFileCenter}}@$analysisFileCenter@" $yamlFileRecenter + +# 4) EnKF ensemble geometry information +# set cyclingDADirEnKF = "${DAWorkDir}/${cycleDate}" +set namelistFileEnsemble = "${cyclingDADirEnKF}/${outerNamelistFile}" +set streamsFileEnsemble = "${cyclingDADirEnKF}/${outerStreamsFile}" +sed -i "s@{{namelistFileEnsemble}}@$namelistFileEnsemble@" $yamlFileRecenter +sed -i "s@{{streamsFileEnsemble}}@$streamsFileEnsemble@" $yamlFileRecenter + +# 5) EnKF analysis ensemble information +# Add an _orig suffix to label the original analysis ensemble member +set analysisFileEnsembleBase = "${ANFilePrefix}_orig.${mpasFileSuffix}" +# set analysisFileEnsemble = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${ANFilePrefix}_orig.${mpasFileSuffix}" +set analysisFileEnsemble = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${analysisFileEnsembleBase}" +sed -i "s@{{analysisFileEnsemble}}@$analysisFileEnsemble@" $yamlFileRecenter +sed -i "s@{{paddingEnsembleMembers}}@$ensPbMemNDigits@" $yamlFileRecenter +sed -i "s@{{numberEnsembleMembers}}@$ensPbNMembers@" $yamlFileRecenter + +# 6) Recentered output +# The recentered analysis files obtain the standard analysis file name, so that a subsequent +# forecast can be started from them +set analysisFileRecenterBase = "${ANFilePrefix}.${mpasFileSuffix}" +# set analysisFileRecenter = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%iMember%/${ANFilePrefix}.${mpasFileSuffix}" +set analysisFileRecenter = "${cyclingDADirEnKF}/${analysisSubDir}/${ensPbMemPrefix}%{member}%/${analysisFileRecenterBase}" +sed -i "s@{{analysisFileRecenter}}@$analysisFileRecenter@" $yamlFileRecenter + +# Copy original analysis files (recall that the recentered files have the original name) +@ i = 1 +while ( $i <= $ensPbNMembers ) + set memberDirBase = `printf "${ensPbMemPrefix}%0${ensPbMemNDigits}d" $i` + set memberDir = "${cyclingDADirEnKF}/${analysisSubDir}/${memberDirBase}" + cp -v "${memberDir}/${analysisFileRecenterBase}" "${memberDir}/${analysisFileEnsembleBase}" + @ i++ +end + +# Link and run the recentering executable. This overwrites the analysis files +set jediOutputFile = "recenter.log" +ln -sfv "${RecenterBuildDir}/${RecenterEXE}" ./ +mpiexec "./${RecenterEXE}" "$yamlFileRecenter" "./${jediOutputFile}" >& recenter.log.all + +# Make sure the application terminated successfully +grep 'Run: Finishing oops.* with status = 0' "$jediOutputFile" +if ( $status != 0 ) then + echo "ERROR: recenter application failed" > ./FAIL + exit 1 +# to do: is a further cleanup along these lines necessary? +# else +# rm ${appName}.log.0* +endif diff --git a/config/jedi/applications/recenter.yaml b/config/jedi/applications/recenter.yaml new file mode 100644 index 00000000..a3ab560a --- /dev/null +++ b/config/jedi/applications/recenter.yaml @@ -0,0 +1,118 @@ +# Variables to recenter: _state variables +# --------------------------------------- +# All variables from the da_stream I/O stream are recentered, except for the following: +# 1) static variables (terrain, hydrostatic balances): +# - ter +# - xland +# - dzs +# - zs +# - pressure_base +# - rho_base +# - theta_base +# +# 2) real-valued variables that are identical across ensemble members: +# - sfc_albbck +# - sfc_emibck +# +# 3) scalar integer variables identical across ensemble members +# (a real-valued mean would not be meaningful) +# - isice_lu +# - iswater_lu +# +# 4) variables containing information about past states +# - xicem +# - h_oml_initial +# +# 5) redundant variables +# - air_pressure (note: we do recenter pressure_p) + +_state variables: &daStateVars +- water_vapor_mixing_ratio_wrt_dry_air +- cloud_liquid_water +- rain_water +- cloud_liquid_ice +- snow_water +- graupel +- cloud_ice_number_concentration +- rain_number_concentration +- cloud_droplet_number_concentration +- u +- w +- dry_air_density +- pressure_p +- air_potential_temperature +- relative_humidity +- eastward_wind +- northward_wind +- air_pressure_at_surface +- cldfrac +- re_cloud +- re_ice +- re_snow +- refl10cm +- refl10cm_max +- rainc +- rainnc +- lai +- sfc_albedo +- mavail +- sfc_emiss +- thc +- ust +- z0 +- znt +- skin_temperature_at_surface +- snow +- snowc +- snowh +- sst +- tmn +- vegetation_area_fraction +- seaice +- seaice_fraction +- eastward_wind_at_10m +- northward_wind_at_10m +- water_vapor_mixing_ratio_wrt_moist_air_at_2m +- air_temperature_at_2m +- precipw +- sh2o +- smois +- tslb + +_current date: ¤tDate {{cycleDateISO8601}} + +recenter variables: *daStateVars + +center geometry: + nml_file: {{namelistFileCenter}} + streams_file: {{streamsFileCenter}} + deallocate non-da fields: false + +center: + filename: {{analysisFileCenter}} + date: *currentDate + state variables: *daStateVars + stream name: da_state + transform model to analysis: false + +ensemble geometry: + nml_file: {{namelistFileEnsemble}} + streams_file: {{streamsFileEnsemble}} + deallocate non-da fields: false + +ensemble: + members from template: + template: + date: *currentDate + state variables: *daStateVars + filename: {{analysisFileEnsemble}} + stream name: da_state + transform model to analysis: false + pattern: %iMember% + start: 1 + zero padding: {{paddingEnsembleMembers}} + nmembers: {{numberEnsembleMembers}} + +recentered output: + filename: {{analysisFileRecenter}} + stream name: da_state diff --git a/initialize/applications/DA.py b/initialize/applications/DA.py index 74f2c27d..a6e2043e 100644 --- a/initialize/applications/DA.py +++ b/initialize/applications/DA.py @@ -176,6 +176,7 @@ def export(self, previousForecast:str, ef:ExtendedForecast): for st in self.__subtasks: self._tasks += st._tasks self._dependencies += st._dependencies + self._xtriggers += st._xtriggers # depends on previous Forecast self.tf.addDependencies([previousForecast]) diff --git a/initialize/applications/EnKF.py b/initialize/applications/EnKF.py index 3689dcc1..5965cea1 100755 --- a/initialize/applications/EnKF.py +++ b/initialize/applications/EnKF.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 from collections import OrderedDict +from getpass import getuser +import os.path from initialize.applications.Members import Members @@ -86,6 +88,7 @@ class EnKF(Component): 'biasCorrection': [False, bool], # directories that stores varBC coefficients that are updated with variational DA + # if coupledToVarDA == True the staticVarBcDir is automatically set to the coupled var DA directory 'staticVarBcDir': ['/glade/campaign/mmm/parc/ivette/pandac/year7Exp/ivette_3dhybrid-allsky-60-60-iter_O30kmI60km_ensB-SE80+RTPP70_VarBC_v3.0.2_newBenchmark_allsky-amsua/CyclingDA', str], ## tropprsMethod @@ -118,6 +121,34 @@ class EnKF(Component): ## IR/VIS land surface coefficients classification # OPTIONS: USGS, IGBP, NPOESS 'IRVISlandCoeff': ['IGBP', str], + + ## Coupling to an external variational DA run + # When coupledToVarDA is true, this EnKF run can be recentered (recenterAnalyses == True) on the variational DA run and/or + # it can use the variational bias correction coefficients (biasCorrection == True) of the variational DA run + 'coupledToVarDA': [False, bool], + + ## Recentering of analysis ensemble + # When recenterAnalyses is true, the analysis ensemble of this EnKF is recentered on the analysis of a + # concurrently running var DA run. Requires coupledToVarDA == True. + # The recentering application is not implemented yet. + 'recenterAnalyses': [False, bool], + } + + optionalVariables = { + ## Coupling to an external variational DA run + # Must be specified if coupledToVarDA == True, not used otherwise + # Specify the full name of the workflow, constructed as: + # Experiment['prefix'] + Experiment['name'] + Experiment['suffix'] + Experiment['suite identifier'] + # The default values are: + # Experiment['prefix'] = "${USER}_" + # Experiment['name']: see Cycle.py + # Experiment['suffix'] = "" + # Experiment['suite identifier'] = "" + # It is best practice to specify the experiment prefix, name, and suffix in the coupled yaml files + # to make sure the coupled workflow can be identified correctly. + # Note: the workflow automatically prepends 'MPAS-Workflow' to workflowNameVarDA where needed + # to make the identifier consistent with submit.csh + 'workflowNameVarDA': str, } def __init__(self, @@ -183,6 +214,31 @@ def __init__(self, # TODO: this needs to be non-zero for EnKF workflows that use IAU, get value from forecast self._set('ensPbOffsetHR', 0) + # coupling to variational DA + if self['coupledToVarDA']: + # A coupled EnKF run should use some information from the variational DA. + # Stop the run if no information is used in the current configuration to make sure + # user can adjust settings. + if not self['biasCorrection'] and not self['recenterAnalyses']: + raise ValueError(("Coupled EnKF does not use any information from var DA." + "biasCorrectionEnKF and/or recenterAnalyses should be set to true.")) + # A coupled EnKF run requires the name of the variational DA workflow to + # set the external dependency + if self['workflowNameVarDA'] is None: + raise ValueError("workflowNameVarDA has to be specified for a coupled EnKF") + # Export the var DA run information to the csh file + self._set('workflowNameVarDA', self['workflowNameVarDA']) + workDirVarDA = os.path.join('/glade', 'derecho', 'scratch', getuser(), + 'pandac', self['workflowNameVarDA']) + self._set('workDirVarDA', workDirVarDA) + # Overwrite the static var BC directory to make sure we are reading the satbias files from + # the coupled var DA run + self._set('staticVarBcDir', f'{os.path.join(workDirVarDA, "CyclingDA")}') + else: + # Recentering the analysis ensemble is not defined if the EnKF run is not coupled. + if self['recenterAnalyses']: + raise NotImplementedError("Recentering of the analyses ensemble is undefined if the EnKF run is not coupled.") + self._cshVars = list(self._vtable.keys()) ######################## @@ -262,6 +318,60 @@ def __init__(self, inherit = '''+self.tf.execute+''', BATCH script = $origin/bin/EnKF.csh Solver '''+solvertask.job()+solvertask.directives()] + + if self['coupledToVarDA']: + # The coupled EnKF run uses the analysis file and/or the variational bias correction files + # from the variational DA run. This introduces a dependency between the EnKF and the + # variational DA at the current cycle point. The following external trigger defines this dependency. + # Note: submit.csh prepends 'MPAS-Workflow/' to the workflow name to generate the workflow ID + workflowIdVarDA = os.path.join('MPAS-Workflow', self['workflowNameVarDA']) + callIntervalInS = 30 # call interval for external trigger (default is 10) + self._xtriggers +=[('\n' + ' var_da = workflow_state(' + f'workflow="{workflowIdVarDA}", ' + 'task="DAFinished__", point="%(point)s")' + f':PT{callIntervalInS}S')] + + # Add the dependency on the var DA run to the graph + self._dependencies += [('\n' + ' @var_da => ' + self.tf.pre)] + if self['recenterAnalyses']: + # Add the dependency: the EnKFDiagOMA task is optional. If the task is enabled, the recentering + # task should run after it. If it is not enabled, the recentering task should run after the solver task. + if self['diagEnKFOMA'] and self['retainObsFeedback']: + self._dependencies += [('\n' + ' EnKFDiagOMA => RecenterEnKF')] + else: + self._dependencies += [('\n' + ' EnKFSolver => RecenterEnKF')] + # Add the task + keyListRecenter = { + # Notes: + # - the recenter application is currently small enough to run on develop + # - the develop queue does not support a job_priority flag, so removed this for now + 'retry': {'t': str}, + 'baseSeconds': {'t': int}, + 'secondsPerMember': {'t': int}, + 'nodes': {'t': int}, + 'PEPerNode': {'t': int}, + 'memory': {'t': str}, + 'queue': {'def': hpc['SharedQueue']}, + 'account': {'def': hpc['CriticalAccount']}, + 'email': {'def': True, 't': bool}, + } + resourceRecenter = meshes['Outer'].name + '.' + solver + '.recenter' + recenterJob = Resource(self._conf, keyListRecenter, ('job', resourceRecenter)) + recenterJob._set('seconds', recenterJob['baseSeconds'] + recenterJob['secondsPerMember'] * NN) + recenterTask = TaskLookup[hpc.system](recenterJob) + self._tasks += [('\n' + ' [[RecenterEnKF]]' + '\n' + f' inherit = {self.tf.execute}, BATCH' + '\n' + ' script = $origin/bin/RecenterAnalysisEnsemble.csh' + '\n' + f'{recenterTask.job() + recenterTask.directives()}' + '\n')] if self['diagEnKFOMA'] and self['retainObsFeedback']: self._tasks += [''' diff --git a/initialize/applications/Variational.py b/initialize/applications/Variational.py index 65fb5a5b..61185ca4 100644 --- a/initialize/applications/Variational.py +++ b/initialize/applications/Variational.py @@ -8,6 +8,8 @@ ''' from collections import OrderedDict +from getpass import getuser +import os.path from initialize.applications.Members import Members @@ -40,6 +42,20 @@ class Variational(Component): 'ensembleCovarianceWeight': float, 'staticCovarianceWeight': float, + ## Coupling to an external EnKF run + # Must be specified if coupledToEnKF == True, not used otherwise + # Specify the full name of the workflow, constructed as: + # Experiment['prefix'] + Experiment['name'] + Experiment['suffix'] + Experiment['suite identifier'] + # The default values are: + # Experiment['prefix'] = "${USER}_" + # Experiment['name']: see Cycle.py + # Experiment['suffix'] = "" + # Experiment['suite identifier'] = "" + # It is best practice to specify the experiment prefix, name, and suffix in the coupled yaml files + # to make sure the coupled workflow can be identified correctly. + # Note: the workflow automatically prepends 'MPAS-Workflow' to workflowNameEnKF where needed + # to make the identifier consistent with submit.csh + 'workflowNameEnKF': str, } variablesWithDefaults = { @@ -154,6 +170,11 @@ class Variational(Component): ## IR/VIS land surface coefficients classification # OPTIONS: USGS, IGBP, NPOESS 'IRVISlandCoeff': ['IGBP', str], + + ## Coupling to an external EnKF run + # When coupledToEnKF is true, this variational run uses the forecast ensemble of a + # concurrently running external EnKF run to construct the ensemble B matrix. + 'coupledToEnKF': [False, bool], } def __init__(self, @@ -286,6 +307,22 @@ def __init__(self, self._setOrDie('.'.join(['covariance', r, 'bumpCovVBalDir']), str, None, 'bumpCovVBalDir') self._setOrDie('.'.join(['covariance', r, 'hybridCoefficientsDir']), str, None, 'hybridCoefficientsDir') + # coupling to EnKF + if self['coupledToEnKF']: + # The coupling to EnKF is meaningful if we run a single variational DA, but undefined + # if we run an EDA + if self.NN > 1: + raise NotImplementedError("Behavior of EDA coupled to EnKF is undefined. Set members.n == 1.") + # A coupled variational run requires the name of the EnKF workflow to + # set the external dependency + if self['workflowNameEnKF'] is None: + raise ValueError("workflowNameEnKF has to be specified for a coupled Var DA") + # Overwrite the ensemble directory to make sure we are reading the forecasts from the + # coupled EnKF run + workDirEnKF = os.path.join('/glade', 'derecho', 'scratch', getuser(), + 'pandac', self['workflowNameEnKF']) + self._set('ensPbDir0', f'{os.path.join(workDirEnKF, "CyclingFC", "{{prevDateTime}}")}') + self._cshVars = list(self._vtable.keys()) ######################## @@ -355,7 +392,26 @@ def __init__(self, [[Variationals]] inherit = '''+self.tf.execute+''' '''+vartask.job()+vartask.directives()] - + + if self['coupledToEnKF']: + # The coupled variational run uses the forecast ensemble from the EnKF to generate + # the ensemble B. This introduces a dependency between the variational DA at the current + # cycle point and the EnKF forecast from the previous cycle point. The following external + # trigger defines this dependency. + # Note: submit.csh prepends 'MPAS-Workflow/' to the workflow name to generate the workflow ID + workflowIdEnKF = os.path.join('MPAS-Workflow', self['workflowNameEnKF']) + callIntervalInS = 30 # call interval for external trigger (default is 10) + self._xtriggers +=[('\n' + ' enkf_forecast = workflow_state(' + f'workflow="{workflowIdEnKF}", ' + 'task="ForecastFinished__", point="%(point)s", ' + f'offset="-PT{workflow["CyclingWindowHR"]}H")' + f':PT{callIntervalInS}S')] + + # Add the external dependency to the graph + self._dependencies += [('\n' + ' @enkf_forecast => ' + self.tf.pre)] + if EDASize == 1: # single instance or ensemble of Variational(s) for mm in range(1, self.NN+1, 1): diff --git a/initialize/config/Component.py b/initialize/config/Component.py index 7b18e0d3..9aa1be08 100644 --- a/initialize/config/Component.py +++ b/initialize/config/Component.py @@ -37,6 +37,7 @@ def __init__(self, config:Config): self._queues = [] self._tasks = [] self._dependencies = [] + self._xtriggers = [] ################### # extract SubConfig diff --git a/initialize/framework/Build.py b/initialize/framework/Build.py index ff441d01..42f607bf 100644 --- a/initialize/framework/Build.py +++ b/initialize/framework/Build.py @@ -110,6 +110,10 @@ def __init__(self, config:Config, model:Model=None): self._set('SACAEXE', 'mpasjedi_saca.x') self._set('SACABuildDir', self['mpas bundle']+'/bin') + ## Ensemble recentering + self._set('RecenterEXE', 'mpasjedi_ens_recenter.x') + self._set('RecenterBuildDir', self['mpas bundle']+'/bin') + if model is not None: # MPAS-Model diff --git a/initialize/suites/Cycle.py b/initialize/suites/Cycle.py index bd6ca884..a0cd4bb5 100644 --- a/initialize/suites/Cycle.py +++ b/initialize/suites/Cycle.py @@ -115,3 +115,7 @@ def __init__(self, conf:Config): 'initic', 'observations', ] + + self.xtriggerComponents += [ + 'da' + ] diff --git a/initialize/suites/SuiteBase.py b/initialize/suites/SuiteBase.py index ca63c071..2b86ec58 100644 --- a/initialize/suites/SuiteBase.py +++ b/initialize/suites/SuiteBase.py @@ -28,10 +28,12 @@ def __init__(self, conf:Config): self.queueComponents = [] self.dependencyComponents = [] self.taskComponents = [] + self.xtriggerComponents = [] self._queues = [] self._dependencies = [] self._tasks = [] + self._xtriggers = [] self.logPrefix = self.__class__.__name__+': ' host = os.getenv('NCAR_HOST') @@ -122,6 +124,11 @@ def submit(self): self._tasks += [''' # '''+ k] self._tasks += self.c[k]._tasks + + for k in self.xtriggerComponents: + self._xtriggers += [''' + # ''' + k] + self._xtriggers += self.c[k]._xtriggers self.__export() @@ -170,6 +177,9 @@ def __export(self): # default: 3 runahead limit = P'''+str(self.c['workflow']['max active cycle points']-1)+''' + [[xtriggers]] +'''+''.join(self._xtriggers)+''' + [[queues]] '''+''.join(self._queues)+''' diff --git a/scenarios/3denvar_OIE120km_coupled.yaml b/scenarios/3denvar_OIE120km_coupled.yaml new file mode 100644 index 00000000..c28d7740 --- /dev/null +++ b/scenarios/3denvar_OIE120km_coupled.yaml @@ -0,0 +1,48 @@ +experiment: + prefix: "stoedtli_" + name: '3denvar_120km_coupled_' + suffix: "1" + +firstbackground: + resource: "PANDAC.GFS" + +forecast: + #execute: False # the default is True + post: ['verifymodel'] # use this when doing extended forecast + +externalanalyses: + resource: "GFS.PANDAC" + +members: + n: 1 + +model: + outerMesh: 120km + innerMesh: 120km + ensembleMesh: 120km + +observations: + resource: PANDACArchiveForVarBC + +variational: + DAType: 3denvar + biasCorrection: True + nInnerIterations: [60,60,] + ensemble: + forecasts: + # resource: "PANDAC.GETKF_uncoupled" + resource: "PANDAC.GETKF_coupled" + #execute: False # the default is True + post: [] # this turns off verifyobs + coupledToEnKF: True + workflowNameEnKF: "stoedtli_getkf_120km_20mem_coupled_1" + +workflow: + first cycle point: 20180414T18 + #restart cycle point: 20180415T00 + final cycle point: 20180422T00 + #CyclingWindowHR: 24 # default is 6 for cycling DA + #max active cycle points: 4 # used for independent 'extendedforecast' + +build: + mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build diff --git a/scenarios/3denvar_OIE120km_uncoupled.yaml b/scenarios/3denvar_OIE120km_uncoupled.yaml new file mode 100644 index 00000000..e7767e79 --- /dev/null +++ b/scenarios/3denvar_OIE120km_uncoupled.yaml @@ -0,0 +1,46 @@ +experiment: + prefix: "stoedtli_" + name: '3denvar_120km_uncoupled_' + suffix: "1" + +firstbackground: + resource: "PANDAC.GFS" + +forecast: + execute: False # the default is True + post: ['verifymodel'] # use this when doing extended forecast + +externalanalyses: + resource: "GFS.PANDAC" + +members: + n: 1 + +model: + outerMesh: 120km + innerMesh: 120km + ensembleMesh: 120km + +observations: + resource: PANDACArchiveForVarBC + +variational: + DAType: 3denvar + biasCorrection: True + nInnerIterations: [60,60,] + ensemble: + forecasts: + resource: "PANDAC.GETKF_uncoupled" + execute: False # the default is True + post: [] # this turns off verifyobs + coupledToEnKF: False + +workflow: + first cycle point: 20180414T18 + #restart cycle point: 20180415T00 + final cycle point: 20180422T00 + #CyclingWindowHR: 24 # default is 6 for cycling DA + #max active cycle points: 4 # used for independent 'extendedforecast' + +build: + mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build diff --git a/scenarios/defaults/enkf.yaml b/scenarios/defaults/enkf.yaml index dabc43f3..26328d54 100644 --- a/scenarios/defaults/enkf.yaml +++ b/scenarios/defaults/enkf.yaml @@ -127,30 +127,26 @@ enkf: secondsPerMember: 3 GETKF: observer: - # cost for record (20 members, 95% variance retained [13 eig], PBS JOB email) - # 8 x 32 PE : 12.2 min., 118 GB (single precision) - nodes: 2 - PEPerNode: 128 + nodes: 1 + PEPerNode: 64 memory: 235GB - baseSeconds: 800 - secondsPerMember: 50 + baseSeconds: 200 + secondsPerMember: 20 solver: - # cost for record (5 members, PBS JOB email) - # 4 x 32 PE : ?.? min., ?? GB (single precision) - # 8 x 32 PE : ?.? min., ?? GB (single precision) - nodes: 2 + nodes: 1 PEPerNode: 128 memory: 235GB baseSeconds: 200 - secondsPerMember: 300 + secondsPerMember: 20 diagoma: - # cost for record (5 members, PBS JOB email) - # 2 x 32 PE : 1.7 min., 14.6 GB (single precision) - # 2 x 32 PE : 3.5 min., 31 GB (double precision) - # cost for record (20 members, PBS JOB email) - # 2 x 32 PE : 3.0 min., 31.7 GB (single precision) - nodes: 2 - PEPerNode: 128 + nodes: 1 + PEPerNode: 64 memory: 235GB - baseSeconds: 300 - secondsPerMember: 3 + baseSeconds: 200 + secondsPerMember: 10 + recenter: + nodes: 1 + PEPerNode: 1 + memory: 45GB + baseSeconds: 100 + secondsPerMember: 10 diff --git a/scenarios/defaults/variational.yaml b/scenarios/defaults/variational.yaml index eddb2e24..3d72d32e 100644 --- a/scenarios/defaults/variational.yaml +++ b/scenarios/defaults/variational.yaml @@ -112,6 +112,26 @@ variational: # zero padding: 3 # nmembers: 80 + GETKF_coupled: + 120km: + directory0: 'setByCoupledWorkflow' + memberPrefix: mem + memberNDigits: 3 + maxMembers: 20 + + 60km: + directory0: 'setByCoupledWorkflow' + memberPrefix: mem + memberNDigits: 3 + maxMembers: 20 + + GETKF_uncoupled: + 120km: + directory0: '/glade/derecho/scratch/stoedtli/pandac/stoedtli_getkf_120km_20mem_uncoupled_1/CyclingFC/{{prevDateTime}}' + memberPrefix: mem + memberNDigits: 3 + maxMembers: 20 + # externally-produced localization files localization: #{{ensembleMesh}}: diff --git a/scenarios/getkf_OIE120km_coupled.yaml b/scenarios/getkf_OIE120km_coupled.yaml new file mode 100644 index 00000000..f73beec5 --- /dev/null +++ b/scenarios/getkf_OIE120km_coupled.yaml @@ -0,0 +1,54 @@ +experiment: + prefix: 'stoedtli_' + name: 'getkf_120km_20mem_coupled_' + suffix: '1' + +enkf: + solver: GETKF + # localization + horizontal localization lengthscale: 1.2e6 + vertical localization lengthscale: 6000.0 + vertical localization lengthscale units: height + fraction of retained variance: 0.95 + # inflation + rtpp value: 0.5 + rtps value: 0.9 + # ensemble observation equivalent (background, analysis) + retainObsFeedback: True + concatenateObsFeedback: False + # coupling to var DA (bias correction, recentering) + coupledToVarDA: True + workflowNameVarDA: 'stoedtli_3denvar_120km_coupled_1' + recenterAnalyses: True + biasCorrection: True + #staticVarBcDir: /glade/campaign/mmm/parc/jban/pandac/year7Exp/jban_3dhybrid-60-60-iter_O30kmI60km_mhsRaw_clrsky/CyclingDA + post: [] + +externalanalyses: + resource: "GFS.PANDAC" + +firstbackground: + resource: "PANDAC.EnsPertB" + +forecast: + post: [] + +members: + n: 20 + +model: + outerMesh: 120km + # TODO: make inner and ensemble meshes unnecessary + # related to {{PrepareExternalAnalysisInner}} and {{PrepareExternalAnalysisEnsemble}} + innerMesh: 120km + ensembleMesh: 120km + +observations: + resource: PANDACArchiveForVarBC + +workflow: + first cycle point: 20180414T18 + final cycle point: 20180422T00 + +build: + mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build diff --git a/scenarios/getkf_OIE120km_uncoupled.yaml b/scenarios/getkf_OIE120km_uncoupled.yaml new file mode 100644 index 00000000..2552fcc1 --- /dev/null +++ b/scenarios/getkf_OIE120km_uncoupled.yaml @@ -0,0 +1,55 @@ +experiment: + prefix: 'stoedtli_' + name: 'getkf_120km_20mem_uncoupled_' + suffix: '1' + +enkf: + solver: GETKF + # localization + horizontal localization lengthscale: 1.2e6 + vertical localization lengthscale: 6000.0 + vertical localization lengthscale units: height + fraction of retained variance: 0.95 + # inflation + rtpp value: 0.5 + rtps value: 0.9 + # ensemble observation equivalent (background, analysis) + retainObsFeedback: True + concatenateObsFeedback: False + # coupling to var DA (bias correction, recentering) + coupledToVarDA: False + #workflowNameVarDA: 'stoedtli_3dhybrid_coupled_test_3' + #recenterAnalyses: False # not implemented yet + biasCorrection: True + staticVarBcDir: /glade/campaign/mmm/parc/jban/pandac/year7Exp/jban_3dhybrid-60-60-iter_O30kmI60km_mhsRaw_clrsky/CyclingDA + post: [] + +externalanalyses: + resource: "GFS.PANDAC" + +firstbackground: + resource: "PANDAC.EnsPertB" + +forecast: + post: [] + +members: + n: 20 + +model: + outerMesh: 120km + # TODO: make inner and ensemble meshes unnecessary + # related to {{PrepareExternalAnalysisInner}} and {{PrepareExternalAnalysisEnsemble}} + innerMesh: 120km + ensembleMesh: 120km + +observations: + resource: PANDACArchiveForVarBC + +workflow: + first cycle point: 20180414T18 + #restart cycle point: 20180415T06 + final cycle point: 20180422T00 + +build: + mpas bundle: /glade/derecho/scratch/stoedtli/recenter_multires/mpas-bundle/build