From 1dab06441276618dadea354e1832f9e4d68e19b6 Mon Sep 17 00:00:00 2001 From: map <24995792+morianemo@users.noreply.github.com> Date: Wed, 30 Jul 2025 19:02:12 +0100 Subject: [PATCH 01/16] PYFLOW-22 pass as famliy extern limit or variable --- pyflow/__init__.py | 3 ++ pyflow/extern.py | 67 +++++++++++++++++++++++++++++++++++++++++--- tests/test_extern.py | 18 ++++++++++-- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/pyflow/__init__.py b/pyflow/__init__.py index 8e6f63b..910f82d 100644 --- a/pyflow/__init__.py +++ b/pyflow/__init__.py @@ -48,10 +48,13 @@ from .expressions import Deferred, all_complete, sequence from .extern import ( Extern, + ExternEdit, ExternEvent, ExternFamily, + ExternLimit, ExternMeter, ExternNode, + ExternSuite, ExternTask, ExternYMD, ) diff --git a/pyflow/extern.py b/pyflow/extern.py index 489db20..9589d6b 100644 --- a/pyflow/extern.py +++ b/pyflow/extern.py @@ -1,6 +1,6 @@ import datetime -from .attributes import Event, Meter, RepeatDate +from .attributes import Event, Meter, RepeatDate, Edit, Limit, Variable from .base import Root from .nodes import Family, Suite, Task @@ -11,7 +11,7 @@ def is_extern_known(ext): return ext in KNOWN_EXTERNS -def ExternNode(path, tail_cls=Family): +def ExternNode(path, tail_cls=Family, **args): """ Maps an external node, i.e. a node that is not built from the same repository. @@ -39,7 +39,7 @@ def ExternNode(path, tail_cls=Family): cls = Family with current: - return tail_cls(path_cpts[-1], extern=True) + return tail_cls(path_cpts[-1], extern=True, **args) def ExternAttribute(path, cls, *args): @@ -49,6 +49,47 @@ def ExternAttribute(path, cls, *args): return cls(attr, *args) +def ExternEdit(path): + """ + Maps an external variable (that may also be a repeat) + + Parameters: + path(*str*): Path of the external variable. + + Returns: + RepeatDate_: An object that corresponds to an external item. + + Example:: + + pyflow.ExternYMD('/a/b/c/d:YMD') + """ + KNOWN_EXTERNS.add(path) + path, attr = path.split(":") + kind = Family if '/' in path[1:] else Suite + return ExternNode(path, kind, variables=[Variable(attr, 1), ]) + + +def ExternLimit(path): + """ + Maps an external limit. + + Parameters: + path(*str*): Path of the item. + + Returns: + RepeatDate_: An object that corresponds to an external item. + + Example:: + + pyflow.ExternYMD('/a/limits:hpc') + """ + KNOWN_EXTERNS.add(path) + node, attr = path.split(":") + kind = Family if '/' in path[1:] else Suite + # return ExternNode(path, kind, limits=[Limit(attr, 1), ]) + return ExternAttribute(path, Limit, 1) + + def ExternYMD(path): """ Maps an external repeat date, i.e. a repeat date that is not built from the same repository. @@ -123,6 +164,24 @@ def Extern(path): return ExternNode(path) +def ExternSuite(path): + """ + Maps an external suite. + + Parameters: + path(str): Path of the external suite. + + Returns: + Family_: An object that corresponds to an external suite. + + Example:: + + pyflow.ExternSuite('/a') + """ + + return ExternNode(path, Suite) + + def ExternFamily(path): """ Maps an external family, i.e. a family that is not built from the same repository. @@ -138,7 +197,7 @@ def ExternFamily(path): pyflow.ExternFamily('/f/g/h/i') """ - return ExternNode(path) + return ExternNode(path, Family) def ExternTask(path): diff --git a/tests/test_extern.py b/tests/test_extern.py index 1baa6a8..20bb7b9 100644 --- a/tests/test_extern.py +++ b/tests/test_extern.py @@ -4,13 +4,19 @@ import pytest from pyflow import ( + Edit, Event, ExternEvent, + ExternEdit, ExternFamily, + ExternLimit, ExternMeter, + ExternSuite, ExternTask, ExternYMD, Family, + InLimit, + Limit, Meter, Notebook, RepeatDate, @@ -68,14 +74,17 @@ def test_extern(): def test_extern_attributes(): with Suite("s") as s: eymd = ExternYMD("/a/b/c/d:YMD") + evar = ExternYMD("/a/main:SUITE_START") + elimit = ExternLimit("/limits/lim:hpc") eevent = ExternEvent("/e/f/g/h:ev") emeter = ExternMeter("/g/h/i/j:mt") Task("t1", YMD=(now, now)).follow = eymd Task("t2").triggers = eevent Task("t3").triggers = emeter == 10 - - # Check that the externs have real types --> will have correct functionality available + Task("t4").completes = evar != eymd + Task("t5", inlimits= [elimit, ]) + # Check that the externs have real types --> will have correct functionality available assert isinstance(eymd, RepeatDate) assert eymd.name == "YMD" @@ -113,6 +122,11 @@ def test_extern_safety(): with Suite("s"): externs.append(ExternTask("/a/b/c/d")) externs.append(ExternFamily("/e/f/g/h")) + # externs.append(ExternSuite("/limits")) + externs.append(ExternLimit("/limits/lim:hpc")) # OK + # externs.append(ExternLimit("/limits:hpc")) # NOK + externs.append(ExternEdit("/a/main:SUITE_START")) + # externs.append(ExternEdit("/a:SUITE_START")) # NOK with externs[-1]: # n.b. should never do this in reality, but trying to break things... From 37fbcc9ef2b111dd138cfe4b7d69b64414325bd9 Mon Sep 17 00:00:00 2001 From: map <24995792+morianemo@users.noreply.github.com> Date: Wed, 30 Jul 2025 19:03:24 +0100 Subject: [PATCH 02/16] PYFLOW-22 pass as famliy extern limit or variable --- tests/test_extern.py | 3 ++- tests/test_host.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_extern.py b/tests/test_extern.py index 20bb7b9..3f2d6d2 100644 --- a/tests/test_extern.py +++ b/tests/test_extern.py @@ -76,6 +76,7 @@ def test_extern_attributes(): eymd = ExternYMD("/a/b/c/d:YMD") evar = ExternYMD("/a/main:SUITE_START") elimit = ExternLimit("/limits/lim:hpc") + slimit = ExternLimit("/limits:hpc") eevent = ExternEvent("/e/f/g/h:ev") emeter = ExternMeter("/g/h/i/j:mt") @@ -83,7 +84,7 @@ def test_extern_attributes(): Task("t2").triggers = eevent Task("t3").triggers = emeter == 10 Task("t4").completes = evar != eymd - Task("t5", inlimits= [elimit, ]) + Task("t5", inlimits= [elimit, slimit ]) # Check that the externs have real types --> will have correct functionality available assert isinstance(eymd, RepeatDate) diff --git a/tests/test_host.py b/tests/test_host.py index 09fb0b4..d6f3d6c 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -264,7 +264,7 @@ def test_troika_host(): ) submit_args = { - "tasks": 2, # deprecated option, will be translated to total_tasks + "total_tasks": 2, # deprecated option, will be translated to total_tasks "gpus": 1, "sthost": "/foo/bar", "distribution": "test", # generates TROIKA pragma for recent version of troika, SBATCH for older versions @@ -317,7 +317,7 @@ def test_troika_host(): def test_host_submit_args(): submit_args = { "troika": { - "tasks": 2, # deprecated option, will be translated to total_tasks + "total_tasks": 2, # deprecated option, will be translated to total_tasks "gpus": 1, "sthost": "/foo/bar", "distribution": "test", # generates TROIKA pragma for recent version of troika, SBATCH for older versions From 98c46fb8ad47a0e766d261d76cba4272db59b4db Mon Sep 17 00:00:00 2001 From: map <24995792+morianemo@users.noreply.github.com> Date: Wed, 30 Jul 2025 21:17:20 +0100 Subject: [PATCH 03/16] PYFLOW-22 pass tests --- pyflow/extern.py | 8 +++----- tests/test_extern.py | 17 +++++++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pyflow/extern.py b/pyflow/extern.py index 9589d6b..1d4c13d 100644 --- a/pyflow/extern.py +++ b/pyflow/extern.py @@ -45,7 +45,8 @@ def ExternNode(path, tail_cls=Family, **args): def ExternAttribute(path, cls, *args): KNOWN_EXTERNS.add(path) path, attr = path.split(":") - with ExternNode(path): + kind = Family if '/' in path[1:] else Suite + with ExternNode(path, kind): return cls(attr, *args) @@ -64,9 +65,7 @@ def ExternEdit(path): pyflow.ExternYMD('/a/b/c/d:YMD') """ KNOWN_EXTERNS.add(path) - path, attr = path.split(":") - kind = Family if '/' in path[1:] else Suite - return ExternNode(path, kind, variables=[Variable(attr, 1), ]) + return ExternAttribute(path, Variable, 1) # context manager protocol def ExternLimit(path): @@ -86,7 +85,6 @@ def ExternLimit(path): KNOWN_EXTERNS.add(path) node, attr = path.split(":") kind = Family if '/' in path[1:] else Suite - # return ExternNode(path, kind, limits=[Limit(attr, 1), ]) return ExternAttribute(path, Limit, 1) diff --git a/tests/test_extern.py b/tests/test_extern.py index 3f2d6d2..155a4dc 100644 --- a/tests/test_extern.py +++ b/tests/test_extern.py @@ -72,9 +72,12 @@ def test_extern(): def test_extern_attributes(): + sext = ExternSuite("/limits") # extern shall not be under a node suite/family/task + evar = ExternEdit("/a/main:SUITE_START") + svar = ExternEdit("/a:SUITE_START") + # svar = ExternEdit("/b:SUITE_START") # OK with Suite("s") as s: eymd = ExternYMD("/a/b/c/d:YMD") - evar = ExternYMD("/a/main:SUITE_START") elimit = ExternLimit("/limits/lim:hpc") slimit = ExternLimit("/limits:hpc") eevent = ExternEvent("/e/f/g/h:ev") @@ -85,6 +88,8 @@ def test_extern_attributes(): Task("t3").triggers = emeter == 10 Task("t4").completes = evar != eymd Task("t5", inlimits= [elimit, slimit ]) + Task("t6").completes = svar != eymd + Task("ts").completes = sext.complete # Check that the externs have real types --> will have correct functionality available assert isinstance(eymd, RepeatDate) @@ -119,15 +124,15 @@ def test_extern_attributes(): def test_extern_safety(): externs = [] + externs.append(ExternSuite("/limits")) + externs.append(ExternLimit("/limits:hpc")) + externs.append(ExternLimit("/limits/lim:hpc")) + externs.append(ExternEdit("/a/main:SUITE_START")) + externs.append(ExternEdit("/a:SUITE_START")) with Suite("s"): externs.append(ExternTask("/a/b/c/d")) externs.append(ExternFamily("/e/f/g/h")) - # externs.append(ExternSuite("/limits")) - externs.append(ExternLimit("/limits/lim:hpc")) # OK - # externs.append(ExternLimit("/limits:hpc")) # NOK - externs.append(ExternEdit("/a/main:SUITE_START")) - # externs.append(ExternEdit("/a:SUITE_START")) # NOK with externs[-1]: # n.b. should never do this in reality, but trying to break things... From 867354e41b1ae9ac95a7e69301588c73d9bbe12d Mon Sep 17 00:00:00 2001 From: map <24995792+morianemo@users.noreply.github.com> Date: Thu, 31 Jul 2025 10:47:14 +0100 Subject: [PATCH 04/16] PYFLOW-22 defined to avoid object does not support the context manager protocol error message --- pyflow/extern.py | 2 -- tests/test_extern.py | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pyflow/extern.py b/pyflow/extern.py index 1d4c13d..af9b2ef 100644 --- a/pyflow/extern.py +++ b/pyflow/extern.py @@ -83,8 +83,6 @@ def ExternLimit(path): pyflow.ExternYMD('/a/limits:hpc') """ KNOWN_EXTERNS.add(path) - node, attr = path.split(":") - kind = Family if '/' in path[1:] else Suite return ExternAttribute(path, Limit, 1) diff --git a/tests/test_extern.py b/tests/test_extern.py index 155a4dc..b552b75 100644 --- a/tests/test_extern.py +++ b/tests/test_extern.py @@ -22,6 +22,7 @@ RepeatDate, Suite, Task, + Variable, ) from pyflow.extern import KNOWN_EXTERNS @@ -71,10 +72,16 @@ def test_extern(): assert excinfo.value.args == ("Attempting to add unknown extern reference",) +# @pytest.mark.xfail +def test_extern_fail(): + pass + def test_extern_attributes(): sext = ExternSuite("/limits") # extern shall not be under a node suite/family/task evar = ExternEdit("/a/main:SUITE_START") svar = ExternEdit("/a:SUITE_START") + limit = ExternLimit("/limits:hpc") # extern shall not be under a node suite/family/task + # svar = ExternEdit("/b:SUITE_START") # OK with Suite("s") as s: eymd = ExternYMD("/a/b/c/d:YMD") @@ -92,6 +99,14 @@ def test_extern_attributes(): Task("ts").completes = sext.complete # Check that the externs have real types --> will have correct functionality available + assert isinstance(elimit, Limit) + assert limit.name == "hpc" + assert limit.fullname == "/limits:hpc" + + assert isinstance(svar, Variable) + assert svar.name == "SUITE_START" + assert svar.fullname == "/a:SUITE_START" + assert isinstance(eymd, RepeatDate) assert eymd.name == "YMD" assert eymd.fullname == "/a/b/c/d:YMD" From d6fcd461aed49e6d3dfcdf8eba73d9ef8ea848b7 Mon Sep 17 00:00:00 2001 From: map <24995792+morianemo@users.noreply.github.com> Date: Thu, 31 Jul 2025 11:35:03 +0100 Subject: [PATCH 05/16] PYFLOW-22 order import --- tests/test_extern.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_extern.py b/tests/test_extern.py index b552b75..59248b2 100644 --- a/tests/test_extern.py +++ b/tests/test_extern.py @@ -6,8 +6,8 @@ from pyflow import ( Edit, Event, - ExternEvent, ExternEdit, + ExternEvent, ExternFamily, ExternLimit, ExternMeter, From 3903e5c1c51aba4623814765da65346e1c4cf0ca Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Wed, 13 Aug 2025 15:53:56 +0000 Subject: [PATCH 06/16] rewrite handling of repeat --- pyflow/__init__.py | 5 +- pyflow/attributes.py | 139 +++++++++++++++++++-------------------- pyflow/base.py | 4 ++ pyflow/extern.py | 58 +++++++++++----- pyflow/nodes.py | 23 +++++-- tests/test8.json | 3 +- tests/test_attributes.py | 9 ++- tests/test_extern.py | 46 +++++++------ tests/test_follow.py | 16 +++-- 9 files changed, 178 insertions(+), 125 deletions(-) diff --git a/pyflow/__init__.py b/pyflow/__init__.py index 910f82d..305eb3d 100644 --- a/pyflow/__init__.py +++ b/pyflow/__init__.py @@ -48,14 +48,15 @@ from .expressions import Deferred, all_complete, sequence from .extern import ( Extern, - ExternEdit, + ExternAttribute, ExternEvent, ExternFamily, ExternLimit, ExternMeter, - ExternNode, + ExternRepeat, ExternSuite, ExternTask, + ExternVariable, ExternYMD, ) from .header import FileHeader, FileTail, Header, InlineCodeHeader diff --git a/pyflow/attributes.py b/pyflow/attributes.py index e8358b5..0d1b341 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -3,6 +3,7 @@ import datetime import re +from . import warn from .anchor import AnchorMixin from .base import Base, GenerateError from .cron import Crontab @@ -92,26 +93,6 @@ def generate_stub(self): shape = "box" -class RepeatDay(Attribute): - """ - An attribute that allows a node to be repeated infinitely. - - Parameters: - value(int): The repeat step. - - Example:: - - pyflow.attributes.RepeatDay(1) - """ - - def __init__(self, value): - super().__init__("_repeat") - self._value = value - - def _build(self, ecflow_parent): - ecflow_parent.add_repeat(ecflow.RepeatDay(int(self.value))) - - class Time(Attribute): """ An attribute for setting a time dependency of the node. @@ -341,7 +322,49 @@ def __init__(self, **kwargs): Variable(key, val) -class RepeatString(Exportable): +class Repeat(Exportable): + """ + A virtual class that defines a repeat attribute + """ + + def __init__(self, name, values=None): + super().__init__(name, values) + if self.parent._repeat is not None: + warn( + "Overwriting an existing repeat value!", + category=UserWarning, + stacklevel=2, + ) + self.parent._repeat = self + print(self.parent.host) + # self.parent._nodes["repeat"] = self + # print(self.parent) + + def settings(self): + raise NotImplementedError("Subclasses must implement settings()") + + +class RepeatDay(Repeat): + """ + An attribute that allows a node to be repeated infinitely. + + Parameters: + value(int): The repeat step. + + Example:: + + pyflow.attributes.RepeatDay(1) + """ + + def __init__(self, value): + super().__init__("_repeat") + self._value = value + + def _build(self, ecflow_parent): + ecflow_parent.add_repeat(ecflow.RepeatDay(int(self.value))) + + +class RepeatString(Repeat): """ An attribute that allows a node to be repeated by a string value. @@ -412,7 +435,7 @@ def __sub__(self, other): return Sub(self, other) -class RepeatEnumerated(Exportable): +class RepeatEnumerated(Repeat): """ An attribute that allows a node to be repeated by an enumerated list. @@ -447,7 +470,7 @@ def __sub__(self, other): return Sub(self, other) -class RepeatDateList(Exportable): +class RepeatDateList(Repeat): """ An attribute that allows a node to be repeated over a list of dates. @@ -489,7 +512,7 @@ def __sub__(self, other): return Sub(self, other) -class RepeatInteger(Exportable): +class RepeatInteger(Repeat): """ An attribute that allows a node to be repeated by an integer range. @@ -527,7 +550,7 @@ def __sub__(self, other): return Sub(self, other) -class RepeatDate(Exportable): +class RepeatDate(Repeat): """ An attribute that allows a node to be repeated by a date value. @@ -712,12 +735,6 @@ def day_of_week(self): return Mod(Add(Div(self, 86400), 4), 7) -def string_or_enumerated(name, value): - if all(isinstance(v, int) for v in value): - return RepeatEnumerated(name, value) - return RepeatString(name, value) - - def is_date(value): return ( isinstance(value, (datetime.date, datetime.datetime)) @@ -796,27 +813,7 @@ def make_variable(node, name, value): with node: if isinstance(value, (tuple, list)): - if len(value) in [2, 3]: - if is_date(value[0]) and is_date(value[1]): - if len(value) == 3: - if isinstance(value[2], int): - return RepeatDate( - name, - as_date(value[0]), - as_date(value[1]), - value[2], - ) - else: - return RepeatDate(name, as_date(value[0]), as_date(value[1]), 1) - - if isinstance(value[0], int) and isinstance(value[1], int): - if len(value) == 3: - if isinstance(value[2], int): - return RepeatInteger(name, value[0], value[1], value[2]) - else: - return RepeatInteger(name, value[0], value[1], 1) - - return string_or_enumerated(name, value) + raise Exception("Repeat attributes must be added after the node creation") if isinstance(value, (str, int, float)): return Variable(name, value) @@ -1239,26 +1236,26 @@ def generate_stub(self): class Follow(_Trigger): """ - An attribute for setting a condition for running the node behind another repeated node which has completed. - - Parameters: - value(RepeatDate_): The repeat date attribute of the followed node. - - Example:: - - pyflow.attributes.Follow(pyflow.RepeatDate('REPEAT_DATE', - datetime.date(year=2019, month=1, day=1), - datetime.date(year=2019, month=12, day=31))) - """ + An attribute for setting a condition for running the node behind another repeated node which has completed. - def __init__(self, value): - super().__init__("_follow_%s" % (value,), value) - if not hasattr(value, "settings"): - raise Exception( - "Cannot follow a node of type %s (%r)" % (type(value), value) - ) - self.parent[value.name] = value.settings() - self._value = value.parent.complete | (self.parent[value.name] < value) + Parameters: + value(RepeatDate_): The repeat date attribute of the followed node. + + Example:: + pyflow.RepeatDate('REPEAT_DATE', + datetime.date(year=2019, month=1, day=1), + datetime.date(year=2019, month=12, day=31)) + pyflow.attributes.Follow() + """ + + def __init__(self, repeat): + super().__init__(f"_follow_{repeat.name}") + if not isinstance(repeat, Repeat): + raise TypeError(f"Follow attribute {self.name} requires a Repeat instance") + print(self.parent.name) + if self.parent.repeat is None: + raise TypeError(f"Follow attribute {self.name} requires a parent repeat") + self._value = repeat.parent.complete | (self.parent.repeat < repeat) ################################################################### diff --git a/pyflow/base.py b/pyflow/base.py index 92ff78d..9fabcfe 100644 --- a/pyflow/base.py +++ b/pyflow/base.py @@ -21,6 +21,10 @@ def remove_node(self, node): def add_node(self, node): pass + @property + def repeat(self): + return None + @property def host(self): return None diff --git a/pyflow/extern.py b/pyflow/extern.py index af9b2ef..9d9a047 100644 --- a/pyflow/extern.py +++ b/pyflow/extern.py @@ -1,6 +1,7 @@ import datetime -from .attributes import Event, Meter, RepeatDate, Edit, Limit, Variable +from . import warn +from .attributes import Attribute, Event, Limit, Meter, Repeat, RepeatDate, Variable from .base import Root from .nodes import Family, Suite, Task @@ -42,30 +43,30 @@ def ExternNode(path, tail_cls=Family, **args): return tail_cls(path_cpts[-1], extern=True, **args) -def ExternAttribute(path, cls, *args): +def ExternAttribute(path, cls=Attribute, *args): KNOWN_EXTERNS.add(path) path, attr = path.split(":") - kind = Family if '/' in path[1:] else Suite + kind = Family if "/" in path[1:] else Suite with ExternNode(path, kind): return cls(attr, *args) -def ExternEdit(path): +def ExternVariable(path): """ - Maps an external variable (that may also be a repeat) + Maps an external variable. Parameters: - path(*str*): Path of the external variable. + path(*str*): Path of the item. Returns: - RepeatDate_: An object that corresponds to an external item. + Variable: An object that corresponds to an external variable. Example:: - pyflow.ExternYMD('/a/b/c/d:YMD') + pyflow.ExternYMD('/a/b:var') """ KNOWN_EXTERNS.add(path) - return ExternAttribute(path, Variable, 1) # context manager protocol + return ExternAttribute(path, Variable, 1) def ExternLimit(path): @@ -86,6 +87,26 @@ def ExternLimit(path): return ExternAttribute(path, Limit, 1) +def ExternRepeat(path): + """ + Maps an external repeat, i.e. a repeat that is not built from the same repository. + Cannot be a generic attribute as the repeat can be used with the follow() approach, + which requires an object of type Repeat. + + Parameters: + path(*str*): Path of the external repeat. + + Returns: + RepeatDate_: An object that corresponds to an external repeat. + + Example:: + + pyflow.ExternRepeat('/a/b/c/d:YMD') + """ + + return ExternAttribute(path, Repeat) + + def ExternYMD(path): """ Maps an external repeat date, i.e. a repeat date that is not built from the same repository. @@ -100,7 +121,11 @@ def ExternYMD(path): pyflow.ExternYMD('/a/b/c/d:YMD') """ - + warn( + "'ExternYMD' is deprecated, use ExternAttribute instead", + DeprecationWarning, + stacklevel=1, + ) return ExternAttribute( path, RepeatDate, datetime.datetime.now(), datetime.datetime.now() ) @@ -120,7 +145,6 @@ def ExternEvent(path): pyflow.ExternEvent('/e/f/g/h:ev') """ - return ExternAttribute(path, Event) @@ -138,7 +162,6 @@ def ExternMeter(path): pyflow.ExternMeter('/g/h/i/j:mt') """ - return ExternAttribute(path, Meter, 0) @@ -156,7 +179,11 @@ def Extern(path): pyflow.Extern('/f/g/h/i') """ - + warn( + "'Extern' is deprecated, use ExternSuite, ExternFamily or ExternTask instead", + DeprecationWarning, + stacklevel=1, + ) return ExternNode(path) @@ -174,7 +201,6 @@ def ExternSuite(path): pyflow.ExternSuite('/a') """ - return ExternNode(path, Suite) @@ -192,7 +218,6 @@ def ExternFamily(path): pyflow.ExternFamily('/f/g/h/i') """ - return ExternNode(path, Family) @@ -210,5 +235,4 @@ def ExternTask(path): pyflow.ExternTask('/a/b/c/d') """ - - return ExternNode(path, tail_cls=Task) + return ExternNode(path, Task) diff --git a/pyflow/nodes.py b/pyflow/nodes.py index db7ef62..023ba0c 100644 --- a/pyflow/nodes.py +++ b/pyflow/nodes.py @@ -25,7 +25,6 @@ Limit, Manual, Meter, - RepeatDay, Time, Today, Trigger, @@ -142,7 +141,6 @@ def __init__( limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -162,6 +160,7 @@ def __init__( self._modules = modules or [] self._purge_modules = purge_modules self._extern = extern + self._repeat = None # can't be set in constructor, needs to be done in context # If we have changed the host, then set the relevant directories self._host = host @@ -302,6 +301,19 @@ def append_node(self, node): self.add_node(node) return self + @property + def repeat(self): + """ + Returns the currently active repeat object. + If not found in current node, search in parents. + + Returns: + Repeat_: Currently active repeat object. + """ + if self._repeat is not None: + return self._repeat + return self.parent.repeat + @property def host(self): """ @@ -832,7 +844,6 @@ def __init__( limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -960,7 +971,6 @@ def __init__( limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -969,6 +979,7 @@ def __init__( variables(Variable_): An attribute for setting an **ecFlow** variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the anchor family. Example:: @@ -1035,7 +1046,6 @@ def __init__(self, name, host=None, exit_hook=None, *args, **kwargs): limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -1045,6 +1055,7 @@ def __init__(self, name, host=None, exit_hook=None, *args, **kwargs): generated_variables(GeneratedVariable_): An attribute for setting an **ecFlow** generated variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the suite. Example:: @@ -1245,7 +1256,6 @@ def __init__( limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -1520,7 +1530,6 @@ def generate_script(self): ("labels", Label), ("limits", Limit), ("meters", Meter), - ("repeat", RepeatDay), ("tasks", Task), ("time", Time), ("today", Today), diff --git a/tests/test8.json b/tests/test8.json index 85e8970..a953a38 100644 --- a/tests/test8.json +++ b/tests/test8.json @@ -69,6 +69,5 @@ }, "f2": { "FOO": 42 - }, - "repeat": true + } } diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 22fcaaf..d90d7b2 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -5,6 +5,7 @@ import pytest import pyflow +from build.lib.pyflow.attributes import RepeatDate from pyflow.base import GenerateError @@ -305,8 +306,12 @@ def test_string_repeat(self): def test_combined_string_repeats(self): with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1", YMD=["20170101", "20180101"]) - t2 = pyflow.Task("t2", YMD=["20170101", "20180101"]) + t1 = pyflow.Task("t1") + with t1: + RepeatDate("YMD", "20170101", "20180101") + t2 = pyflow.Task("t2") + with t2: + RepeatDate("YMD", "20170101", "20180101") t2.triggers = t1.YMD >= t2.YMD assert str(t2.triggers.value) == "(/s/t1:YMD ge /s/t2:YMD)" diff --git a/tests/test_extern.py b/tests/test_extern.py index 59248b2..66e5b33 100644 --- a/tests/test_extern.py +++ b/tests/test_extern.py @@ -4,18 +4,16 @@ import pytest from pyflow import ( - Edit, Event, - ExternEdit, ExternEvent, ExternFamily, ExternLimit, ExternMeter, + ExternRepeat, ExternSuite, ExternTask, - ExternYMD, + ExternVariable, Family, - InLimit, Limit, Meter, Notebook, @@ -24,7 +22,7 @@ Task, Variable, ) -from pyflow.extern import KNOWN_EXTERNS +from pyflow.extern import KNOWN_EXTERNS, Repeat now = datetime.datetime.now() @@ -35,8 +33,9 @@ def test_extern(): et = ExternTask("/a/b/c/d") ef = ExternFamily("/f/g/h/i") + es = ExternSuite("/j") - t1.triggers = et & ef + t1.triggers = et & ef & es # Check that the externs have real types --> will have correct functionality available @@ -46,6 +45,9 @@ def test_extern(): assert isinstance(ef, Family) assert ef.name == "i" assert ef.fullname == "/f/g/h/i" + assert isinstance(es, Suite) + assert es.name == "j" + assert es.fullname == "/j" # Check that they work! @@ -69,32 +71,38 @@ def test_extern(): with pytest.raises(AssertionError) as excinfo: s.ecflow_definition() - assert excinfo.value.args == ("Attempting to add unknown extern reference",) + assert excinfo.value.args == ( + "Attempting to add unknown extern reference /a/b/c/d", + ) # @pytest.mark.xfail def test_extern_fail(): pass + def test_extern_attributes(): sext = ExternSuite("/limits") # extern shall not be under a node suite/family/task - evar = ExternEdit("/a/main:SUITE_START") - svar = ExternEdit("/a:SUITE_START") - limit = ExternLimit("/limits:hpc") # extern shall not be under a node suite/family/task + evar = ExternVariable("/a/main:SUITE_START") + svar = ExternVariable("/a:SUITE_START") + limit = ExternLimit( + "/limits:hpc" + ) # extern shall not be under a node suite/family/task # svar = ExternEdit("/b:SUITE_START") # OK with Suite("s") as s: - eymd = ExternYMD("/a/b/c/d:YMD") + eymd = ExternRepeat("/a/b/c/d:YMD") elimit = ExternLimit("/limits/lim:hpc") slimit = ExternLimit("/limits:hpc") eevent = ExternEvent("/e/f/g/h:ev") emeter = ExternMeter("/g/h/i/j:mt") - Task("t1", YMD=(now, now)).follow = eymd + t1 = Task("t1", repeat=RepeatDate("YMD", now, now)) + t1.follow = eymd Task("t2").triggers = eevent Task("t3").triggers = emeter == 10 Task("t4").completes = evar != eymd - Task("t5", inlimits= [elimit, slimit ]) + Task("t5").inlimits = [elimit, slimit] Task("t6").completes = svar != eymd Task("ts").completes = sext.complete # Check that the externs have real types --> will have correct functionality available @@ -107,7 +115,7 @@ def test_extern_attributes(): assert svar.name == "SUITE_START" assert svar.fullname == "/a:SUITE_START" - assert isinstance(eymd, RepeatDate) + assert isinstance(eymd, Repeat) assert eymd.name == "YMD" assert eymd.fullname == "/a/b/c/d:YMD" @@ -142,18 +150,18 @@ def test_extern_safety(): externs.append(ExternSuite("/limits")) externs.append(ExternLimit("/limits:hpc")) externs.append(ExternLimit("/limits/lim:hpc")) - externs.append(ExternEdit("/a/main:SUITE_START")) - externs.append(ExternEdit("/a:SUITE_START")) + externs.append(ExternVariable("/a/main:SUITE_START")) + externs.append(ExternVariable("/a:SUITE_START")) with Suite("s"): - externs.append(ExternTask("/a/b/c/d")) - externs.append(ExternFamily("/e/f/g/h")) + externs.append(ExternFamily("/a/b/c/d")) + externs.append(ExternTask("/e/f/g/h")) with externs[-1]: # n.b. should never do this in reality, but trying to break things... externs.append(Task("e3")) - externs.append(ExternYMD("i/j/k/l:YMD")) + externs.append(ExternRepeat("i/j/k/l:YMD")) externs.append(ExternEvent("m/n/o/p:ev")) externs.append(ExternMeter("q/s/t/u:mt")) diff --git a/tests/test_follow.py b/tests/test_follow.py index 81caa29..78a5c97 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -1,19 +1,25 @@ import datetime -from pyflow import Notebook, Suite, Task +from pyflow import Notebook, RepeatDate, Suite, Task now = datetime.datetime.now() def test_follow(): with Suite("s") as s: - t1 = Task("t1", YMD=(now, now)) - t2 = Task("t2") - Task("t3") + with Task("t1") as t1: + r1 = RepeatDate("YMD", now, now) + with Task("t2") as t2: + RepeatDate("YMD", now, now) + t3 = Task("t3") + with t3: + RepeatDate("YMD", now, now) t2.triggers = t1.complete - t2.follow = t1.YMD + t2.follow = r1 + t3.follow = t2.repeat + print(s) s.check_definition() s.generate_node() From 408576ad2a841cb1ec44ed9570805790f197ed98 Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 10:26:54 +0000 Subject: [PATCH 07/16] debugging new repeat approach --- pyflow/attributes.py | 33 +++++++++++++++++++-------------- pyflow/nodes.py | 16 ++++++++++++++++ tests/test8.json | 4 ---- tests/test_attributes.py | 32 +++++++++++++++----------------- tests/test_contextmanager.py | 5 +++-- tests/test_extern.py | 4 ++-- tests/test_follow.py | 31 ++++++++++++++++++++----------- tests/test_json.py | 2 +- tests/test_suite.py | 18 +++++++++++------- 9 files changed, 87 insertions(+), 58 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 0d1b341..1589836 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -335,10 +335,7 @@ def __init__(self, name, values=None): category=UserWarning, stacklevel=2, ) - self.parent._repeat = self - print(self.parent.host) - # self.parent._nodes["repeat"] = self - # print(self.parent) + self.parent._repeat = self # set the repeat at the node level def settings(self): raise NotImplementedError("Subclasses must implement settings()") @@ -445,7 +442,7 @@ class RepeatEnumerated(Repeat): Example:: - pyflow.RepeatEnumerated("REPEAT_STRING", ["a", "b", "c", "d", "e"]) + pyflow.RepeatEnumerated("REPEAT_STRING", [1, 3, 4, 5]) """ def __init__(self, name, value): @@ -1239,7 +1236,7 @@ class Follow(_Trigger): An attribute for setting a condition for running the node behind another repeated node which has completed. Parameters: - value(RepeatDate_): The repeat date attribute of the followed node. + value(Repeat_ or Node_): The repeat attribute of the followed node or the followed node. Example:: pyflow.RepeatDate('REPEAT_DATE', @@ -1248,14 +1245,22 @@ class Follow(_Trigger): pyflow.attributes.Follow() """ - def __init__(self, repeat): - super().__init__(f"_follow_{repeat.name}") - if not isinstance(repeat, Repeat): - raise TypeError(f"Follow attribute {self.name} requires a Repeat instance") - print(self.parent.name) - if self.parent.repeat is None: - raise TypeError(f"Follow attribute {self.name} requires a parent repeat") - self._value = repeat.parent.complete | (self.parent.repeat < repeat) + def __init__(self, value): + super().__init__(f"_follow_{value.name}") + from .nodes import Node # yeah it's bad but there's a circular import + + if isinstance(value, Node): + parent = value + repeat = value.repeat + elif isinstance(value, Repeat): + parent = value.parent + repeat = value + else: + raise TypeError(f"Follow attribute {self.name} requires a Repeat or a Node instance") + + if repeat is None: + raise TypeError(f"Follow attribute {self.name} requires a repeat") + self._value = parent.complete | (self.parent.repeat < repeat) ################################################################### diff --git a/pyflow/nodes.py b/pyflow/nodes.py index 023ba0c..0c52c12 100644 --- a/pyflow/nodes.py +++ b/pyflow/nodes.py @@ -111,6 +111,7 @@ def __init__( purge_modules=False, extern=False, workdir=None, + repeat=None, **kwargs, ): """ @@ -150,6 +151,7 @@ def __init__( generated_variables(GeneratedVariable_): An attribute for setting an **ecFlow** generated variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the node. """ @@ -161,6 +163,11 @@ def __init__( self._purge_modules = purge_modules self._extern = extern self._repeat = None # can't be set in constructor, needs to be done in context + if repeat is not None: + if not isinstance(repeat, (list, tuple)): + raise TypeError("Repeat attribute must be passed as a list or tuple") + with self: + self._repeat = repeat[0](*repeat[1:]) # If we have changed the host, then set the relevant directories self._host = host @@ -313,6 +320,13 @@ def repeat(self): if self._repeat is not None: return self._repeat return self.parent.repeat + + @repeat.setter + def repeat(self, value): + if not isinstance(value, (list, tuple)): + raise TypeError("Repeat attribute must be passed as a list or tuple") + with self: + self._repeat = value[0](*value[1:]) @property def host(self): @@ -853,6 +867,7 @@ def __init__( generated_variables(GeneratedVariable_): An attribute for setting an **ecFlow** generated variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the family. Example:: @@ -1265,6 +1280,7 @@ def __init__( generated_variables(GeneratedVariable_): An attribute for setting an **ecFlow** generated variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the task. Example:: diff --git a/tests/test8.json b/tests/test8.json index a953a38..ce617c7 100644 --- a/tests/test8.json +++ b/tests/test8.json @@ -1,10 +1,6 @@ { "FOO": 42, "f1": { - "YMD": [ - "2010-01-01", - "2011-01-01" - ], "labels": { "foo_label": "bar" }, diff --git a/tests/test_attributes.py b/tests/test_attributes.py index d90d7b2..ec0b66c 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -47,9 +47,9 @@ def test_reassign_variable(): with pyflow.Suite("s") as s: with pyflow.Task("t1") as t: t.FOO = 61 - t.FOO = (1, 10) + t.FOO = 100 with pyflow.Task("t2") as t: - t.FOO = (1, 10) + t.FOO = 100 assert s.t1.FOO.value == s.t2.FOO.value @@ -277,9 +277,9 @@ def test_date(): assert "date *.*.3" in str(s.ecflow_definition()) with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1", DATE=("20180105", "20180206")) + t1 = pyflow.Task("t1", repeat=(pyflow.RepeatString, "DATE", ["20180105", "20180206"])) - assert t1.DATE.value == ("20180105", "20180206") + assert t1.DATE.value == ['20180105', '20180206'] assert 'repeat string DATE "20180105" "20180206"' in str(s.ecflow_definition()) @@ -289,7 +289,7 @@ class TestRepeats: def test_string_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f1") as f1: - f1.STRING_REPEAT = [str(v) for v in reversed(range(10))] + pyflow.RepeatString("STRING_REPEAT", [str(v) for v in reversed(range(10))]) t1 = pyflow.Task("t1") t1.triggers = (f1.STRING_REPEAT == "7") & (f1.STRING_REPEAT == 3) @@ -306,13 +306,9 @@ def test_string_repeat(self): def test_combined_string_repeats(self): with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1") - with t1: - RepeatDate("YMD", "20170101", "20180101") - t2 = pyflow.Task("t2") - with t2: - RepeatDate("YMD", "20170101", "20180101") - t2.triggers = t1.YMD >= t2.YMD + t1 = pyflow.Task("t1", repeat=(pyflow.RepeatDate, "YMD", "20170101", "20180101")) + t2 = pyflow.Task("t2", repeat=(pyflow.RepeatDate, "YMD", "20170101", "20180101")) + t2.triggers = t1.YMD >= t2.repeat assert str(t2.triggers.value) == "(/s/t1:YMD ge /s/t2:YMD)" s.check_definition() @@ -320,7 +316,7 @@ def test_combined_string_repeats(self): def test_enumerated_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f2") as f2: - f2.ENUMERATED_REPEAT = list(reversed(range(10))) + pyflow.RepeatEnumerated("ENUMERATED_REPEAT", list(range(10))) t2 = pyflow.Task("t2") t2.triggers = (f2.ENUMERATED_REPEAT == "7") & ( @@ -341,8 +337,10 @@ def test_enumerated_repeat(self): def test_combined_enumerated_repeats(self): with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1", ENUMERATED_REPEAT=list(range(10))) - t2 = pyflow.Task("t2", ENUMERATED_REPEAT=list(range(10))) + with pyflow.Task("t1") as t1: + pyflow.RepeatEnumerated("ENUMERATED_REPEAT", list(range(10))) + with pyflow.Task("t2") as t2: + pyflow.RepeatEnumerated("ENUMERATED_REPEAT", list(range(10))) t2.triggers = t1.ENUMERATED_REPEAT >= t2.ENUMERATED_REPEAT assert ( str(t2.triggers.value) @@ -387,7 +385,7 @@ def test_combined_integer_repeats(self): def test_date_datetime_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f4") as f4: - f4.DATE_REPEAT = (datetime(2018, 1, 1), datetime(2019, 12, 31)) + pyflow.RepeatDateTime("DATE_REPEAT", datetime(2018, 1, 1), datetime(2019, 12, 31)) t4 = pyflow.Task("t4") t4.triggers = (f4.DATE_REPEAT >= "20180301") & ( @@ -408,7 +406,7 @@ def test_date_datetime_repeat(self): def test_date_date_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f4") as f4: - f4.DATE_REPEAT = (date(2018, 1, 1), date(2019, 12, 31)) + pyflow.RepeatDate("DATE_REPEAT", date(2018, 1, 1), date(2019, 12, 31)) t4 = pyflow.Task("t4") t4.triggers = (f4.DATE_REPEAT >= "20180301") & ( diff --git a/tests/test_contextmanager.py b/tests/test_contextmanager.py index f7fcebc..7770ab2 100644 --- a/tests/test_contextmanager.py +++ b/tests/test_contextmanager.py @@ -14,6 +14,7 @@ Tasks, Trigger, Variable, + RepeatEnumerated ) @@ -22,7 +23,7 @@ def test_suite(): Limit("foo", 1) Limit("bar", 2) - with Family("f", BAR=[1, 2, 3]): + with Family("f", repeat=(RepeatEnumerated, "BAR", [1, 2, 3])): Task("t1") Task("t2") Task("t3").triggers = (s.f.t1 == "complete") | "2 < 8" @@ -30,7 +31,7 @@ def test_suite(): with Family("g") as g: InLimit("foo") - g.QUUX = [1, 2] + RepeatEnumerated("QUUX", [1, 2]) Task("t4").triggers = s.f.t1 == "aborted" with Task("t5"): diff --git a/tests/test_extern.py b/tests/test_extern.py index 66e5b33..bd1bd14 100644 --- a/tests/test_extern.py +++ b/tests/test_extern.py @@ -29,7 +29,7 @@ def test_extern(): with Suite("s") as s: - t1 = Task("t1", YMD=(now, now)) + t1 = Task("t1", repeat=(RepeatDate, "YMD", now, now)) et = ExternTask("/a/b/c/d") ef = ExternFamily("/f/g/h/i") @@ -97,7 +97,7 @@ def test_extern_attributes(): eevent = ExternEvent("/e/f/g/h:ev") emeter = ExternMeter("/g/h/i/j:mt") - t1 = Task("t1", repeat=RepeatDate("YMD", now, now)) + t1 = Task("t1", repeat=(RepeatDate, "YMD", now, now)) t1.follow = eymd Task("t2").triggers = eevent Task("t3").triggers = emeter == 10 diff --git a/tests/test_follow.py b/tests/test_follow.py index 78a5c97..6d6bc19 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -1,6 +1,6 @@ import datetime -from pyflow import Notebook, RepeatDate, Suite, Task +from pyflow import Notebook, RepeatDate, Suite, Task, Family now = datetime.datetime.now() @@ -8,22 +8,31 @@ def test_follow(): with Suite("s") as s: with Task("t1") as t1: - r1 = RepeatDate("YMD", now, now) - with Task("t2") as t2: - RepeatDate("YMD", now, now) - t3 = Task("t3") - with t3: - RepeatDate("YMD", now, now) - - t2.triggers = t1.complete + r1 = RepeatDate("YMD1", now, now) + with Family("f1") as f1: + f1.repeat = (RepeatDate, "YMD2", now, now) + t2 = Task("t2") + t3 = Task("t3", repeat=(RepeatDate, "YMD3", now, now)) + t2.follow = r1 - t3.follow = t2.repeat + t3.follow = t2 - print(s) s.check_definition() s.generate_node() s.deploy_suite(target=Notebook) + print(s) + print(t3.repeat) + print(str(t2.triggers)) + print(str(t3.triggers)) + assert ( + str(t2.triggers) == + "Triggers< task t2\n trigger ../t1 eq complete or ../f1:YMD2 lt ../t1:YMD1\n>" + ) + assert ( + str(t3.triggers) == + "Triggers< task t3\n trigger f1/t2 eq complete or t3:YMD3 lt f1:YMD2\n repeat date YMD3 20250814 20250814 1\n>" + ) if __name__ == "__main__": diff --git a/tests/test_json.py b/tests/test_json.py index 29f53da..cc56ee8 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -9,11 +9,11 @@ def test_json(): x = json.loads(f.read()) s = Suite("s", json=x) + print(s) s.check_definition() s.generate_node() - if __name__ == "__main__": import pytest diff --git a/tests/test_suite.py b/tests/test_suite.py index 03724f1..3dad574 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -2,7 +2,7 @@ import pytest -from pyflow import Family, Suite, Task +from pyflow import Family, Suite, Task, RepeatString, RepeatDate from pyflow.base import GenerateError @@ -18,8 +18,10 @@ def test_suite(): t1["DATE"] = 19900101 t1.FOOO = 42 - t1.BAR = ["ab", "cd", "ef"] - t2.YMD = (datetime.datetime(2000, 1, 1), datetime.datetime(2010, 1, 1)) + with t1: + RepeatString("BAR", ["ab", "cd", "ef"]) + with t2: + RepeatDate("YMD", datetime.datetime(2000, 1, 1), datetime.datetime(2010, 1, 1)) f += Family("g") f.g += Task("t4") @@ -63,7 +65,7 @@ def test_suite(): "a": { "inlimits": s.l1, "FOO": 42, - "YMD": (now, then), + "repeat": (RepeatDate, "YMD", now, then), "labels": [("info", "hi"), ("status", "ok")], "meters": ("progress", 0, 100), } @@ -84,8 +86,10 @@ def test_suite_builtin_triggers(): t1["DATE"] = 19900101 t1.FOOO = 42 - t1.BAR = ["ab", "cd", "ef"] - t2.YMD = (datetime.datetime(2000, 1, 1), datetime.datetime(2010, 1, 1)) + with t1: + RepeatString("BAR", ["ab", "cd", "ef"]) + with t2: + RepeatDate("YMD", datetime.datetime(2000, 1, 1), datetime.datetime(2010, 1, 1)) f += Family("g") f.g += Task("t4") @@ -129,7 +133,7 @@ def test_suite_builtin_triggers(): "a": { "inlimits": s.l1, "FOO": 42, - "YMD": (now, then), + "repeat": (RepeatDate, "YMD", now, then), "labels": [("info", "hi"), ("status", "ok")], "meters": ("progress", 0, 100), } From 129322dca9b95cf1b872f2c9066ded59176151c1 Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 10:31:33 +0000 Subject: [PATCH 08/16] formatting --- pyflow/attributes.py | 8 ++++++-- pyflow/nodes.py | 2 +- tests/test_attributes.py | 23 ++++++++++++++++------- tests/test_contextmanager.py | 4 ++-- tests/test_follow.py | 12 ++++++------ tests/test_json.py | 1 + tests/test_suite.py | 2 +- 7 files changed, 33 insertions(+), 19 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 1589836..9e2a097 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -810,7 +810,9 @@ def make_variable(node, name, value): with node: if isinstance(value, (tuple, list)): - raise Exception("Repeat attributes must be added after the node creation") + raise Exception( + "Repeat construction through a list is not supported anymore" + ) if isinstance(value, (str, int, float)): return Variable(name, value) @@ -1256,7 +1258,9 @@ def __init__(self, value): parent = value.parent repeat = value else: - raise TypeError(f"Follow attribute {self.name} requires a Repeat or a Node instance") + raise TypeError( + f"Follow attribute {self.name} requires a Repeat or a Node instance" + ) if repeat is None: raise TypeError(f"Follow attribute {self.name} requires a repeat") diff --git a/pyflow/nodes.py b/pyflow/nodes.py index 0c52c12..b04e42b 100644 --- a/pyflow/nodes.py +++ b/pyflow/nodes.py @@ -320,7 +320,7 @@ def repeat(self): if self._repeat is not None: return self._repeat return self.parent.repeat - + @repeat.setter def repeat(self, value): if not isinstance(value, (list, tuple)): diff --git a/tests/test_attributes.py b/tests/test_attributes.py index ec0b66c..c74d2f7 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -5,7 +5,6 @@ import pytest import pyflow -from build.lib.pyflow.attributes import RepeatDate from pyflow.base import GenerateError @@ -277,9 +276,11 @@ def test_date(): assert "date *.*.3" in str(s.ecflow_definition()) with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1", repeat=(pyflow.RepeatString, "DATE", ["20180105", "20180206"])) + t1 = pyflow.Task( + "t1", repeat=(pyflow.RepeatString, "DATE", ["20180105", "20180206"]) + ) - assert t1.DATE.value == ['20180105', '20180206'] + assert t1.DATE.value == ["20180105", "20180206"] assert 'repeat string DATE "20180105" "20180206"' in str(s.ecflow_definition()) @@ -289,7 +290,9 @@ class TestRepeats: def test_string_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f1") as f1: - pyflow.RepeatString("STRING_REPEAT", [str(v) for v in reversed(range(10))]) + pyflow.RepeatString( + "STRING_REPEAT", [str(v) for v in reversed(range(10))] + ) t1 = pyflow.Task("t1") t1.triggers = (f1.STRING_REPEAT == "7") & (f1.STRING_REPEAT == 3) @@ -306,8 +309,12 @@ def test_string_repeat(self): def test_combined_string_repeats(self): with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1", repeat=(pyflow.RepeatDate, "YMD", "20170101", "20180101")) - t2 = pyflow.Task("t2", repeat=(pyflow.RepeatDate, "YMD", "20170101", "20180101")) + t1 = pyflow.Task( + "t1", repeat=(pyflow.RepeatDate, "YMD", "20170101", "20180101") + ) + t2 = pyflow.Task( + "t2", repeat=(pyflow.RepeatDate, "YMD", "20170101", "20180101") + ) t2.triggers = t1.YMD >= t2.repeat assert str(t2.triggers.value) == "(/s/t1:YMD ge /s/t2:YMD)" @@ -385,7 +392,9 @@ def test_combined_integer_repeats(self): def test_date_datetime_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f4") as f4: - pyflow.RepeatDateTime("DATE_REPEAT", datetime(2018, 1, 1), datetime(2019, 12, 31)) + pyflow.RepeatDateTime( + "DATE_REPEAT", datetime(2018, 1, 1), datetime(2019, 12, 31) + ) t4 = pyflow.Task("t4") t4.triggers = (f4.DATE_REPEAT >= "20180301") & ( diff --git a/tests/test_contextmanager.py b/tests/test_contextmanager.py index 7770ab2..5ca6eb6 100644 --- a/tests/test_contextmanager.py +++ b/tests/test_contextmanager.py @@ -9,12 +9,12 @@ Label, Limit, Meter, + RepeatEnumerated, Suite, Task, Tasks, Trigger, Variable, - RepeatEnumerated ) @@ -28,7 +28,7 @@ def test_suite(): Task("t2") Task("t3").triggers = (s.f.t1 == "complete") | "2 < 8" - with Family("g") as g: + with Family("g"): InLimit("foo") RepeatEnumerated("QUUX", [1, 2]) diff --git a/tests/test_follow.py b/tests/test_follow.py index 6d6bc19..e61b0bc 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -1,13 +1,13 @@ import datetime -from pyflow import Notebook, RepeatDate, Suite, Task, Family +from pyflow import Family, Notebook, RepeatDate, Suite, Task now = datetime.datetime.now() def test_follow(): with Suite("s") as s: - with Task("t1") as t1: + with Task("t1"): r1 = RepeatDate("YMD1", now, now) with Family("f1") as f1: f1.repeat = (RepeatDate, "YMD2", now, now) @@ -26,12 +26,12 @@ def test_follow(): print(str(t2.triggers)) print(str(t3.triggers)) assert ( - str(t2.triggers) == - "Triggers< task t2\n trigger ../t1 eq complete or ../f1:YMD2 lt ../t1:YMD1\n>" + str(t2.triggers) + == "Triggers< task t2\n trigger ../t1 eq complete or ../f1:YMD2 lt ../t1:YMD1\n>" ) assert ( - str(t3.triggers) == - "Triggers< task t3\n trigger f1/t2 eq complete or t3:YMD3 lt f1:YMD2\n repeat date YMD3 20250814 20250814 1\n>" + str(t3.triggers) + == "Triggers< task t3\n trigger f1/t2 eq complete or t3:YMD3 lt f1:YMD2\n repeat date YMD3 20250814 20250814 1\n>" # noqa: E501 ) diff --git a/tests/test_json.py b/tests/test_json.py index cc56ee8..9318013 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -14,6 +14,7 @@ def test_json(): s.check_definition() s.generate_node() + if __name__ == "__main__": import pytest diff --git a/tests/test_suite.py b/tests/test_suite.py index 3dad574..d462ffb 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -2,7 +2,7 @@ import pytest -from pyflow import Family, Suite, Task, RepeatString, RepeatDate +from pyflow import Family, RepeatDate, RepeatString, Suite, Task from pyflow.base import GenerateError From daffd92ae1680c8a33465ab601ab75637c28ead6 Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 11:01:36 +0000 Subject: [PATCH 09/16] minor changes --- .../introductory-course/helper-functionality.ipynb | 4 ++-- pyflow/attributes.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/content/introductory-course/helper-functionality.ipynb b/docs/content/introductory-course/helper-functionality.ipynb index 25adaa1..9c4460b 100644 --- a/docs/content/introductory-course/helper-functionality.ipynb +++ b/docs/content/introductory-course/helper-functionality.ipynb @@ -158,7 +158,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "b2e2a324-3df0-4874-8c69-d3d93ea39bc4", "metadata": {}, "outputs": [ @@ -196,7 +196,7 @@ " pf.RepeatDate(\"YMD\", datetime.date(2019, 1, 1), datetime.date(2019, 12, 31))\n", " with pf.Family('follower') as follower:\n", " pf.RepeatDate(\"YMD\", datetime.date(2019, 1, 1), datetime.date(2019, 12, 31))\n", - " follower.follow = leader.YMD\n", + " follower.follow = leader\n", "\n", "s" ] diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 9e2a097..179a205 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -1238,13 +1238,12 @@ class Follow(_Trigger): An attribute for setting a condition for running the node behind another repeated node which has completed. Parameters: - value(Repeat_ or Node_): The repeat attribute of the followed node or the followed node. + value(Repeat_ or Node_): The followed node or the repeat attribute of the followed node. Example:: - pyflow.RepeatDate('REPEAT_DATE', - datetime.date(year=2019, month=1, day=1), - datetime.date(year=2019, month=12, day=31)) - pyflow.attributes.Follow() + t1 = Task("t1", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) + t2 = Task("t2", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) + t2.follow = t1 """ def __init__(self, value): From 7b62b1afae07480c63ceaec02080edf3fa536cde Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 11:10:23 +0000 Subject: [PATCH 10/16] fix test --- tests/test_follow.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_follow.py b/tests/test_follow.py index e61b0bc..2d46ea1 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -25,14 +25,10 @@ def test_follow(): print(t3.repeat) print(str(t2.triggers)) print(str(t3.triggers)) - assert ( - str(t2.triggers) - == "Triggers< task t2\n trigger ../t1 eq complete or ../f1:YMD2 lt ../t1:YMD1\n>" - ) - assert ( - str(t3.triggers) - == "Triggers< task t3\n trigger f1/t2 eq complete or t3:YMD3 lt f1:YMD2\n repeat date YMD3 20250814 20250814 1\n>" # noqa: E501 - ) + print(t2.triggers) + assert "trigger ../t1 eq complete or ../f1:YMD2 lt ../t1:YMD1" in str(t2.triggers) + assert "trigger f1/t2 eq complete or t3:YMD3 lt f1:YMD2" in str(t3.triggers) + assert "repeat date YMD3 20250814 20250814 1" in str(t3.triggers) if __name__ == "__main__": From b14b8310e6b7eb2ce507a2124ba47ba9423feb7a Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 11:10:37 +0000 Subject: [PATCH 11/16] fix test --- pyflow/attributes.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 179a205..cd1a4df 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -1235,15 +1235,15 @@ def generate_stub(self): class Follow(_Trigger): """ - An attribute for setting a condition for running the node behind another repeated node which has completed. + An attribute for setting a condition for running the node behind another repeated node which has completed. - Parameters: - value(Repeat_ or Node_): The followed node or the repeat attribute of the followed node. + Parameters: + value(Repeat_ or Node_): The followed node or the repeat attribute of the followed node. - Example:: - t1 = Task("t1", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) - t2 = Task("t2", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) - t2.follow = t1 + Example:: + t1 = Task("t1", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) + t2 = Task("t2", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) + t2.follow = t1 """ def __init__(self, value): From cc822403adc9561872932a1f2551424e69250868 Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 11:18:44 +0000 Subject: [PATCH 12/16] minor debug --- docs/content/api-reference.rst | 10 ++++++---- docs/content/introductory-course/flow-control.ipynb | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/content/api-reference.rst b/docs/content/api-reference.rst index ef743fc..f7df6d5 100644 --- a/docs/content/api-reference.rst +++ b/docs/content/api-reference.rst @@ -226,19 +226,21 @@ External .. _Extern: -.. autoclass:: pyflow.Extern +.. autoclass:: pyflow.ExternSuite -.. autoclass:: pyflow.ExternNode +.. autoclass:: pyflow.ExternFamily .. autoclass:: pyflow.ExternTask -.. autoclass:: pyflow.ExternFamily +.. autoclass:: pyflow.ExternVariable + +.. autoclass:: pyflow.ExternLimit .. autoclass:: pyflow.ExternEvent .. autoclass:: pyflow.ExternMeter -.. autoclass:: pyflow.ExternYMD +.. autoclass:: pyflow.ExternRepeat Deployment diff --git a/docs/content/introductory-course/flow-control.ipynb b/docs/content/introductory-course/flow-control.ipynb index 16542fb..075ed91 100644 --- a/docs/content/introductory-course/flow-control.ipynb +++ b/docs/content/introductory-course/flow-control.ipynb @@ -1001,7 +1001,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "a2937746-a676-49ea-9332-315e763167ca", "metadata": {}, "outputs": [ @@ -1034,7 +1034,7 @@ " etask = pf.ExternTask('/a/b/c/d')\n", " efamily = pf.ExternFamily('/f/g/h/i')\n", " \n", - " eymd = pf.ExternYMD('/a/b/c/d:YMD')\n", + " eymd = pf.ExternRepeat('/a/b/c/d:YMD')\n", " eevent = pf.ExternEvent('/e/f/g/h:ev')\n", " emeter = pf.ExternMeter('/g/h/i/j:mt')\n", " \n", From 22a7f39648b1b4ae58b7a8447f0d437bb69416cd Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 11:21:00 +0000 Subject: [PATCH 13/16] minor debug --- pyflow/attributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index cd1a4df..0109d94 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -1238,7 +1238,7 @@ class Follow(_Trigger): An attribute for setting a condition for running the node behind another repeated node which has completed. Parameters: - value(Repeat_ or Node_): The followed node or the repeat attribute of the followed node. + value(Repeat_ or Task_ or Family_ or Suite_): The followed node or the repeat attribute of the followed node. Example:: t1 = Task("t1", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) From 1817dabf147be3e105b98dd5c38af1f2f8280f24 Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 12:19:31 +0000 Subject: [PATCH 14/16] removing settings method in repeat and cleaning code --- pyflow/attributes.py | 15 --------------- pyflow/extern.py | 18 ++++++++---------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 0109d94..2d23bd7 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -337,9 +337,6 @@ def __init__(self, name, values=None): ) self.parent._repeat = self # set the repeat at the node level - def settings(self): - raise NotImplementedError("Subclasses must implement settings()") - class RepeatDay(Repeat): """ @@ -457,9 +454,6 @@ def values(self): """*list*: The list of enumerated values.""" return [str(x) for x in self.value] - def settings(self): - return self.value - def __add__(self, other): return Add(self, other) @@ -499,9 +493,6 @@ def values(self): v = [int(x) for x in v] return v - def settings(self): - return self.value - def __add__(self, other): return Add(self, other) @@ -600,9 +591,6 @@ def __sub__(self, other): result = Sub(self.julian, other.julian) return result - def settings(self): - return self._start, self._end, self._increment - @property def julian(self): """*int*: The Julian date of the repeat date.""" @@ -701,9 +689,6 @@ def __add__(self, other): def __sub__(self, other): return Sub(self, other) - def settings(self): - return self._start, self._end, self._increment - def _delta_to_string(self, delta): # there is no strftime for timedelta so we make our own total_seconds = int(delta.total_seconds()) diff --git a/pyflow/extern.py b/pyflow/extern.py index 9d9a047..5affc56 100644 --- a/pyflow/extern.py +++ b/pyflow/extern.py @@ -12,7 +12,7 @@ def is_extern_known(ext): return ext in KNOWN_EXTERNS -def ExternNode(path, tail_cls=Family, **args): +def ExternNode(path, tail_cls=Family): """ Maps an external node, i.e. a node that is not built from the same repository. @@ -40,10 +40,10 @@ def ExternNode(path, tail_cls=Family, **args): cls = Family with current: - return tail_cls(path_cpts[-1], extern=True, **args) + return tail_cls(path_cpts[-1], extern=True) -def ExternAttribute(path, cls=Attribute, *args): +def ExternAttribute(path, cls, *args): KNOWN_EXTERNS.add(path) path, attr = path.split(":") kind = Family if "/" in path[1:] else Suite @@ -59,13 +59,12 @@ def ExternVariable(path): path(*str*): Path of the item. Returns: - Variable: An object that corresponds to an external variable. + Variable_: An object that corresponds to an external variable. Example:: - pyflow.ExternYMD('/a/b:var') + pyflow.ExternVariable('/a/b:var') """ - KNOWN_EXTERNS.add(path) return ExternAttribute(path, Variable, 1) @@ -77,13 +76,12 @@ def ExternLimit(path): path(*str*): Path of the item. Returns: - RepeatDate_: An object that corresponds to an external item. + Limit_: An object that corresponds to an external item. Example:: - pyflow.ExternYMD('/a/limits:hpc') + pyflow.ExternLimit('/a/limits:hpc') """ - KNOWN_EXTERNS.add(path) return ExternAttribute(path, Limit, 1) @@ -195,7 +193,7 @@ def ExternSuite(path): path(str): Path of the external suite. Returns: - Family_: An object that corresponds to an external suite. + Suite_: An object that corresponds to an external suite. Example:: From db4f3b51e1d602c9b47643343d650a9bf040ca03 Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 12:21:24 +0000 Subject: [PATCH 15/16] minor change --- pyflow/extern.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflow/extern.py b/pyflow/extern.py index 5affc56..e3e280c 100644 --- a/pyflow/extern.py +++ b/pyflow/extern.py @@ -1,7 +1,7 @@ import datetime from . import warn -from .attributes import Attribute, Event, Limit, Meter, Repeat, RepeatDate, Variable +from .attributes import Event, Limit, Meter, Repeat, RepeatDate, Variable from .base import Root from .nodes import Family, Suite, Task From 59b372adf36f7a02dcdf1d821b897893d6059144 Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 14 Aug 2025 12:22:32 +0000 Subject: [PATCH 16/16] minor change --- tests/test_host.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_host.py b/tests/test_host.py index d6f3d6c..fc18260 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -264,7 +264,7 @@ def test_troika_host(): ) submit_args = { - "total_tasks": 2, # deprecated option, will be translated to total_tasks + "total_tasks": 2, "gpus": 1, "sthost": "/foo/bar", "distribution": "test", # generates TROIKA pragma for recent version of troika, SBATCH for older versions @@ -317,7 +317,7 @@ def test_troika_host(): def test_host_submit_args(): submit_args = { "troika": { - "total_tasks": 2, # deprecated option, will be translated to total_tasks + "total_tasks": 2, "gpus": 1, "sthost": "/foo/bar", "distribution": "test", # generates TROIKA pragma for recent version of troika, SBATCH for older versions