diff --git a/docs/content/api-reference.rst b/docs/content/api-reference.rst index ef743fc..f7df6d5 100644 --- a/docs/content/api-reference.rst +++ b/docs/content/api-reference.rst @@ -226,19 +226,21 @@ External .. _Extern: -.. autoclass:: pyflow.Extern +.. autoclass:: pyflow.ExternSuite -.. autoclass:: pyflow.ExternNode +.. autoclass:: pyflow.ExternFamily .. autoclass:: pyflow.ExternTask -.. autoclass:: pyflow.ExternFamily +.. autoclass:: pyflow.ExternVariable + +.. autoclass:: pyflow.ExternLimit .. autoclass:: pyflow.ExternEvent .. autoclass:: pyflow.ExternMeter -.. autoclass:: pyflow.ExternYMD +.. autoclass:: pyflow.ExternRepeat Deployment diff --git a/docs/content/introductory-course/flow-control.ipynb b/docs/content/introductory-course/flow-control.ipynb index 16542fb..075ed91 100644 --- a/docs/content/introductory-course/flow-control.ipynb +++ b/docs/content/introductory-course/flow-control.ipynb @@ -1001,7 +1001,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "a2937746-a676-49ea-9332-315e763167ca", "metadata": {}, "outputs": [ @@ -1034,7 +1034,7 @@ " etask = pf.ExternTask('/a/b/c/d')\n", " efamily = pf.ExternFamily('/f/g/h/i')\n", " \n", - " eymd = pf.ExternYMD('/a/b/c/d:YMD')\n", + " eymd = pf.ExternRepeat('/a/b/c/d:YMD')\n", " eevent = pf.ExternEvent('/e/f/g/h:ev')\n", " emeter = pf.ExternMeter('/g/h/i/j:mt')\n", " \n", diff --git a/docs/content/introductory-course/helper-functionality.ipynb b/docs/content/introductory-course/helper-functionality.ipynb index 25adaa1..9c4460b 100644 --- a/docs/content/introductory-course/helper-functionality.ipynb +++ b/docs/content/introductory-course/helper-functionality.ipynb @@ -158,7 +158,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "b2e2a324-3df0-4874-8c69-d3d93ea39bc4", "metadata": {}, "outputs": [ @@ -196,7 +196,7 @@ " pf.RepeatDate(\"YMD\", datetime.date(2019, 1, 1), datetime.date(2019, 12, 31))\n", " with pf.Family('follower') as follower:\n", " pf.RepeatDate(\"YMD\", datetime.date(2019, 1, 1), datetime.date(2019, 12, 31))\n", - " follower.follow = leader.YMD\n", + " follower.follow = leader\n", "\n", "s" ] diff --git a/pyflow/__init__.py b/pyflow/__init__.py index 8e6f63b..305eb3d 100644 --- a/pyflow/__init__.py +++ b/pyflow/__init__.py @@ -48,11 +48,15 @@ from .expressions import Deferred, all_complete, sequence from .extern import ( Extern, + ExternAttribute, ExternEvent, ExternFamily, + ExternLimit, ExternMeter, - ExternNode, + ExternRepeat, + ExternSuite, ExternTask, + ExternVariable, ExternYMD, ) from .header import FileHeader, FileTail, Header, InlineCodeHeader diff --git a/pyflow/attributes.py b/pyflow/attributes.py index e8358b5..2d23bd7 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -3,6 +3,7 @@ import datetime import re +from . import warn from .anchor import AnchorMixin from .base import Base, GenerateError from .cron import Crontab @@ -92,26 +93,6 @@ def generate_stub(self): shape = "box" -class RepeatDay(Attribute): - """ - An attribute that allows a node to be repeated infinitely. - - Parameters: - value(int): The repeat step. - - Example:: - - pyflow.attributes.RepeatDay(1) - """ - - def __init__(self, value): - super().__init__("_repeat") - self._value = value - - def _build(self, ecflow_parent): - ecflow_parent.add_repeat(ecflow.RepeatDay(int(self.value))) - - class Time(Attribute): """ An attribute for setting a time dependency of the node. @@ -341,7 +322,43 @@ def __init__(self, **kwargs): Variable(key, val) -class RepeatString(Exportable): +class Repeat(Exportable): + """ + A virtual class that defines a repeat attribute + """ + + def __init__(self, name, values=None): + super().__init__(name, values) + if self.parent._repeat is not None: + warn( + "Overwriting an existing repeat value!", + category=UserWarning, + stacklevel=2, + ) + self.parent._repeat = self # set the repeat at the node level + + +class RepeatDay(Repeat): + """ + An attribute that allows a node to be repeated infinitely. + + Parameters: + value(int): The repeat step. + + Example:: + + pyflow.attributes.RepeatDay(1) + """ + + def __init__(self, value): + super().__init__("_repeat") + self._value = value + + def _build(self, ecflow_parent): + ecflow_parent.add_repeat(ecflow.RepeatDay(int(self.value))) + + +class RepeatString(Repeat): """ An attribute that allows a node to be repeated by a string value. @@ -412,7 +429,7 @@ def __sub__(self, other): return Sub(self, other) -class RepeatEnumerated(Exportable): +class RepeatEnumerated(Repeat): """ An attribute that allows a node to be repeated by an enumerated list. @@ -422,7 +439,7 @@ class RepeatEnumerated(Exportable): Example:: - pyflow.RepeatEnumerated("REPEAT_STRING", ["a", "b", "c", "d", "e"]) + pyflow.RepeatEnumerated("REPEAT_STRING", [1, 3, 4, 5]) """ def __init__(self, name, value): @@ -437,9 +454,6 @@ def values(self): """*list*: The list of enumerated values.""" return [str(x) for x in self.value] - def settings(self): - return self.value - def __add__(self, other): return Add(self, other) @@ -447,7 +461,7 @@ def __sub__(self, other): return Sub(self, other) -class RepeatDateList(Exportable): +class RepeatDateList(Repeat): """ An attribute that allows a node to be repeated over a list of dates. @@ -479,9 +493,6 @@ def values(self): v = [int(x) for x in v] return v - def settings(self): - return self.value - def __add__(self, other): return Add(self, other) @@ -489,7 +500,7 @@ def __sub__(self, other): return Sub(self, other) -class RepeatInteger(Exportable): +class RepeatInteger(Repeat): """ An attribute that allows a node to be repeated by an integer range. @@ -527,7 +538,7 @@ def __sub__(self, other): return Sub(self, other) -class RepeatDate(Exportable): +class RepeatDate(Repeat): """ An attribute that allows a node to be repeated by a date value. @@ -580,9 +591,6 @@ def __sub__(self, other): result = Sub(self.julian, other.julian) return result - def settings(self): - return self._start, self._end, self._increment - @property def julian(self): """*int*: The Julian date of the repeat date.""" @@ -681,9 +689,6 @@ def __add__(self, other): def __sub__(self, other): return Sub(self, other) - def settings(self): - return self._start, self._end, self._increment - def _delta_to_string(self, delta): # there is no strftime for timedelta so we make our own total_seconds = int(delta.total_seconds()) @@ -712,12 +717,6 @@ def day_of_week(self): return Mod(Add(Div(self, 86400), 4), 7) -def string_or_enumerated(name, value): - if all(isinstance(v, int) for v in value): - return RepeatEnumerated(name, value) - return RepeatString(name, value) - - def is_date(value): return ( isinstance(value, (datetime.date, datetime.datetime)) @@ -796,27 +795,9 @@ def make_variable(node, name, value): with node: if isinstance(value, (tuple, list)): - if len(value) in [2, 3]: - if is_date(value[0]) and is_date(value[1]): - if len(value) == 3: - if isinstance(value[2], int): - return RepeatDate( - name, - as_date(value[0]), - as_date(value[1]), - value[2], - ) - else: - return RepeatDate(name, as_date(value[0]), as_date(value[1]), 1) - - if isinstance(value[0], int) and isinstance(value[1], int): - if len(value) == 3: - if isinstance(value[2], int): - return RepeatInteger(name, value[0], value[1], value[2]) - else: - return RepeatInteger(name, value[0], value[1], 1) - - return string_or_enumerated(name, value) + raise Exception( + "Repeat construction through a list is not supported anymore" + ) if isinstance(value, (str, int, float)): return Variable(name, value) @@ -1242,23 +1223,32 @@ class Follow(_Trigger): An attribute for setting a condition for running the node behind another repeated node which has completed. Parameters: - value(RepeatDate_): The repeat date attribute of the followed node. + value(Repeat_ or Task_ or Family_ or Suite_): The followed node or the repeat attribute of the followed node. Example:: - - pyflow.attributes.Follow(pyflow.RepeatDate('REPEAT_DATE', - datetime.date(year=2019, month=1, day=1), - datetime.date(year=2019, month=12, day=31))) + t1 = Task("t1", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) + t2 = Task("t2", repeat=(RepeatEnumerated, "NUM", [1, 2, 3])) + t2.follow = t1 """ def __init__(self, value): - super().__init__("_follow_%s" % (value,), value) - if not hasattr(value, "settings"): - raise Exception( - "Cannot follow a node of type %s (%r)" % (type(value), value) + super().__init__(f"_follow_{value.name}") + from .nodes import Node # yeah it's bad but there's a circular import + + if isinstance(value, Node): + parent = value + repeat = value.repeat + elif isinstance(value, Repeat): + parent = value.parent + repeat = value + else: + raise TypeError( + f"Follow attribute {self.name} requires a Repeat or a Node instance" ) - self.parent[value.name] = value.settings() - self._value = value.parent.complete | (self.parent[value.name] < value) + + if repeat is None: + raise TypeError(f"Follow attribute {self.name} requires a repeat") + self._value = parent.complete | (self.parent.repeat < repeat) ################################################################### diff --git a/pyflow/base.py b/pyflow/base.py index 92ff78d..9fabcfe 100644 --- a/pyflow/base.py +++ b/pyflow/base.py @@ -21,6 +21,10 @@ def remove_node(self, node): def add_node(self, node): pass + @property + def repeat(self): + return None + @property def host(self): return None diff --git a/pyflow/extern.py b/pyflow/extern.py index 489db20..e3e280c 100644 --- a/pyflow/extern.py +++ b/pyflow/extern.py @@ -1,6 +1,7 @@ import datetime -from .attributes import Event, Meter, RepeatDate +from . import warn +from .attributes import Event, Limit, Meter, Repeat, RepeatDate, Variable from .base import Root from .nodes import Family, Suite, Task @@ -45,10 +46,65 @@ def ExternNode(path, tail_cls=Family): def ExternAttribute(path, cls, *args): KNOWN_EXTERNS.add(path) path, attr = path.split(":") - with ExternNode(path): + kind = Family if "/" in path[1:] else Suite + with ExternNode(path, kind): return cls(attr, *args) +def ExternVariable(path): + """ + Maps an external variable. + + Parameters: + path(*str*): Path of the item. + + Returns: + Variable_: An object that corresponds to an external variable. + + Example:: + + pyflow.ExternVariable('/a/b:var') + """ + return ExternAttribute(path, Variable, 1) + + +def ExternLimit(path): + """ + Maps an external limit. + + Parameters: + path(*str*): Path of the item. + + Returns: + Limit_: An object that corresponds to an external item. + + Example:: + + pyflow.ExternLimit('/a/limits:hpc') + """ + return ExternAttribute(path, Limit, 1) + + +def ExternRepeat(path): + """ + Maps an external repeat, i.e. a repeat that is not built from the same repository. + Cannot be a generic attribute as the repeat can be used with the follow() approach, + which requires an object of type Repeat. + + Parameters: + path(*str*): Path of the external repeat. + + Returns: + RepeatDate_: An object that corresponds to an external repeat. + + Example:: + + pyflow.ExternRepeat('/a/b/c/d:YMD') + """ + + return ExternAttribute(path, Repeat) + + def ExternYMD(path): """ Maps an external repeat date, i.e. a repeat date that is not built from the same repository. @@ -63,7 +119,11 @@ def ExternYMD(path): pyflow.ExternYMD('/a/b/c/d:YMD') """ - + warn( + "'ExternYMD' is deprecated, use ExternAttribute instead", + DeprecationWarning, + stacklevel=1, + ) return ExternAttribute( path, RepeatDate, datetime.datetime.now(), datetime.datetime.now() ) @@ -83,7 +143,6 @@ def ExternEvent(path): pyflow.ExternEvent('/e/f/g/h:ev') """ - return ExternAttribute(path, Event) @@ -101,7 +160,6 @@ def ExternMeter(path): pyflow.ExternMeter('/g/h/i/j:mt') """ - return ExternAttribute(path, Meter, 0) @@ -119,10 +177,31 @@ def Extern(path): pyflow.Extern('/f/g/h/i') """ - + warn( + "'Extern' is deprecated, use ExternSuite, ExternFamily or ExternTask instead", + DeprecationWarning, + stacklevel=1, + ) return ExternNode(path) +def ExternSuite(path): + """ + Maps an external suite. + + Parameters: + path(str): Path of the external suite. + + Returns: + Suite_: An object that corresponds to an external suite. + + Example:: + + pyflow.ExternSuite('/a') + """ + return ExternNode(path, Suite) + + def ExternFamily(path): """ Maps an external family, i.e. a family that is not built from the same repository. @@ -137,8 +216,7 @@ def ExternFamily(path): pyflow.ExternFamily('/f/g/h/i') """ - - return ExternNode(path) + return ExternNode(path, Family) def ExternTask(path): @@ -155,5 +233,4 @@ def ExternTask(path): pyflow.ExternTask('/a/b/c/d') """ - - return ExternNode(path, tail_cls=Task) + return ExternNode(path, Task) diff --git a/pyflow/nodes.py b/pyflow/nodes.py index 2a21200..e0a1185 100644 --- a/pyflow/nodes.py +++ b/pyflow/nodes.py @@ -25,7 +25,6 @@ Limit, Manual, Meter, - RepeatDay, Time, Today, Trigger, @@ -112,6 +111,7 @@ def __init__( purge_modules=False, extern=False, workdir=None, + repeat=None, **kwargs, ): """ @@ -142,7 +142,6 @@ def __init__( limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -152,6 +151,7 @@ def __init__( generated_variables(GeneratedVariable_): An attribute for setting an **ecFlow** generated variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the node. """ @@ -162,6 +162,12 @@ def __init__( self._modules = modules or [] self._purge_modules = purge_modules self._extern = extern + self._repeat = None # can't be set in constructor, needs to be done in context + if repeat is not None: + if not isinstance(repeat, (list, tuple)): + raise TypeError("Repeat attribute must be passed as a list or tuple") + with self: + self._repeat = repeat[0](*repeat[1:]) # If we have changed the host, then set the relevant directories self._host = host @@ -301,6 +307,26 @@ def append_node(self, node): self.add_node(node) return self + @property + def repeat(self): + """ + Returns the currently active repeat object. + If not found in current node, search in parents. + + Returns: + Repeat_: Currently active repeat object. + """ + if self._repeat is not None: + return self._repeat + return self.parent.repeat + + @repeat.setter + def repeat(self, value): + if not isinstance(value, (list, tuple)): + raise TypeError("Repeat attribute must be passed as a list or tuple") + with self: + self._repeat = value[0](*value[1:]) + @property def host(self): """ @@ -831,7 +857,6 @@ def __init__( limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -841,6 +866,7 @@ def __init__( generated_variables(GeneratedVariable_): An attribute for setting an **ecFlow** generated variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the family. Example:: @@ -959,7 +985,6 @@ def __init__( limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -968,6 +993,7 @@ def __init__( variables(Variable_): An attribute for setting an **ecFlow** variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the anchor family. Example:: @@ -1034,7 +1060,6 @@ def __init__(self, name, host=None, exit_hook=None, *args, **kwargs): limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -1044,6 +1069,7 @@ def __init__(self, name, host=None, exit_hook=None, *args, **kwargs): generated_variables(GeneratedVariable_): An attribute for setting an **ecFlow** generated variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the suite. Example:: @@ -1244,7 +1270,6 @@ def __init__( limits(Limit_): An attribute for a simple load management by limiting the number of tasks submitted by a specific **ecFlow** server. meters(Meter_): An attribute for a range of integer values that can be set from a script. - repeat(RepeatDay_): An attribute that allows a node to be repeated infinitely. tasks(Task_): An attribute for adding a child task on the node. time(Time_): An attribute for setting a time dependency of the node. today(Today_): An attribute for setting a cron dependency of the node for the current day. @@ -1254,6 +1279,7 @@ def __init__( generated_variables(GeneratedVariable_): An attribute for setting an **ecFlow** generated variable. zombies(Zombies_): An attribute that defines how a zombie should be handled in an automated fashion. events(Event_): An attribute for declaring an action that a task can trigger while it is running. + repeat(Repeat_): An attribute for setting a repeat schedule for the node. **kwargs(str): Accept extra keyword arguments as variables to be set on the task. Example:: @@ -1519,7 +1545,6 @@ def generate_script(self): ("labels", Label), ("limits", Limit), ("meters", Meter), - ("repeat", RepeatDay), ("tasks", Task), ("time", Time), ("today", Today), diff --git a/tests/test8.json b/tests/test8.json index 85e8970..ce617c7 100644 --- a/tests/test8.json +++ b/tests/test8.json @@ -1,10 +1,6 @@ { "FOO": 42, "f1": { - "YMD": [ - "2010-01-01", - "2011-01-01" - ], "labels": { "foo_label": "bar" }, @@ -69,6 +65,5 @@ }, "f2": { "FOO": 42 - }, - "repeat": true + } } diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 22fcaaf..c74d2f7 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -46,9 +46,9 @@ def test_reassign_variable(): with pyflow.Suite("s") as s: with pyflow.Task("t1") as t: t.FOO = 61 - t.FOO = (1, 10) + t.FOO = 100 with pyflow.Task("t2") as t: - t.FOO = (1, 10) + t.FOO = 100 assert s.t1.FOO.value == s.t2.FOO.value @@ -276,9 +276,11 @@ def test_date(): assert "date *.*.3" in str(s.ecflow_definition()) with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1", DATE=("20180105", "20180206")) + t1 = pyflow.Task( + "t1", repeat=(pyflow.RepeatString, "DATE", ["20180105", "20180206"]) + ) - assert t1.DATE.value == ("20180105", "20180206") + assert t1.DATE.value == ["20180105", "20180206"] assert 'repeat string DATE "20180105" "20180206"' in str(s.ecflow_definition()) @@ -288,7 +290,9 @@ class TestRepeats: def test_string_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f1") as f1: - f1.STRING_REPEAT = [str(v) for v in reversed(range(10))] + pyflow.RepeatString( + "STRING_REPEAT", [str(v) for v in reversed(range(10))] + ) t1 = pyflow.Task("t1") t1.triggers = (f1.STRING_REPEAT == "7") & (f1.STRING_REPEAT == 3) @@ -305,9 +309,13 @@ def test_string_repeat(self): def test_combined_string_repeats(self): with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1", YMD=["20170101", "20180101"]) - t2 = pyflow.Task("t2", YMD=["20170101", "20180101"]) - t2.triggers = t1.YMD >= t2.YMD + t1 = pyflow.Task( + "t1", repeat=(pyflow.RepeatDate, "YMD", "20170101", "20180101") + ) + t2 = pyflow.Task( + "t2", repeat=(pyflow.RepeatDate, "YMD", "20170101", "20180101") + ) + t2.triggers = t1.YMD >= t2.repeat assert str(t2.triggers.value) == "(/s/t1:YMD ge /s/t2:YMD)" s.check_definition() @@ -315,7 +323,7 @@ def test_combined_string_repeats(self): def test_enumerated_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f2") as f2: - f2.ENUMERATED_REPEAT = list(reversed(range(10))) + pyflow.RepeatEnumerated("ENUMERATED_REPEAT", list(range(10))) t2 = pyflow.Task("t2") t2.triggers = (f2.ENUMERATED_REPEAT == "7") & ( @@ -336,8 +344,10 @@ def test_enumerated_repeat(self): def test_combined_enumerated_repeats(self): with pyflow.Suite("s") as s: - t1 = pyflow.Task("t1", ENUMERATED_REPEAT=list(range(10))) - t2 = pyflow.Task("t2", ENUMERATED_REPEAT=list(range(10))) + with pyflow.Task("t1") as t1: + pyflow.RepeatEnumerated("ENUMERATED_REPEAT", list(range(10))) + with pyflow.Task("t2") as t2: + pyflow.RepeatEnumerated("ENUMERATED_REPEAT", list(range(10))) t2.triggers = t1.ENUMERATED_REPEAT >= t2.ENUMERATED_REPEAT assert ( str(t2.triggers.value) @@ -382,7 +392,9 @@ def test_combined_integer_repeats(self): def test_date_datetime_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f4") as f4: - f4.DATE_REPEAT = (datetime(2018, 1, 1), datetime(2019, 12, 31)) + pyflow.RepeatDateTime( + "DATE_REPEAT", datetime(2018, 1, 1), datetime(2019, 12, 31) + ) t4 = pyflow.Task("t4") t4.triggers = (f4.DATE_REPEAT >= "20180301") & ( @@ -403,7 +415,7 @@ def test_date_datetime_repeat(self): def test_date_date_repeat(self): with pyflow.Suite("s") as s: with pyflow.Family("f4") as f4: - f4.DATE_REPEAT = (date(2018, 1, 1), date(2019, 12, 31)) + pyflow.RepeatDate("DATE_REPEAT", date(2018, 1, 1), date(2019, 12, 31)) t4 = pyflow.Task("t4") t4.triggers = (f4.DATE_REPEAT >= "20180301") & ( diff --git a/tests/test_contextmanager.py b/tests/test_contextmanager.py index f7fcebc..5ca6eb6 100644 --- a/tests/test_contextmanager.py +++ b/tests/test_contextmanager.py @@ -9,6 +9,7 @@ Label, Limit, Meter, + RepeatEnumerated, Suite, Task, Tasks, @@ -22,15 +23,15 @@ def test_suite(): Limit("foo", 1) Limit("bar", 2) - with Family("f", BAR=[1, 2, 3]): + with Family("f", repeat=(RepeatEnumerated, "BAR", [1, 2, 3])): Task("t1") Task("t2") Task("t3").triggers = (s.f.t1 == "complete") | "2 < 8" - with Family("g") as g: + with Family("g"): InLimit("foo") - g.QUUX = [1, 2] + RepeatEnumerated("QUUX", [1, 2]) Task("t4").triggers = s.f.t1 == "aborted" with Task("t5"): diff --git a/tests/test_extern.py b/tests/test_extern.py index 8dfb2fd..2aa0059 100644 --- a/tests/test_extern.py +++ b/tests/test_extern.py @@ -7,29 +7,35 @@ Event, ExternEvent, ExternFamily, + ExternLimit, ExternMeter, + ExternRepeat, + ExternSuite, ExternTask, - ExternYMD, + ExternVariable, Family, + Limit, Meter, Notebook, RepeatDate, Suite, Task, + Variable, ) -from pyflow.extern import KNOWN_EXTERNS +from pyflow.extern import KNOWN_EXTERNS, Repeat now = datetime.datetime.now() def test_extern(): with Suite("s") as s: - t1 = Task("t1", YMD=(now, now)) + t1 = Task("t1", repeat=(RepeatDate, "YMD", now, now)) et = ExternTask("/a/b/c/d") ef = ExternFamily("/f/g/h/i") + es = ExternSuite("/j") - t1.triggers = et & ef + t1.triggers = et & ef & es # Check that the externs have real types --> will have correct functionality available @@ -39,6 +45,9 @@ def test_extern(): assert isinstance(ef, Family) assert ef.name == "i" assert ef.fullname == "/f/g/h/i" + assert isinstance(es, Suite) + assert es.name == "j" + assert es.fullname == "/j" # Check that they work! @@ -68,18 +77,40 @@ def test_extern(): def test_extern_attributes(): + sext = ExternSuite("/limits") # extern shall not be under a node suite/family/task + evar = ExternVariable("/a/main:SUITE_START") + svar = ExternVariable("/a:SUITE_START") + limit = ExternLimit( + "/limits:hpc" + ) # extern shall not be under a node suite/family/task + + # svar = ExternEdit("/b:SUITE_START") # OK with Suite("s") as s: - eymd = ExternYMD("/a/b/c/d:YMD") + eymd = ExternRepeat("/a/b/c/d:YMD") + elimit = ExternLimit("/limits/lim:hpc") + slimit = ExternLimit("/limits:hpc") eevent = ExternEvent("/e/f/g/h:ev") emeter = ExternMeter("/g/h/i/j:mt") - Task("t1", YMD=(now, now)).follow = eymd + t1 = Task("t1", repeat=(RepeatDate, "YMD", now, now)) + t1.follow = eymd Task("t2").triggers = eevent Task("t3").triggers = emeter == 10 + Task("t4").completes = evar != eymd + Task("t5").inlimits = [elimit, slimit] + Task("t6").completes = svar != eymd + Task("ts").completes = sext.complete + # Check that the externs have real types --> will have correct functionality available - # Check that the externs have real types --> will have correct functionality available + assert isinstance(elimit, Limit) + assert limit.name == "hpc" + assert limit.fullname == "/limits:hpc" + + assert isinstance(svar, Variable) + assert svar.name == "SUITE_START" + assert svar.fullname == "/a:SUITE_START" - assert isinstance(eymd, RepeatDate) + assert isinstance(eymd, Repeat) assert eymd.name == "YMD" assert eymd.fullname == "/a/b/c/d:YMD" @@ -111,16 +142,21 @@ def test_extern_attributes(): def test_extern_safety(): externs = [] + externs.append(ExternSuite("/limits")) + externs.append(ExternLimit("/limits:hpc")) + externs.append(ExternLimit("/limits/lim:hpc")) + externs.append(ExternVariable("/a/main:SUITE_START")) + externs.append(ExternVariable("/a:SUITE_START")) with Suite("s"): - externs.append(ExternTask("/a/b/c/d")) - externs.append(ExternFamily("/e/f/g/h")) + externs.append(ExternFamily("/a/b/c/d")) + externs.append(ExternTask("/e/f/g/h")) with externs[-1]: # n.b. should never do this in reality, but trying to break things... externs.append(Task("e3")) - externs.append(ExternYMD("i/j/k/l:YMD")) + externs.append(ExternRepeat("i/j/k/l:YMD")) externs.append(ExternEvent("m/n/o/p:ev")) externs.append(ExternMeter("q/s/t/u:mt")) diff --git a/tests/test_follow.py b/tests/test_follow.py index 81caa29..2d46ea1 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -1,23 +1,34 @@ import datetime -from pyflow import Notebook, Suite, Task +from pyflow import Family, Notebook, RepeatDate, Suite, Task now = datetime.datetime.now() def test_follow(): with Suite("s") as s: - t1 = Task("t1", YMD=(now, now)) - t2 = Task("t2") - Task("t3") + with Task("t1"): + r1 = RepeatDate("YMD1", now, now) + with Family("f1") as f1: + f1.repeat = (RepeatDate, "YMD2", now, now) + t2 = Task("t2") + t3 = Task("t3", repeat=(RepeatDate, "YMD3", now, now)) - t2.triggers = t1.complete - t2.follow = t1.YMD + t2.follow = r1 + t3.follow = t2 s.check_definition() s.generate_node() s.deploy_suite(target=Notebook) + print(s) + print(t3.repeat) + print(str(t2.triggers)) + print(str(t3.triggers)) + print(t2.triggers) + assert "trigger ../t1 eq complete or ../f1:YMD2 lt ../t1:YMD1" in str(t2.triggers) + assert "trigger f1/t2 eq complete or t3:YMD3 lt f1:YMD2" in str(t3.triggers) + assert "repeat date YMD3 20250814 20250814 1" in str(t3.triggers) if __name__ == "__main__": diff --git a/tests/test_host.py b/tests/test_host.py index 09fb0b4..fc18260 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -264,7 +264,7 @@ def test_troika_host(): ) submit_args = { - "tasks": 2, # deprecated option, will be translated to total_tasks + "total_tasks": 2, "gpus": 1, "sthost": "/foo/bar", "distribution": "test", # generates TROIKA pragma for recent version of troika, SBATCH for older versions @@ -317,7 +317,7 @@ def test_troika_host(): def test_host_submit_args(): submit_args = { "troika": { - "tasks": 2, # deprecated option, will be translated to total_tasks + "total_tasks": 2, "gpus": 1, "sthost": "/foo/bar", "distribution": "test", # generates TROIKA pragma for recent version of troika, SBATCH for older versions diff --git a/tests/test_json.py b/tests/test_json.py index 29f53da..9318013 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -9,6 +9,7 @@ def test_json(): x = json.loads(f.read()) s = Suite("s", json=x) + print(s) s.check_definition() s.generate_node() diff --git a/tests/test_suite.py b/tests/test_suite.py index 03724f1..d462ffb 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -2,7 +2,7 @@ import pytest -from pyflow import Family, Suite, Task +from pyflow import Family, RepeatDate, RepeatString, Suite, Task from pyflow.base import GenerateError @@ -18,8 +18,10 @@ def test_suite(): t1["DATE"] = 19900101 t1.FOOO = 42 - t1.BAR = ["ab", "cd", "ef"] - t2.YMD = (datetime.datetime(2000, 1, 1), datetime.datetime(2010, 1, 1)) + with t1: + RepeatString("BAR", ["ab", "cd", "ef"]) + with t2: + RepeatDate("YMD", datetime.datetime(2000, 1, 1), datetime.datetime(2010, 1, 1)) f += Family("g") f.g += Task("t4") @@ -63,7 +65,7 @@ def test_suite(): "a": { "inlimits": s.l1, "FOO": 42, - "YMD": (now, then), + "repeat": (RepeatDate, "YMD", now, then), "labels": [("info", "hi"), ("status", "ok")], "meters": ("progress", 0, 100), } @@ -84,8 +86,10 @@ def test_suite_builtin_triggers(): t1["DATE"] = 19900101 t1.FOOO = 42 - t1.BAR = ["ab", "cd", "ef"] - t2.YMD = (datetime.datetime(2000, 1, 1), datetime.datetime(2010, 1, 1)) + with t1: + RepeatString("BAR", ["ab", "cd", "ef"]) + with t2: + RepeatDate("YMD", datetime.datetime(2000, 1, 1), datetime.datetime(2010, 1, 1)) f += Family("g") f.g += Task("t4") @@ -129,7 +133,7 @@ def test_suite_builtin_triggers(): "a": { "inlimits": s.l1, "FOO": 42, - "YMD": (now, then), + "repeat": (RepeatDate, "YMD", now, then), "labels": [("info", "hi"), ("status", "ok")], "meters": ("progress", 0, 100), }