diff --git a/.github/workflows/readthedocs-pr.yml b/.github/workflows/readthedocs-pr.yml index 7939a4c..6dd0782 100644 --- a/.github/workflows/readthedocs-pr.yml +++ b/.github/workflows/readthedocs-pr.yml @@ -2,7 +2,7 @@ # This does NOT trigger a build of the documentation, this is handled through webhooks. name: Read the Docs PR Preview on: - pull_request: + pull_request_target: types: - opened - synchronize diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d347971..3807566 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -22,3 +22,9 @@ build: os: "ubuntu-20.04" tools: python: "mambaforge-4.10" + jobs: + # Install the checked-out source (e.g. the current PR) into the conda + # environment, so the docs are built against this code. Dependencies are + # already provided by docs/environment.yml, hence --no-deps. + post_install: + - pip install --no-deps . diff --git a/docs/_ext/ecflow_lexers.py b/docs/_ext/ecflow_lexers.py index a202c1c..1c2ee3f 100644 --- a/docs/_ext/ecflow_lexers.py +++ b/docs/_ext/ecflow_lexers.py @@ -33,7 +33,7 @@ class EcflowDefLexer(RegexLexer): bygroups(Keyword, Name.Constant), ), ( - r"(repeat)(\s+(?:date(?:list)?|day|month|year|integer|enumerated|string))(\s+(?:.+?))(\s(?:.*))", + r"(repeat)(\s+(?:date(?:time)?(?:list)?|day|month|year|integer|enumerated|string))(\s+(?:.+?))(\s(?:.*))", # noqa: E501 bygroups(Keyword, Name.Other, Name.Variable, Literal.Date), ), # Required diff --git a/docs/content/api-reference.rst b/docs/content/api-reference.rst index f7df6d5..d518e46 100644 --- a/docs/content/api-reference.rst +++ b/docs/content/api-reference.rst @@ -160,6 +160,14 @@ Repeat .. autoclass:: pyflow.attributes.RepeatDateList +.. _RepeatDateTime: + +.. autoclass:: pyflow.attributes.RepeatDateTime + +.. _RepeatDateTimeList: + +.. autoclass:: pyflow.attributes.RepeatDateTimeList + .. _RepeatDay: .. autoclass:: pyflow.attributes.RepeatDay diff --git a/docs/environment.yml b/docs/environment.yml index db553a6..a244d83 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -18,7 +18,6 @@ dependencies: - sphinx-rtd-theme - sphinx-copybutton - sphinx-tabs - - git+https://github.com/ecmwf/pyflow.git variables: QT_MAC_WANTS_LAYER: 1 diff --git a/docs/requirements.txt b/docs/requirements.txt index e4b5648..cc30bbd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ ipykernel nbsphinx +pypandoc_binary sphinx-rtd-theme==0.5.2 sphinx-copybutton==0.3.1 -sphinx-tabs -git+https://github.com/ecmwf/pyflow.git \ No newline at end of file +sphinx-tabs \ No newline at end of file diff --git a/pyflow/__init__.py b/pyflow/__init__.py index 305eb3d..20c0e80 100644 --- a/pyflow/__init__.py +++ b/pyflow/__init__.py @@ -30,6 +30,7 @@ RepeatDate, RepeatDateList, RepeatDateTime, + RepeatDateTimeList, RepeatDay, RepeatEnumerated, RepeatInteger, diff --git a/pyflow/attributes.py b/pyflow/attributes.py index 6ef5947..06f3653 100644 --- a/pyflow/attributes.py +++ b/pyflow/attributes.py @@ -25,7 +25,7 @@ expression_from_json, make_expression, ) -from .importer import ecflow +from .importer import ecflow, supported from .state import aborted, active, complete, queued, submitted, suspended, unknown NO_TRIGGER = False @@ -641,6 +641,7 @@ def day_of_week(self): setattr(RepeatDate, day, property(lambda self: Eq(self.day_of_week, dow))) +@supported(">=5.12.0") class RepeatDateTime(Exportable): """ An attribute that allows a node to be repeated by a date+time value. @@ -658,11 +659,17 @@ class RepeatDateTime(Exportable): datetime.datetime(year=2019, month=12, day=31, hour=12, minute=0, second=0), datetime.timedelta(hours=12, minutes=0, seconds=0)) - Date and increment can also be strings:: + Start/End values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with + hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``, + or simply a date ``yyyymmdd`` (the missing components are assumed to be 0), and increment can also be a string:: pyflow.RepeatDateTime('REPEAT_DATETIME', '20190101T120000', '20191231T120000', '12:00:00') + Note:: + + This repeat type is only supported in ecFlow 5.12.0 and later. + """ def __init__( @@ -727,6 +734,92 @@ def day_of_week(self): return Mod(Add(Div(self, 86400), 4), 7) +@supported(">=5.17.0") +class RepeatDateTimeList(Repeat): + """ + An attribute that allows a node to be repeated over a list of datetime values. + + Parameters: + name(str): The name of the repeat attribute. + values(list of datetime/date): The list of datetime/date values, as datetime/date objects or strings. + + Example:: + + pyflow.RepeatDateTimeList('REPEAT_DATETIME', + [datetime.date(year=2019, month=1, day=1), + datetime.datetime(year=2019, month=1, day=3, hour=12, minute=0, seconds=0)]) + + Values can also be strings: ISO 8601 basic format `yyyymmddTHHMMSS`, DateTime with + hours and minutes ``yyyymmddTHHMM``, DateTime with hours only ``yyyymmddTHH``, + or simply a date ``yyyymmdd`` (the missing components are assumed to be 0):: + + pyflow.RepeatDateTimeList('REPEAT_DATETIME', ['20190101T120000', '20190103']) + + Note:: + + This repeat type is only supported in ecFlow 5.17.0 and later. + """ + + def __init__(self, name, values): + if values is None: + raise ValueError("values cannot be None") + if not isinstance(values, list): + raise TypeError("values must be a list") + if isinstance(values, list) and not values: + raise ValueError("values cannot be an empty list") + if not all( + isinstance(value, (datetime.datetime, datetime.date, str)) + for value in values + ): + raise TypeError("values must be a list of datetime objects or strings") + + super().__init__(name, values) + + def _build(self, ecflow_parent): + # Format all datetime values as ISO 8601 basic format `yyyymmddTHHMMSS` + values = [as_date(value).strftime("%Y%m%dT%H%M%S") for value in self.values] + + repeat = ecflow.RepeatDateTimeList( + str(self.name), + values, + ) + + ecflow_parent.add_repeat(repeat) + + @property + def values(self): + """*list*: The list of datetime values.""" + return [ + x if isinstance(x, datetime.datetime) else as_date(x) for x in self.value + ] + + def __add__(self, other): + return Add(self, other) + + def __sub__(self, other): + return Sub(self, other) + + @property + def second(self): + """*int*: The second of the repeat datetime.""" + return Mod(self, 60) + + @property + def minute(self): + """*int*: The minute of the repeat datetime.""" + return Mod(Div(self, 60), 60) + + @property + def hour(self): + """*int*: The hour of the repeat datetime.""" + return Mod(Div(self, 3600), 24) + + @property + def day_of_week(self): + """*int*: The day of the week of the repeat datetime.""" + return Mod(Add(Div(self, 86400), 4), 7) + + def is_date(value): return ( isinstance(value, (datetime.date, datetime.datetime)) diff --git a/pyflow/importer.py b/pyflow/importer.py index 870c7b8..3969ee7 100644 --- a/pyflow/importer.py +++ b/pyflow/importer.py @@ -1,5 +1,9 @@ +import functools import os import sys +import types + +from packaging.specifiers import SpecifierSet try: import ecflow @@ -30,3 +34,70 @@ raise ImportError( "Could not find ecflow Python library, try to set ECFLOW_DIR environment variable to correct path" ) + + +def supported(specifier: str, current: str = ecflow.__version__): + """ + A decorator that ensures the decorated class can only be used when + the available ecFlow version satisfies ``specifier``. + + Every method and property of the class is wrapped so that invoking it + (e.g. instantiating the class via ``__init__``, or accessing a property) + raises :class:`NotImplementedError` when ``current`` does not satisfy + ``specifier``. + + The version comparison is evaluated once, when the class is decorated + (i.e. at import time, using ``current`` which defaults to the version of + the imported ecFlow module). + The ``current`` argument allows tests to instrument the behaviour for an arbitrary version. + + Parameters: + specifier(str): A :class:`packaging.specifiers.SpecifierSet` string, e.g. ``">=5.12.0,<6.0.0"``. + current(str): The current version, e.g. ``"5.12.0"``. Defaults to the version of the imported ecFlow module. + """ + + # Evaluate the comparison once, since both versions are fixed for the + # lifetime of the decorated class. + is_supported = SpecifierSet(specifier).contains(current) + + def decorator(cls): + + # Define a wrapper factory that checks the supported version before calling the original function + def make_wrapper(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not is_supported: + raise NotImplementedError( + "{} functionality is only supported for ecFlow {}, but current version is {}".format( + cls.__name__, specifier, current + ) + ) + + return func(*args, **kwargs) + + return wrapper + + # Iterate over a copy as we mutate the class namespace while iterating. + for attr_name, attr_value in list(cls.__dict__.items()): + + if isinstance(attr_value, types.FunctionType): + # Wrapped plain methods directly. + setattr(cls, attr_name, make_wrapper(attr_value)) + + elif isinstance(attr_value, property): + # Ensure properties must remain properties, + # by wrapping accessors and rebuilding the property so attribute access keeps working. + setattr( + cls, + attr_name, + property( + make_wrapper(attr_value.fget) if attr_value.fget else None, + make_wrapper(attr_value.fset) if attr_value.fset else None, + make_wrapper(attr_value.fdel) if attr_value.fdel else None, + attr_value.__doc__, + ), + ) + + return cls + + return decorator diff --git a/pyproject.toml b/pyproject.toml index 440f8ce..a07e905 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dynamic = ["version", "readme"] dependencies = [ "jinja2", + "packaging", "requests", ] diff --git a/tests/test_attributes.py b/tests/test_attributes.py index 594eac9..36ce55e 100644 --- a/tests/test_attributes.py +++ b/tests/test_attributes.py @@ -554,6 +554,97 @@ def test_repeat_datetime(self): s.check_definition() + def test_repeat_datetimelist_basic_usage(self): + from datetime import datetime as dt + + i = dt(2000, 1, 1, 12, 0, 0) + j = dt(2000, 1, 2) + + input_tests = ( + ("A", [i]), + ("B", [i, j]), + ("C", ["20000103T120000", "20000104T000000"]), + ("D", [i, "20010105T123456"]), + ) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + asserts = ( + 'repeat datetimelist A "20000101T120000"', + 'repeat datetimelist B "20000101T120000" "20000102T000000"', + 'repeat datetimelist C "20000103T120000" "20000104T000000"', + 'repeat datetimelist D "20000101T120000" "20010105T123456"', + ) + defn = str(s.ecflow_definition()) + for a in asserts: + assert a in defn + + s.check_definition() + + def test_repeat_datetimelist_allowing_date_and_datetime_object(self): + import datetime + + i = datetime.datetime(2000, 1, 1, 12, 34, 56) + j = datetime.date(2000, 1, 2) + + input_tests = (("A", [i, j]),) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + asserts = 'repeat datetimelist A "20000101T123456" "20000102T000000"' + defn = str(s.ecflow_definition()) + for a in asserts: + assert a in defn + + s.check_definition() + + def test_repeat_datetimelist_with_truncated_string_values(self): + input_tests = ( + ("A", ["20000101T01", "20000102T0102", "20000103T010203"]), + ("B", ["20000104", "20000105T12", "20010106T1234"]), + ) + + with pyflow.Suite("s") as s: + for idx, args in enumerate(input_tests): + with pyflow.Task(f"t{idx}"): + pyflow.RepeatDateTimeList(*args) + + asserts = ( + 'repeat datetimelist A "20000101T010000" "20000102T010200" "20000103T010203"', + 'repeat datetimelist B "20000104T000000" "20000105T120000" "20010106T123400"', + ) + defn = str(s.ecflow_definition()) + for a in asserts: + assert a in defn + + s.check_definition() + + def test_repeat_datetimelist_with_none_value(self): + with pytest.raises(ValueError, match="values cannot be None"): + pyflow.RepeatDateTimeList("N", None) + + def test_repeat_datetimelist_with_empty_values_list(self): + with pytest.raises(ValueError, match="values cannot be an empty list"): + pyflow.RepeatDateTimeList("E", []) + + def test_repeat_datetimelist_with_invalid_type_values_list(self): + with pytest.raises( + TypeError, match="values must be a list of datetime objects or strings" + ): + pyflow.RepeatDateTimeList("I", [20050101]) + + def test_repeat_datetimelist_with_literal_type_value(self): + from datetime import datetime as dt + + with pytest.raises(TypeError, match="values must be a list"): + pyflow.RepeatDateTimeList("I", dt(2000, 1, 1, 0, 0, 0)) + def test_repeat_date_list(self): i = date(year=2019, month=12, day=31) j = date(year=2020, month=1, day=1) diff --git a/tests/test_supported.py b/tests/test_supported.py new file mode 100644 index 0000000..cd58406 --- /dev/null +++ b/tests/test_supported.py @@ -0,0 +1,176 @@ +import ecflow +import pytest +from packaging import version + +import pyflow +from pyflow.importer import supported + + +def make_widget(specifier, current): + """Build a class decorated with an explicitly instrumented specifier/current version.""" + + @supported(specifier, current=current) + class Widget: + def __init__(self, value=0): + self.value = value + + def a_method(self): + "A Widget method." + return self.value + + @property + def a_property(self): + "A Widget property." + return self.value * 2 + + return Widget + + +def test_widget_use_is_disallowed_on_older_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + with pytest.raises(NotImplementedError): + Widget() + + +def test_widget_use_is_allowed_on_newer_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.13.0") + assert Widget(3).value == 3 + + +def test_widget_use_is_allowed_on_boundary_version(): + """The exact lower-bound version must be accepted (>= semantics).""" + Widget = make_widget(specifier=">=5.12.0", current="5.12.0") + assert Widget(7).value == 7 + + +def test_widget_use_is_disallowed_on_just_below_boundary(): + Widget = make_widget(specifier=">=5.12.0", current="5.11.9") + with pytest.raises(NotImplementedError): + Widget() + + +def test_error_message_contains_specifier_and_name(): + Widget = make_widget(specifier=">=5.20.1", current="5.6.7") + with pytest.raises(NotImplementedError) as exc: + Widget() + message = str(exc.value) + assert "Widget" in message + assert ">=5.20.1" in message + assert "5.6.7" in message + + +def test_methods_are_guarded_when_unsupported(): + """Ensure non-__init__ methods also raise when unsupported.""" + + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + # __init__ is guarded too, so build with an unguarded instance via __new__. + w = Widget.__new__(Widget) + with pytest.raises(NotImplementedError): + w.a_method() + + +def test_methods_work_when_supported(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(5) + assert w.a_method() == 5 + + +def test_properties_remain_properties(): + """ + Ensure properties behave as properties (return a computed value) after decoration. + """ + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(4) + assert w.a_property == 8 + assert isinstance(type(w).__dict__["a_property"], property) + + +def test_properties_are_disallowed_on_older_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.0.0") + w = Widget.__new__(Widget) + with pytest.raises(NotImplementedError): + _ = w.a_property + + +def test_properties_are_allowed_on_newer_version(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + w = Widget(4) + assert w.a_property == 8 + + +def test_wraps_preserves_metadata(): + Widget = make_widget(specifier=">=5.12.0", current="5.20.0") + assert Widget.a_method.__name__ == "a_method" + assert Widget.a_method.__doc__ == "A Widget method." + assert Widget.__dict__["a_property"].__doc__ == "A Widget property." + + +def test_current_defaults_to_installed_ecflow_version(): + """Without an explicit ``current``, the installed ecFlow version is used.""" + + @supported(">=9999.0.0") + class Future: + def __init__(self): + pass + + with pytest.raises(NotImplementedError): + Future() + + @supported(">=0.0.1") + class Ancient: + def __init__(self): + self.ok = True + + assert Ancient().ok is True + + +def test_compound_specifier_excludes_upper_bound(): + """A compound specifier like >=5.12.0,<5.13.0 rejects versions outside the range.""" + Widget = make_widget(specifier=">=5.12.0,<5.13.0", current="5.13.0") + with pytest.raises(NotImplementedError): + Widget() + + +def test_compound_specifier_allows_version_in_range(): + Widget = make_widget(specifier=">=5.12.0,<5.13.0", current="5.12.5") + assert Widget(1).value == 1 + + +# ----------------------------------------------------------------------------- + + +def _installed_below(min_version): + return version.parse(ecflow.__version__) < version.parse(min_version) + + +def test_repeat_datetime_builds_on_installed_ecflow(): + if _installed_below("5.12.0"): + pytest.skip("RepeatDateTime requires ecFlow >= 5.12.0") + + with pyflow.Suite("s"): + with pyflow.Task("t"): + repeat = pyflow.RepeatDateTime( + "REPEAT_DATETIME", + "20190101T120000", + "20191231T120000", + "12:00:00", + ) + + assert repeat.name == "REPEAT_DATETIME" + assert not callable(repeat.second) + assert isinstance(type(repeat).__dict__["second"], property) + + +def test_repeat_datetimelist_builds_on_installed_ecflow(): + if _installed_below("5.17.0"): + pytest.skip("RepeatDateTimeList requires ecFlow >= 5.17.0") + + with pyflow.Suite("s"): + with pyflow.Task("t"): + repeat = pyflow.RepeatDateTimeList( + "REPEAT_DATETIME", ["20190101T120000", "20190103"] + ) + + assert repeat.name == "REPEAT_DATETIME" + assert not callable(repeat.values) + assert isinstance(type(repeat).__dict__["values"], property)