DM-22802: Changes requested by the DRAGONS team

python/lsst/pex/config/callStack.py

@@ -102,7 +102,7 @@ class StackFrame:
     getStackFrame
     """
 
-    _STRIP = "/python/lsst/"
+    _STRIP = "/DRAGONS/"
     """String to strip from the ``filename`` in the constructor."""
 
     def __init__(self, filename, lineno, function, content=None):

python/lsst/pex/config/comparison.py

@@ -98,6 +98,8 @@ def compareScalars(name, v1, v2, output, rtol=1e-8, atol=1e-8, dtype=None):
     -----
     Floating point comparisons are performed by `numpy.allclose`.
     """
+    if isinstance(dtype, tuple):
+        dtype = type(v1)
     if v1 is None or v2 is None:
         result = v1 == v2
     elif dtype in (float, complex):

python/lsst/pex/config/config.py

@@ -38,6 +38,7 @@
 import copy
 import importlib
 import io
+import itertools
 import math
 import numbers
 import os
@@ -57,6 +58,14 @@
 except ImportError:
     yaml = None
 
+try:
+    from astrodata import AstroData
+except ImportError:
+
+    class AstroData:
+        pass
+
+
 from .callStack import getCallStack, getStackFrame
 from .comparison import compareConfigs, compareScalars, getComparisonName
 
@@ -140,10 +149,19 @@ def _autocast(x, dtype):
     -----
     Will convert numpy scalar types to the standard Python equivalents.
     """
-    if dtype is float and isinstance(x, numbers.Real):
+    if isinstance(x, numbers.Real) and (
+        dtype is float or (isinstance(dtype, tuple) and float in dtype and int not in dtype)
+    ):
         return float(x)
     if dtype is int and isinstance(x, numbers.Integral):
         return int(x)
+    if isinstance(x, str):
+        for type in (int, float, bool):
+            if dtype == type or (isinstance(dtype, tuple) and type in dtype):
+                try:
+                    return type(x)
+                except ValueError:  # Carry on and try a different coercion
+                    pass
     return x
 
 
@@ -235,9 +253,9 @@ def getFields(classtype):
             for b in bases:
                 fields.update(getFields(b))
 
-            for k, v in classtype.__dict__.items():
-                if isinstance(v, Field):
-                    fields[k] = v
+            field_dict = {k: v for k, v in classtype.__dict__.items() if isinstance(v, Field)}
+            for k, v in sorted(field_dict.items(), key=lambda x: x[1]._creation_order):
+                fields[k] = v
             return fields
 
         fields = getFields(cls)
@@ -295,7 +313,7 @@ def __init__(self, field, config, msg):
 
         self.configSource = config._source
         error = (
-            f"{self.fieldType.__name__} '{self.fullname}' failed validation: {msg}\n"
+            f"{self.fieldType.__name__} '{self.fullname}' ({field.doc}) failed validation: {msg}\n"
             f"For more information see the Field definition at:\n{self.fieldSource.format()}"
             f" and the Config definition at:\n{self.configSource.format()}"
         )
@@ -402,10 +420,12 @@ class Field(Generic[FieldTypeVar]):
     Class.
     """
 
-    supportedTypes = {str, bool, float, int, complex}
+    supportedTypes = {str, bool, float, int, complex, tuple, AstroData}
     """Supported data types for field values (`set` of types).
     """
 
+    _counter = itertools.count()
+
     @staticmethod
     def _parseTypingArgs(
         params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any]
@@ -469,7 +489,12 @@ def __init__(self, doc, dtype=None, default=None, check=None, optional=False, de
             raise ValueError(
                 "dtype must either be supplied as an argument or as a type argument to the class"
             )
-        if dtype not in self.supportedTypes:
+        if isinstance(dtype, list):
+            dtype = tuple(dtype)
+        if isinstance(dtype, tuple):
+            if any(x not in self.supportedTypes for x in dtype):
+                raise ValueError(f"Unsupported Field dtype in {_typeStr(dtype)}")
+        elif dtype not in self.supportedTypes:
             raise ValueError(f"Unsupported Field dtype {_typeStr(dtype)}")
 
         source = getStackFrame()
@@ -528,6 +553,8 @@ def _setup(self, doc, dtype, default, check, optional, source, deprecated):
         `~lsst.pex.config.callStack.StackFrame`).
         """
 
+        self._creation_order = next(Field._counter)
+
     def rename(self, instance):
         r"""Rename the field in a `~lsst.pex.config.Config` (for internal use
         only).
@@ -615,9 +642,16 @@ def _validateValue(self, value):
             return
 
         if not isinstance(value, self.dtype):
-            msg = (
-                f"Value {value} is of incorrect type {_typeStr(value)}. Expected type {_typeStr(self.dtype)}"
-            )
+            if isinstance(self.dtype, tuple):
+                msg = (
+                    f"Value {value} is of incorrect type {_typeStr(value)}. "
+                    f"Expected types {[_typeStr(dt) for dt in self.dtype]}"
+                )
+            else:
+                msg = (
+                    f"Value {value} is of incorrect type {_typeStr(value)}. "
+                    f"Expected type {_typeStr(self.dtype)}"
+                )
             raise TypeError(msg)
         if self.check is not None and not self.check(value):
             msg = f"Value {value} is not a valid value"
@@ -784,6 +818,12 @@ def __set__(
         if instance._frozen:
             raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
 
+        if at is None:
+            at = getCallStack()
+        # setDefaults() gets a free pass due to our mashing of inheritance
+        if self.name not in instance._fields:
+            raise AttributeError(f"{instance.__class__.__name__} has no attribute {self.name}")
+
         history = instance._history.setdefault(self.name, [])
         if value is not None:
             value = _autocast(value, self.dtype)
@@ -793,9 +833,9 @@ def __set__(
                 raise FieldValidationError(self, instance, str(e)) from e
 
         instance._storage[self.name] = value
-        if at is None:
-            at = getCallStack()
-        history.append((value, at, label))
+        # We don't want to put an actual AD object here, so just the filename
+        value_to_append = value.filename if isinstance(value, AstroData) else value
+        history.append((value_to_append, at, label))
 
     def __delete__(self, instance, at=None, label="deletion"):
         """Delete an attribute from a `lsst.pex.config.Config` instance.
@@ -970,6 +1010,9 @@ class behavior.
     _history: dict[str, list[Any]]
     _imports: set[Any]
 
+    # Only _fields are exposure. _storage retains items that have been
+    # deleted.
+
     def __iter__(self):
         """Iterate over fields."""
         return self._fields.__iter__()
@@ -982,7 +1025,7 @@ def keys(self):
         names : `~collections.abc.KeysView`
             List of `lsst.pex.config.Field` names.
         """
-        return self._storage.keys()
+        return list(self._fields)
 
     def values(self):
         """Get field values.
@@ -992,7 +1035,7 @@ def values(self):
         values : `~collections.abc.ValuesView`
             Iterator of field values.
         """
-        return self._storage.values()
+        return self.toDict().values()
 
     def items(self):
         """Get configurations as ``(field name, field value)`` pairs.
@@ -1005,7 +1048,22 @@ def items(self):
             0. Field name.
             1. Field value.
         """
-        return self._storage.items()
+        return self.toDict().items()
+
+    def doc(self, field):
+        """Return docstring for field.
+
+        Parameters
+        ----------
+        field : `str`
+            Field to select.
+
+        Returns
+        -------
+        doc : `str`
+            Associated docstring.
+        """
+        return self._fields[field].doc
 
     def __contains__(self, name):
         """Return `True` if the specified field exists in this config.
@@ -1020,7 +1078,7 @@ def __contains__(self, name):
         in : `bool`
             `True` if the specified field exists in the config.
         """
-        return self._storage.__contains__(name)
+        return self._storage.__contains__(name) and name in self._fields
 
     def __new__(cls, *args, **kw):
         """Allocate a new `lsst.pex.config.Config` object.
@@ -1046,9 +1104,7 @@ def __new__(cls, *args, **kw):
         instance._history = {}
         instance._imports = set()
         # load up defaults
-        for field in instance._fields.values():
-            instance._history[field.name] = []
-            field.__set__(instance, field.default, at=at + [field.source], label="default")
+        instance.reset(at=at)
         # set custom default-overrides
         instance.setDefaults()
         # set constructor overrides
@@ -1068,6 +1124,20 @@ def __reduce__(self):
         self.saveToStream(stream)
         return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
 
+    def reset(self, at=None):
+        """Reset all values to their defaults.
+
+        Parameters
+        ----------
+        at : `lists` [ `StackFrame` ] or `None`, optional
+            Location in stack.
+        """
+        if at is None:
+            at = getCallStack()
+        for field in self._fields.values():
+            self._history[field.name] = []
+            field.__set__(self, field.default, at=at + [field.source], label="default")
+
     def setDefaults(self):
         """Subclass hook for computing defaults.
 
@@ -1136,7 +1206,7 @@ def update(self, **kw):
                 field = self._fields[name]
                 field.__set__(self, value, at=at, label=label)
             except KeyError as e:
-                e.add_note(f"No field of name {name} exists in config type {_typeStr(self)}")
+                e.add_note(f"{type(self).__name__.replace('Config', '')} has no field named {name}")
                 raise
 
     def load(self, filename, root="config"):
@@ -1520,7 +1590,7 @@ def validate(self):
         for field in self._fields.values():
             field.validate(self)
 
-    def formatHistory(self, name, **kwargs):
+    def formatHistory(self, name=None, **kwargs):
         """Format a configuration field's history to a human-readable format.
 
         Parameters
@@ -1539,7 +1609,7 @@ def formatHistory(self, name, **kwargs):
         --------
         lsst.pex.config.history.format
         """
-        import lsst.pex.config.history as pexHist
+        from . import history as pexHist
 
         return pexHist.format(self, name, **kwargs)
 
@@ -1583,11 +1653,12 @@ def __setattr__(self, attr, value, at=None, label="assignment"):
             raise AttributeError(f"{_typeStr(self)} has no attribute {attr}")
 
     def __delattr__(self, attr, at=None, label="deletion"):
+        # CJS: Hacked to allow setDefaults() to delete non-existent fields
+        if at is None:
+            at = getCallStack()
         if attr in self._fields:
-            if at is None:
-                at = getCallStack()
-            self._fields[attr].__delete__(self, at=at, label=label)
-        else:
+            del self._fields[attr]
+        elif not any(stk.function == "setDefaults" for stk in at):
             object.__delattr__(self, attr)
 
     def __eq__(self, other):

python/lsst/pex/config/history.py

@@ -170,7 +170,7 @@ def _colorize(text, category):
     return str(text)
 
 
-def format(config, name=None, writeSourceLine=True, prefix="", verbose=False):
+def format(config, name=None, writeSourceLine=True, prefix="", verbose=False, debug=False):
     """Format the history record for a configuration, or a specific
     configuration field.
 
@@ -189,12 +189,17 @@ def format(config, name=None, writeSourceLine=True, prefix="", verbose=False):
         even before any source line. The default is an empty string.
     verbose : `bool`, optional
         Default is `False`.
+    debug : `bool`, optional
+        Enable debug detail.
     """
+    msg = []
+    verbose |= debug  # verbose=False and debug=True seems wrong!
     if name is None:
         for i, name in enumerate(config.history.keys()):
             if i > 0:
-                print()
-            print(format(config, name))
+                msg.append("")
+            msg.append(format(config, name))
+        return "\n".join(msg)
 
     outputs = []
     for value, stack, _ in config.history.get(name, []):
@@ -207,7 +212,7 @@ def format(config, name=None, writeSourceLine=True, prefix="", verbose=False):
                 "execfile",
                 "wrapper",
             ) or os.path.split(frame.filename)[1] in ("argparse.py", "argumentParser.py"):
-                if not verbose:
+                if not debug:
                     continue
 
             line = []
@@ -235,6 +240,9 @@ def format(config, name=None, writeSourceLine=True, prefix="", verbose=False):
 
             output.append(line)
 
+            if not verbose:
+                break
+
         outputs.append([value, output])
 
     if outputs: