Gemini/DRAGONS changes

python/lsst/pex/config/callStack.py

@@ -89,7 +89,7 @@ class StackFrame(object):
         The actual content being executed. If not provided, it will be
         loaded from the file.
     """
-    _STRIP = "/python/lsst/"  # String to strip from the filename
+    _STRIP = "/DRAGONS/"  # String to strip from the filename
 
     def __init__(self, filename, lineno, function, content=None):
         loc = filename.rfind(self._STRIP)

python/lsst/pex/config/comparison.py

@@ -53,6 +53,8 @@ def compareScalars(name, v1, v2, output, rtol=1E-8, atol=1E-8, dtype=None):
 
     Floating point comparisons are performed by numpy.allclose; refer to that for details.
     """
+    if isinstance(dtype, tuple):
+        dtype = type(v1)
     if v1 is None or v2 is None:
         result = (v1 == v2)
     elif dtype in (float, complex):

python/lsst/pex/config/config.py

@@ -25,6 +25,7 @@
 from past.builtins import long
 from past.builtins import basestring
 from past.builtins import unicode
+from collections import OrderedDict
 
 import os
 import io
@@ -33,11 +34,14 @@
 import copy
 import tempfile
 import shutil
+import itertools
 
 from .comparison import getComparisonName, compareScalars, compareConfigs
 from .callStack import getStackFrame, getCallStack
 from future.utils import with_metaclass
 
+from astrodata import AstroData
+
 __all__ = ("Config", "Field", "FieldValidationError")
 
 
@@ -63,12 +67,19 @@ def _autocast(x, dtype):
     If appropriate perform type casting of value x to type dtype,
     otherwise return the original value x
     """
-    if dtype == float and isinstance(x, int):
+    if isinstance(x, int) and (dtype == float or (isinstance(dtype, tuple)
+                               and float in dtype and int not in dtype)):
         return float(x)
-    if dtype == int and isinstance(x, long):
+    if isinstance(x, long) and (dtype == int or (isinstance(dtype, tuple)
+                                                 and int in dtype)):
         return int(x)
-    if isinstance(x, str):
-        return oldStringType(x)
+    if isinstance(x, str) or isinstance(x, oldStringType):
+        for type in (int, float, bool, oldStringType):
+            if dtype == type or (isinstance(dtype, tuple) and type in dtype):
+                try:
+                    return type(x)
+                except ValueError:  # Carry on and try a different coercion
+                    pass
     return x
 
 
@@ -90,7 +101,8 @@ def _typeStr(x):
 
 
 class ConfigMeta(type):
-    """A metaclass for Config
+    """
+    A metaclass for Config
 
     Adds a dictionary containing all Field class attributes
     as a class attribute called '_fields', and adds the name of each field as
@@ -99,19 +111,19 @@ class ConfigMeta(type):
     """
     def __init__(self, name, bases, dict_):
         type.__init__(self, name, bases, dict_)
-        self._fields = {}
+        self._fields = OrderedDict()
         self._source = getStackFrame()
 
         def getFields(classtype):
-            fields = {}
+            fields = OrderedDict()
             bases = list(classtype.__bases__)
             bases.reverse()
             for b in bases:
                 fields.update(getFields(b))
 
-            for k, v in classtype.__dict__.items():
-                if isinstance(v, Field):
-                    fields[k] = v
+            field_dict = {k: v for k, v in classtype.__dict__.items() if isinstance(v, Field)}
+            for k, v in sorted(field_dict.items(), key=lambda x: x[1]._creation_order):
+                fields[k] = v
             return fields
 
         fields = getFields(self)
@@ -142,11 +154,13 @@ def __init__(self, field, config, msg):
         self.history = config.history.setdefault(field.name, [])
         self.fieldSource = field.source
         self.configSource = config._source
-        error = "%s '%s' failed validation: %s\n"\
-                "For more information read the Field definition at:\n%s"\
-                "And the Config definition at:\n%s" % \
-            (self.fieldType.__name__, self.fullname, msg,
-             self.fieldSource.format(), self.configSource.format())
+        #error = "%s '%s' failed validation: %s\n"\
+        #        "For more information read the Field definition at:\n%s"\
+        #        "And the Config definition at:\n%s" % \
+        #    (self.fieldType.__name__, self.fullname, msg,
+        #     self.fieldSource.format(), self.configSource.format())
+        error = ("{} '{}' ({}) failed validation: {}".
+                 format(self.fieldType.__name__, self.fullname, field.doc, msg))
         ValueError.__init__(self, error)
 
 
@@ -161,7 +175,9 @@ class Example(Config):
     """
     # Must be able to support str and future str as we can not guarantee that
     # code will pass in a future str type on Python 2
-    supportedTypes = set((str, unicode, basestring, oldStringType, bool, float, int, complex))
+    supportedTypes = set((str, unicode, basestring, oldStringType, bool, float, int, complex,
+                          tuple, AstroData))
+    _counter = itertools.count()
 
     def __init__(self, doc, dtype, default=None, check=None, optional=False):
         """Initialize a Field.
@@ -175,12 +191,19 @@ def __init__(self, doc, dtype, default=None, check=None, optional=False):
                      method; this will be ignored if set to None.
         optional --- When False, Config validate() will fail if value is None
         """
-        if dtype not in self.supportedTypes:
+        if isinstance(dtype, list):
+            dtype = tuple(dtype)
+        if isinstance(dtype, tuple):
+            if any([x not in self.supportedTypes for x in dtype]):
+                raise ValueError("Unsupported Field dtype in %s" % repr(dtype))
+        elif dtype not in self.supportedTypes:
             raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype))
 
         # Use standard string type if we are given a future str
         if dtype == str:
             dtype = oldStringType
+        elif isinstance(dtype, tuple):
+            dtype = tuple(oldStringType if dt==str else dt for dt in dtype)
 
         source = getStackFrame()
         self._setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
@@ -197,6 +220,7 @@ def _setup(self, doc, dtype, default, check, optional, source):
         self.check = check
         self.optional = optional
         self.source = source
+        self._creation_order = next(Field._counter)
 
     def rename(self, instance):
         """
@@ -243,8 +267,12 @@ def _validateValue(self, value):
             return
 
         if not isinstance(value, self.dtype):
-            msg = "Value %s is of incorrect type %s. Expected type %s" % \
-                (value, _typeStr(value), _typeStr(self.dtype))
+            if isinstance(self.dtype, tuple):
+                msg = "Value %s is of incorrect type %s. Expected types %s" % \
+                  ( value, _typeStr(value), [_typeStr(dt) for dt in self.dtype])
+            else:
+                msg = "Value %s is of incorrect type %s. Expected type %s" % \
+                    (value, _typeStr(value), _typeStr(self.dtype))
             raise TypeError(msg)
         if self.check is not None and not self.check(value):
             msg = "Value %s is not a valid value" % str(value)
@@ -301,7 +329,11 @@ def __get__(self, instance, owner=None, at=None, label="default"):
         if instance is None or not isinstance(instance, Config):
             return self
         else:
-            return instance._storage[self.name]
+            if self.name in instance:
+                return instance._storage[self.name]
+            else:
+                raise AttributeError("'{}' object has no attribute '{}'".
+                                     format(instance.__class__.__name__, self.name))
 
     def __set__(self, instance, value, at=None, label='assignment'):
         """
@@ -328,6 +360,14 @@ def __set__(self, instance, value, at=None, label='assignment'):
         if instance._frozen:
             raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
 
+        if at is None:
+            at = getCallStack()
+        # setDefaults() gets a free pass due to our mashing of inheritance
+        if self.name not in instance._fields:
+            #if any('setDefaults' in stk.function for stk in at):
+            #    return
+            raise AttributeError("{} has no attribute {}".format(instance.__class__.__name__, self.name))
+
         history = instance._history.setdefault(self.name, [])
         if value is not None:
             value = _autocast(value, self.dtype)
@@ -337,9 +377,9 @@ def __set__(self, instance, value, at=None, label='assignment'):
                 raise FieldValidationError(self, instance, str(e))
 
         instance._storage[self.name] = value
-        if at is None:
-            at = getCallStack()
-        history.append((value, at, label))
+        # We don't want to put an actual AD object here, so just the filename
+        value_to_append = value.filename if isinstance(value, AstroData) else value
+        history.append((value_to_append, at, label))
 
     def __delete__(self, instance, at=None, label='deletion'):
         """
@@ -426,6 +466,9 @@ class Config(with_metaclass(ConfigMeta, object)):
     attributes.
 
     Config also emulates a dict of field name: field value
+
+    CJS: Edited these so only the _fields are exposed. _storage retains
+    items that have been deleted 
     """
 
     def __iter__(self):
@@ -436,39 +479,49 @@ def __iter__(self):
     def keys(self):
         """!Return the list of field names
         """
-        return list(self._storage.keys())
+        return list(self._fields)
 
     def values(self):
         """!Return the list of field values
         """
-        return list(self._storage.values())
+        return self.toDict().values()
 
     def items(self):
         """!Return the list of (field name, field value) pairs
         """
-        return list(self._storage.items())
+        return self.toDict().items()
 
     def iteritems(self):
         """!Iterate over (field name, field value) pairs
         """
-        return iter(self._storage.items())
+        return self.toDict().iteritems()
 
     def itervalues(self):
         """!Iterate over field values
         """
-        return iter(self.storage.values())
+        return self.toDict().itervalues()
 
     def iterkeys(self):
         """!Iterate over field names
         """
-        return iter(self.storage.keys())
+        return self.toDict().iterkeys()
+
+    def iterfields(self):
+        """!Iterate over field objects
+        """
+        return iter(self._fields.values())
+
+    def doc(self, field):
+        """Return docstring for field
+        """
+        return self._fields[field].doc
 
     def __contains__(self, name):
         """!Return True if the specified field exists in this config
 
         @param[in] name  field name to test for
         """
-        return self._storage.__contains__(name)
+        return self._storage.__contains__(name) and name in self._fields
 
     def __new__(cls, *args, **kw):
         """!Allocate a new Config object.
@@ -493,9 +546,7 @@ def __new__(cls, *args, **kw):
         instance._history = {}
         instance._imports = set()
         # load up defaults
-        for field in instance._fields.values():
-            instance._history[field.name] = []
-            field.__set__(instance, field.default, at=at + [field.source], label="default")
+        instance.reset(at=at)
         # set custom default-overides
         instance.setDefaults()
         # set constructor overides
@@ -513,6 +564,14 @@ def __reduce__(self):
         self.saveToStream(stream)
         return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
 
+    def reset(self, at=None):
+        """Reset all values to their defaults"""
+        if at is None:
+            at = getCallStack()
+        for field in self._fields.values():
+            self._history[field.name] = []
+            field.__set__(self, field.default, at=at + [field.source], label="default")
+
     def setDefaults(self):
         """
         Derived config classes that must compute defaults rather than using the
@@ -538,7 +597,8 @@ def update(self, **kw):
                 field = self._fields[name]
                 field.__set__(self, value, at=at, label=label)
             except KeyError:
-                raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self)))
+                #raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self)))
+                raise KeyError("{} has no field named {}".format(type(self).__name__.replace('Config', ''), name))
 
     def load(self, filename, root="config"):
         """!Modify this config in place by executing the Python code in the named file.
@@ -655,7 +715,7 @@ def toDict(self):
         Correct behavior is dependent on proper implementation of  Field.toDict. If implementing a new
         Field type, you may need to implement your own toDict method.
         """
-        dict_ = {}
+        dict_ = OrderedDict()
         for name, field in self._fields.items():
             dict_[name] = field.toDict(self)
         return dict_
@@ -689,14 +749,14 @@ def validate(self):
         for field in self._fields.values():
             field.validate(self)
 
-    def formatHistory(self, name, **kwargs):
+    def formatHistory(self, name=None, **kwargs):
         """!Format the specified config field's history to a more human-readable format
 
         @param[in] name  name of field whose history is wanted
         @param[in] kwargs  keyword arguments for lsst.pex.config.history.format
         @return a string containing the formatted history
         """
-        import lsst.pex.config.history as pexHist
+        from . import history as pexHist
         return pexHist.format(self, name, **kwargs)
 
     """
@@ -731,11 +791,13 @@ def __setattr__(self, attr, value, at=None, label="assignment"):
             raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr))
 
     def __delattr__(self, attr, at=None, label="deletion"):
+        # CJS: Hacked to allow setDefaults() to delete non-existent fields
+        if at is None:
+            at = getCallStack()
         if attr in self._fields:
-            if at is None:
-                at = getCallStack()
-            self._fields[attr].__delete__(self, at=at, label=label)
-        else:
+            #self._fields[attr].__delete__(self, at=at, label=label)
+            del self._fields[attr]
+        elif not any(stk.function== 'setDefaults' for stk in at):
             object.__delattr__(self, attr)
 
     def __eq__(self, other):