diff --git a/python/lsst/pex/config/callStack.py b/python/lsst/pex/config/callStack.py index b2ef7871..7e0aa229 100644 --- a/python/lsst/pex/config/callStack.py +++ b/python/lsst/pex/config/callStack.py @@ -89,7 +89,7 @@ class StackFrame(object): The actual content being executed. If not provided, it will be loaded from the file. """ - _STRIP = "/python/lsst/" # String to strip from the filename + _STRIP = "/DRAGONS/" # String to strip from the filename def __init__(self, filename, lineno, function, content=None): loc = filename.rfind(self._STRIP) diff --git a/python/lsst/pex/config/comparison.py b/python/lsst/pex/config/comparison.py index 7f452d8d..a8101761 100644 --- a/python/lsst/pex/config/comparison.py +++ b/python/lsst/pex/config/comparison.py @@ -53,6 +53,8 @@ def compareScalars(name, v1, v2, output, rtol=1E-8, atol=1E-8, dtype=None): Floating point comparisons are performed by numpy.allclose; refer to that for details. """ + if isinstance(dtype, tuple): + dtype = type(v1) if v1 is None or v2 is None: result = (v1 == v2) elif dtype in (float, complex): diff --git a/python/lsst/pex/config/config.py b/python/lsst/pex/config/config.py index 145d4644..a8789698 100644 --- a/python/lsst/pex/config/config.py +++ b/python/lsst/pex/config/config.py @@ -25,6 +25,7 @@ from past.builtins import long from past.builtins import basestring from past.builtins import unicode +from collections import OrderedDict import os import io @@ -33,11 +34,14 @@ import copy import tempfile import shutil +import itertools from .comparison import getComparisonName, compareScalars, compareConfigs from .callStack import getStackFrame, getCallStack from future.utils import with_metaclass +from astrodata import AstroData + __all__ = ("Config", "Field", "FieldValidationError") @@ -63,12 +67,19 @@ def _autocast(x, dtype): If appropriate perform type casting of value x to type dtype, otherwise return the original value x """ - if dtype == float and isinstance(x, int): + if isinstance(x, int) and (dtype == float or (isinstance(dtype, tuple) + and float in dtype and int not in dtype)): return float(x) - if dtype == int and isinstance(x, long): + if isinstance(x, long) and (dtype == int or (isinstance(dtype, tuple) + and int in dtype)): return int(x) - if isinstance(x, str): - return oldStringType(x) + if isinstance(x, str) or isinstance(x, oldStringType): + for type in (int, float, bool, oldStringType): + if dtype == type or (isinstance(dtype, tuple) and type in dtype): + try: + return type(x) + except ValueError: # Carry on and try a different coercion + pass return x @@ -90,7 +101,8 @@ def _typeStr(x): class ConfigMeta(type): - """A metaclass for Config + """ + A metaclass for Config Adds a dictionary containing all Field class attributes as a class attribute called '_fields', and adds the name of each field as @@ -99,19 +111,19 @@ class ConfigMeta(type): """ def __init__(self, name, bases, dict_): type.__init__(self, name, bases, dict_) - self._fields = {} + self._fields = OrderedDict() self._source = getStackFrame() def getFields(classtype): - fields = {} + fields = OrderedDict() bases = list(classtype.__bases__) bases.reverse() for b in bases: fields.update(getFields(b)) - for k, v in classtype.__dict__.items(): - if isinstance(v, Field): - fields[k] = v + field_dict = {k: v for k, v in classtype.__dict__.items() if isinstance(v, Field)} + for k, v in sorted(field_dict.items(), key=lambda x: x[1]._creation_order): + fields[k] = v return fields fields = getFields(self) @@ -142,11 +154,13 @@ def __init__(self, field, config, msg): self.history = config.history.setdefault(field.name, []) self.fieldSource = field.source self.configSource = config._source - error = "%s '%s' failed validation: %s\n"\ - "For more information read the Field definition at:\n%s"\ - "And the Config definition at:\n%s" % \ - (self.fieldType.__name__, self.fullname, msg, - self.fieldSource.format(), self.configSource.format()) + #error = "%s '%s' failed validation: %s\n"\ + # "For more information read the Field definition at:\n%s"\ + # "And the Config definition at:\n%s" % \ + # (self.fieldType.__name__, self.fullname, msg, + # self.fieldSource.format(), self.configSource.format()) + error = ("{} '{}' ({}) failed validation: {}". + format(self.fieldType.__name__, self.fullname, field.doc, msg)) ValueError.__init__(self, error) @@ -161,7 +175,9 @@ class Example(Config): """ # Must be able to support str and future str as we can not guarantee that # code will pass in a future str type on Python 2 - supportedTypes = set((str, unicode, basestring, oldStringType, bool, float, int, complex)) + supportedTypes = set((str, unicode, basestring, oldStringType, bool, float, int, complex, + tuple, AstroData)) + _counter = itertools.count() def __init__(self, doc, dtype, default=None, check=None, optional=False): """Initialize a Field. @@ -175,12 +191,19 @@ def __init__(self, doc, dtype, default=None, check=None, optional=False): method; this will be ignored if set to None. optional --- When False, Config validate() will fail if value is None """ - if dtype not in self.supportedTypes: + if isinstance(dtype, list): + dtype = tuple(dtype) + if isinstance(dtype, tuple): + if any([x not in self.supportedTypes for x in dtype]): + raise ValueError("Unsupported Field dtype in %s" % repr(dtype)) + elif dtype not in self.supportedTypes: raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) # Use standard string type if we are given a future str if dtype == str: dtype = oldStringType + elif isinstance(dtype, tuple): + dtype = tuple(oldStringType if dt==str else dt for dt in dtype) source = getStackFrame() self._setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source) @@ -197,6 +220,7 @@ def _setup(self, doc, dtype, default, check, optional, source): self.check = check self.optional = optional self.source = source + self._creation_order = next(Field._counter) def rename(self, instance): """ @@ -243,8 +267,12 @@ def _validateValue(self, value): return if not isinstance(value, self.dtype): - msg = "Value %s is of incorrect type %s. Expected type %s" % \ - (value, _typeStr(value), _typeStr(self.dtype)) + if isinstance(self.dtype, tuple): + msg = "Value %s is of incorrect type %s. Expected types %s" % \ + ( value, _typeStr(value), [_typeStr(dt) for dt in self.dtype]) + else: + msg = "Value %s is of incorrect type %s. Expected type %s" % \ + (value, _typeStr(value), _typeStr(self.dtype)) raise TypeError(msg) if self.check is not None and not self.check(value): msg = "Value %s is not a valid value" % str(value) @@ -301,7 +329,11 @@ def __get__(self, instance, owner=None, at=None, label="default"): if instance is None or not isinstance(instance, Config): return self else: - return instance._storage[self.name] + if self.name in instance: + return instance._storage[self.name] + else: + raise AttributeError("'{}' object has no attribute '{}'". + format(instance.__class__.__name__, self.name)) def __set__(self, instance, value, at=None, label='assignment'): """ @@ -328,6 +360,14 @@ def __set__(self, instance, value, at=None, label='assignment'): if instance._frozen: raise FieldValidationError(self, instance, "Cannot modify a frozen Config") + if at is None: + at = getCallStack() + # setDefaults() gets a free pass due to our mashing of inheritance + if self.name not in instance._fields: + #if any('setDefaults' in stk.function for stk in at): + # return + raise AttributeError("{} has no attribute {}".format(instance.__class__.__name__, self.name)) + history = instance._history.setdefault(self.name, []) if value is not None: value = _autocast(value, self.dtype) @@ -337,9 +377,9 @@ def __set__(self, instance, value, at=None, label='assignment'): raise FieldValidationError(self, instance, str(e)) instance._storage[self.name] = value - if at is None: - at = getCallStack() - history.append((value, at, label)) + # We don't want to put an actual AD object here, so just the filename + value_to_append = value.filename if isinstance(value, AstroData) else value + history.append((value_to_append, at, label)) def __delete__(self, instance, at=None, label='deletion'): """ @@ -426,6 +466,9 @@ class Config(with_metaclass(ConfigMeta, object)): attributes. Config also emulates a dict of field name: field value + + CJS: Edited these so only the _fields are exposed. _storage retains + items that have been deleted """ def __iter__(self): @@ -436,39 +479,49 @@ def __iter__(self): def keys(self): """!Return the list of field names """ - return list(self._storage.keys()) + return list(self._fields) def values(self): """!Return the list of field values """ - return list(self._storage.values()) + return self.toDict().values() def items(self): """!Return the list of (field name, field value) pairs """ - return list(self._storage.items()) + return self.toDict().items() def iteritems(self): """!Iterate over (field name, field value) pairs """ - return iter(self._storage.items()) + return self.toDict().iteritems() def itervalues(self): """!Iterate over field values """ - return iter(self.storage.values()) + return self.toDict().itervalues() def iterkeys(self): """!Iterate over field names """ - return iter(self.storage.keys()) + return self.toDict().iterkeys() + + def iterfields(self): + """!Iterate over field objects + """ + return iter(self._fields.values()) + + def doc(self, field): + """Return docstring for field + """ + return self._fields[field].doc def __contains__(self, name): """!Return True if the specified field exists in this config @param[in] name field name to test for """ - return self._storage.__contains__(name) + return self._storage.__contains__(name) and name in self._fields def __new__(cls, *args, **kw): """!Allocate a new Config object. @@ -493,9 +546,7 @@ def __new__(cls, *args, **kw): instance._history = {} instance._imports = set() # load up defaults - for field in instance._fields.values(): - instance._history[field.name] = [] - field.__set__(instance, field.default, at=at + [field.source], label="default") + instance.reset(at=at) # set custom default-overides instance.setDefaults() # set constructor overides @@ -513,6 +564,14 @@ def __reduce__(self): self.saveToStream(stream) return (unreduceConfig, (self.__class__, stream.getvalue().encode())) + def reset(self, at=None): + """Reset all values to their defaults""" + if at is None: + at = getCallStack() + for field in self._fields.values(): + self._history[field.name] = [] + field.__set__(self, field.default, at=at + [field.source], label="default") + def setDefaults(self): """ Derived config classes that must compute defaults rather than using the @@ -538,7 +597,8 @@ def update(self, **kw): field = self._fields[name] field.__set__(self, value, at=at, label=label) except KeyError: - raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self))) + #raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self))) + raise KeyError("{} has no field named {}".format(type(self).__name__.replace('Config', ''), name)) def load(self, filename, root="config"): """!Modify this config in place by executing the Python code in the named file. @@ -655,7 +715,7 @@ def toDict(self): Correct behavior is dependent on proper implementation of Field.toDict. If implementing a new Field type, you may need to implement your own toDict method. """ - dict_ = {} + dict_ = OrderedDict() for name, field in self._fields.items(): dict_[name] = field.toDict(self) return dict_ @@ -689,14 +749,14 @@ def validate(self): for field in self._fields.values(): field.validate(self) - def formatHistory(self, name, **kwargs): + def formatHistory(self, name=None, **kwargs): """!Format the specified config field's history to a more human-readable format @param[in] name name of field whose history is wanted @param[in] kwargs keyword arguments for lsst.pex.config.history.format @return a string containing the formatted history """ - import lsst.pex.config.history as pexHist + from . import history as pexHist return pexHist.format(self, name, **kwargs) """ @@ -731,11 +791,13 @@ def __setattr__(self, attr, value, at=None, label="assignment"): raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr)) def __delattr__(self, attr, at=None, label="deletion"): + # CJS: Hacked to allow setDefaults() to delete non-existent fields + if at is None: + at = getCallStack() if attr in self._fields: - if at is None: - at = getCallStack() - self._fields[attr].__delete__(self, at=at, label=label) - else: + #self._fields[attr].__delete__(self, at=at, label=label) + del self._fields[attr] + elif not any(stk.function== 'setDefaults' for stk in at): object.__delattr__(self, attr) def __eq__(self, other): diff --git a/python/lsst/pex/config/history.py b/python/lsst/pex/config/history.py index 96be7bb2..116914c8 100644 --- a/python/lsst/pex/config/history.py +++ b/python/lsst/pex/config/history.py @@ -131,14 +131,17 @@ def _colorize(text, category): return str(text) -def format(config, name=None, writeSourceLine=True, prefix="", verbose=False): +def format(config, name=None, writeSourceLine=True, prefix="", verbose=False, debug=False): """Format the history record for config.name""" + msg = [] + verbose |= debug # verbose=False and debug=True seems wrong! if name is None: for i, name in enumerate(config.history.keys()): if i > 0: - print() - print(format(config, name)) + msg.append('') + msg.append(format(config, name)) + return '\n'.join(msg) outputs = [] for value, stack, label in config.history[name]: @@ -146,7 +149,7 @@ def format(config, name=None, writeSourceLine=True, prefix="", verbose=False): for frame in stack: if frame.function in ("__new__", "__set__", "__setattr__", "execfile", "wrapper") or \ os.path.split(frame.filename)[1] in ("argparse.py", "argumentParser.py"): - if not verbose: + if not debug: continue line = [] @@ -159,6 +162,9 @@ def format(config, name=None, writeSourceLine=True, prefix="", verbose=False): output.append(line) + if not verbose: + break + outputs.append([value, output]) # # Find the maximum widths of the value and file:lineNo fields @@ -173,7 +179,6 @@ def format(config, name=None, writeSourceLine=True, prefix="", verbose=False): # # actually generate the config history # - msg = [] fullname = "%s.%s" % (config._name, name) if config._name is not None else name msg.append(_colorize(re.sub(r"^root\.", "", fullname), "NAME")) for value, output in outputs: diff --git a/python/lsst/pex/config/listField.py b/python/lsst/pex/config/listField.py index 9009f64c..6ff031d3 100644 --- a/python/lsst/pex/config/listField.py +++ b/python/lsst/pex/config/listField.py @@ -162,15 +162,23 @@ class ListField(Field): If maxLength is not None, then instances of the field must be no longer than maxLength + If single is True, a single object of dtype rather than a list is OK + Additionally users can provide two check functions: listCheck - used to validate the list as a whole, and itemCheck - used to validate each item individually """ def __init__(self, doc, dtype, default=None, optional=False, listCheck=None, itemCheck=None, - length=None, minLength=None, maxLength=None): - if dtype not in Field.supportedTypes: - raise ValueError("Unsupported dtype %s" % _typeStr(dtype)) + length=None, minLength=None, maxLength=None, single=False): + + if isinstance(dtype, list): + dtype = tuple(dtype) + if isinstance(dtype, tuple): + if any([x not in self.supportedTypes for x in dtype]): + raise ValueError("Unsupported Field dtype in %s" % repr(dtype)) + elif dtype not in self.supportedTypes: + raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype)) if length is not None: if length <= 0: raise ValueError("'length' (%d) must be positive" % length) @@ -190,13 +198,19 @@ def __init__(self, doc, dtype, default=None, optional=False, raise ValueError("'itemCheck' must be callable") source = getStackFrame() - self._setup(doc=doc, dtype=List, default=default, check=None, optional=optional, source=source) + if single: + dtype_setup = (List,) + dtype if isinstance(dtype, tuple) else (List, dtype) + else: + dtype_setup = List + self._setup(doc=doc, dtype=dtype_setup, + default=default, check=None, optional=optional, source=source) self.listCheck = listCheck self.itemCheck = itemCheck self.itemtype = dtype self.length = length self.minLength = minLength self.maxLength = maxLength + self.single = single def validate(self, instance): """ @@ -207,7 +221,7 @@ def validate(self, instance): """ Field.validate(self, instance) value = self.__get__(instance) - if value is not None: + if not self.single and value is not None: lenValue = len(value) if self.length is not None and not lenValue == self.length: msg = "Required list length=%d, got length=%d" % (self.length, lenValue) @@ -230,7 +244,14 @@ def __set__(self, instance, value, at=None, label="assignment"): at = getCallStack() if value is not None: - value = List(instance, self, value, at, label) + if not self.single or isinstance(value, (list, tuple)): + value = List(instance, self, value, at, label) + else: + value = _autocast(value, self.dtype) + try: + self._validateValue(value) + except BaseException as e: + raise FieldValidationError(self, instance, str(e)) else: history = instance._history.setdefault(self.name, []) history.append((value, at, label)) @@ -239,7 +260,10 @@ def __set__(self, instance, value, at=None, label="assignment"): def toDict(self, instance): value = self.__get__(instance) - return list(value) if value is not None else None + if isinstance(value, List): + return list(value) + else: + return value def _compare(self, instance1, instance2, shortcut, rtol, atol, output): """Helper function for Config.compare; used to compare two fields for equality. @@ -264,9 +288,15 @@ def _compare(self, instance1, instance2, shortcut, rtol, atol, output): return False if l1 is None and l2 is None: return True + equal = True + if not isinstance(l1, List): + if not isinstance(l2, List): + return compareScalars(name, l1, l2, dtype=self.dtype[1], rtol=rtol, + atol=atol, output=output) + else: + return False if not compareScalars("size for %s" % name, len(l1), len(l2), output=output): return False - equal = True for n, v1, v2 in zip(range(len(l1)), l1, l2): result = compareScalars("%s[%d]" % (name, n), v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)