From b633f07278bc3202d3f83588339b6b39375f07dc Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Fri, 8 Nov 2013 21:20:02 -0800 Subject: [PATCH 01/18] [WIP] Added support for declarative document types. --- elasticutils/__init__.py | 41 +++++ elasticutils/exceptions.py | 3 + elasticutils/fields.py | 186 +++++++++++++++++++++++ elasticutils/tests/test_document_type.py | 43 ++++++ 4 files changed, 273 insertions(+) create mode 100644 elasticutils/exceptions.py create mode 100644 elasticutils/fields.py create mode 100644 elasticutils/tests/test_document_type.py diff --git a/elasticutils/__init__.py b/elasticutils/__init__.py index 87ea7db..d642763 100644 --- a/elasticutils/__init__.py +++ b/elasticutils/__init__.py @@ -7,6 +7,7 @@ from elasticsearch.helpers import bulk_index from elasticutils._version import __version__ # noqa +from elasticutils.fields import SearchField log = logging.getLogger('elasticutils') @@ -2194,3 +2195,43 @@ def refresh_index(cls, es=None, index=None): index = cls.get_index() es.indices.refresh(index=index) + + +class DeclarativeMappingMeta(type): + + def __new__(cls, name, bases, attrs): + # TODO: See about keeping attrs in order so the mapping comes out the + # same as the Python code. + fields = [(name_, attrs.pop(name_)) for name_, column in attrs.items() + if isinstance(column, SearchField)] + attrs['fields'] = fields + return super(DeclarativeMappingMeta, cls).__new__(cls, name, bases, + attrs) + + +class DocumentType(object): + __metaclass__ = DeclarativeMappingMeta + + def get_mapping(self): + """ + Returns mapping based on defined fields. + """ + fields = {} + for name, field in self.fields: + f = {'type': field.field_type} + + if field.index_fieldname: + name = field.index_fieldname + + if not field.analyzed: + f['index'] = 'not_analyzed' + + for attr in ('analyzer', 'boost'): + if getattr(field, attr, None): + f[attr] = getattr(field, attr) + + fields[name] = f + + mapping = {'properties': fields} + + return mapping diff --git a/elasticutils/exceptions.py b/elasticutils/exceptions.py new file mode 100644 index 0000000..a2d2aeb --- /dev/null +++ b/elasticutils/exceptions.py @@ -0,0 +1,3 @@ +class SearchFieldError(Exception): + """Raised when a field encounters an error.""" + pass diff --git a/elasticutils/fields.py b/elasticutils/fields.py new file mode 100644 index 0000000..46c4c4f --- /dev/null +++ b/elasticutils/fields.py @@ -0,0 +1,186 @@ +import re + +# TODO: Don't rely on Django. +from django.utils import datetime_safe + +from .exceptions import SearchFieldError + + +DATETIME_REGEX = re.compile('^(?P\d{4})-(?P\d{2})-(?P\d{2})' + '(T|\s+)(?P\d{2}):(?P\d{2}):' + '(?P\d{2}).*?$') + + +class SearchField(object): + + field_type = None + + # TODO: Determine more attributes that need setting. + def __init__(self, analyzer=None, index_fieldname=None, boost=None, + is_multivalued=False, analyzed=True): + self.analyzer = analyzer + self.index_fieldname = index_fieldname + self.boost = boost + self.is_multivalued = is_multivalued + self.analyzed = analyzed + + def prepare(self, obj): + """ + Takes data from the provided object and prepares it for storage in the + index. + + Extending classes should override this method. + """ + return obj + + def convert(self, value): + """ + Handles conversion between the data found and the type of the field. + + Extending classes should override this method and provide correct data + coercion. + """ + return value + + +class CharField(SearchField): + field_type = 'string' + + def prepare(self, obj): + return self.convert(super(CharField, self).prepare(obj)) + + def convert(self, value): + if value is None: + return None + + return unicode(value) + + +class IntegerField(SearchField): + # TODO: Check other integer types and add them. + field_type = 'integer' + + def __init__(self, type='integer', *args, **kwargs): + if type in ('short', 'integer', 'long'): + self.field_type = type + super(IntegerField, self).__init__(*args, **kwargs) + + def prepare(self, obj): + return self.convert(super(IntegerField, self).prepare(obj)) + + def convert(self, value): + if value is None: + return None + + return int(value) + + +class FloatField(SearchField): + # TODO: Check other float types and add them. + field_type = 'float' + + def prepare(self, obj): + return self.convert(super(FloatField, self).prepare(obj)) + + def convert(self, value): + if value is None: + return None + + return float(value) + + +class DecimalField(SearchField): + field_type = 'string' + + def prepare(self, obj): + return self.convert(super(DecimalField, self).prepare(obj)) + + def convert(self, value): + if value is None: + return None + + # TODO: return Decimal(str(value))? + return unicode(value) + + +class BooleanField(SearchField): + field_type = 'boolean' + + def prepare(self, obj): + return self.convert(super(BooleanField, self).prepare(obj)) + + def convert(self, value): + if value is None: + return None + + return bool(value) + + +class DateField(SearchField): + # TODO: Check other date attributes needed here. + field_type = 'date' + + def convert(self, value): + if value is None: + return None + + if isinstance(value, basestring): + match = DATETIME_REGEX.search(value) + + if match: + data = match.groupdict() + return datetime_safe.date(int(data['year']), + int(data['month']), int(data['day'])) + else: + raise SearchFieldError( + "Date provided to '%s' field doesn't appear to be a valid " + "date string: '%s'" % (self.instance_name, value)) + + return value + + +class DateTimeField(SearchField): + field_type = 'datetime' + + def convert(self, value): + if value is None: + return None + + if isinstance(value, basestring): + match = DATETIME_REGEX.search(value) + + if match: + data = match.groupdict() + return datetime_safe.datetime(int(data['year']), + int(data['month']), + int(data['day']), + int(data['hour']), + int(data['minute']), + int(data['second'])) + else: + raise SearchFieldError( + "Datetime provided to '%s' field doesn't appear to be a " + "valid datetime string: '%s'" % ( + self.instance_name, value)) + + return value + + +# TODO: Not sure we need this actually. Multivalued field types can be their +# base class (e.g. IntegerField(multivalue=True) so Python knows how to return +# data. +class MultiValueField(SearchField): + field_type = 'string' + + def __init__(self, **kwargs): + super(MultiValueField, self).__init__(**kwargs) + self.is_multivalued = True + + def prepare(self, obj): + return self.convert(super(MultiValueField, self).prepare(obj)) + + def convert(self, value): + if value is None: + return None + + return list(value) diff --git a/elasticutils/tests/test_document_type.py b/elasticutils/tests/test_document_type.py new file mode 100644 index 0000000..ce6c8fb --- /dev/null +++ b/elasticutils/tests/test_document_type.py @@ -0,0 +1,43 @@ +from unittest import TestCase + +from nose.tools import eq_ + +from elasticutils import DocumentType +from elasticutils.fields import * + + +class BookDocumentType(DocumentType): + + id = IntegerField(type='long') + name = CharField(analyzer='snowball') + name2 = CharField(analyzed=False, index_fieldname='name_sort') + authors = CharField(is_multivalued=True) + published_date = DateField() + price = DecimalField() + is_autographed = BooleanField() + sales = IntegerField() + + +class DocumentTypeTest(TestCase): + + def setUp(self): + self._type = BookDocumentType + + def test_mapping(self): + mapping = self._type().get_mapping() + + # Check top level element. + eq_(mapping.keys(), ['properties']) + + fields = mapping['properties'] + + eq_(fields['id']['type'], 'long') + eq_(fields['name']['type'], 'string') + eq_(fields['name']['analyzer'], 'snowball') + eq_(fields['name_sort']['type'], 'string') + eq_(fields['name_sort']['index'], 'not_analyzed') + eq_(fields['authors']['type'], 'string') + eq_(fields['published_date']['type'], 'date') + eq_(fields['price']['type'], 'string') + eq_(fields['is_autographed']['type'], 'boolean') + eq_(fields['sales']['type'], 'integer') From a637e3e51ee22a7daab3c1132108dac538cbcfed Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Tue, 12 Nov 2013 22:08:01 -0800 Subject: [PATCH 02/18] Maintain field order defined in document type class --- elasticutils/__init__.py | 7 ++++--- elasticutils/fields.py | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/elasticutils/__init__.py b/elasticutils/__init__.py index d642763..7dab8cf 100644 --- a/elasticutils/__init__.py +++ b/elasticutils/__init__.py @@ -1,3 +1,4 @@ +import collections import copy import logging from datetime import datetime @@ -2200,10 +2201,10 @@ def refresh_index(cls, es=None, index=None): class DeclarativeMappingMeta(type): def __new__(cls, name, bases, attrs): - # TODO: See about keeping attrs in order so the mapping comes out the - # same as the Python code. fields = [(name_, attrs.pop(name_)) for name_, column in attrs.items() if isinstance(column, SearchField)] + # Put fields in order defined in the class. + fields.sort(key=lambda f: f[1]._creation_order) attrs['fields'] = fields return super(DeclarativeMappingMeta, cls).__new__(cls, name, bases, attrs) @@ -2216,7 +2217,7 @@ def get_mapping(self): """ Returns mapping based on defined fields. """ - fields = {} + fields = collections.OrderedDict() for name, field in self.fields: f = {'type': field.field_type} diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 46c4c4f..055bab6 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -15,6 +15,9 @@ class SearchField(object): field_type = None + # Used to maintain the order of fields as defined in the class. + _creation_order = 0 + # TODO: Determine more attributes that need setting. def __init__(self, analyzer=None, index_fieldname=None, boost=None, is_multivalued=False, analyzed=True): @@ -24,6 +27,11 @@ def __init__(self, analyzer=None, index_fieldname=None, boost=None, self.is_multivalued = is_multivalued self.analyzed = analyzed + # Store this fields order. + self._creation_order = SearchField._creation_order + # Increment order number for future fields. + SearchField._creation_order += 1 + def prepare(self, obj): """ Takes data from the provided object and prepares it for storage in the From dc6e3bb10c6bb398366b658328861dda9b7fdbcd Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Tue, 12 Nov 2013 22:30:14 -0800 Subject: [PATCH 03/18] Removed Django dependency and updated DateField regex --- elasticutils/fields.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 055bab6..c419de6 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -1,11 +1,10 @@ +import datetime import re -# TODO: Don't rely on Django. -from django.utils import datetime_safe - from .exceptions import SearchFieldError +DATE_REGEX = re.compile('^(?P\d{4})-(?P\d{2})-(?P\d{2}).*?$') DATETIME_REGEX = re.compile('^(?P\d{4})-(?P\d{2})-(?P\d{2})' '(T|\s+)(?P\d{2}):(?P\d{2}):' '(?P\d{2}).*?$') @@ -133,12 +132,12 @@ def convert(self, value): return None if isinstance(value, basestring): - match = DATETIME_REGEX.search(value) + match = DATE_REGEX.search(value) if match: data = match.groupdict() - return datetime_safe.date(int(data['year']), - int(data['month']), int(data['day'])) + return datetime.date( + int(data['year']), int(data['month']), int(data['day'])) else: raise SearchFieldError( "Date provided to '%s' field doesn't appear to be a valid " @@ -159,12 +158,10 @@ def convert(self, value): if match: data = match.groupdict() - return datetime_safe.datetime(int(data['year']), - int(data['month']), - int(data['day']), - int(data['hour']), - int(data['minute']), - int(data['second'])) + return datetime.datetime( + int(data['year']), int(data['month']), int(data['day']), + int(data['hour']), int(data['minute']), + int(data['second'])) else: raise SearchFieldError( "Datetime provided to '%s' field doesn't appear to be a " From 179e42fddeb4ca168e7dd700b900cb707315ea2e Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Tue, 12 Nov 2013 22:33:35 -0800 Subject: [PATCH 04/18] Added all integer types --- elasticutils/fields.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index c419de6..efcb7b3 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -64,11 +64,10 @@ def convert(self, value): class IntegerField(SearchField): - # TODO: Check other integer types and add them. field_type = 'integer' def __init__(self, type='integer', *args, **kwargs): - if type in ('short', 'integer', 'long'): + if type in ('byte', 'short', 'integer', 'long'): self.field_type = type super(IntegerField, self).__init__(*args, **kwargs) From fea7d187ad3086851a7cc3d72f7df64e692e3b44 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Tue, 12 Nov 2013 22:37:16 -0800 Subject: [PATCH 05/18] Added all float field types --- elasticutils/fields.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index efcb7b3..8555b03 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -1,5 +1,6 @@ import datetime import re +from decimal import Decimal from .exceptions import SearchFieldError @@ -82,9 +83,13 @@ def convert(self, value): class FloatField(SearchField): - # TODO: Check other float types and add them. field_type = 'float' + def __init__(self, type='float', *args, **kwargs): + if type in ('float', 'double'): + self.field_type = type + super(FloatField, self).__init__(*args, **kwargs) + def prepare(self, obj): return self.convert(super(FloatField, self).prepare(obj)) @@ -105,8 +110,7 @@ def convert(self, value): if value is None: return None - # TODO: return Decimal(str(value))? - return unicode(value) + return Decimal(str(value)) class BooleanField(SearchField): From 35651e643d05338d9629ab6b3fb20b209c8aac84 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Tue, 12 Nov 2013 22:53:17 -0800 Subject: [PATCH 06/18] Call it `value` and make docstrings clearer --- elasticutils/fields.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 8555b03..804e625 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -32,21 +32,21 @@ def __init__(self, analyzer=None, index_fieldname=None, boost=None, # Increment order number for future fields. SearchField._creation_order += 1 - def prepare(self, obj): + def prepare(self, value): """ - Takes data from the provided object and prepares it for storage in the - index. + Handles conversion between the value sent to Elasticsearch and the type + of the field. Extending classes should override this method. """ - return obj + return value def convert(self, value): """ - Handles conversion between the data found and the type of the field. + Handles conversion between the data received from Elasticsearch and the + type of the field. - Extending classes should override this method and provide correct data - coercion. + Extending classes should override this method. """ return value @@ -54,8 +54,8 @@ def convert(self, value): class CharField(SearchField): field_type = 'string' - def prepare(self, obj): - return self.convert(super(CharField, self).prepare(obj)) + def prepare(self, value): + return self.convert(super(StringField, self).prepare(value)) def convert(self, value): if value is None: @@ -72,8 +72,8 @@ def __init__(self, type='integer', *args, **kwargs): self.field_type = type super(IntegerField, self).__init__(*args, **kwargs) - def prepare(self, obj): - return self.convert(super(IntegerField, self).prepare(obj)) + def prepare(self, value): + return self.convert(super(IntegerField, self).prepare(value)) def convert(self, value): if value is None: @@ -90,8 +90,8 @@ def __init__(self, type='float', *args, **kwargs): self.field_type = type super(FloatField, self).__init__(*args, **kwargs) - def prepare(self, obj): - return self.convert(super(FloatField, self).prepare(obj)) + def prepare(self, value): + return self.convert(super(FloatField, self).prepare(value)) def convert(self, value): if value is None: @@ -103,8 +103,8 @@ def convert(self, value): class DecimalField(SearchField): field_type = 'string' - def prepare(self, obj): - return self.convert(super(DecimalField, self).prepare(obj)) + def prepare(self, value): + return self.convert(super(DecimalField, self).prepare(value)) def convert(self, value): if value is None: @@ -116,8 +116,8 @@ def convert(self, value): class BooleanField(SearchField): field_type = 'boolean' - def prepare(self, obj): - return self.convert(super(BooleanField, self).prepare(obj)) + def prepare(self, value): + return self.convert(super(BooleanField, self).prepare(value)) def convert(self, value): if value is None: @@ -184,8 +184,8 @@ def __init__(self, **kwargs): super(MultiValueField, self).__init__(**kwargs) self.is_multivalued = True - def prepare(self, obj): - return self.convert(super(MultiValueField, self).prepare(obj)) + def prepare(self, value): + return self.convert(super(MultiValueField, self).prepare(value)) def convert(self, value): if value is None: From db99b924fb3f63b6357e96c0fce0feba5fb90997 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Tue, 12 Nov 2013 22:54:01 -0800 Subject: [PATCH 07/18] Added TODOs to support various field attributes --- elasticutils/fields.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 804e625..7659ab2 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -51,7 +51,9 @@ def convert(self, value): return value -class CharField(SearchField): +# TODO: Support all attributes for string types: +# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#string +class StringField(SearchField): field_type = 'string' def prepare(self, value): @@ -64,6 +66,8 @@ def convert(self, value): return unicode(value) +# TODO: Support all attributes for number types: +# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#number class IntegerField(SearchField): field_type = 'integer' @@ -113,6 +117,8 @@ def convert(self, value): return Decimal(str(value)) +# TODO: Support all attributes for boolean types: +# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#boolean class BooleanField(SearchField): field_type = 'boolean' @@ -126,8 +132,9 @@ def convert(self, value): return bool(value) +# TODO: Support all attributes for date types: +# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#date class DateField(SearchField): - # TODO: Check other date attributes needed here. field_type = 'date' def convert(self, value): From dbbd2e8c2c86529b276506ac24a7f0016abb6aa5 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Tue, 12 Nov 2013 22:54:22 -0800 Subject: [PATCH 08/18] Added binary field type --- elasticutils/fields.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 7659ab2..9d14b78 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -1,3 +1,4 @@ +import base64 import datetime import re from decimal import Decimal @@ -181,6 +182,22 @@ def convert(self, value): return value +class BinaryField(SearchField): + field_type = 'binary' + + def prepare(self, value): + if value is None: + return None + + return base64.b64encode(value) + + def convert(self, value): + if value is None: + return None + + return base64.b64decode(value) + + # TODO: Not sure we need this actually. Multivalued field types can be their # base class (e.g. IntegerField(multivalue=True) so Python knows how to return # data. From d9dd074354f18fcbc05c938e5c42d629749fed55 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 7 Dec 2013 09:25:50 -0800 Subject: [PATCH 09/18] Moved field definition to field class. --- elasticutils/__init__.py | 16 +++------------- elasticutils/fields.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/elasticutils/__init__.py b/elasticutils/__init__.py index 7dab8cf..1a9af40 100644 --- a/elasticutils/__init__.py +++ b/elasticutils/__init__.py @@ -2219,19 +2219,9 @@ def get_mapping(self): """ fields = collections.OrderedDict() for name, field in self.fields: - f = {'type': field.field_type} - - if field.index_fieldname: - name = field.index_fieldname - - if not field.analyzed: - f['index'] = 'not_analyzed' - - for attr in ('analyzer', 'boost'): - if getattr(field, attr, None): - f[attr] = getattr(field, attr) - - fields[name] = f + name = field.index_fieldname or name + defn = field.get_definition() + fields[name] = defn mapping = {'properties': fields} diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 9d14b78..b16a3b1 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -51,6 +51,21 @@ def convert(self, value): """ return value + def get_definition(self): + """ + Returns the resprentation for this field's definition in the mapping. + """ + f = {'type': self.field_type} + + if not self.analyzed: + f['index'] = 'not_analyzed' + + for attr in ('analyzer', 'boost'): + if getattr(self, attr, None): + f[attr] = getattr(self, attr) + + return f + # TODO: Support all attributes for string types: # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#string From f01d1edfcb7de9d73b59f616c80ebba302cb5473 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 7 Dec 2013 10:30:31 -0800 Subject: [PATCH 10/18] Fixed tests after changing field class name --- elasticutils/tests/test_document_type.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elasticutils/tests/test_document_type.py b/elasticutils/tests/test_document_type.py index ce6c8fb..80745d1 100644 --- a/elasticutils/tests/test_document_type.py +++ b/elasticutils/tests/test_document_type.py @@ -9,9 +9,9 @@ class BookDocumentType(DocumentType): id = IntegerField(type='long') - name = CharField(analyzer='snowball') - name2 = CharField(analyzed=False, index_fieldname='name_sort') - authors = CharField(is_multivalued=True) + name = StringField(analyzer='snowball') + name2 = StringField(analyzed=False, index_fieldname='name_sort') + authors = StringField(is_multivalued=True) published_date = DateField() price = DecimalField() is_autographed = BooleanField() From f981f7a46eb1d18010ecae79faebedb3e18e288d Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 7 Dec 2013 13:09:18 -0800 Subject: [PATCH 11/18] Added add string field attributes and misc refactorings. --- elasticutils/fields.py | 54 ++++++++++------ elasticutils/tests/test_document_type.py | 19 +++--- elasticutils/tests/test_fields.py | 79 ++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 28 deletions(-) create mode 100644 elasticutils/tests/test_fields.py diff --git a/elasticutils/fields.py b/elasticutils/fields.py index b16a3b1..7cfd792 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -19,14 +19,9 @@ class SearchField(object): # Used to maintain the order of fields as defined in the class. _creation_order = 0 - # TODO: Determine more attributes that need setting. - def __init__(self, analyzer=None, index_fieldname=None, boost=None, - is_multivalued=False, analyzed=True): - self.analyzer = analyzer + def __init__(self, index_fieldname=None, is_multivalued=False): self.index_fieldname = index_fieldname - self.boost = boost self.is_multivalued = is_multivalued - self.analyzed = analyzed # Store this fields order. self._creation_order = SearchField._creation_order @@ -55,22 +50,25 @@ def get_definition(self): """ Returns the resprentation for this field's definition in the mapping. """ - f = {'type': self.field_type} + return {'type': self.field_type} - if not self.analyzed: - f['index'] = 'not_analyzed' - for attr in ('analyzer', 'boost'): - if getattr(self, attr, None): - f[attr] = getattr(self, attr) - - return f - - -# TODO: Support all attributes for string types: -# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#string class StringField(SearchField): field_type = 'string' + attrs = ('analyzer', 'boost', 'ignore_above', 'include_in_all', 'index', + 'index_analyzer', 'index_options', 'null_value', 'omit_norms', + 'position_offset_gap', 'search_analyzer', 'store', 'term_vector') + attr_casts = { + 'float': ['boost'], + 'int': ['position_offset_gap'], + 'bool': ['omit_norms', 'include_in_all'], + } + + def __init__(self, *args, **kwargs): + for attr in self.attrs: + setattr(self, attr, kwargs.pop(attr, None)) + + super(StringField, self).__init__(*args, **kwargs) def prepare(self, value): return self.convert(super(StringField, self).prepare(value)) @@ -81,6 +79,26 @@ def convert(self, value): return unicode(value) + def get_definition(self): + f = super(StringField, self).get_definition() + + for attr in self.attrs: + val = getattr(self, attr, None) + if val is not None: + if attr in self.attr_casts['bool']: + if not val or val == 'false': + f[attr] = False + else: + f[attr] = True + elif attr in self.attr_casts['float']: + f[attr] = float(val) + elif attr in self.attr_casts['int']: + f[attr] = int(val) + else: + f[attr] = val + + return f + # TODO: Support all attributes for number types: # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#number diff --git a/elasticutils/tests/test_document_type.py b/elasticutils/tests/test_document_type.py index 80745d1..ff06886 100644 --- a/elasticutils/tests/test_document_type.py +++ b/elasticutils/tests/test_document_type.py @@ -2,20 +2,19 @@ from nose.tools import eq_ -from elasticutils import DocumentType -from elasticutils.fields import * +from elasticutils import DocumentType, fields class BookDocumentType(DocumentType): - id = IntegerField(type='long') - name = StringField(analyzer='snowball') - name2 = StringField(analyzed=False, index_fieldname='name_sort') - authors = StringField(is_multivalued=True) - published_date = DateField() - price = DecimalField() - is_autographed = BooleanField() - sales = IntegerField() + id = fields.IntegerField(type='long') + name = fields.StringField(analyzer='snowball') + name_sort = fields.StringField(index='not_analyzed') + authors = fields.StringField(is_multivalued=True) + published_date = fields.DateField() + price = fields.DecimalField() + is_autographed = fields.BooleanField() + sales = fields.IntegerField() class DocumentTypeTest(TestCase): diff --git a/elasticutils/tests/test_fields.py b/elasticutils/tests/test_fields.py new file mode 100644 index 0000000..c4e6132 --- /dev/null +++ b/elasticutils/tests/test_fields.py @@ -0,0 +1,79 @@ +from unittest import TestCase + +from nose.tools import eq_ + +from elasticutils import fields + + +class TestStringField(TestCase): + + def test_type(self): + eq_(fields.StringField().field_type, 'string') + + def test_prepare(self): + eq_(fields.StringField().prepare('test'), 'test') + + def test_convert(self): + eq_(fields.StringField().convert(None), None) + eq_(fields.StringField().convert('test'), 'test') + + def test_index_kwarg(self): + field = fields.StringField(index='not_analyzed') + eq_(field.get_definition()['index'], 'not_analyzed') + + def test_store(self): + field = fields.StringField(store='yes') + eq_(field.get_definition()['store'], 'yes') + + def test_term_vector(self): + field = fields.StringField(term_vector='with_offsets') + eq_(field.get_definition()['term_vector'], 'with_offsets') + + def test_boost(self): + field = fields.StringField(boost=2.5) + eq_(field.get_definition()['boost'], 2.5) + field = fields.StringField(boost='2.5') + eq_(field.get_definition()['boost'], 2.5) + + def test_null_value(self): + field = fields.StringField(null_value='na') + eq_(field.get_definition()['null_value'], 'na') + + def test_boolean_attributes(self): + for attr in ('omit_norms', 'include_in_all'): + # Test truthiness. + field = fields.StringField(**{attr: True}) + eq_(field.get_definition()[attr], True) + field = fields.StringField(**{attr: 'true'}) + eq_(field.get_definition()[attr], True) + # Test falsiness. + field = fields.StringField(**{attr: False}) + eq_(field.get_definition()[attr], False) + field = fields.StringField(**{attr: ''}) + eq_(field.get_definition()[attr], False) + + def test_index_options(self): + field = fields.StringField(index_options='positions') + eq_(field.get_definition()['index_options'], 'positions') + + def test_analyzer(self): + field = fields.StringField(analyzer='snowball') + eq_(field.get_definition()['analyzer'], 'snowball') + + def test_index_analyzer(self): + field = fields.StringField(index_analyzer='snowball') + eq_(field.get_definition()['index_analyzer'], 'snowball') + + def test_search_analyzer(self): + field = fields.StringField(search_analyzer='snowball') + eq_(field.get_definition()['search_analyzer'], 'snowball') + + def test_ignore_above(self): + field = fields.StringField(ignore_above='1024') + eq_(field.get_definition()['ignore_above'], '1024') + + def test_position_offset_gap(self): + field = fields.StringField(position_offset_gap=2) + eq_(field.get_definition()['position_offset_gap'], 2) + field = fields.StringField(position_offset_gap='2') + eq_(field.get_definition()['position_offset_gap'], 2) From 2b49ef29ceeadad9084283a6ac0780de00934ce3 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 7 Dec 2013 14:32:19 -0800 Subject: [PATCH 12/18] Added IntegerField and FloatField --- elasticutils/fields.py | 78 ++++++++++++---------- elasticutils/tests/test_fields.py | 104 +++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 35 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 7cfd792..9c3b7fd 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -16,12 +16,17 @@ class SearchField(object): field_type = None + attrs = ('boost', 'include_in_all', 'index', 'null_value', 'store') + bool_casts = ('include_in_all',) + float_casts = ('boost',) + int_casts = () + # Used to maintain the order of fields as defined in the class. _creation_order = 0 - def __init__(self, index_fieldname=None, is_multivalued=False): - self.index_fieldname = index_fieldname - self.is_multivalued = is_multivalued + def __init__(self, *args, **kwargs): + self.index_fieldname = kwargs.pop('index_fieldname', None) + self.is_multivalued = kwargs.pop('is_multivalued', None) # Store this fields order. self._creation_order = SearchField._creation_order @@ -50,19 +55,34 @@ def get_definition(self): """ Returns the resprentation for this field's definition in the mapping. """ - return {'type': self.field_type} + f = {'type': self.field_type} + + for attr in self.attrs: + val = getattr(self, attr, None) + if val is not None: + if attr in self.bool_casts: + if not val or val == 'false': + f[attr] = False + else: + f[attr] = True + elif attr in self.float_casts: + f[attr] = float(val) + elif attr in self.int_casts: + f[attr] = int(val) + else: + f[attr] = str(val) + + return f class StringField(SearchField): field_type = 'string' - attrs = ('analyzer', 'boost', 'ignore_above', 'include_in_all', 'index', - 'index_analyzer', 'index_options', 'null_value', 'omit_norms', - 'position_offset_gap', 'search_analyzer', 'store', 'term_vector') - attr_casts = { - 'float': ['boost'], - 'int': ['position_offset_gap'], - 'bool': ['omit_norms', 'include_in_all'], - } + attrs = SearchField.attrs + ( + 'analyzer', 'ignore_above', + 'index_analyzer', 'index_options', 'omit_norms', + 'position_offset_gap', 'search_analyzer', 'term_vector') + bool_casts = SearchField.bool_casts + ('omit_norms',) + int_casts = SearchField.int_casts + ('position_offset_gap',) def __init__(self, *args, **kwargs): for attr in self.attrs: @@ -79,35 +99,22 @@ def convert(self, value): return unicode(value) - def get_definition(self): - f = super(StringField, self).get_definition() - - for attr in self.attrs: - val = getattr(self, attr, None) - if val is not None: - if attr in self.attr_casts['bool']: - if not val or val == 'false': - f[attr] = False - else: - f[attr] = True - elif attr in self.attr_casts['float']: - f[attr] = float(val) - elif attr in self.attr_casts['int']: - f[attr] = int(val) - else: - f[attr] = val - return f +class NumberField(SearchField): + attrs = SearchField.attrs + ('ignore_malformed', 'precision_step') + bool_casts = SearchField.bool_casts + ('ignore_malformed',) + int_casts = SearchField.int_casts + ('precision_step',) -# TODO: Support all attributes for number types: -# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#number -class IntegerField(SearchField): +class IntegerField(NumberField): field_type = 'integer' + int_casts = NumberField.int_casts + ('null_value',) def __init__(self, type='integer', *args, **kwargs): if type in ('byte', 'short', 'integer', 'long'): self.field_type = type + for attr in self.attrs: + setattr(self, attr, kwargs.pop(attr, None)) super(IntegerField, self).__init__(*args, **kwargs) def prepare(self, value): @@ -120,12 +127,15 @@ def convert(self, value): return int(value) -class FloatField(SearchField): +class FloatField(NumberField): field_type = 'float' + float_casts = NumberField.float_casts + ('null_value',) def __init__(self, type='float', *args, **kwargs): if type in ('float', 'double'): self.field_type = type + for attr in self.attrs: + setattr(self, attr, kwargs.pop(attr, None)) super(FloatField, self).__init__(*args, **kwargs) def prepare(self, value): diff --git a/elasticutils/tests/test_fields.py b/elasticutils/tests/test_fields.py index c4e6132..5464aa0 100644 --- a/elasticutils/tests/test_fields.py +++ b/elasticutils/tests/test_fields.py @@ -17,7 +17,7 @@ def test_convert(self): eq_(fields.StringField().convert(None), None) eq_(fields.StringField().convert('test'), 'test') - def test_index_kwarg(self): + def test_index(self): field = fields.StringField(index='not_analyzed') eq_(field.get_definition()['index'], 'not_analyzed') @@ -77,3 +77,105 @@ def test_position_offset_gap(self): eq_(field.get_definition()['position_offset_gap'], 2) field = fields.StringField(position_offset_gap='2') eq_(field.get_definition()['position_offset_gap'], 2) + + +class TestIntegerField(TestCase): + + def test_type(self): + eq_(fields.IntegerField().field_type, 'integer') + eq_(fields.IntegerField(type='byte').field_type, 'byte') + eq_(fields.IntegerField(type='short').field_type, 'short') + eq_(fields.IntegerField(type='long').field_type, 'long') + eq_(fields.IntegerField(type='foo').field_type, 'integer') + + def test_prepare(self): + eq_(fields.IntegerField().prepare(100), 100) + + def test_convert(self): + eq_(fields.IntegerField().convert(None), None) + eq_(fields.IntegerField().convert(100), 100) + + def test_index(self): + field = fields.IntegerField(index='no') + eq_(field.get_definition()['index'], 'no') + + def test_store(self): + field = fields.IntegerField(store='yes') + eq_(field.get_definition()['store'], 'yes') + + def test_precision_step(self): + field = fields.IntegerField(precision_step=4) + eq_(field.get_definition()['precision_step'], 4) + + def test_boost(self): + field = fields.IntegerField(boost=2.5) + eq_(field.get_definition()['boost'], 2.5) + field = fields.IntegerField(boost='2.5') + eq_(field.get_definition()['boost'], 2.5) + + def test_null_value(self): + field = fields.IntegerField(null_value=1) + eq_(field.get_definition()['null_value'], 1) + + def test_boolean_attributes(self): + for attr in ('ignore_malformed', 'include_in_all'): + # Test truthiness. + field = fields.IntegerField(**{attr: True}) + eq_(field.get_definition()[attr], True) + field = fields.IntegerField(**{attr: 'true'}) + eq_(field.get_definition()[attr], True) + # Test falsiness. + field = fields.IntegerField(**{attr: False}) + eq_(field.get_definition()[attr], False) + field = fields.IntegerField(**{attr: ''}) + eq_(field.get_definition()[attr], False) + + +class TestFloatField(TestCase): + + def test_type(self): + eq_(fields.FloatField().field_type, 'float') + eq_(fields.FloatField(type='double').field_type, 'double') + eq_(fields.FloatField(type='foo').field_type, 'float') + + def test_prepare(self): + eq_(fields.FloatField().prepare(100), 100.0) + + def test_convert(self): + eq_(fields.FloatField().convert(None), None) + eq_(fields.FloatField().convert(100), 100.0) + + def test_index(self): + field = fields.FloatField(index='no') + eq_(field.get_definition()['index'], 'no') + + def test_store(self): + field = fields.FloatField(store='yes') + eq_(field.get_definition()['store'], 'yes') + + def test_precision_step(self): + field = fields.FloatField(precision_step=4) + eq_(field.get_definition()['precision_step'], 4) + + def test_boost(self): + field = fields.FloatField(boost=2.5) + eq_(field.get_definition()['boost'], 2.5) + field = fields.FloatField(boost='2.5') + eq_(field.get_definition()['boost'], 2.5) + + def test_null_value(self): + field = fields.FloatField(null_value=1.0) + eq_(field.get_definition()['null_value'], 1.0) + + def test_boolean_attributes(self): + for attr in ('ignore_malformed', 'include_in_all'): + # Test truthiness. + field = fields.FloatField(**{attr: True}) + eq_(field.get_definition()[attr], True) + field = fields.FloatField(**{attr: 'true'}) + eq_(field.get_definition()[attr], True) + # Test falsiness. + field = fields.FloatField(**{attr: False}) + eq_(field.get_definition()[attr], False) + field = fields.FloatField(**{attr: ''}) + eq_(field.get_definition()[attr], False) From 171fe31e4ccd7659cd60b481ff4869ec2c010eb1 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 7 Dec 2013 19:09:53 -0800 Subject: [PATCH 13/18] Added DecimalField and BooleanField --- elasticutils/fields.py | 21 ++++++++--- elasticutils/tests/test_fields.py | 60 +++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 9c3b7fd..ffe443e 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -148,11 +148,18 @@ def convert(self, value): return float(value) -class DecimalField(SearchField): - field_type = 'string' +class DecimalField(StringField): + + def __init__(self, *args, **kwargs): + for attr in self.attrs: + setattr(self, attr, kwargs.pop(attr, None)) + super(DecimalField, self).__init__(*args, **kwargs) def prepare(self, value): - return self.convert(super(DecimalField, self).prepare(value)) + if value is None: + return None + + return str(value) def convert(self, value): if value is None: @@ -161,10 +168,14 @@ def convert(self, value): return Decimal(str(value)) -# TODO: Support all attributes for boolean types: -# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#boolean class BooleanField(SearchField): field_type = 'boolean' + bool_casts = SearchField.bool_casts + ('null_value',) + + def __init__(self, *args, **kwargs): + for attr in self.attrs: + setattr(self, attr, kwargs.pop(attr, None)) + super(BooleanField, self).__init__(*args, **kwargs) def prepare(self, value): return self.convert(super(BooleanField, self).prepare(value)) diff --git a/elasticutils/tests/test_fields.py b/elasticutils/tests/test_fields.py index 5464aa0..eff9f58 100644 --- a/elasticutils/tests/test_fields.py +++ b/elasticutils/tests/test_fields.py @@ -1,3 +1,4 @@ +from decimal import Decimal from unittest import TestCase from nose.tools import eq_ @@ -179,3 +180,62 @@ def test_boolean_attributes(self): eq_(field.get_definition()[attr], False) field = fields.FloatField(**{attr: ''}) eq_(field.get_definition()[attr], False) + + +class TestDecimalField(TestCase): + """DecimalField subclasses StringField, so we just test a few things.""" + + def test_type(self): + eq_(fields.DecimalField().field_type, 'string') + + def test_prepare(self): + eq_(fields.DecimalField().prepare(Decimal('100.0')), '100.0') + + def test_convert(self): + eq_(fields.DecimalField().convert(None), None) + eq_(fields.DecimalField().convert('100.0'), Decimal('100.0')) + + +class TestBooleanField(TestCase): + + def test_type(self): + eq_(fields.BooleanField().field_type, 'boolean') + + def test_prepare(self): + eq_(fields.BooleanField().prepare(True), True) + eq_(fields.BooleanField().prepare(False), False) + + def test_convert(self): + eq_(fields.BooleanField().convert(None), None) + eq_(fields.BooleanField().convert(True), True) + eq_(fields.BooleanField().convert(False), False) + + def test_index(self): + field = fields.BooleanField(index='no') + eq_(field.get_definition()['index'], 'no') + + def test_store(self): + field = fields.BooleanField(store='yes') + eq_(field.get_definition()['store'], 'yes') + + def test_boost(self): + field = fields.BooleanField(boost=2.5) + eq_(field.get_definition()['boost'], 2.5) + field = fields.BooleanField(boost='2.5') + eq_(field.get_definition()['boost'], 2.5) + + def test_null_value(self): + field = fields.BooleanField(null_value=True) + eq_(field.get_definition()['null_value'], True) + + def test_boolean_attributes(self): + # Test truthiness. + field = fields.BooleanField(include_in_all=True) + eq_(field.get_definition()['include_in_all'], True) + field = fields.BooleanField(include_in_all='true') + eq_(field.get_definition()['include_in_all'], True) + # Test falsiness. + field = fields.BooleanField(include_in_all=False) + eq_(field.get_definition()['include_in_all'], False) + field = fields.BooleanField(include_in_all='') + eq_(field.get_definition()['include_in_all'], False) From 0706d6e5fd2bae8315d42605659958b0fefdae99 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 7 Dec 2013 20:19:46 -0800 Subject: [PATCH 14/18] Added DateField and DateTimeField --- elasticutils/fields.py | 35 +++++++--- elasticutils/tests/test_fields.py | 102 ++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 10 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index ffe443e..9d475c1 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -100,15 +100,15 @@ def convert(self, value): return unicode(value) -class NumberField(SearchField): +class NumberFieldBase(SearchField): attrs = SearchField.attrs + ('ignore_malformed', 'precision_step') bool_casts = SearchField.bool_casts + ('ignore_malformed',) int_casts = SearchField.int_casts + ('precision_step',) -class IntegerField(NumberField): +class IntegerField(NumberFieldBase): field_type = 'integer' - int_casts = NumberField.int_casts + ('null_value',) + int_casts = NumberFieldBase.int_casts + ('null_value',) def __init__(self, type='integer', *args, **kwargs): if type in ('byte', 'short', 'integer', 'long'): @@ -127,9 +127,9 @@ def convert(self, value): return int(value) -class FloatField(NumberField): +class FloatField(NumberFieldBase): field_type = 'float' - float_casts = NumberField.float_casts + ('null_value',) + float_casts = NumberFieldBase.float_casts + ('null_value',) def __init__(self, type='float', *args, **kwargs): if type in ('float', 'double'): @@ -187,11 +187,27 @@ def convert(self, value): return bool(value) -# TODO: Support all attributes for date types: -# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#date -class DateField(SearchField): +class DateFieldBase(SearchField): + attrs = SearchField.attrs + ('format', 'ignore_malformed', + 'precision_step') + bool_casts = SearchField.bool_casts + ('ignore_malformed',) + int_casts = SearchField.int_casts + ('precision_step',) + + +class DateField(DateFieldBase): field_type = 'date' + def __init__(self, *args, **kwargs): + for attr in self.attrs: + setattr(self, attr, kwargs.pop(attr, None)) + super(DateField, self).__init__(*args, **kwargs) + + def prepare(self, value): + if isinstance(value, (datetime.date, datetime.datetime)): + return value.isoformat() + + return value + def convert(self, value): if value is None: return None @@ -211,8 +227,7 @@ def convert(self, value): return value -class DateTimeField(SearchField): - field_type = 'datetime' +class DateTimeField(DateField): def convert(self, value): if value is None: diff --git a/elasticutils/tests/test_fields.py b/elasticutils/tests/test_fields.py index eff9f58..ae3a72e 100644 --- a/elasticutils/tests/test_fields.py +++ b/elasticutils/tests/test_fields.py @@ -1,3 +1,4 @@ +import datetime from decimal import Decimal from unittest import TestCase @@ -239,3 +240,104 @@ def test_boolean_attributes(self): eq_(field.get_definition()['include_in_all'], False) field = fields.BooleanField(include_in_all='') eq_(field.get_definition()['include_in_all'], False) + + +class TestDateField(TestCase): + + def test_type(self): + eq_(fields.DateField().field_type, 'date') + + def test_prepare(self): + eq_(fields.DateField().prepare(datetime.date(2013, 11, 22)), + '2013-11-22') + + def test_convert(self): + eq_(fields.DateField().convert('2013-11-22'), + datetime.date(2013, 11, 22)) + eq_(fields.DateField().convert('2013-11-22T12:34:56'), + datetime.date(2013, 11, 22)) + + def test_index(self): + field = fields.DateField(index='no') + eq_(field.get_definition()['index'], 'no') + + def test_store(self): + field = fields.DateField(store='yes') + eq_(field.get_definition()['store'], 'yes') + + def test_boost(self): + field = fields.DateField(boost=2.5) + eq_(field.get_definition()['boost'], 2.5) + field = fields.DateField(boost='2.5') + eq_(field.get_definition()['boost'], 2.5) + + def test_null_value(self): + field = fields.DateField(null_value='2013-11-22') + eq_(field.get_definition()['null_value'], '2013-11-22') + + def test_precision_step(self): + field = fields.IntegerField(precision_step=4) + eq_(field.get_definition()['precision_step'], 4) + + def test_boolean_attributes(self): + for attr in ('ignore_malformed', 'include_in_all'): + # Test truthiness. + field = fields.FloatField(**{attr: True}) + eq_(field.get_definition()[attr], True) + field = fields.FloatField(**{attr: 'true'}) + eq_(field.get_definition()[attr], True) + # Test falsiness. + field = fields.FloatField(**{attr: False}) + eq_(field.get_definition()[attr], False) + field = fields.FloatField(**{attr: ''}) + eq_(field.get_definition()[attr], False) + + +class TestDateTimeField(TestCase): + + def test_type(self): + eq_(fields.DateTimeField().field_type, 'date') + + def test_prepare(self): + eq_(fields.DateTimeField().prepare( + datetime.datetime(2013, 11, 22, 12, 34, 56)), + '2013-11-22T12:34:56') + + def test_convert(self): + eq_(fields.DateTimeField().convert('2013-11-22T12:34:56'), + datetime.datetime(2013, 11, 22, 12, 34, 56)) + + def test_index(self): + field = fields.DateTimeField(index='no') + eq_(field.get_definition()['index'], 'no') + + def test_store(self): + field = fields.DateTimeField(store='yes') + eq_(field.get_definition()['store'], 'yes') + + def test_boost(self): + field = fields.DateTimeField(boost=2.5) + eq_(field.get_definition()['boost'], 2.5) + field = fields.DateTimeField(boost='2.5') + eq_(field.get_definition()['boost'], 2.5) + + def test_null_value(self): + field = fields.DateTimeField(null_value='2013-11-22') + eq_(field.get_definition()['null_value'], '2013-11-22') + + def test_precision_step(self): + field = fields.IntegerField(precision_step=4) + eq_(field.get_definition()['precision_step'], 4) + + def test_boolean_attributes(self): + for attr in ('ignore_malformed', 'include_in_all'): + # Test truthiness. + field = fields.FloatField(**{attr: True}) + eq_(field.get_definition()[attr], True) + field = fields.FloatField(**{attr: 'true'}) + eq_(field.get_definition()[attr], True) + # Test falsiness. + field = fields.FloatField(**{attr: False}) + eq_(field.get_definition()[attr], False) + field = fields.FloatField(**{attr: ''}) + eq_(field.get_definition()[attr], False) From a863ecb5df4830cff91c37b67a1d2c3d731c916d Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 7 Dec 2013 20:38:30 -0800 Subject: [PATCH 15/18] Added BinaryField --- elasticutils/fields.py | 24 ++++-------------------- elasticutils/tests/test_fields.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 9d475c1..ae81dc1 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -253,6 +253,10 @@ def convert(self, value): class BinaryField(SearchField): field_type = 'binary' + attrs = () + bool_casts = () + float_casts = () + int_casts = () def prepare(self, value): if value is None: @@ -265,23 +269,3 @@ def convert(self, value): return None return base64.b64decode(value) - - -# TODO: Not sure we need this actually. Multivalued field types can be their -# base class (e.g. IntegerField(multivalue=True) so Python knows how to return -# data. -class MultiValueField(SearchField): - field_type = 'string' - - def __init__(self, **kwargs): - super(MultiValueField, self).__init__(**kwargs) - self.is_multivalued = True - - def prepare(self, value): - return self.convert(super(MultiValueField, self).prepare(value)) - - def convert(self, value): - if value is None: - return None - - return list(value) diff --git a/elasticutils/tests/test_fields.py b/elasticutils/tests/test_fields.py index ae3a72e..695ece3 100644 --- a/elasticutils/tests/test_fields.py +++ b/elasticutils/tests/test_fields.py @@ -1,3 +1,4 @@ +import base64 import datetime from decimal import Decimal from unittest import TestCase @@ -341,3 +342,17 @@ def test_boolean_attributes(self): eq_(field.get_definition()[attr], False) field = fields.FloatField(**{attr: ''}) eq_(field.get_definition()[attr], False) + + +class TestBinaryField(TestCase): + + def test_type(self): + eq_(fields.BinaryField().field_type, 'binary') + + def test_prepare(self): + eq_(fields.BinaryField().prepare(None), None) + eq_(fields.BinaryField().prepare('test'), base64.b64encode('test')) + + def test_convert(self): + eq_(fields.BinaryField().convert(None), None) + eq_(fields.BinaryField().convert(base64.b64encode('test')), 'test') From 614cc094b11c7a01f437b814f95288cfa4ddbb28 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 1 Mar 2014 14:41:41 -0800 Subject: [PATCH 16/18] Removed casting --- elasticutils/fields.py | 27 +---------------- elasticutils/tests/test_fields.py | 50 ------------------------------- 2 files changed, 1 insertion(+), 76 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index ae81dc1..e85b6eb 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -17,9 +17,6 @@ class SearchField(object): field_type = None attrs = ('boost', 'include_in_all', 'index', 'null_value', 'store') - bool_casts = ('include_in_all',) - float_casts = ('boost',) - int_casts = () # Used to maintain the order of fields as defined in the class. _creation_order = 0 @@ -60,17 +57,7 @@ def get_definition(self): for attr in self.attrs: val = getattr(self, attr, None) if val is not None: - if attr in self.bool_casts: - if not val or val == 'false': - f[attr] = False - else: - f[attr] = True - elif attr in self.float_casts: - f[attr] = float(val) - elif attr in self.int_casts: - f[attr] = int(val) - else: - f[attr] = str(val) + f[attr] = val return f @@ -81,8 +68,6 @@ class StringField(SearchField): 'analyzer', 'ignore_above', 'index_analyzer', 'index_options', 'omit_norms', 'position_offset_gap', 'search_analyzer', 'term_vector') - bool_casts = SearchField.bool_casts + ('omit_norms',) - int_casts = SearchField.int_casts + ('position_offset_gap',) def __init__(self, *args, **kwargs): for attr in self.attrs: @@ -102,13 +87,10 @@ def convert(self, value): class NumberFieldBase(SearchField): attrs = SearchField.attrs + ('ignore_malformed', 'precision_step') - bool_casts = SearchField.bool_casts + ('ignore_malformed',) - int_casts = SearchField.int_casts + ('precision_step',) class IntegerField(NumberFieldBase): field_type = 'integer' - int_casts = NumberFieldBase.int_casts + ('null_value',) def __init__(self, type='integer', *args, **kwargs): if type in ('byte', 'short', 'integer', 'long'): @@ -129,7 +111,6 @@ def convert(self, value): class FloatField(NumberFieldBase): field_type = 'float' - float_casts = NumberFieldBase.float_casts + ('null_value',) def __init__(self, type='float', *args, **kwargs): if type in ('float', 'double'): @@ -170,7 +151,6 @@ def convert(self, value): class BooleanField(SearchField): field_type = 'boolean' - bool_casts = SearchField.bool_casts + ('null_value',) def __init__(self, *args, **kwargs): for attr in self.attrs: @@ -190,8 +170,6 @@ def convert(self, value): class DateFieldBase(SearchField): attrs = SearchField.attrs + ('format', 'ignore_malformed', 'precision_step') - bool_casts = SearchField.bool_casts + ('ignore_malformed',) - int_casts = SearchField.int_casts + ('precision_step',) class DateField(DateFieldBase): @@ -254,9 +232,6 @@ def convert(self, value): class BinaryField(SearchField): field_type = 'binary' attrs = () - bool_casts = () - float_casts = () - int_casts = () def prepare(self, value): if value is None: diff --git a/elasticutils/tests/test_fields.py b/elasticutils/tests/test_fields.py index 695ece3..b20d03a 100644 --- a/elasticutils/tests/test_fields.py +++ b/elasticutils/tests/test_fields.py @@ -35,8 +35,6 @@ def test_term_vector(self): def test_boost(self): field = fields.StringField(boost=2.5) eq_(field.get_definition()['boost'], 2.5) - field = fields.StringField(boost='2.5') - eq_(field.get_definition()['boost'], 2.5) def test_null_value(self): field = fields.StringField(null_value='na') @@ -44,16 +42,10 @@ def test_null_value(self): def test_boolean_attributes(self): for attr in ('omit_norms', 'include_in_all'): - # Test truthiness. field = fields.StringField(**{attr: True}) eq_(field.get_definition()[attr], True) - field = fields.StringField(**{attr: 'true'}) - eq_(field.get_definition()[attr], True) - # Test falsiness. field = fields.StringField(**{attr: False}) eq_(field.get_definition()[attr], False) - field = fields.StringField(**{attr: ''}) - eq_(field.get_definition()[attr], False) def test_index_options(self): field = fields.StringField(index_options='positions') @@ -78,8 +70,6 @@ def test_ignore_above(self): def test_position_offset_gap(self): field = fields.StringField(position_offset_gap=2) eq_(field.get_definition()['position_offset_gap'], 2) - field = fields.StringField(position_offset_gap='2') - eq_(field.get_definition()['position_offset_gap'], 2) class TestIntegerField(TestCase): @@ -113,8 +103,6 @@ def test_precision_step(self): def test_boost(self): field = fields.IntegerField(boost=2.5) eq_(field.get_definition()['boost'], 2.5) - field = fields.IntegerField(boost='2.5') - eq_(field.get_definition()['boost'], 2.5) def test_null_value(self): field = fields.IntegerField(null_value=1) @@ -122,16 +110,10 @@ def test_null_value(self): def test_boolean_attributes(self): for attr in ('ignore_malformed', 'include_in_all'): - # Test truthiness. field = fields.IntegerField(**{attr: True}) eq_(field.get_definition()[attr], True) - field = fields.IntegerField(**{attr: 'true'}) - eq_(field.get_definition()[attr], True) - # Test falsiness. field = fields.IntegerField(**{attr: False}) eq_(field.get_definition()[attr], False) - field = fields.IntegerField(**{attr: ''}) - eq_(field.get_definition()[attr], False) class TestFloatField(TestCase): @@ -163,8 +145,6 @@ def test_precision_step(self): def test_boost(self): field = fields.FloatField(boost=2.5) eq_(field.get_definition()['boost'], 2.5) - field = fields.FloatField(boost='2.5') - eq_(field.get_definition()['boost'], 2.5) def test_null_value(self): field = fields.FloatField(null_value=1.0) @@ -172,16 +152,10 @@ def test_null_value(self): def test_boolean_attributes(self): for attr in ('ignore_malformed', 'include_in_all'): - # Test truthiness. field = fields.FloatField(**{attr: True}) eq_(field.get_definition()[attr], True) - field = fields.FloatField(**{attr: 'true'}) - eq_(field.get_definition()[attr], True) - # Test falsiness. field = fields.FloatField(**{attr: False}) eq_(field.get_definition()[attr], False) - field = fields.FloatField(**{attr: ''}) - eq_(field.get_definition()[attr], False) class TestDecimalField(TestCase): @@ -223,24 +197,16 @@ def test_store(self): def test_boost(self): field = fields.BooleanField(boost=2.5) eq_(field.get_definition()['boost'], 2.5) - field = fields.BooleanField(boost='2.5') - eq_(field.get_definition()['boost'], 2.5) def test_null_value(self): field = fields.BooleanField(null_value=True) eq_(field.get_definition()['null_value'], True) def test_boolean_attributes(self): - # Test truthiness. field = fields.BooleanField(include_in_all=True) eq_(field.get_definition()['include_in_all'], True) - field = fields.BooleanField(include_in_all='true') - eq_(field.get_definition()['include_in_all'], True) - # Test falsiness. field = fields.BooleanField(include_in_all=False) eq_(field.get_definition()['include_in_all'], False) - field = fields.BooleanField(include_in_all='') - eq_(field.get_definition()['include_in_all'], False) class TestDateField(TestCase): @@ -269,8 +235,6 @@ def test_store(self): def test_boost(self): field = fields.DateField(boost=2.5) eq_(field.get_definition()['boost'], 2.5) - field = fields.DateField(boost='2.5') - eq_(field.get_definition()['boost'], 2.5) def test_null_value(self): field = fields.DateField(null_value='2013-11-22') @@ -282,16 +246,10 @@ def test_precision_step(self): def test_boolean_attributes(self): for attr in ('ignore_malformed', 'include_in_all'): - # Test truthiness. field = fields.FloatField(**{attr: True}) eq_(field.get_definition()[attr], True) - field = fields.FloatField(**{attr: 'true'}) - eq_(field.get_definition()[attr], True) - # Test falsiness. field = fields.FloatField(**{attr: False}) eq_(field.get_definition()[attr], False) - field = fields.FloatField(**{attr: ''}) - eq_(field.get_definition()[attr], False) class TestDateTimeField(TestCase): @@ -319,8 +277,6 @@ def test_store(self): def test_boost(self): field = fields.DateTimeField(boost=2.5) eq_(field.get_definition()['boost'], 2.5) - field = fields.DateTimeField(boost='2.5') - eq_(field.get_definition()['boost'], 2.5) def test_null_value(self): field = fields.DateTimeField(null_value='2013-11-22') @@ -332,16 +288,10 @@ def test_precision_step(self): def test_boolean_attributes(self): for attr in ('ignore_malformed', 'include_in_all'): - # Test truthiness. field = fields.FloatField(**{attr: True}) eq_(field.get_definition()[attr], True) - field = fields.FloatField(**{attr: 'true'}) - eq_(field.get_definition()[attr], True) - # Test falsiness. field = fields.FloatField(**{attr: False}) eq_(field.get_definition()[attr], False) - field = fields.FloatField(**{attr: ''}) - eq_(field.get_definition()[attr], False) class TestBinaryField(TestCase): From 87257e6de5f4442c1d921fab543f8b5f97ddc119 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 1 Mar 2014 14:56:07 -0800 Subject: [PATCH 17/18] Moved to open-ended kwargs for fields for future proofing --- elasticutils/fields.py | 58 +++++++++--------------------------------- 1 file changed, 12 insertions(+), 46 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index e85b6eb..1269a57 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -15,15 +15,20 @@ class SearchField(object): field_type = None - - attrs = ('boost', 'include_in_all', 'index', 'null_value', 'store') + attrs = [] # Used to maintain the order of fields as defined in the class. _creation_order = 0 def __init__(self, *args, **kwargs): - self.index_fieldname = kwargs.pop('index_fieldname', None) - self.is_multivalued = kwargs.pop('is_multivalued', None) + # These are special. + for attr in ('index_fieldname', 'is_multivalue'): + setattr(self, attr, kwargs.pop(attr, None)) + + # Set all kwargs on self for later access. + for attr in kwargs.keys(): + self.attrs.append(attr) + setattr(self, attr, kwargs.pop(attr, None)) # Store this fields order. self._creation_order = SearchField._creation_order @@ -64,16 +69,6 @@ def get_definition(self): class StringField(SearchField): field_type = 'string' - attrs = SearchField.attrs + ( - 'analyzer', 'ignore_above', - 'index_analyzer', 'index_options', 'omit_norms', - 'position_offset_gap', 'search_analyzer', 'term_vector') - - def __init__(self, *args, **kwargs): - for attr in self.attrs: - setattr(self, attr, kwargs.pop(attr, None)) - - super(StringField, self).__init__(*args, **kwargs) def prepare(self, value): return self.convert(super(StringField, self).prepare(value)) @@ -85,18 +80,12 @@ def convert(self, value): return unicode(value) -class NumberFieldBase(SearchField): - attrs = SearchField.attrs + ('ignore_malformed', 'precision_step') - - -class IntegerField(NumberFieldBase): +class IntegerField(SearchField): field_type = 'integer' def __init__(self, type='integer', *args, **kwargs): if type in ('byte', 'short', 'integer', 'long'): self.field_type = type - for attr in self.attrs: - setattr(self, attr, kwargs.pop(attr, None)) super(IntegerField, self).__init__(*args, **kwargs) def prepare(self, value): @@ -109,14 +98,12 @@ def convert(self, value): return int(value) -class FloatField(NumberFieldBase): +class FloatField(SearchField): field_type = 'float' def __init__(self, type='float', *args, **kwargs): if type in ('float', 'double'): self.field_type = type - for attr in self.attrs: - setattr(self, attr, kwargs.pop(attr, None)) super(FloatField, self).__init__(*args, **kwargs) def prepare(self, value): @@ -131,11 +118,6 @@ def convert(self, value): class DecimalField(StringField): - def __init__(self, *args, **kwargs): - for attr in self.attrs: - setattr(self, attr, kwargs.pop(attr, None)) - super(DecimalField, self).__init__(*args, **kwargs) - def prepare(self, value): if value is None: return None @@ -152,11 +134,6 @@ def convert(self, value): class BooleanField(SearchField): field_type = 'boolean' - def __init__(self, *args, **kwargs): - for attr in self.attrs: - setattr(self, attr, kwargs.pop(attr, None)) - super(BooleanField, self).__init__(*args, **kwargs) - def prepare(self, value): return self.convert(super(BooleanField, self).prepare(value)) @@ -167,19 +144,9 @@ def convert(self, value): return bool(value) -class DateFieldBase(SearchField): - attrs = SearchField.attrs + ('format', 'ignore_malformed', - 'precision_step') - - -class DateField(DateFieldBase): +class DateField(SearchField): field_type = 'date' - def __init__(self, *args, **kwargs): - for attr in self.attrs: - setattr(self, attr, kwargs.pop(attr, None)) - super(DateField, self).__init__(*args, **kwargs) - def prepare(self, value): if isinstance(value, (datetime.date, datetime.datetime)): return value.isoformat() @@ -231,7 +198,6 @@ def convert(self, value): class BinaryField(SearchField): field_type = 'binary' - attrs = () def prepare(self, value): if value is None: From 011c5380e8dc830eb5115fb0902d21e118d78dd7 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Sat, 1 Mar 2014 15:11:12 -0800 Subject: [PATCH 18/18] Renamed conversion methods to `to_python` and `to_es` --- elasticutils/fields.py | 62 +++++++++++--------- elasticutils/tests/test_fields.py | 96 +++++++++++++++++-------------- 2 files changed, 90 insertions(+), 68 deletions(-) diff --git a/elasticutils/fields.py b/elasticutils/fields.py index 1269a57..30ea205 100644 --- a/elasticutils/fields.py +++ b/elasticutils/fields.py @@ -35,19 +35,17 @@ def __init__(self, *args, **kwargs): # Increment order number for future fields. SearchField._creation_order += 1 - def prepare(self, value): + def to_es(self, value): """ - Handles conversion between the value sent to Elasticsearch and the type - of the field. + Converts a Python value to an Elasticsearch value. Extending classes should override this method. """ return value - def convert(self, value): + def to_python(self, value): """ - Handles conversion between the data received from Elasticsearch and the - type of the field. + Converts an Elasticsearch value to a Python value. Extending classes should override this method. """ @@ -70,10 +68,13 @@ def get_definition(self): class StringField(SearchField): field_type = 'string' - def prepare(self, value): - return self.convert(super(StringField, self).prepare(value)) + def to_es(self, value): + if value is None: + return None + + return unicode(value) - def convert(self, value): + def to_python(self, value): if value is None: return None @@ -88,10 +89,13 @@ def __init__(self, type='integer', *args, **kwargs): self.field_type = type super(IntegerField, self).__init__(*args, **kwargs) - def prepare(self, value): - return self.convert(super(IntegerField, self).prepare(value)) + def to_es(self, value): + if value is None: + return None + + return int(value) - def convert(self, value): + def to_python(self, value): if value is None: return None @@ -106,10 +110,13 @@ def __init__(self, type='float', *args, **kwargs): self.field_type = type super(FloatField, self).__init__(*args, **kwargs) - def prepare(self, value): - return self.convert(super(FloatField, self).prepare(value)) + def to_es(self, value): + if value is None: + return None + + return float(value) - def convert(self, value): + def to_python(self, value): if value is None: return None @@ -118,13 +125,13 @@ def convert(self, value): class DecimalField(StringField): - def prepare(self, value): + def to_es(self, value): if value is None: return None - return str(value) + return str(float(value)) - def convert(self, value): + def to_python(self, value): if value is None: return None @@ -134,10 +141,13 @@ def convert(self, value): class BooleanField(SearchField): field_type = 'boolean' - def prepare(self, value): - return self.convert(super(BooleanField, self).prepare(value)) + def to_es(self, value): + if value is None: + return None + + return bool(value) - def convert(self, value): + def to_python(self, value): if value is None: return None @@ -147,13 +157,13 @@ def convert(self, value): class DateField(SearchField): field_type = 'date' - def prepare(self, value): + def to_es(self, value): if isinstance(value, (datetime.date, datetime.datetime)): return value.isoformat() return value - def convert(self, value): + def to_python(self, value): if value is None: return None @@ -174,7 +184,7 @@ def convert(self, value): class DateTimeField(DateField): - def convert(self, value): + def to_python(self, value): if value is None: return None @@ -199,13 +209,13 @@ def convert(self, value): class BinaryField(SearchField): field_type = 'binary' - def prepare(self, value): + def to_es(self, value): if value is None: return None return base64.b64encode(value) - def convert(self, value): + def to_python(self, value): if value is None: return None diff --git a/elasticutils/tests/test_fields.py b/elasticutils/tests/test_fields.py index b20d03a..5553795 100644 --- a/elasticutils/tests/test_fields.py +++ b/elasticutils/tests/test_fields.py @@ -13,12 +13,13 @@ class TestStringField(TestCase): def test_type(self): eq_(fields.StringField().field_type, 'string') - def test_prepare(self): - eq_(fields.StringField().prepare('test'), 'test') + def test_to_es(self): + eq_(fields.StringField().to_es(None), None) + eq_(fields.StringField().to_es('test'), 'test') - def test_convert(self): - eq_(fields.StringField().convert(None), None) - eq_(fields.StringField().convert('test'), 'test') + def test_to_python(self): + eq_(fields.StringField().to_python(None), None) + eq_(fields.StringField().to_python('test'), 'test') def test_index(self): field = fields.StringField(index='not_analyzed') @@ -81,12 +82,15 @@ def test_type(self): eq_(fields.IntegerField(type='long').field_type, 'long') eq_(fields.IntegerField(type='foo').field_type, 'integer') - def test_prepare(self): - eq_(fields.IntegerField().prepare(100), 100) + def test_to_es(self): + eq_(fields.IntegerField().to_python(None), None) + eq_(fields.IntegerField().to_es(100), 100) + eq_(fields.IntegerField().to_es('100'), 100) - def test_convert(self): - eq_(fields.IntegerField().convert(None), None) - eq_(fields.IntegerField().convert(100), 100) + def test_to_python(self): + eq_(fields.IntegerField().to_python(None), None) + eq_(fields.IntegerField().to_python(100), 100) + eq_(fields.IntegerField().to_es('100'), 100) def test_index(self): field = fields.IntegerField(index='no') @@ -123,12 +127,15 @@ def test_type(self): eq_(fields.FloatField(type='double').field_type, 'double') eq_(fields.FloatField(type='foo').field_type, 'float') - def test_prepare(self): - eq_(fields.FloatField().prepare(100), 100.0) + def test_to_es(self): + eq_(fields.FloatField().to_python(None), None) + eq_(fields.FloatField().to_es(100), 100.0) + eq_(fields.FloatField().to_es('100'), 100.0) - def test_convert(self): - eq_(fields.FloatField().convert(None), None) - eq_(fields.FloatField().convert(100), 100.0) + def test_to_python(self): + eq_(fields.FloatField().to_python(None), None) + eq_(fields.FloatField().to_python(100), 100.0) + eq_(fields.FloatField().to_es('100'), 100.0) def test_index(self): field = fields.FloatField(index='no') @@ -164,12 +171,14 @@ class TestDecimalField(TestCase): def test_type(self): eq_(fields.DecimalField().field_type, 'string') - def test_prepare(self): - eq_(fields.DecimalField().prepare(Decimal('100.0')), '100.0') + def test_to_es(self): + eq_(fields.DecimalField().to_python(None), None) + eq_(fields.DecimalField().to_es(Decimal('100.0')), '100.0') + eq_(fields.DecimalField().to_es(Decimal('100')), '100.0') - def test_convert(self): - eq_(fields.DecimalField().convert(None), None) - eq_(fields.DecimalField().convert('100.0'), Decimal('100.0')) + def test_to_python(self): + eq_(fields.DecimalField().to_python(None), None) + eq_(fields.DecimalField().to_python('100.0'), Decimal('100.0')) class TestBooleanField(TestCase): @@ -177,14 +186,15 @@ class TestBooleanField(TestCase): def test_type(self): eq_(fields.BooleanField().field_type, 'boolean') - def test_prepare(self): - eq_(fields.BooleanField().prepare(True), True) - eq_(fields.BooleanField().prepare(False), False) + def test_to_es(self): + eq_(fields.BooleanField().to_python(None), None) + eq_(fields.BooleanField().to_es(True), True) + eq_(fields.BooleanField().to_es(False), False) - def test_convert(self): - eq_(fields.BooleanField().convert(None), None) - eq_(fields.BooleanField().convert(True), True) - eq_(fields.BooleanField().convert(False), False) + def test_to_python(self): + eq_(fields.BooleanField().to_python(None), None) + eq_(fields.BooleanField().to_python(True), True) + eq_(fields.BooleanField().to_python(False), False) def test_index(self): field = fields.BooleanField(index='no') @@ -214,14 +224,15 @@ class TestDateField(TestCase): def test_type(self): eq_(fields.DateField().field_type, 'date') - def test_prepare(self): - eq_(fields.DateField().prepare(datetime.date(2013, 11, 22)), + def test_to_es(self): + eq_(fields.DateField().to_es(datetime.date(2013, 11, 22)), '2013-11-22') - def test_convert(self): - eq_(fields.DateField().convert('2013-11-22'), + def test_to_python(self): + eq_(fields.DateField().to_python(None), None) + eq_(fields.DateField().to_python('2013-11-22'), datetime.date(2013, 11, 22)) - eq_(fields.DateField().convert('2013-11-22T12:34:56'), + eq_(fields.DateField().to_python('2013-11-22T12:34:56'), datetime.date(2013, 11, 22)) def test_index(self): @@ -257,13 +268,14 @@ class TestDateTimeField(TestCase): def test_type(self): eq_(fields.DateTimeField().field_type, 'date') - def test_prepare(self): - eq_(fields.DateTimeField().prepare( + def test_to_es(self): + eq_(fields.DateTimeField().to_es( datetime.datetime(2013, 11, 22, 12, 34, 56)), '2013-11-22T12:34:56') - def test_convert(self): - eq_(fields.DateTimeField().convert('2013-11-22T12:34:56'), + def test_to_python(self): + eq_(fields.DateTimeField().to_python(None), None) + eq_(fields.DateTimeField().to_python('2013-11-22T12:34:56'), datetime.datetime(2013, 11, 22, 12, 34, 56)) def test_index(self): @@ -299,10 +311,10 @@ class TestBinaryField(TestCase): def test_type(self): eq_(fields.BinaryField().field_type, 'binary') - def test_prepare(self): - eq_(fields.BinaryField().prepare(None), None) - eq_(fields.BinaryField().prepare('test'), base64.b64encode('test')) + def test_to_es(self): + eq_(fields.BinaryField().to_es(None), None) + eq_(fields.BinaryField().to_es('test'), base64.b64encode('test')) - def test_convert(self): - eq_(fields.BinaryField().convert(None), None) - eq_(fields.BinaryField().convert(base64.b64encode('test')), 'test') + def test_to_python(self): + eq_(fields.BinaryField().to_python(None), None) + eq_(fields.BinaryField().to_python(base64.b64encode('test')), 'test')