diff --git a/.gitignore b/.gitignore index 825a5ee..62e3d88 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ MANIFEST build dist docs/_build +*.egg-info/ +env/ diff --git a/Changelog b/Changelog index f76b5d9..ac4150d 100644 --- a/Changelog +++ b/Changelog @@ -1,30 +1,72 @@ -* 0.7 : unreleased - - Provide default string-based field for schema field types unknown to - Sunburnt. (@davidjb) - - Escape forward slash characters for compatibility with Solr 4.0 (@davidjb) - - Fix handling of queries with ``boost_relevancy`` applied - boost was - previously lost in some cases. (@davidjb) - - Ensure 'more like this' results are transformed using a query's - execute() `constructor`, as are normal query results. (@davidjb) +0.8.1lu (unreleased) +-------------------- -* 0.6 : 2012-01-01 - - Change license to MIT/X11 - - Integrate with Django Paginator (thanks @rlskoeser) - - Add UUIDField support - - Add MLT Handler support (thanks @ogrisel) +- Nothing changed yet. -* 0.5 : 2011-05-18 - - Add proper documentation - - Add field_limit() option (thanks, @danaspiegel) - - Add basic support for new Solr-3.1 datatypes - - Add support for __any queries (thanks, @skarab) - - Add experimental JSON output module +0.8lu (2013-12-17) +------------------ - - Rationalize data normalization and quoting codepaths, fixing several bugs along the way - - Fixes for correct serialization of complex nested query objects +- Added boundaryScanner options to HighlightOptions. + [jod] +- Fixed tests. + [jod] -* 0.4 : 2011-01-04 +- Use setuptools in setup.py. + [jod] - - First release on PyPi +- Added https://wiki.apache.org/solr/PostingsHighlighter + [jod] + +0.7.1lu (2013-11-04) +-------------------- + +- Add support for hierarchical facet queryies, i.e. facet.pivot. (@florianpilz) + +0.7lu (2013-07-02) +------------------ + +- Provide default string-based field for schema field types unknown to + +- Escape forward slash characters for compatibility with Solr 4.0 (@davidjb) + Sunburnt. (@davidjb) + +- Fix handling of queries with ``boost_relevancy`` applied - boost was + previously lost in some cases. (@davidjb) + +- Ensure 'more like this' results are transformed using a query's + execute() `constructor`, as are normal query results. (@davidjb) + +0.6 (2012-01-01) +---------------- + +- Change license to MIT/X11 + +- Integrate with Django Paginator (thanks @rlskoeser) + +- Add UUIDField support + +- Add MLT Handler support (thanks @ogrisel) + +0.5 (2011-05-18) +---------------- + +- Add proper documentation + +- Add field_limit() option (thanks, @danaspiegel) + +- Add basic support for new Solr-3.1 datatypes + +- Add support for __any queries (thanks, @skarab) + +- Add experimental JSON output module + +- Rationalize data normalization and quoting codepaths, fixing several bugs along the way + +- Fixes for correct serialization of complex nested query objects + +0.4 (2011-01-04) +---------------- + +- First release on PyPi diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b4688ce --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[nosetests] +match=^test +nocapture=1 +cover-package=sunburnt +with-coverage=1 +cover-erase=1 diff --git a/setup.py b/setup.py index 63bc561..a1b6069 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,17 @@ #!/usr/bin/env python -import distutils.core, os, re +from setuptools import setup -version_number_re = "\s*__version__\s*=\s*((\"([^\"]|\\\\\")*\"|'([^']|\\\\')*'))" -version_file = os.path.join(os.path.dirname(__file__), 'sunburnt', '__init__.py') -version_number = re.search(version_number_re, open(version_file).read()).groups()[0][1:-1] +version = '0.8.1lu.dev0' -distutils.core.setup( +setup( name='sunburnt', - version=version_number, + version=version, description='Python interface to Solr', author='Toby White', author_email='toby@timetric.com', packages=['sunburnt'], - requires=['lxml', 'pytz'], + install_requires=['httplib2', 'lxml', 'pytz', 'setuptools'], classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', diff --git a/sunburnt/__init__.py b/sunburnt/__init__.py index f9c00de..b4bcef5 100644 --- a/sunburnt/__init__.py +++ b/sunburnt/__init__.py @@ -3,6 +3,4 @@ from .strings import RawString from .sunburnt import SolrError, SolrInterface -__version__ = '0.7' - __all__ = ['RawString', 'SolrError', 'SolrInterface'] diff --git a/sunburnt/dates.py b/sunburnt/dates.py index fd58976..7896703 100644 --- a/sunburnt/dates.py +++ b/sunburnt/dates.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import datetime, math, re, warnings +import pytz try: import mx.DateTime @@ -55,6 +56,7 @@ def datetime_from_w3_datestring(s): del d['tzd_sign'] del d['tzd_hour'] del d['tzd_minute'] + d['tzinfo'] = pytz.utc try: dt = datetime_factory(**d) + tz_delta except DateTimeRangeError: diff --git a/sunburnt/schema.py b/sunburnt/schema.py index 3d0332c..bd522e6 100644 --- a/sunburnt/schema.py +++ b/sunburnt/schema.py @@ -595,9 +595,14 @@ def doc(self, doc): if not doc: return self.DOC() else: - return self.DOC(*reduce(operator.add, - [self.fields(name, values) - for name, values in doc.items()])) + # XXX remove all None fields this is needed for adding date fields + fields = [] + for name, values in doc.items(): + if values is None: + doc.pop(name) + continue + fields.append(self.fields(name, values)) + return self.DOC(*reduce(operator.add, fields)) def add(self, docs): if hasattr(docs, "items") or not hasattr(docs, "__iter__"): @@ -669,7 +674,7 @@ def __str__(self): class SolrFacetCounts(object): - members= ["facet_dates", "facet_fields", "facet_queries"] + members= ["facet_dates", "facet_fields", "facet_queries", "facet_pivot"] def __init__(self, **kwargs): for member in self.members: setattr(self, member, kwargs.get(member, ())) diff --git a/sunburnt/search.py b/sunburnt/search.py index 13055c6..bf91cf7 100644 --- a/sunburnt/search.py +++ b/sunburnt/search.py @@ -376,8 +376,9 @@ def add_boost(self, kwargs, boost_score): class BaseSearch(object): """Base class for common search options management""" option_modules = ('query_obj', 'filter_obj', 'paginator', - 'more_like_this', 'highlighter', 'faceter', - 'sorter', 'facet_querier', 'field_limiter',) + 'more_like_this', 'highlighter', 'postings_highlighter', + 'faceter', 'sorter', 'facet_querier', 'field_limiter', + 'pivoter') result_constructor = dict @@ -386,7 +387,9 @@ def _init_common_modules(self): self.filter_obj = LuceneQuery(self.schema, u'fq') self.paginator = PaginateOptions(self.schema) self.highlighter = HighlightOptions(self.schema) + self.postings_highlighter = PostingsHighlightOptions(self.schema) self.faceter = FacetOptions(self.schema) + self.pivoter = FacetPivotOptions(self.schema) self.sorter = SortOptions(self.schema) self.field_limiter = FieldLimitOptions(self.schema) self.facet_querier = FacetQueryOptions(self.schema) @@ -446,6 +449,11 @@ def facet_by(self, field, **kwargs): newself.faceter.update(field, **kwargs) return newself + def pivot_by(self, fields, **kwargs): + newself = self.clone() + newself.pivoter.update(fields, **kwargs) + return newself + def facet_query(self, *args, **kwargs): newself = self.clone() newself.facet_querier.update(self.Q(*args, **kwargs)) @@ -456,6 +464,11 @@ def highlight(self, fields=None, **kwargs): newself.highlighter.update(fields, **kwargs) return newself + def postings_highlight(self, fields=None, **kwargs): + newself = self.clone() + newself.postings_highlighter.update(fields, **kwargs) + return newself + def mlt(self, fields, query_fields=None, **kwargs): newself = self.clone() newself.more_like_this.update(fields, query_fields, **kwargs) @@ -776,6 +789,31 @@ def field_names_in_opts(self, opts, fields): opts["facet.field"] = sorted(fields) +class FacetPivotOptions(Options): + option_name = "facet.pivot" + opts = { + "mincount":lambda self, x: int(x) >= 0 and int(x) or self.invalid_value(), + } + + def __init__(self, schema, original=None): + self.schema = schema + if original is None: + self.fields = collections.defaultdict(dict) + else: + self.fields = copy.copy(original.fields) + + def field_names_in_opts(self, opts, fields): + opts["facet"] = True + if fields: + field_opts = {} + for field in fields: + field_opts = dict(field_opts.items() + self.fields[field].items()) + del(self.fields[field]) + self.fields[None] = field_opts + opts["facet.pivot"] = ','.join(sorted(fields)) + + + class HighlightOptions(Options): option_name = "hl" opts = {"snippets":int, @@ -794,8 +832,46 @@ class HighlightOptions(Options): "highlightMultiTerm":bool, "regex.slop":float, "regex.pattern":unicode, - "regex.maxAnalyzedChars":int + "regex.maxAnalyzedChars":int, + "boundaryScanner": unicode, + "bs.maxScan": unicode, + "bs.chars": unicode, + "bs.type": unicode, + "bs.language": unicode, + "bs.country": unicode, + } + def __init__(self, schema, original=None): + self.schema = schema + if original is None: + self.fields = collections.defaultdict(dict) + else: + self.fields = copy.copy(original.fields) + + def field_names_in_opts(self, opts, fields): + if fields: + opts["hl.fl"] = ",".join(sorted(fields)) + + +class PostingsHighlightOptions(Options): + + option_name = "hl" + opts = {"snippets": int, + "tag.pre": unicode, + "tag.post": unicode, + "tag.ellipsis": unicode, + "defaultSummary": bool, + "encoder": unicode, + "score.k1": float, + "score.b": float, + "score.pivot": float, + "bs.type": unicode, + "bs.language": unicode, + "bs.country": unicode, + "bs.variant": unicode, + "maxAnalyzedChars": unicode, + "multiValuedSeperatorChar": unicode } + def __init__(self, schema, original=None): self.schema = schema if original is None: diff --git a/sunburnt/test_schema.py b/sunburnt/test_schema.py index a8c9875..1f66355 100644 --- a/sunburnt/test_schema.py +++ b/sunburnt/test_schema.py @@ -51,7 +51,7 @@ "2009-07-23T03:24:34.1Z": datetime.datetime(2009, 07, 23, 3, 24, 34, 100000, pytz.utc), "2009-07-23T03:24:34.123Z": - datetime.datetime(2009, 07, 23, 3, 24, 34, 123000, pytz.utc) + datetime.datetime(2009, 07, 23, 3, 24, 34, 122999, pytz.utc) } def check_solr_date_from_date(s, date, canonical_date): @@ -59,7 +59,7 @@ def check_solr_date_from_date(s, date, canonical_date): check_solr_date_from_string(s, canonical_date) def check_solr_date_from_string(s, date): - assert solr_date(s)._dt_obj == date + assert solr_date(s)._dt_obj == date, "Unequal representations of %r: %r" % (solr_date(s)._dt_obj, date) def test_solr_date_from_pydatetimes(): for k, v in samples_from_pydatetimes.items(): diff --git a/sunburnt/test_search.py b/sunburnt/test_search.py index 8acc2c9..b03b435 100644 --- a/sunburnt/test_search.py +++ b/sunburnt/test_search.py @@ -16,7 +16,10 @@ HAS_MX_DATETIME = False from .schema import SolrSchema, SolrError -from .search import SolrSearch, MltSolrSearch, PaginateOptions, SortOptions, FieldLimitOptions, FacetOptions, HighlightOptions, MoreLikeThisOptions, params_from_dict +from .search import (SolrSearch, MltSolrSearch, PaginateOptions, SortOptions, + FieldLimitOptions, FacetOptions, FacetPivotOptions, + HighlightOptions, MoreLikeThisOptions, params_from_dict, + PostingsHighlightOptions) from .strings import RawString from .sunburnt import SolrInterface @@ -308,6 +311,14 @@ def test_bad_query_data(): ({"fields":["int_field", "text_field"], "prefix":"abc", "limit":3}, {"facet":True, "facet.field":["int_field", "text_field"], "f.int_field.facet.prefix":"abc", "f.int_field.facet.limit":3, "f.text_field.facet.prefix":"abc", "f.text_field.facet.limit":3, }), ), + FacetPivotOptions:( + ({"fields":["text_field"]}, + {"facet":True, "facet.pivot":"text_field"}), + ({"fields":["int_field", "text_field"]}, + {"facet":True, "facet.pivot":"int_field,text_field"}), + ({"fields":["int_field", "text_field"], "mincount":2}, + {"facet":True, "facet.pivot":"int_field,text_field", "facet.pivot.mincount":2}), + ), SortOptions:( ({"field":"int_field"}, {"sort":"int_field asc"}), @@ -330,6 +341,35 @@ def test_bad_query_data(): ({"fields":["int_field", "text_field"], "snippets":3, "fragsize":5}, {"hl":True, "hl.fl":"int_field,text_field", "f.int_field.hl.snippets":3, "f.int_field.hl.fragsize":5, "f.text_field.hl.snippets":3, "f.text_field.hl.fragsize":5}), ), + PostingsHighlightOptions:( + ({"fields":"int_field"}, + {"hl":True, "hl.fl":"int_field"}), + ({"fields":["int_field", "text_field"]}, + {"hl":True, "hl.fl":"int_field,text_field"}), + ({"snippets":3}, + {"hl":True, "hl.snippets":3}), + ({"fields":["int_field", "text_field"], "snippets":1, + "tag.pre":"<em>", "tag.post": "<em>", + "tag.ellipsis": "...", "defaultSummary": True, "encoder": "simple", + "score.k1": 1.2, "score.b": 0.75, "score.pivot": 87, + "bs.type": "SENTENCE", "maxAnalyzedChars": 10000,}, + {'f.text_field.hl.score.b': 0.75, 'f.int_field.hl.encoder': u'simple', + 'f.int_field.hl.tag.pre': u'<em>', 'f.text_field.hl.tag.pre': + u'<em>', 'f.text_field.hl.defaultSummary': True, + 'f.text_field.hl.tag.post': u'<em>', 'f.text_field.hl.bs.type': + u'SENTENCE', 'f.int_field.hl.tag.ellipsis': u'...', + 'f.text_field.hl.score.k1': 1.2, 'f.text_field.hl.tag.ellipsis': + u'...', 'f.int_field.hl.score.pivot': 87.0, + 'f.int_field.hl.tag.post': u'<em>', 'f.int_field.hl.bs.type': + u'SENTENCE', 'f.int_field.hl.score.b': 0.75, + 'f.text_field.hl.maxAnalyzedChars': u'10000', 'hl': True, + 'f.text_field.hl.encoder': u'simple', 'hl.fl': + 'int_field,text_field', 'f.int_field.hl.snippets': 1, + 'f.text_field.hl.snippets': 1, 'f.int_field.hl.maxAnalyzedChars': + u'10000', 'f.int_field.hl.score.k1': 1.2, + 'f.int_field.hl.defaultSummary': True, 'f.text_field.hl.score.pivot': + 87.0}), + ), MoreLikeThisOptions:( ({"fields":"int_field"}, {"mlt":True, "mlt.fl":"int_field"}), @@ -359,7 +399,8 @@ def test_bad_query_data(): def check_good_option_data(OptionClass, kwargs, output): optioner = OptionClass(schema) optioner.update(**kwargs) - assert optioner.options() == output + assert optioner.options() == output, "Unequal: %r, %r" % ( + optioner.options(), output) def test_good_option_data(): for OptionClass, option_data in good_option_data.items(): diff --git a/sunburnt/tests/__init__.py b/sunburnt/tests/__init__.py new file mode 100644 index 0000000..792d600 --- /dev/null +++ b/sunburnt/tests/__init__.py @@ -0,0 +1 @@ +#