diff --git a/.travis.yml b/.travis.yml index 9764a41..5bfd826 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,45 @@ language: python -# Existing Python versions -python: - - 3.4 - - 3.5 - - 3.6 -# Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs + matrix: include: - - python: 3.7 + - env: ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian ES_DSL_VERS=2.2.0 ES_VERS=2.2.1 + python: 3.6 + - env: ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt ES_DSL_VERS=5.3.0 ES_VERS=5.6.0 + python: 3.6 + - env: ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt ES_DSL_VERS=6.3.1 ES_VERS=6.4.3 + python: 3.6 + - env: ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian ES_DSL_VERS=2.2.0 ES_VERS=2.2.1 + python: 3.7 + sudo: true + dist: xenial + - env: ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt ES_DSL_VERS=5.3.0 ES_VERS=5.6.0 + python: 3.7 + sudo: true + dist: xenial + - env: ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt ES_DSL_VERS=6.3.1 ES_VERS=6.4.3 + python: 3.7 dist: xenial sudo: true + +before_install: + - sudo rm /etc/apt/sources.list; sudo touch /etc/apt/sources.list + # - sudo rm -rf /etc/apt/sources.list.d; sudo mkdir sources.list.d + - wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + - echo "deb $ES_APT_URL stable main" | sudo tee -a /etc/apt/sources.list.d/elastic.list + - sudo apt-get update && sudo apt-get install elasticsearch=$ES_VERS -y --allow-downgrades + - sudo service elasticsearch start + - sleep 25 + - curl localhost:9200 + install: - pip install . - pip install -r requirements-dev.txt - pip install coveralls + - pip install elasticsearch-dsl==$ES_DSL_VERS + script: - make tests - make quality + after_success: - coveralls diff --git a/luqum/tests/book.json b/luqum/tests/book.json new file mode 100644 index 0000000..afe689c --- /dev/null +++ b/luqum/tests/book.json @@ -0,0 +1,182 @@ +{ + "books": [ + { + "title": "Harry Potter and the Philosopher's Stone", + "edition": "Bloomsbury", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [ + { + "name": "Thomas Taylor", + "nationality": "UK", + "birthdate": "1973-05-22" + }, + { + "name": "Mary GrandPré", + "nationality":"US", + "birthdate": "1954-02-13" + } + ], + "publication_date": "1997-06-26", + "n_pages": "223" + }, + { + "title": "Harry Potter and the Chamber of Secrets", + "edition": "Bloomsbury", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [ + { + "name": "Cliff Wright", + "nationality": "UK", + "birthdate": "1953-10-24" + }, + { + "name": "Mary GrandPré", + "nationality": "US", + "birthdate": "1954-02-13" + } + ], + "publication_date": "1998-07-02", + "n_pages": "251" + }, + { + "title": "Harry Potter and the Prisoner of Azkaban", + "edition": "Bloomsbury", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [ + { + "name": "Cliff Wright", + "nationality": "UK", + "birthdate": "1953-10-24" + }, + { + "name": "Mary GrandPré", + "nationality": "US", + "birthdate": "1954-02-13" + } + ], + "publication_date": "1999-07-08", + "n_pages": "317" + }, + { + "title": "Harry Potter and the Goblet of Fire", + "edition": "Bloomsbury", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [ + { + "name": "Giles Greenfield", + "nationality": "UK" + }, + { + "name": "Mary GrandPré", + "nationality": "US", + "birthdate": "1954-02-13" + } + ], + "publication_date": "2000-07-08", + "n_pages": "636" + }, + { + "title": "Harry Potter and the Order of the Phoenix", + "edition": "Bloomsbury", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [ + { + "name":"Jason Cockcroft", + "nationality":"UK" + }, + { + "name": "Mary GrandPré", + "nationality": "US", + "birthdate": "1954-02-13" + } + ], + "publication_date": "2003-06-21", + "n_pages": "766" + }, + { + "title": "Harry Potter and the Half-Blood Prince", + "edition": "Bloomsbury", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [ + { + "name": "Jason Cockcroft", + "nationality": "UK" + }, + { + "name": "Mary GrandPré", + "nationality": "US", + "birthdate": "1954-02-13" + } + ], + "publication_date": "2005-07-16", + "n_pages": "607" + }, + { + "title": "Harry Potter and the Deathly Hallows", + "edition": "Bloomsbury", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [ + { + "name": "Jason Cockcroft", + "nationality": "UK" + }, + { + "name": "Mary GrandPré", + "nationality": "US", + "birthdate": "1954-02-13" + } + ], + "publication_date": "2007-07-21", + "n_pages": "607" + }, + { + "title": "Harry Potter and the Cursed Child", + "edition": "Little, Brown and Company", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [], + "publication_date": "2016-07-30", + "n_pages": "360" + }, + { + "title": "The Tales of Beedle the Bard", + "edition": "Lumos (charity)", + "author": { + "name": "J. K. Rowling", + "birthdate": "1965-07-31" + }, + "illustrators": [ + { + "name":"J. K. Rowling", + "nationality": "UK", + "birthdate": "1965-07-31" + } + ], + "publication_date": "2008-12-04", + "n_pages": "157" + } + ] +} diff --git a/luqum/tests/test_es_integration.py b/luqum/tests/test_es_integration.py new file mode 100644 index 0000000..9f8f2f1 --- /dev/null +++ b/luqum/tests/test_es_integration.py @@ -0,0 +1,222 @@ +import json +from unittest import TestCase + +import elasticsearch_dsl +from elasticsearch import Elasticsearch +from elasticsearch.helpers import bulk +from elasticsearch_dsl import Date, Index, Integer, Nested, Object, Search +from elasticsearch_dsl.connections import connections +from luqum.elasticsearch import ElasticsearchQueryBuilder, SchemaAnalyzer +from luqum.parser import parser + +MAJOR_ES = elasticsearch_dsl.VERSION[0] +if MAJOR_ES > 2: + from elasticsearch_dsl import Keyword + +ES6 = False +if MAJOR_ES == 6: + from elasticsearch_dsl import Text, Document, InnerDoc + ES6 = True +else: + from elasticsearch_dsl import ( + String as Text, + DocType as Document, + InnerObjectWrapper as InnerDoc, + ) + +client = Elasticsearch() +co = connections.create_connection(hosts=["localhost"], timeout=20) + +if MAJOR_ES > 2: + class Illustrator(InnerDoc): + name = Text() + birthdate = Date() + nationality = Keyword() + + +class Book(Document): + title = Text() + edition = Text() + author = Object(properties={"name": Text(), "birthdate": Date()}) + publication_date = Date() + n_pages = Integer() + + if ES6: + illustrators = Nested(Illustrator) + + class Index: + name = "bk" + + else: + illustrators = Nested( + properties={ + "name": Text(), + "birthdate": Date(), + "nationality": Keyword() if MAJOR_ES > 2 else Text(index="not_analyzed") + } + ) + + class Meta: + index = "bk" + + def save(self, **kwargs): + return super().save(**kwargs) + + +def add_data(): + search = connections.get_connection() + Book.init() + with open("luqum/tests/book.json") as f: + datas = json.load(f) + + actions = ( + {"_op_type": "index", "_id": i, "_source": d} + for i, d in enumerate(datas["books"]) + ) + if ES6: + doc_type = "doc" + else: + doc_type = "book" + + bulk(search, actions, index="bk", doc_type=doc_type, refresh=True) + + +class LuqumRequestTestCase(TestCase): + @classmethod + def setUpClass(cls): + cls.search = Search(using=client, index="bk") + MESSAGES_SCHEMA = {"mappings": Book._doc_type.mapping.to_dict()} + schema_analizer = SchemaAnalyzer(MESSAGES_SCHEMA) + cls.es_builder = ElasticsearchQueryBuilder( + **schema_analizer.query_builder_options(), + ) + add_data() + + def _ask_luqum(self, req): + tree = parser.parse(req) + query = self.es_builder(tree) + return [x.title for x in self.search.filter(query).execute()] + + def test_simple_field_search(self): + self.assertListEqual( + self._ask_luqum('title:"Chamber"'), + ["Harry Potter and the Chamber of Secrets"], + ) + + def test_nested_field_search(self): + self.assertListEqual( + self._ask_luqum("illustrators:(name:Giles)"), + ["Harry Potter and the Goblet of Fire"], + ) + + def test_or_condition_search(self): + self.assertListEqual( + self._ask_luqum( + 'illustrators:(name:"Giles Greenfield" OR name:"Cliff Wright")' + ), + [ + "Harry Potter and the Prisoner of Azkaban", + "Harry Potter and the Chamber of Secrets", + "Harry Potter and the Goblet of Fire", + ], + ) + + def test_and_condition_search(self): + self.assertListEqual( + self._ask_luqum( + 'illustrators:(name:"Cliff Wright") AND illustrators:(name:"Mary GrandPré")' + ), + [ + "Harry Potter and the Prisoner of Azkaban", + "Harry Potter and the Chamber of Secrets", + ], + ) + + def test_date_range_search(self): + self.assertListEqual( + self._ask_luqum("publication_date:[2005-01-01 TO 2010-12-31]"), + [ + "Harry Potter and the Half-Blood Prince", + "The Tales of Beedle the Bard", + "Harry Potter and the Deathly Hallows", + ], + ) + + def test_int_range_search(self): + self.assertListEqual( + self._ask_luqum("n_pages:[500 TO *]"), + [ + "Harry Potter and the Half-Blood Prince", + "Harry Potter and the Order of the Phoenix", + "Harry Potter and the Deathly Hallows", + "Harry Potter and the Goblet of Fire", + ], + ) + + def test_int_search(self): + self.assertListEqual( + self._ask_luqum("n_pages:360"), ["Harry Potter and the Cursed Child"] + ) + + def test_proximity_search(self): + self.assertListEqual( + self._ask_luqum('title:"Harry Secrets"~5'), + ["Harry Potter and the Chamber of Secrets"], + ) + + def test_fuzzy_search(self): + self.assertListEqual( + self._ask_luqum("title:Gublet~2"), ["Harry Potter and the Goblet of Fire"] + ) + + def test_object_field_search(self): + self.assertListEqual( + self._ask_luqum('illustrators:(name:"J. K. Rowling")'), + ["The Tales of Beedle the Bard"], + ) + + def test_fail_search(self): + self.assertListEqual(self._ask_luqum("title:secret"), []) + + def test_wildcard_matching(self): + self.assertListEqual( + self._ask_luqum("title:secret*"), + ["Harry Potter and the Chamber of Secrets"], + ) + + def test_wildcard1_search(self): + self.assertListEqual( + self._ask_luqum("title:P*ix"), ["Harry Potter and the Order of the Phoenix"] + ) + + def test_not_search(self): + self.assertListEqual( + self._ask_luqum("-title:Harry"), ["The Tales of Beedle the Bard"] + ) + + def test_not_analysed_field_search(self): + self.assertListEqual(self._ask_luqum("illustrators:nationality:uk"), []) + + def test_complex_search(self): + + self.assertListEqual( + self._ask_luqum( + """ + title:phoenux~2 AND + illustrators:name:Grand* AND + illustrators:( + -name:grandpr* AND ( + name:J*on OR birthdate:[1950-01-01 TO 1970-01-01] + ) + ) + """ + ), + ["Harry Potter and the Order of the Phoenix"], + ) + + @classmethod + def tearDownClass(cls): + if ES6: + Book._index.delete() + else: + Index("bk").delete(ignore=404) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0f49049..17df8af 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ -r requirements.txt -nose==1.3.7 coverage==4.0.3 -Sphinx==1.4.1 +elasticsearch-dsl==6.3.1 flake8==2.5.4 +nose==1.3.7 +Sphinx==1.4.1 diff --git a/requirements.txt b/requirements.txt index 0ac3887..f63a7e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ ply==3.11 +tox==3.7.0