Merge pull request #281 from ael-code/elasticsearch-2-bulk

Add support for Elasticsearch 2.x

.travis.yml

@@ -1,30 +1,39 @@
 language: python
 
+python:
+ - 2.7
+
 services:
   - elasticsearch
 
+env:
+    - VERSION_ES=1.x
+
 matrix:
+    fast_finish: true
+
     include:
         - python: "2.7"
-          env: TEST_SUITE=test VERSION_GEVENT=1.0.1
-        - python: "2.7"
-          env: TEST_SUITE=test VERSION_GEVENT=1.0.2
+          sudo: true
+          env: VERSION_ES=2.x
+
         - python: "2.7"
-          env: TEST_SUITE=test VERSION_GEVENT=1.1.0
+          env: TEST_SUITE=flake
+
         - python: "2.7"
           env: TEST_SUITE=build_sphinx
-        - python: "2.7"
-          env: TEST_SUITE=flake
 
-before_install:
-    - '[[ -n $VERSION_GEVENT ]] && pip install --force gevent==$VERSION_GEVENT || true'
 
 install:
-    - 'if [[ $TEST_SUITE == flake ]]; then pip install flake8; return $? ; fi'
-    - 'if [[ $TEST_SUITE != flake ]]; then python setup.py install; return $? ; fi'
-    - 'if [[ $TEST_SUITE == build_sphinx ]]; then pip install sphinx; return $? ; fi'
+    - 'if [[ $TEST_SUITE == flake ]]; then pip install flake8; return $?; fi'
+    - 'if [[ $TEST_SUITE != flake ]]; then python setup.py install; return $?; fi'
+    - 'if [[ $TEST_SUITE == build_sphinx ]]; then pip install sphinx; return $?; fi'
+
+before_script:
+    - bash -x ./travis/install_dependencies.sh
 
 script:
+    - 'if [[ -z $TEST_SUITE ]]; then sleep 10; python setup.py test; return $?; fi'
     - 'if [[ $TEST_SUITE == flake ]]; then flake8; return $?; fi'
-    - 'if [[ $TEST_SUITE == test ]]; then sleep 10 && python setup.py test ; return $?; fi'
     - 'if [[ $TEST_SUITE == build_sphinx ]]; then python setup.py build_sphinx; return $?; fi'
+

archivant/test/class_template.py

@@ -27,8 +27,8 @@ def setUp(self):
         self.arc = Archivant(conf)
 
     def tearDown(self):
-        self.arc._db.es.delete_by_query(index=self.TEST_ES_INDEX,
-                                        body={'query': {'match_all': {}}})
+        self.arc._db.delete_all()
+        self.refresh_index()
         rmtree(self.tmpDir)
         self.tmpDir = None
         self.arc = None

archivant/test/test_insert_volume.py

@@ -22,7 +22,6 @@ def test_insert_volume_check_metadata(self):
         added_volume = self.arc.get_volume(id)
         ok_('id' in added_volume)
         eq_(added_volume['type'], 'volume')
-        eq_(len(volume_metadata), len(added_volume['metadata']))
         for k, v in volume_metadata.items():
             eq_(added_volume['metadata'][k], v)
 

cli/libreant_db.py

@@ -50,6 +50,62 @@ def libreant_db(debug, settings, fsdb_path, es_indexname, es_hosts):
             die(str(e))
 
 
+@libreant_db.command(name="upgrade")
+@click.option('-c', '--check-only', is_flag=True,
+              help='Does not perform any action on database, it will exit with code 123 if an upgraded is needed, and 0 if it\'s not. '+
+                   'Any other exit code means that an error occurred.')
+@click.option('-y', '--yes', is_flag=True, help='Assume Yes to all queries and do not prompt')
+def upgrade(check_only, yes):
+    '''
+    Upgrade libreant database.
+
+    This command can be used after an update of libreant
+    in order to upgrade the database and make it aligned with the new version.
+    '''
+    from elasticsearch import Elasticsearch
+    from libreantdb import DB, migration
+    from libreantdb.exceptions import MappingsException
+
+    try:
+        db = DB(Elasticsearch(hosts=conf['ES_HOSTS']),
+                              index_name=conf['ES_INDEXNAME'])
+        if not db.es.indices.exists(db.index_name):
+            die("The specified index does not exists: {}".format(db.index_name))
+
+        # Migrate old special `_timestamp` field into the new `_insertion_date`
+        num_to_update = migration.elements_without_insertion_date(db.es, db.index_name)
+        if num_to_update > 0:
+            if check_only:
+                exit(123)
+
+            if yes or click.confirm("{} entries miss the '_insertion_date' field. Do you want to proceed and update those entries?".format(num_to_update),
+                             prompt_suffix='',
+                             default=False):
+                migration.migrate_timestamp(db.es, db.index_name)
+            else:
+                exit(0)
+
+        # Upgrade the index mappings and reindex if necessary
+        try:
+            db.update_mappings()
+        except MappingsException:
+            if check_only:
+                exit(123)
+            count = db.es.count(index=db.index_name)['count']
+            if yes or click.confirm("Some old or wrong mappings has been found for the index '"+ db.index_name +"'.\n"\
+                                    "In order to upgrade them it is necessary to reindex '"+ str(count) +"' entries.\n"\
+                                    "Are you sure you want to proceed?",
+                                     prompt_suffix='',
+                                     default=False):
+                    db.reindex()
+
+    except Exception as e:
+        if conf.get('DEBUG', False):
+            raise
+        else:
+            die(str(e))
+
+
 @libreant_db.command(name="export-volume", help="export a volume")
 @click.argument('volumeid')
 @click.option('-p', '--pretty', is_flag=True, help='format the output on multiple lines')

cli/tests/test_libreant_db.py

@@ -96,7 +96,7 @@ def test_no_metadata(self):
         eq_(export_res.exit_code, 0)
         volume_data = json.loads(export_res.output)
         eq_(volume_data['metadata']['_language'], 'en')
-        eq_(len(volume_data['metadata'].keys()), 1)
+        eq_(len(volume_data['metadata'].keys()), 2) # _language and _insertion_date properties
 
     def test_no_language(self):
         '''--language is required'''