Merge 302cd3f into 71edac5

inveniosoftware · Aug 7, 2019 · 25b749a · 25b749a
2 parents 71edac5 + 302cd3f
commit 25b749a
Show file tree

Hide file tree

Showing 17 changed files with 312 additions and 69 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -23,6 +23,9 @@
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
 addons:
+  apt:
+    packages:
+      - rabbitmq-server
   postgresql: 9.4
 
 notifications:
@@ -32,43 +35,56 @@ sudo: false
 
 services:
   - postgresql
-  - redis
-  - rabbitmq
+  - redis-server
 
 language: python
 
 matrix:
   fast_finish: true
+  allow_failures:
+    - env: REQUIREMENTS=devel EXTRAS=all,elasticsearch6 ES_URL=$ES6_DOWNLOAD_URL
+    - env: REQUIREMENTS=devel EXTRAS=all,elasticsearch7 ES_URL=$ES7_DOWNLOAD_URL
 
 cache:
   - pip
 
 env:
-  - REQUIREMENTS=lowest ES_HOST=127.0.0.1 ES_VERSION=2.2.0 SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://postgres@localhost:5432/invenio"
-  - REQUIREMENTS=release ES_HOST=127.0.0.1 ES_VERSION=2.2.0 SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://postgres@localhost:5432/invenio"
-  - REQUIREMENTS=devel ES_HOST=127.0.0.1 ES_VERSION=2.2.0 SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://postgres@localhost:5432/invenio"
+  global:
+    - ES5_DOWNLOAD_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.4.tar.gz"
+    - ES6_DOWNLOAD_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.2.tar.gz"
+    - ES7_DOWNLOAD_URL="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.3.0-linux-x86_64.tar.gz"
+  matrix:
+  - REQUIREMENTS=lowest EXTRAS=all,elasticsearch5 ES_URL=$ES5_DOWNLOAD_URL
+  - REQUIREMENTS=lowest EXTRAS=all,elasticsearch6 ES_URL=$ES6_DOWNLOAD_URL
+  - REQUIREMENTS=release EXTRAS=all,elasticsearch6 ES_URL=$ES6_DOWNLOAD_URL
+  - REQUIREMENTS=release EXTRAS=all,elasticsearch7 ES_URL=$ES7_DOWNLOAD_URL DEPLOY=true
+  - REQUIREMENTS=devel EXTRAS=all,elasticsearch6 ES_URL=$ES6_DOWNLOAD_URL
+  - REQUIREMENTS=devel EXTRAS=all,elasticsearch7 ES_URL=$ES7_DOWNLOAD_URL
 
 python:
   - "2.7"
-  # - "3.3"
-  # - "3.4"
   - "3.5"
 
 before_install:
   - "mkdir /tmp/elasticsearch"
-  - "wget -O - https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/${ES_VERSION}/elasticsearch-${ES_VERSION}.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1"
-  - "/tmp/elasticsearch/bin/plugin install -b mapper-attachments"
-  - "/tmp/elasticsearch/bin/elasticsearch &"
-  - "sleep 20"
+  - "wget -O - $ES_URL | tar xz --directory=/tmp/elasticsearch --strip-components=1"
+  - "if [ ${ES_URL} == ${ES2_DOWNLOAD_URL} ]; then /tmp/elasticsearch/bin/plugin install -b mapper-attachments; fi"
+  - "/tmp/elasticsearch/bin/elasticsearch > /tmp/local-es.log &"
   - "travis_retry pip install --upgrade pip setuptools py"
   - "travis_retry pip install twine wheel coveralls requirements-builder"
-  - "requirements-builder -e all --level=min setup.py > .travis-lowest-requirements.txt"
-  - "requirements-builder -e all --level=pypi setup.py > .travis-release-requirements.txt"
-  - "requirements-builder -e all --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements.txt"
+  - "requirements-builder -e $EXTRAS --level=min setup.py > .travis-lowest-requirements.txt"
+  - "requirements-builder -e $EXTRAS --level=pypi setup.py > .travis-release-requirements.txt"
+  - "requirements-builder -e $EXTRAS --level=dev --req requirements-devel.txt setup.py > .travis-devel-requirements.txt"
 
 install:
   - "travis_retry pip install -r .travis-${REQUIREMENTS}-requirements.txt"
-  - "travis_retry pip install -e .[all]"
+  - "travis_retry pip install -e .[${EXTRAS}]"
+  - "pip freeze"
+
+before_script:
+  # Allow ES to start
+  - sleep 5
+  - curl http://127.0.0.1:9200 && echo "Elasticsearch is up!" || cat /tmp/local-es.log
 
 script:
   - "./run-tests.sh"
@@ -84,5 +100,5 @@ deploy:
   distributions: "compile_catalog sdist bdist_wheel"
   on:
     tags: true
-    python: "2.7"
-    condition: $REQUIREMENTS = release
+    python: "3.5"
+    condition: $DEPLOY = true
diff --git a/examples/app-fixtures.sh b/examples/app-fixtures.sh
@@ -15,5 +15,5 @@ export FLASK_APP=app.py
 flask users create info@inveniosoftware.org -a --password 123456
 
 # Load fixtures
-flask fixtures records
 flask fixtures location
+flask fixtures records
diff --git a/examples/app.py b/examples/app.py
@@ -75,7 +75,7 @@
 from invenio_indexer import InvenioIndexer
 from invenio_indexer.api import RecordIndexer
 from invenio_jsonschemas import InvenioJSONSchemas
-from invenio_oauth2server import InvenioOAuth2Server
+from invenio_oauth2server import InvenioOAuth2Server, InvenioOAuth2ServerREST
 from invenio_oauth2server.views import server_blueprint, settings_blueprint
 from invenio_pidstore import InvenioPIDStore
 from invenio_records import InvenioRecords
@@ -160,6 +160,7 @@
 InvenioPIDStore(app)
 InvenioAdmin(app)
 InvenioOAuth2Server(app)
+InvenioOAuth2ServerREST(app)
 
 InvenioFilesREST(app)
 
@@ -181,32 +182,18 @@ def fixtures():
 @cli.with_appcontext
 def records():
     """Load records."""
-    import pkg_resources
-    from dojson.contrib.marc21 import marc21
-    from dojson.contrib.marc21.utils import create_record, split_blob
     from flask_login import login_user, logout_user
     from invenio_accounts.models import User
     from invenio_deposit.api import Deposit
 
-    users = User.query.all()
-
-    # pkg resources the demodata
-    data_path = pkg_resources.resource_filename(
-        'invenio_records', 'data/marc21/bibliographic.xml'
-    )
-    with open(data_path) as source:
-        with current_app.test_request_context():
-            indexer = RecordIndexer()
-            with db.session.begin_nested():
-                for index, data in enumerate(split_blob(source.read()),
-                                             start=1):
-                    login_user(users[index % len(users)])
-                    # do translate
-                    record = marc21.do(create_record(data))
-                    # create record
-                    indexer.index(Deposit.create(record))
-                    logout_user()
-            db.session.commit()
+    user = User.query.one()
+
+    with current_app.test_request_context():
+        login_user(user)
+        with db.session.begin_nested():
+            Deposit.create({'title': 'Test'})
+        logout_user()
+        db.session.commit()
 
 
 @fixtures.command()

diff --git a/examples/requirements.txt b/examples/requirements.txt
@@ -23,11 +23,11 @@
 # as an Intergovernmental Organization or submit itself to any jurisdiction.
 
 dojson>=1.0.0
-invenio-access>=1.0.0a9
-invenio-accounts>=1.0.0a13
-invenio-admin>=1.0.0a3
-invenio-i18n>=1.0.0b1
-invenio-pidstore>=1.0.0a9
-invenio-rest[cors]>=1.0.0a9
-invenio-theme>=1.0.0a17
+invenio-access>=1.0.0
+invenio-accounts>=1.0.0
+invenio-admin>=1.0.0
+invenio-i18n>=1.0.0
+invenio-pidstore>=1.0.0
+invenio-rest[cors]>=1.0.0
+invenio-theme>=1.0.0
 urllib3>=1.21.1,<1.25
diff --git a/invenio_deposit/mappings/v5/__init__.py b/invenio_deposit/mappings/v5/__init__.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2019 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Mappings for ES 7."""
diff --git a/invenio_deposit/mappings/v5/deposits/deposit-v1.0.0.json b/invenio_deposit/mappings/v5/deposits/deposit-v1.0.0.json
@@ -0,0 +1,65 @@
+{
+  "mappings": {
+    "deposit-v1.0.0": {
+      "properties": {
+        "$schema": {
+          "type": "keyword"
+        },
+        "_deposit": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "keyword"
+            },
+            "pid": {
+              "type": "object",
+              "properties": {
+                "revision_id": {
+                  "type": "integer"
+                },
+                "type": {
+                  "type": "text"
+                },
+                "value": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "created_by": {
+              "type": "integer"
+            },
+            "owners": {
+              "type": "integer"
+            },
+            "status": {
+              "type": "keyword"
+            }
+          }
+        },
+        "_files": {
+          "type": "object",
+          "properties": {
+            "key": {
+              "type": "keyword"
+            },
+            "bucket": {
+              "type": "keyword"
+            },
+            "size": {
+              "type": "integer"
+            },
+            "version_id": {
+              "type": "keyword"
+            }
+          }
+        },
+        "title": {
+          "type": "text"
+        },
+        "control_number": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
diff --git a/invenio_deposit/mappings/v6/__init__.py b/invenio_deposit/mappings/v6/__init__.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2019 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Mappings for ES 7."""
diff --git a/invenio_deposit/mappings/v6/deposits/deposit-v1.0.0.json b/invenio_deposit/mappings/v6/deposits/deposit-v1.0.0.json
@@ -0,0 +1,65 @@
+{
+  "mappings": {
+    "deposit-v1.0.0": {
+      "properties": {
+        "$schema": {
+          "type": "keyword"
+        },
+        "_deposit": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "keyword"
+            },
+            "pid": {
+              "type": "object",
+              "properties": {
+                "revision_id": {
+                  "type": "integer"
+                },
+                "type": {
+                  "type": "text"
+                },
+                "value": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "created_by": {
+              "type": "integer"
+            },
+            "owners": {
+              "type": "integer"
+            },
+            "status": {
+              "type": "keyword"
+            }
+          }
+        },
+        "_files": {
+          "type": "object",
+          "properties": {
+            "key": {
+              "type": "keyword"
+            },
+            "bucket": {
+              "type": "keyword"
+            },
+            "size": {
+              "type": "integer"
+            },
+            "version_id": {
+              "type": "keyword"
+            }
+          }
+        },
+        "title": {
+          "type": "text"
+        },
+        "control_number": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
diff --git a/invenio_deposit/mappings/v7/__init__.py b/invenio_deposit/mappings/v7/__init__.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2019 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Mappings for ES 7."""
diff --git a/invenio_deposit/mappings/v7/deposits/deposit-v1.0.0.json b/invenio_deposit/mappings/v7/deposits/deposit-v1.0.0.json
@@ -0,0 +1,63 @@
+{
+  "mappings": {
+    "properties": {
+      "$schema": {
+          "type": "keyword"
+      },
+      "_deposit": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "keyword"
+          },
+          "pid": {
+            "type": "object",
+            "properties": {
+              "revision_id": {
+                "type": "integer"
+              },
+              "type": {
+                "type": "text"
+              },
+              "value": {
+                "type": "keyword"
+              }
+            }
+          },
+          "created_by": {
+            "type": "integer"
+          },
+          "owners": {
+              "type": "integer"
+          },
+          "status": {
+            "type": "keyword"
+          }
+        }
+      },
+      "_files": {
+        "type": "object",
+        "properties": {
+          "key": {
+            "type": "keyword"
+          },
+          "bucket": {
+            "type": "keyword"
+          },
+          "size": {
+            "type": "integer"
+          },
+          "version_id": {
+            "type": "keyword"
+          }
+        }
+      },
+      "title": {
+        "type": "text"
+      },
+      "control_number": {
+        "type": "text"
+      }
+    }
+  }
+}