Merge 98bae65 into 6d9cd29

.travis.yml

@@ -34,8 +34,8 @@ cache:
   - pip
 
 env:
-#  - REQUIREMENTS=lowest
-#  - REQUIREMENTS=release
+  - REQUIREMENTS=lowest
+  - REQUIREMENTS=release
   - REQUIREMENTS=devel
 
 python:
@@ -60,3 +60,14 @@ script:
 
 after_success:
   - coveralls
+
+deploy:
+  provider: pypi
+  user: lnielsen
+  password:
+    secure: "F9LXazIs6Dc3DE847YyJ0w4e8ahzvKJbz2BJ8H8T/kxNq4+DNM5GVxlZEvKx46VKr8SYEXWcdLUkeCegFk+mqpM75hiJAAzHCMd/AL9Njs51dxqMUK5CjXFBCASZVZPMcQsN/xemArX6s78wCm+mjkj9XqABGdksppz0QOGDouOV/+ihdfOtl2MTXnlfEOP9hr4wh3gStbaC7yU8FZSv98dxS0ep6pW0GAFk0I1S4LTP4NCDQ0cy7pyZivXwiGPIcN+TO42f4F25TqhtskRoZ3hwyaTzYjNYtUukvM+xN3BBcJ5q6hW0X6PGj9vjj3XamGtZBXb8MMDbzSNtLhAiDCMMTMUrIiXBmZFiimdevNlR9JF91d7zJXwPdWueHtlufXXYQ6ZyLCHnZj27e9MILQAs49WsHTQyb7TxZrtYetILRdhxSeUG7gwvn5gkGvI82Nn4OObUAa0nEMhmsI2ZoSKtvBoLXNn5f10tpu+XLgHNPb19gPZujnrqWIStOOSS17HD3pZwtPOclJIYVECUzrB8ERQ8AczeHZ7WC1U+IqqLxvNgxj/I+QNvWFqo0zkllbRVycJMQ2Nu8v2McFBUiBHoqijVZoiPoEBP1pwXR/8XNbLBBsS+346ID5+SOv1cS5a7huCpV2MJz+TNtuztv/7yhmCddNwd+euEZyCQz6c="
+  distributions: "sdist bdist_wheel"
+  on:
+    tags: true
+    python: "2.7"
+    condition: $REQUIREMENTS = release

CHANGES.rst

@@ -25,6 +25,6 @@
 Changes
 =======
 
-Version 1.0.0 (released TBD)
+Version 1.0.0a1 (released 2016-03-21)
 
 - Initial public release.

RELEASE-NOTES.rst

@@ -2,7 +2,7 @@
  Invenio-Migrator v1.0.0a1
 ===========================
 
-Invenio-Migrator v1.0.0a1 was released on TBD, 2016.
+Invenio-Migrator v1.0.0a1 was released on March 21, 2016.
 
 About
 -----

invenio_migrator/cli.py

@@ -30,8 +30,10 @@
 import sys
 
 import click
+from celery import chain
 from flask_cli import with_appcontext
 
+from .proxies import current_migrator
 from .tasks.records import import_record
 
 
@@ -54,7 +56,13 @@ def loadrecords(source, source_type):
     click.echo('Sending tasks to queue...')
     with click.progressbar(data) as records:
         for item in records:
-            import_record.delay(item, source_type=source_type)
+            if current_migrator.records_post_task:
+                chain(
+                    import_record.s(item, source_type=source_type),
+                    current_migrator.records_post_task.s()
+                )()
+            else:
+                import_record.delay(item, source_type=source_type)
 
 
 @dumps.command()

invenio_migrator/legacy/cli.py

@@ -61,17 +61,22 @@ def dump(thing, query, from_date, file_prefix, chunk_size, argument):
             '{1}'.format(thing, collect_things_entry_points()))
 
     click.echo("Querying {0}...".format(thing))
-    items = thing_func.get(query, from_date, **kwargs)
+    count, items = thing_func.get(query, from_date, **kwargs)
 
     count = 0
     click.echo("Dumping {0}...".format(thing))
-    with click.progressbar(length=len(items)) as bar:
+    with click.progressbar(length=count) as bar:
         for i, chunk_ids in enumerate(grouper(items, chunk_size)):
             with open('{0}_{1}.json'.format(file_prefix, i), 'w') as fp:
                 fp.write("[\n")
                 for _id in chunk_ids:
-                    json.dump(thing_func.dump(_id, from_date, **kwargs), fp)
-                    fp.write(",")
+                    try:
+                        json.dump(
+                            thing_func.dump(_id, from_date, **kwargs), fp)
+                        fp.write(",")
+                    except Exception:
+                        click.secho(
+                            "Failed dump {0} {1}".format(thing, _id), fg='red')
                     count += 1
                     bar.update(count)
 

invenio_migrator/legacy/records.py

@@ -89,7 +89,7 @@ def get(query, from_date, **kwargs):
     if query:
         return recids.intersection(set(search_pattern(p=query)))
 
-    return recids
+    return len(recids), recids
 
 
 def dump(recid, from_date, with_json=True, latest_only=False, **kwargs):
@@ -116,7 +116,7 @@ def dump(recid, from_date, with_json=True, latest_only=False, **kwargs):
     for revision in revision_iter:
         revision_date = datetime.datetime.strptime(
             revision[1], '%Y%m%d%H%M%S')
-        if revision_date < date:
+        if date is not None and revision_date < date:
             continue
 
         marcxml = get_marcxml_of_revision_id(*revision)

invenio_migrator/records.py

@@ -26,6 +26,8 @@
 
 from __future__ import absolute_import, print_function
 
+from os.path import splitext
+
 import arrow
 from dojson.contrib.marc21 import marc21
 from dojson.contrib.marc21.utils import create_record
@@ -108,6 +110,7 @@ def create_pids(record_uuid, pids):
             PersistentIdentifier.create(
                 pid_type=p.pid_type,
                 pid_value=p.pid_value,
+                pid_provider=p.provider.pid_provider if p.provider else None,
                 object_type='rec',
                 object_uuid=record_uuid,
                 status=PIDStatus.REGISTERED,
@@ -149,11 +152,16 @@ def create_files(cls, record, files, existing_files):
         record['files'] = []
         for key, meta in files.items():
             obj = cls.create_file(b, key, meta)
+            ext = splitext(obj.key)[1].lower()
+            if ext.startswith('.'):
+                ext = ext[1:]
             record['files'].append(dict(
                 bucket=str(obj.bucket.id),
                 filename=obj.key,
+                version_id=str(obj.version_id),
                 size=obj.file.size,
                 checksum=obj.file.checksum,
+                type=ext,
             ))
         record.commit()
         db.session.commit()

invenio_migrator/tasks/records.py

@@ -32,7 +32,7 @@
 from ..proxies import current_migrator
 
 
-@shared_task(ignore_result=True)
+@shared_task()
 def import_record(data, source_type=None, latest_only=False):
     """Migrate a record from a migration dump.
 
@@ -51,10 +51,9 @@ def import_record(data, source_type=None, latest_only=False):
     )
     try:
         record = current_migrator.records_dumploader_cls.create(recorddump)
+        db.session.commit()
     except Exception:
         db.session.rollback()
         raise
 
-    # Call post task if available
-    if current_migrator.records_post_task:
-        current_migrator.records_post_task.delay(record.id)
+    return str(record.id)

setup.py

@@ -51,7 +51,7 @@
         'dojson>=1.0.0',
         'Flask-CeleryExt>=0.2.0',
         'invenio-db[versioning]>=1.0.0a9',
-        'invenio-files-rest>=1.0.0.dev20150000',
+        'invenio-files-rest>=1.0.0a1',
         'invenio-pidstore>=1.0.0a6',
         'invenio-records>=1.0.0a9',
     ],