CLI imports/exports

cli.py

@@ -1,35 +1,111 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-"""Shell access to dribdat."""
+"""Management functions for dribdat."""
 import os
 import click
 import datetime as dt
-from flask.cli import AppGroup
-from dateutil.parser import parse
 from dribdat.app import init_app
 from dribdat.settings import DevConfig, ProdConfig
-from dribdat.user.models import Event
-from flask.helpers import get_debug_flag
+from dribdat.apipackage import fetch_datapackage, load_file_datapackage
 
-CONFIG = DevConfig if get_debug_flag() else ProdConfig
+HERE = os.path.abspath(os.path.dirname(__file__))
+TEST_PATH = os.path.join(HERE, 'tests')
 
-app = init_app(CONFIG)
 
-event_cli = AppGroup('event')
+def create_app(script_info=None):
+    """Initialise the app object."""
+    if os.environ.get("DRIBDAT_ENV") == 'prod':
+        app = init_app(ProdConfig)
+    else:
+        app = init_app(DevConfig)
+    return app
+
 
+@click.command()
+@click.argument('kind', nargs=-1, required=False)
+def socialize(kind):
+    """Reset user profile data."""
+    """Parameter: which models to refresh (users, ..)"""
+    with create_app().app_context():
+        # TODO: use the kind parameter to refresh projects, etc.
+        from dribdat.user.models import User
+        q = [u.socialize() for u in User.query.all()]
+        print("Updated %d users." % len(q))
 
-@event_cli.command('start')
+
+@click.command()
 @click.argument('name', required=True)
 @click.argument('start', required=False)
-def event_start(name, start=None):
+@click.argument('finish', required=False)
+def event_start(name, start=None, finish=None):
     """Create a new event."""
     if start is None:
         start = dt.datetime.now() + dt.timedelta(days=1)
     else:
-        start = parse(start)
-    event = Event(name="test", starts_at=start)
-    event.save()
-    print("Created event %d" % event.id)
+        start = dt.datetime.parse(start)
+    if finish is None:
+        finish = dt.datetime.now() + dt.timedelta(days=2)
+    else:
+        finish = dt.datetime.parse(finish)
+    with create_app().app_context():
+        from dribdat.user.models import Event
+        event = Event(name=name, starts_at=start, ends_at=finish)
+        event.save()
+        print("Created event %d" % event.id)
+
+
+@click.command()
+@click.argument('url', required=True)
+@click.argument('level', required=False)
+def imports(url, level='full'):
+    """Import a new event from a URI or file."""
+    # Configuration
+    dry_run = True
+    all_data = False
+    if level == 'basic':
+        dry_run = False
+    elif level == 'full':
+        dry_run = False
+        all_data = True
+    else:
+        level = 'dry run'
+    print("Starting %s import" % level)
+    with create_app().app_context():
+        if url.startswith('http'):
+            results = fetch_datapackage(url, dry_run, all_data)
+        else:
+            results = load_file_datapackage(url, dry_run, all_data)
+    event_names = ', '.join([r['name'] for r in results['events']])
+    if 'errors' in results:
+        print(results['errors'])
+    else:
+        print("Created events: %s" % event_names)
+
+
+@click.command()
+@click.argument('kind', nargs=-1, required=False)
+def exports(kind):
+    """Export some data."""
+    with create_app().app_context():
+        if 'people' in kind:
+            from dribdat.user.models import User
+            for pp in User.query.filter_by(active=True).all():
+                print(pp.email)
+        elif 'events' in kind:
+            from dribdat.user.models import Event
+            for pp in Event.query.filter_by(is_hidden=False).all():
+                print('\t'.join([pp.name, str(pp.starts_at), str(pp.ends_at)]))
+
+
+@click.group(name='j')
+def cli():
+    """dribdat command line interfoot."""
+    pass
+
 
+cli.add_command(socialize)
+cli.add_command(imports)
+cli.add_command(exports)
 
-app.cli.add_command(event_start())
+if __name__ == '__main__':
+    cli()

dribdat/apipackage.py

@@ -3,7 +3,11 @@
 
 import logging
 import requests
-from datetime import datetime as dt
+import json
+import tempfile
+from os import path
+from werkzeug.utils import secure_filename
+from datetime import datetime as dtt
 from frictionless import Package, Resource
 from .user.models import Event, Project, Activity, Category, User, Role
 from .utils import format_date
@@ -47,7 +51,7 @@ def PackageEvent(event, author=None, host_url='', full_contents=False):
         }],
         contributors=contributors,
         homepage=event.webpage_url or '',
-        created=format_date(dt.now(), '%Y-%m-%dT%H:%M'),
+        created=format_date(dtt.now(), '%Y-%m-%dT%H:%M'),
         version="0.2.0",
     )
 
@@ -206,7 +210,7 @@ def importActivities(data, DRY_RUN=False):
     pname = ""
     for act in data:
         aname = act['name']
-        tstamp = dt.utcfromtimestamp(act['time'])
+        tstamp = dtt.utcfromtimestamp(act['time'])
         activity = Activity.query.filter_by(name=aname,
                                             timestamp=tstamp).first()
         if activity:
@@ -254,10 +258,34 @@ def ImportEventPackage(data, DRY_RUN=False, ALL_DATA=False):
     return updates
 
 
-def ImportEventByURL(url, DRY_RUN=False, ALL_DATA=False):
+def fetch_datapackage(url, DRY_RUN=False, ALL_DATA=False):
+    """Get event data from a URL."""
     try:
         data = requests.get(url, timeout=REQUEST_TIMEOUT).json()
+        return ImportEventPackage(data, DRY_RUN, ALL_DATA)
+    except json.decoder.JSONDecodeError:
+        return {'errors': ['Could not load package due to JSON error']}
     except requests.exceptions.RequestException:
         logging.error("Could not connect to %s" % url)
         return {}
-    return ImportEventPackage(data, DRY_RUN, ALL_DATA)
+
+
+def import_datapackage(filedata, dry_run, all_data):
+    """Saves a temporary file and provides details."""
+    ext = filedata.filename.split('.')[-1].lower()
+    if ext not in ['json']:
+        return {'errors': ['Invalid format (allowed: JSON)']}
+    with tempfile.TemporaryDirectory() as tmpdir:
+        filepath = path.join(tmpdir, secure_filename(filedata.filename))
+        filedata.save(filepath)
+        return load_file_datapackage(filepath, dry_run, all_data)
+
+
+def load_file_datapackage(filepath, dry_run, all_data):
+    """Get event data from a file."""
+    try:
+        with open(filepath, mode='rb') as file:
+            data = json.load(file)
+        return ImportEventPackage(data, dry_run, all_data)
+    except json.decoder.JSONDecodeError:
+        return {'errors': ['Could not load package due to JSON error']}

dribdat/public/api.py

@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 """API calls for dribdat."""
 import boto3
+import tempfile
 from flask import (
     Blueprint, current_app,
     Response, request, redirect,
     stream_with_context, send_file,
     jsonify, flash, url_for, escape
 )
 from flask_login import login_required, current_user
-from werkzeug.utils import secure_filename
 from sqlalchemy import or_
 from ..extensions import db
 from ..utils import timesince, random_password
@@ -23,9 +23,9 @@
     expand_project_urls,
     gen_csv,
 )
-import tempfile
-import json
-from os import path
+from ..apipackage import (
+    fetch_datapackage, import_datapackage
+)
 
 blueprint = Blueprint('api', __name__, url_prefix='/api')
 
@@ -277,32 +277,18 @@ def event_load_datapackage():  # noqa: C901
         status = "Complete"
     # File handling
     if filedata and filedata.filename != '':
-        results = prepare_datapackage(filedata, dry_run, all_data)
-        event_names = ', '.join([r['name'] for r in results['events']])
-        if 'errors' in results:
-            return jsonify(status='Error', errors=results['errors'])
-        else:
-            flash("Events uploaded: %s" % event_names, 'success')
-            return redirect(url_for("admin.events"))
+        results = import_datapackage(filedata, dry_run, all_data)
+    else:
+        results = fetch_datapackage(url, dry_run, all_data)
+    event_names = ', '.join([r['name'] for r in results['events']])
+    if 'errors' in results:
+        return jsonify(status='Error', errors=results['errors'])
+    else:
+        flash("Events uploaded: %s" % event_names, 'success')
+        return redirect(url_for("admin.events"))
     return jsonify(status=status, results=results)
 
 
-def prepare_datapackage(filedata, dry_run, all_data):
-    """Saves a temporary file and provides details."""
-    ext = filedata.filename.split('.')[-1].lower()
-    if ext not in ['json']:
-        return {'errors': ['Invalid format (allowed: JSON)']}
-    with tempfile.TemporaryDirectory() as tmpdir:
-        filepath = path.join(tmpdir, secure_filename(filedata.filename))
-        filedata.save(filepath)
-        try:
-            with open(filepath, mode='rb') as file:
-                data = json.load(file)
-            return ImportEventPackage(data, dry_run, all_data)
-        except json.decoder.JSONDecodeError:
-            return {'errors': ['Could not load package due to JSON error']}
-
-
 @blueprint.route('/event/push/datapackage', methods=["PUT", "POST"])
 def event_push_datapackage():
     """Upload event data from a Data Package."""

manage.py

@@ -57,52 +57,13 @@ def test(name):
     return subprocess.call(['pytest', feat_test])
 
 
-@click.command()
-@click.argument('kind', nargs=-1, required=False)
-def socialize(kind):
-    """Reset user profile data."""
-    """Parameter: which models to refresh (users, ..)"""
-    with create_app().app_context():
-        # TODO: use the kind parameter to refresh projects, etc.
-        from dribdat.user.models import User
-        q = [u.socialize() for u in User.query.all()]
-        print("Updated %d users." % len(q))
-
-
-@click.command()
-@click.argument('name', required=True)
-@click.argument('start', required=False)
-def event_start(name, start=None):
-    """Create a new event."""
-    if start is None:
-        start = dt.datetime.now() + dt.timedelta(days=1)
-    else:
-        start = parse(start)
-    event = Event(name="test", starts_at=start)
-    event.save()
-    print("Created event %d" % event.id)
-
-
-@click.command()
-@click.argument('kind', nargs=-1, required=False)
-def exports(kind):
-    """Export some data."""
-    with create_app().app_context():
-        if 'people' in kind:
-            from dribdat.user.models import User
-            for pp in User.query.filter_by(active=True).all():
-                print(pp.email)
-
-
 @click.group(cls=FlaskGroup, create_app=create_app)
 def cli():
     """Script for managing this application."""
     pass
 
 
 cli.add_command(test)
-cli.add_command(socialize)
-cli.add_command(exports)
 
 if __name__ == '__main__':
     cli()