Tags and Collections in batchaddmedia, Collections in addmedia

mediagoblin/gmg_commands/addmedia.py

@@ -20,7 +20,7 @@
 
 import six
 
-from mediagoblin.db.models import LocalUser
+from mediagoblin.db.models import LocalUser, Collection
 from mediagoblin.gmg_commands import util as commands_util
 from mediagoblin.submit.lib import (
     submit_media, get_upload_file_limits,
@@ -51,17 +51,19 @@ def parser_setup(subparser):
         "-T", "--tags",
         help=(
             "Comma separated list of tags for this media entry."))
+    subparser.add_argument(
+        "-C", "--collections",
+        help=(
+            "Comma separated list of slugs of collections to add this media entry to."))
     subparser.add_argument(
         "-s", "--slug",
         help=(
             "Slug for this media entry. "
             "Will be autogenerated if unspecified."))
-
     subparser.add_argument(
         '--celery',
         action='store_true',
-        help="Don't process eagerly, pass off to celery")
-
+        help="Don't process eagerly, pass off to celery. WARNING: If there is an error during processing (transcoding error, etc) you will not see it here.")
 
 def addmedia(args):
     # Run eagerly unless explicetly set not to
@@ -95,6 +97,30 @@ def maybe_unicodeify(some_string):
             return six.text_type(some_string, 'utf-8')
         return some_string
 
+    if args.collections:
+        collections_string = args.collections
+    else:
+        collections_string = ''
+    collection_slugs = tuple(
+        collection_slug.strip() for collection_slug
+        in collections_string.split(',')
+        if collection_slug.strip()
+    )
+    collections = Collection.query.filter_by(
+        actor=user.id,
+        type=Collection.USER_DEFINED_TYPE,
+    ).filter(
+        Collection.slug.in_(collection_slugs),
+    )
+    invalid_collections_slugs = (
+        set(collection_slugs) - set(c.slug for c in collections)
+    )
+    if invalid_collections_slugs:
+        raise ValueError(
+            'Couldn\'t find these collections: %s'
+            % ", ".join(invalid_collections_slugs)
+        )
+
     try:
         submit_media(
             mg_app=app,
@@ -103,6 +129,7 @@ def maybe_unicodeify(some_string):
             title=maybe_unicodeify(args.title),
             description=maybe_unicodeify(args.description),
             license=maybe_unicodeify(args.license),
+            collections=collections,
             tags_string=maybe_unicodeify(args.tags) or u"",
             upload_limit=upload_limit, max_file_size=max_file_size)
     except FileUploadLimit:

mediagoblin/gmg_commands/batchaddmedia.py

@@ -17,6 +17,7 @@
 from __future__ import print_function
 
 import codecs
+from collections import OrderedDict
 import csv
 import os
 
@@ -25,7 +26,7 @@
 
 from six.moves.urllib.parse import urlparse
 
-from mediagoblin.db.models import LocalUser
+from mediagoblin.db.models import LocalUser, Collection
 from mediagoblin.gmg_commands import util as commands_util
 from mediagoblin.submit.lib import (
     submit_media, get_upload_file_limits,
@@ -52,7 +53,42 @@ def parser_setup(subparser):
     subparser.add_argument(
         '--celery',
         action='store_true',
-        help=_(u"Don't process eagerly, pass off to celery"))
+        help=_("Don't process eagerly, pass off to celery. WARNING: If there is an error during processing (transcoding error, etc) you will not see it here, and this script will continue adding media from the csv. This may be relevant if, for instance, you're adding media to a collection, and the order of entries is important. If such an error were to occur in the middle of your csv, you might have to delete many items before retrying the problem file in order to keep order correct."))
+
+def _get_collection_slugs(media_data):
+    return {
+        collection_slug.strip() for collection_slug
+        in media_data.get('collections', u'').split(',')
+        if collection_slug.strip()
+    }
+
+
+def _get_collections_lookup(user, media_metadata):
+    all_collection_slugs = set.union(set(), *(
+        _get_collection_slugs(media_data)
+        for media_data in media_metadata.itervalues()
+    ))
+    all_collections_lookup = {
+        c.slug: c
+        for c in Collection.query.filter_by(
+            actor=user.id,
+            type=Collection.USER_DEFINED_TYPE,
+        ).filter(
+            Collection.slug.in_(
+                tuple(all_collection_slugs)
+            ),
+        )
+    }
+    invalid_collections_slugs = (
+        set(all_collection_slugs)
+        - set(c.slug for c in all_collections_lookup.itervalues())
+    )
+    if invalid_collections_slugs:
+        raise ValueError(
+            'Couldn\'t find these collections: %s'
+            % ", ".join(invalid_collections_slugs)
+        )
+    return all_collections_lookup
 
 
 def batchaddmedia(args):
@@ -74,11 +110,9 @@ def batchaddmedia(args):
         return
 
     upload_limit, max_file_size = get_upload_file_limits(user)
-    temp_files = []
 
     if os.path.isfile(args.metadata_path):
         metadata_path = args.metadata_path
-
     else:
         error = _(u'File at {path} not found, use -h flag for help'.format(
                     path=args.metadata_path))
@@ -101,7 +135,21 @@ def maybe_unicodeify(some_string):
         contents = all_metadata.read()
         media_metadata = parse_csv_file(contents)
 
-    for media_id, file_metadata in media_metadata.iteritems():
+    # Grab the collections, or fail before changing anything
+    all_collections_lookup = _get_collections_lookup(user, media_metadata)
+
+    for media_id, media_data in media_metadata.iteritems():
+        file_metadata = {
+            k:v
+            for (k, v)
+            in media_data.iteritems()
+            if k in {'location',
+                     'license',
+                     'title',
+                     'dc:title',
+                     'description',
+                     'dc:description'}
+        }
         files_attempted += 1
         # In case the metadata was not uploaded initialize an empty dictionary.
         json_ld_metadata = compact_and_validate({})
@@ -115,6 +163,7 @@ def maybe_unicodeify(some_string):
         title = file_metadata.get('title') or file_metadata.get('dc:title')
         description = (file_metadata.get('description') or
             file_metadata.get('dc:description'))
+        tags_string = media_data.get('tags', u'')
 
         license = file_metadata.get('license')
         try:
@@ -131,6 +180,11 @@ def maybe_unicodeify(some_string):
         url = urlparse(original_location)
         filename = url.path.split()[-1]
 
+        collections = [
+            all_collections_lookup[collection_slug]
+            for collection_slug in _get_collection_slugs(media_data)
+        ]
+
         if url.scheme == 'http':
             res = requests.get(url.geturl(), stream=True)
             media_file = res.raw
@@ -159,8 +213,10 @@ def maybe_unicodeify(some_string):
                 description=maybe_unicodeify(description),
                 license=maybe_unicodeify(license),
                 metadata=json_ld_metadata,
-                tags_string=u"",
-                upload_limit=upload_limit, max_file_size=max_file_size)
+                tags_string=tags_string,
+                upload_limit=upload_limit,
+                max_file_size=max_file_size,
+                collections=collections)
             print(_(u"""Successfully submitted {filename}!
 Be sure to look at the Media Processing Panel on your website to be sure it
 uploaded successfully.""".format(filename=filename)))
@@ -201,7 +257,7 @@ def parse_csv_file(file_contents):
     list_of_contents = file_contents.split('\n')
     key, lines = (list_of_contents[0].split(','),
                   list_of_contents[1:])
-    objects_dict = {}
+    objects_dict = OrderedDict()
 
     # Build a dictionary
     for index, line in enumerate(lines):

mediagoblin/submit/lib.py

@@ -27,11 +27,12 @@
 from mediagoblin.tools.response import json_response
 from mediagoblin.tools.text import convert_to_tag_list_of_dicts
 from mediagoblin.tools.federation import create_activity, create_generator
-from mediagoblin.db.models import MediaEntry, ProcessingMetaData
+from mediagoblin.db.models import MediaEntry, ProcessingMetaData, Collection
 from mediagoblin.processing import mark_entry_failed
 from mediagoblin.processing.task import ProcessMedia
 from mediagoblin.notifications import add_comment_subscription
 from mediagoblin.media_types import sniff_media
+from mediagoblin.user_pages.lib import add_media_to_collection
 
 
 _log = logging.getLogger(__name__)
@@ -104,7 +105,8 @@ def submit_media(mg_app, user, submitted_file, filename,
                  title=None, description=None,
                  license=None, metadata=None, tags_string=u"",
                  upload_limit=None, max_file_size=None,
-                 callback_url=None, urlgen=None,):
+                 callback_url=None, urlgen=None,
+                 collections=None):
     """
     Args:
      - mg_app: The MediaGoblinApp instantiated for this process
@@ -118,13 +120,15 @@ def submit_media(mg_app, user, submitted_file, filename,
      - description: description for this media entry
      - license: license for this media entry
      - tags_string: comma separated string of tags to be associated
-       with this entry
+       with this entry.
+     - collections: collections to add this item to
      - upload_limit: size in megabytes that's the per-user upload limit
      - max_file_size: maximum size each file can be that's uploaded
      - callback_url: possible post-hook to call after submission
      - urlgen: if provided, used to do the feed_url update and assign a public
                ID used in the API (very important).
     """
+    collections = collections or []
     if upload_limit and user.uploaded >= upload_limit:
         raise UserPastUploadLimit()
 
@@ -205,6 +209,11 @@ def submit_media(mg_app, user, submitted_file, filename,
     create_activity("post", entry, entry.actor)
     entry.save()
 
+    # Add to collections
+    for collection in collections:
+        add_media_to_collection(collection, entry)
+        create_activity("add", entry, user, target=collection)
+
     # Pass off to processing
     #
     # (... don't change entry after this point to avoid race