diff --git a/scripts/cloud_storage_migration.py b/scripts/cloud_storage_migration.py new file mode 100755 index 0000000..ddad3dc --- /dev/null +++ b/scripts/cloud_storage_migration.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# coding=utf-8 +# +# Copyright 2011 Friday Film Club. All Rights Reserved. + +"""Migrate images from blobstore to cloud storage + +You'll need to `pip install python-magic` +""" + +import os +import sys +sys.path.append('/usr/local/google_appengine') +import dev_appserver +import posixpath +import cloudstorage +import magic +import re +import getpass + +dev_appserver.fix_sys_path() + +dir = os.path.dirname(__file__) +sys.path.insert(0, os.path.join(dir, '../src')) + +from google.appengine.ext.remote_api import remote_api_stub +from google.appengine.ext import blobstore, ndb + +GCS_ROOT = '/ffcapp.appspot.com/images/' + +os.environ['SERVER_SOFTWARE'] = 'Development (remote_api)/1.0' + +import models + +APP_NAME = 's~ffcapp' +RE_SPECIAL_CHARS_ = re.compile(r'[^a-zA-Z0-9 ]') +os.environ['AUTH_DOMAIN'] = 'gmail.com' +os.environ['USER_EMAIL'] = 'adamjmcgrath@gmail.com' + +def slugify(my_string): + """Remove special characters and replace spaces with hyphens.""" + return '-'.join(re.sub(RE_SPECIAL_CHARS_, '', my_string).lower().split(' ')) + +def auth_func(): + return (os.environ['USER_EMAIL'], getpass.getpass()) + +def save_image(blob_key, folder_name, file_name): + img_file = blobstore.BlobInfo(blob_key) + img_file_o = img_file.open() + img_data = img_file_o.read() + type = magic.from_buffer(img_data, mime=True) + + file_path = posixpath.join( + GCS_ROOT, folder_name, file_name) + gcs_file = cloudstorage.open(file_path, 'w', content_type=type) + print 'Saving file: %s' % file_path + gcs_file.write(img_data) + gcs_file.close() + img_file_o.close() + return blobstore.BlobKey(blobstore.create_gs_key('/gs' + file_path)) + + +def migrate_screenshot(q): + clue = q.clues[0] + if not clue: + return + title = q.answer_title + clue_entity = clue.get() + old_key = str(clue_entity.image) + clue_entity.image = save_image( + clue_entity.image, 'questions', slugify(title) + '-screenshot') + clue_entity.put() + print ('screenshot: %s | old: %s | new: %s' % (title, old_key, str(clue_entity.image))) + + +def migrate_packshot(q): + title = q.answer_title + old_key = str(q.packshot) + q.packshot = save_image( + q.packshot, 'questions', slugify(title) + '-packshot') + q.put() + print ('packshot: %s | old: %s | new: %s' % (title, old_key, str(q.packshot))) + + +def migrate_questions(): + for q in models.Question.query().fetch(2): + migrate_packshot(q) + migrate_screenshot(q) + + +def main(): + # Use 'localhost:8080' for dev server. + remote_api_stub.ConfigureRemoteApi(APP_NAME, '/_ah/remote_api', + auth_func, servername='ffcapp.appspot.com') + + # migrate_questions() + # TODO leagues, users + + +if __name__ == '__main__': + main() diff --git a/src/forms.py b/src/forms.py index 707ca13..d235fbf 100644 --- a/src/forms.py +++ b/src/forms.py @@ -12,7 +12,6 @@ import json import re import posixpath -import uuid import cloudstorage import webapp2 @@ -27,7 +26,7 @@ _USERNAME_RE = re.compile(r'^[\w\d_]{3,16}$') -GCS_FOLDER = '/ffcapp.appspot.com/images/' +GCS_ROOT = '/ffcapp.appspot.com/images/' def validate_username(form, field): @@ -101,6 +100,10 @@ def __init__(self, name, gcs_folder, img_size=None, **kwargs): self.img_size = img_size super(ImageField, self).__init__(name, **kwargs) + def file_name(self, req, obj): + img_file = req.POST.get(self.name) + return img_file.filename + def populate_obj(self, obj, name): """Populate the question model with a GCS image.""" req = webapp2.get_request() @@ -109,19 +112,44 @@ def populate_obj(self, obj, name): img_file = req.POST.get(self.name) file_name = posixpath.join( - GCS_FOLDER, self.gcs_folder, uuid.uuid4(), img_file.filename) + GCS_ROOT, self.gcs_folder, self.file_name(req, obj)) gcs_file = cloudstorage.open(file_name, 'w', content_type=img_file.type) logging.info('Saving file: %s' % file_name) gcs_file.write(img_file.value) gcs_file.close() - setattr(obj, name, blobstore.create_gs_key('/gs' + file_name)) + setattr(obj, name, + blobstore.BlobKey(blobstore.create_gs_key('/gs' + file_name))) + + +class ProfileImageField(ImageField): + + def file_name(self, req, obj): + return obj.username_lower + + +class PackshotImageField(ImageField): + + def file_name(self, req, obj): + return '%s-packshot' % models.slugify(obj.answer_title) + + +class ScreenshotImageField(ImageField): + + def file_name(self, req, obj): + return '%s-screenshot' % models.slugify(obj.question.get().answer_title) + + +class LeagueImageField(ImageField): + + def file_name(self, req, obj): + return obj.name_slug class ClueForm(Form): """A clue form.""" text = fields.TextAreaField('Text') - image = ImageField('Image', 'clues') + image = ScreenshotImageField('Image', 'questions') class ClueFormField(fields.FormField): @@ -208,7 +236,41 @@ def populate_obj(self, entity, name): entity.users = [ndb.Key('User', int(key)) for key in self.data.split(',')] -class Question(Form): +class OrderedFieldForm(Form): + """ + Set the order in which the fields populate the model. + + So we can rely on the model having populated fields in subsequent + populate object calls. + """ + + # The field order list must contain the name of every field. + field_order = [] + + def populate_obj(self, obj): + """ + Populates the attributes of the passed `obj` with data from the form's + fields. + """ + items = sorted( + self._fields.items(), key=lambda tup: self.field_order.index(tup[0])) + + for name, field in items: + field.populate_obj(obj, name) + + +class Question(OrderedFieldForm): + + field_order = [ + 'answer', + 'clues', + 'week', + 'season', + 'imdb_url', + 'email_msg', + 'packshot', + ] + """A question form.""" def __init__(self, **kwargs): super(Question, self).__init__( **kwargs) @@ -218,7 +280,7 @@ def __init__(self, **kwargs): answer = FilmField('Film', [validators.Required()], id='film') clues = CluesFieldList(ClueFormField(ClueForm), min_entries=4) email_msg = fields.TextAreaField('Email Message') - packshot = ImageField('Image', 'questions') + packshot = PackshotImageField('Image', 'questions') imdb_url = fields.TextField('IMDB Link', default='http://www.imdb.com/title/XXX/') week = WeekField(choices=WeekField.week_choices()) @@ -230,17 +292,34 @@ class Registration(Form): username = fields.TextField('', [validate_username]) -class User(Form): +class User(OrderedFieldForm): + + field_order = [ + 'username', + 'email', + 'favourite_film', + 'pic', + ] + username = fields.TextField('', [validate_username]) email = fields.TextField(validators=[validators.Email()]) - pic = ImageField('pic', 'profiles') + pic = ProfileImageField('pic', 'profiles') favourite_film = FilmField() -class League(Form): +class League(OrderedFieldForm): + + field_order = [ + 'id', + 'name', + 'pic', + 'owner', + 'users', + ] + id = fields.HiddenField('') name = fields.TextField('', [validate_league_name]) - pic = ImageField('pic', 'leagues') + pic = LeagueImageField('pic', 'leagues') owner = CurrentUserField() users = LeagueUsersField() diff --git a/src/models.py b/src/models.py index 82a0c4f..5ab91de 100755 --- a/src/models.py +++ b/src/models.py @@ -247,7 +247,7 @@ def blob_from_url(url, username): logging.info('Saving file: %s' % file_name) gcs_file.write(result.content) gcs_file.close() - return blobstore.create_gs_key('/gs' + file_name) + return blobstore.BlobKey(blobstore.create_gs_key('/gs' + file_name)) @staticmethod def to_leaderboard_json(user):