Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Add gitattributes

  • Loading branch information...
commit bcf4ea93efb2161cef3295c64c9c1625b142a3dc 1 parent 1866f38
Tyler Hicks-Wright tghw authored
14 .gitattributes
View
@@ -0,0 +1,14 @@
+# Auto detect text files and perform LF normalization
+* text=auto
+
+# Standard to msysgit
+*.doc diff=astextplain
+*.DOC diff=astextplain
+*.docx diff=astextplain
+*.DOCX diff=astextplain
+*.dot diff=astextplain
+*.DOT diff=astextplain
+*.pdf diff=astextplain
+*.PDF diff=astextplain
+*.rtf diff=astextplain
+*.RTF diff=astextplain
178 app.yaml
View
@@ -1,89 +1,89 @@
-application: tghwputty
-version: 1
-runtime: python27
-threadsafe: yes
-api_version: 1
-default_expiration: "365d"
-
-inbound_services:
-- mail
-- channel_presence
-- warmup
-
-handlers:
-
-- url: /favicon\.ico
- static_files: static/img/favicon.ico
- upload: static/img/favicon\.ico
- mime_type: image/x-icon
-
-- url: /respond-proxy\.html
- static_files: static/respond/respond-proxy.html
- upload: static/respond/respond-proxy\.html
-
-- url: /static/img/(.*\.(gif|png|jpg))
- static_files: static/img/\1
- upload: static/img/(.*\.(gif|png|jpg))
-
-- url: /static/img/docs/(.*\.(gif|png|jpg))
- static_files: static/img/docs/\1
- upload: static/img/docs/(.*\.(gif|png|jpg))
-
-- url: /static/css/(.*\.css)
- mime_type: text/css
- static_files: static/css/\1
- upload: static/css/(.*\.css)
-
-- url: /static/js/(.*\.js)
- mime_type: text/javascript
- static_files: static/js/\1
- upload: static/js/(.*\.js)
-
-- url: /static/codemirror/(.*\.css)
- mime_type: text/css
- static_files: static/codemirror/\1
- upload: static/codemirror/(.*\.css)
-
-- url: /static/codemirror/(.*\.js)
- mime_type: text/javascript
- static_files: static/codemirror/\1
- upload: static/codemirror/(.*\.js)
-
-- url: /_ah/mail/.+
- script: incoming_email_handler.application
- login: admin
-
-- url: /_migrate
- script: bootstrap.app
- login: admin
-
-- url: /gae_mini_profiler/static/js/(.*\.tmpl)
- mime_type: text/html
- static_files: libs/gae_mini_profiler/static/js/\1
- upload: libs/gae_mini_profiler/static/js/(.*\.tmpl)
-
-- url: /gae_mini_profiler/static
- static_dir: libs/gae_mini_profiler/static
-
-- url: /gae_mini_profiler/.*
- script: libs.gae_mini_profiler.main.app
-
-- url: /livecount/counter_admin
- script: libs.livecount.counter_admin.application
- login: admin
-
-- url: /livecount/.*
- script: libs.livecount.counter.application
- login: admin
-
-- url: /tasks/.*
- script: bootstrap.app
- login: admin
-
-- url: .*\.(jpg|gif|png)
- static_files: static/img/404.png
- upload: static/img/404.png
-
-- url: .*
- script: bootstrap.app
-
+application: tghwputty
+version: 1
+runtime: python27
+threadsafe: yes
+api_version: 1
+default_expiration: "365d"
+
+inbound_services:
+- mail
+- channel_presence
+- warmup
+
+handlers:
+
+- url: /favicon\.ico
+ static_files: static/img/favicon.ico
+ upload: static/img/favicon\.ico
+ mime_type: image/x-icon
+
+- url: /respond-proxy\.html
+ static_files: static/respond/respond-proxy.html
+ upload: static/respond/respond-proxy\.html
+
+- url: /static/img/(.*\.(gif|png|jpg))
+ static_files: static/img/\1
+ upload: static/img/(.*\.(gif|png|jpg))
+
+- url: /static/img/docs/(.*\.(gif|png|jpg))
+ static_files: static/img/docs/\1
+ upload: static/img/docs/(.*\.(gif|png|jpg))
+
+- url: /static/css/(.*\.css)
+ mime_type: text/css
+ static_files: static/css/\1
+ upload: static/css/(.*\.css)
+
+- url: /static/js/(.*\.js)
+ mime_type: text/javascript
+ static_files: static/js/\1
+ upload: static/js/(.*\.js)
+
+- url: /static/codemirror/(.*\.css)
+ mime_type: text/css
+ static_files: static/codemirror/\1
+ upload: static/codemirror/(.*\.css)
+
+- url: /static/codemirror/(.*\.js)
+ mime_type: text/javascript
+ static_files: static/codemirror/\1
+ upload: static/codemirror/(.*\.js)
+
+- url: /_ah/mail/.+
+ script: incoming_email_handler.application
+ login: admin
+
+- url: /_migrate
+ script: bootstrap.app
+ login: admin
+
+- url: /gae_mini_profiler/static/js/(.*\.tmpl)
+ mime_type: text/html
+ static_files: libs/gae_mini_profiler/static/js/\1
+ upload: libs/gae_mini_profiler/static/js/(.*\.tmpl)
+
+- url: /gae_mini_profiler/static
+ static_dir: libs/gae_mini_profiler/static
+
+- url: /gae_mini_profiler/.*
+ script: libs.gae_mini_profiler.main.app
+
+- url: /livecount/counter_admin
+ script: libs.livecount.counter_admin.application
+ login: admin
+
+- url: /livecount/.*
+ script: libs.livecount.counter.application
+ login: admin
+
+- url: /tasks/.*
+ script: bootstrap.app
+ login: admin
+
+- url: .*\.(jpg|gif|png)
+ static_files: static/img/404.png
+ upload: static/img/404.png
+
+- url: .*
+ script: bootstrap.app
+
58 app/decorators.py
View
@@ -1,29 +1,29 @@
-import json
-from functools import wraps
-from flask import Response, redirect, request
-from google.appengine.api import users
-
-def requires_auth(func):
- @wraps(func)
- def wrapper(*args, **kwargs):
- if not users.get_current_user():
- return redirect(users.create_login_url(request.path))
- return func(*args, **kwargs)
- return wrapper
-
-def requires_admin(func):
- @wraps(func)
- def wrapper(*args, **kwargs):
- if not users.get_current_user() or not users.is_current_user_admin():
- return redirect(users.create_login_url(request.path))
- return func(*args, **kwargs)
- return wrapper
-
-def as_json(func):
- @wraps(func)
- def wrapper(*args, **kwargs):
- res = func(*args, **kwargs)
- if isinstance(res, Response):
- return res
- return json.dumps(res)
- return wrapper
+import json
+from functools import wraps
+from flask import Response, redirect, request
+from google.appengine.api import users
+
+def requires_auth(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ if not users.get_current_user():
+ return redirect(users.create_login_url(request.path))
+ return func(*args, **kwargs)
+ return wrapper
+
+def requires_admin(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ if not users.get_current_user() or not users.is_current_user_admin():
+ return redirect(users.create_login_url(request.path))
+ return func(*args, **kwargs)
+ return wrapper
+
+def as_json(func):
+ @wraps(func)
+ def wrapper(*args, **kwargs):
+ res = func(*args, **kwargs)
+ if isinstance(res, Response):
+ return res
+ return json.dumps(res)
+ return wrapper
956 app/models.py
View
@@ -1,478 +1,478 @@
-from logging import Formatter, Filter, StreamHandler
-import scss
-import settings
-import json
-from datetime import datetime
-from StringIO import StringIO
-from google.appengine.ext import db
-from google.appengine.api import users as gae_users
-from google.appengine.api import taskqueue
-from google.appengine.api import channel as gae_channels
-from google.appengine.api import files
-from google.appengine.api.datastore_errors import BadKeyError
-from flask import abort, url_for, render_template
-
-def dt_handler(obj):
- if isinstance(obj, datetime):
- return obj.isoformat() + 'Z'
- return None
-
-class ExceptionFilter(Filter):
- def filter(self, record):
- if record.getMessage().lower().startswith('exception'):
- return 0
- return 1
-
-# Old and dead...
-class UserGroup(db.Model):
- name = db.StringProperty(required=True)
- users = db.ListProperty(gae_users.User)
- admins = db.ListProperty(gae_users.User)
-
-class SchemaVersion(db.Model):
- version = db.IntegerProperty(required=True, default=0)
-
-class Site(db.Model):
- name = db.StringProperty(required=True)
- owner = db.UserProperty()
- users = db.ListProperty(gae_users.User)
- admins = db.ListProperty(gae_users.User)
- example = db.BooleanProperty(default=False)
-
- def delete(self):
- for page in self.page_set.fetch(10000):
- page.delete()
- db.delete(self)
-
- @staticmethod
- def get_or_404(id):
- site = Site.get_by_id(id)
- if not site or gae_users.get_current_user() not in site.users:
- abort(404)
- return site
-
- @staticmethod
- def get_admin_or_404(id):
- site = Site.get_by_id(id)
- if not site or gae_users.get_current_user() not in site.admins:
- abort(404)
- return site
-
-class Invitation(db.Model):
- hash = db.StringProperty(required=True)
- site = db.ReferenceProperty(Site, required=True)
- email = db.EmailProperty(required=True)
- admin = db.BooleanProperty(default=False)
- inviter = db.UserProperty(required=True)
- has_been_logged_out = db.BooleanProperty(default=False)
-
-class Page(db.Model):
- name = db.StringProperty(required=True)
- url = db.LinkProperty(required=True)
- site = db.ReferenceProperty(Site)
- _styles = db.ListProperty(db.Key)
- channels = db.ListProperty(db.Key)
- preview_img = db.BlobProperty(required=False, default=None)
- preview_urls = db.ListProperty(db.Link, default=None) # *additional* preview urls
- import_state = db.IntegerProperty(default=0)
- on_cdn = db.BooleanProperty(default=False)
-
- _style_cache = None
- def _set_styles(self, styles):
- self._style_cache = styles
- self._styles = [style.key() for style in styles]
- def _get_styles(self):
- if not self._style_cache:
- self._style_cache = [Style.get(k) for k in self._styles]
- return self._style_cache
- styles = property(_get_styles, _set_styles)
-
- def delete(self):
- for key in self.channels:
- channel = PageChannel.get(key)
- if channel:
- channel.send_message({'cmd': 'lock'})
- channel.delete()
- for style in self.styles:
- style.delete()
- db.delete(self)
-
- def clean_channels(self):
- stale = []
- for key in self.channels:
- channel = PageChannel.get(key)
- if not channel or channel.is_stale():
- stale.append(key)
- if stale:
- for key in stale:
- self.channels.remove(key)
- channel = PageChannel.get(key)
- if channel:
- # If the channel is still here, it's probably stale.
- # Send 'lock' and remove, so it can't clobber anyone else.
- channel.send_message({'cmd': 'lock'})
- channel.delete()
- self.put()
-
- def get_channels(self):
- channels = []
- stale = []
- for key in self.channels:
- channel = PageChannel.get(key)
- if channel:
- channels.append(channel)
- else:
- stale.append(key)
- if stale:
- for key in stale:
- self.channels.remove(key)
- self.put()
- return channels
-
- def update_locks(self):
- owner = None
- channels = self.get_channels()
- if channels:
- owner_user = channels[0].user
- owner = dict(name=owner_user.nickname(), email=owner_user.email())
- channels[0].send_message(dict(cmd='unlock', user=owner))
- lock_msg = dict(cmd='lock', user=owner)
- for channel in channels[1:]:
- channel.send_message(lock_msg)
-
- def add_channel(self, channel):
- self.remove_channel(channel)
- self.channels.append(channel.key())
- self.put()
-
- def add_channel_first(self, channel):
- self.remove_channel(channel)
- self.channels.insert(0, channel.key())
- self.put()
-
- def remove_channel(self, channel, delete=False):
- if channel.key() in self.channels:
- self.channels.remove(channel.key())
- self.put()
- if delete:
- channel.delete()
-
- def put(self, *args, **kwargs):
- self._set_styles(self.styles)
- super(Page, self).put(*args, **kwargs)
-
- def queue_preview(self):
- taskqueue.add(queue_name='fetch-preview', url=url_for('tasks.fetch_preview'), params={'page_key': self.key()})
-
- def queue_upload(self):
- taskqueue.add(queue_name='upload-css', url=url_for('tasks.upload_style'), params={'page_key': self.key()})
-
- def queue_refresh(self):
- self.queue_upload()
- self.queue_preview()
-
- def _css(self, preview, compress):
- css = StringIO()
- for style in self.styles:
- rev = style.preview_rev if (preview and style.preview_rev) else style.published_rev
- if compress:
- css.write(rev.compressed)
- else:
- css.write(scss.Scss().compile('@option compress:no;' + rev.raw))
- return css.getvalue()
-
- def compressed_css(self, preview):
- return self._css(preview, compress=True)
-
- def uncompressed_css(self, preview):
- return self._css(preview, compress=False)
-
- def last_modified(self, preview):
- max_last_edit = datetime.min
- for style in self.styles:
- rev = style.preview_rev if (preview and style.preview_rev) else style.published_rev
- max_last_edit = max(max_last_edit, rev.dt_last_edit)
- return max_last_edit
-
- def styles_json(self):
- # NOTE: It is okay to return an array here because we only display this
- # to users via editor.html. If we ever return this directly as the
- # response, we'll want to wrap it to avoid the exploit described at
- # http://haacked.com/archive/2009/06/25/json-hijacking.aspx
- styles_obj = [style.json_obj() for style in self.styles]
- return json.dumps(styles_obj, default=dt_handler, sort_keys=True, indent=4*' ' if settings.debug else None)
-
- def upload_to_cdn(self):
- if not settings.use_google_cloud_storage:
- return
- path = files.gs.create('/gs/%s/%s.css' % (settings.google_bucket, str(self.key())), mime_type='text/css', acl='public-read', cache_control='private,max-age=300')
- try:
- fd = files.open(path, 'a')
- fd.write(self.compressed_css(False).encode('utf-8'))
- self.on_cdn = True
- self.save()
- except Exception:
- self.on_cdn = False
- self.save()
- raise
- finally:
- fd.close()
- files.finalize(path)
-
- @staticmethod
- def get_or_404(key):
- page = None
- if isinstance(key, int) or (isinstance(key, basestring) and key.isdigit()):
- page = Page.get_by_id(int(key))
- else:
- try:
- key_obj = db.Key(key)
- except BadKeyError:
- abort(404)
- if(key_obj.kind() == 'Style'):
- page = Page.gql('WHERE _styles=:1', key_obj).get()
- else:
- page = Page.get(key)
- if not page:
- abort(404)
- return page
-
- @staticmethod
- def get_edit_or_404(page_id):
- page = Page.get_or_404(page_id)
- if gae_users.get_current_user() not in page.site.users:
- abort(404)
- return page
-
- @staticmethod
- def get_admin_or_404(page_id):
- page = Page.get_or_404(page_id)
- site = page.site
- if not site or gae_users.get_current_user() not in site.admins:
- abort(404)
- return page
-
- @staticmethod
- def new_page(site, name, url):
- '''
- Do all the work in adding a new page to a site.
- '''
- style = Style(name = name, site = site)
- style.put()
- first_rev = StyleRevision(parent=style)
- first_rev.raw = render_template('first_run.css')
- first_rev.put()
- style.published_rev = first_rev
- style.put()
- page = Page(
- name = name,
- url = url,
- site = site,
- _styles = [style.key()]
- )
- page.put()
- page.queue_refresh()
- return page
-
-class StyleRevision(db.Model):
- # parent = Style
- rev = db.IntegerProperty(required=True, default=0)
- dt_created = db.DateTimeProperty(auto_now_add=True)
- dt_last_edit = db.DateTimeProperty(auto_now=True)
- raw = db.TextProperty(required=False, default='')
- compressed = db.TextProperty(required=False, default=None)
- # Old and dead...
- css = db.TextProperty(required=False)
- _cached = db.TextProperty(required=False)
-
- def update(self, raw):
- self.raw = raw
- log = StringIO()
- handler = StreamHandler(log)
- handler.addFilter(ExceptionFilter())
- handler.setFormatter(Formatter('<span class="level">%(levelname)s</span>: <span class="message">%(message)s</span><br />'))
- scss.log.addHandler(handler)
- self.compressed = scss.Scss().compile(self.raw)
- scss.log.removeHandler(handler)
- handler.flush()
- self.put()
- return log.getvalue()
-
-class Style(db.Model):
- site = db.ReferenceProperty(Site)
- name = db.StringProperty(required=True)
- published_rev = db.ReferenceProperty(StyleRevision, default=None, collection_name='style_published')
- preview_rev = db.ReferenceProperty(StyleRevision, default=None, collection_name='style_preview')
-
- # Old and dead
- user_group = db.ReferenceProperty(UserGroup)
- url = db.LinkProperty(required=False)
-
- def delete(self, *args, **kwargs):
- revisions = StyleRevision.all(keys_only=True).ancestor(self).fetch(10000)
- db.delete(revisions)
- db.delete(self)
-
- def json_obj(self):
- if self.preview_rev:
- preview_rev = self.preview_rev
- else:
- if not self.published_rev:
- rev = StyleRevision(parent=self)
- rev.put()
- self.published_rev = rev
- self.put()
- preview_rev = self.published_rev
- return {
- 'id': self.key().id(),
- 'name': self.name,
- 'preview_scss': preview_rev.raw,
- 'preview_dt_last_edit': preview_rev.dt_last_edit,
- 'published_scss': self.published_rev.raw,
- 'published_dt_last_edit': self.published_rev.dt_last_edit,
- }
-
- @staticmethod
- def get_or_404(style_id):
- if isinstance(style_id, basestring) and not style_id.isdigit():
- style = Style.get(style_id)
- else:
- style = Style.get_by_id(style_id)
- if not style:
- abort(404)
- return style
-
- @staticmethod
- def get_edit_or_404(style_id):
- style = Style.get_or_404(style_id)
- if gae_users.get_current_user() not in style.site.users:
- abort(404)
- return style
-
- @staticmethod
- def get_admin_or_404(style_id):
- style = Style.get_or_404(style_id)
- site = style.site
- if not site or gae_users.get_current_user() not in site.admins:
- abort(404)
- return style
-
-class PageChannel(db.Model):
- user = db.UserProperty(required=True)
- page = db.ReferenceProperty(Page, required=True)
- token = db.StringProperty(required=True)
- client_id = db.StringProperty(required=True)
- dt_connected = db.DateTimeProperty(auto_now_add=True)
- dt_last_update = db.DateTimeProperty(auto_now_add=True)
-
- def is_stale(self):
- return (datetime.utcnow() - self.dt_last_update).seconds > 3600
-
- def send_message(self, message):
- if not isinstance(message, basestring):
- message = json.dumps(message, default=dt_handler, sort_keys=True, indent=4*' ' if settings.debug else None)
- gae_channels.send_message(self.client_id, message)
-
- @staticmethod
- def get_or_404(token=None, client_id=None):
- channel = None
- if token:
- channel = PageChannel.gql('WHERE token=:1', token).get()
- elif client_id:
- channel = PageChannel.gql('WHERE client_id=:1', client_id).get()
- if not channel:
- abort(404)
- return channel
-
-class UserSettings(db.Model):
- user = db.UserProperty(required=True)
- seen_example = db.BooleanProperty(default=False)
- seen_guiders = db.StringListProperty()
- # the last version (list of ints) this person has viewed the release notes for
- seen_version = db.ListProperty(int, default=None)
- locale = db.StringProperty(default=None)
- chimped = db.BooleanProperty(default=False)
-
- @staticmethod
- def has_seen_example():
- user = gae_users.get_current_user()
- if not user or not user.user_id():
- raise Exception("Logged in user expected")
- settings = UserSettings.get_or_insert(user.user_id(), user=user)
- return settings.seen_example
-
- @staticmethod
- def mark_example_as_seen():
- user = gae_users.get_current_user()
- if not user or not user.user_id():
- raise Exception("Logged in user expected")
- settings = UserSettings.get_or_insert(user.user_id(), user=user)
- settings.seen_example = True
- settings.put()
-
- @staticmethod
- def show_guider(guider_name):
- user = gae_users.get_current_user()
- if not user or not user.user_id():
- return False
- settings = UserSettings.get_or_insert(user.user_id(), user=user)
- return (guider_name not in settings.seen_guiders)
-
- @staticmethod
- def mark_guider_as_seen(guider_name):
- user = gae_users.get_current_user()
- if not user or not user.user_id():
- return
- settings = UserSettings.get_or_insert(user.user_id(), user=user)
- if not guider_name in settings.seen_guiders:
- settings.seen_guiders.append(guider_name)
- settings.put()
-
- @staticmethod
- def has_seen_version(version):
- user = gae_users.get_current_user()
- if not user or not user.user_id():
- return True # don't bother displaying "new version available" to non-authenticated users
- settings = UserSettings.get_or_insert(user.user_id(), user=user)
- if not settings.seen_version:
- settings.seen_version = [0, 0, 0]
- settings.put()
- return settings.seen_version >= version
-
- @staticmethod
- def mark_version_as_seen(version):
- user = gae_users.get_current_user()
- if not user or not user.user_id():
- return
- settings = UserSettings.get_or_insert(user.user_id(), user=user)
- settings.seen_version = version
- settings.put()
-
- @staticmethod
- def get_locale():
- user = gae_users.get_current_user()
- if not user or not user.user_id():
- return None
- settings = UserSettings.get_or_insert(user.user_id(), user=user)
- return settings.locale
-
- @staticmethod
- def set_locale(locale):
- user = gae_users.get_current_user()
- if not user or not user.user_id():
- return
- settings = UserSettings.get_or_insert(user.user_id(), user=user)
- settings.locale = locale
- settings.put()
-
-class Importer(db.Model):
- page = db.ReferenceProperty(Page)
- urls = db.StringListProperty()
- style = db.TextProperty()
- errors = db.StringListProperty()
-
-class Credential(db.Model):
- name = db.StringProperty()
- user = db.StringProperty(default='')
- passwd = db.StringProperty(default='')
- api_key = db.StringProperty(default='')
+from logging import Formatter, Filter, StreamHandler
+import scss
+import settings
+import json
+from datetime import datetime
+from StringIO import StringIO
+from google.appengine.ext import db
+from google.appengine.api import users as gae_users
+from google.appengine.api import taskqueue
+from google.appengine.api import channel as gae_channels
+from google.appengine.api import files
+from google.appengine.api.datastore_errors import BadKeyError
+from flask import abort, url_for, render_template
+
+def dt_handler(obj):
+ if isinstance(obj, datetime):
+ return obj.isoformat() + 'Z'
+ return None
+
+class ExceptionFilter(Filter):
+ def filter(self, record):
+ if record.getMessage().lower().startswith('exception'):
+ return 0
+ return 1
+
+# Old and dead...
+class UserGroup(db.Model):
+ name = db.StringProperty(required=True)
+ users = db.ListProperty(gae_users.User)
+ admins = db.ListProperty(gae_users.User)
+
+class SchemaVersion(db.Model):
+ version = db.IntegerProperty(required=True, default=0)
+
+class Site(db.Model):
+ name = db.StringProperty(required=True)
+ owner = db.UserProperty()
+ users = db.ListProperty(gae_users.User)
+ admins = db.ListProperty(gae_users.User)
+ example = db.BooleanProperty(default=False)
+
+ def delete(self):
+ for page in self.page_set.fetch(10000):
+ page.delete()
+ db.delete(self)
+
+ @staticmethod
+ def get_or_404(id):
+ site = Site.get_by_id(id)
+ if not site or gae_users.get_current_user() not in site.users:
+ abort(404)
+ return site
+
+ @staticmethod
+ def get_admin_or_404(id):
+ site = Site.get_by_id(id)
+ if not site or gae_users.get_current_user() not in site.admins:
+ abort(404)
+ return site
+
+class Invitation(db.Model):
+ hash = db.StringProperty(required=True)
+ site = db.ReferenceProperty(Site, required=True)
+ email = db.EmailProperty(required=True)
+ admin = db.BooleanProperty(default=False)
+ inviter = db.UserProperty(required=True)
+ has_been_logged_out = db.BooleanProperty(default=False)
+
+class Page(db.Model):
+ name = db.StringProperty(required=True)
+ url = db.LinkProperty(required=True)
+ site = db.ReferenceProperty(Site)
+ _styles = db.ListProperty(db.Key)
+ channels = db.ListProperty(db.Key)
+ preview_img = db.BlobProperty(required=False, default=None)
+ preview_urls = db.ListProperty(db.Link, default=None) # *additional* preview urls
+ import_state = db.IntegerProperty(default=0)
+ on_cdn = db.BooleanProperty(default=False)
+
+ _style_cache = None
+ def _set_styles(self, styles):
+ self._style_cache = styles
+ self._styles = [style.key() for style in styles]
+ def _get_styles(self):
+ if not self._style_cache:
+ self._style_cache = [Style.get(k) for k in self._styles]
+ return self._style_cache
+ styles = property(_get_styles, _set_styles)
+
+ def delete(self):
+ for key in self.channels:
+ channel = PageChannel.get(key)
+ if channel:
+ channel.send_message({'cmd': 'lock'})
+ channel.delete()
+ for style in self.styles:
+ style.delete()
+ db.delete(self)
+
+ def clean_channels(self):
+ stale = []
+ for key in self.channels:
+ channel = PageChannel.get(key)
+ if not channel or channel.is_stale():
+ stale.append(key)
+ if stale:
+ for key in stale:
+ self.channels.remove(key)
+ channel = PageChannel.get(key)
+ if channel:
+ # If the channel is still here, it's probably stale.
+ # Send 'lock' and remove, so it can't clobber anyone else.
+ channel.send_message({'cmd': 'lock'})
+ channel.delete()
+ self.put()
+
+ def get_channels(self):
+ channels = []
+ stale = []
+ for key in self.channels:
+ channel = PageChannel.get(key)
+ if channel:
+ channels.append(channel)
+ else:
+ stale.append(key)
+ if stale:
+ for key in stale:
+ self.channels.remove(key)
+ self.put()
+ return channels
+
+ def update_locks(self):
+ owner = None
+ channels = self.get_channels()
+ if channels:
+ owner_user = channels[0].user
+ owner = dict(name=owner_user.nickname(), email=owner_user.email())
+ channels[0].send_message(dict(cmd='unlock', user=owner))
+ lock_msg = dict(cmd='lock', user=owner)
+ for channel in channels[1:]:
+ channel.send_message(lock_msg)
+
+ def add_channel(self, channel):
+ self.remove_channel(channel)
+ self.channels.append(channel.key())
+ self.put()
+
+ def add_channel_first(self, channel):
+ self.remove_channel(channel)
+ self.channels.insert(0, channel.key())
+ self.put()
+
+ def remove_channel(self, channel, delete=False):
+ if channel.key() in self.channels:
+ self.channels.remove(channel.key())
+ self.put()
+ if delete:
+ channel.delete()
+
+ def put(self, *args, **kwargs):
+ self._set_styles(self.styles)
+ super(Page, self).put(*args, **kwargs)
+
+ def queue_preview(self):
+ taskqueue.add(queue_name='fetch-preview', url=url_for('tasks.fetch_preview'), params={'page_key': self.key()})
+
+ def queue_upload(self):
+ taskqueue.add(queue_name='upload-css', url=url_for('tasks.upload_style'), params={'page_key': self.key()})
+
+ def queue_refresh(self):
+ self.queue_upload()
+ self.queue_preview()
+
+ def _css(self, preview, compress):
+ css = StringIO()
+ for style in self.styles:
+ rev = style.preview_rev if (preview and style.preview_rev) else style.published_rev
+ if compress:
+ css.write(rev.compressed)
+ else:
+ css.write(scss.Scss().compile('@option compress:no;' + rev.raw))
+ return css.getvalue()
+
+ def compressed_css(self, preview):
+ return self._css(preview, compress=True)
+
+ def uncompressed_css(self, preview):
+ return self._css(preview, compress=False)
+
+ def last_modified(self, preview):
+ max_last_edit = datetime.min
+ for style in self.styles:
+ rev = style.preview_rev if (preview and style.preview_rev) else style.published_rev
+ max_last_edit = max(max_last_edit, rev.dt_last_edit)
+ return max_last_edit
+
+ def styles_json(self):
+ # NOTE: It is okay to return an array here because we only display this
+ # to users via editor.html. If we ever return this directly as the
+ # response, we'll want to wrap it to avoid the exploit described at
+ # http://haacked.com/archive/2009/06/25/json-hijacking.aspx
+ styles_obj = [style.json_obj() for style in self.styles]
+ return json.dumps(styles_obj, default=dt_handler, sort_keys=True, indent=4*' ' if settings.debug else None)
+
+ def upload_to_cdn(self):
+ if not settings.use_google_cloud_storage:
+ return
+ path = files.gs.create('/gs/%s/%s.css' % (settings.google_bucket, str(self.key())), mime_type='text/css', acl='public-read', cache_control='private,max-age=300')
+ try:
+ fd = files.open(path, 'a')
+ fd.write(self.compressed_css(False).encode('utf-8'))
+ self.on_cdn = True
+ self.save()
+ except Exception:
+ self.on_cdn = False
+ self.save()
+ raise
+ finally:
+ fd.close()
+ files.finalize(path)
+
+ @staticmethod
+ def get_or_404(key):
+ page = None
+ if isinstance(key, int) or (isinstance(key, basestring) and key.isdigit()):
+ page = Page.get_by_id(int(key))
+ else:
+ try:
+ key_obj = db.Key(key)
+ except BadKeyError:
+ abort(404)
+ if(key_obj.kind() == 'Style'):
+ page = Page.gql('WHERE _styles=:1', key_obj).get()
+ else:
+ page = Page.get(key)
+ if not page:
+ abort(404)
+ return page
+
+ @staticmethod
+ def get_edit_or_404(page_id):
+ page = Page.get_or_404(page_id)
+ if gae_users.get_current_user() not in page.site.users:
+ abort(404)
+ return page
+
+ @staticmethod
+ def get_admin_or_404(page_id):
+ page = Page.get_or_404(page_id)
+ site = page.site
+ if not site or gae_users.get_current_user() not in site.admins:
+ abort(404)
+ return page
+
+ @staticmethod
+ def new_page(site, name, url):
+ '''
+ Do all the work in adding a new page to a site.
+ '''
+ style = Style(name = name, site = site)
+ style.put()
+ first_rev = StyleRevision(parent=style)
+ first_rev.raw = render_template('first_run.css')
+ first_rev.put()
+ style.published_rev = first_rev
+ style.put()
+ page = Page(
+ name = name,
+ url = url,
+ site = site,
+ _styles = [style.key()]
+ )
+ page.put()
+ page.queue_refresh()
+ return page
+
+class StyleRevision(db.Model):
+ # parent = Style
+ rev = db.IntegerProperty(required=True, default=0)
+ dt_created = db.DateTimeProperty(auto_now_add=True)
+ dt_last_edit = db.DateTimeProperty(auto_now=True)
+ raw = db.TextProperty(required=False, default='')
+ compressed = db.TextProperty(required=False, default=None)
+ # Old and dead...
+ css = db.TextProperty(required=False)
+ _cached = db.TextProperty(required=False)
+
+ def update(self, raw):
+ self.raw = raw
+ log = StringIO()
+ handler = StreamHandler(log)
+ handler.addFilter(ExceptionFilter())
+ handler.setFormatter(Formatter('<span class="level">%(levelname)s</span>: <span class="message">%(message)s</span><br />'))
+ scss.log.addHandler(handler)
+ self.compressed = scss.Scss().compile(self.raw)
+ scss.log.removeHandler(handler)
+ handler.flush()
+ self.put()
+ return log.getvalue()
+
+class Style(db.Model):
+ site = db.ReferenceProperty(Site)
+ name = db.StringProperty(required=True)
+ published_rev = db.ReferenceProperty(StyleRevision, default=None, collection_name='style_published')
+ preview_rev = db.ReferenceProperty(StyleRevision, default=None, collection_name='style_preview')
+
+ # Old and dead
+ user_group = db.ReferenceProperty(UserGroup)
+ url = db.LinkProperty(required=False)
+
+ def delete(self, *args, **kwargs):
+ revisions = StyleRevision.all(keys_only=True).ancestor(self).fetch(10000)
+ db.delete(revisions)
+ db.delete(self)
+
+ def json_obj(self):
+ if self.preview_rev:
+ preview_rev = self.preview_rev
+ else:
+ if not self.published_rev:
+ rev = StyleRevision(parent=self)
+ rev.put()
+ self.published_rev = rev
+ self.put()
+ preview_rev = self.published_rev
+ return {
+ 'id': self.key().id(),
+ 'name': self.name,
+ 'preview_scss': preview_rev.raw,
+ 'preview_dt_last_edit': preview_rev.dt_last_edit,
+ 'published_scss': self.published_rev.raw,
+ 'published_dt_last_edit': self.published_rev.dt_last_edit,
+ }
+
+ @staticmethod
+ def get_or_404(style_id):
+ if isinstance(style_id, basestring) and not style_id.isdigit():
+ style = Style.get(style_id)
+ else:
+ style = Style.get_by_id(style_id)
+ if not style:
+ abort(404)
+ return style
+
+ @staticmethod
+ def get_edit_or_404(style_id):
+ style = Style.get_or_404(style_id)
+ if gae_users.get_current_user() not in style.site.users:
+ abort(404)
+ return style
+
+ @staticmethod
+ def get_admin_or_404(style_id):
+ style = Style.get_or_404(style_id)
+ site = style.site
+ if not site or gae_users.get_current_user() not in site.admins:
+ abort(404)
+ return style
+
+class PageChannel(db.Model):
+ user = db.UserProperty(required=True)
+ page = db.ReferenceProperty(Page, required=True)
+ token = db.StringProperty(required=True)
+ client_id = db.StringProperty(required=True)
+ dt_connected = db.DateTimeProperty(auto_now_add=True)
+ dt_last_update = db.DateTimeProperty(auto_now_add=True)
+
+ def is_stale(self):
+ return (datetime.utcnow() - self.dt_last_update).seconds > 3600
+
+ def send_message(self, message):
+ if not isinstance(message, basestring):
+ message = json.dumps(message, default=dt_handler, sort_keys=True, indent=4*' ' if settings.debug else None)
+ gae_channels.send_message(self.client_id, message)
+
+ @staticmethod
+ def get_or_404(token=None, client_id=None):
+ channel = None
+ if token:
+ channel = PageChannel.gql('WHERE token=:1', token).get()
+ elif client_id:
+ channel = PageChannel.gql('WHERE client_id=:1', client_id).get()
+ if not channel:
+ abort(404)
+ return channel
+
+class UserSettings(db.Model):
+ user = db.UserProperty(required=True)
+ seen_example = db.BooleanProperty(default=False)
+ seen_guiders = db.StringListProperty()
+ # the last version (list of ints) this person has viewed the release notes for
+ seen_version = db.ListProperty(int, default=None)
+ locale = db.StringProperty(default=None)
+ chimped = db.BooleanProperty(default=False)
+
+ @staticmethod
+ def has_seen_example():
+ user = gae_users.get_current_user()
+ if not user or not user.user_id():
+ raise Exception("Logged in user expected")
+ settings = UserSettings.get_or_insert(user.user_id(), user=user)
+ return settings.seen_example
+
+ @staticmethod
+ def mark_example_as_seen():
+ user = gae_users.get_current_user()
+ if not user or not user.user_id():
+ raise Exception("Logged in user expected")
+ settings = UserSettings.get_or_insert(user.user_id(), user=user)
+ settings.seen_example = True
+ settings.put()
+
+ @staticmethod
+ def show_guider(guider_name):
+ user = gae_users.get_current_user()
+ if not user or not user.user_id():
+ return False
+ settings = UserSettings.get_or_insert(user.user_id(), user=user)
+ return (guider_name not in settings.seen_guiders)
+
+ @staticmethod
+ def mark_guider_as_seen(guider_name):
+ user = gae_users.get_current_user()
+ if not user or not user.user_id():
+ return
+ settings = UserSettings.get_or_insert(user.user_id(), user=user)
+ if not guider_name in settings.seen_guiders:
+ settings.seen_guiders.append(guider_name)
+ settings.put()
+
+ @staticmethod
+ def has_seen_version(version):
+ user = gae_users.get_current_user()
+ if not user or not user.user_id():
+ return True # don't bother displaying "new version available" to non-authenticated users
+ settings = UserSettings.get_or_insert(user.user_id(), user=user)
+ if not settings.seen_version:
+ settings.seen_version = [0, 0, 0]
+ settings.put()
+ return settings.seen_version >= version
+
+ @staticmethod
+ def mark_version_as_seen(version):
+ user = gae_users.get_current_user()
+ if not user or not user.user_id():
+ return
+ settings = UserSettings.get_or_insert(user.user_id(), user=user)
+ settings.seen_version = version
+ settings.put()
+
+ @staticmethod
+ def get_locale():
+ user = gae_users.get_current_user()
+ if not user or not user.user_id():
+ return None
+ settings = UserSettings.get_or_insert(user.user_id(), user=user)
+ return settings.locale
+
+ @staticmethod
+ def set_locale(locale):
+ user = gae_users.get_current_user()
+ if not user or not user.user_id():
+ return
+ settings = UserSettings.get_or_insert(user.user_id(), user=user)
+ settings.locale = locale
+ settings.put()
+
+class Importer(db.Model):
+ page = db.ReferenceProperty(Page)
+ urls = db.StringListProperty()
+ style = db.TextProperty()
+ errors = db.StringListProperty()
+
+class Credential(db.Model):
+ name = db.StringProperty()
+ user = db.StringProperty(default='')
+ passwd = db.StringProperty(default='')
+ api_key = db.StringProperty(default='')
170 app/rpc.py
View
@@ -1,85 +1,85 @@
-import logging
-from datetime import datetime
-import json
-from google.appengine.api import memcache
-from flask import Module, request, abort, jsonify
-from flaskext.csrf import csrf_exempt
-from models import Page, PageChannel, Style, StyleRevision
-from decorators import requires_auth, as_json
-
-rpc = Module(__name__, 'rpc')
-
-@rpc.route('/page/<int:page_id>/rpc', methods=['POST'])
-@requires_auth
-@as_json
-def page_rpc(page_id):
- page = Page.get_edit_or_404(page_id)
- try:
- message = json.loads(request.form.get('message', ''))
- except Exception:
- abort(400)
- data = message.get('data', None)
- token = message.get('from', None)
- if not token or not data:
- logging.warn('RPC received no token or data.')
- abort(400)
- cmd = data.get('cmd', None)
- if not cmd:
- logging.warn('RPC received no cmd.')
- abort(400)
-
- channel = PageChannel.gql('WHERE token=:1', token).get()
- if not channel:
- # We've timed out the channel. User should refresh the page.
- logging.debug('Could not find token: %s', token)
- return dict(cmd='refresh')
- channel.dt_last_update = datetime.utcnow()
- channel.put()
-
- # Commands
- if cmd == 'open':
- page.add_channel(channel)
- page.update_locks()
- return 'OK'
- elif cmd == 'claimLock':
- page.clean_channels()
- page.add_channel_first(channel)
- page.update_locks()
- return 'OK'
- elif cmd == 'save':
- style_id = data.get('style_id', '')
- style = Style.get_edit_or_404(style_id)
- if not style.preview_rev:
- preview_rev = StyleRevision(parent=style, rev=style.published_rev.rev + 1)
- preview_rev.put()
- style.preview_rev = preview_rev
- style.put()
- log = style.preview_rev.update(data.get('scss', ''))
- publish = data.get('fPublish', False)
- preview = not publish
- if publish:
- style.published_rev = style.preview_rev
- style.preview_rev = None
- style.put()
- page_key = str(page.key())
- memcache.delete(page_key + '-css')
- memcache.delete(page_key + '-css-etag')
- page.queue_refresh()
- return jsonify({'css': page.compressed_css(preview), 'log': log})
- else:
- logging.warn('Got a bad command: %s', cmd)
- abort(400) # Bad cmd
-
-@rpc.route('/_ah/channel/<presence>/', methods=['POST'])
-@csrf_exempt
-def _channel_presence(presence):
- client_id = request.form.get('from', '')
- channel = PageChannel.get_or_404(client_id=client_id)
- page = channel.page
- if presence == 'connected':
- page.add_channel(channel)
- elif presence == 'disconnected':
- page.remove_channel(channel, True)
- page.update_locks()
- return 'OK'
-
+import logging
+from datetime import datetime
+import json
+from google.appengine.api import memcache
+from flask import Module, request, abort, jsonify
+from flaskext.csrf import csrf_exempt
+from models import Page, PageChannel, Style, StyleRevision
+from decorators import requires_auth, as_json
+
+rpc = Module(__name__, 'rpc')
+
+@rpc.route('/page/<int:page_id>/rpc', methods=['POST'])
+@requires_auth
+@as_json
+def page_rpc(page_id):
+ page = Page.get_edit_or_404(page_id)
+ try:
+ message = json.loads(request.form.get('message', ''))
+ except Exception:
+ abort(400)
+ data = message.get('data', None)
+ token = message.get('from', None)
+ if not token or not data:
+ logging.warn('RPC received no token or data.')
+ abort(400)
+ cmd = data.get('cmd', None)
+ if not cmd:
+ logging.warn('RPC received no cmd.')
+ abort(400)
+
+ channel = PageChannel.gql('WHERE token=:1', token).get()
+ if not channel:
+ # We've timed out the channel. User should refresh the page.
+ logging.debug('Could not find token: %s', token)
+ return dict(cmd='refresh')
+ channel.dt_last_update = datetime.utcnow()
+ channel.put()
+
+ # Commands
+ if cmd == 'open':
+ page.add_channel(channel)
+ page.update_locks()
+ return 'OK'
+ elif cmd == 'claimLock':
+ page.clean_channels()
+ page.add_channel_first(channel)
+ page.update_locks()
+ return 'OK'
+ elif cmd == 'save':
+ style_id = data.get('style_id', '')
+ style = Style.get_edit_or_404(style_id)
+ if not style.preview_rev:
+ preview_rev = StyleRevision(parent=style, rev=style.published_rev.rev + 1)
+ preview_rev.put()
+ style.preview_rev = preview_rev
+ style.put()
+ log = style.preview_rev.update(data.get('scss', ''))
+ publish = data.get('fPublish', False)
+ preview = not publish
+ if publish:
+ style.published_rev = style.preview_rev
+ style.preview_rev = None
+ style.put()
+ page_key = str(page.key())
+ memcache.delete(page_key + '-css')
+ memcache.delete(page_key + '-css-etag')
+ page.queue_refresh()
+ return jsonify({'css': page.compressed_css(preview), 'log': log})
+ else:
+ logging.warn('Got a bad command: %s', cmd)
+ abort(400) # Bad cmd
+
+@rpc.route('/_ah/channel/<presence>/', methods=['POST'])
+@csrf_exempt
+def _channel_presence(presence):
+ client_id = request.form.get('from', '')
+ channel = PageChannel.get_or_404(client_id=client_id)
+ page = channel.page
+ if presence == 'connected':
+ page.add_channel(channel)
+ elif presence == 'disconnected':
+ page.remove_channel(channel, True)
+ page.update_locks()
+ return 'OK'
+
64 bootstrap.py
View
@@ -1,32 +1,32 @@
-#!/usr/bin/env python
-
-"""Google App Engine uses this file to run your Flask application."""
-
-import os
-import settings
-from utils import adjust_sys_path
-
-adjust_sys_path()
-if settings.debug:
- adjust_sys_path('ziplibs')
- # Enable ctypes for Jinja debugging
- from google.appengine.tools.dev_appserver import HardenedModulesHook
- HardenedModulesHook._WHITE_LIST_C_MODULES += ['_ctypes', 'gestalt']
-else:
- adjust_sys_path(os.path.join('ziplibs.zip', 'ziplibs'))
-
-from app import create_app
-from werkzeug_debugger_appengine import get_debugged_app
-from flaskext.csrf import csrf
-from gae_mini_profiler import profiler, config as profiler_config
-
-profiler_config.enabled_profiler_emails = settings.admin_emails
-
-app = create_app()
-csrf(app)
-# If we're on the local server, let's enable Flask debugging.
-# For more information: http://goo.gl/RNofH
-if settings.debug:
- app.debug = True
- app = get_debugged_app(app)
-app = profiler.ProfilerWSGIMiddleware(app)
+#!/usr/bin/env python
+
+"""Google App Engine uses this file to run your Flask application."""
+
+import os
+import settings
+from utils import adjust_sys_path
+
+adjust_sys_path()
+if settings.debug:
+ adjust_sys_path('ziplibs')
+ # Enable ctypes for Jinja debugging
+ from google.appengine.tools.dev_appserver import HardenedModulesHook
+ HardenedModulesHook._WHITE_LIST_C_MODULES += ['_ctypes', 'gestalt']
+else:
+ adjust_sys_path(os.path.join('ziplibs.zip', 'ziplibs'))
+
+from app import create_app
+from werkzeug_debugger_appengine import get_debugged_app
+from flaskext.csrf import csrf
+from gae_mini_profiler import profiler, config as profiler_config
+
+profiler_config.enabled_profiler_emails = settings.admin_emails
+
+app = create_app()
+csrf(app)
+# If we're on the local server, let's enable Flask debugging.
+# For more information: http://goo.gl/RNofH
+if settings.debug:
+ app.debug = True
+ app = get_debugged_app(app)
+app = profiler.ProfilerWSGIMiddleware(app)
320 compress.py
View
@@ -1,160 +1,160 @@
-import os
-import shutil
-import subprocess
-import sys
-import md5
-import re
-
-sys.path.append(os.path.abspath("."))
-from js_css_packages import packages
-
-COMBINED_FILENAME = "combined"
-COMPRESSED_FILENAME = "compressed"
-PACKAGE_SUFFIX = "-package"
-HASHED_FILENAME_PREFIX = "hashed-"
-PATH_PACKAGES = "js_css_packages/packages.py"
-PATH_PACKAGES_TEMP = "js_css_packages/packages.compresstemp.py"
-
-def revert_js_css_hashes(base_path=None):
- path = PATH_PACKAGES
- if base_path:
- path = os.path.join(base_path, PATH_PACKAGES)
- root = os.path.abspath(os.path.dirname(__file__))
- if os.path.exists(os.path.join(root, '.hg')):
- print "Reverting %s" % path
- popen_results(['hg', 'revert', '--no-backup', path])
- elif os.path.exists(os.path.join(root, '.git')):
- print "Reverting %s" % path
- popen_results(['git', 'checkout', '--', path])
-
-
-def compress_all_javascript(base_path):
- path = os.path.join(base_path, 'static', 'js')
- dict_packages = packages.javascript
- compress_all_packages(base_path, path, dict_packages, ".js")
-
-def compress_all_stylesheets(base_path):
- path = os.path.join(base_path, 'static', 'css')
- dict_packages = packages.stylesheets
- compress_all_packages(base_path, path, dict_packages, ".css")
-
-# Combine all .js\.css files in all "-package" suffixed directories
-# into a single combined.js\.css file for each package, then
-# minify into a single compressed.js\.css file.
-def compress_all_packages(base_path, path, dict_packages, suffix):
- if not os.path.exists(path):
- raise Exception("Path does not exist: %s" % path)
-
- for package_name in dict_packages:
- package = dict_packages[package_name]
-
- dir_name = "%s-package" % package_name
- package_path = os.path.join(path, dir_name)
-
- compress_package(base_path, package_name, package_path, package["files"], suffix)
-
-def compress_package(base_path, name, path, files, suffix):
- if not os.path.exists(path):
- raise Exception("Path does not exist: %s" % path)
-
- remove_working_files(path, suffix)
-
- path_combined = combine_package(path, files, suffix)
- path_compressed = minify_package(path, path_combined, suffix)
- path_hashed = hash_package(base_path, name, path, path_compressed, suffix)
-
- if not os.path.exists(path_hashed):
- raise Exception("Did not successfully compress and hash: %s" % path)
-
- return path_hashed
-
-# Remove previous combined.js\.css and compress.js\.css files
-def remove_working_files(path, suffix):
- filenames = os.listdir(path)
- for filename in filenames:
- if filename.endswith(COMBINED_FILENAME + suffix) \
- or filename.endswith(COMPRESSED_FILENAME + suffix) \
- or filename.startswith(HASHED_FILENAME_PREFIX):
- os.remove(os.path.join(path, filename))
-
-# Use YUICompressor to minify the combined file
-def minify_package(path, path_combined, suffix):
- path_compressed = os.path.join(path, COMPRESSED_FILENAME + suffix)
- path_compressor = os.path.join(os.path.dirname(__file__), "yuicompressor-2.4.6.jar")
-
- print "Compressing %s into %s" % (path_combined, path_compressed)
- print popen_results(["java", "-jar", path_compressor, "--charset", "utf-8", path_combined, "-o", path_compressed])
-
- if not os.path.exists(path_compressed):
- raise Exception("Unable to YUICompress: %s" % path_combined)
-
- return path_compressed
-
-def hash_package(base_path, name, path, path_compressed, suffix):
- f = open(path_compressed, "r")
- content = f.read()
- f.close()
-
- hash_sig = md5.new(content).hexdigest()
- path_hashed = os.path.join(path, "hashed-%s%s" % (hash_sig, suffix))
-
- print "Copying %s into %s" % (path_compressed, path_hashed)
- shutil.copyfile(path_compressed, path_hashed)
-
- if not os.path.exists(path_hashed):
- raise Exception("Unable to copy to hashed file: %s" % path_compressed)
-
- insert_hash_sig(base_path, name, hash_sig, suffix)
-
- return path_hashed
-
-def insert_hash_sig(base_path, name, hash_sig, suffix):
- package_file_path = os.path.join(base_path, PATH_PACKAGES)
- temp_file_path = os.path.join(base_path, PATH_PACKAGES_TEMP)
- print "Inserting %s sig (%s) into %s\n" % (name, hash_sig, package_file_path)
-
- f = open(package_file_path, "r")
- content = f.read()
- f.close()
-
- re_search = "@%s@%s" % (name, suffix)
- re_replace = "%s%s" % (hash_sig, suffix)
- hashed_content = re.sub(re_search, re_replace, content)
-
- if content == hashed_content:
- raise Exception("Hash sig insertion failed: %s" % name)
-
- f = open(temp_file_path, "w")
- f.write(hashed_content)
- f.close()
-
- shutil.move(temp_file_path, package_file_path)
-
-# Combine all files into a single combined.js\.css
-def combine_package(path, files, suffix):
- path_combined = os.path.join(path, COMBINED_FILENAME + suffix)
-
- print "Building %s" % path_combined
-
- content = []
- for static_filename in files:
- path_static = os.path.join(path, static_filename)
- print " ...adding %s" % path_static
- f = open(path_static, 'r')
- content.append(f.read())
- f.close()
-
- if os.path.exists(path_combined):
- raise Exception("File about to be compressed already exists: %s" % path_combined)
-
- f = open(path_combined, "w")
- separator = "\n" if suffix.endswith(".css") else ";\n"
- f.write(separator.join(content))
- f.close()
-
- return path_combined
-
-def popen_results(args):
- proc = subprocess.Popen(args, stdout=subprocess.PIPE)
- return proc.communicate()[0]
-
+import os
+import shutil
+import subprocess
+import sys
+import md5
+import re
+
+sys.path.append(os.path.abspath("."))
+from js_css_packages import packages
+
+COMBINED_FILENAME = "combined"
+COMPRESSED_FILENAME = "compressed"
+PACKAGE_SUFFIX = "-package"
+HASHED_FILENAME_PREFIX = "hashed-"
+PATH_PACKAGES = "js_css_packages/packages.py"
+PATH_PACKAGES_TEMP = "js_css_packages/packages.compresstemp.py"
+
+def revert_js_css_hashes(base_path=None):
+ path = PATH_PACKAGES
+ if base_path:
+ path = os.path.join(base_path, PATH_PACKAGES)
+ root = os.path.abspath(os.path.dirname(__file__))
+ if os.path.exists(os.path.join(root, '.hg')):
+ print "Reverting %s" % path
+ popen_results(['hg', 'revert', '--no-backup', path])
+ elif os.path.exists(os.path.join(root, '.git')):
+ print "Reverting %s" % path
+ popen_results(['git', 'checkout', '--', path])
+
+
+def compress_all_javascript(base_path):
+ path = os.path.join(base_path, 'static', 'js')
+ dict_packages = packages.javascript
+ compress_all_packages(base_path, path, dict_packages, ".js")
+
+def compress_all_stylesheets(base_path):
+ path = os.path.join(base_path, 'static', 'css')
+ dict_packages = packages.stylesheets
+ compress_all_packages(base_path, path, dict_packages, ".css")
+
+# Combine all .js\.css files in all "-package" suffixed directories
+# into a single combined.js\.css file for each package, then
+# minify into a single compressed.js\.css file.
+def compress_all_packages(base_path, path, dict_packages, suffix):
+ if not os.path.exists(path):
+ raise Exception("Path does not exist: %s" % path)
+
+ for package_name in dict_packages:
+ package = dict_packages[package_name]
+
+ dir_name = "%s-package" % package_name
+ package_path = os.path.join(path, dir_name)
+
+ compress_package(base_path, package_name, package_path, package["files"], suffix)
+
+def compress_package(base_path, name, path, files, suffix):
+ if not os.path.exists(path):
+ raise Exception("Path does not exist: %s" % path)
+
+ remove_working_files(path, suffix)
+
+ path_combined = combine_package(path, files, suffix)
+ path_compressed = minify_package(path, path_combined, suffix)
+ path_hashed = hash_package(base_path, name, path, path_compressed, suffix)
+
+ if not os.path.exists(path_hashed):
+ raise Exception("Did not successfully compress and hash: %s" % path)
+
+ return path_hashed
+
+# Remove previous combined.js\.css and compress.js\.css files
+def remove_working_files(path, suffix):
+ filenames = os.listdir(path)
+ for filename in filenames:
+ if filename.endswith(COMBINED_FILENAME + suffix) \
+ or filename.endswith(COMPRESSED_FILENAME + suffix) \
+ or filename.startswith(HASHED_FILENAME_PREFIX):
+ os.remove(os.path.join(path, filename))
+
+# Use YUICompressor to minify the combined file
+def minify_package(path, path_combined, suffix):
+ path_compressed = os.path.join(path, COMPRESSED_FILENAME + suffix)
+ path_compressor = os.path.join(os.path.dirname(__file__), "yuicompressor-2.4.6.jar")
+
+ print "Compressing %s into %s" % (path_combined, path_compressed)
+ print popen_results(["java", "-jar", path_compressor, "--charset", "utf-8", path_combined, "-o", path_compressed])
+
+ if not os.path.exists(path_compressed):
+ raise Exception("Unable to YUICompress: %s" % path_combined)
+
+ return path_compressed
+
+def hash_package(base_path, name, path, path_compressed, suffix):
+ f = open(path_compressed, "r")
+ content = f.read()
+ f.close()
+
+ hash_sig = md5.new(content).hexdigest()
+ path_hashed = os.path.join(path, "hashed-%s%s" % (hash_sig, suffix))
+
+ print "Copying %s into %s" % (path_compressed, path_hashed)
+ shutil.copyfile(path_compressed, path_hashed)
+
+ if not os.path.exists(path_hashed):
+ raise Exception("Unable to copy to hashed file: %s" % path_compressed)
+
+ insert_hash_sig(base_path, name, hash_sig, suffix)
+
+ return path_hashed
+
+def insert_hash_sig(base_path, name, hash_sig, suffix):
+ package_file_path = os.path.join(base_path, PATH_PACKAGES)
+ temp_file_path = os.path.join(base_path, PATH_PACKAGES_TEMP)
+ print "Inserting %s sig (%s) into %s\n" % (name, hash_sig, package_file_path)
+
+ f = open(package_file_path, "r")
+ content = f.read()
+ f.close()
+
+ re_search = "@%s@%s" % (name, suffix)
+ re_replace = "%s%s" % (hash_sig, suffix)
+ hashed_content = re.sub(re_search, re_replace, content)
+
+ if content == hashed_content:
+ raise Exception("Hash sig insertion failed: %s" % name)
+
+ f = open(temp_file_path, "w")
+ f.write(hashed_content)
+ f.close()
+
+ shutil.move(temp_file_path, package_file_path)
+
+# Combine all files into a single combined.js\.css
+def combine_package(path, files, suffix):
+ path_combined = os.path.join(path, COMBINED_FILENAME + suffix)
+
+ print "Building %s" % path_combined
+
+ content = []
+ for static_filename in files:
+ path_static = os.path.join(path, static_filename)
+ print " ...adding %s" % path_static
+ f = open(path_static, 'r')
+ content.append(f.read())
+ f.close()
+
+ if os.path.exists(path_combined):
+ raise Exception("File about to be compressed already exists: %s" % path_combined)
+
+ f = open(path_combined, "w")
+ separator = "\n" if suffix.endswith(".css") else ";\n"
+ f.write(separator.join(content))
+ f.close()
+
+ return path_combined
+
+def popen_results(args):
+ proc = subprocess.Popen(args, stdout=subprocess.PIPE)
+ return proc.communicate()[0]
+
354 fabfile.py
View
@@ -1,177 +1,177 @@
-from __future__ import with_statement
-
-import functools
-import os
-import sys
-import webbrowser
-import zipfile
-from fabric.api import env, local, prompt
-
-import compress
-
-#Some environment information to customize
-if os.name == 'posix':
- APPENGINE_PATH = '/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine'
- PYTHON = '/usr/bin/python2.7'
-else:
- APPENGINE_PATH = r'C:\Program Files (x86)\Google\google_appengine'
- PYTHON = r'C:\Python27\python.exe'
-APPENGINE_APP_CFG = os.path.join(APPENGINE_PATH, 'appcfg.py')
-print APPENGINE_APP_CFG
-
-env.gae_src = os.path.dirname(__file__)
-
-#default values
-env.dryrun = False
-
-EXTRA_PATHS = [
- APPENGINE_PATH,
- os.path.join(APPENGINE_PATH, 'lib', 'antlr3'),
- os.path.join(APPENGINE_PATH, 'lib', 'django'),
- os.path.join(APPENGINE_PATH, 'lib', 'fancy_urllib'),
- os.path.join(APPENGINE_PATH, 'lib', 'ipaddr'),
- os.path.join(APPENGINE_PATH, 'lib', 'webob'),
- os.path.join(APPENGINE_PATH, 'lib', 'yaml', 'lib'),
-]
-
-sys.path = EXTRA_PATHS + sys.path
-
-from google.appengine.api import appinfo
-
-def _include_appcfg(func):
- '''Decorator that ensures the current Fabric env has a GAE app.yaml config
- attached to it.'''
- @functools.wraps(func)
- def decorated_func(*args, **kwargs):
- if not hasattr(env, 'app'):
- appcfg = appinfo.LoadSingleAppInfo(open(os.path.join(env.gae_src, 'app.yaml')))
- env.app = appcfg
- return func(*args, **kwargs)
- return decorated_func
-
-def dryrun():
- env.dryrun = True
-
-@_include_appcfg
-def deploy():
- env.deploy_path = env.gae_src
-
- compress_js(env.deploy_path)
- compress_css(env.deploy_path)
- ziplibs(env.deploy_path)
- _clean_babel()
-
- if not env.dryrun:
- print 'Deploying %s' % env.app.version
- local('%s "%s" -A %s -V %s --oauth2 update %s' % (PYTHON, APPENGINE_APP_CFG, env.app.application, env.app.version, env.deploy_path), capture=False)
- webbrowser.open('https://%s.appspot.com/' % env.app.application)
- else:
- print 'This is where we\'d actually deploy to App Engine, but this is a dryrun so we skip that part.'
-
- clean_packages(env.deploy_path)
-
-def compress_js(path=None):
- if not path: path = env.gae_src
- print 'Compressing JavaScript'
- compress.compress_all_javascript(path)
-
-def compress_css(path=None):
- if not path: path = env.gae_src
- print 'Compressing stylesheets'
- compress.compress_all_stylesheets(path)
-
-def clean_packages(base_path=None):
- compress.revert_js_css_hashes(base_path)
-
-def update_translations():
- local('pybabel extract -F babel.cfg -o app/messages.pot --project=WebPutty --copyright-holder="Fog Creek Software, Inc." --msgid-bugs-address=customer-service@fogcreek.com app')
- local('pybabel update -i app/messages.pot -d app/translations')
- _update_piglatin()
- _update_english()
- local('pybabel compile -d app/translations')
-
-def add_locale():
- prompt('Locale:', 'locale')
- if env.locale:
- local('pybabel init -i app/messages.pot -d app/translations -l %s' % env.locale)
- else:
- print 'You must enter a locale.'
-
-def _update_english():
- from babel.messages.pofile import read_po, write_po
- from babel.messages.catalog import Catalog
- with open('app/messages.pot', 'r') as f:
- template = read_po(f)
- catalog = Catalog()
- for message in template:
- catalog.add(message.id, message.id, locations=message.locations)
- with open('app/translations/en/LC_MESSAGES/messages.po', 'w') as f:
- write_po(f, catalog)
- with open('app/translations/en_US/LC_MESSAGES/messages.po', 'w') as f:
- write_po(f, catalog)
-
-def _update_piglatin():
- from babel.messages.pofile import read_po, write_po
- from babel.messages.catalog import Catalog
- with open('app/messages.pot', 'r') as f:
- template = read_po(f)
- catalog = Catalog()
- for message in template:
- trans = ' '.join([_piglatin_translate(w) for w in message.id.split(' ')])
- catalog.add(message.id, trans, locations=message.locations)
- with open('app/translations/aa/LC_MESSAGES/messages.po', 'w') as f:
- write_po(f, catalog)
-
-def _piglatin_translate(word):
- """ convert one word into pig latin """
- word = unicode(word)
- m = len(word)
- vowels = "a", "e", "i", "o", "u", "y"
- if m < 3 or word == "the" or word.startswith('%'): # short words are not converted
- return word
- else:
- for i in vowels:
- if word.find(i) < m and word.find(i) != -1:
- m = word.find(i)
- if m==0:
- return word + u"way"
- else:
- return word[m:] + word[:m] + u"ay"
-
-def _clean_babel():
- from settings import available_locales
- locale_codes = [t[0] for t in available_locales]
- if not env.deploy_path:
- return
- print 'Cleaning up unused localedata.'
- localedata_dir = os.path.join(env.deploy_path, 'libs', 'babel', 'localedata')
- for name in os.listdir(localedata_dir):
- remove = True
- for code in locale_codes:
- if name.startswith(code):
- remove = False
- break
- if remove:
- os.unlink(os.path.join(localedata_dir, name))
-
-def ziplibs(root_dir=None):
- if not root_dir:
- root_dir = os.path.abspath(os.path.dirname(__file__))
- to_zip = os.path.join(root_dir, 'ziplibs')
- print 'Cleaning %s of pyc files.' % to_zip
- def rem_ext(ext, dirname, names):
- for name in names:
- if name.endswith(ext):
- os.unlink(os.path.join(dirname, name))
- os.path.walk(to_zip, rem_ext, '.pyc')
- print 'Zipping ziplibs.'
- zip_file = zipfile.ZipFile(to_zip + '.zip', 'w', compression=zipfile.ZIP_DEFLATED)
- def add_file(args, dir_name, names):
- zip_file, common_base = args
- for name in names:
- zip_file.write(os.path.join(dir_name, name), os.path.join(dir_name[len(common_base):], name))
- os.path.walk(to_zip, add_file, (zip_file, os.path.dirname(to_zip)))
- zip_file.close()
-
-def lint():
- local('pylint --rcfile=.pylintrc app')
+from __future__ import with_statement
+
+import functools
+import os
+import sys
+import webbrowser
+import zipfile
+from fabric.api import env, local, prompt
+
+import compress
+
+#Some environment information to customize
+if os.name == 'posix':
+ APPENGINE_PATH = '/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine'
+ PYTHON = '/usr/bin/python2.7'
+else:
+ APPENGINE_PATH = r'C:\Program Files (x86)\Google\google_appengine'
+ PYTHON = r'C:\Python27\python.exe'
+APPENGINE_APP_CFG = os.path.join(APPENGINE_PATH, 'appcfg.py')
+print APPENGINE_APP_CFG
+
+env.gae_src = os.path.dirname(__file__)
+
+#default values
+env.dryrun = False
+
+EXTRA_PATHS = [
+ APPENGINE_PATH,
+ os.path.join(APPENGINE_PATH, 'lib', 'antlr3'),
+ os.path.join(APPENGINE_PATH, 'lib', 'django'),
+ os.path.join(APPENGINE_PATH, 'lib', 'fancy_urllib'),
+ os.path.join(APPENGINE_PATH, 'lib', 'ipaddr'),
+ os.path.join(APPENGINE_PATH, 'lib', 'webob'),
+ os.path.join(APPENGINE_PATH, 'lib', 'yaml', 'lib'),
+]
+
+sys.path = EXTRA_PATHS + sys.path
+
+from google.appengine.api import appinfo
+
+def _include_appcfg(func):
+ '''Decorator that ensures the current Fabric env has a GAE app.yaml config
+ attached to it.'''
+ @functools.wraps(func)
+ def decorated_func(*args, **kwargs):
+ if not hasattr(env, 'app'):
+ appcfg = appinfo.LoadSingleAppInfo(open(os.path.join(env.gae_src, 'app.yaml')))
+ env.app = appcfg
+ return func(*args, **kwargs)
+ return decorated_func
+
+def dryrun():
+ env.dryrun = True
+
+@_include_appcfg
+def deploy():
+ env.deploy_path = env.gae_src
+
+ compress_js(env.deploy_path)
+ compress_css(env.deploy_path)
+ ziplibs(env.deploy_path)
+ _clean_babel()
+
+ if not env.dryrun:
+ print 'Deploying %s' % env.app.version
+ local('%s "%s" -A %s -V %s --oauth2 update %s' % (PYTHON, APPENGINE_APP_CFG, env.app.application, env.app.version, env.deploy_path), capture=False)
+ webbrowser.open('https://%s.appspot.com/' % env.app.application)
+ else:
+ print 'This is where we\'d actually deploy to App Engine, but this is a dryrun so we skip that part.'
+
+ clean_packages(env.deploy_path)
+
+def compress_js(path=None):
+ if not path: path = env.gae_src
+ print 'Compressing JavaScript'
+ compress.compress_all_javascript(path)
+
+def compress_css(path=None):
+ if not path: path = env.gae_src
+ print 'Compressing stylesheets'
+ compress.compress_all_stylesheets(path)
+
+def clean_packages(base_path=None):
+ compress.revert_js_css_hashes(base_path)
+
+def update_translations():
+ local('pybabel extract -F babel.cfg -o app/messages.pot --project=WebPutty --copyright-holder="Fog Creek Software, Inc." --msgid-bugs-address=customer-service@fogcreek.com app')
+ local('pybabel update -i app/messages.pot -d app/translations')
+ _update_piglatin()
+ _update_english()
+ local('pybabel compile -d app/translations')
+
+def add_locale():
+ prompt('Locale:', 'locale')
+ if env.locale:
+ local('pybabel init -i app/messages.pot -d app/translations -l %s' % env.locale)
+ else:
+ print 'You must enter a locale.'
+
+def _update_english():
+ from babel.messages.pofile import read_po, write_po
+ from babel.messages.catalog import Catalog
+ with open('app/messages.pot', 'r') as f:
+ template = read_po(f)
+ catalog = Catalog()
+ for message in template:
+ catalog.add(message.id, message.id, locations=message.locations)
+ with open('app/translations/en/LC_MESSAGES/messages.po', 'w') as f:
+ write_po(f, catalog)
+ with open('app/translations/en_US/LC_MESSAGES/messages.po', 'w') as f:
+ write_po(f, catalog)
+
+def _update_piglatin():
+ from babel.messages.pofile import read_po, write_po
+ from babel.messages.catalog import Catalog
+ with open('app/messages.pot', 'r') as f:
+ template = read_po(f)
+ catalog = Catalog()
+ for message in template:
+ trans = ' '.join([_piglatin_translate(w) for w in message.id.split(' ')])
+ catalog.add(message.id, trans, locations=message.locations)
+ with open('app/translations/aa/LC_MESSAGES/messages.po', 'w') as f:
+ write_po(f, catalog)
+
+def _piglatin_translate(word):
+ """ convert one word into pig latin """
+ word = unicode(word)
+ m = len(word)
+ vowels = "a", "e", "i", "o", "u", "y"
+ if m < 3 or word == "the" or word.startswith('%'): # short words are not converted
+ return word
+ else:
+ for i in vowels:
+ if word.find(i) < m and word.find(i) != -1:
+ m = word.find(i)
+ if m==0:
+ return word + u"way"
+ else:
+ return word[m:] + word[:m] + u"ay"
+
+def _clean_babel():
+ from settings import available_locales
+ locale_codes = [t[0] for t in available_locales]
+ if not env.deploy_path:
+ return
+ print 'Cleaning up unused localedata.'
+ localedata_dir = os.path.join(env.deploy_path, 'libs', 'babel', 'localedata')
+ for name in os.listdir(localedata_dir):
+ remove = True
+ for code in locale_codes:
+ if name.startswith(code):
+ remove = False
+ break
+ if remove:
+ os.unlink(os.path.join(localedata_dir, name))
+
+def ziplibs(root_dir=None):
+ if not root_dir:
+ root_dir = os.path.abspath(os.path.dirname(__file__))
+ to_zip = os.path.join(root_dir, 'ziplibs')
+ print 'Cleaning %s of pyc files.' % to_zip
+ def rem_ext(ext, dirname, names):
+ for name in names:
+ if name.endswith(ext):
+ os.unlink(os.path.join(dirname, name))
+ os.path.walk(to_zip, rem_ext, '.pyc')
+ print 'Zipping ziplibs.'
+ zip_file = zipfile.ZipFile(to_zip + '.zip', 'w', compression=zipfile.ZIP_DEFLATED)
+ def add_file(args, dir_name, names):
+ zip_file, common_base = args
+ for name in names:
+ zip_file.write(os.path.join(dir_name, name), os.path.join(dir_name[len(common_base):], name))
+ os.path.walk(to_zip, add_file, (zip_file, os.path.dirname(to_zip)))
+ zip_file.close()
+
+def lint():
+ local('pylint --rcfile=.pylintrc app')
94 incoming_email_handler.py
View
@@ -1,47 +1,47 @@
-import logging
-import settings
-from textwrap import dedent
-from google.appengine.ext import webapp
-from google.appengine.api import mail
-from google.appengine.ext.webapp.mail_handlers import InboundMailHandler
-
-class ReceiveEmail(InboundMailHandler):
- def receive(self,message):
- headers = dedent("""\
- From: %s
- To: %s
- Subject: %s
- """ % (
- message.sender,
- message.to,
- message.subject,
- ))
-
- log_msg = headers + 'Body:\n'
- for content_type, body in message.bodies():
- log_msg += '%s\n%s\n-------------------------------\n' % (content_type, body.decode())
-
- if settings.log_all_incoming:
- logging.info(log_msg)
-
- email_subject = '%s (from %s)' % (message.subject, message.sender)
-
- email_body = headers
- for content_type, body in message.bodies('text/plain'):
- email_body += body.decode()
-
- email_html = '<p>' + headers.replace('\n', '<br />') + '</p>'
- for content_type, body in message.bodies('text/html'):
- email_html += body.decode()
-
- mail.send_mail(
- sender = settings.incoming_sender_email,
- reply_to = message.sender,
- to = settings.forward_mail_to,
- subject = email_subject,
- body = email_body,
- html = email_html,
- )
- return 'OK'
-
-application = webapp.WSGIApplication([ReceiveEmail.mapping()], debug=True)
+import logging
+import settings
+from textwrap import dedent
+from google.appengine.ext import webapp
+from google.appengine.api import mail
+from google.appengine.ext.webapp.mail_handlers import InboundMailHandler
+
+class ReceiveEmail(InboundMailHandler):
+ def receive(self,message):
+ headers = dedent("""\
+ From: %s
+ To: %s
+ Subject: %s
+ """ % (
+ message.sender,
+ message.to,
+ message.subject,
+ ))
+
+ log_msg = headers + 'Body:\n'
+ for content_type, body in message.bodies():
+ log_msg += '%s\n%s\n-------------------------------\n' % (content_type, body.decode())
+
+ if settings.log_all_incoming:
+ logging.info(log_msg)
+
+ email_subject = '%s (from %s)' % (message.subject, message.sender)
+
+ email_body = headers
+ for content_type, body in message.bodies('text/plain'):
+ email_body += body.decode()
+
+ email_html = '<p>' + headers.replace('\n', '<br />') + '</p>'
+ for content_type, body in message.bodies('text/html'):
+ email_html += body.decode()
+
+ mail.send_mail(
+ sender = settings.incoming_sender_email,
+ reply_to = message.sender,
+ to = settings.forward_mail_to,
+ subject = email_subject,
+ body = email_body,
+ html = email_html,
+ )
+ return 'OK'
+
+application = webapp.WSGIApplication([ReceiveEmail.mapping()], debug=True)
446 libs/livecount/counter.py
View
@@ -1,223 +1,223 @@
-#
-# Copyright 2011 Greg Bayer <greg@gbayer.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-from datetime import datetime, timedelta
-import logging
-import time
-
-from google.appengine.api import memcache
-from google.appengine.api.labs import taskqueue
-from google.appengine.ext import db
-from google.appengine.ext import webapp
-
-
-"""
- Livecount is a memcache-based counter api with asynchronous writes to persist counts to datastore.
-
- Semantics:
- - Write-Behind
- - Read-Through
-"""
-
-class PeriodType(object):
- SECOND = "second"
- MINUTE = "minute"
- HOUR = "hour"
- DAY = "day"
- WEEK = "week"
- MONTH = "month"
- YEAR = "year"
- ALL = "all"
-
- @staticmethod
- def find_scope(period_type, period):
- if period_type == PeriodType.SECOND:
- return str(period)[0:19] # 2011-06-13 18:11:32
- elif period_type == PeriodType.MINUTE:
- return str(period)[0:16] # 2011-06-13 18:11
- elif period_type == PeriodType.HOUR:
- return str(period)[0:13] # 2011-06-13 18
- elif period_type == PeriodType.DAY:
- return str(period)[0:10] # 2011-06-13
- elif period_type == PeriodType.WEEK:
- if not isinstance(period, datetime):
- period = PeriodType.str_to_datetime(period)
- return str(period- timedelta(period.weekday()))[0:10]+"week" # 2011-06-13week; use Monday as marker
- elif period_type == PeriodType.MONTH:
- return str(period)[0:7] # 2011-06
- elif period_type == PeriodType.YEAR:
- return str(period)[0:4] # 2011
- else:
- return "all"
-
- @staticmethod
- def str_to_datetime(datetime_str):
- time_format = "%Y-%m-%d %H:%M:%S"
- return datetime.fromtimestamp(time.mktime(time.strptime(datetime_str.split('.')[0], time_format)))
-
-
-class LivecountCounter(db.Model):
- namespace = db.StringProperty(default="default")
- period_type = db.StringProperty(default=PeriodType.ALL)
- period = db.StringProperty()
- name = db.StringProperty()
- count = db.IntegerProperty()
-
- @staticmethod
- def KeyName(namespace, period_type, period, name):
- scoped_period = PeriodType.find_scope(period_type, period)
- return namespace + ":" + period_type + ":" + scoped_period + ":" + name
-
- @staticmethod
- def PartialKeyName(period_type, period, name):
- scoped_period = PeriodType.find_scope(period_type, period)
- return period_type + ":" + scoped_period + ":" + name
-
-
-def load_and_get_count(name, namespace='default', period_type='all', period=datetime.now()):
- # Try memcache first
- partial_key = LivecountCounter.PartialKeyName(period_type, period, name)
- count = memcache.get(partial_key, namespace=namespace)
-
- # Not in memcache
- if count is None:
- # See if this counter already exists in the datastore
- full_key = LivecountCounter.KeyName(namespace, period_type, period, name)
- record = LivecountCounter.get_by_key_name(full_key)
- # If counter exists in the datastore, but is not currently in memcache, add it
- if record:
- count = record.count
- memcache.add(partial_key, count, namespace=namespace)
-
- return count
-
-
-def load_and_increment_counter(name, period=datetime.now(), period_types=[PeriodType.ALL], namespace='default', delta=1, batch_size=None):
- """
- Setting batch size allows control of how often a writeback worker is created.
- By default, this happens at every increment to ensure maximum durability.
- If there is already a worker waiting to write the value of a counter, another will not be created.
- """
- # Warning: There is a race condition here. If two processes try to load
- # the same value from the datastore, one's update may be lost.
- # TODO: Think more about whether we care about this...
-
- for period_type in period_types:
- current_count = None
-
- result = None
- partial_key = LivecountCounter.PartialKeyName(period_type, period, name)
- if delta >= 0:
- result = memcache.incr(partial_key, delta, namespace=namespace)
- else: # Since increment by negative number is not supported, convert to decrement
- result = memcache.decr(partial_key, -delta, namespace=namespace)
-
- if result is None:
- # See if this counter already exists in the datastore
- full_key = LivecountCounter.KeyName(namespace, period_type, period, name)
- record = LivecountCounter.get_by_key_name(full_key)
- if record:
- # Load last value from datastore
- new_counter_value = record.count + delta
- if new_counter_value < 0: new_counter_value = 0 # To match behavior of memcache.decr(), don't allow negative values
- memcache.add(partial_key, new_counter_value, namespace=namespace)
- if batch_size: current_count = record.count
- else:
- # Start new counter
- memcache.add(partial_key, delta, namespace=namespace)
- if batch_size: current_count = delta
- else:
- if batch_size: current_count = memcache.get(partial_key, namespace=namespace)
-
- # If batch_size is set, only try creating one worker per batch
- if not batch_size or (batch_size and current_count % batch_size == 0):
- if memcache.add(partial_key + '_dirty', delta, namespace=namespace):
- #logging.info("Adding task to taskqueue. counter value = " + str(memcache.get(partial_key, namespace=namespace)))
- taskqueue.add(queue_name='livecount-writebacks', url='/livecount/worker', params={'name': name, 'period': period, 'period_type': period_type, 'namespace': namespace}) # post parameter
-
-
-def load_and_decrement_counter(name, period=datetime.now(), period_types=[PeriodType.ALL], namespace='default', delta=1, batch_size=None):
- load_and_increment_counter(name, period, period_types, namespace, -delta, batch_size)
-
-
-def GetMemcacheStats():
- stats = memcache.get_stats()
- return stats
-
-
-class LivecountCounterWorker(webapp.RequestHandler):
- def post(self):
- namespace = self.request.get('namespace')
- period_type = self.request.get('period_type')
- period = self.request.get('period')
- name = self.request.get('name')
-
- full_key = LivecountCounter.KeyName(namespace, period_type, period, name)
- partial_key = LivecountCounter.PartialKeyName(period_type, period, name)
-
- memcache.delete(partial_key + '_dirty', namespace=namespace)
- cached_count = memcache.get(partial_key, namespace=namespace)
- if cached_count is None:
- logging.error('LivecountCounterWorker: Failure for partial key=%s', partial_key)
- return
-
- # add new row in datastore
- scoped_period = PeriodType.find_scope(period_type, period)
- LivecountCounter(key_name=full_key, namespace=namespace, period_type=period_type, period=scoped_period, name=name, count=cached_count).put()
-
-
-class WritebackAllCountersHandler(webapp.RequestHandler):
- """
- Writes back all counters from memory to the datastore
- """
- def get(self):
- namespace = self.request.get('namespace')
- delete = self.request.get('delete')
-
- logging.info("Writing back all counters from memory to the datastore. Namespace=%s. Delete from memory=%s." % (str(namespace), str(delete)))
- result = False
- while not result:
- result=in_memory_counter.WritebackAllCounters(namespace, delete)
-
- self.response.out.write("Done. WritebackAllCounters succeeded = " + str(result))
-
-
-class ClearEntireCacheHandler(webapp.RequestHandler):
- """
- Clears entire memcache
- """
- def get(self):
- logging.info("Deleting all counters in memcache. Any counts not previously flushed will be lost.")
-
- result = in_memory_counter.ClearEntireCache()
- self.response.out.write("Done. ClearEntireCache succeeded = " + str(result))
-
-
-class RedirectToCounterAdminHandler(webapp.RequestHandler):
- """ For convenience / demo purposes, redirect to counter admin page.
- """
- def get(self):
- self.redirect('/livecount/counter_admin')
-
-
-logging.getLogger().setLevel(logging.DEBUG)
-application = webapp.WSGIApplication([
- ('/livecount/worker', LivecountCounterWorker),
- ('/livecount/writeback_all_counters', WritebackAllCountersHandler),
- ('/livecount/clear_entire_cache', ClearEntireCacheHandler),
- ('/', RedirectToCounterAdminHandler)
- ], debug=True)
+#
+# Copyright 2011 Greg Bayer <greg@gbayer.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+from datetime import datetime, timedelta
+import logging
+import time
+
+from google.appengine.api import memcache
+from google.appengine.api.labs import taskqueue
+from google.appengine.ext import db
+from google.appengine.ext import webapp
+
+
+"""
+ Livecount is a memcache-based counter api with asynchronous writes to persist counts to datastore.
+
+ Semantics:
+ - Write-Behind
+ - Read-Through
+"""
+
+class PeriodType(object):
+ SECOND = "second"
+ MINUTE = "minute"
+ HOUR = "hour"
+ DAY = "day"
+ WEEK = "week"
+ MONTH = "month"
+ YEAR = "year"
+ ALL = "all"
+
+ @staticmethod
+ def find_scope(period_type, period):
+ if period_type == PeriodType.SECOND:
+ return str(period)[0:19] # 2011-06-13 18:11:32
+ elif period_type == PeriodType.MINUTE:
+ return str(period)[0:16] # 2011-06-13 18:11
+ elif period_type == PeriodType.HOUR:
+ return str(period)[0:13] # 2011-06-13 18
+ elif period_type == PeriodType.DAY:
+ return str(period)[0:10] # 2011-06-13
+ elif period_type == PeriodType.WEEK:
+ if not isinstance(period, datetime):
+ period = PeriodType.str_to_datetime(period)
+ return str(period- timedelta(period.weekday()))[0:10]+"week" # 2011-06-13week; use Monday as marker
+ elif period_type == PeriodType.MONTH:
+ return str(period)[0:7] # 2011-06
+ elif period_type == PeriodType.YEAR:
+ return str(period)[0:4] # 2011
+ else:
+ return "all"
+
+ @staticmethod
+ def str_to_datetime(datetime_str):
+ time_format = "%Y-%m-%d %H:%M:%S"
+ return datetime.fromtimestamp(time.mktime(time.strptime(datetime_str.split('.')[0], time_format)))
+
+
+class LivecountCounter(db.Model):
+ namespace = db.StringProperty(default="default")
+ period_type = db.StringProperty(default=PeriodType.ALL)
+ period = db.StringProperty()
+ name = db.StringProperty()
+ count = db.IntegerProperty()
+
+ @staticmethod
+ def KeyName(namespace, period_type, period, name):
+ scoped_period = PeriodType.find_scope(period_type, period)
+ return namespace + ":" + period_type + ":" + scoped_period + ":" + name
+
+ @staticmethod
+ def PartialKeyName(period_type, period, name):
+ scoped_period = PeriodType.find_scope(period_type, period)
+ return period_type + ":" + scoped_period + ":" + name
+
+
+def load_and_get_count(name, namespace='default', period_type='all', period=datetime.now()):
+ # Try memcache first
+ partial_key = LivecountCounter.PartialKeyName(period_type, period, name)