Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Add some basic caching functions, and cache transcript views. - #42

  • Loading branch information...
commit d04edb7aeec29e1e961ea4959c7cc211531e4fb4 1 parent 886c53b
@gldnspud gldnspud authored
View
1  development-local.ini.example
@@ -1,6 +1,7 @@
[app:main]
use = config:development.ini
fanscribed.audio = %(here)s/../audio
+fanscribed.cache = %(here)s/../cache
fanscribed.repos = %(here)s/../repos
fanscribed.repo_templates = %(here)s/../repo_templates
fanscribed.snippet_cache = %(here)s/fanscribed/static/snippets/
View
49 fanscribed/cache.py
@@ -0,0 +1,49 @@
+"""Simple filesystem-based cache."""
+
+import hashlib
+import os
+import random
+import time
+
+from fanscribed.common import app_settings
+
+
+def _cache_path():
+ path = app_settings()['fanscribed.cache']
+ if not os.path.isdir(path):
+ os.makedirs(path)
+ return path
+
+
+def get_cached_content(key):
+ """Return (content, mtime) associated with ``key``, or ``(None, None)`` if not found."""
+ # Convert key to a hash.
+ hashed_key = hashlib.sha1(str(key)).hexdigest()
+ # Load it if it exists.
+ content_path = os.path.join(_cache_path(), hashed_key)
+ try:
+ with open(content_path, 'rb') as f:
+ print 'Cache: {0} hit'.format(key)
+ return f.read(), os.fstat(f.fileno()).st_mtime
+ except IOError:
+ print 'Cache: {0} miss'.format(key)
+ return None, None
+
+
+def cache_content(key, content, mtime=None):
+ """Cache the given content."""
+ # Convert key to a hash.
+ hashed_key = hashlib.sha1(str(key)).hexdigest()
+ # Convert to bytes as needed. TODO: should we be doing this here?
+ if isinstance(content, unicode):
+ content = content.encode('utf8')
+ # Write to a temporary file, then rename, to make cache writes atomic
+ # in the event of race conditions.
+ cache_path = _cache_path()
+ initial_path = os.path.join(cache_path, '{0}-{1}'.format(hashed_key, random.random()))
+ final_path = os.path.join(cache_path, hashed_key)
+ with open(initial_path, 'wb') as f:
+ f.write(content)
+ if mtime is not None:
+ os.utime(initial_path, (time.time(), mtime))
+ os.rename(initial_path, final_path)
View
13 fanscribed/common.py
@@ -0,0 +1,13 @@
+"""Functions commonly used in other modules."""
+
+from pyramid.threadlocal import get_current_registry
+
+
+_settings = None
+
+
+def app_settings():
+ global _settings
+ if _settings is None:
+ _settings = get_current_registry().settings
+ return _settings
View
10 fanscribed/repos.py
@@ -7,7 +7,7 @@
import git
-from pyramid.threadlocal import get_current_registry
+from fanscribed.common import app_settings
# Twenty minute lock timeout.
@@ -25,17 +25,13 @@ def _lock_secret():
return ''.join(random.choice(string.letters) for x in xrange(16))
-def _settings():
- return get_current_registry().settings
-
-
def _snippet_ms():
- snippet_seconds = int(_settings()['fanscribed.snippet_seconds'])
+ snippet_seconds = int(app_settings()['fanscribed.snippet_seconds'])
return snippet_seconds * 1000
def repo_from_request(request):
- repos_path = _settings()['fanscribed.repos']
+ repos_path = app_settings()['fanscribed.repos']
repo_path = os.path.join(repos_path, request.host)
# Make sure repo path is underneath outer repos path.
assert '..' not in os.path.relpath(repo_path, repos_path)
View
74 fanscribed/views.py
@@ -9,10 +9,13 @@
import git
from pyramid.httpexceptions import HTTPFound
+from pyramid.renderers import render
from pyramid.response import Response
from pyramid.threadlocal import get_current_registry
from pyramid.view import view_config
+from fanscribed import cache
+from fanscribed.common import app_settings
from fanscribed import mp3
from fanscribed import repos
@@ -95,12 +98,8 @@ def _slugify(text):
)
-def _settings():
- return get_current_registry().settings
-
-
def _snippet_ms():
- snippet_seconds = int(_settings()['fanscribed.snippet_seconds'])
+ snippet_seconds = int(app_settings()['fanscribed.snippet_seconds'])
return snippet_seconds * 1000
@@ -172,34 +171,41 @@ def edit(request):
request_method='GET',
route_name='view',
context='fanscribed:resources.Root',
- renderer='fanscribed:templates/view.mako',
)
def view(request):
repo = repos.repo_from_request(request)
- master = repo.tree('master')
- transcription_info = repos.transcription_info(master)
- raw_snippets = {}
- for obj in master:
- if isinstance(obj, git.Blob):
- name, ext = os.path.splitext(obj.name)
- if ext == '.txt':
- try:
- starting_point = int(name)
- except ValueError:
- pass
- else:
- raw_snippets[starting_point] = obj.data_stream.read().decode('utf8')
- # Go through all snippets, whether they've been transcribed or not.
- snippets = []
- speakers_map = repos.speakers_map(master)
- for starting_point in range(0, transcription_info['duration'], _snippet_ms()):
- text = raw_snippets.get(starting_point, '').strip()
- lines = _split_lines_and_expand_abbreviations(text, speakers_map)
- snippets.append((starting_point, lines))
- return dict(
- _standard_response(repo, master),
- snippets=sorted(snippets),
- )
+ # Return cached if found.
+ cache_key = repo.head.commit.hexsha
+ content, mtime = cache.get_cached_content(cache_key)
+ if content is None:
+ mtime = repo.head.commit.authored_date
+ master = repo.tree('master')
+ transcription_info = repos.transcription_info(master)
+ raw_snippets = {}
+ for obj in master:
+ if isinstance(obj, git.Blob):
+ name, ext = os.path.splitext(obj.name)
+ if ext == '.txt':
+ try:
+ starting_point = int(name)
+ except ValueError:
+ pass
+ else:
+ raw_snippets[starting_point] = obj.data_stream.read().decode('utf8')
+ # Go through all snippets, whether they've been transcribed or not.
+ snippets = []
+ speakers_map = repos.speakers_map(master)
+ for starting_point in range(0, transcription_info['duration'], _snippet_ms()):
+ text = raw_snippets.get(starting_point, '').strip()
+ lines = _split_lines_and_expand_abbreviations(text, speakers_map)
+ snippets.append((starting_point, lines))
+ data = dict(
+ _standard_response(repo, master),
+ snippets=sorted(snippets),
+ )
+ content = render('fanscribed:templates/view.mako', data, request=request)
+ cache.cache_content(cache_key, content, mtime)
+ return Response(content, date=mtime)
@view_config(
@@ -263,7 +269,7 @@ def transcription_json(request):
master = repo.tree('master')
info = repos.transcription_info(master)
# Inject additional information into the info dict.
- settings = _settings()
+ settings = app_settings()
info['snippet_ms'] = int(settings['fanscribed.snippet_seconds']) * 1000
info['snippet_padding_ms'] = int(float(settings['fanscribed.snippet_padding_seconds']) * 1000)
return Response(body=json.dumps(info), content_type='application/json')
@@ -305,7 +311,7 @@ def snippet_info(request):
def _banned_message(request):
"""Returns a reason why you're banned, or None if you're not banned."""
ip_address = request.headers.get('X-Forwarded-For', request.remote_addr).strip()
- ip_bans_filename = _settings().get('fanscribed.ip_address_bans')
+ ip_bans_filename = app_settings().get('fanscribed.ip_address_bans')
email = request.POST.get('identity_email')
if ip_bans_filename:
with open(ip_bans_filename, 'rU') as f:
@@ -315,7 +321,7 @@ def _banned_message(request):
if ip_address == candidate_ip_address:
return reason.strip()
if email:
- email_bans_filename = _settings().get('fanscribed.email_bans')
+ email_bans_filename = app_settings().get('fanscribed.email_bans')
if email_bans_filename:
with open(email_bans_filename, 'rU') as f:
for line in f.readlines():
@@ -556,7 +562,7 @@ def cancel_review(request):
)
def snippet_mp3(request):
# Get information needed from settings and repository.
- settings = _settings()
+ settings = app_settings()
full_mp3 = os.path.join(
settings['fanscribed.audio'],
'{0}.mp3'.format(request.host),
View
1  production-hosted.ini.example
@@ -1,6 +1,7 @@
[app:main]
use = config:production.ini
fanscribed.audio = %(here)s/../audio
+fanscribed.cache = %(here)s/../cache
fanscribed.repos = %(here)s/../repos
fanscribed.repo_templates = %(here)s/../repo_templates
fanscribed.snippet_cache = %(here)s/fanscribed/static/snippets/
Please sign in to comment.
Something went wrong with that request. Please try again.