Permalink
Browse files

Merge branch 'develop' of github.com:gaurav/MapOfLife into develop

  • Loading branch information...
2 parents ba3c934 + 3263be5 commit a36fc9cad395f8a420b97788f3ab3474a786e386 @gaurav gaurav committed Feb 21, 2012
View
@@ -46,6 +46,7 @@ bulkloader-progress*
*.pydevproject
*.iml
*.swp
+#*.*#
# OS files #
############
View
@@ -40,6 +40,33 @@ $ dev_appserver.py ./
BOOM! You should be able to access the app at [http://localhost:8080](http://localhost:8080).
+# Backends
+
+We're using [App Engine backends](http://code.google.com/appengine/docs/python/backends/) for executing long running jobs that pre-cache CartoDB queries. For example, one of these jobs queries CartoDB for a distict list of `scientificname`, and for each one, performs a second CartoDB search profile query (e.g., a row per source/type) and stores the results in the `CacheItem` entity.
+
+Backends are configured in `backends.yaml`.
+
+To use backends with the development server via `dev_appserver.py`, you'll need to start it with the following command:
+
+```bash
+$ dev_appserver.py --use_sqlite --backends .
+```
+
+Backends are fired off using the taskqueue API, so keep an eye on the taskqueue admin console:
+
+```
+http://localhost:8080/_ah/admin/queues
+```
+
+When you're ready to deploy or update backends to the production servers on App Engine, use this command:
+
+```shell
+$ appcfg backends -V {app version} . update
+```
+
+Definitely check out [all of the available commands](http://code.google.com/appengine/docs/python/backends/overview.html#Commands) for backends.
+
+
# Emacs
You can setup Emacs with a JavaScript REPL which is really nice for hacking on MOL code since it's mainly written in JavaScript. If you need to install Emacs, it's easy, and here's a [great starting point](https://github.om/whizbangsystems/emacs-starter-kit). Just follow the instructions in the README.
View
@@ -0,0 +1,29 @@
+"""This module contains request handlers for admin-only access."""
+
+import datetime
+import logging
+
+from google.appengine.api import backends
+from google.appengine.api import taskqueue
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp.util import run_wsgi_app
+
+class SearchCacheHandler(webapp.RequestHandler):
+ def get(self):
+ taskqueue.add(
+ url='/backend/build_search_cache',
+ queue_name='build-search-cache',
+ eta=datetime.datetime.now(),
+ target='search-cache-builder-backend')
+ logging.info("BOOM")
+ self.response.set_status(202) # Accepted
+
+application = webapp.WSGIApplication(
+ [('/admin/build-search-cache', SearchCacheHandler)],
+ debug=True)
+
+def main():
+ run_wsgi_app(application)
+
+if __name__ == "__main__":
+ main()
View
@@ -10,7 +10,7 @@ builtins:
handlers:
- url: /admin/.*
- script: app.py
+ script: admin_handler.py
login: admin
- url: /js
@@ -26,6 +26,11 @@ handlers:
static_files: static/\1
upload: static/(.*\.(gif|png|jpg))
+- url: /cache/.*
+ script: cache_handler.py
+
+- url: /backend/build_search_cache(/.*)?
+ script: search_cache_backend.py
- url: /.*
script: app.py
View
@@ -0,0 +1,8 @@
+backends:
+
+- name: search-cache-builder-backend
+ start: search_cache_backend.py
+ class: B8
+ instances: 1
+ options: dynamic, public
+
View
@@ -0,0 +1,73 @@
+"""This module contains a cache that supports string and blob values. It supports
+both because blobs are unable to encode unicode characters properly.
+"""
+
+__author__ = 'Aaron Steele'
+
+# Standard Python imports
+import logging
+
+# Google App Engine imports
+from google.appengine.ext.ndb import model
+
+class CacheItem(model.Model):
+ """An item in the cache. Supports blob and string cached values since blob
+ can't handle unicode characters.
+ """
+ blob = model.BlobProperty('b')
+ string = model.StringProperty('s', indexed=False)
+ created = model.DateTimeProperty('c', auto_now_add=True)
+
+ @classmethod
+ def create(cls, key, value, dumps=False, value_type='string'):
+ entity = None
+ if value_type == 'string':
+ if dumps:
+ entity = cls(id=key.strip().lower(), string=simplejson.dumps(value))
+ else:
+ entity = cls(id=key.strip().lower(), string=value)
+ elif value_type == 'blob':
+ entity = cls(id=key.strip().lower(), blob=value)
+ return entity
+
+ @classmethod
+ def get(cls, key, loads=False, value_type='string'):
+ value = None
+ item = model.Key(cls.__name__, key.strip().lower()).get()
+ if item:
+ if value_type == 'string':
+ if loads:
+ value = simplejson.loads(item.string)
+ else:
+ value = item.string
+ elif value_type == 'blob':
+ value = item.blob
+ return value
+
+ @classmethod
+ def add(cls, key, value, dumps=False, value_type='string'):
+ cls.create(key, value, dumps, value_type).put()
+
+def create_entry(key, value, dumps=False, value_type='string'):
+ return CacheItem.create(key, value, dumps, value_type)
+
+def get(key, loads=False, value_type='string'):
+ """Gets a cached item value by key.
+
+ Arguments:
+ key - The cache item key.
+ loads - If true call simplejson.loads() on cached item (default false).
+ value_type - The type of cache value (string or blob, default string).
+ """
+ return CacheItem.get(key, loads, value_type)
+
+def add(key, value, dumps=False, value_type='string'):
+ """Adds a value to the cache by key.
+
+ Arguments:
+ key - The cache item key.
+ value - The cache item value.
+ dumps - If true call simplejson.dumps() to value before caching (default false).
+ value_type - The type of cache value (string or blob, default string).
+ """
+ CacheItem.add(key, value, dumps, value_type)
View
@@ -0,0 +1,41 @@
+"""This module contains a cache handler."""
+
+__author__ = 'Aaron Steele'
+
+# MOL imports
+import cache
+
+# Standard Python imports
+import logging
+import urllib
+
+# Google App Engine imports
+from google.appengine.api import urlfetch
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp.util import run_wsgi_app
+
+class GetHandler(webapp.RequestHandler):
+ """Request handler for cache requests."""
+
+ def post(self):
+ """Returns a cached value by key or None if it doesn't exist."""
+ key = self.request.get('key', 'empty')
+ sql = self.request.get('sql')
+ value = cache.get(key)
+ if not value:
+ logging.info('Cache miss on %s' % key)
+ url = 'http://mol.cartodb.com/api/v2/sql?%s' % urllib.urlencode(dict(q=sql))
+ value = urlfetch.fetch(url, deadline=60).content
+ cache.add(key, value)
+ self.response.headers["Content-Type"] = "application/json"
+ self.response.out.write(value)
+
+application = webapp.WSGIApplication(
+ [('/cache/get', GetHandler),],
+ debug=True)
+
+def main():
+ run_wsgi_app(application)
+
+if __name__ == "__main__":
+ main()
View
@@ -4,6 +4,6 @@ cd js
rm -rf ../static/js/mol.js
-cat mol.js mol.core.js mol.bus.js mol.mvp.js mol.services.js mol.services.cartodb.js mol.map.js mol.map.layers.js mol.map.menu.js mol.map.results.js mol.map.search.js mol.map.tiles.js > ../static/js/mol.js
+cat mol.js mol.core.js mol.bus.js mol.mvp.js mol.services.js mol.services.cartodb.js mol.map.js mol.map.loading.js mol.map.layers.js mol.map.menu.js mol.map.results.js mol.map.search.js mol.map.tiles.js > ../static/js/mol.js
cd ..
View
@@ -192,16 +192,27 @@ button:hover{
padding-top: 2px;
padding-bottom: 2px;
}
+.mol-LayerControl-Layers .layer .source{
+ float: left;
+ width: 10%;
+ padding-top: 2px;
+ padding-bottom: 2px;
+}
.mol-LayerControl-Layers .layer img.type{
height: 18px;
width: 18px;
}
+.mol-LayerControl-Layers .layer img.source{
+ height: 18px;
+ width: 18px;
+}
.mol-LayerControl-Layers .layer .buttonContainer {
- width: 23%;
+ width: 29%;
float: right;
position: relative;
margin-right: 5px;
}
+
.mol-LayerControl-Layers .layer .toggle {
opacity: 0.0;
position:absolute;
@@ -244,14 +255,23 @@ button:hover{
height: 24px;
}
+.mol-LayerControl-Layers .layer .close {
+ float: right;
+ position: relative;
+ width: 7%;
+ font-style: italic;
+ font-size: 1.2em;
+ height: 24px;
+}
+
.mol-LayerControl-Layers .layer .opacity {
float: right;
- position: absolute;
- #width: 75px;
+ position: relative;
+ width: 20%;
font-style: italic;
font-size: 1.2em;
height: 24px;
- #margin-left: 220px;
+ margin-right: 35px;
}
.mol-LayerControl-Layers .layer .zoom {
Oops, something went wrong.

0 comments on commit a36fc9c

Please sign in to comment.