Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

some modifications to make ckan.py run nicely from cron. removed lod.…

…js since it will presumably change quite a bit
  • Loading branch information...
commit 352f99dd11de3830612c2c06ef5fabd845f043aa 1 parent 78b6420
@edsu authored
Showing with 61 additions and 32 deletions.
  1. +0 −1  .gitignore
  2. +12 −9 README
  3. +49 −22 ckan.py
View
1  .gitignore
@@ -1,2 +1 @@
-lod.js
ckan.log
View
21 README
@@ -1,19 +1,22 @@
This is a visualization of the Linked Open Data [1] cloud using the Protovis
JavaScript library [2]. The underlying data is generated by talking to the
-Comprehensive Knowledge Archive Network (CKAN) API [3].
+Comprehensive Knowledge Archive Network (CKAN) API [3].
-To generate the lod.js data file you should be able to run ckan.py:
+In order to view index.html in your browser you'll first need to generate
+the lod.js file using ckan.py.
- ./ckan.py > lod.js
+ ./ckan.py
-It can take a few minutes to run, so go make yourself a cup of tea while
-it's running. After that you should be able to load index.html in
-your browser.
+If you would like to generate the lod.js on a regular basis just add it to
+your crontab, e.g. to update data every hour:
-Ed Summers
-ehs@pobox.com
+ 0 * * * * cd /var/www/inkdroid.org/lod-graph; ./ckan.py
+
+Contributors:
+
+Richard Cyganiak <richard@cyganiak.de>
+Ed Summers <ehs@pobox.com>
[1] http://richard.cyganiak.de/2007/10/lod/
[2] http://vis.stanford.edu/protovis/
[3] http://knowledgeforge.net/ckan/doc/ckan/api.html
-
View
71 ckan.py
@@ -1,20 +1,17 @@
#!/usr/bin/env python
"""
-This is a little script that will pull down all the package information
-for packages in the lodcloud group on CKAN, and write out JSON dataset
-for ProtoVis to stdout. Typically you'll want to:
-
- ./ckan.py > lod.js
-
-You should see information written to ckan.log about how the download
-is proceeding.
+Generate lod.js Protovis data file.
"""
+import os
import sys
import json
import urllib
import logging
+import traceback
+
+from datetime import datetime
LOG_FILE = "ckan.log"
@@ -22,13 +19,26 @@
def main(argv):
- logging.basicConfig(filename=LOG_FILE, level=LOG_LEVEL)
- packages = lod_packages()
- protovis = protovis_json(packages)
- print "var lod = " + json.dumps(protovis, indent=2)
+ """talk to ckan rest api and generate lod.js
+ """
+ configure_logging()
+ log = logging.getLogger()
+ log.info("starting to load data from ckan")
+
+ try:
+ packages = lod_packages()
+ javascript = protovis_javascript(packages)
+ write_javascript(javascript)
+ except BaseException, e:
+ traceback.print_exc()
+ log.fatal("exiting after unexpected error: %s" % e)
+
+ log.info("finished ckan load")
def lod_packages():
+ """returns a list of package metadata from ckan
+ """
log = logging.getLogger()
packages = []
count = 0
@@ -41,13 +51,34 @@ def lod_packages():
return packages
-def protovis_json(packages):
+def ckan(path):
+ """gets a JSON resource via the CKAN API
+ """
+ j = urllib.urlopen('http://ckan.net/api/rest/' + path).read()
+ return json.loads(j)
+
+
+def protovis_javascript(packages):
+ """generates protovis javascript data file
+ """
protovis = {'nodes': get_nodes(packages),
'links': get_links(packages)}
- return protovis
+ javascript = "var lod = " + json.dumps(protovis, indent=2)
+ return javascript
+
+
+def write_javascript(javascript):
+ """safely writes protovis javascript to lod.js
+ """
+ timestamp = datetime.strftime(datetime.now(), "%Y%m%dT%H%M%S")
+ tmp_file = "lod.js-%s" % timestamp
+ open(tmp_file, "w").write(javascript)
+ os.rename(tmp_file, "lod.js")
def get_nodes(packages):
+ """constructs a list of nodes suitable for protovis
+ """
nodes = []
for package in packages:
if package['ratings_average'] == None:
@@ -73,6 +104,8 @@ def get_nodes(packages):
def get_links(packages):
+ """returns links between the nodes suitable for protovis
+ """
log = logging.getLogger()
# first get a dictionary lookup for all the packages by name
@@ -101,16 +134,10 @@ def get_links(packages):
return links
-def ckan(path):
- j = urllib.urlopen('http://ckan.net/api/rest/' + path).read()
- return json.loads(j)
-
-
def configure_logging():
- logging.basicConfig()
logger = logging.getLogger()
- logger.setLevel(logging.INFO)
- handler = logging.FileHandler(log_file)
+ logger.setLevel(LOG_LEVEL)
+ handler = logging.FileHandler(LOG_FILE)
formatter = logging.Formatter("""[%(asctime)s %(levelname)s %(name)s] %(message)s""")
handler.setFormatter(formatter)
logger.addHandler(handler)
Please sign in to comment.
Something went wrong with that request. Please try again.