Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 68 lines (58 sloc) 2.16 KB
#!/usr/bin/env python
import json
import shutil
import urllib2
from datetime import datetime
"""Quick and dirty script for generating an index of Khan Academy content.
Looks at topic titles, exercise titles, and video titles and keywords to
produce a simple hash that can be used to test whether or not a given keyword
is related to any videos or exercises. The source data is from the topic tree
public API method."""
def parse(tree, stopwords, data={}):
children = tree.get("children")
if children:
for child in children:
data = parse(child, stopwords, data)
kind = tree.get("kind")
if kind == "Topic":
keywords = tree["title"].split()
if kind == "Video":
keywords = tree["title"].split() + tree["keywords"].split(", ")
elif kind == "Exercise":
keywords = tree["display_name"].split()
keywords = []
keywords = filter(lambda k: k not in stopwords, keywords)
keywords = filter(lambda k: len(k) >= 4, keywords)
for keyword in keywords:
data[keyword] = 1
return data
def main():
print "Retrieving topic tree"
topic_tree = json.loads(urllib2.urlopen(API_ENDPOINT).read())
print "Topic tree retrieval failed. Something is wrong with the API."
print "Generating index"
with open("stopwords.json") as f:
stopwords = json.loads(
stopwords = stopwords + [str(i) for i in range(10)] + [""]
data = parse(topic_tree, stopwords)
with open("extension/khantent.json", 'w') as f:
print "Index of %s items created" % len(data)
print "Updating manifest.json"
with open("manifest.template.json") as f:
manifest = json.loads(
# TODO(dylan): I'm not totally sure that this solves auto-versioning
version = datetime.strftime(, "%-y%m.%-d%-H.%-M%-S")
manifest["version"] = version
with open("extension/manifest.json", 'w') as f:
print "Creating archive of version %s" % version
shutil.make_archive("khanpedia", "zip", "extension")
if __name__ == '__main__':