diff --git a/docloader.py b/docloader.py deleted file mode 100755 index d80e7da..0000000 --- a/docloader.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -# -*- python -*- - -import sys, zipfile, os, os.path -import getopt -import json -import ast -import tempfile - -from couchbase import client - -def usage(err=0): - print >> sys.stderr, """ -Usage: %s [-u user [-p password]] [-n host:port] [-b bucket] |zipfile - -Example: - %s -u user -p secret9876 localhost:8091 default mydir -""" % (os.path.basename(sys.argv[0]), (os.path.basename(sys.argv[0]))) - sys.exit(err) - -def parse_args(args): - user = None - pswd = None - host = 'localhost:8091' - bucket = 'default' - - try: - opts, args = getopt.getopt(args, 'hu:p:n:b:', ['help']) - except getopt.GetoptError, e: - usage("ERROR: " + e.msg) - - for (o, a) in opts: - if o == '--help' or o == '-h': - usage() - elif o == '-u': - user = a - elif o == '-p': - pswd = a - elif o == '-n': - host = a - elif o == '-b': - bucket = a - else: - usage("ERROR: unknown option - " + o) - - if not args or len(args) < 1: - usage("ERROR: missing upload directory or zipped file.") - - return user, pswd, host, bucket, args - -def save_doc(bucket,fp, views): - buf = fp.read() - result = ast.literal_eval(buf) - if isinstance(result, dict): - try: - doc_id = bucket.save(result) - except: - doc_id = "_design/testing" - if result['_id'] and 'views' in result: - for key in result['views'].iterkeys(): - viewpath = result['_id'] + '/_view/' + key - views.append(viewpath) - -def listFiles(bucket, dir, views): - basedir = dir - print "Files in ", os.path.abspath(dir), ": " - subdirlist = [] - for item in os.listdir(dir): - if os.path.isfile(os.path.join(basedir, item)): - with open(os.path.join(basedir, item), 'r') as fp: - #print item - save_doc(bucket, fp, views) - else: - subdirlist.append(os.path.join(basedir, item)) - for subdir in subdirlist: - listFiles(bucket, subdir, views) - -def unzip_file_and_upload(bucket,file, views): - zfobj = zipfile.ZipFile(file) - for name in zfobj.namelist(): - if name.endswith('/'): - print "dir:", name - else: - #print 'file:', name - temp = tempfile.NamedTemporaryFile(delete=False) - fname = temp.name - temp.write(zfobj.read(name)) - temp.close() - with open(fname, 'r') as fp: - save_doc(bucket, fp, views) - os.remove(fname) - -def populate_docs(bucket, dir, views): - if dir.endswith('.zip'): - unzip_file_and_upload(bucket, dir, views) - else: - listFiles(bucket, dir, views) - -def main(): - user, pswd, host, bucket, args = parse_args(sys.argv[1:]) - - cb = client.Server(host, user, pswd) - - try: - newbucket = cb.create(bucket, ram_quota_mb=100, replica=1) - except: - newbucket = cb[bucket] - - #upload documents - dir = args[0] - views = [] - populate_docs(newbucket, dir, views) - - # execute views at least once - for viewpath in views: - rows = newbucket.view(viewpath) - for row in rows: - print row - -if __name__ == '__main__': main() diff --git a/testapp/1.json b/testapp/1.json deleted file mode 100644 index 54c7fd8..0000000 --- a/testapp/1.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "_id": "007abf9d-f4a7-4001-15e8-26450000041c", - "_rev": "3-04c15a31dd051965e1a3c5430e8d2055", - "doc_type": "Stats", - "uuid": "007abf9d-f4a7-4001-15e8-26450000041c", - "numNodes": 1, - "timestamp": "2011-08-19T16:51:43Z", - "buckets": { - "total": 8, - "membase": 8, - "memcached": 0 - }, - "ram": { - "total": 12815089664, - "quotaTotal": 4294967296, - "quotaUsed": 2147483648 - }, - "dogbowled": "True", - "gae_state": "done", - "version": "1.7.0", - "componentsVersion": { - "kernel": "2.14.3", - "os_mon": "2.2.5", - "stdlib": "1.17.3", - "mnesia": "4.4.17", - "sasl": "2.1.9.3", - "ns_server": "1.7.0" - }, - "nodes": { - "uptime": [ - "105142" - ], - "os": [ - "windows" - ] - }, - "browser": "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)" -} diff --git a/testapp/2.json b/testapp/2.json deleted file mode 100644 index 1b5c891..0000000 --- a/testapp/2.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "_id": "00bcbb1c-6aa7-4001-15e8-231d000000e2", - "_rev": "1-104c40cac9fbe67b70b83e027e3ed765", - "doc_type": "Stats", - "uuid": "00bcbb1c-6aa7-4001-15e8-231d000000e2", - "numNodes": 1, - "timestamp": "2011-08-19T16:51:41Z", - "buckets": { - "total": 1, - "membase": 0, - "memcached": 1 - }, - "ram": { - "total": 4293541888, - "quotaTotal": 3434086400, - "quotaUsed": 536870912 - }, - "dogbowled": "True", - "gae_state": "done", - "version": "1.7.0", - "componentsVersion": { - "kernel": "2.14.3", - "os_mon": "2.2.5", - "stdlib": "1.17.3", - "mnesia": "4.4.17", - "sasl": "2.1.9.3", - "ns_server": "1.7.0" - }, - "nodes": { - "uptime": [ - "685616" - ], - "os": [ - "windows" - ] - }, - "browser": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0" -} \ No newline at end of file diff --git a/testapp/3.json b/testapp/3.json deleted file mode 100644 index a359b39..0000000 --- a/testapp/3.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "_id": "0619ef39-fca6-4001-15e8-9c180800275a", - "_rev": "1-831a36c9cb2bdddb299035fd83daabea", - "doc_type": "Stats", - "uuid": "0619ef39-fca6-4001-15e8-9c180800275a", - "numNodes": 4, - "timestamp": "2011-08-19T16:51:44Z", - "buckets": { - "total": 1, - "membase": 1, - "memcached": 0 - }, - "ram": { - "total": 67414867968, - "quotaTotal": 4194304000, - "quotaUsed": 4194304000 - }, - "dogbowled": "True", - "gae_state": "done", - "version": "1.7.0", - "componentsVersion": { - "kernel": "2.14.3", - "os_mon": "2.2.5", - "stdlib": "1.17.3", - "mnesia": "4.4.17", - "inets": "5.5.2", - "sasl": "2.1.9.3", - "ns_server": "1.7.0" - }, - "nodes": { - "uptime": [ - "158950", - "157937", - "7479", - "158421" - ], - "os": [ - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-gnu" - ] - }, - "browser": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.18) Gecko/20110614 Firefox/3.6.18" -} \ No newline at end of file