/
docloader
executable file
·130 lines (108 loc) · 4.43 KB
/
docloader
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#!/usr/bin/env python
# -*- python -*-
import sys
import zipfile
import os
import os.path
import simplejson as json
from tempfile import TemporaryFile
from optparse import OptionParser
from couchbase import client
class DocLoader:
def __init__(self):
pass
def parse_args(self):
usage = "usage: %prog [options] <directory>|zipfile\n\n" + \
"Example: %prog -u Administrator -p password -n 127.0.0.1:8091 " + \
"-b mybucket -s 100 gamesim-sample.zip"
parser = OptionParser(usage)
username = os.environ.get('REST_USERNAME', 'Administrator')
password = os.environ.get('REST_PASSWORD', 'password')
parser.add_option('-u', dest='username', default=username,
help='Username', metavar='Administrator')
parser.add_option('-p', dest='password', default=password,
help='Password', metavar='password')
parser.add_option('-b', dest='bucket', default='gamesim-sample',
help='Bucket', metavar='gamesim-sample')
parser.add_option('-n', dest='node', default='127.0.0.1:8091',
help='Node address', metavar='127.0.0.1:8091')
parser.add_option('-s', dest='ram_quota', default=100, type='int',
help='RAM quota in MB', metavar=100)
self.options, self.args = parser.parse_args()
if not self.args:
parser.print_help()
sys.exit()
print self.options, self.args
def save_doc(self, bucket, dockey, fp, views):
buf = fp.read()
result = json.loads(buf)
if isinstance(result, dict):
if '_id' not in result:
bucket.set(dockey, 0, 0, json.dumps(result))
else:
try:
result['_id'] = result['_id'].encode('UTF-8')
doc_id = bucket.save(result)
print "just now saving", doc_id
except:
doc_id = "_design/testing"
if result['_id'] and 'views' in result:
for key in result['views'].iterkeys():
viewpath = result['_id'] + '/_view/' + key
views.append(viewpath)
def gen_dockey(self, filename):
return os.path.basename(filename).split('.')[0]
def list_files(self, bucket, views, subdir=None):
if not subdir:
subdir = self.args[0]
#print "Files in ", os.path.abspath(dir), ": "
subdirlist = []
for item in os.listdir(subdir):
if os.path.isfile(os.path.join(subdir, item)):
try:
fp = open(os.path.join(subdir, item), 'r')
print "working with ", item
dockey = self.gen_dockey(item)
self.save_doc(bucket, dockey, fp, views)
fp.close()
except IOError, error:
print error
else:
subdirlist.append(os.path.join(subdir, item))
for dir in subdirlist:
self.list_files(bucket, views, dir)
def unzip_file_and_upload(self, bucket, views):
zfobj = zipfile.ZipFile(self.args[0])
for name in zfobj.namelist():
if not name.endswith('/'):
print 'working with ', name
dockey = self.gen_dockey(name)
temp_file = TemporaryFile()
temp_file.write(zfobj.read(name))
temp_file.seek(0)
self.save_doc(bucket, dockey, temp_file, views)
temp_file.close()
def populate_docs(self, bucket, views):
if self.args[0].endswith('.zip'):
self.unzip_file_and_upload(bucket, views)
else:
self.list_files(bucket, views)
def main():
docloader = DocLoader()
docloader.parse_args()
cb = client.Server(docloader.options.node, docloader.options.username,
docloader.options.password)
try:
newbucket = cb.create(docloader.options.bucket, replica=1,
ram_quota_mb=docloader.options.ram_quota)
except:
newbucket = cb[docloader.options.bucket]
#upload documents
views = []
docloader.populate_docs(newbucket, views)
# execute views at least once
for viewpath in views:
rows = newbucket.view(viewpath, stale="update_after")
if __name__ == '__main__':
main()
os._exit(0)