Browse files

Modifying access to couchbase in exporter. Now, port 8092 is used.

  • Loading branch information...
1 parent 28cb75e commit ceb98041b455a18678ad43399b44e766913bad68 @hardlifeofapo committed Aug 30, 2012
Showing with 21 additions and 48 deletions.
  1. +0 −27 .gitignore
  2. +17 −16 exporter.py
  3. +4 −5 restorer.py
View
27 .gitignore
@@ -1,27 +0,0 @@
-*.py[co]
-
-# Packages
-*.egg
-*.egg-info
-dist
-build
-eggs
-parts
-bin
-var
-sdist
-develop-eggs
-.installed.cfg
-
-# Installer logs
-pip-log.txt
-
-# Unit test / coverage reports
-.coverage
-.tox
-
-#Translations
-*.mo
-
-#Mr Developer
-.mr.developer.cfg
View
33 exporter.py
@@ -5,53 +5,54 @@
'''
__author__ = "Pablo Casado (p.casado.arias@gmail.com)"
-__credits__ = []
+__credits__ = ["Francis Varga (nerd@crowdpark.com)"]
__license__ = "To de decided"
-__version__ = "0.0.1"
+__version__ = "0.0.2"
__maintainer__ = "Pablo Casado"
__email__ = "p.casado.arias@gmail.com"
__status__ = "Development"
-import time, sys, uuid, json, codecs, getopt, datetime
+import time, sys, uuid, json, codecs, getopt, datetime, subprocess
+import commands
from couchbase import Couchbase
from boto.s3.connection import S3Connection
from boto.s3.key import Key
-ACCESS_KEY_ID = 'YOUR_PUBLIC_KEY_HERE'
-SECRET_ACCESS_KEY = 'YOUR_PRIVATE_KEY_HERE'
+ACCESS_KEY_ID = 'PUBLIC_KEY'
+SECRET_ACCESS_KEY = 'PRIVATE_KEY'
CB_BUCKET_NAME = 'default'
S3_BUCKET_NAME = 'buck_up'
-SERVER_NAME = 'your_ip_goes_here'
+SERVER_NAME = '176.58.119.212'
SERVER_PORT = '8091'
-ALL_DOCS_VIEW_NAME = '_design/all/_view/all'
-USERNAME = 'your_username' #couchbase username
-PASSWORD = 'your_password' #couchbase password
class Exporter(object):
def __init__(self):
# connect to a couchbase server and select bucket where docs are stored
self.conn = S3Connection(ACCESS_KEY_ID, SECRET_ACCESS_KEY)
now = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M")
- #self.s3BucketName = "%s_%s_%s" % (ACCESS_KEY_ID.lower(), S3_BUCKET_NAME, now)
- #self.couchbase = Couchbase("%s:%s" % (SERVER_NAME, SERVER_PORT), username=USERNAME, password=PASSWORD)
- #self.cb_bucket = self.couchbase[CB_BUCKET_NAME]
-
+ self.s3BucketName = "%s_%s_%s" % (ACCESS_KEY_ID.lower(), S3_BUCKET_NAME, now)
def run(self):
ini = int( time.time() )
- items = self.cb_bucket.view(ALL_DOCS_VIEW_NAME, include_docs="true" )
+ url = '/usr/bin/curl http://%s:%s/%s/_all_docs?include_docs=true'% (SERVER_NAME, 8092, CB_BUCKET_NAME )
+ p = subprocess.Popen(url, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ output, errors = p.communicate()
+ output = json.loads(output)
+ items = output['rows']
+
# Create does not create again a bucket if that bucket already exists.
s3_bucket = self.conn.create_bucket(self.s3BucketName)
for item in items:
k = Key(s3_bucket)
k.key = item["doc"]["_id"]
- print json.dumps(item["doc"], sort_keys=True)
+ print "Saving %s" % (item["doc"]["_id"])
+ # json.dumps is needed to prevent unicode representation of strings in python, like u'a_string'
k.set_contents_from_string( json.dumps(item["doc"], sort_keys=True) )
-
+
fin = int( time.time() )
total = (fin - ini) #in seconds
View
9 restorer.py
@@ -5,7 +5,7 @@
'''
__author__ = "Pablo Casado (p.casado.arias@gmail.com)"
-__credits__ = []
+__credits__ = ["Francis Varga (nerd@crowdpark.com)"]
__license__ = "To de decided"
__version__ = "0.0.1"
__maintainer__ = "Pablo Casado"
@@ -18,13 +18,12 @@
from boto.s3.connection import S3Connection
from boto.s3.key import Key
-ACCESS_KEY_ID = 'YOUR_PUBLIC_KEY_HERE'
-SECRET_ACCESS_KEY = 'YOUR_PRIVATE_KEY_HERE'
+ACCESS_KEY_ID = 'PUBLIC_KEY'
+SECRET_ACCESS_KEY = 'PRIVATE_KEY'
CB_BUCKET_NAME = 'default'
S3_BUCKET_NAME = 'buck_up'
SERVER_NAME = 'your_ip_goes_here'
SERVER_PORT = '8091'
-ALL_DOCS_VIEW_NAME = '_design/all/_view/all'
USERNAME = 'your_couchbase_username' #couchbase username
PASSWORD = 'your_couchbase_password' #couchbase password
@@ -53,7 +52,7 @@ def run(self):
for key in rs:
key = s3_bucket.get_key(key.name)
contents = json.loads(key.get_contents_as_string())
-
+
# See http://www.couchbase.com/issues/browse/MB-5302
del(contents['_id'])
del(contents['_rev'])

0 comments on commit ceb9804

Please sign in to comment.