Permalink
Browse files

Add rough prototypes

  • Loading branch information...
1 parent 50a1cc0 commit a7a485f0aa22e397a02c6d391e2c941ec890db73 @trondn trondn committed Aug 1, 2012
Showing with 216 additions and 0 deletions.
  1. +20 −0 README.md
  2. +58 −0 deploy.py
  3. +52 −0 genmanifest.py
  4. +86 −0 populate.py
View
@@ -0,0 +1,20 @@
+Preamble
+--------
+
+Please see http://hub.internal.couchbase.com/confluence/display/cbeng/Building+Couchbase+Server+2.x
+
+Example 1 - Snappy
+------------------
+
+* Compile Snappy:
+ `./configure --prefix=/opt/couchbase`
+ `gmake install DESTDIR=/tmp/snappy`
+
+* Generate manifest file
+ `python genmanifest.py /tmp/snappy/opt/couchbase > snappy-1.0.5-solaris32.json`
+
+* Deploy the binaries to our software depot
+ `python deploy.py /tmp/snappy/opt/couchbase`
+
+* Install it to the directory named install
+ `python populate.py snappy-1.0.5-solaris32.json install`
View
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+# Put all files from a staging directory into a depot
+# @todo we should read the manifest file and only
+# upload the files listed there..
+# @todo add support for none-filesystem-locations
+# @todo we should verify that the sha1 of the uploaded
+# file match
+
+import sys
+import os
+import hashlib
+import shutil
+
+def gen_depot(stage_dir, depot_dir):
+ """Put files from staging directory into the depot"""
+
+ for root, dirs, files in os.walk(stage_dir):
+ for file_ in files:
+ fullpath = os.path.join(root, file_)
+ contents = open(fullpath, 'rb').read()
+ sha1 = hashlib.sha1(contents).hexdigest()
+ dest_file = os.path.join(depot_dir, sha1)
+ # NOTE vmx 2012-06-25: Currently we just overwrite the file if it
+ # already exists
+ shutil.copy(fullpath, dest_file)
+ print "Copied ", fullpath, "to", dest_file
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv
+
+ if sys.platform == "win32":
+ depot_dir = os.environ.get("COUCHBASE_DEPOT_DIR", "c:/depot/");
+ else:
+ depot_dir = os.environ.get("COUCHBASE_DEPOT_DIR", "/export/depot/");
+
+ if (len(argv) < 2) or (len(argv) > 3):
+ print "Usage: gendepot.py stage-directory [depot-directory]"
+ return
+
+ stage_dir = os.path.abspath(argv[1])
+
+ if len(argv) == 3:
+ depot_dir = os.path.abspath(argv[2])
+
+ if not os.path.exists(stage_dir):
+ print "Error: stage directory not found"
+ return
+ if not os.path.exists(depot_dir):
+ print "Error: depot directory not found"
+ return
+
+ gen_depot(stage_dir, depot_dir)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
View
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+# Create a manifest file from a staging directory
+
+import sys
+import os
+import hashlib
+import json
+import stat
+
+
+# Based on # https://stomp.colorado.edu/blog/blog/2010/10/22/on-python-stat-octal-and-file-system-permissions/ (2012-06-25)
+def get_permission(path):
+ """Returns the permission of a file."""
+ return oct(stat.S_IMODE(os.stat(path).st_mode))
+
+def gen_manifest(stage_dir):
+ """Generate a manifest from a directory"""
+ manifest = {'files': []}
+
+ for root, dirs, files in os.walk(stage_dir):
+ for file_ in files:
+ fullpath = os.path.join(root, file_)
+ contents = open(fullpath, 'rb').read()
+ sha1 = hashlib.sha1(contents).hexdigest()
+ filename = os.path.relpath(fullpath, stage_dir)
+ mode = get_permission(fullpath)
+ manifest['files'].append({'path': filename, 'sha1': sha1,
+ 'mode': mode})
+ return manifest
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv
+
+ if len(argv) != 2:
+ print "Usage: genmanifest.py [directory]"
+ return
+
+ stage_dir = os.path.abspath(argv[1])
+ #print "Creating manifest for", stage_dir
+
+ if not os.path.exists(stage_dir):
+ print "Error: directory not found"
+ return
+
+ manifest = gen_manifest(stage_dir)
+ print json.dumps(manifest)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
View
@@ -0,0 +1,86 @@
+#! /usr/bin/env python
+# @todo install dependencies recursively
+# @todo install the maifest file to "etc/packages"
+# @todo check the mode on the installed file
+
+import json
+import sys
+import hashlib
+import os.path
+import urllib
+import exceptions
+
+class ChecksumException(Exception):
+ def __init__(self, value):
+ self.value = value
+ def __str__(self):
+ return repr(self.value)
+
+def processDependencies(deps):
+ """Installs all the dependencies"""
+ # @todo recursively grab the manifest and run it
+ for k in deps:
+ print k["package"]
+
+def checkFile(file):
+ """Check the sha1 of and installed file and verifies its permission"""
+ fullpath = os.path.join(install_dir, file["path"])
+ print "\rChecksumming " + fullpath + " "
+ digest = hashlib.new("sha1")
+ digest.update(open(fullpath, "rb").read())
+ if digest.hexdigest() != file["sha1"]:
+ raise ChecksumException("FATAL Incorrect SHA1")
+ # @todo check the mode
+ return True
+
+def installFile(file):
+ """Download the file from the depot and write it to the correct location"""
+ src = os.path.join(depot, file["sha1"])
+ dest = os.path.join(install_dir, file["path"])
+ try:
+ os.makedirs(os.path.dirname(dest))
+ # It doesn't matter if directory already exists
+ except OSError:
+ pass
+ print "Installing " + dest + " "
+ urllib.urlretrieve(src, dest)
+ os.chmod(dest, int(file["mode"], 8))
+ if not checkFile(file):
+ os._exit(os.EX_DATAERR)
+ print "OK"
+
+def processFiles(files):
+ for f in files:
+ fullpath = os.path.join(install_dir, f['path'])
+ if os.path.exists(fullpath):
+ checkFile(f)
+ else:
+ installFile(f)
+
+def processDirectories(dirs):
+ for d in dirs:
+ if not os.path.exists(d["path"]):
+ os.makedirs(d["path"], 0755)
+
+def installPackage(manifestfile):
+ file = open(manifestfile, "rb")
+ manifest = json.load(file)
+ try:
+ if 'depends' in manifest:
+ processDependencies(manifest["depends"])
+ if 'dirs' in manifest:
+ processDirectories(manifest["dirs"])
+ if 'files' in manifest:
+ processFiles(manifest["files"])
+ except ChecksumException, e:
+ sys.exit(e.value)
+
+
+
+if __name__ == '__main__':
+ if sys.platform == "win32":
+ depot = os.environ.get("COUCHBASE_DEPOT_DIR", "file://c:/depot/");
+ else:
+ depot = os.environ.get("COUCHBASE_DEPOT_DIR", "file:///export/depot/");
+ install_dir = os.path.abspath(sys.argv[2])
+ installPackage(sys.argv[1])

0 comments on commit a7a485f

Please sign in to comment.