Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Add rough prototypes

  • Loading branch information...
commit a7a485f0aa22e397a02c6d391e2c941ec890db73 1 parent 50a1cc0
Trond Norbye authored
20  README.md
Source Rendered
... ...
@@ -0,0 +1,20 @@
  1
+Preamble
  2
+--------
  3
+
  4
+Please see http://hub.internal.couchbase.com/confluence/display/cbeng/Building+Couchbase+Server+2.x
  5
+
  6
+Example 1 - Snappy
  7
+------------------
  8
+
  9
+* Compile Snappy:
  10
+  `./configure --prefix=/opt/couchbase`
  11
+  `gmake install DESTDIR=/tmp/snappy`
  12
+
  13
+* Generate manifest file
  14
+  `python genmanifest.py /tmp/snappy/opt/couchbase > snappy-1.0.5-solaris32.json`
  15
+
  16
+* Deploy the binaries to our software depot
  17
+  `python deploy.py /tmp/snappy/opt/couchbase`
  18
+
  19
+* Install it to the directory named install
  20
+  `python populate.py snappy-1.0.5-solaris32.json install`
58  deploy.py
... ...
@@ -0,0 +1,58 @@
  1
+#!/usr/bin/env python
  2
+
  3
+# Put all files from a staging directory into a depot
  4
+# @todo we should read the manifest file and only
  5
+#       upload the files listed there..
  6
+# @todo add support for none-filesystem-locations
  7
+# @todo we should verify that the sha1 of the uploaded
  8
+#       file match
  9
+
  10
+import sys
  11
+import os
  12
+import hashlib
  13
+import shutil
  14
+
  15
+def gen_depot(stage_dir, depot_dir):
  16
+    """Put files from staging directory into the depot"""
  17
+
  18
+    for root, dirs, files in os.walk(stage_dir):
  19
+        for file_ in files:
  20
+            fullpath = os.path.join(root, file_)
  21
+            contents = open(fullpath, 'rb').read()
  22
+            sha1 = hashlib.sha1(contents).hexdigest()
  23
+            dest_file = os.path.join(depot_dir, sha1)
  24
+            # NOTE vmx 2012-06-25: Currently we just overwrite the file if it
  25
+            #     already exists
  26
+            shutil.copy(fullpath, dest_file)
  27
+            print "Copied ", fullpath, "to", dest_file
  28
+
  29
+def main(argv=None):
  30
+    if argv is None:
  31
+        argv = sys.argv
  32
+
  33
+    if sys.platform == "win32":
  34
+        depot_dir = os.environ.get("COUCHBASE_DEPOT_DIR", "c:/depot/");
  35
+    else:
  36
+        depot_dir = os.environ.get("COUCHBASE_DEPOT_DIR", "/export/depot/");
  37
+
  38
+    if (len(argv) < 2) or (len(argv) > 3):
  39
+        print "Usage: gendepot.py stage-directory [depot-directory]"
  40
+        return
  41
+
  42
+    stage_dir = os.path.abspath(argv[1])
  43
+
  44
+    if len(argv) == 3:
  45
+        depot_dir = os.path.abspath(argv[2])
  46
+
  47
+    if not os.path.exists(stage_dir):
  48
+        print "Error: stage directory not found"
  49
+        return
  50
+    if not os.path.exists(depot_dir):
  51
+        print "Error: depot directory not found"
  52
+        return
  53
+
  54
+    gen_depot(stage_dir, depot_dir)
  55
+
  56
+
  57
+if __name__ == '__main__':
  58
+    sys.exit(main())
52  genmanifest.py
... ...
@@ -0,0 +1,52 @@
  1
+#!/usr/bin/env python
  2
+
  3
+# Create a manifest file from a staging directory
  4
+
  5
+import sys
  6
+import os
  7
+import hashlib
  8
+import json
  9
+import stat
  10
+
  11
+
  12
+# Based on # https://stomp.colorado.edu/blog/blog/2010/10/22/on-python-stat-octal-and-file-system-permissions/ (2012-06-25)
  13
+def get_permission(path):
  14
+    """Returns the permission of a file."""
  15
+    return oct(stat.S_IMODE(os.stat(path).st_mode))
  16
+
  17
+def gen_manifest(stage_dir):
  18
+    """Generate a manifest from a directory"""
  19
+    manifest = {'files': []}
  20
+
  21
+    for root, dirs, files in os.walk(stage_dir):
  22
+        for file_ in files:
  23
+            fullpath = os.path.join(root, file_)
  24
+            contents = open(fullpath, 'rb').read()
  25
+            sha1 = hashlib.sha1(contents).hexdigest()
  26
+            filename = os.path.relpath(fullpath, stage_dir)
  27
+            mode = get_permission(fullpath)
  28
+            manifest['files'].append({'path': filename, 'sha1': sha1,
  29
+                                      'mode': mode})
  30
+    return manifest
  31
+
  32
+def main(argv=None):
  33
+    if argv is None:
  34
+        argv = sys.argv
  35
+
  36
+    if len(argv) != 2:
  37
+        print "Usage: genmanifest.py [directory]"
  38
+        return
  39
+
  40
+    stage_dir = os.path.abspath(argv[1])
  41
+    #print "Creating manifest for", stage_dir
  42
+
  43
+    if not os.path.exists(stage_dir):
  44
+        print "Error: directory not found"
  45
+        return
  46
+
  47
+    manifest = gen_manifest(stage_dir)
  48
+    print json.dumps(manifest)
  49
+
  50
+
  51
+if __name__ == '__main__':
  52
+    sys.exit(main())
86  populate.py
... ...
@@ -0,0 +1,86 @@
  1
+#! /usr/bin/env python
  2
+# @todo install dependencies recursively
  3
+# @todo install the maifest file to "etc/packages"
  4
+# @todo check the mode on the installed file
  5
+
  6
+import json
  7
+import sys
  8
+import hashlib
  9
+import os.path
  10
+import urllib
  11
+import exceptions
  12
+
  13
+class ChecksumException(Exception):
  14
+    def __init__(self, value):
  15
+        self.value = value
  16
+    def __str__(self):
  17
+        return repr(self.value)
  18
+
  19
+def processDependencies(deps):
  20
+    """Installs all the dependencies"""
  21
+    # @todo recursively grab the manifest and run it
  22
+    for k in deps:
  23
+        print k["package"]
  24
+
  25
+def checkFile(file):
  26
+    """Check the sha1 of and installed file and verifies its permission"""
  27
+    fullpath = os.path.join(install_dir, file["path"])
  28
+    print "\rChecksumming " + fullpath + " "
  29
+    digest = hashlib.new("sha1")
  30
+    digest.update(open(fullpath, "rb").read())
  31
+    if digest.hexdigest() != file["sha1"]:
  32
+        raise ChecksumException("FATAL Incorrect SHA1")
  33
+    # @todo check the mode
  34
+    return True
  35
+
  36
+def installFile(file):
  37
+    """Download the file from the depot and write it to the correct location"""
  38
+    src = os.path.join(depot, file["sha1"])
  39
+    dest = os.path.join(install_dir, file["path"])
  40
+    try:
  41
+        os.makedirs(os.path.dirname(dest))
  42
+    # It doesn't matter if directory already exists
  43
+    except OSError:
  44
+        pass
  45
+    print "Installing " + dest + " "
  46
+    urllib.urlretrieve(src, dest)
  47
+    os.chmod(dest, int(file["mode"], 8))
  48
+    if not checkFile(file):
  49
+        os._exit(os.EX_DATAERR)
  50
+    print "OK"
  51
+
  52
+def processFiles(files):
  53
+    for f in files:
  54
+        fullpath = os.path.join(install_dir, f['path'])
  55
+        if os.path.exists(fullpath):
  56
+            checkFile(f)
  57
+        else:
  58
+            installFile(f)
  59
+
  60
+def processDirectories(dirs):
  61
+    for d in dirs:
  62
+        if not os.path.exists(d["path"]):
  63
+            os.makedirs(d["path"], 0755)
  64
+
  65
+def installPackage(manifestfile):
  66
+    file = open(manifestfile, "rb")
  67
+    manifest = json.load(file)
  68
+    try:
  69
+        if 'depends' in manifest:
  70
+            processDependencies(manifest["depends"])
  71
+        if 'dirs' in manifest:
  72
+            processDirectories(manifest["dirs"])
  73
+        if 'files' in manifest:
  74
+            processFiles(manifest["files"])
  75
+    except ChecksumException, e:
  76
+        sys.exit(e.value)
  77
+
  78
+
  79
+
  80
+if __name__ == '__main__':
  81
+    if sys.platform == "win32":
  82
+        depot = os.environ.get("COUCHBASE_DEPOT_DIR", "file://c:/depot/");
  83
+    else:
  84
+        depot = os.environ.get("COUCHBASE_DEPOT_DIR", "file:///export/depot/");
  85
+    install_dir = os.path.abspath(sys.argv[2])
  86
+    installPackage(sys.argv[1])

0 notes on commit a7a485f

Please sign in to comment.
Something went wrong with that request. Please try again.