Bug 506404: Scripts and tools for nanojit, including hgtool.py. r=bhe…

…arsum

--HG--
extra : rebase_source : de6cc65f8aa77ce7b16b34d5263d6528fef0e12d

buildfarm/utils/hgtool.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python
+"""%prog [-p|--props-file] [-r|--rev revision] [-b|--branch branch] repo [dest]
+
+Tool to do safe operations with hg.
+
+revision/branch on commandline will override those in props-file"""
+
+# Import snippet to find tools lib
+import os, sys
+sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib/python"))
+
+from util.hg import mercurial
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+    import os, logging
+
+    parser = OptionParser(__doc__)
+    parser.set_defaults(
+            revision=os.environ.get('HG_REV'),
+            branch=os.environ.get('HG_BRANCH', 'default'),
+            propsfile=os.environ.get('PROPERTIES_FILE'),
+            tbox=bool(os.environ.get('PROPERTIES_FILE')),
+            loglevel=logging.INFO,
+            )
+    parser.add_option("-r", "--rev", dest="revision", help="which revision to update to")
+    parser.add_option("-b", "--branch", dest="branch", help="which branch to update to")
+    parser.add_option("-p", "--props-file", dest="propsfile",
+        help="build json file containing revision information")
+    parser.add_option("--tbox", dest="tbox", action="store_true",
+        help="output TinderboxPrint messages")
+    parser.add_option("--no-tbox", dest="tbox", action="store_false",
+        help="don't output TinderboxPrint messages")
+
+    options, args = parser.parse_args()
+
+    logging.basicConfig(level=options.loglevel, format="%(message)s")
+
+    if len(args) not in (1, 2):
+        parser.error("Invalid number of arguments")
+
+    repo = args[0]
+    if len(args) == 2:
+        dest = args[1]
+    else:
+        dest = os.path.basename(repo)
+
+    # Parse propsfile
+    if options.propsfile:
+        try:
+            import json
+        except ImportError:
+            import simplejson as json
+        js = json.load(open(options.propsfile))
+        if options.revision is None:
+            options.revision = js['sourcestamp']['revision']
+        if options.branch is None:
+            options.branch = js['sourcestamp']['branch']
+
+    got_revision = mercurial(repo, dest, options.branch, options.revision)
+
+    if options.tbox:
+        print "TinderboxPrint: revision: %s" % got_revision
+    else:
+        print "Got revision %s" % got_revision

lib/python/test/init_hgrepo.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Set up an hg repo for testing
+dest=$1
+if [ -z "$dest" ]; then
+    echo You must specify a destination directory 1>&2
+    exit 1
+fi
+
+rm -rf $dest
+mkdir $dest
+cd $dest
+hg init
+
+echo "Hello world $RANDOM" > hello.txt
+hg add hello.txt
+hg commit -m "Adding hello"
+
+hg branch branch2 > /dev/null
+echo "So long, farewell" >> hello.txt
+hg commit -m "Changing hello on branch"
+
+hg checkout default
+echo "Is this thing on?" >> hello.txt
+hg commit -m "Last change on default"

lib/python/test/test_util_commands.py

@@ -0,0 +1,25 @@
+import unittest, subprocess
+
+from util.commands import run_cmd, get_output
+
+class TestRunCmd(unittest.TestCase):
+    def testSimple(self):
+        self.assertEquals(run_cmd(['true']), 0)
+
+    def testFailure(self):
+        self.assertRaises(subprocess.CalledProcessError, run_cmd, ['false'])
+
+    def testOutput(self):
+        output = get_output(['echo', 'hello'])
+        self.assertEquals(output, 'hello\n')
+
+    def testStdErr(self):
+        output = get_output(['bash', '-c', 'echo hello 1>&2'], include_stderr=True)
+        self.assertEquals(output, 'hello\n')
+
+    def testNoStdErr(self):
+        output = get_output(['bash', '-c', 'echo hello 1>&2'])
+        self.assertEquals(output, '')
+
+    def testBadOutput(self):
+        self.assertRaises(subprocess.CalledProcessError, get_output, ['false'])

lib/python/test/test_util_hg.py

@@ -0,0 +1,201 @@
+import unittest
+import tempfile
+import shutil
+import os
+import subprocess
+
+from util.hg import clone, pull, update, hg_ver, mercurial, _make_absolute
+from util.commands import run_cmd, get_output
+
+def getRevisions(dest):
+    retval = []
+    for rev in get_output(['hg', 'log', '-R', dest, '--template', '{node|short}\n']).split('\n'):
+        rev = rev.strip()
+        if not rev:
+            continue
+        retval.append(rev)
+    return retval
+
+class TestMakeAbsolute(unittest.TestCase):
+    def testAboslutePath(self):
+        self.assertEquals(_make_absolute("/foo/bar"), "/foo/bar")
+
+    def testRelativePath(self):
+        self.assertEquals(_make_absolute("foo/bar"), os.path.abspath("foo/bar"))
+
+    def testHTTPPaths(self):
+        self.assertEquals(_make_absolute("http://foo/bar"), "http://foo/bar")
+
+    def testAbsoluteFilePath(self):
+        self.assertEquals(_make_absolute("file:///foo/bar"), "file:///foo/bar")
+
+    def testRelativeFilePath(self):
+        self.assertEquals(_make_absolute("file://foo/bar"), "file://%s/foo/bar" % os.getcwd())
+
+class TestHg(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        self.repodir = os.path.join(self.tmpdir, 'repo')
+        run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__),
+                self.repodir])
+
+        self.revisions = getRevisions(self.repodir)
+        self.wc = os.path.join(self.tmpdir, 'wc')
+        self.pwd = os.getcwd()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+        os.chdir(self.pwd)
+
+    def testClone(self):
+        rev = clone(self.repodir, self.wc, update_dest=False)
+        self.assertEquals(rev, None)
+        self.assertEquals(self.revisions, getRevisions(self.wc))
+        self.assertEquals(sorted(os.listdir(self.wc)), ['.hg'])
+
+    def testCloneIntoNonEmptyDir(self):
+        os.mkdir(self.wc)
+        open(os.path.join(self.wc, 'test.txt'), 'w').write('hello')
+        clone(self.repodir, self.wc, update_dest=False)
+        self.failUnless(not os.path.exists(os.path.join(self.wc, 'test.txt')))
+
+    def testCloneUpdate(self):
+        rev = clone(self.repodir, self.wc, update_dest=True)
+        self.assertEquals(rev, self.revisions[0])
+
+    def testCloneBranch(self):
+        clone(self.repodir, self.wc, branch='branch2',
+                update_dest=False)
+        # On hg 1.6, we should only have a subset of the revisions
+        if hg_ver() >= (1,6,0):
+            self.assertEquals(self.revisions[1:],
+                    getRevisions(self.wc))
+        else:
+            self.assertEquals(self.revisions,
+                    getRevisions(self.wc))
+
+    def testCloneUpdateBranch(self):
+        rev = clone(self.repodir, os.path.join(self.tmpdir, 'wc'),
+                branch="branch2", update_dest=True)
+        self.assertEquals(rev, self.revisions[1], self.revisions)
+
+    def testCloneRevision(self):
+        clone(self.repodir, self.wc,
+                revision=self.revisions[0], update_dest=False)
+        # We'll only get a subset of the revisions
+        self.assertEquals(self.revisions[:1] + self.revisions[2:],
+                getRevisions(self.wc))
+
+    def testUpdateRevision(self):
+        rev = clone(self.repodir, self.wc, update_dest=False)
+        self.assertEquals(rev, None)
+
+        rev = update(self.wc, revision=self.revisions[1])
+        self.assertEquals(rev, self.revisions[1])
+
+    def testPull(self):
+        # Clone just the first rev
+        clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
+        self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
+
+        # Now pull in new changes
+        rev = pull(self.repodir, self.wc, update_dest=False)
+        self.assertEquals(rev, None)
+        self.assertEquals(getRevisions(self.wc), self.revisions)
+
+    def testPullRevision(self):
+        # Clone just the first rev
+        clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
+        self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
+
+        # Now pull in just the last revision
+        rev = pull(self.repodir, self.wc, revision=self.revisions[0], update_dest=False)
+        self.assertEquals(rev, None)
+
+        # We'll be missing the middle revision (on another branch)
+        self.assertEquals(getRevisions(self.wc), self.revisions[:1] + self.revisions[2:])
+
+    def testPullBranch(self):
+        # Clone just the first rev
+        clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
+        self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
+
+        # Now pull in the other branch
+        rev = pull(self.repodir, self.wc, branch="branch2", update_dest=False)
+        self.assertEquals(rev, None)
+
+        # On hg 1.6, we'll be missing the last revision (on another branch)
+        if hg_ver() >= (1,6,0):
+            self.assertEquals(getRevisions(self.wc), self.revisions[1:])
+        else:
+            self.assertEquals(getRevisions(self.wc), self.revisions)
+
+    def testPullUnrelated(self):
+        # Create a new repo
+        repo2 = os.path.join(self.tmpdir, 'repo2')
+        run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2])
+
+        self.assertNotEqual(self.revisions, getRevisions(repo2))
+
+        # Clone the original repo
+        clone(self.repodir, self.wc, update_dest=False)
+
+        # Try and pull in changes from the new repo
+        self.assertRaises(subprocess.CalledProcessError, pull, repo2, self.wc, update_dest=False)
+
+    def testMercurial(self):
+        rev = mercurial(self.repodir, self.wc)
+        self.assertEquals(rev, self.revisions[0])
+
+    def testMercurialRelativeDir(self):
+        os.chdir(os.path.dirname(self.repodir))
+
+        repo = os.path.basename(self.repodir)
+        wc = os.path.basename(self.wc)
+
+        rev = mercurial(repo, wc, revision=self.revisions[-1])
+        self.assertEquals(rev, self.revisions[-1])
+        open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
+
+        rev = mercurial(repo, wc)
+        self.assertEquals(rev, self.revisions[0])
+        # Make sure our local file didn't go away
+        self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
+
+    def testMercurialUpdateTip(self):
+        rev = mercurial(self.repodir, self.wc, revision=self.revisions[-1])
+        self.assertEquals(rev, self.revisions[-1])
+        open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
+
+        rev = mercurial(self.repodir, self.wc)
+        self.assertEquals(rev, self.revisions[0])
+        # Make sure our local file didn't go away
+        self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
+
+    def testMercurialUpdateRev(self):
+        rev = mercurial(self.repodir, self.wc, revision=self.revisions[-1])
+        self.assertEquals(rev, self.revisions[-1])
+        open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
+
+        rev = mercurial(self.repodir, self.wc, revision=self.revisions[0])
+        self.assertEquals(rev, self.revisions[0])
+        # Make sure our local file didn't go away
+        self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
+
+    def testMercurialChangeRepo(self):
+        # Create a new repo
+        repo2 = os.path.join(self.tmpdir, 'repo2')
+        run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2])
+
+        self.assertNotEqual(self.revisions, getRevisions(repo2))
+
+        # Clone the original repo
+        mercurial(self.repodir, self.wc)
+        self.assertEquals(getRevisions(self.wc), self.revisions)
+        open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
+
+        # Clone the new one
+        mercurial(repo2, self.wc)
+        self.assertEquals(getRevisions(self.wc), getRevisions(repo2))
+        # Make sure our local file went away
+        self.failUnless(not os.path.exists(os.path.join(self.wc, 'test.txt')))

lib/python/util/commands.py

@@ -0,0 +1,77 @@
+"""Functions for running commands"""
+import subprocess, os
+
+import logging
+log = logging.getLogger(__name__)
+
+def run_cmd(cmd, **kwargs):
+    """Run cmd (a list of arguments).  Raise subprocess.CalledProcessError if
+    the command exits with non-zero.  If the command returns successfully,
+    return 0."""
+    if 'cwd' in kwargs and kwargs['cwd']:
+        log.info("%s in %s", " ".join(cmd), kwargs['cwd'])
+    else:
+        log.info(" ".join(cmd))
+    return subprocess.check_call(cmd, **kwargs)
+
+def get_output(cmd, include_stderr=False, **kwargs):
+    """Run cmd (a list of arguments) and return the output.  If include_stderr
+    is set, stderr will be included in the output, otherwise it will be sent to
+    the caller's stderr stream.
+
+    Warning that you shouldn't use this function to capture a large amount of
+    output (> a few thousand bytes), it will deadlock."""
+    if include_stderr:
+        stderr = subprocess.STDOUT
+    else:
+        stderr = None
+
+    if 'cwd' in kwargs and kwargs['cwd']:
+        log.info("%s in %s", " ".join(cmd), kwargs['cwd'])
+    else:
+        log.info(" ".join(cmd))
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr,
+            **kwargs)
+    proc.wait()
+    if proc.returncode != 0:
+        raise subprocess.CalledProcessError(proc.returncode, cmd)
+    output = proc.stdout.read()
+    return output
+
+def remove_path(path):
+    """This is a replacement for shutil.rmtree that works better under
+    windows. Thanks to Bear at the OSAF for the code.
+    (Borrowed from buildbot.slave.commands)"""
+    log.debug("Removing %s", path)
+    if not os.path.exists(path):
+        # This handles broken links
+        if os.path.islink(path):
+            os.remove(path)
+        return
+
+    if os.path.islink(path):
+        os.remove(path)
+        return
+
+    # Verify the directory is read/write/execute for the current user
+    os.chmod(path, 0700)
+
+    for name in os.listdir(path):
+        full_name = os.path.join(path, name)
+        # on Windows, if we don't have write permission we can't remove
+        # the file/directory either, so turn that on
+        if os.name == 'nt':
+            if not os.access(full_name, os.W_OK):
+                # I think this is now redundant, but I don't have an NT
+                # machine to test on, so I'm going to leave it in place
+                # -warner
+                os.chmod(full_name, 0600)
+
+        if os.path.isdir(full_name):
+            remove_path(full_name)
+        else:
+            # Don't try to chmod links
+            if not os.path.islink(full_name):
+                os.chmod(full_name, 0700)
+            os.remove(full_name)
+    os.rmdir(path)