diff --git a/buildfarm/utils/hgtool.py b/buildfarm/utils/hgtool.py new file mode 100755 index 000000000..53ad263a2 --- /dev/null +++ b/buildfarm/utils/hgtool.py @@ -0,0 +1,65 @@ +#!/usr/bin/python +"""%prog [-p|--props-file] [-r|--rev revision] [-b|--branch branch] repo [dest] + +Tool to do safe operations with hg. + +revision/branch on commandline will override those in props-file""" + +# Import snippet to find tools lib +import os, sys +sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib/python")) + +from util.hg import mercurial + +if __name__ == '__main__': + from optparse import OptionParser + import os, logging + + parser = OptionParser(__doc__) + parser.set_defaults( + revision=os.environ.get('HG_REV'), + branch=os.environ.get('HG_BRANCH', 'default'), + propsfile=os.environ.get('PROPERTIES_FILE'), + tbox=bool(os.environ.get('PROPERTIES_FILE')), + loglevel=logging.INFO, + ) + parser.add_option("-r", "--rev", dest="revision", help="which revision to update to") + parser.add_option("-b", "--branch", dest="branch", help="which branch to update to") + parser.add_option("-p", "--props-file", dest="propsfile", + help="build json file containing revision information") + parser.add_option("--tbox", dest="tbox", action="store_true", + help="output TinderboxPrint messages") + parser.add_option("--no-tbox", dest="tbox", action="store_false", + help="don't output TinderboxPrint messages") + + options, args = parser.parse_args() + + logging.basicConfig(level=options.loglevel, format="%(message)s") + + if len(args) not in (1, 2): + parser.error("Invalid number of arguments") + + repo = args[0] + if len(args) == 2: + dest = args[1] + else: + dest = os.path.basename(repo) + + # Parse propsfile + if options.propsfile: + try: + import json + except ImportError: + import simplejson as json + js = json.load(open(options.propsfile)) + if options.revision is None: + options.revision = js['sourcestamp']['revision'] + if options.branch is None: + options.branch = js['sourcestamp']['branch'] + + got_revision = mercurial(repo, dest, options.branch, options.revision) + + if options.tbox: + print "TinderboxPrint: revision: %s" % got_revision + else: + print "Got revision %s" % got_revision diff --git a/lib/python/test/init_hgrepo.sh b/lib/python/test/init_hgrepo.sh new file mode 100755 index 000000000..c978ebe73 --- /dev/null +++ b/lib/python/test/init_hgrepo.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Set up an hg repo for testing +dest=$1 +if [ -z "$dest" ]; then + echo You must specify a destination directory 1>&2 + exit 1 +fi + +rm -rf $dest +mkdir $dest +cd $dest +hg init + +echo "Hello world $RANDOM" > hello.txt +hg add hello.txt +hg commit -m "Adding hello" + +hg branch branch2 > /dev/null +echo "So long, farewell" >> hello.txt +hg commit -m "Changing hello on branch" + +hg checkout default +echo "Is this thing on?" >> hello.txt +hg commit -m "Last change on default" diff --git a/lib/python/test/test_util_commands.py b/lib/python/test/test_util_commands.py new file mode 100644 index 000000000..1b24a1762 --- /dev/null +++ b/lib/python/test/test_util_commands.py @@ -0,0 +1,25 @@ +import unittest, subprocess + +from util.commands import run_cmd, get_output + +class TestRunCmd(unittest.TestCase): + def testSimple(self): + self.assertEquals(run_cmd(['true']), 0) + + def testFailure(self): + self.assertRaises(subprocess.CalledProcessError, run_cmd, ['false']) + + def testOutput(self): + output = get_output(['echo', 'hello']) + self.assertEquals(output, 'hello\n') + + def testStdErr(self): + output = get_output(['bash', '-c', 'echo hello 1>&2'], include_stderr=True) + self.assertEquals(output, 'hello\n') + + def testNoStdErr(self): + output = get_output(['bash', '-c', 'echo hello 1>&2']) + self.assertEquals(output, '') + + def testBadOutput(self): + self.assertRaises(subprocess.CalledProcessError, get_output, ['false']) diff --git a/lib/python/test/test_util_hg.py b/lib/python/test/test_util_hg.py new file mode 100644 index 000000000..4ea3acdd8 --- /dev/null +++ b/lib/python/test/test_util_hg.py @@ -0,0 +1,201 @@ +import unittest +import tempfile +import shutil +import os +import subprocess + +from util.hg import clone, pull, update, hg_ver, mercurial, _make_absolute +from util.commands import run_cmd, get_output + +def getRevisions(dest): + retval = [] + for rev in get_output(['hg', 'log', '-R', dest, '--template', '{node|short}\n']).split('\n'): + rev = rev.strip() + if not rev: + continue + retval.append(rev) + return retval + +class TestMakeAbsolute(unittest.TestCase): + def testAboslutePath(self): + self.assertEquals(_make_absolute("/foo/bar"), "/foo/bar") + + def testRelativePath(self): + self.assertEquals(_make_absolute("foo/bar"), os.path.abspath("foo/bar")) + + def testHTTPPaths(self): + self.assertEquals(_make_absolute("http://foo/bar"), "http://foo/bar") + + def testAbsoluteFilePath(self): + self.assertEquals(_make_absolute("file:///foo/bar"), "file:///foo/bar") + + def testRelativeFilePath(self): + self.assertEquals(_make_absolute("file://foo/bar"), "file://%s/foo/bar" % os.getcwd()) + +class TestHg(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.repodir = os.path.join(self.tmpdir, 'repo') + run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), + self.repodir]) + + self.revisions = getRevisions(self.repodir) + self.wc = os.path.join(self.tmpdir, 'wc') + self.pwd = os.getcwd() + + def tearDown(self): + shutil.rmtree(self.tmpdir) + os.chdir(self.pwd) + + def testClone(self): + rev = clone(self.repodir, self.wc, update_dest=False) + self.assertEquals(rev, None) + self.assertEquals(self.revisions, getRevisions(self.wc)) + self.assertEquals(sorted(os.listdir(self.wc)), ['.hg']) + + def testCloneIntoNonEmptyDir(self): + os.mkdir(self.wc) + open(os.path.join(self.wc, 'test.txt'), 'w').write('hello') + clone(self.repodir, self.wc, update_dest=False) + self.failUnless(not os.path.exists(os.path.join(self.wc, 'test.txt'))) + + def testCloneUpdate(self): + rev = clone(self.repodir, self.wc, update_dest=True) + self.assertEquals(rev, self.revisions[0]) + + def testCloneBranch(self): + clone(self.repodir, self.wc, branch='branch2', + update_dest=False) + # On hg 1.6, we should only have a subset of the revisions + if hg_ver() >= (1,6,0): + self.assertEquals(self.revisions[1:], + getRevisions(self.wc)) + else: + self.assertEquals(self.revisions, + getRevisions(self.wc)) + + def testCloneUpdateBranch(self): + rev = clone(self.repodir, os.path.join(self.tmpdir, 'wc'), + branch="branch2", update_dest=True) + self.assertEquals(rev, self.revisions[1], self.revisions) + + def testCloneRevision(self): + clone(self.repodir, self.wc, + revision=self.revisions[0], update_dest=False) + # We'll only get a subset of the revisions + self.assertEquals(self.revisions[:1] + self.revisions[2:], + getRevisions(self.wc)) + + def testUpdateRevision(self): + rev = clone(self.repodir, self.wc, update_dest=False) + self.assertEquals(rev, None) + + rev = update(self.wc, revision=self.revisions[1]) + self.assertEquals(rev, self.revisions[1]) + + def testPull(self): + # Clone just the first rev + clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False) + self.assertEquals(getRevisions(self.wc), self.revisions[-1:]) + + # Now pull in new changes + rev = pull(self.repodir, self.wc, update_dest=False) + self.assertEquals(rev, None) + self.assertEquals(getRevisions(self.wc), self.revisions) + + def testPullRevision(self): + # Clone just the first rev + clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False) + self.assertEquals(getRevisions(self.wc), self.revisions[-1:]) + + # Now pull in just the last revision + rev = pull(self.repodir, self.wc, revision=self.revisions[0], update_dest=False) + self.assertEquals(rev, None) + + # We'll be missing the middle revision (on another branch) + self.assertEquals(getRevisions(self.wc), self.revisions[:1] + self.revisions[2:]) + + def testPullBranch(self): + # Clone just the first rev + clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False) + self.assertEquals(getRevisions(self.wc), self.revisions[-1:]) + + # Now pull in the other branch + rev = pull(self.repodir, self.wc, branch="branch2", update_dest=False) + self.assertEquals(rev, None) + + # On hg 1.6, we'll be missing the last revision (on another branch) + if hg_ver() >= (1,6,0): + self.assertEquals(getRevisions(self.wc), self.revisions[1:]) + else: + self.assertEquals(getRevisions(self.wc), self.revisions) + + def testPullUnrelated(self): + # Create a new repo + repo2 = os.path.join(self.tmpdir, 'repo2') + run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2]) + + self.assertNotEqual(self.revisions, getRevisions(repo2)) + + # Clone the original repo + clone(self.repodir, self.wc, update_dest=False) + + # Try and pull in changes from the new repo + self.assertRaises(subprocess.CalledProcessError, pull, repo2, self.wc, update_dest=False) + + def testMercurial(self): + rev = mercurial(self.repodir, self.wc) + self.assertEquals(rev, self.revisions[0]) + + def testMercurialRelativeDir(self): + os.chdir(os.path.dirname(self.repodir)) + + repo = os.path.basename(self.repodir) + wc = os.path.basename(self.wc) + + rev = mercurial(repo, wc, revision=self.revisions[-1]) + self.assertEquals(rev, self.revisions[-1]) + open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!") + + rev = mercurial(repo, wc) + self.assertEquals(rev, self.revisions[0]) + # Make sure our local file didn't go away + self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt'))) + + def testMercurialUpdateTip(self): + rev = mercurial(self.repodir, self.wc, revision=self.revisions[-1]) + self.assertEquals(rev, self.revisions[-1]) + open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!") + + rev = mercurial(self.repodir, self.wc) + self.assertEquals(rev, self.revisions[0]) + # Make sure our local file didn't go away + self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt'))) + + def testMercurialUpdateRev(self): + rev = mercurial(self.repodir, self.wc, revision=self.revisions[-1]) + self.assertEquals(rev, self.revisions[-1]) + open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!") + + rev = mercurial(self.repodir, self.wc, revision=self.revisions[0]) + self.assertEquals(rev, self.revisions[0]) + # Make sure our local file didn't go away + self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt'))) + + def testMercurialChangeRepo(self): + # Create a new repo + repo2 = os.path.join(self.tmpdir, 'repo2') + run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2]) + + self.assertNotEqual(self.revisions, getRevisions(repo2)) + + # Clone the original repo + mercurial(self.repodir, self.wc) + self.assertEquals(getRevisions(self.wc), self.revisions) + open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!") + + # Clone the new one + mercurial(repo2, self.wc) + self.assertEquals(getRevisions(self.wc), getRevisions(repo2)) + # Make sure our local file went away + self.failUnless(not os.path.exists(os.path.join(self.wc, 'test.txt'))) diff --git a/lib/python/util/__init__.py b/lib/python/util/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/python/util/commands.py b/lib/python/util/commands.py new file mode 100644 index 000000000..8ef2ffb5d --- /dev/null +++ b/lib/python/util/commands.py @@ -0,0 +1,77 @@ +"""Functions for running commands""" +import subprocess, os + +import logging +log = logging.getLogger(__name__) + +def run_cmd(cmd, **kwargs): + """Run cmd (a list of arguments). Raise subprocess.CalledProcessError if + the command exits with non-zero. If the command returns successfully, + return 0.""" + if 'cwd' in kwargs and kwargs['cwd']: + log.info("%s in %s", " ".join(cmd), kwargs['cwd']) + else: + log.info(" ".join(cmd)) + return subprocess.check_call(cmd, **kwargs) + +def get_output(cmd, include_stderr=False, **kwargs): + """Run cmd (a list of arguments) and return the output. If include_stderr + is set, stderr will be included in the output, otherwise it will be sent to + the caller's stderr stream. + + Warning that you shouldn't use this function to capture a large amount of + output (> a few thousand bytes), it will deadlock.""" + if include_stderr: + stderr = subprocess.STDOUT + else: + stderr = None + + if 'cwd' in kwargs and kwargs['cwd']: + log.info("%s in %s", " ".join(cmd), kwargs['cwd']) + else: + log.info(" ".join(cmd)) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, + **kwargs) + proc.wait() + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, cmd) + output = proc.stdout.read() + return output + +def remove_path(path): + """This is a replacement for shutil.rmtree that works better under + windows. Thanks to Bear at the OSAF for the code. + (Borrowed from buildbot.slave.commands)""" + log.debug("Removing %s", path) + if not os.path.exists(path): + # This handles broken links + if os.path.islink(path): + os.remove(path) + return + + if os.path.islink(path): + os.remove(path) + return + + # Verify the directory is read/write/execute for the current user + os.chmod(path, 0700) + + for name in os.listdir(path): + full_name = os.path.join(path, name) + # on Windows, if we don't have write permission we can't remove + # the file/directory either, so turn that on + if os.name == 'nt': + if not os.access(full_name, os.W_OK): + # I think this is now redundant, but I don't have an NT + # machine to test on, so I'm going to leave it in place + # -warner + os.chmod(full_name, 0600) + + if os.path.isdir(full_name): + remove_path(full_name) + else: + # Don't try to chmod links + if not os.path.islink(full_name): + os.chmod(full_name, 0700) + os.remove(full_name) + os.rmdir(path) diff --git a/lib/python/util/hg.py b/lib/python/util/hg.py new file mode 100644 index 000000000..6de96ce0c --- /dev/null +++ b/lib/python/util/hg.py @@ -0,0 +1,128 @@ +"""Functions for interacting with hg""" +import os, re, subprocess + +from util.commands import run_cmd, get_output, remove_path + +import logging +log = logging.getLogger(__name__) + +def _make_absolute(repo): + if repo.startswith("file://"): + path = repo[len("file://"):] + repo = "file://%s" % os.path.abspath(path) + elif "://" not in repo: + repo = os.path.abspath(repo) + return repo + +def get_revision(path): + """Returns which revision directory `path` currently has checked out.""" + return get_output(['hg', 'parent', '--template', '{node|short}'], cwd=path) + +def hg_ver(): + """Returns the current version of hg, as a tuple of + (major, minor, build)""" + ver_string = get_output(['hg', '-q', 'version']) + match = re.search("\(version ([0-9.]+)\)", ver_string) + if match: + bits = match.group(1).split(".") + if len(bits) < 3: + bits += (0,) + ver = tuple(int(b) for b in bits) + else: + ver = (0, 0, 0) + log.debug("Running hg version %s", ver) + return ver + +def update(dest, branch=None, revision=None): + """Updates working copy `dest` to `branch` or `revision`. If neither is + set then the working copy will be updated to the latest revision on the + current branch. Local changes will be discarded.""" + # If we have a revision, switch to that + if revision is not None: + cmd = ['hg', 'update', '-C', '-r', revision] + run_cmd(cmd, cwd=dest) + else: + # Check & switch branch + local_branch = get_output(['hg', 'branch'], cwd=dest).strip() + + cmd = ['hg', 'update', '-C'] + + # If this is different, checkout the other branch + if branch and branch != local_branch: + cmd.append(branch) + + run_cmd(cmd, cwd=dest) + return get_revision(dest) + +def clone(repo, dest, branch=None, revision=None, update_dest=True): + """Clones hg repo and places it at `dest`, replacing whatever else is + there. The working copy will be empty. + + If `revision` is set, only the specified revision and its ancestors will be + cloned. + + If `update_dest` is set, then `dest` will be updated to `revision` if set, + otherwise to `branch`, otherwise to the head of default.""" + if os.path.exists(dest): + remove_path(dest) + + cmd = ['hg', 'clone', '-U'] + if revision: + cmd.extend(['-r', revision]) + elif branch: + # hg >= 1.6 supports -b branch for cloning + ver = hg_ver() + if ver >= (1, 6, 0): + cmd.extend(['-b', branch]) + + cmd.extend([repo, dest]) + run_cmd(cmd) + + if update_dest: + return update(dest, branch, revision) + +def pull(repo, dest, branch=None, revision=None, update_dest=True): + """Pulls changes from hg repo and places it in `dest`. + + If `revision` is set, only the specified revision and its ancestors will be + pulled. + + If `update_dest` is set, then `dest` will be updated to `revision` if set, + otherwise to `branch`, otherwise to the head of default. """ + # Convert repo to an absolute path if it's a local repository + repo = _make_absolute(repo) + cmd = ['hg', 'pull'] + if revision is not None: + cmd.extend(['-r', revision]) + elif branch: + # hg >= 1.6 supports -b branch for cloning + if hg_ver() >= (1, 6, 0): + cmd.extend(['-b', branch]) + cmd.append(repo) + run_cmd(cmd, cwd=dest) + + if update_dest: + return update(dest, branch, revision) + +def mercurial(repo, dest, branch=None, revision=None): + """Makes sure that `dest` is has `revision` or `branch` checked out from + `repo`. + + Do what it takes to make that happen, including possibly clobbering + dest.""" + + # If dest exists, try pulling first + log.debug("mercurial: %s %s", repo, dest) + if os.path.exists(dest): + log.debug("%s exists, pulling", dest) + try: + # TODO: If revision is set, try updating before pulling? + return pull(repo, dest, branch, revision) + except subprocess.CalledProcessError: + log.warning("Error pulling changes into %s from %s; clobbering", dest, + repo) + log.debug("Exception:", exc_info=True) + remove_path(dest) + + # If it doesn't exist, clone it! + return clone(repo, dest, branch, revision) diff --git a/scripts/fuzzing/fuzzer.sh b/scripts/fuzzing/fuzzer.sh index 67aa8a8fa..33184a73b 100755 --- a/scripts/fuzzing/fuzzer.sh +++ b/scripts/fuzzing/fuzzer.sh @@ -4,9 +4,8 @@ eval `ssh-agent` ssh-add ~/.ssh/ffxbld_dsa trap "ssh-agent -k" EXIT -if [ -d fuzzing ]; then - (cd fuzzing; hg pull; hg update -C) -else - hg clone $HG_REPO -fi +SCRIPTS_DIR="$(dirname $0)/../.." + +python $SCRIPTS_DIR/buildfarm/utils/hgtool.py $HG_REPO fuzzing + python fuzzing/dom/automation/bot.py --remote-host "$FUZZ_REMOTE_HOST" --basedir "$FUZZ_BASE_DIR" diff --git a/scripts/nanojit/nanojit.sh b/scripts/nanojit/nanojit.sh new file mode 100755 index 000000000..086e95e54 --- /dev/null +++ b/scripts/nanojit/nanojit.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e +SCRIPTS_DIR="$(dirname $0)/../.." + +if [ -z "$HG_REPO" ]; then + HG_REPO="http://hg.mozilla.org/projects/nanojit-central" +fi + +python $SCRIPTS_DIR/buildfarm/utils/hgtool.py $HG_REPO nanojit-src +(cd nanojit-src; autoconf) + +if [ ! -d nanojit-build ]; then + mkdir nanojit-build +fi +cd nanojit-build +../nanojit-src/configure +make +make check