Permalink
Browse files

Add support for testing extensions

If the extensions/v8-jsshell sub-module is checked out, it is also
exported to the guest system (as git://host/v8-jsshell.git) thus
enabling it to be checked out there using "git submodule update".
(The extensions/v8-jsshell/v8 submodule is fetched from github.com.)

Extensions is enabled by a test (similar to how Critic is installed
and upgraded) that calls virtualbox.Instance.extend().  To improve
performance, the actual jsshell binary is cached (per instance and
extensions/v8-jsshell sub-module SHA-1) so that it doesn't need to
be built during every test run.  When a cached binary is found, the
"--fetch" and "--build" extend.py actions are skipped.

Extension testing is currently opt-in; the testing framework needs
to be started with --test-extensions for it to be enabled.  If not,
the virtualbox.Instance.extend() function raises a NotSupported
exception, which is like a TestFailed extension in that it causes
dependent tests to be skipped, but is not considered an error.
  • Loading branch information...
1 parent f14b12a commit 5bbbda8e91a3eef541b6fa2b1cb7c51e30f3108e @jensl committed with mo Apr 25, 2013
View
@@ -3,3 +3,4 @@
*.pyc
*.pyo
*~
+testing/cache
View
@@ -25,6 +25,10 @@ class TestFailure(Error):
"""Error raised for "expected" test failures."""
pass
+class NotSupported(Error):
+ """Error raised when a test (and its dependencies) are unsupported."""
+ pass
+
import virtualbox
import frontend
import expect
@@ -0,0 +1,7 @@
+Author = "Jens Lindstr\u00f6m"
+Description = "Extension used to test system extension support."
+
+[Page check]
+Description = "Simple page to check that the extension is installed and working."
+Script = check.js
+Function = check
@@ -0,0 +1,10 @@
+/* -*- mode: js; indent-tabs-mode: nil -*- */
+
+"use strict";
+
+function check() {
+ writeln(200);
+ writeln("Content-Type: text/json");
+ writeln();
+ writeln(JSON.stringify({ "status": "ok" }));
+}
@@ -0,0 +1 @@
+Hello world!
View
@@ -28,26 +28,56 @@
def main():
parser = argparse.ArgumentParser(description="Critic testing framework")
- parser.add_argument("--debug", help="Enable DEBUG level logging", action="store_true")
- parser.add_argument("--debug-mails", help="Log every mail sent by the tested system", action="store_true")
- parser.add_argument("--quiet", help="Disable INFO level logging", action="store_true")
- parser.add_argument("--coverage", help="Enable coverage measurement mode", action="store_true")
- parser.add_argument("--commit", help="Commit (symbolic ref or SHA-1) to test [default=HEAD]", default="HEAD")
- parser.add_argument("--upgrade-from", help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
- parser.add_argument("--strict-fs-permissions", help="Set strict file-system permissions in guest OS", action="store_true")
- parser.add_argument("--vbox-host", help="Host that's running VirtualBox [default=host]", default="host")
- parser.add_argument("--vm-identifier", help="VirtualBox instance name or UUID", required=True)
- parser.add_argument("--vm-hostname", help="VirtualBox instance hostname [default=VM_IDENTIFIER]")
- parser.add_argument("--vm-snapshot", help="VirtualBox snapshot (name or UUID) to restore [default=clean]", default="clean")
- parser.add_argument("--vm-ssh-port", help="VirtualBox instance SSH port [default=22]", type=int, default=22)
- parser.add_argument("--vm-http-port", help="VirtualBox instance HTTP port [default=80]", type=int, default=80)
- parser.add_argument("--git-daemon-port", help="Port to tell 'git daemon' to bind to", type=int)
- parser.add_argument("--pause-before", help="Pause testing before specified test(s)", action="append")
- parser.add_argument("--pause-after", help="Pause testing before specified test(s)", action="append")
- parser.add_argument("--pause-on-failure", help="Pause testing after each failed test", action="store_true")
- parser.add_argument("--pause-upgrade-loop", help="Support upgrading the tested system while paused", action="store_true")
- parser.add_argument("--pause-upgrade-hook", help="Command to run (locally) before upgrading", action="append")
- parser.add_argument("test", help="Specific tests to run [default=all]", nargs="*")
+
+ parser.add_argument("--debug", action="store_true",
+ help="Enable DEBUG level logging")
+ parser.add_argument("--debug-mails", action="store_true",
+ help="Log every mail sent by the tested system")
+ parser.add_argument("--quiet", action="store_true",
+ help="Disable INFO level logging")
+
+ parser.add_argument("--coverage", action="store_true",
+ help="Enable coverage measurement mode")
+ parser.add_argument("--commit", default="HEAD",
+ help="Commit (symbolic ref or SHA-1) to test [default=HEAD]")
+ parser.add_argument("--upgrade-from",
+ help="Commit (symbolic ref or SHA-1) to install first and upgrade from")
+ parser.add_argument("--strict-fs-permissions", action="store_true",
+ help="Set strict file-system permissions in guest OS")
+ parser.add_argument("--test-extensions", action="store_true",
+ help="Test extensions")
+
+ parser.add_argument("--vbox-host", default="host",
+ help="Host that's running VirtualBox [default=host]")
+ parser.add_argument("--vm-identifier", required=True,
+ help="VirtualBox instance name or UUID")
+ parser.add_argument("--vm-hostname",
+ help="VirtualBox instance hostname [default=VM_IDENTIFIER")
+ parser.add_argument("--vm-snapshot", default="clean",
+ help="VirtualBox snapshot (name or UUID) to restore [default=clean]")
+ parser.add_argument("--vm-ssh-port", type=int, default=22,
+ help="VirtualBox instance SSH port [default=22]")
+ parser.add_argument("--vm-http-port", type=int, default=80,
+ help="VirtualBox instance HTTP port [default=80]")
+
+ parser.add_argument("--git-daemon-port", type=int,
+ help="Port to tell 'git daemon' to bind to")
+ parser.add_argument("--cache-dir", default="testing/cache",
+ help="Directory where cache files are stored")
+
+ parser.add_argument("--pause-before", action="append",
+ help="Pause testing before specified test(s)")
+ parser.add_argument("--pause-after", action="append",
+ help="Pause testing before specified test(s)")
+ parser.add_argument("--pause-on-failure", action="store_true",
+ help="Pause testing after each failed test")
+ parser.add_argument("--pause-upgrade-loop", action="store_true",
+ help="Support upgrading the tested system while paused")
+ parser.add_argument("--pause-upgrade-hook", action="append",
+ help="Command to run (locally) before upgrading")
+
+ parser.add_argument("test", nargs="*",
+ help="Specific tests to run [default=all]")
arguments = parser.parse_args()
@@ -118,6 +148,18 @@ def exception(self, message):
logger.error("Required software missing; see testing/USAGE.md for details.")
return
+ if arguments.test_extensions:
+ # Check that the v8-jsshell submodule is checked out if extension
+ # testing was requested.
+ output = subprocess.check_output(["git", "submodule", "status",
+ "installation/externals/v8-jsshell"])
+ if output.startswith("-"):
+ logger.error("""\
+The v8-jsshell submodule must be checked for extension testing. Please run
+ git submodule update --init installation/externals/v8-jsshell
+first or run this script without --test-extensions.""")
+ return
+
# Note: we are not ignoring typical temporary editor files such as the
# ".#<name>" files created by Emacs when a buffer has unsaved changes. This
# is because unsaved changes in an editor is probably also something you
@@ -180,16 +222,10 @@ def exception(self, message):
http_port=arguments.vm_http_port)
instance = testing.virtualbox.Instance(
- vboxhost=arguments.vbox_host,
- identifier=arguments.vm_identifier,
- snapshot=arguments.vm_snapshot,
- hostname=arguments.vm_hostname,
- ssh_port=arguments.vm_ssh_port,
+ arguments,
install_commit=(install_commit, install_commit_description),
upgrade_commit=(upgrade_commit, upgrade_commit_description),
- frontend=frontend,
- strict_fs_permissions=arguments.strict_fs_permissions,
- coverage=arguments.coverage)
+ frontend=frontend)
except testing.Error as error:
logger.error(error.message)
return
@@ -335,6 +371,12 @@ def run_tests(directory):
has_failed = True
else:
return
+ except testing.NotSupported as not_supported:
+ logger.info("Test not supported: %s" % not_supported.message)
+ if "/" in directory:
+ has_failed = True
+ else:
+ return
else:
if path in pause_after:
pause()
View
@@ -32,7 +32,11 @@ def __init__(self, command, output):
def _git(args, **kwargs):
argv = ["git"] + args
- testing.logger.debug("Running: %s" % " ".join(argv))
+ if "cwd" in kwargs:
+ cwd = " (in %s)" % kwargs["cwd"]
+ else:
+ cwd = ""
+ testing.logger.debug("Running: %s%s" % (" ".join(argv), cwd))
try:
return subprocess.check_output(
argv, stdin=open("/dev/null"), stderr=subprocess.STDOUT, **kwargs)
@@ -52,7 +56,7 @@ def __init__(self, host, port, tested_commit, vm_hostname):
else:
self.url = "git://%s/critic.git" % host
- testing.logger.debug("Creating temporary repository: %s" % self.path)
+ testing.logger.debug("Creating temporary repositories in: %s" % self.base_path)
_git(["clone", "--bare", os.getcwd(), "critic.git"],
cwd=self.base_path)
@@ -64,6 +68,55 @@ def __init__(self, host, port, tested_commit, vm_hostname):
self.push(tested_commit)
+ def submodule_sha1(repository_path, parent_sha1, submodule_path):
+ try:
+ lstree = _git(["ls-tree", parent_sha1, submodule_path],
+ cwd=repository_path)
+ except GitCommandError:
+ # Sub-module doesn't exist? Will probably fail later, but
+ # doesn't need to fail here.
+ return None
+ mode, object_type, sha1, path = lstree.strip().split(None, 3)
+ if object_type != "commit":
+ # Odd. The repository doesn't look at all like we expect.
+ return None
+ return sha1
+
+ if os.path.exists("installation/externals/v8-jsshell/.git"):
+ v8_jsshell_path = os.path.join(os.getcwd(), "installation/externals/v8-jsshell")
+ _git(["clone", "--bare", v8_jsshell_path, "v8-jsshell.git"],
+ cwd=self.base_path)
+ self.v8_jsshell_path = os.path.join(self.base_path, "v8-jsshell.git")
+ v8_jsshell_sha1 = submodule_sha1(os.getcwd(), tested_commit,
+ "installation/externals/v8-jsshell")
+ if v8_jsshell_sha1:
+ _git(["push", "--quiet", "--force", self.v8_jsshell_path,
+ v8_jsshell_sha1 + ":refs/heads/master"],
+ cwd=v8_jsshell_path)
+ else:
+ self.v8_jsshell_path = None
+ v8_jsshell_sha1 = None
+
+ if os.path.exists("installation/externals/v8-jsshell/v8/.git"):
+ v8_path = os.path.join(os.getcwd(), "installation/externals/v8-jsshell/v8")
+ _git(["clone", "--bare", v8_path, "v8/v8.git"],
+ cwd=self.base_path)
+ self.v8_path = os.path.join(self.base_path, "v8/v8.git")
+ if port:
+ self.v8_url = "git://%s:%d/v8/v8.git" % (host, port)
+ else:
+ self.v8_url = "git://%s/v8/v8.git" % host
+ if v8_jsshell_sha1:
+ v8_sha1 = submodule_sha1("installation/externals/v8-jsshell",
+ v8_jsshell_sha1, "v8")
+ if v8_sha1:
+ _git(["push", "--quiet", "--force", self.v8_path,
+ v8_sha1 + ":refs/heads/master"],
+ cwd=v8_path)
+ else:
+ self.v8_path = None
+ self.v8_url = None
+
def push(self, commit):
_git(["push", "--quiet", "--force", self.path,
"%s:refs/heads/master" % commit])
@@ -74,6 +127,10 @@ def export(self):
if self.port:
argv.append("--port=%d" % self.port)
argv.append(self.path)
+ if self.v8_jsshell_path:
+ argv.append(self.v8_jsshell_path)
+ if self.v8_path:
+ argv.append(self.v8_path)
self.daemon = subprocess.Popen(argv)
@@ -85,7 +142,11 @@ def export(self):
testing.logger.error("Failed to export repository!")
return False
- testing.logger.info("Exported repository: %s" % self.path)
+ testing.logger.debug("Exported repository: %s" % self.path)
+ if self.v8_jsshell_path:
+ testing.logger.debug("Exported repository: %s" % self.v8_jsshell_path)
+ if self.v8_path:
+ testing.logger.debug("Exported repository: %s" % self.v8_path)
return True
def run(self, args):
@@ -0,0 +1,2 @@
+# Enable extensions.
+instance.extend(repository)
@@ -0,0 +1,22 @@
+frontend.page("tutorial",
+ expect={ "document_title": testing.expect.document_title(u"Tutorials"),
+ "content_title": testing.expect.paleyellow_title(0, u"Tutorials"),
+ "pageheader_links": testing.expect.pageheader_links("anonymous",
+ "extensions"),
+ "script_user": testing.expect.script_no_user() })
+
+frontend.page("tutorial",
+ params={ "item": "extensions" },
+ expect={ "document_title": testing.expect.document_title(u"Critic Extensions"),
+ "content_title": testing.expect.paleyellow_title(0, u"Critic Extensions"),
+ "pageheader_links": testing.expect.pageheader_links("anonymous",
+ "extensions"),
+ "script_user": testing.expect.script_no_user() })
+
+frontend.page("tutorial",
+ params={ "item": "extensions-api" },
+ expect={ "document_title": testing.expect.document_title(u"Critic Extensions API"),
+ "content_title": testing.expect.paleyellow_title(0, u"Critic Extensions API"),
+ "pageheader_links": testing.expect.pageheader_links("anonymous",
+ "extensions"),
+ "script_user": testing.expect.script_no_user() })
@@ -0,0 +1,25 @@
+frontend.page(
+ "manageextensions",
+ expect={ "document_title": testing.expect.document_title(u"Manage Extensions"),
+ "content_title": testing.expect.paleyellow_title(0, u"Available Extensions"),
+ "pageheader_links": testing.expect.pageheader_links("anonymous",
+ "extensions"),
+ "script_user": testing.expect.script_anonymous_user() })
+
+frontend.page(
+ "manageextensions",
+ params={ "what": "available" },
+ expect={ "document_title": testing.expect.document_title(u"Manage Extensions"),
+ "content_title": testing.expect.paleyellow_title(0, u"Available Extensions"),
+ "pageheader_links": testing.expect.pageheader_links("anonymous",
+ "extensions"),
+ "script_user": testing.expect.script_anonymous_user() })
+
+frontend.page(
+ "manageextensions",
+ params={ "what": "installed" },
+ expect={ "document_title": testing.expect.document_title(u"Manage Extensions"),
+ "content_title": testing.expect.paleyellow_title(0, u"Installed Extensions"),
+ "pageheader_links": testing.expect.pageheader_links("anonymous",
+ "extensions"),
+ "script_user": testing.expect.script_anonymous_user() })
@@ -0,0 +1,74 @@
+def check_extension(installed):
+ def check(document):
+ tr_items = document.findAll("tr", attrs={ "class": "item" })
+
+ for tr_item in tr_items:
+ td_name = tr_item.find("td", attrs={ "class": "name" })
+ testing.expect.check("Extension:", td_name.string)
+
+ td_value = tr_item.find("td", attrs={ "class": "value" })
+ span_name = td_value.find("span", attrs={ "class": "name" })
+
+ if span_name.contents[0].string != "SystemExtension":
+ # Wrong extension.
+ continue
+
+ testing.expect.check(1, len(span_name.contents))
+
+ span_installed = td_value.find("span", attrs={ "class": "installed" })
+ if installed:
+ testing.expect.check(" [installed]", span_installed.string)
+ elif span_installed:
+ testing.expect.check("<no installed indicator>",
+ "<found installed indicator>")
+
+ return
+ else:
+ testing.expect.check("<SystemExtension entry>",
+ "<expected content not found>")
+
+ return check
+
+def check_helloworld(document):
+ testing.expect.check("Hello world!\n", document)
+
+try:
+ instance.execute(
+ ["sudo", "mkdir", "/var/lib/critic/extensions",
+ "&&",
+ "sudo", "cp", "-R", "~/critic/testing/input/SystemExtension",
+ "/var/lib/critic/extensions",
+ "&&",
+ "sudo", "chown", "-R", "critic.critic",
+ "/var/lib/critic/extensions",
+ "&&",
+ "sudo", "chmod", "-R", "u+rwX,go+rX",
+ "/var/lib/critic/extensions"])
+except testing.InstanceError as error:
+ raise testing.TestFailure(error.message)
+
+with frontend.signin("alice"):
+ frontend.page(
+ "manageextensions",
+ expect={ "system_extension": check_extension(installed=False) })
+
+ frontend.operation(
+ "installextension",
+ data={ "extension_name": "SystemExtension" })
+
+ frontend.page(
+ "manageextensions",
+ expect={ "system_extension": check_extension(installed=True) })
+
+ frontend.operation(
+ "check",
+ data={})
+
+ frontend.page(
+ "extension-resource/SystemExtension/HelloWorld.txt",
+ expected_content_type="text/plain",
+ expect={ "hello_world": check_helloworld })
+
+frontend.page(
+ "manageextensions",
+ expect={ "system_extension": check_extension(False) })
Oops, something went wrong.

0 comments on commit 5bbbda8

Please sign in to comment.