Skip to content
Browse files

Add script (extend.py) for enabling extensions

Also add the js/v8 flavor extension library code.
  • Loading branch information...
1 parent cfffe85 commit f14b12a2ad7c074abedc0b074751b2ebc28c92f6 @jensl committed with mo Feb 12, 2014
Showing with 8,100 additions and 927 deletions.
  1. +4 −0 .gitmodules
  2. +364 −0 extend.py
  3. +13 −14 install.py
  4. +2 −0 installation/__init__.py
  5. +8 −0 installation/apache.py
  6. +77 −68 installation/config.py
  7. +21 −28 installation/data/dbschema.extensions.sql
  8. +11 −9 installation/database.py
  9. +37 −0 installation/extensions.py
  10. +1 −0 installation/externals/v8-jsshell
  11. +9 −0 installation/initd.py
  12. +1 −0 installation/prefs.py
  13. +59 −59 installation/prereqs.py
  14. +7 −2 installation/system.py
  15. +13 −9 installation/templates/configuration/extensions.py
  16. +29 −0 pythonversion.py
  17. +25 −0 src/background/maintenance.py
  18. +0 −80 src/batchprocessor.py
  19. +27 −12 src/critic.py
  20. +0 −5 src/data/preferences.json
  21. +9 −0 src/dbutils/user.py
  22. +1 −1 src/extensions/__init__.py
  23. +25 −27 src/extensions/execute.py
  24. +259 −131 src/extensions/extension.py
  25. +114 −22 src/extensions/installation.py
  26. +100 −76 src/extensions/manifest.py
  27. +25 −17 src/extensions/resource.py
  28. +0 −1 src/extensions/role/__init__.py
  29. +222 −166 src/extensions/role/inject.py
  30. +70 −37 src/extensions/role/page.py
  31. +0 −98 src/extensions/role/processchanges.py
  32. +66 −55 src/extensions/role/processcommits.py
  33. +1,054 −0 src/library/js/v8/critic-batch.js
  34. +185 −0 src/library/js/v8/critic-branch.js
  35. +506 −0 src/library/js/v8/critic-changeset.js
  36. +422 −0 src/library/js/v8/critic-comment.js
  37. +255 −0 src/library/js/v8/critic-commitset.js
  38. +255 −0 src/library/js/v8/critic-dashboard.js
  39. +234 −0 src/library/js/v8/critic-file.js
  40. +109 −0 src/library/js/v8/critic-filters.js
  41. +129 −0 src/library/js/v8/critic-filterstransaction.js
  42. +888 −0 src/library/js/v8/critic-git.js
  43. +192 −0 src/library/js/v8/critic-html.js
  44. +163 −0 src/library/js/v8/critic-launcher-fork.js
  45. +59 −0 src/library/js/v8/critic-launcher.js
  46. +116 −0 src/library/js/v8/critic-log.js
  47. +24 −0 src/library/js/v8/critic-mail.js
  48. +738 −0 src/library/js/v8/critic-review.js
  49. +339 −0 src/library/js/v8/critic-statistics.js
  50. +68 −0 src/library/js/v8/critic-storage.js
  51. +150 −0 src/library/js/v8/critic-text.js
  52. +246 −0 src/library/js/v8/critic-user.js
  53. +316 −0 src/library/js/v8/critic.js
  54. +53 −10 src/operation/extensioninstallation.py
Sorry, we could not display the entire diff because it was too big.
View
4 .gitmodules
@@ -1,3 +1,7 @@
[submodule "installation/externals/chosen"]
path = installation/externals/chosen
url = ../chosen.git
+
+[submodule "installation/externals/v8-jsshell"]
+ path = installation/externals/v8-jsshell
+ url = ../v8-jsshell.git
View
364 extend.py
@@ -0,0 +1,364 @@
+# -*- mode: python; encoding: utf-8 -*-
+#
+# Copyright 2012 Jens Lindström, Opera Software ASA
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import os
+import sys
+
+# To avoid accidentally creating files owned by root.
+sys.dont_write_bytecode = True
+
+# Python version check is done before imports below so that python
+# 2.6/2.5 users can see the error message.
+import pythonversion
+pythonversion.check()
+
+import argparse
+import subprocess
+import multiprocessing
+import tempfile
+import pwd
+
+import installation
+
+parser = argparse.ArgumentParser(description="Critic extension support installation script",
+ epilog="""\
+Critic extension support is activated by simply running (as root):
+
+ # python extend.py
+
+For finer control over the script's operation you can invoke it with one
+or more of the action arguments:
+
+ --prereqs, --fetch, --build, --install and --enable
+
+This can for instance be used to build the v8-jsshell executable on a
+system where Critic has not been installed.""",
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+
+# Uses default values for everything that has a default value (and isn't
+# overridden by other command-line arguments) and signals an error for anything
+# that doesn't have a default value and isn't set by a command-line argument.
+parser.add_argument("--headless", help=argparse.SUPPRESS, action="store_true")
+
+class DefaultBinDir:
+ pass
+
+basic = parser.add_argument_group("basic options")
+basic.add_argument("--etc-dir", help="directory where the Critic system configuration is stored [default=/etc/critic]", action="store", default="/etc/critic")
+basic.add_argument("--identity", help="system identity to upgrade [default=main]", action="store", default="main")
+basic.add_argument("--bin-dir", help="directory where the extension host executable is installed [default=/usr/lib/critic/$IDENTITY/bin]", action="store", default=DefaultBinDir)
+basic.add_argument("--no-compiler-check", help="disable compiler version check", action="store_true")
+basic.add_argument("--dry-run", "-n", help="produce output but don't modify the system at all", action="store_true")
+basic.add_argument("--libcurl-flavor", help="libcurl flavor (openssl, gnutls or nss) or install", choices=["openssl", "gnutls", "nss"])
+
+actions = parser.add_argument_group("actions")
+actions.add_argument("--prereqs", help="(check for and) install prerequisite software", action="store_true")
+actions.add_argument("--fetch", help="fetch the extension host source code", action="store_true")
+actions.add_argument("--build", help="build the extension host executable", action="store_true")
+actions.add_argument("--install", help="install the extension host executable", action="store_true")
+actions.add_argument("--enable", help="enable extension support in Critic's configuration", action="store_true")
+
+actions.add_argument("--with-v8-jsshell", help="v8-jsshell repository URL [default=../v8-jsshell.git]", metavar="URL")
+actions.add_argument("--with-v8", help="v8 repository URL [default=git://github.com/v8/v8.git]", metavar="URL")
+
+# Useful to speed up repeated building from clean repositories; used
+# by the testing framework.
+actions.add_argument("--export-v8-dependencies", help=argparse.SUPPRESS)
+actions.add_argument("--import-v8-dependencies", help=argparse.SUPPRESS)
+
+arguments = parser.parse_args()
+
+if arguments.headless:
+ installation.input.headless = True
+
+import installation
+
+is_root = os.getuid() == 0
+
+prereqs = arguments.prereqs
+fetch = arguments.fetch
+build = arguments.build
+install = arguments.install
+enable = arguments.enable
+
+if not any([prereqs, fetch, build, install, enable]) \
+ and arguments.export_v8_dependencies is None \
+ and arguments.import_v8_dependencies is None:
+ prereqs = fetch = build = install = enable = True
+
+libcurl = False
+
+if any([prereqs, install, enable]) and not is_root:
+ print """
+ERROR: You need to run this script as root.
+"""
+ sys.exit(1)
+
+git = os.environ.get("GIT", "git")
+
+if install or enable:
+ data = installation.utils.read_install_data(arguments)
+
+ if data is not None:
+ git = data["installation.prereqs.git"]
+
+ installed_sha1 = data["sha1"]
+ current_sha1 = installation.utils.run_git([git, "rev-parse", "HEAD"],
+ cwd=installation.root_dir).strip()
+
+ if installed_sha1 != current_sha1:
+ print """
+ERROR: You should to run upgrade.py to upgrade to the current commit before
+ using this script to enable extension support.
+"""
+ sys.exit(1)
+
+if arguments.bin_dir is DefaultBinDir:
+ bin_dir = os.path.join("/usr/lib/critic", arguments.identity, "bin")
+else:
+ bin_dir = arguments.bin_dir
+
+if "CXX" in os.environ:
+ compiler = os.environ["CXX"]
+
+ try:
+ subprocess.check_output([compiler, "--help"])
+ except OSError as error:
+ print """
+ERROR: %r (from $CXX) does not appear to be a valid compiler.
+""" % compiler
+ sys.exit(1)
+else:
+ compiler = "g++"
+
+def check_libcurl():
+ fd, empty_cc = tempfile.mkstemp(".cc")
+ os.close(fd)
+
+ try:
+ subprocess.check_output([compiler, "-include", "curl/curl.h", "-c", empty_cc, "-o", "/dev/null"],
+ stderr=subprocess.STDOUT)
+ return True
+ except subprocess.CalledProcessError as error:
+ if "curl/curl.h" in error.output:
+ return False
+ raise
+ finally:
+ os.unlink(empty_cc)
+
+def missing_packages():
+ packages = []
+
+ if not installation.prereqs.find_executable("svn"):
+ packages.append("subversion")
+ if not installation.prereqs.find_executable("make"):
+ packages.append("make")
+ if "CXX" not in os.environ and not installation.prereqs.find_executable("g++"):
+ packages.append("g++")
+ pg_config = installation.prereqs.find_executable("pg_config")
+ if pg_config:
+ try:
+ subprocess.check_output(["pg_config"], stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError:
+ # Just installing the PostgreSQL database server might install
+ # a dummy pg_config that just outputs an error message.
+ pg_config = None
+ if not pg_config:
+ packages.append("libpq-dev")
+
+ return packages
+
+if prereqs:
+ packages = missing_packages()
+
+ if packages:
+ installation.prereqs.install_packages(arguments, *packages)
+
+ if not check_libcurl():
+ if arguments.libcurl_flavor:
+ installation.prereqs.install_packages(
+ arguments, "libcurl4-%s-dev" % arguments.libcurl_flavor)
+ else:
+ print """
+No version of libcurl-dev appears to be install. There are usually multiple
+versions available to install using different libraries (openssl, gnutls or nss)
+for secure communication. If curl is already installed, you probably need to
+install a matching version of libcurl-dev.
+
+This script can install any one of them, or build the extension host executable
+without URL loading support ("none").
+
+Available choices are: "openssl", "gnutls", "nss"
+Also: "none", "abort"
+"""
+
+ def check(string):
+ if string not in ("openssl", "gnutls", "nss", "none", "abort"):
+ return 'please answer "openssl", "gnutls", "nss", "none" or "abort"'
+
+ choice = installation.input.string("Install libcurl-dev version?", "none")
+
+ if choice in ("openssl", "gnutls", "nss"):
+ installation.prereqs.install_packages(arguments, "libcurl4-%s-dev" % choice)
+ elif choice == "abort":
+ print """
+ERROR: Installation aborted.
+"""
+ sys.exit(1)
+
+env = os.environ.copy()
+
+if build and not arguments.no_compiler_check:
+ version = subprocess.check_output([compiler, "--version"])
+ if version.startswith("g++"):
+ version = subprocess.check_output([compiler, "-dumpversion"]).strip().split(".")
+ if (int(version[0]), int(version[1])) < (4, 7):
+ print """
+ERROR: GCC version 4.7 or later required to build v8-jsshell.
+HINT: Set $CXX to use a different compiler than '%s', or use
+ --no-compiler-check to try to build anyway.
+""" % compiler
+ sys.exit(1)
+ else:
+ if "clang" in version:
+ note_clang = "NOTE: CLang (version 3.2 and earlier) is known not to work.\n"
+ else:
+ note_clang = ""
+
+ print """
+ERROR: GCC (version 4.7 or later) required to build v8-jsshell.
+%sHINT: Set $CXX to use a different compiler than '%s', or use
+ --no-compiler-check to try to build anyway.
+""" % (note_clang, compiler)
+ sys.exit(1)
+
+env["compiler"] = compiler
+env["v8static"] = "yes"
+env["postgresql"] = "yes"
+
+if check_libcurl():
+ env["libcurl"] = "yes"
+
+root = os.path.dirname(os.path.abspath(sys.argv[0]))
+v8_jsshell = os.path.join(root, "installation/externals/v8-jsshell")
+
+def do_unprivileged_work():
+ if is_root:
+ stat = os.stat(sys.argv[0])
+ os.environ["USER"] = pwd.getpwuid(stat.st_uid).pw_name
+ os.environ["HOME"] = pwd.getpwuid(stat.st_uid).pw_dir
+ os.setgid(stat.st_gid)
+ os.setuid(stat.st_uid)
+
+ if fetch:
+ def fetch_submodule(cwd, submodule, url=None):
+ subprocess.check_call(
+ [git, "submodule", "init", submodule],
+ cwd=cwd)
+ if url:
+ subprocess.check_call(
+ [git, "config", "submodule.%s.url" % submodule, url],
+ cwd=cwd)
+ subprocess.check_call(
+ [git, "submodule", "update", submodule],
+ cwd=cwd)
+
+ fetch_submodule(root, "installation/externals/v8-jsshell",
+ arguments.with_v8_jsshell)
+ fetch_submodule(v8_jsshell, "v8", arguments.with_v8)
+
+ if arguments.import_v8_dependencies or arguments.export_v8_dependencies:
+ argv = ["make", "v8dependencies"]
+
+ if arguments.import_v8_dependencies:
+ argv.append("v8importdepsfrom=" + arguments.import_v8_dependencies)
+ if arguments.export_v8_dependencies:
+ argv.append("v8exportdepsto=" + arguments.export_v8_dependencies)
+
+ subprocess.check_call(argv, cwd=v8_jsshell)
+
+ if build:
+ subprocess.check_call(
+ ["make", "-j%d" % multiprocessing.cpu_count()],
+ cwd=v8_jsshell, env=env)
+
+if fetch or build \
+ or arguments.import_v8_dependencies \
+ or arguments.export_v8_dependencies:
+ if is_root:
+ unprivileged = multiprocessing.Process(target=do_unprivileged_work)
+ unprivileged.start()
+ unprivileged.join()
+ else:
+ do_unprivileged_work()
+
+if install or enable:
+ etc_path = os.path.join(arguments.etc_dir, arguments.identity)
+
+ sys.path.insert(0, etc_path)
+
+ import configuration
+
+ executable = configuration.extensions.FLAVORS.get("js/v8", {}).get("executable")
+
+ if not executable or not os.access(executable, os.X_OK):
+ executable = os.path.join(bin_dir, "v8-jsshell")
+
+if install:
+ if not os.path.isdir(os.path.dirname(executable)):
+ os.makedirs(os.path.dirname(executable))
+
+ subprocess.check_call(
+ ["install", os.path.join(v8_jsshell, "out", "jsshell"), executable])
+
+if enable and not configuration.extensions.ENABLED:
+ try:
+ subprocess.check_output(
+ ["su", "-s", "/bin/bash",
+ "-c", "psql -q -c 'SELECT 1 FROM extensions LIMIT 1'",
+ configuration.base.SYSTEM_USER_NAME],
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError:
+ installation.database.psql_import(
+ "installation/data/dbschema.extensions.sql",
+ configuration.base.SYSTEM_USER_NAME)
+
+ data = { "installation.system.username": configuration.base.SYSTEM_USER_NAME,
+ "installation.system.groupname": configuration.base.SYSTEM_GROUP_NAME,
+ "installation.extensions.enabled": True,
+ "installation.extensions.critic_v8_jsshell": executable,
+ "installation.extensions.default_flavor": "js/v8" }
+
+ installation.system.fetch_uid_gid()
+
+ installation.paths.mkdir(configuration.extensions.INSTALL_DIR)
+ installation.paths.mkdir(configuration.extensions.WORKCOPY_DIR)
+
+ compilation_failed = []
+
+ if installation.config.update_file(os.path.join(etc_path, "configuration"),
+ "extensions.py", data, arguments,
+ compilation_failed):
+ if compilation_failed:
+ print
+ print "ERROR: Update aborted."
+ print
+
+ installation.config.undo()
+ sys.exit(1)
+
+ installation.initd.restart(arguments.identity)
+ installation.apache.restart()
View
27 install.py
@@ -19,30 +19,23 @@
import stat
import traceback
-if os.getuid() != 0:
- print """
-ERROR: This script must be run as root.
-"""
- sys.exit(1)
+# To avoid accidentally creating files owned by root.
+sys.dont_write_bytecode = True
# Python version check is done before imports below so
# that python 2.6/2.5 users can see the error message.
-if sys.version_info[0] != 2 or sys.version_info[1] < 7:
- print """\
-Unsupported Python version! Critic requires Python 2.7.x or later,
-but not Python 3.x. This script must be run in the Python interpreter
-that will be used to run Critic."""
- sys.exit(2)
+import pythonversion
+pythonversion.check("""\
+NOTE: This script must be run in the Python interpreter that will be
+used to run Critic.
+""")
if sys.flags.optimize > 0:
print """
ERROR: Please run this script without -O or -OO options.
"""
sys.exit(1)
-# To avoid accidentally creating files owned by root.
-sys.dont_write_bytecode = True
-
import argparse
import installation
@@ -79,6 +72,12 @@
arguments = parser.parse_args()
+if os.getuid() != 0:
+ print """
+ERROR: This script must be run as root.
+"""
+ sys.exit(1)
+
if os.path.exists(os.path.join(installation.root_dir, ".installed")):
print """
ERROR: Found an .installed file in the directory you're installing from.
View
2 installation/__init__.py
@@ -43,12 +43,14 @@
import prefs
import git
import migrate
+import extensions
modules = [prereqs,
system,
paths,
files,
database,
+ extensions,
config,
apache,
criticctl,
View
8 installation/apache.py
@@ -65,6 +65,14 @@ def stop():
return False
return True
+def restart():
+ print
+ try:
+ subprocess.check_call(["service", "apache2", "restart"])
+ except subprocess.CalledProcessError:
+ return False
+ return True
+
def prepare(mode, arguments, data):
global pass_auth, site_suffix, default_site
View
145 installation/config.py
@@ -587,102 +587,111 @@ def install(data):
return True
-def upgrade(arguments, data):
+def update_file(target_dir, entry, data, arguments, compilation_failed):
global modified_files
import configuration
source_dir = os.path.join(installation.root_dir, "installation", "templates", "configuration")
- target_dir = os.path.join(data["installation.paths.etc_dir"], arguments.identity, "configuration")
compilation_failed = False
system_uid = pwd.getpwnam(data["installation.system.username"]).pw_uid
system_gid = grp.getgrnam(data["installation.system.groupname"]).gr_gid
- no_changes = True
-
- for entry in os.listdir(source_dir):
- source_path = os.path.join(source_dir, entry)
- target_path = os.path.join(target_dir, entry)
- backup_path = os.path.join(target_dir, "_" + entry)
+ source_path = os.path.join(source_dir, entry)
+ target_path = os.path.join(target_dir, entry)
+ backup_path = os.path.join(target_dir, "_" + entry)
- source = open(source_path, "r").read().decode("utf-8") % data
+ source = open(source_path, "r").read().decode("utf-8") % data
- if not os.path.isfile(target_path):
- write_target = True
- no_changes = False
- else:
- if open(target_path).read().decode("utf-8") == source: continue
-
- no_changes = False
+ if not os.path.isfile(target_path):
+ write_target = True
+ else:
+ if open(target_path).read().decode("utf-8") == source:
+ return False
- def generateVersion(label, path):
- if label == "updated":
- with open(path, "w") as target:
- target.write(source.encode("utf-8"))
+ def generateVersion(label, path):
+ if label == "updated":
+ with open(path, "w") as target:
+ target.write(source.encode("utf-8"))
- update_query = installation.utils.UpdateModifiedFile(
- arguments,
- message="""\
+ update_query = installation.utils.UpdateModifiedFile(
+ arguments,
+ message="""\
A configuration file is about to be updated. Please check that no
local modifications are being overwritten.
- Current version: %(current)s
- Updated version: %(updated)s
+Current version: %(current)s
+Updated version: %(updated)s
Please note that if any configuration options were added in the
updated version, the system will most likely break if you do not
either install the updated version or manually transfer the new
configuration options to the existing version.
""",
- versions={ "current": target_path,
- "updated": target_path + ".new" },
- options=[ ("i", "install the updated version"),
- ("k", "keep the current version"),
- ("d", ("current", "updated")) ],
- generateVersion=generateVersion)
-
- write_target = update_query.prompt() == "i"
-
- if write_target:
- print "Updated file: %s" % target_path
-
- if not arguments.dry_run:
- if os.path.isfile(target_path):
- os.rename(target_path, backup_path)
- renamed.append((target_path, backup_path))
-
- with open(target_path, "w") as target:
- created_file.append(target_path)
- if entry in SENSITIVE_FILES:
- # May contain secrets (passwords.)
- mode = 0600
- else:
- # Won't contain secrets.
- mode = 0640
- os.chmod(target_path, mode)
- os.chown(target_path, system_uid, system_gid)
- target.write(source.encode("utf-8"))
+ versions={ "current": target_path,
+ "updated": target_path + ".new" },
+ options=[ ("i", "install the updated version"),
+ ("k", "keep the current version"),
+ ("d", ("current", "updated")) ],
+ generateVersion=generateVersion)
+
+ write_target = update_query.prompt() == "i"
+
+ if write_target:
+ print "Updated file: %s" % target_path
- path = os.path.join("configuration", entry)
- if not compile_file(path):
- compilation_failed = True
+ if not arguments.dry_run:
+ if os.path.isfile(target_path):
+ os.rename(target_path, backup_path)
+ renamed.append((target_path, backup_path))
+
+ with open(target_path, "w") as target:
+ created_file.append(target_path)
+ if entry in SENSITIVE_FILES:
+ # May contain secrets (passwords.)
+ mode = 0600
else:
- # The module's name (relative the 'configuration' package)
- # is the base name minus the trailing ".py".
- module_name = os.path.basename(target_path)[:-3]
+ # Won't contain secrets.
+ mode = 0640
+ os.chmod(target_path, mode)
+ os.chown(target_path, system_uid, system_gid)
+ target.write(source.encode("utf-8"))
+
+ path = os.path.join("configuration", entry)
+ if not compile_file(path):
+ compilation_failed.append(path)
+ else:
+ # The module's name (relative the 'configuration' package)
+ # is the base name minus the trailing ".py".
+ module_name = os.path.basename(target_path)[:-3]
+
+ if module_name != "__init__" \
+ and hasattr(configuration, module_name):
+ # Reload the updated module so that code executing later
+ # sees added configuration options. (It will also see
+ # removed configuration options, but that is unlikely to
+ # be a problem.)
+ reload(getattr(configuration, module_name))
+
+ modified_files += 1
+
+ return True
- if module_name != "__init__" \
- and hasattr(configuration, module_name):
- # Reload the updated module so that code executing later
- # sees added configuration options. (It will also see
- # removed configuration options, but that is unlikely to
- # be a problem.)
- reload(getattr(configuration, module_name))
+def upgrade(arguments, data):
+ global modified_files
+
+ import configuration
+
+ source_dir = os.path.join(installation.root_dir, "installation", "templates", "configuration")
+ target_dir = os.path.join(data["installation.paths.etc_dir"], arguments.identity, "configuration")
+ compilation_failed = []
- os.chmod(target_path + "c", mode)
+ no_changes = True
- modified_files += 1
+ for entry in os.listdir(source_dir):
+ if update_file(target_dir, entry, data, arguments, compilation_failed):
+ no_changes = False
if compilation_failed:
return False
View
49 installation/data/dbschema.extensions.sql
@@ -14,23 +14,37 @@
-- License for the specific language governing permissions and limitations under
-- the License.
+-- Disable notices about implicitly created indexes and sequences.
+SET client_min_messages TO WARNING;
+
CREATE TABLE extensions
( id SERIAL PRIMARY KEY,
- author INTEGER NOT NULL REFERENCES users,
- name VARCHAR(64),
+ author INTEGER REFERENCES users, -- NULL means system extension
+ name VARCHAR(64) NOT NULL,
UNIQUE (author, name) );
CREATE TABLE extensionversions
( id SERIAL PRIMARY KEY,
- sha1 CHAR(40),
extension INTEGER NOT NULL REFERENCES extensions,
+ name VARCHAR(256) NOT NULL,
+ sha1 CHAR(40) NOT NULL,
UNIQUE (sha1) );
+-- Installed extensions.
+-- If uid=NULL, it is a "universal install" (affecting all users.)
+-- If version=NULL, the "LIVE" version is installed.
+CREATE TABLE extensioninstalls
+ ( id SERIAL PRIMARY KEY,
+ uid INTEGER REFERENCES users,
+ extension INTEGER NOT NULL REFERENCES extensions,
+ version INTEGER REFERENCES extensionversions,
+
+ UNIQUE (uid, extension) );
+
CREATE TABLE extensionroles
( id SERIAL PRIMARY KEY,
- uid INTEGER NOT NULL REFERENCES users,
version INTEGER NOT NULL REFERENCES extensionversions,
script VARCHAR(64) NOT NULL,
function VARCHAR(64) NOT NULL );
@@ -40,7 +54,7 @@ CREATE TABLE extensionpageroles
path VARCHAR(64) NOT NULL );
CREATE VIEW extensionroles_page AS
- SELECT uid, version, path, script, function
+ SELECT version, path, script, function
FROM extensionroles
JOIN extensionpageroles ON (role=id);
@@ -49,33 +63,12 @@ CREATE TABLE extensioninjectroles
path VARCHAR(64) NOT NULL );
CREATE VIEW extensionroles_inject AS
- SELECT uid, version, path, script, function
+ SELECT version, path, script, function
FROM extensionroles
JOIN extensioninjectroles ON (role=id);
CREATE TABLE extensionprocesscommitsroles
- ( role INTEGER NOT NULL REFERENCES extensionroles ON DELETE CASCADE,
- filter INTEGER REFERENCES filters );
-
-CREATE VIEW extensionroles_processcommits AS
- SELECT uid, version, filter, script, function
- FROM extensionroles
- JOIN extensionprocesscommitsroles ON (role=id);
-
-CREATE TABLE extensionprocesschangesroles
- ( role INTEGER NOT NULL REFERENCES extensionroles ON DELETE CASCADE,
- skip INTEGER NOT NULL REFERENCES batches );
-
-CREATE VIEW extensionroles_processchanges AS
- SELECT id, skip, uid, version, script, function
- FROM extensionroles
- JOIN extensionprocesschangesroles ON (role=id);
-
-CREATE TABLE extensionprocessedbatches
- ( role INTEGER NOT NULL REFERENCES extensionroles,
- batch INTEGER NOT NULL REFERENCES batches,
-
- PRIMARY KEY (batch, role) );
+ ( role INTEGER NOT NULL REFERENCES extensionroles ON DELETE CASCADE );
CREATE TABLE extensionstorage
( extension INTEGER NOT NULL REFERENCES extensions,
View
20 installation/database.py
@@ -27,12 +27,15 @@
database_created = False
language_created = False
-def psql_import(sql_file):
+def psql_import(sql_file, as_user=None):
+ if as_user is None:
+ as_user = installation.system.username
temp_file = tempfile.mkstemp()[1]
- shutil.copy(sql_file, temp_file)
+ shutil.copy(os.path.join(installation.root_dir, sql_file), temp_file)
# Make sure file is readable by postgres user
os.chmod(temp_file, 0644)
- subprocess.check_output(["su", "-s", "/bin/sh", "-c", "psql -v ON_ERROR_STOP=1 -f %s" % temp_file, installation.system.username])
+ subprocess.check_output(
+ ["su", "-s", "/bin/sh", "-c", "psql -v ON_ERROR_STOP=1 -f %s" % temp_file, as_user])
os.unlink(temp_file)
def add_arguments(mode, parser):
@@ -125,12 +128,11 @@ def install(data):
subprocess.check_output(["su", "-c", "psql -v ON_ERROR_STOP=1 -c 'GRANT ALL ON DATABASE \"critic\" TO \"%s\";'" % installation.system.username, "postgres"])
- data_dir = os.path.join(installation.root_dir, "installation/data")
-
- psql_import(os.path.join(data_dir, "dbschema.sql"))
- psql_import(os.path.join(data_dir, "dbschema.comments.sql"))
- psql_import(os.path.join(data_dir, "comments.pgsql"))
- psql_import(os.path.join(data_dir, "roles.sql"))
+ psql_import("installation/data/dbschema.sql")
+ psql_import("installation/data/dbschema.comments.sql")
+ psql_import("installation/data/dbschema.extensions.sql")
+ psql_import("installation/data/comments.pgsql")
+ psql_import("installation/data/roles.sql")
import psycopg2
View
37 installation/extensions.py
@@ -0,0 +1,37 @@
+# -*- mode: python; encoding: utf-8 -*-
+#
+# Copyright 2012 Jens Lindström, Opera Software ASA
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+def prepare(mode, arguments, data):
+ data["installation.extensions.enabled"] = False
+ data["installation.extensions.critic_v8_jsshell"] = "NOT_INSTALLED"
+ data["installation.extensions.default_flavor"] = "js/v8"
+
+ if mode == "upgrade":
+ import configuration
+ data["installation.extensions.enabled"] = \
+ configuration.extensions.ENABLED
+ try:
+ data["installation.extensions.critic_v8_jsshell"] = \
+ configuration.extensions.FLAVORS["js/v8"]["executable"]
+ except (KeyError, AttributeError):
+ pass
+ try:
+ data["installation.extensions.default_flavor"] = \
+ configuration.extensions.DEFAULT_FLAVOR
+ except AttributeError:
+ pass
+
+ return True
1 installation/externals/v8-jsshell
@@ -0,0 +1 @@
+Subproject commit add55f8db48e315c807604be2bbb41e88bd7fe29
View
9 installation/initd.py
@@ -49,6 +49,15 @@ def start(identity="main"):
return True
+def restart(identity="main"):
+ print
+ try:
+ subprocess.check_call(["service", "critic-%s" % identity, "restart"])
+ except subprocess.CalledProcessError:
+ return False
+
+ return True
+
def install(data):
global servicemanager_started, rclinks_added
View
1 installation/prefs.py
@@ -90,6 +90,7 @@ def update_preference(db, item, data, type_changed):
def remove_preference(db, item):
cursor = db.cursor()
+ cursor.execute("DELETE FROM userpreferences WHERE item=%s", (item,))
cursor.execute("DELETE FROM preferences WHERE item=%s", (item,))
def load_preferences(db):
View
118 installation/prereqs.py
@@ -54,8 +54,45 @@ def blankline():
print
need_blankline = False
+def install_packages(arguments, *packages):
+ global aptget, aptget_approved, aptget_updated, need_blankline, all_ok
+ if aptget is None:
+ aptget = find_executable("apt-get")
+ if aptget and not aptget_approved:
+ all_ok = False
+ print """\
+Found 'apt-get' executable in your $PATH. This script can attempt to install
+missing software using it.
+"""
+ aptget_approved = installation.input.yes_or_no(
+ prompt="Do you want to use 'apt-get' to install missing packages?",
+ default=True)
+ if not aptget_approved: aptget = False
+ if aptget:
+ installed_anything = False
+ aptget_env = os.environ.copy()
+ if arguments.headless:
+ aptget_env["DEBIAN_FRONTEND"] = "noninteractive"
+ if not aptget_updated:
+ subprocess.check_output(
+ [aptget, "-qq", "update"],
+ env=aptget_env)
+ aptget_updated = True
+ aptget_output = subprocess.check_output(
+ [aptget, "-qq", "-y", "install"] + list(packages),
+ env=aptget_env)
+ for line in aptget_output.splitlines():
+ match = re.search(r"([^ ]+) \(.* \.\.\./([^)]+\.deb)\) \.\.\.", line)
+ if match:
+ need_blankline = True
+ installed_anything = True
+ print "Installed: %s (%s)" % (match.group(1), match.group(2))
+ return installed_anything
+ else:
+ return False
+
def check(mode, arguments):
- global git, tar, psql, passlib_available, aptget, apache2ctl, a2enmod, a2ensite, a2dissite
+ global git, tar, psql, passlib_available, apache2ctl, a2enmod, a2ensite, a2dissite
if mode == "install":
print """
@@ -66,46 +103,9 @@ def check(mode, arguments):
success = True
all_ok = True
- aptget = find_executable("apt-get")
-
- def install(*packages):
- global aptget, aptget_approved, aptget_updated, need_blankline, all_ok
- if aptget and not aptget_approved:
- all_ok = False
- print """\
-Found 'apt-get' executable in your $PATH. This script can attempt to install
-missing software using it.
-"""
- aptget_approved = installation.input.yes_or_no(
- prompt="Do you want to use 'apt-get' to install missing packages?",
- default=True)
- if not aptget_approved: aptget = None
- if aptget:
- installed_anything = False
- aptget_env = os.environ.copy()
- if arguments.headless:
- aptget_env["DEBIAN_FRONTEND"] = "noninteractive"
- if not aptget_updated:
- subprocess.check_output(
- [aptget, "-qq", "update"],
- env=aptget_env)
- aptget_updated = True
- aptget_output = subprocess.check_output(
- [aptget, "-qq", "-y", "install"] + list(packages),
- env=aptget_env)
- for line in aptget_output.splitlines():
- match = re.search(r"([^ ]+) \(.* \.\.\./([^)]+\.deb)\) \.\.\.", line)
- if match:
- need_blankline = True
- installed_anything = True
- print "Installed: %s (%s)" % (match.group(1), match.group(2))
- return installed_anything
- else:
- return False
-
git = find_executable("git")
if not git:
- if aptget_approved and install("git-core"):
+ if aptget_approved and install_packages(arguments, "git-core"):
git = find_executable("git")
if not git:
blankline()
@@ -118,7 +118,7 @@ def install(*packages):
https://github.com/git/git
"""
- if not aptget_approved and install("git-core"):
+ if not aptget_approved and install_packages(arguments, "git-core"):
git = find_executable("git")
if not git: success = False
@@ -127,7 +127,7 @@ def install(*packages):
psql = find_executable("psql")
if not psql:
- if aptget_approved and install("postgresql", "postgresql-client"):
+ if aptget_approved and install_packages(arguments, "postgresql", "postgresql-client"):
psql = find_executable("psql")
if not psql:
blankline()
@@ -137,7 +137,7 @@ def install(*packages):
and its client utilities are installed. In Debian/Ubuntu, the packages you need
to install are 'postgresql' and 'postgresql-client'.
"""
- if not aptget_approved and install("postgresql", "postgresql-client"):
+ if not aptget_approved and install_packages(arguments, "postgresql", "postgresql-client"):
psql = find_executable("psql")
if not psql: success = False
@@ -157,7 +157,7 @@ def install(*packages):
apache2ctl = find_executable("apache2ctl")
if not apache2ctl:
- if aptget_approved and install("apache2"):
+ if aptget_approved and install_packages(arguments, "apache2"):
apache2ctl = find_executable("apache2ctl")
if not apache2ctl:
blankline()
@@ -166,13 +166,13 @@ def install(*packages):
No 'apache2ctl' executable found in $PATH. Make sure the Apache web server is
installed. In Debian/Ubuntu, the package you need to install is 'apache2'.
"""
- if not aptget_approved and install("apache2"):
+ if not aptget_approved and install_packages(arguments, "apache2"):
apache2ctl = find_executable("apache2ctl")
if not apache2ctl: success = False
a2enmod = find_executable("a2enmod")
if not a2enmod:
- if aptget_approved and install("apache2"):
+ if aptget_approved and install_packages(arguments, "apache2"):
a2enmod = find_executable("a2enmod")
if not a2enmod:
blankline()
@@ -181,13 +181,13 @@ def install(*packages):
No 'a2enmod' executable found in $PATH. Make sure the Apache web server is
installed. In Debian/Ubuntu, the package you need to install is 'apache2'.
"""
- if not aptget_approved and install("apache2"):
+ if not aptget_approved and install_packages(arguments, "apache2"):
a2enmod = find_executable("a2enmod")
if not a2enmod: success = False
a2ensite = find_executable("a2ensite")
if not a2ensite:
- if aptget_approved and install("apache2"):
+ if aptget_approved and install_packages(arguments, "apache2"):
a2ensite = find_executable("a2ensite")
if not a2ensite:
blankline()
@@ -196,13 +196,13 @@ def install(*packages):
No 'a2ensite' executable found in $PATH. Make sure the Apache web server is
installed. In Debian/Ubuntu, the package you need to install is 'apache2'.
"""
- if not aptget_approved and install("apache2"):
+ if not aptget_approved and install_packages(arguments, "apache2"):
a2ensite = find_executable("a2ensite")
if not a2ensite: success = False
a2dissite = find_executable("a2dissite")
if not a2dissite:
- if aptget_approved and install("apache2"):
+ if aptget_approved and install_packages(arguments, "apache2"):
a2dissite = find_executable("a2dissite")
if not a2dissite:
blankline()
@@ -211,7 +211,7 @@ def install(*packages):
No 'a2dissite' executable found in $PATH. Make sure the Apache web server is
installed. In Debian/Ubuntu, the package you need to install is 'apache2'.
"""
- if not aptget_approved and install("apache2"):
+ if not aptget_approved and install_packages(arguments, "apache2"):
a2dissite = find_executable("a2dissite")
if not a2dissite: success = False
@@ -231,7 +231,7 @@ def install(*packages):
mod_wsgi_available_path = os.path.join("/etc", "apache2", "mods-available", "wsgi.load")
mod_wsgi_available = os.path.isfile(mod_wsgi_available_path)
if not mod_wsgi_available:
- if aptget_approved and install("libapache2-mod-wsgi"):
+ if aptget_approved and install_packages(arguments, "libapache2-mod-wsgi"):
mod_wsgi_available = os.path.isfile(mod_wsgi_available_path)
if not mod_wsgi_available:
blankline()
@@ -243,7 +243,7 @@ def install(*packages):
http://code.google.com/p/modwsgi/wiki/DownloadTheSoftware?tm=2
"""
- if not aptget_approved and install("libapache2-mod-wsgi"):
+ if not aptget_approved and install_packages(arguments, "libapache2-mod-wsgi"):
mod_wsgi_available = os.path.isfile(mod_wsgi_available_path)
if not mod_wsgi_available: success = False
@@ -256,7 +256,7 @@ def check_psycopg2():
check_psycopg2()
if not psycopg2_available:
- if aptget_approved and install("python-psycopg2"):
+ if aptget_approved and install_packages(arguments, "python-psycopg2"):
check_psycopg2()
if not psycopg2_available:
blankline()
@@ -268,7 +268,7 @@ def check_psycopg2():
http://www.initd.org/psycopg/download/
"""
- if not aptget_approved and install("python-psycopg2"):
+ if not aptget_approved and install_packages(arguments, "python-psycopg2"):
check_psycopg2()
if not psycopg2_available:
success = False
@@ -282,7 +282,7 @@ def check_pygments():
check_pygments()
if not pygments_available:
- if aptget_approved and install("python-pygments"):
+ if aptget_approved and install_packages(arguments, "python-pygments"):
check_pygments()
if not pygments_available:
blankline()
@@ -294,7 +294,7 @@ def check_pygments():
http://pygments.org/download/
"""
- if not aptget_approved and install("python-pygments"):
+ if not aptget_approved and install_packages(arguments, "python-pygments"):
check_pygments()
if not pygments_available:
success = False
@@ -338,7 +338,7 @@ def check_passlib():
"Do you want to install the 'passlib' module?",
default=False)
if install_passlib:
- if install("python-passlib"):
+ if install_packages(arguments, "python-passlib"):
check_passlib()
if not passlib_available:
print """
@@ -356,7 +356,7 @@ def check_requests():
check_requests()
if not requests_available:
- if aptget_approved and install("python-requests"):
+ if aptget_approved and install_packages(arguments, "python-requests"):
check_requests()
if not requests_available:
blankline()
@@ -368,7 +368,7 @@ def check_requests():
https://github.com/kennethreitz/requests
"""
- if not aptget_approved and install("python-requests"):
+ if not aptget_approved and install_packages(arguments, "python-requests"):
check_requests()
if not requests_available:
success = False
View
9 installation/system.py
@@ -33,6 +33,12 @@
create_system_group = None
created_system_group = False
+def fetch_uid_gid():
+ global uid, gid
+
+ uid = pwd.getpwnam(username).pw_uid
+ gid = grp.getgrnam(groupname).gr_gid
+
def prepare(mode, arguments, data):
global hostname, username, email, create_system_user
global groupname, create_system_group
@@ -128,8 +134,7 @@ def prepare(mode, arguments, data):
try: groupname = configuration.base.SYSTEM_GROUP_NAME
except AttributeError: groupname = data["installation.system.groupname"]
- uid = pwd.getpwnam(username).pw_uid
- gid = grp.getgrnam(groupname).gr_gid
+ fetch_uid_gid()
data["installation.system.hostname"] = hostname
data["installation.system.username"] = username
View
22 installation/templates/configuration/extensions.py
@@ -19,24 +19,28 @@
# Whether extension support is enabled. If False, the rest of the
# configuration in this file is irrelevant.
-ENABLED = False
+ENABLED = %(installation.extensions.enabled)s
-# Where to search for extensions.
-SEARCH_ROOT = "/home"
+# Where to search for system extensions.
+SYSTEM_EXTENSIONS_DIR = os.path.join(configuration.paths.DATA_DIR, "extensions")
+
+# Name of directory under users' $HOME in which to search for user extensions.
+# If set to None, user extensions support is disabled.
+USER_EXTENSIONS_DIR = "CriticExtensions"
FLAVORS = {
"js/v8":
- { "executable": "/usr/bin/critic-v8-jsshell",
+ { "executable": "%(installation.extensions.critic_v8_jsshell)s",
"library": os.path.join(configuration.paths.INSTALL_DIR, "library", "js", "v8") }
}
-DEFAULT_FLAVOR = "js/v8"
-
-# Directory where the Javascript extension library is installed.
-JS_LIBRARY_DIR = os.path.join(configuration.paths.INSTALL_DIR, "library", "js")
+DEFAULT_FLAVOR = "%(installation.extensions.default_flavor)s"
# Directory into which extension version snapshots are installed.
-INSTALL_DIR = os.path.join(configuration.paths.DATA_DIR, "extensions")
+INSTALL_DIR = os.path.join(configuration.paths.DATA_DIR, "extension-snapshots")
+
+# Directory into which extension repository work copies are created.
+WORKCOPY_DIR = os.path.join(configuration.paths.DATA_DIR, "temporary", "EXTENSIONS")
# Long timeout, in seconds. Used for extension "Page" roles.
LONG_TIMEOUT = 300
View
29 pythonversion.py
@@ -0,0 +1,29 @@
+# -*- mode: python; encoding: utf-8 -*-
+#
+# Copyright 2013 Jens Lindström, Opera Software ASA
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import sys
+
+def check(message=None):
+ if sys.version_info[0] != 2 or sys.version_info[1] < 7:
+ print """
+ERROR: Unsupported Python version! Critic requires Python 2.7.x or
+later, and does not support Python 3.x.
+"""
+
+ if message:
+ print message
+
+ sys.exit(2)
View
25 src/background/maintenance.py
@@ -16,6 +16,8 @@
import sys
import os
+import time
+import shutil
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "..")))
@@ -76,6 +78,29 @@ def __maintenance(self):
if self.terminated:
return
+ if configuration.extensions.ENABLED:
+ now = time.time()
+ max_age = 7 * 24 * 60 * 60
+
+ base_path = os.path.join(configuration.paths.DATA_DIR,
+ "temporary", "EXTENSIONS")
+
+ for user_name in os.listdir(base_path):
+ user_dir = os.path.join(base_path, user_name)
+
+ for extension_id in os.listdir(user_dir):
+ extension_dir = os.path.join(user_dir, extension_id)
+
+ for repository_name in os.listdir(extension_dir):
+ repository_dir = os.path.join(extension_dir,
+ repository_name)
+ age = now - os.stat(repository_dir).st_mtime
+
+ if age > max_age:
+ self.info("Removing repository work copy: %s"
+ % repository_dir)
+ shutil.rmtree(repository_dir)
+
def start_service():
maintenance = Maintenance()
maintenance.run()
View
80 src/batchprocessor.py
@@ -1,80 +0,0 @@
-# -*- mode: python; encoding: utf-8 -*-
-#
-# Copyright 2012 Jens Lindström, Opera Software ASA
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-import sys
-
-if __name__ != "__main__":
- print >>sys.stderr, "Don't include batchprocessor.py as a module!"
- sys.exit(1)
-
-import traceback
-import time
-import cStringIO
-
-import dbaccess
-import extensions.role.processchanges
-import reviewing.mail
-
-POLLING_INTERVAL = 5
-
-# Process loop. Terminates on any kind of error (such as if the DB connection
-# is lost.)
-def processLoop():
- db = dbaccess.connect()
- cursor = db.cursor()
-
- while True:
- cursor.execute("""SELECT DISTINCT roles.uid, batches.id
- FROM extensionroles_processchanges AS roles
- JOIN batches ON (batches.id > roles.skip)
- JOIN reviewusers ON (reviewusers.review=batches.review AND reviewusers.uid=roles.uid)
- LEFT OUTER JOIN extensionprocessedbatches AS processed ON (processed.batch=batches.id AND processed.role=roles.id)
- WHERE processed.batch IS NULL""")
-
- queue = cursor.fetchall()
-
- if not queue:
- # Nothing to do right now; sleep a little to avoid a tight loop of
- # DB queries. (Not that the query will be expensive at all in the
- # foreseeable future, but)
-
- time.sleep(POLLING_INTERVAL)
- else:
- for user_id, batch_id in queue:
- output = cStringIO.StringIO()
-
- extensions.role.processchanges.execute(db, user_id, batch_id, output)
-
- output = output.getvalue()
-
- if output.strip():
- pending_mails = reviewing.mail.sendExtensionOutput(db, user_id, batch_id, output)
- reviewing.mail.sendPendingMails(pending_mails)
-
-# Main loop.
-while True:
- try:
- try:
- processLoop()
- except KeyboardInterrupt:
- raise
- except:
- print >>sys.stderr, "".join(traceback.format_exception(*sys.exc_info()))
- time.sleep(5)
- except KeyboardInterrupt:
- break
-
-print "exiting"
View
39 src/critic.py
@@ -90,6 +90,7 @@
import page.rebasetrackingreview
import page.createuser
import page.verifyemail
+import page.manageextensions
try:
from customization.email import getUserEmailAddress
@@ -363,11 +364,20 @@ def addSuggestions():
return json_encode(suggestions)
def loadmanifest(req, _db, _user):
- author = req.getParameter("author")
- name = req.getParameter("name")
+ key = req.getParameter("key")
+
+ if "/" in key:
+ author_name, extension_name = key.split("/", 1)
+ else:
+ author_name, extension_name = None, key
+
+ try:
+ extension = extensions.extension.Extension(author_name, extension_name)
+ except extensions.extension.ExtensionError as error:
+ return str(error)
try:
- extensions.manifest.Manifest.load(extensions.getExtensionPath(author, name))
+ extension.getManifest()
return "That's a valid manifest, friend."
except extensions.manifest.ManifestError as error:
return str(error)
@@ -510,21 +520,21 @@ def processcommits(req, db, user):
"services": page.services.renderServices,
"rebasetrackingreview": page.rebasetrackingreview.RebaseTrackingReview(),
"createuser": page.createuser.CreateUser(),
- "verifyemail": page.verifyemail.renderVerifyEmail }
+ "verifyemail": page.verifyemail.renderVerifyEmail,
+ "manageextensions": page.manageextensions.renderManageExtensions }
if configuration.extensions.ENABLED:
import extensions
import extensions.role.page
import extensions.role.processcommits
import operation.extensioninstallation
- import page.manageextensions
OPERATIONS["installextension"] = operation.extensioninstallation.InstallExtension()
OPERATIONS["uninstallextension"] = operation.extensioninstallation.UninstallExtension()
OPERATIONS["reinstallextension"] = operation.extensioninstallation.ReinstallExtension()
+ OPERATIONS["clearextensionstorage"] = operation.extensioninstallation.ClearExtensionStorage()
OPERATIONS["loadmanifest"] = loadmanifest
OPERATIONS["processcommits"] = processcommits
- PAGES["manageextensions"] = page.manageextensions.renderManageExtensions
if configuration.base.AUTHENTICATION_MODE != "host" and configuration.base.SESSION_TYPE == "cookie":
import operation.usersession
@@ -966,15 +976,20 @@ def revparseWithReview(item):
if repository:
try:
items = filter(None, map(revparse, path.split("..")))
+ query = None
if len(items) == 1:
- req.query = ("repository=%d&sha1=%s&%s"
- % (repository.id, items[0], req.query))
- req.path = "showcommit"
- continue
+ query = ("repository=%d&sha1=%s"
+ % (repository.id, items[0]))
elif len(items) == 2:
- req.query = ("repository=%d&from=%s&to=%s&%s"
- % (repository.id, items[0], items[1], req.query))
+ query = ("repository=%d&from=%s&to=%s"
+ % (repository.id, items[0], items[1]))
+
+ if query:
+ if req.query:
+ query += "&" + req.query
+
+ req.query = query
req.path = "showcommit"
continue
except gitutils.GitReferenceError:
View
5 src/data/preferences.json
@@ -150,11 +150,6 @@
"default": 250,
"description": "Maximum number of lines of commit stats to include in the email sent when a review is submitted. If exceeded, no stats are included at all."
},
- "email.subjectLine.extensionOutput": {
- "type": "string",
- "default": "Extension Output: %(summary)s",
- "description": "Python format string for subject line of email sent when an extension's ProcessChanges hook produces output (or fails.)"
- },
"email.subjectLine.newReview": {
"type": "string",
"default": "New Review: %(summary)s",
View
9 src/dbutils/user.py
@@ -353,6 +353,15 @@ def getAbsence(self, db):
else:
return "absent until %04d-%02d-%02d" % (row[0].year, row[0].month, row[0].day)
+ def hasGitEmail(self, db, address):
+ cursor = db.cursor()
+ cursor.execute("""SELECT 1
+ FROM usergitemails
+ WHERE email=%s
+ AND uid=%s""",
+ (address, self.id))
+ return bool(cursor.fetchone())
+
@staticmethod
def cache(db, user):
storage = db.storage["User"]
View
2 src/extensions/__init__.py
@@ -19,7 +19,7 @@
import configuration
def getExtensionPath(author_name, extension_name):
- return os.path.join(configuration.extensions.SEARCH_ROOT, author_name, "CriticExtensions", extension_name)
+ return extension.Extension(author_name, extension_name).getPath()
def getExtensionInstallPath(sha1):
return os.path.join(configuration.extensions.INSTALL_DIR, sha1)
View
52 src/extensions/execute.py
@@ -21,7 +21,8 @@
from textutils import json_encode
from communicate import Communicate
-def executeProcess(manifest, role, extension_id, user_id, argv, timeout, stdin=None, rlimit_cpu=5, rlimit_rss=256):
+def executeProcess(manifest, role_name, script, function, extension_id, user_id,
+ argv, timeout, stdin=None, rlimit_rss=256):
flavor = manifest.flavor
if manifest.flavor not in configuration.extensions.FLAVORS:
@@ -30,32 +31,29 @@ def executeProcess(manifest, role, extension_id, user_id, argv, timeout, stdin=N
executable = configuration.extensions.FLAVORS[flavor]["executable"]
library = configuration.extensions.FLAVORS[flavor]["library"]
- process_argv = [executable,
- "--rlimit-cpu=%ds" % rlimit_cpu,
- "--rlimit-rss=%dm" % rlimit_rss,
- os.path.join(library, "critic-launcher.js")]
+ process_argv = [executable, os.path.join(library, "critic-launcher.js")]
- stdin_data = "%s\n" % json_encode({ "criticjs_path": os.path.join(library, "critic2.js"),
- "rlimit": { "cpu": rlimit_cpu,
- "rss": rlimit_rss },
- "hostname": configuration.base.HOSTNAME,
- "dbname": configuration.database.PARAMETERS["database"],
- "dbuser": configuration.database.PARAMETERS["user"],
- "git": configuration.executables.GIT,
- "python": configuration.executables.PYTHON,
- "python_path": "%s:%s" % (configuration.paths.CONFIG_DIR,
- configuration.paths.INSTALL_DIR),
- "repository_work_copy_path": os.path.join(configuration.paths.DATA_DIR, "temporary", "EXTENSIONS"),
- "changeset_address": configuration.services.CHANGESET["address"],
- "maildelivery_pid_path": configuration.services.MAILDELIVERY["pidfile_path"],
- "is_development": configuration.debug.IS_DEVELOPMENT,
- "extension_path": manifest.path,
- "extension_id": extension_id,
- "user_id": user_id,
- "role": role.name(),
- "script_path": role.script,
- "fn": role.function,
- "argv": argv })
+ stdin_data = "%s\n" % json_encode({
+ "criticjs_path": os.path.join(library, "critic.js"),
+ "rlimit": { "rss": rlimit_rss },
+ "hostname": configuration.base.HOSTNAME,
+ "dbname": configuration.database.PARAMETERS["database"],
+ "dbuser": configuration.database.PARAMETERS["user"],
+ "git": configuration.executables.GIT,
+ "python": configuration.executables.PYTHON,
+ "python_path": "%s:%s" % (configuration.paths.CONFIG_DIR,
+ configuration.paths.INSTALL_DIR),
+ "repository_work_copy_path": configuration.extensions.WORKCOPY_DIR,
+ "changeset_address": configuration.services.CHANGESET["address"],
+ "maildelivery_pid_path": configuration.services.MAILDELIVERY["pidfile_path"],
+ "is_development": configuration.debug.IS_DEVELOPMENT,
+ "extension_path": manifest.path,
+ "extension_id": extension_id,
+ "user_id": user_id,
+ "role": role_name,
+ "script_path": script,
+ "fn": function,
+ "argv": argv })
if stdin is not None:
stdin_data += stdin
@@ -64,6 +62,6 @@ def executeProcess(manifest, role, extension_id, user_id, argv, timeout, stdin=N
communicate = Communicate(process)
communicate.setInput(stdin_data)
- communicate.setTimout(timeout)
+ communicate.setTimeout(timeout)
return communicate.run()[0]
View
390 src/extensions/extension.py
@@ -16,77 +16,194 @@
import os
import subprocess
+import pwd
-import base
import configuration
import dbutils
+import htmlutils
-from extensions.manifest import Manifest, PageRole, InjectRole, ProcessChangesRole, ProcessCommitsRole
+from extensions.manifest import Manifest, ManifestError
+from extensions import getExtensionInstallPath
-class Extension:
+class ExtensionError(Exception):
+ def __init__(self, message, extension=None):
+ super(ExtensionError, self).__init__(message)
+ self.extension = extension
+
+class Extension(object):
def __init__(self, author_name, extension_name):
+ if os.path.sep in extension_name:
+ raise ExtensionError(
+ "Invalid extension name: %s" % extension_name)
+
self.__author_name = author_name
self.__extension_name = extension_name
- self.__path = os.path.join(configuration.extensions.SEARCH_ROOT, author_name, "CriticExtensions", extension_name)
+ self.__manifest = {}
+
+ if author_name:
+ try:
+ user_home_dir = pwd.getpwnam(author_name).pw_dir
+ except KeyError:
+ raise ExtensionError(
+ "No such system user: %s" % author_name,
+ extension=self)
+
+ self.__path = os.path.join(
+ user_home_dir,
+ configuration.extensions.USER_EXTENSIONS_DIR,
+ extension_name)
+ else:
+ self.__path = os.path.join(
+ configuration.extensions.SYSTEM_EXTENSIONS_DIR,
+ extension_name)
+
+ if not (os.path.isdir(self.__path) and
+ os.access(self.__path, os.R_OK | os.X_OK)):
+ raise ExtensionError(
+ "Invalid or inaccessible extension dir: %s" % self.__path,
+ extension=self)
+
+ def isSystemExtension(self):
+ return self.__author_name is None
def getAuthorName(self):
+ if self.isSystemExtension():
+ return None
return self.__author_name
def getName(self):
return self.__extension_name
+ def getTitle(self, db, html=False):
+ if html:
+ title = "<b>%s</b>" % htmlutils.htmlify(self.getName())
+ else:
+ title = self.getName()
+
+ if not self.isSystemExtension():
+ author = self.getAuthor(db)
+
+ try:
+ manifest = self.getManifest()
+ except ManifestError:
+ # Can't access information from the manifest, so assume "yes".
+ is_author = True
+ else:
+ is_author = manifest.isAuthor(db, author)
+
+ if is_author:
+ title += " by "
+ else:
+ title += " hosted by "
+
+ if html:
+ title += htmlutils.htmlify(author.fullname)
+ else:
+ title += author.fullname
+
+ return title
+
def getKey(self):
- return "%s/%s" % (self.__author_name, self.__extension_name)
+ if self.isSystemExtension():
+ return self.__extension_name
+ else:
+ return "%s/%s" % (self.__author_name, self.__extension_name)
def getPath(self):
return self.__path
def getVersions(self):
try:
- branches = subprocess.check_output([configuration.executables.GIT, "branch"], cwd=self.__path).splitlines()
- return [branch[10:].strip() for branch in branches if branch[2:].startswith("version/")]
+ output = subprocess.check_output(
+ [configuration.executables.GIT, "for-each-ref",
+ "--format=%(refname)", "refs/heads/version/"],
+ stderr=subprocess.STDOUT, cwd=self.__path)
except subprocess.CalledProcessError:
# Not a git repository => no versions (except "Live").
return []
- def readManifest(self, version=None):
- if version is None:
- source = None
- else:
- source = subprocess.check_output([configuration.executables.GIT, "cat-file", "blob", "version/%s:MANIFEST" % version],
- cwd=self.__path)
+ versions = []
+ for ref in output.splitlines():
+ if ref.startswith("refs/heads/version/"):
+ versions.append(ref[len("refs/heads/version/"):])
+ return versions
+
+ def getManifest(self, version=None, sha1=None):
+ path = self.__path
+ source = None
+
+ if sha1 is not None:
+ if sha1 in self.__manifest:
+ return self.__manifest[sha1]
+
+ install_path = getExtensionInstallPath(sha1)
+ with open(os.path.join(install_path, "MANIFEST")) as manifest_file:
+ source = manifest_file.read()
- manifest = Manifest(self.__path, source)
+ path = "<snapshot of commit %s>" % sha1[:8]
+ elif version in self.__manifest:
+ return self.__manifest[version]
+
+ if source is None and version is not None:
+ source = subprocess.check_output(
+ [configuration.executables.GIT, "cat-file", "blob",
+ "version/%s:MANIFEST" % version],
+ cwd=self.__path)
+
+ manifest = Manifest(path, source)
manifest.read()
+
+ if sha1 is not None:
+ self.__manifest[sha1] = manifest
+ else:
+ self.__manifest[version] = manifest
+
return manifest
def getCurrentSHA1(self, version):
- return subprocess.check_output([configuration.executables.GIT, "rev-parse", "--verify", "version/%s" % version],
- cwd=self.__path).strip()
+ return subprocess.check_output(
+ [configuration.executables.GIT, "rev-parse", "--verify",
+ "version/%s" % version],
+ cwd=self.__path).strip()
def prepareVersionSnapshot(self, version):
sha1 = self.getCurrentSHA1(version)
- if not os.path.isdir(os.path.join(configuration.extensions.INSTALL_DIR, sha1)):
- git_archive = subprocess.Popen([configuration.executables.GIT, "archive", "--format=tar", "--prefix=%s/" % sha1, sha1],
- stdout=subprocess.PIPE, cwd=self.__path)
- subprocess.check_call([configuration.executables.TAR, "x"], stdin=git_archive.stdout,
- cwd=configuration.extensions.INSTALL_DIR)
+ if not os.path.isdir(getExtensionInstallPath(sha1)):
+ git_archive = subprocess.Popen(
+ [configuration.executables.GIT, "archive", "--format=tar",
+ "--prefix=%s/" % sha1, sha1],
+ stdout=subprocess.PIPE, cwd=self.__path)
+ subprocess.check_call(
+ [configuration.executables.TAR, "x"],
+ stdin=git_archive.stdout,
+ cwd=configuration.extensions.INSTALL_DIR)
return sha1
def getAuthor(self, db):
- return dbutils.User.fromName(db, self.__author_name)
+ if self.isSystemExtension():
+ return None
+ return dbutils.User.fromName(db, self.getAuthorName())
def getExtensionID(self, db, create=False):
- author_id = self.getAuthor(db).id
-
cursor = db.cursor()
- cursor.execute("""SELECT extensions.id
- FROM extensions
- WHERE extensions.author=%s
- AND extensions.name=%s""",
- (author_id, self.__extension_name))
+
+ if self.isSystemExtension():
+ author_id = None
+ cursor.execute("""SELECT extensions.id
+ FROM extensions
+ WHERE extensions.author IS NULL
+ AND extensions.name=%s""",
+ (self.__extension_name,))
+ else:
+ author_id = self.getAuthor(db).id
+ cursor.execute("""SELECT extensions.id
+ FROM extensions
+ WHERE extensions.author=%s
+ AND extensions.name=%s""",
+ (author_id, self.__extension_name))
+
row = cursor.fetchone()
if row:
@@ -116,132 +233,143 @@ def getInstalledVersion(self, db, user):
return (False, False)
cursor = db.cursor()
- cursor.execute("""SELECT DISTINCT extensionversions.sha1, extensionversions.name
- FROM extensionversions
- JOIN extensionroles ON (extensionroles.version=extensionversions.id)
- WHERE extensionversions.extension=%s
- AND extensionroles.uid=%s""",
- (extension_id, user.id))
- versions = set(cursor)
-
- if len(versions) > 1:
- raise base.ImplementationError(
- "multiple extension versions installed (should not happen)")
-
- if versions:
- return versions.pop()
+
+ if user is None:
+ cursor.execute("""SELECT extensionversions.sha1, extensionversions.name
+ FROM extensioninstalls
+ LEFT OUTER JOIN extensionversions ON (extensionversions.id=extensioninstalls.version)
+ WHERE extensioninstalls.uid IS NULL
+ AND extensioninstalls.extension=%s""",
+ (extension_id,))
else:
- return (False, False)
+ cursor.execute("""SELECT extensionversions.sha1, extensionversions.name
+ FROM extensioninstalls
+ LEFT OUTER JOIN extensionversions ON (extensionversions.id=extensioninstalls.version)
+ WHERE extensioninstalls.uid=%s
+ AND extensioninstalls.extension=%s""",
+ (user.id, extension_id))
+
+ row = cursor.fetchone()
- def getInstallationStatus(self, db, user, version=None):
- author = self.getAuthor(db)
- manifest = self.readManifest(version)
+ if row:
+ return row
+ else:
+ return (False, False)
+ @staticmethod
+ def fromId(db, extension_id):
cursor = db.cursor()
- version_ids = set()
- installed_roles = 0
-
- for role in manifest.roles:
- if isinstance(role, PageRole):
- cursor.execute("""SELECT extensionversions.id
- FROM extensions
- JOIN extensionversions ON (extensionversions.extension=extensions.id)
- JOIN extensionroles_page ON (extensionroles_page.version=extensionversions.id)
- WHERE extensions.author=%s
- AND extensions.name=%s
- AND extensionroles_page.uid=%s
- AND extensionroles_page.path=%s
- AND extensionroles_page.script=%s
- AND extensionroles_page.function=%s""",
- (author.id, self.__extension_name, user.id, role.regexp, role.script, role.function))
- elif isinstance(role, InjectRole):
- cursor.execute("""SELECT extensionversions.id
- FROM extensions
- JOIN extensionversions ON (extensionversions.extension=extensions.id)
- JOIN extensionroles_inject ON (extensionroles_inject.version=extensionversions.id)
- WHERE extensions.author=%s
- AND extensions.name=%s
- AND extensionroles_inject.uid=%s
- AND extensionroles_inject.path=%s
- AND extensionroles_inject.script=%s
- AND extensionroles_inject.function=%s""",
- (author.id, self.__extension_name, user.id, role.regexp, role.script, role.function))
- elif isinstance(role, ProcessCommitsRole):
- cursor.execute("""SELECT extensionversions.id
- FROM extensions
- JOIN extensionversions ON (extensionversions.extension=extensions.id)
- JOIN extensionroles_processcommits ON (extensionroles_processcommits.version=extensionversions.id)
- WHERE extensions.author=%s
- AND extensions.name=%s
- AND extensionroles_processcommits.uid=%s
- AND extensionroles_processcommits.script=%s
- AND extensionroles_processcommits.function=%s""",
- (author.id, self.__extension_name, user.id, role.script, role.function))
- elif isinstance(role, ProcessChangesRole):
- cursor.execute("""SELECT extensionversions.id
- FROM extensions
- JOIN extensionversions ON (extensionversions.extension=extensions.id)
- JOIN extensionroles_processchanges ON (extensionroles_processchanges.version=extensionversions.id)
- WHERE extensions.author=%s
- AND extensions.name=%s
- AND extensionroles_processchanges.uid=%s
- AND extensionroles_processchanges.script=%s
- AND extensionroles_processchanges.function=%s""",
- (author.id, self.__extension_name, user.id, role.script, role.function))
- else:
- continue
+ cursor.execute("""SELECT users.name, extensions.name
+ FROM extensions
+ LEFT OUTER JOIN users ON (users.id=extensions.author)
+ WHERE extensions.id=%s""",
+ (extension_id,))
+ author_name, extension_name = cursor.fetchone()
+ return Extension(author_name, extension_name)
- row = cursor.fetchone()
- if row:
- version_ids.add(row[0])
- installed_roles += 1
- role.installed = True
- else:
- role.installed = False
+ @staticmethod
+ def getInstalls(db, user):
+ """
+ Return a list of extension installs in effect for the specified user
- if installed_roles == 0:
- manifest.status = "none"
- elif installed_roles < len(manifest.roles) or len(version_ids) != 1:
- manifest.status = "partial"
- else:
- manifest.status = "installed"
+ If 'user' is None, all universal extension installs are listed.
- return manifest
+ Each install is returned as a tuple containing four elements, the
+ extension id, the version id, the version SHA-1 and a boolean which is
+ true if the install is universal. For a LIVE version, the version id
+ and the version SHA-1 are None.
+
+ The list of installs is ordered by precedence; most significant install
+ first, least significant install last.
+ """
+
+ cursor = db.cursor()
+ cursor.execute("""SELECT extensioninstalls.id, extensioninstalls.extension,
+ extensionversions.id, extensionversions.sha1,
+ extensioninstalls.uid IS NULL
+ FROM extensioninstalls
+ LEFT OUTER JOIN extensionversions ON (extensionversions.id=extensioninstalls.version)
+ WHERE uid=%s OR uid IS NULL
+ ORDER BY uid NULLS FIRST""",
+ (user.id if user else None,))
+
+ install_per_extension = {}
+
+ # Since we ordered by 'uid' with nulls ("universal installs") first,
+ # we'll overwrite universal installs with per-user installs, as intended.
+ for install_id, extension_id, version_id, version_sha1, is_universal in cursor:
+ install_per_extension[extension_id] = (install_id, version_id,
+ version_sha1, is_universal)
+
+ installs = [(install_id, extension_id, version_id, version_sha1, is_universal)
+ for extension_id, (install_id, version_id, version_sha1, is_universal)
+ in install_per_extension.items()]
+
+ # Sort installs by install id, higher first. This means a later install
+ # takes precedence over an earlier, if they both handle the same path.
+ installs.sort(reverse=True)
+
+ # Drop the install_id; it is not relevant past this point.
+ return [(extension_id, version_id, version_sha1, is_universal)
+ for _, extension_id, version_id, version_sha1, is_universal
+ in installs]
@staticmethod
def getUpdatedExtensions(db, user):
cursor = db.cursor()
- cursor.execute("""SELECT DISTINCT users.name, users.fullname, extensions.name, extensionversions.name, extensionversions.sha1
+ cursor.execute("""SELECT users.name, users.fullname, extensions.name,
+ extensionversions.name, extensionversions.sha1
FROM users
JOIN extensions ON (extensions.author=users.id)
JOIN extensionversions ON (extensionversions.extension=extensions.id)
- JOIN extensionroles ON (extensionroles.version=extensionversions.id)
- WHERE extensionroles.uid=%s
- AND extensionversions.sha1 IS NOT NULL""",
+ JOIN extensioninstalls ON (extensioninstalls.version=extensionversions.id)
+ WHERE extensioninstalls.uid=%s""",
(user.id,))
updated = []
- for author_name, author_fullname, extension_name, version_name, sha1 in cursor:
+ for author_name, author_fullname, extension_name, version_name, version_sha1 in cursor:
extension = Extension(author_name, extension_name)
- if extension.getCurrentSHA1(version_name) != sha1:
+ if extension.getCurrentSHA1(version_name) != version_sha1:
updated.append((author_fullname, extension_name))
return updated
@staticmethod
- def find():
- extensions = []
+ def find(db):
+ def search(user_name, search_dir):
+ if not (os.path.isdir(search_dir) and
+ os.access(search_dir, os.X_OK | os.R_OK)):
+ return []
- for user_directory in os.listdir(configuration.extensions.SEARCH_ROOT):
- try:
- for extension_directory in os.listdir(os.path.join(configuration.extensions.SEARCH_ROOT, user_directory, "CriticExtensions")):
- extension_path = os.path.join(configuration.extensions.SEARCH_ROOT, user_directory, "CriticExtensions", extension_directory)
+ extensions = []
+
+ for extension_name in os.listdir(search_dir):
+ extension_dir = os.path.join(search_dir, extension_name)
+ manifest_path = os.path.join(extension_dir, "MANIFEST")
+
+ if not (os.path.isdir(extension_dir) and
+ os.access(extension_dir, os.X_OK | os.R_OK) and
+ os.access(manifest_path, os.R_OK)):
+ continue
+
+ extensions.append(Extension(user_name, extension_name))
+
+ return extensions
+
+ extensions = search(None, configuration.extensions.SYSTEM_EXTENSIONS_DIR)
+
+ if configuration.extensions.USER_EXTENSIONS_DIR:
+ cursor = db.cursor()
+ cursor.execute("SELECT name FROM users ORDER BY name ASC")
+
+ for (user_name,) in cursor:
+ try:
+ pwd_entry = pwd.getpwnam(user_name)
+ except KeyError:
+ continue
- if os.path.isdir(extension_path) and os.access(extension_path, os.X_OK | os.R_OK):
- manifest_path = os.path.join(extension_path, "MANIFEST")
+ user_dir = os.path.join(
+ pwd_entry.pw_dir, configuration.extensions.USER_EXTENSIONS_DIR)
- if os.path.isfile(manifest_path) and os.access(manifest_path, os.R_OK):
- extensions.append(Extension(user_directory, extension_directory))
- except OSError:
-