Merge pull request #206 from PolyJIT/f/cli-projects

This turns out to be one of the rather messy pull requests for 3.4.

In general(!), there is not much of an API change happening with 3.4, hence the minor release.
However, there are a few crucial things that will break user-code (sorry):

Let's list the breaking features in order:
* cli: You can no longer use -P as a switch for selecting projects. The cli takes projects as a 
    variable length list.
* projects: Projects no longer provide methods for build/prepare/download. Instead, there 
    are only 2 methods used per project.
    1) compile: All steps necessary to compile the project.
    2) run_tests: All steps necessary to run the project's tests. Assume that the project is 
        already compiled.
    We reduce a project's API to match the kinds of experiments we want to support. In 
    general, there are only 2 kinds of experiments: Compile-Time & Run-Time based ones.

    In addition, projects can now make use of the new download API. This provides a 
    generated method to download the project's source files: download and a new method 
    that lists all available versions for a given project. Almost all supported projects make 
    use of this new API from now on.
* actions: A new action to serve as a delegate for a project's compile method has been
    added (actions.Compile).
* uchroot/unionfs: Both have gotten a major rework & refactoring. No new features. 
    Please refer to the API docs for details.
* settings: The BB_ENV_* section is now generalized, i.e., any dictionary key entered here 
    will be treated as an environment variable. For now, we still only make use of PATH and 
    LD_LIBRARY_PATH, but we will honor all variables in a future commit.
* gentoo: Complete rewrite of the containerize feature. Before, all actions would need to 
   be wrapped into a call to uchroot, which made path handling kinda convoluted. In this 
   rewrite, we redirect into a container-environment once and execute the gentoo based 
   projects inside the container.
   This installs a copy of benchbuild inside the container and calls benchbuild inside with 
   the same arguments as outside. This is very experimental and will be refined in future 
   commits.
* settings (2): Configuration.value() has become a property now (Configuration.value). 
   Furthermore, we added conversion functions for bool and int. This enables the use of 
   configuration elements in boolean expressions and ranges.
* download: downloader got renamed to download.
* uchroot: removed the uchroot_* prefix from all functions.
* container: removed the container suffix/prefix from all functions.
* extensions: all extensions have been split into separate modules.

* Minor Bugfixes, Linter, Refactorings (renamings). Most not visible to users of 
   benchbuild.

.travis.yml

@@ -14,28 +14,22 @@ addons:
       - libfuse-dev
       - clang
 
-services:
-  - postgresql
-
 install:
   - pip install .
   - pip install -r requirements.txt
   - pip install codecov
   - pip install pytest-cov
 
 env:
-  - BB_UNIONFS_ENABLE=false BB_PLUGINS_EXPERIMENTS='["benchbuild.experiments.raw", "benchbuild.experiments.empty"]';
-
-before_script:
-  - psql -c 'create database pprof;' -U postgres
+  - BB_UNIONFS_ENABLE=false BB_PLUGINS_EXPERIMENTS='["benchbuild.experiments.raw", "benchbuild.experiments.empty"]'
 
 script:
+  - coverage run -a `which benchbuild` bootstrap -s
   - pytest --cov=./ benchbuild tests
   - coverage run -a `which benchbuild` config view
   - coverage run -a `which benchbuild` project view
   - coverage run -a `which benchbuild` experiment view
-  - coverage run -a `which benchbuild` bootstrap -s
-  - coverage run -a `which benchbuild` -vvvvv run --full -P test
+  - coverage run -a `which benchbuild` -vvvvv run --full test
   - codecov
 
 branches:

benchbuild/cli/bootstrap.py

@@ -2,16 +2,11 @@
 
 from plumbum import cli
 
-import benchbuild.utils.bootstrap as bs
+from benchbuild.utils import bootstrap
 from benchbuild.cli.main import BenchBuild
-from benchbuild.settings import CFG
+from benchbuild import settings
 
-provide_package = bs.provide_package
-provide_packages = bs.provide_packages
-find_package = bs.find_package
-install_package = bs.find_package
-install_uchroot = bs.install_uchroot
-check_uchroot_config = bs.check_uchroot_config
+CFG = settings.CFG
 
 
 @BenchBuild.subcommand("bootstrap")
@@ -27,26 +22,12 @@ def main(self, *args):
         del args  # Unused
 
         print("Checking benchbuild binary dependencies...")
-        provide_package("cmake")
-        provide_package("fusermount")
-        provide_package("unionfs")
-        provide_package("postgres")
-
-        has_uchroot = find_package("uchroot")
-        if not has_uchroot:
-            install_uchroot()
-            has_uchroot = find_package("uchroot")
-            if not has_uchroot:
-                print("NOT INSTALLED")
-        if has_uchroot:
-            check_uchroot_config()
-
-        provide_packages([
-            "mkdir", "git", "tar", "mv", "rm", "bash", "rmdir", "time",
-            "chmod", "cp", "ln", "make", "unzip", "cat", "patch", "find",
-            "echo", "grep", "sed", "sh", "autoreconf", "ruby", "curl", "tail",
-            "kill", "virtualenv", "timeout"
-        ])
+        bootstrap.provide_package("cmake")
+        bootstrap.provide_package("fusermount")
+        bootstrap.provide_package("unionfs")
+        bootstrap.provide_package(
+            'uchroot', installer=bootstrap.install_uchroot)
+        bootstrap.provide_packages(CFG['bootstrap']['packages'].value)
 
         if self.store_config:
             config_path = ".benchbuild.yml"

benchbuild/cli/experiment.py

@@ -2,12 +2,10 @@
 import sqlalchemy as sa
 import urwid
 from plumbum import cli
-import functools
 
-import benchbuild.experiment as exp
-import benchbuild.experiments as experiments
-import benchbuild.utils.schema as schema
+from benchbuild import experiment, experiments
 from benchbuild.cli.main import BenchBuild
+from benchbuild.utils import schema
 
 
 @BenchBuild.subcommand("experiment")
@@ -25,7 +23,7 @@ class BBExperimentView(cli.Application):
 
     def main(self):
         experiments.discover()
-        all_exps = exp.ExperimentRegistry.experiments
+        all_exps = experiment.ExperimentRegistry.experiments
         for exp_cls in all_exps.values():
             print(exp_cls.NAME)
             docstring = exp_cls.__doc__ or "-- no docstring --"
@@ -121,13 +119,13 @@ def maybe_exit(key):
 
         # yapf: disable
         palette = [(None, 'light gray', 'black'),
-                    ('heading', 'black', 'light gray'),
-                    ('line', 'black', 'light gray'),
-                    ('options', 'dark gray', 'black'),
-                    ('focus heading', 'white', 'dark red'),
-                    ('focus line', 'black', 'dark red'),
-                    ('focus options', 'black', 'light gray'),
-                    ('selected', 'white', 'dark blue')]
+                   ('heading', 'black', 'light gray'),
+                   ('line', 'black', 'light gray'),
+                   ('options', 'dark gray', 'black'),
+                   ('focus heading', 'white', 'dark red'),
+                   ('focus line', 'black', 'dark red'),
+                   ('focus options', 'black', 'light gray'),
+                   ('selected', 'white', 'dark blue')]
 
         # yapf: enable
         top = HorizontalBoxes()
@@ -148,18 +146,18 @@ def get_template():
     return env.get_template('experiment_show.txt.inc')
 
 
-def render_experiment(experiment):
+def render_experiment(_experiment):
     template = get_template()
     sess = schema.Session()
 
     return template.render(
-        name=experiment.name,
-        description=experiment.description,
-        start_date=experiment.begin,
-        end_date=experiment.end,
-        id=experiment.id,
-        num_completed_runs=get_completed_runs(sess, experiment),
-        num_failed_runs=get_failed_runs(sess, experiment))
+        name=_experiment.name,
+        description=_experiment.description,
+        start_date=_experiment.begin,
+        end_date=_experiment.end,
+        id=_experiment.id,
+        num_completed_runs=get_completed_runs(sess, _experiment),
+        num_failed_runs=get_failed_runs(sess, _experiment))
 
 
 def refresh_root_window(root):
@@ -193,4 +191,4 @@ def get_completed_runs(session, exp):
 def get_failed_runs(session, exp):
     return session.query(sa.func.count(schema.Run.id)).\
         filter(schema.Run.experiment_group == exp.id).\
-        filter(schema.Run.status != 'completed').scalar()
+        filter(schema.Run.status != 'completed').scalar()

benchbuild/cli/log.py

@@ -59,15 +59,6 @@ def experiment_ids(self, experiment_ids):
         """ Set the experiment ids to fetch the log for. """
         self._experiment_ids = experiment_ids
 
-    @cli.switch(
-        ["-P", "--project"],
-        str,
-        list=True,
-        help="Projects to fetch the log for.")
-    def project(self, projects):
-        """ Set the projects to fetch the log for. """
-        self._projects = projects
-
     @cli.switch(
         ["-p", "--project-id"],
         str,
@@ -88,19 +79,17 @@ def log_type(self, types):
 
     _experiments = None
     _experiment_ids = None
-    _projects = None
     _project_ids = None
     _types = None
 
-    def main(self):
+    def main(self, *projects):
         """ Run the log command. """
         from benchbuild.utils.schema import Session, Run, RunLog
 
         session = Session()
 
         exps = self._experiments
         exp_ids = self._experiment_ids
-        projects = self._projects
         project_ids = self._project_ids
         types = self._types
 

benchbuild/cli/main.py

@@ -1,4 +1,5 @@
 """Main CLI unit of BenchBuild."""
+import os
 from plumbum import cli
 
 from benchbuild import settings
@@ -8,7 +9,7 @@
 class BenchBuild(cli.Application):
     """Frontend for running/building the benchbuild study framework."""
 
-    VERSION = settings.CFG["version"].value()
+    VERSION = str(settings.CFG["version"])
     _list_env = False
 
     verbosity = cli.CountOf('-v', help="Enable verbose output")
@@ -18,13 +19,15 @@ def main(self, *args):
         self.verbosity = self.verbosity if self.verbosity < 6 else 5
         if self.debug:
             self.verbosity = 3
-        settings.CFG["verbosity"] = self.verbosity
+        verbosity = int(os.getenv('BB_VERBOSITY', self.verbosity))
+
+        settings.CFG["verbosity"] = verbosity
         settings.CFG["debug"] = self.debug
 
         log.configure()
         log.set_defaults()
 
-        if settings.CFG["db"]["create_functions"].value():
+        if settings.CFG["db"]["create_functions"]:
             from benchbuild.utils.schema import init_functions, Session
             init_functions(Session())
 

benchbuild/cli/project.py

@@ -1,7 +1,7 @@
 """Subcommand for project handling."""
 from plumbum import cli
-import benchbuild.project as project
-import benchbuild.experiments.empty as empty
+
+from benchbuild import project
 from benchbuild.cli.main import BenchBuild
 
 
@@ -18,28 +18,21 @@ def main(self):
 class BBProjectView(cli.Application):
     """View available projects."""
 
-    project_names = []
-    group_names = None
-
-    @cli.switch(["-P", "--project"], str, list=True, help="Include project.")
-    def set_projects(self, names):
-        self.project_names = names
+    groups = None
 
     @cli.switch(
         ["-G", "--group"],
         str,
         list=True,
         help="Include projects of this group.")
     def set_group(self, groups):
-        self.group_names = groups
+        self.groups = groups
 
-    def main(self):
-        prjs = project.populate(self.project_names, self.group_names)
-        ee = empty.Empty(projects=prjs)
-        print_projects(ee)
+    def main(self, *projects):
+        print_projects(project.populate(projects, self.groups))
 
 
-def print_projects(exp):
+def print_projects(projects=None):
     """
     Print a list of projects registered for that experiment.
 
@@ -48,10 +41,10 @@ def print_projects(exp):
 
     """
     grouped_by = {}
-    projects = exp.projects
     if not projects:
         print(
             "Your selection didn't include any projects for this experiment.")
+        return
 
     for name in projects:
         prj = projects[name]
@@ -67,11 +60,22 @@ def print_projects(exp):
         group_projects = sorted(grouped_by[name])
         for prj in group_projects:
             prj_cls = projects[prj]
-            print("  name: {id:<32} {version:<24} {src}".format(
-                id="{0}/{1}".format(prj_cls.NAME, prj_cls.GROUP),
-                version="version: {0}".format(prj_cls.VERSION),
-                src="source: {0}".format(prj_cls.SRC_FILE)))
+
+            version_str = None
+            if hasattr(prj_cls, 'versions'):
+                version_str = ", ".join(prj_cls.versions())
+
+            project_id = "{0}/{1}".format(prj_cls.NAME, prj_cls.GROUP)
+
+            project_str = \
+                "  name: {id:<32} version: {version:<24} source: {src}".format(
+                    id=str(project_id),
+                    version=str(prj_cls.VERSION),
+                    src=str(prj_cls.SRC_FILE))
+            print(project_str)
             if prj_cls.__doc__:
-                print("    description: {desc}".format(
-                    desc=prj_cls.__doc__.strip("\n ")))
+                docstr = prj_cls.__doc__.strip("\n ")
+                print("    description: {desc}".format(desc=docstr))
+            if version_str:
+                print("    versions: {versions}".format(versions=version_str))
         print()

benchbuild/cli/report.py

@@ -1,9 +1,8 @@
 from plumbum import cli
 
-import benchbuild.utils.schema as schema
-import benchbuild.experiments as e
-import benchbuild.reports as r
+from benchbuild import experiments, reports
 from benchbuild.cli.main import BenchBuild
+from benchbuild.utils import schema
 
 
 @BenchBuild.subcommand("report")
@@ -22,16 +21,16 @@ def __init__(self, executable):
         str,
         list=True,
         help="Specify the reports to generate")
-    def reports(self, reports):
-        self.report_names = reports
+    def reports(self, _reports):
+        self.report_names = _reports
 
     @cli.switch(
         ["-E", "--experiment"],
         str,
         list=True,
         help="Specify experiments to run")
-    def experiments(self, experiments):
-        self.experiment_names = experiments
+    def experiments(self, _experiments):
+        self.experiment_names = _experiments
 
     @cli.switch(
         ["-e", "--experiment-id"],
@@ -48,17 +47,17 @@ def outfile(self, outfile):
     def main(self, *args):
         del args  # Unused
 
-        e.discover()
-        r.discover()
-        all_reports = r.ReportRegistry.reports
+        experiments.discover()
+        reports.discover()
+        all_reports = reports.ReportRegistry.reports
 
-        def generate_reports(reports, experiments=None):
+        def generate_reports(_reports, _experiments=None):
             if not reports:
                 print("No reports found. Sorry.")
 
-            for rcls in reports:
-                if experiments:
-                    for exp in experiments:
+            for rcls in _reports:
+                if _experiments:
+                    for exp in _experiments:
                         report = rcls(exp, self._experiment_ids, self._outfile,
                                       schema.Session())
                 else:
@@ -67,18 +66,18 @@ def generate_reports(reports, experiments=None):
                 report.generate()
 
         if self.report_names:
-            reports = [
+            _reports = [
                 all_reports[name] for name in all_reports
                 if name in self.report_names
             ]
-            generate_reports(reports, self.experiment_names)
+            generate_reports(_reports, self.experiment_names)
             exit(0)
 
         if self.experiment_names:
-            reports = [
+            _reports = [
                 all_reports[name] for name in all_reports
                 if set(all_reports[name].SUPPORTED_EXPERIMENTS) & set(
                     self.experiment_names)
             ]
-            generate_reports(reports, self.experiment_names)
+            generate_reports(_reports, self.experiment_names)
             exit(0)