Fix temci exec builder and add documentation

doc/temci_build.rst

@@ -7,6 +7,10 @@ and to build these programs at once. This build config also contains the run con
 each program. ``temci build`` compiles a run config and stores it into a file that can be
 directly used with `temci exec <temci_exec.html>`_ (or other configured run drivers).
 
+**For most cases using the builder capabilities of `temci exec <temci_exec.html#building>`_
+should be enough. This also has the advantage of using a single command for all benchmarked
+programs, whether they need to built or not.**
+
 Usage
 -----
 

doc/temci_exec.rst

@@ -131,6 +131,38 @@ Configuring them in ``temci short exec`` is currently not possible.
    It can be used in combination with the :ref:`time<time runner>` and :ref:`perf_stat<perf_stat runner>` runners
    (using the ``--parse_output`` option or setting ``parse_output`` to true in the run block config).
 
+Building
+~~~~~~~~
+``temci exec`` supports to build the programs that are then benchmarked. It supports the same format and the same options as
+`temci build <temci_build.html>`_.
+
+In the most basic case (and the case that is thoroughly tested), just supply a build command:
+
+.. code:: yaml
+
+    - attributes: …
+      run_config: …
+      build_config:
+        cmd: make # a sample build command
+
+Executing the file with ``temci exec`` runs all available build commands.
+
+This can be configured using the following options (set in the ``run`` settings block):
+
+no_build
+    Do not build, default is false
+only_build:
+    Only build the build configs for all blocks, default is false
+abort_after_build_error
+    default true
+
+If building a block fails and ``abort_after_build_error`` is not true
+(e.g. ``--no-abort_after_build_error`` is passed), then temci
+produces an ``EXEC_INPUT_FILE.erroneous.yaml`` that contains the configurations
+of all failing blocks. This file can be used to execute the missing blocks
+again after the error is fixed. Use the ``--append`` option to append the benchmarks
+to the preexisting benchmark result file.
+
 Error Codes
 ~~~~~~~~~~~
 

temci/run/run_processor.py

@@ -193,8 +193,8 @@ def benchmark(self):
         """
         Benchmark and teardown.
         """
-        if not self.only_build:
-            pass
+        if self.only_build:
+            return
         try:
 
             show_progress = Settings().has_log_level("info") and \

tests/test_run.py

@@ -54,6 +54,22 @@ def test_build_before_exec_do_not_arbort():
     }, expect_success=False, raise_exc=False).out
 
 
+def test_build_before_exec_only_build():
+    assert "3333" not in run_temci("exec bla.yaml --runs 1", files={
+        "bla.yaml": [
+            {
+                "attributes": {"description": "3333"},
+                "run_config": {"cmd": "./test", "tags": []},
+                "build_config": {"cmd": "echo 'echo 3333' > test; chmod +x test"}
+            }
+        ]
+    }, settings={
+        "run": {
+            "only_build": True
+        }
+    }, expect_success=False, raise_exc=False).out
+
+
 def test_successful_run_errors():
     d = run_temci("short exec true").yaml_contents["run_output.yaml"][0]
     assert "internal_error" not in d