frontend, backend, rpmbuild: skip excluded chroots

Fix #2137

backend/copr_backend/background_worker_build.py

@@ -35,7 +35,7 @@
 
 MAX_HOST_ATTEMPTS = 3
 MAX_SSH_ATTEMPTS = 5
-MIN_BUILDER_VERSION = "0.54.1.dev"
+MIN_BUILDER_VERSION = "0.68.dev"
 CANCEL_CHECK_PERIOD = 5
 
 MESSAGES = {
@@ -291,10 +291,6 @@ def _parse_results(self):
         """
         Parse `results.json` and update the `self.job` object.
         """
-        if self.job.chroot == "srpm-builds":
-            # We care only about final RPMs
-            return
-
         path = os.path.join(self.job.results_dir, "results.json")
         if not os.path.exists(path):
             raise BackendError("results.json file not found in resultdir")

backend/copr_backend/frontend.py

@@ -9,7 +9,7 @@
 from copr_backend.exceptions import FrontendClientException
 
 # The frontend counterpart is in `backend_general:send_frontend_version`
-MIN_FE_BE_API = 5
+MIN_FE_BE_API = 6
 
 class FrontendClient:
     """

backend/tests/test_background_worker_build.py

@@ -311,7 +311,8 @@ def test_prev_build_backup(f_build_rpm_case):
     assert len(glob.glob(os.path.join(prev_results, rsync_patt))) == 1
 
 
-def test_full_srpm_build(f_build_srpm):
+@_patch_bwbuild_object("BuildBackgroundWorker._parse_results")
+def test_full_srpm_build(_parse_results, f_build_srpm):
     worker = f_build_srpm.bw
     worker.process()
     assert worker.job.pkg_name == "example"
@@ -326,7 +327,9 @@ def test_full_srpm_build(f_build_srpm):
 
 
 @mock.patch("copr_backend.background_worker_build.find_spec_file")
-def test_full_srpm_build_without_specfile(mock_find_spec_file, f_build_srpm):
+@_patch_bwbuild_object("BuildBackgroundWorker._parse_results")
+def test_full_srpm_build_without_specfile(_parse_results, mock_find_spec_file,
+                                          f_build_srpm):
     mock_find_spec_file.return_value = None
 
     worker = f_build_srpm.bw
@@ -784,7 +787,8 @@ def test_createrepo_failure(mc_call_copr_repo, f_build_rpm_case, caplog):
     ], caplog)
 
 @_patch_bwbuild_object("pkg_name_evr")
-def test_pkg_collect_failure(mc_pkg_evr, f_build_srpm, caplog):
+@_patch_bwbuild_object("BuildBackgroundWorker._parse_results")
+def test_pkg_collect_failure(_parse_results, mc_pkg_evr, f_build_srpm, caplog):
     mc_pkg_evr.side_effect = CoprBackendSrpmError("srpm error")
     config = f_build_srpm
     worker = config.bw
@@ -868,7 +872,8 @@ def test_unable_to_start_builder(f_build_srpm, caplog):
     assert_logs_dont_exist(["Retry"], caplog)
 
 @_patch_bwbuild_object("time.sleep", mock.MagicMock())
-def test_retry_vm_factory_take(f_build_srpm, caplog):
+@_patch_bwbuild_object("BuildBackgroundWorker._parse_results")
+def test_retry_vm_factory_take(_parse_results, f_build_srpm, caplog):
     config = f_build_srpm
     rhf = config.resalloc_host_factory
     host = config.host

beaker-tests/Sanity/copr-cli-basic-operations/runtest-exclusivearch.sh

@@ -0,0 +1,51 @@
+#! /bin/bash
+
+# Include Beaker environment
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+# Load config settings
+HERE=$(dirname "$(realpath "$0")")
+source "$HERE/config"
+source "$HERE/helpers"
+
+
+rlJournalStart
+    rlPhaseStartSetup
+        setup_checks
+        rlAssertRpm "jq"
+    rlPhaseEnd
+
+    rlPhaseStartTest
+        chroots=""
+        chroots+=" --chroot fedora-$FEDORA_VERSION-x86_64"
+        chroots+=" --chroot fedora-$FEDORA_VERSION-aarch64"
+        chroots+=" --chroot fedora-$FEDORA_VERSION-ppc64le"
+        chroots+=" --chroot fedora-$FEDORA_VERSION-s390x"
+
+        OUTPUT=`mktemp`
+
+        # Test ExclusiveArch
+        rlRun "copr-cli create ${NAME_PREFIX}ExclusiveArch $chroots"
+        rlRun "copr-cli build-distgit ${NAME_PREFIX}ExclusiveArch --name biosdevname --commit $BRANCH"
+        rlRun "copr monitor ${NAME_PREFIX}ExclusiveArch > $OUTPUT"
+        rlAssertEquals "Skipped chroots" `cat $OUTPUT |grep "skipped" |wc -l` 3
+        rlAssertEquals "Succeeded chroots" `cat $OUTPUT |grep "succeeded" |wc -l` 1
+
+        # This is a more complicated package with `BuildArch: noarch` and
+        # ExclusiveArch for subpackages. Test that we don't fail while parsing it
+        rlRun "copr-cli build-distgit ${NAME_PREFIX}ExclusiveArch --name procyon"
+
+        # Test ExcludeArch
+        rlRun "copr-cli create ${NAME_PREFIX}ExcludeArch $chroots"
+        rlRun "copr-cli build-distgit ${NAME_PREFIX}ExcludeArch --name python-giacpy"
+        rlRun "copr monitor ${NAME_PREFIX}ExcludeArch > $OUTPUT"
+        rlAssertEquals "Skipped chroots" `cat $OUTPUT |grep "skipped" |wc -l` 3
+        rlAssertEquals "Succeeded chroots" `cat $OUTPUT |grep "succeeded" |wc -l` 1
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        rlRun "copr-cli delete ${NAME_PREFIX}ExclusiveArch"
+        rlRun "copr-cli delete ${NAME_PREFIX}ExcludeArch"
+    rlPhaseEnd
+rlJournalPrintText
+rlJournalEnd

frontend/coprs_frontend/coprs/logic/builds_logic.py

@@ -1011,19 +1011,34 @@ def update_state_from_dict(cls, build, upd_dict):
                         buildchroot.status = chroot_status
                         db.session.add(buildchroot)
 
+            def finish(chroot, status):
+                chroot.status = status
+                chroot.ended_on = upd_dict.get("ended_on") or time.time()
+                chroot.started_on = upd_dict.get("started_on", chroot.ended_on)
+                db.session.add(chroot)
+
             build.source_status = new_status
             if new_status == StatusEnum("failed") or \
                    new_status == StatusEnum("skipped"):
                 for ch in build.build_chroots:
-                    ch.status = new_status
-                    ch.ended_on = upd_dict.get("ended_on") or time.time()
-                    ch.started_on = upd_dict.get("started_on", ch.ended_on)
-                    db.session.add(ch)
+                    finish(ch, new_status)
 
             if new_status == StatusEnum("failed"):
                 build.fail_type = FailTypeEnum("srpm_build_error")
                 BuildsLogic.delete_local_source(build)
 
+            # Skip excluded architectures
+            if upd_dict.get("chroot") == "srpm-builds" and upd_dict.get("results"):
+                for chroot in build.build_chroots:
+                    arch = chroot.mock_chroot.arch
+
+                    exclusivearch = upd_dict["results"]["exclusivearch"]
+                    if exclusivearch and arch not in exclusivearch:
+                        finish(chroot, StatusEnum("skipped"))
+
+                    if arch in upd_dict["results"]["excludearch"]:
+                        finish(chroot, StatusEnum("skipped"))
+
             cls.process_update_callback(build)
             db.session.add(build)
             return

frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py

@@ -22,7 +22,7 @@ def send_frontend_version(response):
     setup the version according to our needs.
     For the backend counterpart, see the `MIN_FE_BE_API` constant.
     """
-    response.headers['Copr-FE-BE-API-Version'] = '5'
+    response.headers['Copr-FE-BE-API-Version'] = '6'
     return response
 
 

rpmbuild/copr_rpmbuild/automation/__init__.py

@@ -4,6 +4,7 @@
 """
 
 from copr_rpmbuild.automation.fedora_review import FedoraReview
+from copr_rpmbuild.automation.srpm_results import SRPMResults
 from copr_rpmbuild.automation.rpm_results import RPMResults
 
 
@@ -12,7 +13,7 @@ def run_automation_tools(task, resultdir, mock_config_file, log):
     Iterate over all supported post-build tools (e.g. `fedora-review`,
     `rpmlint`, etc) and run the desired ones for a given task.
     """
-    tools = [FedoraReview, RPMResults]
+    tools = [FedoraReview, SRPMResults, RPMResults]
     for _class in tools:
         tool = _class(task, resultdir, mock_config_file, log)
         if not tool.enabled:

rpmbuild/copr_rpmbuild/automation/rpm_results.py

@@ -3,9 +3,9 @@
 """
 
 import os
-import rpm
 import simplejson
 from copr_rpmbuild.automation.base import AutomationTool
+from copr_rpmbuild.helpers import get_rpm_header
 
 
 class RPMResults(AutomationTool):
@@ -53,7 +53,7 @@ def get_nevra_dict(cls, path):
             msg = "File name doesn't end with '.rpm': {}".format(path)
             raise ValueError(msg)
 
-        hdr = cls.get_rpm_header(path)
+        hdr = get_rpm_header(path)
         arch = "src" if filename.endswith(".src.rpm") else hdr["arch"]
         return {
             "name": hdr["name"],
@@ -62,14 +62,3 @@ def get_nevra_dict(cls, path):
             "release": hdr["release"],
             "arch": arch,
         }
-
-    @staticmethod
-    def get_rpm_header(path):
-        """
-        Examine a RPM package file and return its header
-        See docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s04.html
-        """
-        ts = rpm.TransactionSet()
-        with open(path, "r") as f:
-            hdr = ts.hdrFromFdno(f.fileno())
-            return hdr

rpmbuild/copr_rpmbuild/automation/srpm_results.py

@@ -0,0 +1,39 @@
+"""
+Create `results.json` file for SRPM builds
+"""
+
+import os
+import simplejson
+from copr_rpmbuild.automation.base import AutomationTool
+from copr_rpmbuild.helpers import locate_srpm, get_rpm_header
+
+
+class SRPMResults(AutomationTool):
+    """
+    Create `results.json` for SRPM builds
+    """
+
+    @property
+    def enabled(self):
+        """
+        Do this for every RPM build
+        """
+        return not self.chroot
+
+    def run(self):
+        """
+        Create `results.json`
+        """
+        data = self.get_package_info()
+        path = os.path.join(self.resultdir, "results.json")
+        with open(path, "w", encoding="utf-8") as dst:
+            simplejson.dump(data, dst, indent=4)
+
+    def get_package_info(self):
+        """
+        Return ``dict`` with interesting package metadata
+        """
+        srpm = locate_srpm(self.resultdir)
+        hdr = get_rpm_header(srpm)
+        keys = ["name", "exclusivearch", "excludearch"]
+        return {key: hdr[key] for key in keys}

rpmbuild/copr_rpmbuild/helpers.py

@@ -101,6 +101,17 @@ def locate_srpm(dirpath):
     return srpm_path
 
 
+def get_rpm_header(path):
+    """
+    Examine a RPM package file and return its header
+    See docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s04.html
+    """
+    ts = rpm.TransactionSet()
+    with open(path, "rb") as f:
+        hdr = ts.hdrFromFdno(f.fileno())
+        return hdr
+
+
 def get_package_name(spec_path):
     """
     Obtain name of a package described by spec