diff --git a/backend/copr_backend/background_worker_build.py b/backend/copr_backend/background_worker_build.py index d5ccaf3a1..e9309047a 100644 --- a/backend/copr_backend/background_worker_build.py +++ b/backend/copr_backend/background_worker_build.py @@ -35,7 +35,7 @@ MAX_HOST_ATTEMPTS = 3 MAX_SSH_ATTEMPTS = 5 -MIN_BUILDER_VERSION = "0.54.1.dev" +MIN_BUILDER_VERSION = "0.68.dev" CANCEL_CHECK_PERIOD = 5 MESSAGES = { @@ -291,10 +291,6 @@ def _parse_results(self): """ Parse `results.json` and update the `self.job` object. """ - if self.job.chroot == "srpm-builds": - # We care only about final RPMs - return - path = os.path.join(self.job.results_dir, "results.json") if not os.path.exists(path): raise BackendError("results.json file not found in resultdir") diff --git a/backend/copr_backend/frontend.py b/backend/copr_backend/frontend.py index 46617c50c..8c75204d1 100644 --- a/backend/copr_backend/frontend.py +++ b/backend/copr_backend/frontend.py @@ -9,7 +9,7 @@ from copr_backend.exceptions import FrontendClientException # The frontend counterpart is in `backend_general:send_frontend_version` -MIN_FE_BE_API = 5 +MIN_FE_BE_API = 6 class FrontendClient: """ diff --git a/backend/tests/test_background_worker_build.py b/backend/tests/test_background_worker_build.py index d7c66078d..94d39def4 100644 --- a/backend/tests/test_background_worker_build.py +++ b/backend/tests/test_background_worker_build.py @@ -311,7 +311,8 @@ def test_prev_build_backup(f_build_rpm_case): assert len(glob.glob(os.path.join(prev_results, rsync_patt))) == 1 -def test_full_srpm_build(f_build_srpm): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_full_srpm_build(_parse_results, f_build_srpm): worker = f_build_srpm.bw worker.process() assert worker.job.pkg_name == "example" @@ -326,7 +327,9 @@ def test_full_srpm_build(f_build_srpm): @mock.patch("copr_backend.background_worker_build.find_spec_file") -def test_full_srpm_build_without_specfile(mock_find_spec_file, f_build_srpm): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_full_srpm_build_without_specfile(_parse_results, mock_find_spec_file, + f_build_srpm): mock_find_spec_file.return_value = None worker = f_build_srpm.bw @@ -784,7 +787,8 @@ def test_createrepo_failure(mc_call_copr_repo, f_build_rpm_case, caplog): ], caplog) @_patch_bwbuild_object("pkg_name_evr") -def test_pkg_collect_failure(mc_pkg_evr, f_build_srpm, caplog): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_pkg_collect_failure(_parse_results, mc_pkg_evr, f_build_srpm, caplog): mc_pkg_evr.side_effect = CoprBackendSrpmError("srpm error") config = f_build_srpm worker = config.bw @@ -868,7 +872,8 @@ def test_unable_to_start_builder(f_build_srpm, caplog): assert_logs_dont_exist(["Retry"], caplog) @_patch_bwbuild_object("time.sleep", mock.MagicMock()) -def test_retry_vm_factory_take(f_build_srpm, caplog): +@_patch_bwbuild_object("BuildBackgroundWorker._parse_results") +def test_retry_vm_factory_take(_parse_results, f_build_srpm, caplog): config = f_build_srpm rhf = config.resalloc_host_factory host = config.host diff --git a/beaker-tests/Sanity/copr-cli-basic-operations/runtest-exclusivearch.sh b/beaker-tests/Sanity/copr-cli-basic-operations/runtest-exclusivearch.sh new file mode 100755 index 000000000..0fe9d454d --- /dev/null +++ b/beaker-tests/Sanity/copr-cli-basic-operations/runtest-exclusivearch.sh @@ -0,0 +1,51 @@ +#! /bin/bash + +# Include Beaker environment +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +# Load config settings +HERE=$(dirname "$(realpath "$0")") +source "$HERE/config" +source "$HERE/helpers" + + +rlJournalStart + rlPhaseStartSetup + setup_checks + rlAssertRpm "jq" + rlPhaseEnd + + rlPhaseStartTest + chroots="" + chroots+=" --chroot fedora-$FEDORA_VERSION-x86_64" + chroots+=" --chroot fedora-$FEDORA_VERSION-aarch64" + chroots+=" --chroot fedora-$FEDORA_VERSION-ppc64le" + chroots+=" --chroot fedora-$FEDORA_VERSION-s390x" + + OUTPUT=`mktemp` + + # Test ExclusiveArch + rlRun "copr-cli create ${NAME_PREFIX}ExclusiveArch $chroots" + rlRun "copr-cli build-distgit ${NAME_PREFIX}ExclusiveArch --name biosdevname --commit $BRANCH" + rlRun "copr monitor ${NAME_PREFIX}ExclusiveArch > $OUTPUT" + rlAssertEquals "Skipped chroots" `cat $OUTPUT |grep "skipped" |wc -l` 3 + rlAssertEquals "Succeeded chroots" `cat $OUTPUT |grep "succeeded" |wc -l` 1 + + # This is a more complicated package with `BuildArch: noarch` and + # ExclusiveArch for subpackages. Test that we don't fail while parsing it + rlRun "copr-cli build-distgit ${NAME_PREFIX}ExclusiveArch --name procyon" + + # Test ExcludeArch + rlRun "copr-cli create ${NAME_PREFIX}ExcludeArch $chroots" + rlRun "copr-cli build-distgit ${NAME_PREFIX}ExcludeArch --name python-giacpy" + rlRun "copr monitor ${NAME_PREFIX}ExcludeArch > $OUTPUT" + rlAssertEquals "Skipped chroots" `cat $OUTPUT |grep "skipped" |wc -l` 3 + rlAssertEquals "Succeeded chroots" `cat $OUTPUT |grep "succeeded" |wc -l` 1 + rlPhaseEnd + + rlPhaseStartCleanup + rlRun "copr-cli delete ${NAME_PREFIX}ExclusiveArch" + rlRun "copr-cli delete ${NAME_PREFIX}ExcludeArch" + rlPhaseEnd +rlJournalPrintText +rlJournalEnd diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index dadf3a26a..030f3f17d 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -1011,19 +1011,34 @@ def update_state_from_dict(cls, build, upd_dict): buildchroot.status = chroot_status db.session.add(buildchroot) + def finish(chroot, status): + chroot.status = status + chroot.ended_on = upd_dict.get("ended_on") or time.time() + chroot.started_on = upd_dict.get("started_on", chroot.ended_on) + db.session.add(chroot) + build.source_status = new_status if new_status == StatusEnum("failed") or \ new_status == StatusEnum("skipped"): for ch in build.build_chroots: - ch.status = new_status - ch.ended_on = upd_dict.get("ended_on") or time.time() - ch.started_on = upd_dict.get("started_on", ch.ended_on) - db.session.add(ch) + finish(ch, new_status) if new_status == StatusEnum("failed"): build.fail_type = FailTypeEnum("srpm_build_error") BuildsLogic.delete_local_source(build) + # Skip excluded architectures + if upd_dict.get("chroot") == "srpm-builds" and upd_dict.get("results"): + for chroot in build.build_chroots: + arch = chroot.mock_chroot.arch + + exclusivearch = upd_dict["results"]["exclusivearch"] + if exclusivearch and arch not in exclusivearch: + finish(chroot, StatusEnum("skipped")) + + if arch in upd_dict["results"]["excludearch"]: + finish(chroot, StatusEnum("skipped")) + cls.process_update_callback(build) db.session.add(build) return diff --git a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py index d3be99907..2be533678 100755 --- a/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py +++ b/frontend/coprs_frontend/coprs/views/backend_ns/backend_general.py @@ -22,7 +22,7 @@ def send_frontend_version(response): setup the version according to our needs. For the backend counterpart, see the `MIN_FE_BE_API` constant. """ - response.headers['Copr-FE-BE-API-Version'] = '5' + response.headers['Copr-FE-BE-API-Version'] = '6' return response diff --git a/rpmbuild/copr_rpmbuild/automation/__init__.py b/rpmbuild/copr_rpmbuild/automation/__init__.py index 4d0d29d3b..77439fe29 100644 --- a/rpmbuild/copr_rpmbuild/automation/__init__.py +++ b/rpmbuild/copr_rpmbuild/automation/__init__.py @@ -4,6 +4,7 @@ """ from copr_rpmbuild.automation.fedora_review import FedoraReview +from copr_rpmbuild.automation.srpm_results import SRPMResults from copr_rpmbuild.automation.rpm_results import RPMResults @@ -12,7 +13,7 @@ def run_automation_tools(task, resultdir, mock_config_file, log): Iterate over all supported post-build tools (e.g. `fedora-review`, `rpmlint`, etc) and run the desired ones for a given task. """ - tools = [FedoraReview, RPMResults] + tools = [FedoraReview, SRPMResults, RPMResults] for _class in tools: tool = _class(task, resultdir, mock_config_file, log) if not tool.enabled: diff --git a/rpmbuild/copr_rpmbuild/automation/rpm_results.py b/rpmbuild/copr_rpmbuild/automation/rpm_results.py index c370a56f3..2e5e467cf 100644 --- a/rpmbuild/copr_rpmbuild/automation/rpm_results.py +++ b/rpmbuild/copr_rpmbuild/automation/rpm_results.py @@ -3,9 +3,9 @@ """ import os -import rpm import simplejson from copr_rpmbuild.automation.base import AutomationTool +from copr_rpmbuild.helpers import get_rpm_header class RPMResults(AutomationTool): @@ -53,7 +53,7 @@ def get_nevra_dict(cls, path): msg = "File name doesn't end with '.rpm': {}".format(path) raise ValueError(msg) - hdr = cls.get_rpm_header(path) + hdr = get_rpm_header(path) arch = "src" if filename.endswith(".src.rpm") else hdr["arch"] return { "name": hdr["name"], @@ -62,14 +62,3 @@ def get_nevra_dict(cls, path): "release": hdr["release"], "arch": arch, } - - @staticmethod - def get_rpm_header(path): - """ - Examine a RPM package file and return its header - See docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s04.html - """ - ts = rpm.TransactionSet() - with open(path, "r") as f: - hdr = ts.hdrFromFdno(f.fileno()) - return hdr diff --git a/rpmbuild/copr_rpmbuild/automation/srpm_results.py b/rpmbuild/copr_rpmbuild/automation/srpm_results.py new file mode 100644 index 000000000..cc4bf7b3a --- /dev/null +++ b/rpmbuild/copr_rpmbuild/automation/srpm_results.py @@ -0,0 +1,39 @@ +""" +Create `results.json` file for SRPM builds +""" + +import os +import simplejson +from copr_rpmbuild.automation.base import AutomationTool +from copr_rpmbuild.helpers import locate_srpm, get_rpm_header + + +class SRPMResults(AutomationTool): + """ + Create `results.json` for SRPM builds + """ + + @property + def enabled(self): + """ + Do this for every RPM build + """ + return not self.chroot + + def run(self): + """ + Create `results.json` + """ + data = self.get_package_info() + path = os.path.join(self.resultdir, "results.json") + with open(path, "w", encoding="utf-8") as dst: + simplejson.dump(data, dst, indent=4) + + def get_package_info(self): + """ + Return ``dict`` with interesting package metadata + """ + srpm = locate_srpm(self.resultdir) + hdr = get_rpm_header(srpm) + keys = ["name", "exclusivearch", "excludearch"] + return {key: hdr[key] for key in keys} diff --git a/rpmbuild/copr_rpmbuild/helpers.py b/rpmbuild/copr_rpmbuild/helpers.py index 3fda09806..b26ed76f0 100644 --- a/rpmbuild/copr_rpmbuild/helpers.py +++ b/rpmbuild/copr_rpmbuild/helpers.py @@ -101,6 +101,17 @@ def locate_srpm(dirpath): return srpm_path +def get_rpm_header(path): + """ + Examine a RPM package file and return its header + See docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s04.html + """ + ts = rpm.TransactionSet() + with open(path, "rb") as f: + hdr = ts.hdrFromFdno(f.fileno()) + return hdr + + def get_package_name(spec_path): """ Obtain name of a package described by spec