From 13137f5c28b8405f2428ca715f203be8dd2b72dd Mon Sep 17 00:00:00 2001 From: Andrii Nikitin Date: Thu, 15 Jul 2021 14:36:03 +0200 Subject: [PATCH 1/2] Add support for ISO and HDD in the same flavor --- script/cfg.py | 19 +++++++++++---- script/scriptgen.py | 27 ++++++++++++++++++---- t/abs/TestIsoHdd-1/files_iso.lst | 2 ++ t/abs/TestIsoHdd-1/print_openqa.before | 14 +++++++++++ t/abs/TestIsoHdd-1/print_rsync_iso.before | 6 +++++ t/abs/TestIsoHdd-1/print_rsync_repo.before | 0 xml/abs/TestIsoHdd.xml | 7 ++++++ 7 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 t/abs/TestIsoHdd-1/files_iso.lst create mode 100644 t/abs/TestIsoHdd-1/print_openqa.before create mode 100644 t/abs/TestIsoHdd-1/print_rsync_iso.before create mode 100644 t/abs/TestIsoHdd-1/print_rsync_repo.before create mode 100644 xml/abs/TestIsoHdd.xml diff --git a/script/cfg.py b/script/cfg.py index 0331361e..e3db13c2 100644 --- a/script/cfg.py +++ b/script/cfg.py @@ -91,8 +91,12 @@ def rsync_commands(checksum): break fi done - [ -n "$folder" ] || continue - echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src /var/lib/openqa/factory/hdd/" + # [ -n "$folder" ] || continue + if [[ $src =~ .iso$ ]]; then + echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src /var/lib/openqa/factory/iso/" + else + echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src /var/lib/openqa/factory/hdd/" + fi echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src.sha256 /var/lib/openqa/factory/other/" echo "" done < <(grep ${arch} __envsub/files_iso.lst | sort) @@ -282,11 +286,16 @@ def openqa_call_start_ex(checksum): break fi done - [ -n "$folder" ] || continue + # [ -n "$folder" ] || continue n=$((i++)) echo " ASSET_$((n+255))=$src.sha256 \\\\" - echo " HDD_$n=$src \\\\" - echo " CHECKSUM_HDD_$n=\$(cut -b-64 /var/lib/openqa/factory/other/$src.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \\\\" + if [[ $src =~ .iso$ ]]; then + echo " ISO=$src \\\\" + echo " CHECKSUM_ISO=\$(cut -b-64 /var/lib/openqa/factory/other/$src.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \\\\" + else + echo " HDD_$n=$src \\\\" + echo " CHECKSUM_HDD_$n=\$(cut -b-64 /var/lib/openqa/factory/other/$src.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \\\\" + fi done < <(grep ${arch} __envsub/files_iso.lst | sort) ''' diff --git a/script/scriptgen.py b/script/scriptgen.py index dc015b55..2f95c6a9 100644 --- a/script/scriptgen.py +++ b/script/scriptgen.py @@ -400,6 +400,11 @@ def doFlavor(self, node): self.asset_folders[t.attrib["filemask"]] = assets_folder for t in node.findall(".//*"): + if t.tag == "iso": + self.isos.append(t.attrib["filemask"]) + if t.attrib.get("folder", ""): + self.iso_folder[t.attrib["filemask"]] = t.attrib["folder"] + self.hdd_folder[t.attrib["filemask"]] = t.attrib["folder"] if t.tag == "hdd": self.hdds.append(t.attrib["filemask"]) if node.attrib.get("folder", ""): @@ -522,7 +527,19 @@ def gen_read_files(self, f): self.p(cfg.read_files_isos, f) else: for iso in self.isos: - self.p(cfg.read_files_iso, f, "FOLDER", self.iso_folder.get(iso, ""), "SRCISO", iso) + if "*" in iso: + self.p( + cfg.read_files_iso, + f, + "FOLDER", + self.iso_folder.get(iso, ""), + "SRCISO", + "", + "Media1?.iso$", + iso, + ) + else: + self.p(cfg.read_files_iso, f, "FOLDER", self.iso_folder.get(iso, ""), "SRCISO", iso) if self.repolink: self.p(cfg.read_files_repo_link, f) @@ -664,7 +681,7 @@ def gen_print_array_flavor_filter(self, f): if not self.isos and not self.hdds: for fl, h in zip(self.flavors, self.assets): self.p("flavor_filter[{}]='{}'".format(fl, h), f) - if not self.assets and not self.hdds: + if not self.assets and ((not self.hdds) or len(self.flavors) == 1): for fl, iso in zip(self.flavors, self.isos): if iso != "1" and iso != "0" and iso != "extract_as_repo": if not added_declare_flavor_filter: @@ -691,7 +708,7 @@ def gen_print_array_iso_folder(self, f): self.p("iso_folder[{}]='{}/'".format(k, v), f) def gen_print_array_hdd_folder(self, f): - if len(self.hdd_folder) > 1 and not self.isos and len(self.flavors) == 1: + if ((len(self.hdd_folder) > 1) or (self.hdd_folder and self.isos)) and len(self.flavors) == 1: self.p("declare -A hdd_folder", f) for k, v in self.hdd_folder.items(): self.p("hdd_folder[{}]='{}'".format(k, v), f) @@ -721,7 +738,7 @@ def gen_print_rsync_assets(self, f): def gen_print_rsync_iso(self, f): print(cfg.header, file=f) self.gen_print_array_no_rsync(f) - if len(self.hdds) > 1 and not self.isos and len(self.flavors) == 1: + if ((len(self.hdds) > 1 and (not self.isos)) or (self.isos and self.hdds)) and len(self.flavors) < 2: self.gen_print_array_hdd_folder(f) if self.archs == "armv7hl": self.p(cfg.rsync_hdds, f, "grep ${arch}", "grep ${arch//armv7hl/armv7l}") @@ -905,7 +922,7 @@ def gen_print_openqa(self, f): i = 0 isos = self.isos.copy() - if len(self.hdds) > 1 and not self.isos and len(self.flavors) == 1: + if ((len(self.hdds) > 1 and not self.isos) or (self.hdds and self.isos)) and len(self.flavors) == 1: if self.archs == "armv7hl": self.p(cfg.openqa_call_start_hdds, f, "grep ${arch}", "grep ${arch//armv7hl/armv7l}") else: diff --git a/t/abs/TestIsoHdd-1/files_iso.lst b/t/abs/TestIsoHdd-1/files_iso.lst new file mode 100644 index 00000000..39087ead --- /dev/null +++ b/t/abs/TestIsoHdd-1/files_iso.lst @@ -0,0 +1,2 @@ +test-x86_64-Build1.1.iso +test-x86_64-Build1.1.qcow2 diff --git a/t/abs/TestIsoHdd-1/print_openqa.before b/t/abs/TestIsoHdd-1/print_openqa.before new file mode 100644 index 00000000..bcba125e --- /dev/null +++ b/t/abs/TestIsoHdd-1/print_openqa.before @@ -0,0 +1,14 @@ +/usr/share/openqa/script/client isos post --host localhost \ + ARCH=x86_64 \ + ASSET_256=test-x86_64-Build1.1.iso.sha256 \ + ASSET_257=test-x86_64-Build1.1.qcow2.sha256 \ + BUILD=1.1 \ + CHECKSUM_HDD_2=$(cut -b-64 /var/lib/openqa/factory/other/test-x86_64-Build1.1.qcow2.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \ + CHECKSUM_ISO=$(cut -b-64 /var/lib/openqa/factory/other/test-x86_64-Build1.1.iso.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \ + DISTRI=distri \ + FLAVOR=TEST \ + HDD_2=test-x86_64-Build1.1.qcow2 \ + ISO=test-x86_64-Build1.1.iso \ + VERSION=1 \ + _DEPRIORITIZEBUILD=1 + diff --git a/t/abs/TestIsoHdd-1/print_rsync_iso.before b/t/abs/TestIsoHdd-1/print_rsync_iso.before new file mode 100644 index 00000000..1761b54a --- /dev/null +++ b/t/abs/TestIsoHdd-1/print_rsync_iso.before @@ -0,0 +1,6 @@ +rsync --timeout=3600 -tlp4 --specials obspublish::openqa/TestIsoHdd-1/TestIsoHdd-1:/images/iso/test-x86_64-Build1.1.iso /var/lib/openqa/factory/iso/ +rsync --timeout=3600 -tlp4 --specials obspublish::openqa/TestIsoHdd-1/TestIsoHdd-1:/images/iso/test-x86_64-Build1.1.iso.sha256 /var/lib/openqa/factory/other/ + +rsync --timeout=3600 -tlp4 --specials obspublish::openqa/TestIsoHdd-1/TestIsoHdd-1:/images//test-x86_64-Build1.1.qcow2 /var/lib/openqa/factory/hdd/ +rsync --timeout=3600 -tlp4 --specials obspublish::openqa/TestIsoHdd-1/TestIsoHdd-1:/images//test-x86_64-Build1.1.qcow2.sha256 /var/lib/openqa/factory/other/ + diff --git a/t/abs/TestIsoHdd-1/print_rsync_repo.before b/t/abs/TestIsoHdd-1/print_rsync_repo.before new file mode 100644 index 00000000..e69de29b diff --git a/xml/abs/TestIsoHdd.xml b/xml/abs/TestIsoHdd.xml new file mode 100644 index 00000000..71c55359 --- /dev/null +++ b/xml/abs/TestIsoHdd.xml @@ -0,0 +1,7 @@ + + + + + + + From b57af1fe989d099de5b5b3f7afeb5fe9d6b4a2e3 Mon Sep 17 00:00:00 2001 From: Andrii Nikitin Date: Thu, 15 Jul 2021 15:49:18 +0200 Subject: [PATCH 2/2] Add offset attribute to flavor tag in xml This offset will be used as number part instead of in HDD_1 --- script/cfg.py | 2 -- script/scriptgen.py | 5 ++++- t/abs/TestIsoHdd-1/print_openqa.before | 8 ++++---- xml/abs/TestIsoHdd.xml | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/script/cfg.py b/script/cfg.py index e3db13c2..7d6d0937 100644 --- a/script/cfg.py +++ b/script/cfg.py @@ -91,7 +91,6 @@ def rsync_commands(checksum): break fi done - # [ -n "$folder" ] || continue if [[ $src =~ .iso$ ]]; then echo "rsync --timeout=3600 -tlp4 --specials PRODUCTISOPATH/$folder/$src /var/lib/openqa/factory/iso/" else @@ -286,7 +285,6 @@ def openqa_call_start_ex(checksum): break fi done - # [ -n "$folder" ] || continue n=$((i++)) echo " ASSET_$((n+255))=$src.sha256 \\\\" if [[ $src =~ .iso$ ]]; then diff --git a/script/scriptgen.py b/script/scriptgen.py index 2f95c6a9..24a52b57 100644 --- a/script/scriptgen.py +++ b/script/scriptgen.py @@ -176,6 +176,7 @@ def __init__(self, name, actionGenerator): self.meta_variables = "_OBSOLETE=1" if self.ag.brand != "obs" and not self.ag.staging(): self.meta_variables = "_DEPRIORITIZEBUILD=1" + self.offset = 1 def productpath(self): if self.dist_path: @@ -348,6 +349,8 @@ def doFlavor(self, node): self.distri = node.attrib["distri"] if node.attrib.get("legacy_builds", ""): self.legacy_builds = node.attrib["legacy_builds"] + if node.attrib.get("offset", ""): + self.offset = node.attrib["offset"] for t in node.findall(".//repos"): if t.attrib.get("archs", ""): @@ -926,7 +929,7 @@ def gen_print_openqa(self, f): if self.archs == "armv7hl": self.p(cfg.openqa_call_start_hdds, f, "grep ${arch}", "grep ${arch//armv7hl/armv7l}") else: - self.p(cfg.openqa_call_start_hdds, f) + self.p(cfg.openqa_call_start_hdds, f, "i=1", "i=" + self.offset) elif self.hdds or (self.assets and not self.isos): if self.hdds and self.productpath().startswith("http"): self.p(' echo " HDD_URL_1=PRODUCTPATH/$destiso \\\\"', f) diff --git a/t/abs/TestIsoHdd-1/print_openqa.before b/t/abs/TestIsoHdd-1/print_openqa.before index bcba125e..d30af698 100644 --- a/t/abs/TestIsoHdd-1/print_openqa.before +++ b/t/abs/TestIsoHdd-1/print_openqa.before @@ -1,13 +1,13 @@ /usr/share/openqa/script/client isos post --host localhost \ ARCH=x86_64 \ - ASSET_256=test-x86_64-Build1.1.iso.sha256 \ - ASSET_257=test-x86_64-Build1.1.qcow2.sha256 \ + ASSET_1155=test-x86_64-Build1.1.iso.sha256 \ + ASSET_1156=test-x86_64-Build1.1.qcow2.sha256 \ BUILD=1.1 \ - CHECKSUM_HDD_2=$(cut -b-64 /var/lib/openqa/factory/other/test-x86_64-Build1.1.qcow2.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \ + CHECKSUM_HDD_901=$(cut -b-64 /var/lib/openqa/factory/other/test-x86_64-Build1.1.qcow2.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \ CHECKSUM_ISO=$(cut -b-64 /var/lib/openqa/factory/other/test-x86_64-Build1.1.iso.sha256 | grep -E '[0-9a-f]{5,40}' | head -n1) \ DISTRI=distri \ FLAVOR=TEST \ - HDD_2=test-x86_64-Build1.1.qcow2 \ + HDD_901=test-x86_64-Build1.1.qcow2 \ ISO=test-x86_64-Build1.1.iso \ VERSION=1 \ _DEPRIORITIZEBUILD=1 diff --git a/xml/abs/TestIsoHdd.xml b/xml/abs/TestIsoHdd.xml index 71c55359..74ddb962 100644 --- a/xml/abs/TestIsoHdd.xml +++ b/xml/abs/TestIsoHdd.xml @@ -1,6 +1,6 @@ - +