From b01b7d529af4d543a2d867a484906d6478463932 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Fri, 22 Nov 2019 13:21:06 +0100 Subject: [PATCH 1/7] Move accept logic into accept_command --- osc-staging.py | 14 +------------- osclib/accept_command.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/osc-staging.py b/osc-staging.py index c0036a65f..fb2ccf4ee 100644 --- a/osc-staging.py +++ b/osc-staging.py @@ -420,19 +420,7 @@ def do_staging(self, subcmd, opts, *args): api.days_since_last_freeze(prj))) elif cmd == 'accept': cmd = AcceptCommand(api) - for prj in args[1:]: - if cmd.perform(api.prj_from_letter(prj), opts.force): - cmd.reset_rebuild_data(prj) - else: - return - if not opts.no_cleanup: - if api.item_exists(api.prj_from_letter(prj)): - cmd.cleanup(api.prj_from_letter(prj)) - cmd.accept_other_new() - if opts.project.startswith('openSUSE:'): - cmd.update_factory_version() - if api.item_exists(api.crebuild): - cmd.sync_buildfailures() + cmd.accept_all(args[1:], opts.force, not opts.no_cleanup) elif cmd == 'unselect': UnselectCommand(api).perform(args[1:], opts.cleanup, opts.message) elif cmd == 'select': diff --git a/osclib/accept_command.py b/osclib/accept_command.py index 4b589692d..18de1f8f7 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -55,12 +55,27 @@ def reset_rebuild_data(self, project): stg.find('rebuild').text = 'unknown' stg.find('supportpkg').text = '' - # reset accpted staging project rebuild state to unknown and clean up + # reset accepted staging project rebuild state to unknown and clean up # supportpkg list content = ET.tostring(root) if content != data: self.api.pseudometa_file_save('support_pkg_rebuild', content, 'accept command update') + def accept_all(self, projects, force=False, cleanup=True): + for prj in projects: + if self.perform(self.api.prj_from_letter(prj), force): + self.reset_rebuild_data(prj) + else: + return + if cleanup: + if self.api.item_exists(self.api.prj_from_letter(prj)): + self.cleanup(self.api.prj_from_letter(prj)) + self.accept_other_new() + if self.api.project.startswith('openSUSE:'): + self.update_factory_version() + if self.api.item_exists(self.api.crebuild): + self.sync_buildfailures() + def perform(self, project, force=False): """Accept the staging project for review and submit to Factory / Leap ... From 62f97d15d1baa749aaeccb406f6fba6e6b2fadf7 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Fri, 22 Nov 2019 13:21:55 +0100 Subject: [PATCH 2/7] Fix accept command to force accept through API --- osclib/accept_command.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/osclib/accept_command.py b/osclib/accept_command.py index 18de1f8f7..44cbe9010 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -109,7 +109,11 @@ def perform(self, project, force=False): print(oldspecs) - u = self.api.makeurl(['staging', self.api.project, 'staging_projects', project, 'accept']) + opts = {} + if force: + opts['force'] = '1' + + u = self.api.makeurl(['staging', self.api.project, 'staging_projects', project, 'accept'], opts) f = http_POST(u) while self.api.project_status(project, reload=True, requests=False).get('state') != 'empty': From 63bda17769a01041d5a7502c8d02255094917f5b Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Sun, 24 Nov 2019 11:00:47 +0100 Subject: [PATCH 3/7] Collect packages for all stagings and ready before accepting --- osclib/accept_command.py | 101 +++++++++++++++------------------------ osclib/stagingapi.py | 10 ---- 2 files changed, 38 insertions(+), 73 deletions(-) diff --git a/osclib/accept_command.py b/osclib/accept_command.py index 44cbe9010..908d88c72 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -24,24 +24,20 @@ def __init__(self, api): self.api = api def find_new_requests(self, project): - query = "match=state/@name='new'+and+action/target/@project='{}'".format(project) - url = self.api.makeurl(['search', 'request'], query) + match = f"state/@name='new' and action/target/@project='{project}'" + url = self.api.makeurl(['search', 'request'], { 'match': match }) f = http_GET(url) root = ET.parse(f).getroot() rqs = [] for rq in root.findall('request'): - pkgs = [] - act_type = None - actions = rq.findall('action') - for action in actions: - act_type = action.get('type') - targets = action.findall('target') - for t in targets: - pkgs.append(str(t.get('package'))) - - rqs.append({'id': int(rq.get('id')), 'packages': pkgs, 'type': act_type}) + for action in rq.findall('action'): + for t in action.findall('target'): + rqs.append({'id': int(rq.get('id')), + 'package': str(t.get('package')), + 'type': action.get('type')}) + break return rqs def reset_rebuild_data(self, project): @@ -62,15 +58,43 @@ def reset_rebuild_data(self, project): self.api.pseudometa_file_save('support_pkg_rebuild', content, 'accept command update') def accept_all(self, projects, force=False, cleanup=True): + self.requests = { 'delete': [], 'submit': [] } + + for prj in projects: + project = self.api.prj_from_letter(prj) + + status = self.api.project_status(project) + if status.get('state') != 'acceptable' and not force: + print('The project "{}" is not yet acceptable.'.format(project)) + #return False + + for request in status.findall('staged_requests/request'): + self.requests[request.get('type')].append(request.get('package')) + + other_new = self.find_new_requests(self.api.project) + for req in other_new: + self.requests[req['type']].append(req['package']) + + print(self.requests) + return True for prj in projects: - if self.perform(self.api.prj_from_letter(prj), force): + project = self.api.prj_from_letter(prj) + + if self.perform(project, force): self.reset_rebuild_data(prj) else: return if cleanup: if self.api.item_exists(self.api.prj_from_letter(prj)): self.cleanup(self.api.prj_from_letter(prj)) - self.accept_other_new() + + for req in other_new: + oldspecs = self.api.get_filelist_for_package(pkgname=req['packages'][0], project=self.api.project, extension='spec') + print('Accepting request %d: %s' % (req['id'], ','.join(req['packages']))) + change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.project) + # Check if all .spec files of the package we just accepted has a package container to build + self.create_new_links(self.api.project, req['packages'][0], oldspecs) + if self.api.project.startswith('openSUSE:'): self.update_factory_version() if self.api.item_exists(self.api.crebuild): @@ -85,17 +109,9 @@ def perform(self, project, force=False): """ - status = self.api.check_project_status(project) - - if not status: - print('The project "{}" is not yet acceptable.'.format(project)) - if not force: - return False - status = self.api.project_status(project) packages = [] - rf = RequestFinder(self.api) oldspecs = {} for req in status.findall('staged_requests/request'): packages.append(req.get('package')) @@ -135,49 +151,8 @@ def cleanup(self, project): print("[cleanup] deleted %s/%s" % (project, package)) delete_package(self.api.apiurl, project, package, force=True, msg="autocleanup") - # wipe Test-DVD binaries and breaks kiwi build - if project.startswith('openSUSE:'): - for package in pkglist: - if package.startswith('Test-DVD-'): - # intend to break the kiwi file - arch = package.split('-')[-1] - fakepkgname = 'I-am-breaks-kiwi-build' - oldkiwifile = source_file_load(self.api.apiurl, project, package, 'PRODUCT-'+arch+'.kiwi') - if oldkiwifile is not None: - newkiwifile = re.sub(r'', '' % fakepkgname, oldkiwifile) - source_file_save(self.api.apiurl, project, package, 'PRODUCT-' + arch + '.kiwi', newkiwifile) - - # do wipe binary now - query = { 'cmd': 'wipe' } - query['package'] = package - query['repository'] = 'images' - - url = self.api.makeurl(['build', project], query) - try: - http_POST(url) - except HTTPError as err: - # failed to wipe isos but we can just continue - pass - return True - def accept_other_new(self): - changed = False - - rqlist = self.find_new_requests(self.api.project) - for req in rqlist: - oldspecs = self.api.get_filelist_for_package(pkgname=req['packages'][0], project=self.api.project, extension='spec') - print('Accepting request %d: %s' % (req['id'], ','.join(req['packages']))) - if req['type'] == 'delete': - # Remove devel project/package tag before accepting the request - self.remove_obsoleted_develtag(self.api.project, req['packages'][0]) - change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.project) - # Check if all .spec files of the package we just accepted has a package container to build - self.create_new_links(self.api.project, req['packages'][0], oldspecs) - changed = True - - return changed - def remove_obsoleted_develtag(self, project, package): xpath = { 'package': "@project='%s' and devel/@project=@project and devel/@package='%s'" % (project, package), diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index 64ff28e1f..a8893783a 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -847,16 +847,6 @@ def project_status(self, staging, status=True, requests=True, reload=False): url = self.makeurl(paths, opts) return ET.parse(self.retried_GET(url)).getroot() - def check_project_status(self, project): - """ - Checks a staging project for acceptance. - :param project: project to check - :return true (ok)/false (empty prj) or list of strings with - informations) - - """ - return self.project_status(project, requests=False).get('state') == 'acceptable' - def project_status_build_percent(self, status): final, tobuild = self.project_status_build_sum(status) return final / float(final + tobuild) * 100 From d4d9382a9e9069f92186770385f24715998bb87c Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Sun, 24 Nov 2019 13:18:27 +0100 Subject: [PATCH 4/7] remove links before accepting staging projects --- osclib/accept_command.py | 21 +++++++++++---------- osclib/stagingapi.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/osclib/accept_command.py b/osclib/accept_command.py index 908d88c72..578919233 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -8,7 +8,7 @@ from osc.core import change_request_state, show_package_meta, wipebinaries from osc.core import http_GET, http_PUT, http_DELETE, http_POST -from osc.core import delete_package, search, set_devel_project +from osc.core import delete_package, search from osc.core import Request from osc.util.helper import decode_it from osclib.core import attribute_value_save @@ -57,6 +57,14 @@ def reset_rebuild_data(self, project): if content != data: self.api.pseudometa_file_save('support_pkg_rebuild', content, 'accept command update') + def delete_linked(self): + for package in self.requests['delete']: + for link in self.api.linked_packages(package): + if link['project'] in self.api.rings or link['project'] == self.api.project: + print(f"delete {link['project']}/{link['package']}") + delete_package(self.api.apiurl, link['project'], link['package'], + msg="remove link while accepting delete of {}".format(package)) + def accept_all(self, projects, force=False, cleanup=True): self.requests = { 'delete': [], 'submit': [] } @@ -75,7 +83,8 @@ def accept_all(self, projects, force=False, cleanup=True): for req in other_new: self.requests[req['type']].append(req['package']) - print(self.requests) + print('requests', self.requests) + self.delete_linked() return True for prj in projects: project = self.api.prj_from_letter(prj) @@ -153,14 +162,6 @@ def cleanup(self, project): return True - def remove_obsoleted_develtag(self, project, package): - xpath = { - 'package': "@project='%s' and devel/@project=@project and devel/@package='%s'" % (project, package), - } - collection = search(self.api.apiurl, **xpath)['package'] - for pkg in collection.findall('package'): - set_devel_project(self.api.apiurl, project, pkg.attrib['name'], devprj=None) - def create_new_links(self, project, pkgname, oldspeclist): filelist = self.api.get_filelist_for_package(pkgname=pkgname, project=project, extension='spec') removedspecs = set(oldspeclist) - set(filelist) diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index a8893783a..eeca7194e 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -956,6 +956,18 @@ def get_sub_packages(self, package, project): return ret + def linked_packages(self, package, project=None): + if not project: + project=self.project + + url = self.makeurl(['source', project, package], { 'cmd': 'showlinked' }) + f = http_POST(url) + root = ET.parse(f).getroot() + result = [] + for package in root.findall('package'): + result.append({'project': package.get('project'), 'package': package.get('name')}) + return result + def create_and_wipe_package(self, project, package): """ Helper function for delete requests From 49bfafb1837af220d897b1707be447c31dc90b4d Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Sun, 24 Nov 2019 16:21:32 +0100 Subject: [PATCH 5/7] Accept all stagings at the same time --- osclib/accept_command.py | 84 +++++++++++++++------------------------- 1 file changed, 32 insertions(+), 52 deletions(-) diff --git a/osclib/accept_command.py b/osclib/accept_command.py index 578919233..d06ed67ca 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -67,6 +67,7 @@ def delete_linked(self): def accept_all(self, projects, force=False, cleanup=True): self.requests = { 'delete': [], 'submit': [] } + staging_packages = {} for prj in projects: project = self.api.prj_from_letter(prj) @@ -76,78 +77,57 @@ def accept_all(self, projects, force=False, cleanup=True): print('The project "{}" is not yet acceptable.'.format(project)) #return False + staging_packages[project] = [] for request in status.findall('staged_requests/request'): self.requests[request.get('type')].append(request.get('package')) + staging_packages[project].append(request.get('package')) other_new = self.find_new_requests(self.api.project) for req in other_new: self.requests[req['type']].append(req['package']) - print('requests', self.requests) + print('delete links to packages pending deletion...') self.delete_linked() - return True + + # requests = {'delete': ['AppStream', 'yast2-x11'], 'submit': ['bash']} + opts = {} + if force: + opts['force'] = '1' + + print('triggering staging accepts...') for prj in projects: project = self.api.prj_from_letter(prj) - if self.perform(project, force): - self.reset_rebuild_data(prj) - else: - return - if cleanup: - if self.api.item_exists(self.api.prj_from_letter(prj)): - self.cleanup(self.api.prj_from_letter(prj)) + u = self.api.makeurl(['staging', self.api.project, 'staging_projects', project, 'accept'], opts) + f = http_POST(u) for req in other_new: - oldspecs = self.api.get_filelist_for_package(pkgname=req['packages'][0], project=self.api.project, extension='spec') - print('Accepting request %d: %s' % (req['id'], ','.join(req['packages']))) + print(f"Accepting request {req['id']}: {req['package']}") change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.project) - # Check if all .spec files of the package we just accepted has a package container to build - self.create_new_links(self.api.project, req['packages'][0], oldspecs) - - if self.api.project.startswith('openSUSE:'): - self.update_factory_version() - if self.api.item_exists(self.api.crebuild): - self.sync_buildfailures() - - def perform(self, project, force=False): - """Accept the staging project for review and submit to Factory / - Leap ... - - Then disable the build to disabled - :param project: staging project we are working with - - """ - status = self.api.project_status(project) - packages = [] - - oldspecs = {} - for req in status.findall('staged_requests/request'): - packages.append(req.get('package')) - - print('Checking file list of {}'.format(req.get('package'))) - os = self.api.get_filelist_for_package(pkgname=req.get('package'), - project=self.api.project, - extension='spec') - oldspecs[req.get('package')] = os - #self.create_new_links(self.api.project, req['package'], oldspecs) - - print(oldspecs) + for prj in projects: + project = self.api.prj_from_letter(prj) + print(f'waiting for staging project {project} to be accepted') - opts = {} - if force: - opts['force'] = '1' + while True: + status = self.api.project_status(project, reload=True) + if status.get('state') == 'empty': + break + print('{} requests still staged - waiting'.format(status.find('staged_requests').get('count'))) + time.sleep(1) - u = self.api.makeurl(['staging', self.api.project, 'staging_projects', project, 'accept'], opts) - f = http_POST(u) + self.api.accept_status_comment(project, staging_packages[project]) + self.api.staging_deactivate(project) - while self.api.project_status(project, reload=True, requests=False).get('state') != 'empty': - time.sleep(1) + self.reset_rebuild_data(prj) - self.api.accept_status_comment(project, packages) - self.api.staging_deactivate(project) + if cleanup: + self.cleanup(project) - return True + if self.api.project.startswith('openSUSE:'): + self.update_factory_version() + if self.api.item_exists(self.api.crebuild): + self.sync_buildfailures() def cleanup(self, project): if not self.api.item_exists(project): From 66c48005209afbf0e687cd43acd2476b036c8f3b Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Sun, 24 Nov 2019 17:26:55 +0100 Subject: [PATCH 6/7] Fix local links --- osclib/accept_command.py | 80 +++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/osclib/accept_command.py b/osclib/accept_command.py index d06ed67ca..3447f2528 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -89,7 +89,6 @@ def accept_all(self, projects, force=False, cleanup=True): print('delete links to packages pending deletion...') self.delete_linked() - # requests = {'delete': ['AppStream', 'yast2-x11'], 'submit': ['bash']} opts = {} if force: opts['force'] = '1' @@ -124,6 +123,9 @@ def accept_all(self, projects, force=False, cleanup=True): if cleanup: self.cleanup(project) + for package in self.requests['submit']: + self.fix_linking_packages(package) + if self.api.project.startswith('openSUSE:'): self.update_factory_version() if self.api.item_exists(self.api.crebuild): @@ -142,15 +144,26 @@ def cleanup(self, project): return True - def create_new_links(self, project, pkgname, oldspeclist): - filelist = self.api.get_filelist_for_package(pkgname=pkgname, project=project, extension='spec') - removedspecs = set(oldspeclist) - set(filelist) - for spec in removedspecs: - # Deleting all the packages that no longer have a .spec file - url = self.api.makeurl(['source', project, spec[:-5]]) - print("Deleting package %s from project %s" % (spec[:-5], project)) + def fix_linking_packages(self, package): + project = self.api.project + file_list = self.api.get_filelist_for_package(package, project) + # ignore + if '_multibuild' in file_list or '_link' in file_list: + return + needed_links = set() + for file in file_list: + if file.endswith('.spec') and file != f'{package}.spec': + needed_links.add(file[:-5]) + local_links = set() + for link in self.api.linked_packages(package): + if link['project'] == project: + local_links.add(link['package']) + + # Deleting all the packages that no longer have a .spec file + for link in local_links - needed_links: + print(f"Deleting package {project}/{link}") try: - http_DELETE(url) + delete_package(self.api.apiurl, project, link, msg=f"No longer linking to {package}") except HTTPError as err: if err.code == 404: # the package link was not yet created, which was likely a mistake from earlier @@ -160,37 +173,28 @@ def create_new_links(self, project, pkgname, oldspeclist): raise # Remove package from Rings in case 2nd specfile was removed - if self.api.ring_packages.get(spec[:-5]): - delete_package(self.api.apiurl, self.api.ring_packages.get(spec[:-5]), spec[:-5], force=True, msg="Cleanup package in Rings") + if self.api.ring_packages.get(link): + delete_package(self.api.apiurl, self.api.ring_packages.get(link), link, force=True, msg="Cleanup package in Rings") - if len(filelist) > 1: + for link in needed_links - local_links: # There is more than one .spec file in the package; link package containers as needed - origmeta = source_file_load(self.api.apiurl, project, pkgname, '_meta') - for specfile in filelist: - package = specfile[:-5] # stripping .spec off the filename gives the packagename - if package == pkgname: - # This is the original package and does not need to be linked to itself - continue - # Check if the target package already exists, if it does not, we get a HTTP error 404 to catch - if not self.api.item_exists(project, package): - print("Creating new package %s linked to %s" % (package, pkgname)) - # new package does not exist. Let's link it with new metadata - newmeta = re.sub(r'(', - r''.format(pkgname), - newmeta) - newmeta = re.sub(r'.*', - r'', - newmeta) - newmeta = re.sub(r'', - r'{}'.format(pkgname), - newmeta) - source_file_save(self.api.apiurl, project, package, '_meta', newmeta) - link = "".format(pkgname) - source_file_save(self.api.apiurl, project, package, '_link', link) - return True + meta = ET.fromstring(source_file_load(self.api.apiurl, project, package, '_meta')) + print(f"Creating new link {link}->{package}") + + meta.attrib['name'] = link + bcnt = meta.find('bcntsynctag') + if bcnt is None: + bcnt = ET.SubElement(meta, 'bcntsynctag') + bcnt.text = package + devel = meta.find('devel') + if devel is None: + devel = ET.SubElement(meta, 'devel') + devel.attrib['project'] = project + devel.attrib['package'] = package + + source_file_save(self.api.apiurl, project, link, '_meta', ET.tostring(meta)) + xml = f"" + source_file_save(self.api.apiurl, project, link, '_link', xml) def update_version_attribute(self, project, version): version_attr = attribute_value_load(self.api.apiurl, project, 'ProductVersion') From 8576f228761ea9a89219c65bfb3a4ace083a3c85 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Sun, 24 Nov 2019 19:21:14 +0100 Subject: [PATCH 7/7] Fix test suite for accept --- osclib/accept_command.py | 6 ++++-- osclib/conf.py | 1 + tests/accept_tests.py | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/osclib/accept_command.py b/osclib/accept_command.py index 3447f2528..a06524857 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -131,9 +131,11 @@ def accept_all(self, projects, force=False, cleanup=True): if self.api.item_exists(self.api.crebuild): self.sync_buildfailures() + return True + def cleanup(self, project): if not self.api.item_exists(project): - return False + return pkglist = self.api.list_packages(project) clean_list = set(pkglist) - set(self.api.cnocleanup_packages) @@ -142,7 +144,7 @@ def cleanup(self, project): print("[cleanup] deleted %s/%s" % (project, package)) delete_package(self.api.apiurl, project, package, force=True, msg="autocleanup") - return True + return def fix_linking_packages(self, package): project = self.api.project diff --git a/osclib/conf.py b/osclib/conf.py index e45f88979..e815a9d0b 100644 --- a/osclib/conf.py +++ b/osclib/conf.py @@ -156,6 +156,7 @@ 'staging-required-checks-adi': '', 'installcheck-ignore-duplicated-binaries': '', 'onlyadi': '', + 'nocleanup-packages': '', 'rings': '', 'rebuild': '', 'product': '', diff --git a/tests/accept_tests.py b/tests/accept_tests.py index 7d7990802..1b89f439a 100644 --- a/tests/accept_tests.py +++ b/tests/accept_tests.py @@ -28,7 +28,7 @@ def setup_wf(self): def test_accept_comments(self): wf = self.setup_wf() - self.assertEqual(True, AcceptCommand(wf.api).perform(self.prj)) + self.assertEqual(True, AcceptCommand(wf.api).accept_all(['B'])) # Comments are cleared up accepted_comments = self.c_api.get_comments(project_name=self.prj) @@ -40,7 +40,7 @@ def test_accept_final_comment(self): # snipe out cleanup to see the comments before the final countdown wf.api.staging_deactivate = MagicMock(return_value=True) - self.assertEqual(True, AcceptCommand(wf.api).perform(self.prj)) + self.assertEqual(True, AcceptCommand(wf.api).accept_all(['B'])) comments = self.c_api.get_comments(project_name=self.prj) self.assertGreater(len(comments), len(self.comments))