Skip to content

Commit

Permalink
Merge 5b79718 into a2c4df2
Browse files Browse the repository at this point in the history
  • Loading branch information
@jberry-suse
jberry-suse committed Jan 25, 2017
2 parents a2c4df2 + 5b79718 commit 050b3d4
Show file tree
Hide file tree
Showing 7 changed files with 498 additions and 165 deletions.
172 changes: 159 additions & 13 deletions osc-staging.py
View file
Expand Up @@ -14,10 +14,14 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

from __future__ import print_function

import os
import os.path
import sys
import tempfile
import warnings
import yaml

from osc import cmdln
from osc import oscerr
Expand All @@ -40,6 +44,7 @@
from osclib.cache import Cache
from osclib.unselect_command import UnselectCommand
from osclib.repair_command import RepairCommand
from osclib.request_splitter import RequestSplitter

OSC_STAGING_VERSION = '0.0.1'

Expand Down Expand Up @@ -95,6 +100,9 @@ def _full_project_name(self, project):
@cmdln.option('--wipe-cache', dest='wipe_cache', action='store_true', default=False,
help='wipe GET request cache before executing')
@cmdln.option('-m', '--message', help='message used by ignore command')
@cmdln.option('--filter-by', action='append', help='xpath by which to filter requests')
@cmdln.option('--group-by', action='append', help='xpath by which to group requests')
@cmdln.option('-i', '--interactive', action='store_true', help='interactively modify selection proposal')
def do_staging(self, subcmd, opts, *args):
"""${cmd_name}: Commands to work with staging projects
Expand Down Expand Up @@ -125,6 +133,53 @@ def do_staging(self, subcmd, opts, *args):
"list" will pick the requests not in rings
"select" will add requests to the project
Stagings are expected to be either in short-hand or the full project
name. For example letter or named stagings can be specified simply as
A, B, Gcc6, etc, while adi stagings can be specified as adi:1, adi:2,
etc. Currently, adi stagings are not supported in proposal mode.
Requests may either be the target package or the request ID.
When using --filter-by or --group-by the xpath will be applied to the
request node as returned by OBS. Several values will supplement the
normal request node.
- ./action/source/@devel_project: the devel project for the package
- ./action/target/@ring: the ring to which the package belongs
- ./@ignored: either false or the provided message
Some useful examples:
--filter-by './action/target[starts-with(@package, "yast-")]'
--filter-by './action/source/[@devel_project="YaST:Head"]'
--filter-by './action/target[starts-with(@ring, "1")]'
--filter-by '@id!="1234567"'
--group-by='./action/source/@devel_project'
--group-by='./action/target/@ring'
Multiple filter-by or group-by options may be used at the same time.
Note that when using proposal mode, multiple stagings to consider may be
provided in addition to a list of requests by which to filter. A more
complex example:
select --group-by='./action/source/@devel_project' A B C 123 456 789
This will separate the requests 123, 456, 789 by devel project and only
consider stagings A, B, or C, if available, for placement.
No arguments is also a valid choice and will propose all non-ignored
requests into the first available staging. Note that bootstrapped
stagings are only used when either required or no other stagings are
available.
Another useful example is placing all open requests into a specific
letter staging with:
select A
Interactive mode allows the proposal to be modified before application.
"unselect" will remove from the project - pushing them back to the backlog
Expand All @@ -137,7 +192,8 @@ def do_staging(self, subcmd, opts, *args):
osc staging ignore [-m MESSAGE] REQUEST...
osc staging unignore REQUEST...|all
osc staging list [--supersede]
osc staging select [--no-freeze] [--move [--from PROJECT]] LETTER REQUEST...
osc staging select [--no-freeze] [--move [--from PROJECT] STAGING REQUEST...
osc staging select [--no-freeze] [[--interactive] [--filter-by...] [--group-by...]] [STAGING...] [REQUEST...]
osc staging unselect REQUEST...
osc staging repair REQUEST...
"""
Expand All @@ -153,9 +209,7 @@ def do_staging(self, subcmd, opts, *args):
elif cmd == 'check':
min_args, max_args = 0, 2
elif cmd == 'select':
min_args, max_args = 1, None
if not opts.add:
min_args = 2
min_args, max_args = 0, None
elif cmd == 'unselect':
min_args, max_args = 1, None
elif cmd == 'adi':
Expand Down Expand Up @@ -194,16 +248,16 @@ def do_staging(self, subcmd, opts, *args):
FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap = opts.bootstrap )
elif cmd == 'frozenage':
for prj in args[1:]:
print "%s last frozen %0.1f days ago" % (api.prj_from_letter(prj), api.days_since_last_freeze(api.prj_from_letter(prj)))
print("%s last frozen %0.1f days ago" % (api.prj_from_letter(prj), api.days_since_last_freeze(api.prj_from_letter(prj))))
elif cmd == 'acheck':
# Is it safe to accept? Meaning: /totest contains what it should and is not dirty
version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64")
if version_totest:
version_openqa = api.load_file_content("%s:Staging" % api.project, "dashboard", "version_totest")
totest_dirty = api.is_repo_dirty(api.project, 'totest')
print "version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % (version_openqa, version_totest, totest_dirty)
print("version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % (version_openqa, version_totest, totest_dirty))
else:
print "acheck is unavailable in %s!\n" % (api.project)
print("acheck is unavailable in %s!\n" % (api.project))
elif cmd == 'accept':
# Is it safe to accept? Meaning: /totest contains what it should and is not dirty
version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64")
Expand Down Expand Up @@ -232,16 +286,108 @@ def do_staging(self, subcmd, opts, *args):
if api.item_exists(api.crebuild):
cmd.sync_buildfailures()
else:
print "Not safe to accept: /totest is not yet synced"
print("Not safe to accept: /totest is not yet synced")
elif cmd == 'unselect':
UnselectCommand(api).perform(args[1:])
elif cmd == 'select':
tprj = api.prj_from_letter(args[1])
if opts.add:
api.mark_additional_packages(tprj, [opts.add])
# Include list of all stagings in short-hand and by full name.
existing_stagings = api.get_staging_projects_short(None)
existing_stagings += [p for p in api.get_staging_projects() if not p.endswith(':DVD')]
stagings = []
requests = []
for arg in args[1:]:
# Since requests may be given by either request ID or package
# name and stagings may include multi-letter special stagings
# there is no easy way to distinguish between stagings and
# requests in arguments. Therefore, check if argument is in the
# list of short-hand and full project name stagings, otherwise
# consider it a request. This also allows for special stagings
# with the same name as package, but the staging will be assumed
# first time around. The current practice seems to be to start a
# special staging with a capital letter which makes them unique.
# lastly adi stagings are consistently prefix with adi: which
# also makes it consistent to distinguish them from request IDs.
if arg in existing_stagings and arg not in stagings:
stagings.append(api.extract_staging_short(arg))
elif arg not in requests:
requests.append(arg)

if len(stagings) != 1 or len(requests) == 0 or opts.filter_by or opts.group_by:
if opts.move or opts.from_:
print('--move and --from must be used with explicit staging and request list')
return

splitter = RequestSplitter(api, api.get_open_requests(), in_ring=True)
if len(requests) > 0:
splitter.filter_add_requests(requests)
if len(splitter.filters) == 0:
splitter.filter_add('./action[not(@type="add_role" or @type="change_devel")]')
splitter.filter_add('@ignored="false"')
if opts.filter_by:
for filter_by in opts.filter_by:
splitter.filter_add(filter_by)
if opts.group_by:
for group_by in opts.group_by:
splitter.group_by(group_by)
splitter.split()

result = splitter.propose_assignment(stagings)
if result is not True:
print('Failed to generate proposal: {}'.format(result))
return
proposal = splitter.proposal
if len(proposal) == 0:
print('Empty proposal')
return

if opts.interactive:
with tempfile.NamedTemporaryFile(suffix='.yml') as temp:
temp.write(yaml.safe_dump(splitter.proposal, default_flow_style=False) + '\n\n')
temp.write('# move requests between stagings or comment/remove them\n')
temp.write('# change the target staging for a group\n')
temp.write('# stagings\n')
temp.write('# - considered: {}\n'
.format(', '.join(sorted(splitter.stagings_considerable.keys()))))
temp.write('# - available: {}\n'
.format(', '.join(sorted(splitter.stagings_available.keys()))))
temp.flush()

editor = os.getenv('EDITOR')
if not editor:
editor = 'xdg-open'
return_code = subprocess.call([editor, temp.name])

proposal = yaml.safe_load(open(temp.name).read())

print(yaml.safe_dump(proposal, default_flow_style=False))

print('Accept proposal? [y/n] (y): ', end='')
response = raw_input().lower()
if response != '' and response != 'y':
print('Quit')
return

for group in sorted(proposal.keys()):
g = proposal[group]
if not g['requests']:
# Skipping since all request removed, presumably in interactive.
continue

print('Staging {}'.format(g['staging']))

# SelectCommand expects strings.
request_ids = map(str, g['requests'].keys())
target_project = api.prj_from_short(g['staging'])

SelectCommand(api, target_project) \
.perform(request_ids, opts.move, opts.from_, opts.no_freeze)
else:
SelectCommand(api, tprj).perform(args[2:], opts.move,
opts.from_, opts.no_freeze)
target_project = api.prj_from_short(stagings[0])
if opts.add:
api.mark_additional_packages(target_project, [opts.add])
else:
SelectCommand(api, target_project) \
.perform(requests, opts.move, opts.from_, opts.no_freeze)
elif cmd == 'cleanup_rings':
CleanupRings(api).perform()
elif cmd == 'ignore':
Expand Down
104 changes: 34 additions & 70 deletions osclib/adi_command.py
View file
Expand Up @@ -6,6 +6,7 @@

from osclib.select_command import SelectCommand
from osclib.request_finder import RequestFinder
from osclib.request_splitter import RequestSplitter
from xml.etree import cElementTree as ET

class AdiCommand:
Expand Down Expand Up @@ -47,87 +48,50 @@ def check_adi_projects(self):
self.check_adi_project(p)

def create_new_adi(self, wanted_requests, by_dp=False, split=False):
all_requests = self.api.get_open_requests()

non_ring_packages = []
non_ring_requests = dict()
non_ring_requests_ignored = []

for request in all_requests:
# Consolidate all data from request
request_id = int(request.get('id'))
if len(wanted_requests) and request_id not in wanted_requests:
continue
action = request.findall('action')
if not action:
msg = 'Request {} has no action'.format(request_id)
raise oscerr.WrongArgs(msg)
# we care only about first action
action = action[0]

# Where are we targeting the package
if len(wanted_requests):
source_project = 'wanted'
requests = self.api.get_open_requests()
splitter = RequestSplitter(self.api, requests, in_ring=False)
splitter.filter_add('./action[@type="submit"]')
if len(wanted_requests):
splitter.filter_add_requests(wanted_requests)
# wanted_requests forces all requests into a single group.
else:
if split:
splitter.group_by('./@id')
elif by_dp:
splitter.group_by('./action/source/@devel_project')
else:
source = action.find('source')
if source is not None:
source_project = source.get('project')
else:
source_project = 'none'
splitter.group_by('./action/source/@project')
splitter.split()

# do not process the rest request type than submit
if action.get('type') != 'submit':
continue
for group in sorted(splitter.grouped.keys()):
print(group if group != '' else 'wanted')

target_package = action.find('target').get('package')
source_package = action.find('source').get('package')
name = None
for request in splitter.grouped[group]['requests']:
request_id = int(request.get('id'))
target_package = request.find('./action/target').get('package')
line = '- sr#{}: {:<30}'.format(request_id, target_package)

if not self.api.ring_packages.get(target_package):
# Auto-superseding request in adi command
if self.api.update_superseded_request(request):
if request_id in self.requests_ignored:
print(line + '\n ignored: ' + self.requests_ignored[request_id])
continue

if not len(wanted_requests) and request_id in self.requests_ignored:
non_ring_requests_ignored.append(request_id)
# Auto-superseding request in adi command
if self.api.update_superseded_request(request):
print(line + ' (superseded)')
continue

non_ring_packages.append(target_package)
if split:
# request_id pretended to be index of non_ring_requests
non_ring_requests[request_id] = [request_id]
else:
if by_dp:
devel_project = self.api.get_devel_project(source_project, source_package)
# try target pacakge in Factory
# this is a bit against Leap development in case submissions is from Update,
# or any other project than Factory
if devel_project is None and self.api.project.startswith('openSUSE:'):
devel_project = self.api.get_devel_project('openSUSE:Factory', target_package)
if devel_project is not None:
source_project = devel_project

if source_project not in non_ring_requests:
non_ring_requests[source_project] = []
non_ring_requests[source_project].append(request_id)

if len(non_ring_requests_ignored):
print "Not in a ring, but ignored:"
for request_id in non_ring_requests_ignored:
print "- sr#{}: {}".format(request_id, requests_ignored[request_id])
if len(non_ring_packages):
print "Not in a ring:", ' '.join(sorted(non_ring_packages))
else:
return

for source_project, requests in non_ring_requests.items():
name = self.api.create_adi_project(None)
# Only create staging projec the first time a non superseded
# request is processed from a particular group.
if name is None:
name = self.api.create_adi_project(None)

for request in requests:
if not self.api.rq_to_prj(request, name):
if not self.api.rq_to_prj(request_id, name):
return False

# Notify everybody about the changes
self.api.update_status_comments(name, 'select')
# Notify everybody about the changes.
self.api.update_status_comments(name, 'select')
print(line + ' (staged in {})'.format(name))

def perform(self, packages, move=False, by_dp=False, split=False):
"""
Expand Down

0 comments on commit 050b3d4

Please sign in to comment.