Skip to content

Commit

Permalink
cmd/flux-jobs: Add special case annotation fields
Browse files Browse the repository at this point in the history
As a convenience to users, support the "sched" field as the same
as "annotations.sched" and the "user" field as the same as
"annotations.user".

Update tests and documentation accordingly.
  • Loading branch information
chu11 committed Jul 25, 2020
1 parent 0c6cc7b commit 1920a12
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 13 deletions.
20 changes: 15 additions & 5 deletions doc/man1/flux-jobs.rst
Expand Up @@ -111,11 +111,16 @@ the following conversion flags are supported by *flux-jobs*:
**!H**
convert a duration to hours:minutes:seconds form (e.g. *{runtime!H}*)

Scheduler and user annotations can be retrieved via the *annotations*
field name. Specific keys and sub-object keys can be retrieved
separated by a period ("."). For example, if the scheduler has
annotated the job with a reason pending status, it can be retrieved
via "{annotations.sched.reason_pending}".
Annotations can be retrieved via the *annotations* field name.
Specific keys and sub-object keys can be retrieved separated by a
period ("."). For example, if the scheduler has annotated the job
with a reason pending status, it can be retrieved via
"{annotations.sched.reason_pending}".

As a convenience, the field names *sched* and *user* can be used as
substitutions for *annotations.sched* and *annotations.user*. For
example, a reason pending status can be retrieved via
"{sched.reason_pending}".

The field names that can be specified are:

Expand Down Expand Up @@ -206,6 +211,11 @@ The field names that can be specified are:
**annotations**
annotations metadata, use "." to get specific keys

**sched**
short hand for *annotations.sched*

**user**
short hand for *annotations.user*

RESOURCES
=========
Expand Down
30 changes: 22 additions & 8 deletions src/cmd/flux-jobs.py
Expand Up @@ -182,6 +182,8 @@ def __init__(self, info_resp):

aDict = combined_dict.get("annotations", {})
combined_dict["annotations"] = AnnotationsInfo(aDict)
combined_dict["sched"] = combined_dict["annotations"].sched
combined_dict["user"] = combined_dict["annotations"].user

# Set all keys as self._{key} to be found by getattr and
# memoized_property decorator:
Expand Down Expand Up @@ -377,13 +379,20 @@ def fetch_jobs_flux(args, fields):
"expiration": ("expiration", "state", "result"),
"t_remaining": ("expiration", "state", "result"),
"annotations": ("annotations",),
# Special cases, pointers to sub-dicts in annotations
"sched": ("annotations",),
"user": ("annotations",),
}

attrs = set()
for field in fields:
# Special case for annotations, can be arbitrary field names determined
# by scheduler/user.
if field.startswith("annotations."):
if (
field.startswith("annotations.")
or field.startswith("sched.")
or field.startswith("user.")
):
attrs.update(fields2attrs["annotations"])
else:
attrs.update(fields2attrs[field])
Expand Down Expand Up @@ -636,6 +645,11 @@ def get_field(self, field_name, args, kwargs):
"annotations.sched.t_estimate": "T_ESTIMATE",
"annotations.sched.reason_pending": "REASON",
"annotations.sched.resource_summary": "RESOURCES",
"sched": "SCHED",
"sched.t_estimate": "T_ESTIMATE",
"sched.reason_pending": "REASON",
"sched.resource_summary": "RESOURCES",
"user": "USER",
}

def __init__(self, fmt):
Expand All @@ -652,13 +666,13 @@ def __init__(self, fmt):
"""
format_list = string.Formatter().parse(fmt)
for (_, field, _, _) in format_list:
if (
field
and not field in self.headings
and field.startswith("annotations.")
):
field_heading = field[len("annotations.") :].upper()
self.headings[field] = field_heading
if field and not field in self.headings:
if field.startswith("annotations."):
field_heading = field[len("annotations.") :].upper()
self.headings[field] = field_heading
elif field.startswith("sched.") or field.startswith("user."):
field_heading = field.upper()
self.headings[field] = field_heading
super().__init__(self.headings, fmt, prepend="0.")

def format(self, obj):
Expand Down
26 changes: 26 additions & 0 deletions t/t2800-jobs-cmd.t
Expand Up @@ -648,6 +648,25 @@ test_expect_success 'flux-jobs --format={expiration!D:h},{t_remaining!H:h} works
#
# as the schedulers in those tests do varied but testable annotations

test_expect_success 'flux-jobs annotation "sched" short hands work' '
fmt="{annotations.sched},{annotations.sched.resource_summary}" &&
flux jobs -no "${fmt}" > sched_long_hand.out &&
fmt="{sched},{sched.resource_summary}" &&
flux jobs -no "${fmt}" > sched_short_hand.out &&
test_cmp sched_long_hand.out sched_short_hand.out
'

test_expect_success 'flux-jobs annotation "user" short hands work' '
for id in $(state_ids sched); do
flux job annotate $id foo 42
done &&
fmt="{annotations.user},{annotations.user.foo}" &&
flux jobs -no "${fmt}" > user_long_hand.out &&
fmt="{user},{user.foo}" &&
flux jobs -no "${fmt}" > user_short_hand.out &&
test_cmp user_long_hand.out user_short_hand.out
'

test_expect_success 'flux-jobs emits empty string on invalid annotations fields' '
fmt="{annotations.foo},{annotations.foo:h}" &&
fmt="${fmt},{annotations.sched.bar},{annotations.sched.bar:h}" &&
Expand Down Expand Up @@ -714,6 +733,13 @@ test_expect_success 'flux-jobs: header included with all custom formats' '
annotations.sched.reason_pending==REASON
annotations.sched.resource_summary==RESOURCES
annotations.sched.foobar==SCHED.FOOBAR
sched==SCHED
sched.t_estimate==T_ESTIMATE
sched.reason_pending==REASON
sched.resource_summary==RESOURCES
sched.foobar==SCHED.FOOBAR
user==USER
user.foobar==USER.FOOBAR
EOF
sed "s/\(.*\)==.*/\1=={\1}/" headers.expected > headers.fmt &&
flux jobs --from-stdin --format="$(cat headers.fmt)" \
Expand Down

0 comments on commit 1920a12

Please sign in to comment.