From c67f50831b39b1dc6fd01afd521553064a8e341f Mon Sep 17 00:00:00 2001 From: John Westcott IV <32551173+john-westcott-iv@users.noreply.github.com> Date: Thu, 28 Apr 2022 15:38:20 -0400 Subject: [PATCH] Modifying schedules API to allow for rrulesets #5733 (#12043) * Added schedule_rruleset lookup plugin for awx.awx * Added DB migration for rrule size * Updated schedule docs * The schedule API endpoint will now return an array of errors on rule validation to try and inform the user of all errors instead of just the first --- awx/api/serializers.py | 81 ++-- .../migrations/0160_alter_schedule_rrule.py | 18 + awx/main/models/schedules.py | 141 ++++--- .../tests/functional/api/test_schedules.py | 131 ++++++- .../tests/functional/models/test_schedule.py | 177 ++++++++- .../Schedule/shared/ScheduleForm.js | 30 ++ .../plugins/lookup/schedule_rruleset.py | 349 ++++++++++++++++++ awx_collection/plugins/modules/schedule.py | 18 +- .../targets/lookup_rruleset/tasks/main.yml | 342 +++++++++++++++++ docs/schedules.md | 48 ++- 10 files changed, 1212 insertions(+), 123 deletions(-) create mode 100644 awx/main/migrations/0160_alter_schedule_rrule.py create mode 100644 awx_collection/plugins/lookup/schedule_rruleset.py create mode 100644 awx_collection/tests/integration/targets/lookup_rruleset/tasks/main.yml diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 2424b461b9fe..af76f96f206f 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -4645,61 +4645,60 @@ class Meta: # We reject rrules if: # - DTSTART is not include - # - INTERVAL is not included - # - SECONDLY is used - # - TZID is used - # - BYDAY prefixed with a number (MO is good but not 20MO) - # - BYYEARDAY - # - BYWEEKNO - # - Multiple DTSTART or RRULE elements - # - Can't contain both COUNT and UNTIL - # - COUNT > 999 + # - Multiple DTSTART + # - At least one of RRULE is not included + # - EXDATE or RDATE is included + # For any rule in the ruleset: + # - INTERVAL is not included + # - SECONDLY is used + # - BYDAY prefixed with a number (MO is good but not 20MO) + # - Can't contain both COUNT and UNTIL + # - COUNT > 999 def validate_rrule(self, value): rrule_value = value - multi_by_month_day = r".*?BYMONTHDAY[\:\=][0-9]+,-*[0-9]+" - multi_by_month = r".*?BYMONTH[\:\=][0-9]+,[0-9]+" by_day_with_numeric_prefix = r".*?BYDAY[\:\=][0-9]+[a-zA-Z]{2}" - match_count = re.match(r".*?(COUNT\=[0-9]+)", rrule_value) match_multiple_dtstart = re.findall(r".*?(DTSTART(;[^:]+)?\:[0-9]+T[0-9]+Z?)", rrule_value) match_native_dtstart = re.findall(r".*?(DTSTART:[0-9]+T[0-9]+) ", rrule_value) - match_multiple_rrule = re.findall(r".*?(RRULE\:)", rrule_value) + match_multiple_rrule = re.findall(r".*?(RULE\:[^\s]*)", rrule_value) + errors = [] if not len(match_multiple_dtstart): - raise serializers.ValidationError(_('Valid DTSTART required in rrule. Value should start with: DTSTART:YYYYMMDDTHHMMSSZ')) + errors.append(_('Valid DTSTART required in rrule. Value should start with: DTSTART:YYYYMMDDTHHMMSSZ')) if len(match_native_dtstart): - raise serializers.ValidationError(_('DTSTART cannot be a naive datetime. Specify ;TZINFO= or YYYYMMDDTHHMMSSZZ.')) + errors.append(_('DTSTART cannot be a naive datetime. Specify ;TZINFO= or YYYYMMDDTHHMMSSZZ.')) if len(match_multiple_dtstart) > 1: - raise serializers.ValidationError(_('Multiple DTSTART is not supported.')) - if not len(match_multiple_rrule): - raise serializers.ValidationError(_('RRULE required in rrule.')) - if len(match_multiple_rrule) > 1: - raise serializers.ValidationError(_('Multiple RRULE is not supported.')) - if 'interval' not in rrule_value.lower(): - raise serializers.ValidationError(_('INTERVAL required in rrule.')) - if 'secondly' in rrule_value.lower(): - raise serializers.ValidationError(_('SECONDLY is not supported.')) - if re.match(multi_by_month_day, rrule_value): - raise serializers.ValidationError(_('Multiple BYMONTHDAYs not supported.')) - if re.match(multi_by_month, rrule_value): - raise serializers.ValidationError(_('Multiple BYMONTHs not supported.')) - if re.match(by_day_with_numeric_prefix, rrule_value): - raise serializers.ValidationError(_("BYDAY with numeric prefix not supported.")) - if 'byyearday' in rrule_value.lower(): - raise serializers.ValidationError(_("BYYEARDAY not supported.")) - if 'byweekno' in rrule_value.lower(): - raise serializers.ValidationError(_("BYWEEKNO not supported.")) - if 'COUNT' in rrule_value and 'UNTIL' in rrule_value: - raise serializers.ValidationError(_("RRULE may not contain both COUNT and UNTIL")) - if match_count: - count_val = match_count.groups()[0].strip().split("=") - if int(count_val[1]) > 999: - raise serializers.ValidationError(_("COUNT > 999 is unsupported.")) + errors.append(_('Multiple DTSTART is not supported.')) + if "rrule:" not in rrule_value.lower(): + errors.append(_('One or more rule required in rrule.')) + if "exdate:" in rrule_value.lower(): + raise serializers.ValidationError(_('EXDATE not allowed in rrule.')) + if "rdate:" in rrule_value.lower(): + raise serializers.ValidationError(_('RDATE not allowed in rrule.')) + for a_rule in match_multiple_rrule: + if 'interval' not in a_rule.lower(): + errors.append("{0}: {1}".format(_('INTERVAL required in rrule'), a_rule)) + elif 'secondly' in a_rule.lower(): + errors.append("{0}: {1}".format(_('SECONDLY is not supported'), a_rule)) + if re.match(by_day_with_numeric_prefix, a_rule): + errors.append("{0}: {1}".format(_("BYDAY with numeric prefix not supported"), a_rule)) + if 'COUNT' in a_rule and 'UNTIL' in a_rule: + errors.append("{0}: {1}".format(_("RRULE may not contain both COUNT and UNTIL"), a_rule)) + match_count = re.match(r".*?(COUNT\=[0-9]+)", a_rule) + if match_count: + count_val = match_count.groups()[0].strip().split("=") + if int(count_val[1]) > 999: + errors.append("{0}: {1}".format(_("COUNT > 999 is unsupported"), a_rule)) + try: Schedule.rrulestr(rrule_value) except Exception as e: import traceback logger.error(traceback.format_exc()) - raise serializers.ValidationError(_("rrule parsing failed validation: {}").format(e)) + errors.append(_("rrule parsing failed validation: {}").format(e)) + + if errors: + raise serializers.ValidationError(errors) + return value diff --git a/awx/main/migrations/0160_alter_schedule_rrule.py b/awx/main/migrations/0160_alter_schedule_rrule.py new file mode 100644 index 000000000000..5443b6bd26aa --- /dev/null +++ b/awx/main/migrations/0160_alter_schedule_rrule.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-04-18 21:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0159_deprecate_inventory_source_UoPU_field'), + ] + + operations = [ + migrations.AlterField( + model_name='schedule', + name='rrule', + field=models.TextField(help_text='A value representing the schedules iCal recurrence rule.'), + ), + ] diff --git a/awx/main/models/schedules.py b/awx/main/models/schedules.py index c3fae526f1ce..8f9caec1315e 100644 --- a/awx/main/models/schedules.py +++ b/awx/main/models/schedules.py @@ -81,7 +81,7 @@ class Meta: dtend = models.DateTimeField( null=True, default=None, editable=False, help_text=_("The last occurrence of the schedule occurs before this time, aftewards the schedule expires.") ) - rrule = models.CharField(max_length=255, help_text=_("A value representing the schedules iCal recurrence rule.")) + rrule = models.TextField(help_text=_("A value representing the schedules iCal recurrence rule.")) next_run = models.DateTimeField(null=True, default=None, editable=False, help_text=_("The next time that the scheduled action will run.")) @classmethod @@ -91,22 +91,22 @@ def get_zoneinfo(self): @property def timezone(self): utc = tzutc() + # All rules in a ruleset will have the same dtstart so we can just take the first rule + tzinfo = Schedule.rrulestr(self.rrule)._rrule[0]._dtstart.tzinfo + if tzinfo is utc: + return 'UTC' all_zones = Schedule.get_zoneinfo() all_zones.sort(key=lambda x: -len(x)) - for r in Schedule.rrulestr(self.rrule)._rrule: - if r._dtstart: - tzinfo = r._dtstart.tzinfo - if tzinfo is utc: - return 'UTC' - fname = getattr(tzinfo, '_filename', None) - if fname: - for zone in all_zones: - if fname.endswith(zone): - return zone + fname = getattr(tzinfo, '_filename', None) + if fname: + for zone in all_zones: + if fname.endswith(zone): + return zone logger.warning('Could not detect valid zoneinfo for {}'.format(self.rrule)) return '' @property + # TODO: How would we handle multiple until parameters? The UI is currently using this on the edit screen of a schedule def until(self): # The UNTIL= datestamp (if any) coerced from UTC to the local naive time # of the DTSTART @@ -134,34 +134,48 @@ def coerce_naive_until(cls, rrule): # timezone (America/New_York), and so we'll coerce to UTC _for you_ # automatically. # - if 'until=' in rrule.lower(): - # if DTSTART;TZID= is used, coerce "naive" UNTIL values - # to the proper UTC date - match_until = re.match(r".*?(?PUNTIL\=[0-9]+T[0-9]+)(?PZ?)", rrule) - if not len(match_until.group('utcflag')): - # rrule = DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T170000 - - # Find the UNTIL=N part of the string - # naive_until = UNTIL=20200601T170000 - naive_until = match_until.group('until') - - # What is the DTSTART timezone for: - # DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T170000Z - # local_tz = tzfile('/usr/share/zoneinfo/America/New_York') - local_tz = dateutil.rrule.rrulestr(rrule.replace(naive_until, naive_until + 'Z'), tzinfos=UTC_TIMEZONES)._dtstart.tzinfo - - # Make a datetime object with tzinfo= - # localized_until = datetime.datetime(2020, 6, 1, 17, 0, tzinfo=tzfile('/usr/share/zoneinfo/America/New_York')) - localized_until = make_aware(datetime.datetime.strptime(re.sub('^UNTIL=', '', naive_until), "%Y%m%dT%H%M%S"), local_tz) - - # Coerce the datetime to UTC and format it as a string w/ Zulu format - # utc_until = UNTIL=20200601T220000Z - utc_until = 'UNTIL=' + localized_until.astimezone(pytz.utc).strftime('%Y%m%dT%H%M%SZ') - - # rrule was: DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T170000 - # rrule is now: DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T220000Z - rrule = rrule.replace(naive_until, utc_until) - return rrule + + # Find the DTSTART rule or raise an error, its usually the first rule but that is not strictly enforced + start_date_rule = re.sub('^.*(DTSTART[^\s]+)\s.*$', r'\1', rrule) + if not start_date_rule: + raise ValueError('A DTSTART field needs to be in the rrule') + + rules = re.split(r'\s+', rrule) + for index in range(0, len(rules)): + rule = rules[index] + if 'until=' in rule.lower(): + # if DTSTART;TZID= is used, coerce "naive" UNTIL values + # to the proper UTC date + match_until = re.match(r".*?(?PUNTIL\=[0-9]+T[0-9]+)(?PZ?)", rule) + if not len(match_until.group('utcflag')): + # rule = DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T170000 + + # Find the UNTIL=N part of the string + # naive_until = UNTIL=20200601T170000 + naive_until = match_until.group('until') + + # What is the DTSTART timezone for: + # DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T170000Z + # local_tz = tzfile('/usr/share/zoneinfo/America/New_York') + # We are going to construct a 'dummy' rule for parsing which will include the DTSTART and the rest of the rule + temp_rule = "{} {}".format(start_date_rule, rule.replace(naive_until, naive_until + 'Z')) + # If the rule is an EX rule we have to add an RRULE to it because an EX rule alone will not manifest into a ruleset + if rule.lower().startswith('ex'): + temp_rule = "{} {}".format(temp_rule, 'RRULE:FREQ=MINUTELY;INTERVAL=1;UNTIL=20380601T170000Z') + local_tz = dateutil.rrule.rrulestr(temp_rule, tzinfos=UTC_TIMEZONES, **{'forceset': True})._rrule[0]._dtstart.tzinfo + + # Make a datetime object with tzinfo= + # localized_until = datetime.datetime(2020, 6, 1, 17, 0, tzinfo=tzfile('/usr/share/zoneinfo/America/New_York')) + localized_until = make_aware(datetime.datetime.strptime(re.sub('^UNTIL=', '', naive_until), "%Y%m%dT%H%M%S"), local_tz) + + # Coerce the datetime to UTC and format it as a string w/ Zulu format + # utc_until = UNTIL=20200601T220000Z + utc_until = 'UNTIL=' + localized_until.astimezone(pytz.utc).strftime('%Y%m%dT%H%M%SZ') + + # rule was: DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T170000 + # rule is now: DTSTART;TZID=America/New_York:20200601T120000 RRULE:...;UNTIL=20200601T220000Z + rules[index] = rule.replace(naive_until, utc_until) + return " ".join(rules) @classmethod def rrulestr(cls, rrule, fast_forward=True, **kwargs): @@ -176,20 +190,28 @@ def rrulestr(cls, rrule, fast_forward=True, **kwargs): if r._dtstart and r._dtstart.tzinfo is None: raise ValueError('A valid TZID must be provided (e.g., America/New_York)') - if fast_forward and ('MINUTELY' in rrule or 'HOURLY' in rrule) and 'COUNT=' not in rrule: + # Fast forward is a way for us to limit the number of events in the rruleset + # If we are fastforwading and we don't have a count limited rule that is minutely or hourley + # We will modify the start date of the rule to last week to prevent a large number of entries + if fast_forward: try: + # All rules in a ruleset will have the same dtstart value + # so lets compare the first event to now to see if its > 7 days old first_event = x[0] - # If the first event was over a week ago... if (now() - first_event).days > 7: - # hourly/minutely rrules with far-past DTSTART values - # are *really* slow to precompute - # start *from* one week ago to speed things up drastically - dtstart = x._rrule[0]._dtstart.strftime(':%Y%m%dT') - new_start = (now() - datetime.timedelta(days=7)).strftime(':%Y%m%dT') - new_rrule = rrule.replace(dtstart, new_start) - return Schedule.rrulestr(new_rrule, fast_forward=False) + for rule in x._rrule: + # If any rule has a minutely or hourly rule without a count... + if rule._freq in [dateutil.rrule.MINUTELY, dateutil.rrule.HOURLY] and not rule._count: + # hourly/minutely rrules with far-past DTSTART values + # are *really* slow to precompute + # start *from* one week ago to speed things up drastically + new_start = (now() - datetime.timedelta(days=7)).strftime('%Y%m%d') + # Now we want to repalce the DTSTART:T with the new date (which includes the T) + new_rrule = re.sub('(DTSTART[^:]*):[^T]+T', r'\1:{0}T'.format(new_start), rrule) + return Schedule.rrulestr(new_rrule, fast_forward=False) except IndexError: pass + return x def __str__(self): @@ -206,6 +228,22 @@ def get_job_kwargs(self): job_kwargs['_eager_fields'] = {'launch_type': 'scheduled', 'schedule': self} return job_kwargs + def get_end_date(ruleset): + # if we have a complex ruleset with a lot of options getting the last index of the ruleset can take some time + # And a ruleset without a count/until can come back as datetime.datetime(9999, 12, 31, 15, 0, tzinfo=tzfile('US/Eastern')) + # So we are going to do a quick scan to make sure we would have an end date + for a_rule in ruleset._rrule: + # if this rule does not have until or count in it then we have no end date + if not a_rule._until and not a_rule._count: + return None + + # If we made it this far we should have an end date and can ask the ruleset what the last date is + # However, if the until/count is before dtstart we will get an IndexError when trying to get [-1] + try: + return ruleset[-1].astimezone(pytz.utc) + except IndexError: + return None + def update_computed_fields_no_save(self): affects_fields = ['next_run', 'dtstart', 'dtend'] starting_values = {} @@ -229,12 +267,7 @@ def update_computed_fields_no_save(self): self.dtstart = future_rs[0].astimezone(pytz.utc) except IndexError: self.dtstart = None - self.dtend = None - if 'until' in self.rrule.lower() or 'count' in self.rrule.lower(): - try: - self.dtend = future_rs[-1].astimezone(pytz.utc) - except IndexError: - self.dtend = None + self.dtend = Schedule.get_end_date(future_rs) changed = any(getattr(self, field_name) != starting_values[field_name] for field_name in affects_fields) return changed diff --git a/awx/main/tests/functional/api/test_schedules.py b/awx/main/tests/functional/api/test_schedules.py index bda8e3efbee8..e38fd0964764 100644 --- a/awx/main/tests/functional/api/test_schedules.py +++ b/awx/main/tests/functional/api/test_schedules.py @@ -111,21 +111,41 @@ def test_encrypted_survey_answer(post, patch, admin_user, project, inventory, su [ ("", "This field may not be blank"), ("DTSTART:NONSENSE", "Valid DTSTART required in rrule"), + ("DTSTART:20300308T050000 RRULE:FREQ=DAILY;INTERVAL=1", "DTSTART cannot be a naive datetime"), ("DTSTART:20300308T050000Z DTSTART:20310308T050000", "Multiple DTSTART is not supported"), - ("DTSTART:20300308T050000Z", "RRULE required in rrule"), - ("DTSTART:20300308T050000Z RRULE:NONSENSE", "INTERVAL required in rrule"), + ("DTSTART:20300308T050000Z", "One or more rule required in rrule"), + ("DTSTART:20300308T050000Z RRULE:FREQ=MONTHLY;INTERVAL=1; EXDATE:20220401", "EXDATE not allowed in rrule"), + ("DTSTART:20300308T050000Z RRULE:FREQ=MONTHLY;INTERVAL=1; RDATE:20220401", "RDATE not allowed in rrule"), ("DTSTART:20300308T050000Z RRULE:FREQ=SECONDLY;INTERVAL=5;COUNT=6", "SECONDLY is not supported"), - ("DTSTART:20300308T050000Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=3,4", "Multiple BYMONTHDAYs not supported"), # noqa - ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=1,2", "Multiple BYMONTHs not supported"), # noqa + # Individual rule test + ("DTSTART:20300308T050000Z RRULE:NONSENSE", "INTERVAL required in rrule"), ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=5MO", "BYDAY with numeric prefix not supported"), # noqa - ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYYEARDAY=100", "BYYEARDAY not supported"), # noqa - ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYWEEKNO=20", "BYWEEKNO not supported"), + ("DTSTART:20030925T104941Z RRULE:FREQ=DAILY;INTERVAL=10;COUNT=500;UNTIL=20040925T104941Z", "RRULE may not contain both COUNT and UNTIL"), # noqa ("DTSTART:20300308T050000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=2000", "COUNT > 999 is unsupported"), # noqa + # Individual rule test with multiple rules + ## Bad Rule: RRULE:NONSENSE + ("DTSTART:20300308T050000Z RRULE:NONSENSE RRULE:INTERVAL=1;FREQ=DAILY EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU", "INTERVAL required in rrule"), + ## Bad Rule: RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=5MO + ( + "DTSTART:20300308T050000Z RRULE:INTERVAL=1;FREQ=DAILY EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=5MO", + "BYDAY with numeric prefix not supported", + ), # noqa + ## Bad Rule: RRULE:FREQ=DAILY;INTERVAL=10;COUNT=500;UNTIL=20040925T104941Z + ( + "DTSTART:20030925T104941Z RRULE:INTERVAL=1;FREQ=DAILY EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU RRULE:FREQ=DAILY;INTERVAL=10;COUNT=500;UNTIL=20040925T104941Z", + "RRULE may not contain both COUNT and UNTIL", + ), # noqa + ## Bad Rule: RRULE:FREQ=DAILY;INTERVAL=1;COUNT=2000 + ( + "DTSTART:20300308T050000Z RRULE:INTERVAL=1;FREQ=DAILY EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU RRULE:FREQ=DAILY;INTERVAL=1;COUNT=2000", + "COUNT > 999 is unsupported", + ), # noqa + # Multiple errors, first condition should be returned + ("DTSTART:NONSENSE RRULE:NONSENSE RRULE:FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=3,4", "Valid DTSTART required in rrule"), + # Parsing Tests ("DTSTART;TZID=US-Eastern:19961105T090000 RRULE:FREQ=MINUTELY;INTERVAL=10;COUNT=5", "A valid TZID must be provided"), # noqa ("DTSTART:20300308T050000Z RRULE:FREQ=REGULARLY;INTERVAL=1", "rrule parsing failed validation: invalid 'FREQ': REGULARLY"), # noqa - ("DTSTART:20030925T104941Z RRULE:FREQ=DAILY;INTERVAL=10;COUNT=500;UNTIL=20040925T104941Z", "RRULE may not contain both COUNT and UNTIL"), # noqa ("DTSTART;TZID=America/New_York:20300308T050000Z RRULE:FREQ=DAILY;INTERVAL=1", "rrule parsing failed validation"), - ("DTSTART:20300308T050000 RRULE:FREQ=DAILY;INTERVAL=1", "DTSTART cannot be a naive datetime"), ], ) def test_invalid_rrules(post, admin_user, project, inventory, rrule, error): @@ -143,6 +163,29 @@ def test_invalid_rrules(post, admin_user, project, inventory, rrule, error): assert error in smart_str(resp.content) +def test_multiple_invalid_rrules(post, admin_user, project, inventory): + job_template = JobTemplate.objects.create(name='test-jt', project=project, playbook='helloworld.yml', inventory=inventory) + url = reverse('api:job_template_schedules_list', kwargs={'pk': job_template.id}) + resp = post( + url, + { + 'name': 'Some Schedule', + 'rrule': "EXRULE:FREQ=SECONDLY DTSTART;TZID=US-Eastern:19961105T090000 RRULE:FREQ=MINUTELY;INTERVAL=10;COUNT=5;UNTIL=20220101 DTSTART;TZID=US-Eastern:19961105T090000", + }, + admin_user, + expect=400, + ) + expected_result = { + "rrule": [ + "Multiple DTSTART is not supported.", + "INTERVAL required in rrule: RULE:FREQ=SECONDLY", + "RRULE may not contain both COUNT and UNTIL: RULE:FREQ=MINUTELY;INTERVAL=10;COUNT=5;UNTIL=20220101", + "rrule parsing failed validation: 'NoneType' object has no attribute 'group'", + ] + } + assert expected_result == resp.data + + @pytest.mark.django_db def test_normal_users_can_preview_schedules(post, alice): url = reverse('api:schedule_rrule') @@ -381,6 +424,78 @@ def test_dst_rollback_duplicates(post, admin_user): ] +@pytest.mark.parametrize( + 'rrule, expected_result', + ( + pytest.param( + 'DTSTART;TZID=America/New_York:20300302T150000 RRULE:INTERVAL=1;FREQ=DAILY;UNTIL=20300304T1500 EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU', + ['2030-03-02 15:00:00-05:00', '2030-03-04 15:00:00-05:00'], + id="Every day except sundays", + ), + pytest.param( + 'DTSTART;TZID=US/Eastern:20300428T170000 RRULE:INTERVAL=1;FREQ=DAILY;COUNT=4 EXRULE:INTERVAL=1;FREQ=DAILY;BYMONTH=4;BYMONTHDAY=30', + ['2030-04-28 17:00:00-04:00', '2030-04-29 17:00:00-04:00', '2030-05-01 17:00:00-04:00'], + id="Every day except April 30th", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20300313T164500 RRULE:INTERVAL=5;FREQ=MINUTELY EXRULE:FREQ=MINUTELY;INTERVAL=5;BYDAY=WE;BYHOUR=17,18', + [ + '2030-03-13 16:45:00-04:00', + '2030-03-13 16:50:00-04:00', + '2030-03-13 16:55:00-04:00', + '2030-03-13 19:00:00-04:00', + '2030-03-13 19:05:00-04:00', + '2030-03-13 19:10:00-04:00', + '2030-03-13 19:15:00-04:00', + '2030-03-13 19:20:00-04:00', + '2030-03-13 19:25:00-04:00', + '2030-03-13 19:30:00-04:00', + ], + id="Every 5 minutes but not Wednesdays from 5-7pm", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20300426T100100 RRULE:INTERVAL=15;FREQ=MINUTELY;BYDAY=MO,TU,WE,TH,FR;BYHOUR=10,11 EXRULE:INTERVAL=15;FREQ=MINUTELY;BYDAY=MO,TU,WE,TH,FR;BYHOUR=11;BYMINUTE=3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,34,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59', + [ + '2030-04-26 10:01:00-04:00', + '2030-04-26 10:16:00-04:00', + '2030-04-26 10:31:00-04:00', + '2030-04-26 10:46:00-04:00', + '2030-04-26 11:01:00-04:00', + '2030-04-29 10:01:00-04:00', + '2030-04-29 10:16:00-04:00', + '2030-04-29 10:31:00-04:00', + '2030-04-29 10:46:00-04:00', + '2030-04-29 11:01:00-04:00', + ], + id="Every 15 minutes Monday - Friday from 10:01am to 11:02pm (inclusive)", + ), + pytest.param( + 'DTSTART:20301219T130551Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYMONTHDAY=12,13,14,15,16,17,18', + [ + '2031-01-18 13:05:51+00:00', + '2031-02-15 13:05:51+00:00', + '2031-03-15 13:05:51+00:00', + '2031-04-12 13:05:51+00:00', + '2031-05-17 13:05:51+00:00', + '2031-06-14 13:05:51+00:00', + '2031-07-12 13:05:51+00:00', + '2031-08-16 13:05:51+00:00', + '2031-09-13 13:05:51+00:00', + '2031-10-18 13:05:51+00:00', + ], + id="Any Saturday whose month day is between 12 and 18", + ), + ), +) +def test_complex_schedule(post, admin_user, rrule, expected_result): + # Every day except Sunday, 2022-05-01 is a Sunday + + url = reverse('api:schedule_rrule') + r = post(url, {'rrule': rrule}, admin_user, expect=200) + + assert list(map(str, r.data['local'])) == expected_result + + @pytest.mark.django_db def test_zoneinfo(get, admin_user): url = reverse('api:schedule_zoneinfo') diff --git a/awx/main/tests/functional/models/test_schedule.py b/awx/main/tests/functional/models/test_schedule.py index f939835fceda..b41c390662df 100644 --- a/awx/main/tests/functional/models/test_schedule.py +++ b/awx/main/tests/functional/models/test_schedule.py @@ -251,18 +251,17 @@ def test_utc_until(job_template, until, dtend): @pytest.mark.django_db @pytest.mark.parametrize( - 'dtstart, until', + 'rrule, length', [ - ['DTSTART:20380601T120000Z', '20380601T170000'], # noon UTC to 5PM UTC - ['DTSTART;TZID=America/New_York:20380601T120000', '20380601T170000'], # noon EST to 5PM EST + ['DTSTART:20380601T120000Z RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000', 6], # noon UTC to 5PM UTC (noon, 1pm, 2, 3, 4, 5pm) + ['DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000', 6], # noon EST to 5PM EST ], ) -def test_tzinfo_naive_until(job_template, dtstart, until): - rrule = '{} RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL={}'.format(dtstart, until) # noqa +def test_tzinfo_naive_until(job_template, rrule, length): s = Schedule(name='Some Schedule', rrule=rrule, unified_job_template=job_template) s.save() gen = Schedule.rrulestr(s.rrule).xafter(now(), count=20) - assert len(list(gen)) == 6 # noon, 1PM, 2, 3, 4, 5PM + assert len(list(gen)) == length @pytest.mark.django_db @@ -309,6 +308,12 @@ def test_beginning_of_time(job_template): [ ['DTSTART:20300112T210000Z RRULE:FREQ=DAILY;INTERVAL=1', 'UTC'], ['DTSTART;TZID=US/Eastern:20300112T210000 RRULE:FREQ=DAILY;INTERVAL=1', 'US/Eastern'], + ['DTSTART;TZID=US/Eastern:20300112T210000 RRULE:FREQ=DAILY;INTERVAL=1 EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU', 'US/Eastern'], + # Technically the serializer should never let us get 2 dtstarts in a rule but its still valid and the rrule will prefer the last DTSTART + [ + 'DTSTART;TZID=US/Eastern:20300112T210000 RRULE:FREQ=DAILY;INTERVAL=1 EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU DTSTART;TZID=US/Pacific:20300112T210000', + 'US/Pacific', + ], ], ) def test_timezone_property(job_template, rrule, tz): @@ -389,3 +394,163 @@ def test_duplicate_name_within_template(job_template): s2.save() assert str(ierror.value) == "UNIQUE constraint failed: main_schedule.unified_job_template_id, main_schedule.name" + + +# Test until with multiple entries (should only return the first) +# NOTE: this test may change once we determine how the UI will start to handle this field +@pytest.mark.django_db +@pytest.mark.parametrize( + 'rrule, expected_until', + [ + pytest.param('DTSTART:20380601T120000Z RRULE:FREQ=HOURLY;INTERVAL=1', '', id="No until"), + pytest.param('DTSTART:20380601T120000Z RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000Z', '2038-06-01T17:00:00', id="One until in UTC"), + pytest.param( + 'DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000', + '2038-06-01T17:00:00', + id="One until in local TZ", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T220000 RRULE:FREQ=MINUTELY;INTERVAL=1;UNTIL=20380601T170000', + '2038-06-01T22:00:00', + id="Multiple untils (return only the first one", + ), + ], +) +def test_until_with_complex_schedules(job_template, rrule, expected_until): + sched = Schedule(name='Some Schedule', rrule=rrule, unified_job_template=job_template) + assert sched.until == expected_until + + +# Test coerce_naive_until, this method takes a naive until field and forces it into utc +@pytest.mark.django_db +@pytest.mark.parametrize( + 'rrule, expected_result', + [ + pytest.param( + 'DTSTART:20380601T120000Z RRULE:FREQ=HOURLY;INTERVAL=1', + 'DTSTART:20380601T120000Z RRULE:FREQ=HOURLY;INTERVAL=1', + id="No untils present", + ), + pytest.param( + 'DTSTART:20380601T120000Z RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000Z', + 'DTSTART:20380601T120000Z RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000Z', + id="One until already in UTC", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000', + 'DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T220000Z', + id="One until with local tz", + ), + pytest.param( + 'DTSTART:20380601T120000Z RRULE:FREQ=MINUTLEY;INTERVAL=1;UNTIL=20380601T170000Z EXRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000Z', + 'DTSTART:20380601T120000Z RRULE:FREQ=MINUTLEY;INTERVAL=1;UNTIL=20380601T170000Z EXRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000Z', + id="Multiple untils all in UTC", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=MINUTELY;INTERVAL=1;UNTIL=20380601T170000 EXRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000', + 'DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=MINUTELY;INTERVAL=1;UNTIL=20380601T220000Z EXRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T220000Z', + id="Multiple untils with local tz", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=MINUTELY;INTERVAL=1;UNTIL=20380601T170000Z EXRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T170000', + 'DTSTART;TZID=America/New_York:20380601T120000 RRULE:FREQ=MINUTELY;INTERVAL=1;UNTIL=20380601T170000Z EXRULE:FREQ=HOURLY;INTERVAL=1;UNTIL=20380601T220000Z', + id="Multiple untils mixed", + ), + ], +) +def test_coerce_naive_until(rrule, expected_result): + new_rrule = Schedule.coerce_naive_until(rrule) + assert new_rrule == expected_result + + +# Test skipping days with exclusion +@pytest.mark.django_db +def test_skip_sundays(): + rrule = ''' + DTSTART;TZID=America/New_York:20220310T150000 + RRULE:INTERVAL=1;FREQ=DAILY + EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU + ''' + timezone = pytz.timezone("America/New_York") + friday_apr_29th = datetime(2022, 4, 29, 0, 0, 0, 0, timezone) + monday_may_2nd = datetime(2022, 5, 2, 23, 59, 59, 999, timezone) + ruleset = Schedule.rrulestr(rrule) + gen = ruleset.between(friday_apr_29th, monday_may_2nd, True) + # We should only get Fri, Sat and Mon (skipping Sunday) + assert len(list(gen)) == 3 + saturday_night = datetime(2022, 4, 30, 23, 59, 59, 9999, timezone) + monday_morning = datetime(2022, 5, 2, 0, 0, 0, 0, timezone) + gen = ruleset.between(saturday_night, monday_morning, True) + assert len(list(gen)) == 0 + + +# Test the get_end_date function +@pytest.mark.django_db +@pytest.mark.parametrize( + 'rrule, expected_result', + [ + pytest.param( + 'DTSTART;TZID=America/New_York:20210310T150000 RRULE:INTERVAL=1;FREQ=DAILY;UNTIL=20210430T150000Z EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU;COUNT=5', + datetime(2021, 4, 29, 19, 0, 0, tzinfo=pytz.utc), + id="Single rule in rule set with UTC TZ aware until", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY;UNTIL=20220430T150000 EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU;COUNT=5', + datetime(2022, 4, 30, 19, 0, tzinfo=pytz.utc), + id="Single rule in ruleset with naive until", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY;COUNT=4 EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU;COUNT=5', + datetime(2022, 3, 12, 20, 0, tzinfo=pytz.utc), + id="Single rule in ruleset with count", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY EXRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU;COUNT=5', + None, + id="Single rule in ruleset with no end", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY', + None, + id="Single rule in rule with no end", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY;UNTIL=20220430T150000Z', + datetime(2022, 4, 29, 19, 0, tzinfo=pytz.utc), + id="Single rule in rule with UTZ TZ aware until", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY;UNTIL=20220430T150000', + datetime(2022, 4, 30, 19, 0, tzinfo=pytz.utc), + id="Single rule in rule with naive until", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY;BYDAY=SU RRULE:INTERVAL=1;FREQ=DAILY;BYDAY=MO', + None, + id="Multi rule with no end", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY;BYDAY=SU RRULE:INTERVAL=1;FREQ=DAILY;BYDAY=MO;COUNT=4', + None, + id="Multi rule one with no end and one with an count", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220310T150000 RRULE:INTERVAL=1;FREQ=DAILY;BYDAY=SU;UNTIL=20220430T1500Z RRULE:INTERVAL=1;FREQ=DAILY;BYDAY=MO;COUNT=4', + datetime(2022, 4, 24, 19, 0, tzinfo=pytz.utc), + id="Multi rule one with until and one with an count", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20010430T1500 RRULE:INTERVAL=1;FREQ=DAILY;BYDAY=SU;COUNT=1', + datetime(2001, 5, 6, 19, 0, tzinfo=pytz.utc), + id="Rule with count but ends in the past", + ), + pytest.param( + 'DTSTART;TZID=America/New_York:20220430T1500 RRULE:INTERVAL=1;FREQ=DAILY;BYDAY=SU;UNTIL=20010430T1500', + None, + id="Rule with until that ends in the past", + ), + ], +) +def test_get_end_date(rrule, expected_result): + ruleset = Schedule.rrulestr(rrule) + assert expected_result == Schedule.get_end_date(ruleset) diff --git a/awx/ui/src/components/Schedule/shared/ScheduleForm.js b/awx/ui/src/components/Schedule/shared/ScheduleForm.js index efbb599a603b..87eadeaf3966 100644 --- a/awx/ui/src/components/Schedule/shared/ScheduleForm.js +++ b/awx/ui/src/components/Schedule/shared/ScheduleForm.js @@ -11,6 +11,8 @@ import { FormGroup, Title, ActionGroup, + // To be removed once UI completes complex schedules + Alert, } from '@patternfly/react-core'; import { Config } from 'contexts/Config'; import { SchedulesAPI } from 'api'; @@ -439,6 +441,34 @@ function ScheduleForm({ if (Object.keys(schedule).length > 0) { if (schedule.rrule) { + if (schedule.rrule.split(/\s+/).length > 2) { + return ( +
+ + {t`Schedule Rules`}: + 
+              {schedule.rrule}
+
+ + + + + ); + } + try { const { origOptions: { diff --git a/awx_collection/plugins/lookup/schedule_rruleset.py b/awx_collection/plugins/lookup/schedule_rruleset.py new file mode 100644 index 000000000000..ead49bbc2a41 --- /dev/null +++ b/awx_collection/plugins/lookup/schedule_rruleset.py @@ -0,0 +1,349 @@ +# (c) 2020 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +DOCUMENTATION = """ + lookup: schedule_rruleset + author: John Westcott IV (@john-westcott-iv) + short_description: Generate an rruleset string + requirements: + - pytz + - python-dateutil >= 2.7.0 + description: + - Returns a string based on criteria which represents an rrule + options: + _terms: + description: + - The start date of the ruleset + - Used for all frequencies + - Format should be YYYY-MM-DD [HH:MM:SS] + required: True + type: str + timezone: + description: + - The timezone to use for this rule + - Used for all frequencies + - Format should be as US/Eastern + - Defaults to America/New_York + type: str + rules: + description: + - Array of rules in the rruleset + type: array + required: True + suboptions: + frequency: + description: + - The frequency of the schedule + - none - Run this schedule once + - minute - Run this schedule every x minutes + - hour - Run this schedule every x hours + - day - Run this schedule every x days + - week - Run this schedule weekly + - month - Run this schedule monthly + required: True + choices: ['none', 'minute', 'hour', 'day', 'week', 'month'] + interval: + description: + - The repetition in months, weeks, days hours or minutes + - Used for all types except none + type: int + end_on: + description: + - How to end this schedule + - If this is not defined, this schedule will never end + - If this is a positive integer, this schedule will end after this number of occurrences + - If this is a date in the format YYYY-MM-DD [HH:MM:SS], this schedule ends after this date + - Used for all types except none + type: str + bysetpos: + description: + - Specify an occurrence number, corresponding to the nth occurrence of the rule inside the frequency period. + - A comma-separated list of positions (first, second, third, forth or last) + type: string + bymonth: + description: + - The months this schedule will run on + - A comma-separated list which can contain values 0-12 + type: string + bymonthday: + description: + - The day of the month this schedule will run on + - A comma-separated list which can contain values 0-31 + type: string + byyearday: + description: + - The year day numbers to run this schedule on + - A comma-separated list which can contain values 0-366 + type: string + byweekno: + description: + - The week numbers to run this schedule on + - A comma-separated list which can contain values as described in ISO8601 + type: string + byweekday: + description: + - The days to run this schedule on + - A comma-separated list which can contain values sunday, monday, tuesday, wednesday, thursday, friday + type: string + byhour: + description: + - The hours to run this schedule on + - A comma-separated list which can contain values 0-23 + type: string + byminute: + description: + - The minutes to run this schedule on + - A comma-separated list which can contain values 0-59 + type: string + include: + description: + - If this rule should be included (RRULE) or excluded (EXRULE) + type: bool + default: True +""" + +EXAMPLES = """ + - name: Create a ruleset for everyday except Sundays + set_fact: + complex_rule: "{{ query(awx.awx.schedule_rruleset, '2022-04-30 10:30:45', rules=rrules, timezone='UTC' ) }}" + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'day' + interval: 1 + byweekday: 'sunday' + include: False +""" + +RETURN = """ +_raw: + description: + - String in the rrule format + type: string +""" +import re + +from ansible.module_utils.six import raise_from +from ansible.plugins.lookup import LookupBase +from ansible.errors import AnsibleError +from datetime import datetime + +try: + import pytz + from dateutil import rrule +except ImportError as imp_exc: + raise_from(AnsibleError('{0}'.format(imp_exc)), imp_exc) + + +class LookupModule(LookupBase): + frequencies = { + 'none': rrule.DAILY, + 'minute': rrule.MINUTELY, + 'hour': rrule.HOURLY, + 'day': rrule.DAILY, + 'week': rrule.WEEKLY, + 'month': rrule.MONTHLY, + } + + weekdays = { + 'monday': rrule.MO, + 'tuesday': rrule.TU, + 'wednesday': rrule.WE, + 'thursday': rrule.TH, + 'friday': rrule.FR, + 'saturday': rrule.SA, + 'sunday': rrule.SU, + } + + set_positions = { + 'first': 1, + 'second': 2, + 'third': 3, + 'fourth': 4, + 'last': -1, + } + + # plugin constructor + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @staticmethod + def parse_date_time(date_string): + try: + return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S') + except ValueError: + return datetime.strptime(date_string, '%Y-%m-%d') + + def process_integer(self, field_name, rule, min_value, max_value, rule_number): + # We are going to tolerate multiple types of input here: + # something: 1 - A single integer + # something: "1" - A single str + # something: "1,2,3" - A comma separated string of ints + # something: "1, 2,3" - A comma separated string of ints (with spaces) + # something: ["1", "2", "3"] - A list of strings + # something: [1,2,3] - A list of ints + return_values = [] + # If they give us a single int, lets make it a list of ints + if type(rule[field_name]) == int: + rule[field_name] = [rule[field_name]] + # If its not a list, we need to split it into a list + if type(rule[field_name]) != list: + rule[field_name] = rule[field_name].split(',') + for value in rule[field_name]: + # If they have a list of strs we want to strip the str incase its space delineated + if type(value) == str: + value = value.strip() + # If value happens to be an int (from a list of ints) we need to coerce it into a str for the re.match + if not re.match(r"^\d+$", str(value)) or int(value) < min_value or int(value) > max_value: + raise AnsibleError('In rule {0} {1} must be between {2} and {3}'.format(rule_number, field_name, min_value, max_value)) + return_values.append(int(value)) + return return_values + + def process_list(self, field_name, rule, valid_list, rule_number): + return_values = [] + if type(rule[field_name]) != list: + rule[field_name] = rule[field_name].split(',') + for value in rule[field_name]: + value = value.strip() + if value not in valid_list: + raise AnsibleError('In rule {0} {1} must only contain values in {2}'.format(rule_number, field_name, ', '.join(valid_list.keys()))) + return_values.append(valid_list[value]) + return return_values + + def run(self, terms, variables=None, **kwargs): + if len(terms) != 1: + raise AnsibleError('You may only pass one schedule type in at a time') + + # Validate the start date + try: + start_date = LookupModule.parse_date_time(terms[0]) + except Exception as e: + raise_from(AnsibleError('The start date must be in the format YYYY-MM-DD [HH:MM:SS]'), e) + + if not kwargs.get('rules', None): + raise AnsibleError('You must include rules to be in the ruleset via the rules parameter') + + # All frequencies can use a timezone but rrule can't support the format that AWX uses. + # So we will do a string manip here if we need to + timezone = 'America/New_York' + if 'timezone' in kwargs: + if kwargs['timezone'] not in pytz.all_timezones: + raise AnsibleError('Timezone parameter is not valid') + timezone = kwargs['timezone'] + + rules = [] + got_at_least_one_rule = False + for rule_index in range(0, len(kwargs['rules'])): + rule = kwargs['rules'][rule_index] + rule_number = rule_index + 1 + valid_options = [ + "frequency", + "interval", + "end_on", + "bysetpos", + "bymonth", + "bymonthday", + "byyearday", + "byweekno", + "byweekday", + "byhour", + "byminute", + "include", + ] + invalid_options = list(set(rule.keys()) - set(valid_options)) + if invalid_options: + raise AnsibleError('Rule {0} has invalid options: {1}'.format(rule_number, ', '.join(invalid_options))) + frequency = rule.get('frequency', None) + if not frequency: + raise AnsibleError("Rule {0} is missing a frequency".format(rule_number)) + if frequency not in LookupModule.frequencies: + raise AnsibleError('Frequency of rule {0} is invalid {1}'.format(rule_number, frequency)) + + rrule_kwargs = { + 'freq': LookupModule.frequencies[frequency], + 'interval': rule.get('interval', 1), + 'dtstart': start_date, + } + + # If we are a none frequency we don't need anything else + if frequency == 'none': + rrule_kwargs['count'] = 1 + else: + # All non-none frequencies can have an end_on option + if 'end_on' in rule: + end_on = rule['end_on'] + if re.match(r'^\d+$', end_on): + rrule_kwargs['count'] = end_on + else: + try: + rrule_kwargs['until'] = LookupModule.parse_date_time(end_on) + except Exception as e: + raise_from( + AnsibleError('In rule {0} end_on must either be an integer or in the format YYYY-MM-DD [HH:MM:SS]'.format(rule_number)), e + ) + + if 'bysetpos' in rule: + rrule_kwargs['bysetpos'] = self.process_list('bysetpos', rule, LookupModule.set_positions, rule_number) + + if 'bymonth' in rule: + rrule_kwargs['bymonth'] = self.process_integer('bymonth', rule, 1, 12, rule_number) + + if 'bymonthday' in rule: + rrule_kwargs['bymonthday'] = self.process_integer('bymonthday', rule, 1, 31, rule_number) + + if 'byyearday' in rule: + rrule_kwargs['byyearday'] = self.process_integer('byyearday', rule, 1, 366, rule_number) # 366 for leap years + + if 'byweekno' in rule: + rrule_kwargs['byweekno'] = self.process_integer('byweekno', rule, 1, 52, rule_number) + + if 'byweekday' in rule: + rrule_kwargs['byweekday'] = self.process_list('byweekday', rule, LookupModule.weekdays, rule_number) + + if 'byhour' in rule: + rrule_kwargs['byhour'] = self.process_integer('byhour', rule, 0, 23, rule_number) + + if 'byminute' in rule: + rrule_kwargs['byminute'] = self.process_integer('byminute', rule, 0, 59, rule_number) + + try: + generated_rule = str(rrule.rrule(**rrule_kwargs)) + except Exception as e: + raise_from(AnsibleError('Failed to parse rrule for rule {0} {1}: {2}'.format(rule_number, str(rrule_kwargs), e)), e) + + # AWX requires an interval. rrule will not add interval if it's set to 1 + if rule.get('interval', 1) == 1: + generated_rule = "{0};INTERVAL=1".format(generated_rule) + + if rule_index == 0: + # rrule puts a \n in the rule instead of a space and can't handle timezones + generated_rule = generated_rule.replace('\n', ' ').replace('DTSTART:', 'DTSTART;TZID={0}:'.format(timezone)) + else: + # Only the first rule needs the dtstart in a ruleset so remaining rules we can split at \n + generated_rule = generated_rule.split('\n')[1] + + # If we are an exclude rule we need to flip from an rrule to an ex rule + if not rule.get('include', True): + generated_rule = generated_rule.replace('RRULE', 'EXRULE') + else: + got_at_least_one_rule = True + + rules.append(generated_rule) + + if not got_at_least_one_rule: + raise AnsibleError("A ruleset must contain at least one RRULE") + + rruleset_str = ' '.join(rules) + + # For a sanity check lets make sure our rule can parse. Not sure how we can test this though + try: + rules = rrule.rrulestr(rruleset_str) + except Exception as e: + raise_from(AnsibleError("Failed to parse generated rule set via rruleset {0}".format(e)), e) + + # return self.get_rrule(frequency, kwargs) + return rruleset_str diff --git a/awx_collection/plugins/modules/schedule.py b/awx_collection/plugins/modules/schedule.py index e29051209c4b..b178e001d0d9 100644 --- a/awx_collection/plugins/modules/schedule.py +++ b/awx_collection/plugins/modules/schedule.py @@ -143,6 +143,21 @@ unified_job_template: "Demo Job Template" rrule: "{{ query('awx.awx.schedule_rrule', 'week', start_date='2019-12-19 13:05:51') }}" register: result + +- name: Build a complex schedule for every day except sunday using the rruleset plugin + schedule: + name: "{{ sched1 }}" + state: present + unified_job_template: "Demo Job Template" + rrule: "{{ query(awx.awx.schedule_rruleset, '2022-04-30 10:30:45', rules=rrules, timezone='UTC' ) }}" + vars: + rrules: + - frequency: 'day' + every: 1 + - frequency: 'day' + every: 1 + on_days: 'sunday' + include: False ''' from ..module_utils.controller_api import ControllerAPIModule @@ -255,7 +270,8 @@ def main(): new_fields, endpoint='schedules', item_type='schedule', - associations=association_fields,) + associations=association_fields, + ) if __name__ == '__main__': diff --git a/awx_collection/tests/integration/targets/lookup_rruleset/tasks/main.yml b/awx_collection/tests/integration/targets/lookup_rruleset/tasks/main.yml new file mode 100644 index 000000000000..fe5b3f755d00 --- /dev/null +++ b/awx_collection/tests/integration/targets/lookup_rruleset/tasks/main.yml @@ -0,0 +1,342 @@ +--- +- name: Get our collection package + controller_meta: + register: controller_meta + +- name: Generate the name of our plugin + set_fact: + ruleset_plugin_name: "{{ controller_meta.prefix }}.schedule_rruleset" + rule_plugin_name: "{{ controller_meta.prefix }}.schedule_rrule" + + +- name: Call ruleset with no rules + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45') }}" + ignore_errors: True + register: results + +- assert: + that: + - results is failed + - "'You must include rules to be in the ruleset via the rules parameter' in results.msg" + + +- name: call ruleset with a missing frequency + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - interval: 1 + byweekday: 'sunday' + +- assert: + that: + - results is failed + - "'Rule 2 is missing a frequency' in results.msg" + + +- name: call ruleset with a missing frequency + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - interval: 1 + byweekday: 'sunday' + +- assert: + that: + - results is failed + - "'Rule 2 is missing a frequency' in results.msg" + + +- name: call rruleset with an invalid frequency + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'asdf' + interval: 1 + byweekday: 'sunday' + +- assert: + that: + - results is failed + - "'Frequency of rule 2 is invalid asdf' in results.msg" + + +- name: call rruleset with an invalid end_on + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'day' + interval: 1 + byweekday: 'sunday' + end_on: 'a' + +- assert: + that: + - results is failed + - "'In rule 2 end_on must either be an integer or in the format YYYY-MM-DD [HH:MM:SS]' in results.msg" + + +- name: call rruleset with an invalid byweekday + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'day' + interval: 1 + byweekday: 'junk' + +- assert: + that: + - results is failed + - "'In rule 2 byweekday must only contain values' in results.msg" + + +- name: call rruleset with a monthly rule with invalid bymonthday (a) + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'month' + interval: 1 + bymonthday: 'a' + +- assert: + that: + - results is failed + - "'In rule 2 bymonthday must be between 1 and 31' in results.msg" + + +- name: call rruleset with a monthly rule with invalid bymonthday (-1) + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'month' + interval: 1 + bymonthday: '-1' + +- assert: + that: + - results is failed + - "'In rule 2 bymonthday must be between 1 and 31' in results.msg" + + +- name: call rruleset with a monthly rule with invalid bymonthday (32) + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'month' + interval: 1 + bymonthday: 32 + +- assert: + that: + - results is failed + - "'In rule 2 bymonthday must be between 1 and 31' in results.msg" + + +- name: call rruleset with a monthly rule with invalid bysetpos (junk) + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'month' + interval: 1 + bysetpos: 'junk' + +- assert: + that: + - results is failed + - "'In rule 2 bysetpos must only contain values in first, second, third, fourth, last' in results.msg" + + +- name: call rruleset with an invalid timezone + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules, timezone='junk' ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'day' + interval: 1 + byweekday: 'sunday' + +- assert: + that: + - results is failed + - "'Timezone parameter is not valid' in results.msg" + + +- name: call rruleset with only exclude rules + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + include: False + - frequency: 'day' + interval: 1 + byweekday: 'sunday' + include: False + +- assert: + that: + - results is failed + - "'A ruleset must contain at least one RRULE' in results.msg" + + +- name: Every day except for Sundays + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules, timezone='UTC' ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'day' + interval: 1 + byweekday: 'sunday' + include: False + +- assert: + that: + - results is success + - "'DTSTART;TZID=UTC:20220430T103045 RRULE:FREQ=DAILY;INTERVAL=1 EXRULE:FREQ=DAILY;BYDAY=SU;INTERVAL=1' == complex_rule" + + +- name: Every day except for April 30th + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2023-04-28 17:00:00', rules=rrules, timezone='UTC' ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'day' + interval: 1 + - frequency: 'day' + interval: 1 + bymonth: '4' + bymonthday: 30 + include: False + +- assert: + that: + - results is success + - "'DTSTART;TZID=UTC:20230428T170000 RRULE:FREQ=DAILY;INTERVAL=1 EXRULE:FREQ=DAILY;BYMONTH=4;BYMONTHDAY=30;INTERVAL=1' == complex_rule" + + +- name: Every 5 minutes but not on Mondays from 5-7pm + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules, timezone='UTC' ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'minute' + interval: 5 + - frequency: 'minute' + interval: 5 + byweekday: 'monday' + byhour: + - 17 + - 18 + include: False + +- assert: + that: + - results is success + - "'DTSTART;TZID=UTC:20220430T103045 RRULE:FREQ=MINUTELY;INTERVAL=5 EXRULE:FREQ=MINUTELY;INTERVAL=5;BYDAY=MO;BYHOUR=17,18' == complex_rule" + + +- name: Every 15 minutes Monday to Friday from 10:01am to 6:02pm (inclusive) + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules, timezone='UTC' ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'minute' + byweekday: + - monday + - tuesday + - wednesday + - thursday + - friday + interval: 15 + byhour: [10, 11, 12, 13, 14, 15, 16, 17, 18] + - frequency: 'minute' + interval: 1 + byweekday: "monday,tuesday,wednesday, thursday,friday" + byhour: 18 + byminute: "{{ range(3, 60) | list }}" + include: False + +- assert: + that: + - results is success + - "'DTSTART;TZID=UTC:20220430T103045 RRULE:FREQ=MINUTELY;INTERVAL=15;BYDAY=MO,TU,WE,TH,FR;BYHOUR=10,11,12,13,14,15,16,17,18 EXRULE:FREQ=MINUTELY;BYDAY=MO,TU,WE,TH,FR;BYHOUR=18;BYMINUTE=3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59;INTERVAL=1' == complex_rule" + + +- name: Any Saturday whose month day is between 12 and 18 + set_fact: + complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules, timezone='UTC' ) }}" + ignore_errors: True + register: results + vars: + rrules: + - frequency: 'month' + interval: 1 + byweekday: 'saturday' + bymonthday: "{{ range(12,19) | list }}" + +- assert: + that: + - results is success + - "'DTSTART;TZID=UTC:20220430T103045 RRULE:FREQ=MONTHLY;BYMONTHDAY=12,13,14,15,16,17,18;BYDAY=SA;INTERVAL=1' == complex_rule" diff --git a/docs/schedules.md b/docs/schedules.md index 5617cdf493bf..3b445f259e43 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -18,10 +18,11 @@ an `HTTP POST` to a variety of API endpoints: } ...where `rrule` is a valid -[RFC5545](https://www.rfc-editor.org/rfc/rfc5545.txt) RRULE string. The -specific example above would run a job every day - for seven consecutive days - starting +[RFC5545](https://www.rfc-editor.org/rfc/rfc5545.txt) RRULE string and within AWX [RRULE Limitations](#rrule-limitations). +The specific example above would run a job every day - for seven consecutive days - starting on January 15th, 2030 at noon (UTC). +For more examples see [RRULE Examples](#rrule-examples). ## Specifying Timezones @@ -109,18 +110,18 @@ local and UTC time: ## RRULE Limitations -The following aspects of `RFC5545` are _not_ supported by AWX schedules: - -* Strings with more than a single `DTSTART:` component -* Strings with more than a single `RRULE` component -* The use of `FREQ=SECONDLY` in an `RRULE` -* The use of more than a single `FREQ=BYMONTHDAY` component in an `RRULE` -* The use of more than a single `FREQ=BYMONTHS` component in an `RRULE` -* The use of `FREQ=BYYEARDAY` in an `RRULE` -* The use of `FREQ=BYWEEKNO` in an `RRULE` -* The use of `FREQ=BYWEEKNO` in an `RRULE` -* The use of `COUNT=` in an `RRULE` with a value over 999 +AWX implements the following constraints on top of the `RFC5545` specification: +* The RRULE must start with the `DTSTART` attribute +* At least one `RRULE` entry must be in the rrule +* Strings with more than a single `DTSTART:` component are prohibited +* The use of `RDATE` or `EXDATE`is prohibited +* For any of the rules in the rrule: + * `Interval` must be included + * The use of `FREQ=SECONDLY` is prohibited + * The usage of a `BYDAY` with a prefixed number is prohibited + * The usage of both `COUNT` and `UNTIL` in the same rule is prohibited + * The use of `COUNT=` with a value over 999 is prohibited ## Implementation Details @@ -141,3 +142,24 @@ database for Schedules where `Schedule.next_run` is between `scheduler_last_runtime()` and `utcnow()`. For each of these, a new job is launched, and `Schedule.next_run` is changed to the next chronological datetime in the list of all occurences. + + +## Complex RRULE Examples + +Every day except for April 30th: + + DTSTART;TZID=US/Eastern:20230428T170000 RRULE:INTERVAL=1;FREQ=DAILY EXRULE:INTERVAL=1;FREQ=DAILY;BYMONTH=4;BYMONTHDAY=30 + +Every 5 minutes but not on Mondays from 5-7pm: + + DTSTART;TZID=America/New_York:20220418T164500 RRULE:INTERVAL=5;FREQ=MINUTELY EXRULE:FREQ=MINUTELY;INTERVAL=5;BYDAY=MO;BYHOUR=17,18 + +Every 15 minutes Monday to Friday from 10:01am to 6:02pm (inclusive): + + DTSTART;TZID=America/New_York:20220417T100100 RRULE:INTERVAL=15;FREQ=MINUTELY;BYDAY=MO,TU,WE,TH,FR;BYHOUR=10,11,12,13,14,15,16,17,18 EXRULE:INTERVAL=15;FREQ=MINUTELY;BYDAY=MO,TU,WE,TH,FR;BYHOUR=18;BYMINUTE=3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,34,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59 + +Any Saturday whose month day is between 12 and 18: + + DTSTART:20191219T130551Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYMONTHDAY=12,13,14,15,16,17,18 + +