From 7b05e7a95ae872f8bd472d0aa8ad0690c1ce03f4 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Mon, 21 Oct 2019 15:38:06 -0500 Subject: [PATCH 01/14] add 2 SLO endpoints also add SLO client to dogshell this was missed last time --- datadog/api/service_level_objectives.py | 54 +++++++++++++ datadog/dogshell/__init__.py | 2 + datadog/dogshell/service_level_objective.py | 61 ++++++++++++++- datadog/util/cli.py | 84 +++++++++++++++++++++ datadog/util/format.py | 8 ++ 5 files changed, 207 insertions(+), 2 deletions(-) diff --git a/datadog/api/service_level_objectives.py b/datadog/api/service_level_objectives.py index d2b8bc6bc..d2dc51b3c 100644 --- a/datadog/api/service_level_objectives.py +++ b/datadog/api/service_level_objectives.py @@ -1,3 +1,4 @@ +from datadog.util.format import force_to_epoch_seconds from datadog.api.resources import ( GetableAPIResource, CreateableAPIResource, @@ -137,3 +138,56 @@ def delete_many(cls, ids, **params): body={"ids": ids}, suppress_response_errors_on_codes=[200], ) + + @classmethod + def can_delete(cls, ids, **params): + """ + Check if the following SLOs can be safely deleted. + + This is used to check if SLO has any references to it. + + :param ids: a list of SLO IDs to check + :type ids: list(str) + + :returns: Dictionary representing the API's JSON response + "data.ok" represents a list of SLO ids that have no known references. + "errors" contains a dictionary of SLO ID to known reference(s). + """ + params["ids"] = ids + return super(ServiceLevelObjective, cls)._trigger_class_action( + "GET", + "can_delete", + params=params, + body=None, + suppress_response_errors_on_codes=[200], + ) + + @classmethod + def history(cls, id, from_ts, to_ts, **params): + """ + Get the SLO's history from the given time range. + + :param id: SLO ID to query + :type id: str + + :param from_ts: `from` timestamp in epoch seconds to query + :type from_ts: int|datetime.datetime + + :param to_ts: `to` timestamp in epoch seconds to query, must be > `from_ts` + :type to_ts: int|datetime.datetime + + :returns: Dictionary representing the API's JSON response + "data.ok" represents a list of SLO ids that have no known references. + "errors" contains a dictionary of SLO ID to known reference(s). + """ + params["id"] = id + params["from_ts"] = force_to_epoch_seconds(from_ts) + params["to_ts"] = force_to_epoch_seconds(to_ts) + return super(ServiceLevelObjective, cls)._trigger_class_action( + "GET", + "history", + id=id, + params=params, + body=None, + suppress_response_errors_on_codes=[200], + ) diff --git a/datadog/dogshell/__init__.py b/datadog/dogshell/__init__.py index 67e939889..107d3f7f3 100644 --- a/datadog/dogshell/__init__.py +++ b/datadog/dogshell/__init__.py @@ -17,6 +17,7 @@ from datadog.dogshell.screenboard import ScreenboardClient from datadog.dogshell.search import SearchClient from datadog.dogshell.service_check import ServiceCheckClient +from datadog.dogshell.service_level_objective import ServiceLevelObjectiveClient from datadog.dogshell.tag import TagClient from datadog.dogshell.timeboard import TimeboardClient from datadog.dogshell.dashboard import DashboardClient @@ -65,6 +66,7 @@ def main(): HostClient.setup_parser(subparsers) DowntimeClient.setup_parser(subparsers) ServiceCheckClient.setup_parser(subparsers) + ServiceLevelObjectiveClient.setup_parser(subparsers) args = parser.parse_args() config.load(args.config, args.api_key, args.app_key) diff --git a/datadog/dogshell/service_level_objective.py b/datadog/dogshell/service_level_objective.py index bc8e71066..f4f3976f5 100644 --- a/datadog/dogshell/service_level_objective.py +++ b/datadog/dogshell/service_level_objective.py @@ -3,7 +3,12 @@ import json # 3p -from datadog.util.cli import set_of_ints, comma_set, comma_list_or_empty +from datadog.util.cli import ( + set_of_ints, + comma_set, + comma_list_or_empty, + parse_date_as_epoch_timestamp, +) from datadog.util.format import pretty_json # datadog @@ -11,7 +16,7 @@ from datadog.dogshell.common import report_errors, report_warnings -class MonitorClient(object): +class ServiceLevelObjectiveClient(object): @classmethod def setup_parser(cls, subparsers): parser = subparsers.add_parser( @@ -183,6 +188,30 @@ def setup_parser(cls, subparsers): ) delete_timeframe_parser.set_defaults(func=cls._delete_timeframe) + can_delete_parser = verb_parsers.add_parser( + "can_delete", help="Check if can delete SLOs" + ) + can_delete_parser.add_argument( + "slo_ids", help="comma separated list of SLO IDs to delete", type=comma_set + ) + can_delete_parser.set_defaults(func=cls._can_delete) + + history_parser = verb_parsers.add_parser("history", help="Get the SLO history") + history_parser.add_argument("slo_id", help="SLO to query the history") + history_parser.add_argument( + "from_ts", + required=True, + type=parse_date_as_epoch_timestamp, + help="`from` date or timestamp", + ) + history_parser.add_argument( + "to_ts", + required=True, + type=parse_date_as_epoch_timestamp, + help="`to` date or timestamp", + ) + history_parser.set_defaults(func=cls._history) + @classmethod def _create(cls, args): api._timeout = args.timeout @@ -402,6 +431,34 @@ def _delete_timeframe(cls, args): else: print(json.dumps(res)) + @classmethod + def _can_delete(cls, args): + api._timeout = args.timeout + + res = api.ServiceLevelObjective.can_delete(args.slo_ids) + if res is not None: + report_warnings(res) + report_errors(res) + + if format == "pretty": + print(pretty_json(res)) + else: + print(json.dumps(res)) + + @classmethod + def _history(cls, args): + api._timeout = args.timeout + + res = api.ServiceLevelObjective.history(args.slo_id) + if res is not None: + report_warnings(res) + report_errors(res) + + if format == "pretty": + print(pretty_json(res)) + else: + print(json.dumps(res)) + @classmethod def _escape(cls, s): return s.replace("\r", "\\r").replace("\n", "\\n").replace("\t", "\\t") diff --git a/datadog/util/cli.py b/datadog/util/cli.py index b9e8ffa00..e6a7855ce 100644 --- a/datadog/util/cli.py +++ b/datadog/util/cli.py @@ -1,5 +1,8 @@ +from datetime import datetime, timedelta from argparse import ArgumentTypeError import json +import re +from format import force_to_epoch_seconds def comma_list(list_str, item_func=None): @@ -50,3 +53,84 @@ def int_or_str(item): def set_of_ints(int_csv): return set(list_of_ints(int_csv)) + + +## Date handling + +class DateParsingError(Exception): + """Thrown if parse_date exhausts all possible parsings of a string""" + + +_date_fieldre = re.compile(r"(\d+)\s?(\w+) (ago|ahead)") + + +def _midnight(): + """ Truncate a date to midnight. Default to UTC midnight today.""" + return datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + + +def parse_date_as_epoch_timestamp(date_str): + return force_to_epoch_seconds(parse_date(date_str)) + + +def parse_date(date_str): + if isinstance(date_str, datetime): + return date_str + + # Parse relative dates. + if date_str == "today": + return _midnight() + elif date_str == "yesterday": + return _midnight() - timedelta(days=1) + elif date_str == "tomorrow": + return _midnight() + timedelta(days=1) + elif date_str.endswith(("ago", "ahead")): + m = _date_fieldre.match(date_str) + if m: + fields = m.groups() + else: + fields = date_str.split(" ")[1:] + num = int(fields[0]) + short_unit = fields[1] + time_direction = {"ago": -1, "ahead": 1}[fields[2]] + assert short_unit, short_unit + units = ["weeks", "days", "hours", "minutes", "seconds"] + # translate 'h' -> 'hours' + short_units = dict([(u[:1], u) for u in units]) + unit = short_units.get(short_unit, short_unit) + # translate 'hour' -> 'hours' + if unit[-1] != "s": + unit += "s" # tolerate 1 hour + assert unit in units, "'%s' not in %s" % (unit, units) + return datetime.utcnow() + time_direction * timedelta(**{unit: num}) + elif date_str == "now": + return datetime.utcnow() + + def _from_epoch_timestamp(seconds): + return datetime.utcfromtimestamp(float(seconds)) + + def _from_epoch_ms_timestamp(millis): + in_sec = float(millis) / 1000.0 + return _from_epoch_timestamp(in_sec) + + # Or parse date formats (most specific to least specific) + parse_funcs = [ + lambda d: datetime.strptime(d, "%Y-%m-%d %H:%M:%S.%f"), + lambda d: datetime.strptime(d, "%Y-%m-%d %H:%M:%S"), + lambda d: datetime.strptime(d, "%Y-%m-%dT%H:%M:%S.%f"), + lambda d: datetime.strptime(d, "%Y-%m-%dT%H:%M:%S"), + lambda d: datetime.strptime(d, "%Y-%m-%d %H:%M"), + lambda d: datetime.strptime(d, "%Y-%m-%d-%H"), + lambda d: datetime.strptime(d, "%Y-%m-%d"), + lambda d: datetime.strptime(d, "%Y-%m"), + lambda d: datetime.strptime(d, "%Y"), + _from_epoch_timestamp, # an epoch in seconds + _from_epoch_ms_timestamp, # an epoch in milliseconds + ] + + for parse_func in parse_funcs: + try: + return parse_func(date_str) + except Exception: + pass + raise DateParsingError(u"Could not parse {0} as date".format(date_str)) diff --git a/datadog/util/format.py b/datadog/util/format.py index 4b4d83948..022a343ea 100644 --- a/datadog/util/format.py +++ b/datadog/util/format.py @@ -1,4 +1,6 @@ # stdlib +import calendar +import datetime import json @@ -12,3 +14,9 @@ def construct_url(host, api_version, path): def construct_path(api_version, path): return "{}/{}".format(api_version.strip("/"), path.strip("/")) + + +def force_to_epoch_seconds(epoch_sec_or_dt): + if isinstance(epoch_sec_or_dt, datetime.datetime): + return calendar.timegm(epoch_sec_or_dt.timetuple()) + return epoch_sec_or_dt From e2446fd121c12286eef2e4c2eb288911b0f34ae3 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Mon, 21 Oct 2019 15:40:58 -0500 Subject: [PATCH 02/14] fix import --- datadog/util/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datadog/util/cli.py b/datadog/util/cli.py index e6a7855ce..569e28efe 100644 --- a/datadog/util/cli.py +++ b/datadog/util/cli.py @@ -2,7 +2,7 @@ from argparse import ArgumentTypeError import json import re -from format import force_to_epoch_seconds +from datadog.util.format import force_to_epoch_seconds def comma_list(list_str, item_func=None): @@ -57,6 +57,7 @@ def set_of_ints(int_csv): ## Date handling + class DateParsingError(Exception): """Thrown if parse_date exhausts all possible parsings of a string""" From ad566c0264e66be6b713c8eedc7d473a929d3f2e Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Mon, 21 Oct 2019 16:13:42 -0500 Subject: [PATCH 03/14] pylint --- datadog/util/cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/datadog/util/cli.py b/datadog/util/cli.py index 569e28efe..42c2b6770 100644 --- a/datadog/util/cli.py +++ b/datadog/util/cli.py @@ -55,9 +55,6 @@ def set_of_ints(int_csv): return set(list_of_ints(int_csv)) -## Date handling - - class DateParsingError(Exception): """Thrown if parse_date exhausts all possible parsings of a string""" From 20a22851a35a7c2c2cdcfb4d8b39c91f87438d47 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Mon, 21 Oct 2019 16:31:03 -0500 Subject: [PATCH 04/14] fix dogshell for slo --- datadog/dogshell/service_level_objective.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog/dogshell/service_level_objective.py b/datadog/dogshell/service_level_objective.py index f4f3976f5..c4895e1a9 100644 --- a/datadog/dogshell/service_level_objective.py +++ b/datadog/dogshell/service_level_objective.py @@ -98,7 +98,7 @@ def setup_parser(cls, subparsers): update_parser.add_argument( "--description", help="description of the SLO", default=None ) - create_parser.add_argument( + update_parser.add_argument( "--thresholds", help="comma separated list of :[:[:[:]]", required=True, From da597f2cc1a8b6200bbe4a95dee8b4cebf6e2bd6 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Mon, 21 Oct 2019 16:42:24 -0500 Subject: [PATCH 05/14] remove required it is implicit --- datadog/dogshell/service_level_objective.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/datadog/dogshell/service_level_objective.py b/datadog/dogshell/service_level_objective.py index c4895e1a9..bdc366d95 100644 --- a/datadog/dogshell/service_level_objective.py +++ b/datadog/dogshell/service_level_objective.py @@ -30,7 +30,6 @@ def setup_parser(cls, subparsers): create_parser = verb_parsers.add_parser("create", help="Create a SLO") create_parser.add_argument( "--type", - required=True, help="type of the SLO, e.g.", choices=["metric", "monitor"], ) @@ -47,7 +46,6 @@ def setup_parser(cls, subparsers): create_parser.add_argument( "--thresholds", help="comma separated list of :[:[:[:]]", - required=True, ) create_parser.add_argument( "--numerator", @@ -90,7 +88,6 @@ def setup_parser(cls, subparsers): ) update_parser.add_argument( "--type", - required=True, help="type of the SLO (must specify it's original type)", choices=["metric", "monitor"], ) @@ -101,7 +98,6 @@ def setup_parser(cls, subparsers): update_parser.add_argument( "--thresholds", help="comma separated list of :[:[:[:]]", - required=True, ) update_parser.add_argument( "--tags", @@ -183,7 +179,6 @@ def setup_parser(cls, subparsers): delete_timeframe_parser.add_argument( "timeframes", help="CSV of timeframes to delete, e.g. 7d,30d,90d", - required=True, type=comma_set, ) delete_timeframe_parser.set_defaults(func=cls._delete_timeframe) @@ -200,13 +195,11 @@ def setup_parser(cls, subparsers): history_parser.add_argument("slo_id", help="SLO to query the history") history_parser.add_argument( "from_ts", - required=True, type=parse_date_as_epoch_timestamp, help="`from` date or timestamp", ) history_parser.add_argument( "to_ts", - required=True, type=parse_date_as_epoch_timestamp, help="`to` date or timestamp", ) From 6e8f4538c9098163bee207f13279b2d7dbfad9d1 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Tue, 22 Oct 2019 12:48:31 -0500 Subject: [PATCH 06/14] add util tests add freezegun dependency to do time-unit tests without being flaky --- datadog/util/cli.py | 4 + setup.py | 2 +- tests/unit/util/test_cli.py | 203 ++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 tests/unit/util/test_cli.py diff --git a/datadog/util/cli.py b/datadog/util/cli.py index 42c2b6770..58761e5be 100644 --- a/datadog/util/cli.py +++ b/datadog/util/cli.py @@ -3,6 +3,7 @@ import json import re from datadog.util.format import force_to_epoch_seconds +import time def comma_list(list_str, item_func=None): @@ -72,8 +73,11 @@ def parse_date_as_epoch_timestamp(date_str): def parse_date(date_str): + print("type: {} - {}".format(type(date_str), date_str)) if isinstance(date_str, datetime): return date_str + elif isinstance(date_str, time.struct_time): + return datetime.fromtimestamp(time.mktime(date_str)) # Parse relative dates. if date_str == "today": diff --git a/setup.py b/setup.py index d04067545..ab7be70fc 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def get_readme_md_contents(): name="datadog", version="0.30.0", install_requires=install_reqs, - tests_require=["pytest", "mock"], + tests_require=["pytest", "mock", "freezegun"], packages=["datadog", "datadog.api", "datadog.dogstatsd", "datadog.threadstats", "datadog.util", "datadog.dogshell"], author="Datadog, Inc.", long_description=get_readme_md_contents(), diff --git a/tests/unit/util/test_cli.py b/tests/unit/util/test_cli.py new file mode 100644 index 000000000..b521aadf8 --- /dev/null +++ b/tests/unit/util/test_cli.py @@ -0,0 +1,203 @@ +from argparse import ArgumentTypeError +from freezegun import freeze_time +import calendar +import datetime +import unittest + +from datadog.util.cli import ( + comma_list, + comma_set, + comma_list_or_empty, + list_of_ints, + list_of_ints_and_strs, + set_of_ints, + DateParsingError, + _midnight, + parse_date_as_epoch_timestamp, + parse_date, +) + + +class TestCLI(unittest.TestCase): + def test_comma_list(self): + invalid_cases = [None, ""] + for invalid_case in invalid_cases: + with self.assertRaises(ArgumentTypeError): + comma_list(invalid_case) + + valid_cases = ( + (["foo"], "foo", None), + (["foo", "bar"], "foo,bar", None), + ([1], "1", int), + ([1, 2], "1,2", int), + ) + for expected, list_str, item_func in valid_cases: + actual = comma_list(list_str, item_func) + self.assertListEqual(expected, actual) + + def test_comma_set(self): + invalid_cases = [None, ""] + for invalid_case in invalid_cases: + with self.assertRaises(ArgumentTypeError): + comma_set(invalid_case) + + valid_cases = ( + ({"foo"}, "foo", None), + ({"foo", "bar"}, "foo,bar", None), + ({1}, "1", int), + ({1}, "1,1,1", int), + ({1, 2}, "1,2,1", int), + ) + for expected, list_str, item_func in valid_cases: + actual = comma_set(list_str, item_func) + self.assertSetEqual(expected, actual) + + def test_comma_list_or_empty(self): + valid_cases = ( + ([], None, None), + ([], "", None), + (["foo"], "foo", None), + (["foo", "bar"], "foo,bar", None), + ) + for expected, list_str, item_func in valid_cases: + actual = comma_list_or_empty(list_str) + self.assertListEqual(expected, actual) + + def test_list_of_ints(self): + invalid_cases = [None, "", "foo", '["foo"]'] + for invalid_case in invalid_cases: + with self.assertRaises(ArgumentTypeError): + list_of_ints(invalid_case) + + valid_cases = (([1], "1"), ([1, 2], "1,2"), ([1], "[1]"), ([1, 2], "[1,2]")) + for expected, list_str in valid_cases: + actual = list_of_ints(list_str) + self.assertListEqual(expected, actual) + + def test_list_of_ints_and_strs(self): + invalid_cases = [None, ""] + for invalid_case in invalid_cases: + with self.assertRaises(ArgumentTypeError): + list_of_ints_and_strs(invalid_case) + + valid_cases = ( + (["foo"], "foo"), + (["foo", "bar"], "foo,bar"), + ([1], "1"), + ([1, 2], "1,2"), + (["foo", 2], "foo,2"), + ) + for expected, list_str in valid_cases: + actual = list_of_ints_and_strs(list_str) + self.assertListEqual(expected, actual) + + def test_set_of_ints(self): + invalid_cases = [None, "", "foo", '["foo"]'] + for invalid_case in invalid_cases: + with self.assertRaises(ArgumentTypeError): + set_of_ints(invalid_case) + + valid_cases = ( + ({1}, "1"), + ({1, 2}, "1,2"), + ({1}, "[1]"), + ({1}, "[1,1,1]"), + ({1, 2}, "[1,2,1]"), + ) + for expected, list_str in valid_cases: + actual = set_of_ints(list_str) + self.assertSetEqual(expected, actual) + + @freeze_time("2019-10-23 04:44:32", tz_offset=0) + def test_midnight(self): + d = _midnight() + self.assertEqual(2019, d.year) + self.assertEqual(10, d.month) + self.assertEqual(23, d.day) + self.assertEqual(0, d.hour) + self.assertEqual(0, d.minute) + self.assertEqual(0, d.second) + self.assertEqual(0, d.microsecond) + + @freeze_time("2019-10-23 04:44:32", tz_offset=0) + def test_parse_date(self): + test_date = datetime.datetime(2019, 10, 23, 4, 44, 32, 0) + cases = ( + (test_date, test_date), # already an instance, return + ("today", datetime.datetime(2019, 10, 23, 0, 0, 0)), + ("yesterday", datetime.datetime(2019, 10, 22, 0, 0, 0)), + ("tomorrow", datetime.datetime(2019, 10, 24, 0, 0, 0)), + ("2 days ago", datetime.datetime(2019, 10, 21, 4, 44, 32)), + ("2d ago", datetime.datetime(2019, 10, 21, 4, 44, 32)), + ("2 days ahead", datetime.datetime(2019, 10, 25, 4, 44, 32)), + ("2d ahead", datetime.datetime(2019, 10, 25, 4, 44, 32)), + ("now", datetime.datetime(2019, 10, 23, 4, 44, 32)), + ("2019-10-23 04:44:32.000000", test_date), + ("2019-10-23T04:44:32.000000", test_date), + ("2019-10-23 04:44:32", test_date), + ("2019-10-23T04:44:32", test_date), + ("2019-10-23 04:44", datetime.datetime(2019, 10, 23, 4, 44, 0, 0)), + ("2019-10-23-04", datetime.datetime(2019, 10, 23, 4, 0, 0, 0)), + ("2019-10-23", datetime.datetime(2019, 10, 23, 0, 0, 0, 0)), + ("2019-10", datetime.datetime(2019, 10, 1, 0, 0, 0, 0)), + ("2019", datetime.datetime(2019, 1, 1, 0, 0, 0, 0)), + ("2019-10", datetime.datetime(2019, 10, 1, 0, 0, 0, 0)), + ("1571805872", test_date), # seconds + ("1571805872000", test_date), # millis + ) + + for i, (date_str, expected) in enumerate(cases): + actual = parse_date(date_str) + self.assertEqual( + expected, + actual, + "case {}: failed, date_str={} expected={}".format( + i, date_str, expected + ), + ) + + # test invalid case + with self.assertRaises(DateParsingError): + parse_date("foo") + + @freeze_time("2019-10-23 04:44:32", tz_offset=0) + def test_parse_date_as_epoch_timestamp(self): + # this applies the same rules but always returns epoch seconds + test_date = datetime.datetime(2019, 10, 23, 4, 44, 32, 0) + cases = ( + (test_date, test_date), # already an instance, return + ("today", datetime.datetime(2019, 10, 23, 0, 0, 0)), + ("yesterday", datetime.datetime(2019, 10, 22, 0, 0, 0)), + ("tomorrow", datetime.datetime(2019, 10, 24, 0, 0, 0)), + ("2 days ago", datetime.datetime(2019, 10, 21, 4, 44, 32)), + ("2d ago", datetime.datetime(2019, 10, 21, 4, 44, 32)), + ("2 days ahead", datetime.datetime(2019, 10, 25, 4, 44, 32)), + ("2d ahead", datetime.datetime(2019, 10, 25, 4, 44, 32)), + ("now", datetime.datetime(2019, 10, 23, 4, 44, 32)), + ("2019-10-23 04:44:32.000000", test_date), + ("2019-10-23T04:44:32.000000", test_date), + ("2019-10-23 04:44:32", test_date), + ("2019-10-23T04:44:32", test_date), + ("2019-10-23 04:44", datetime.datetime(2019, 10, 23, 4, 44, 0, 0)), + ("2019-10-23-04", datetime.datetime(2019, 10, 23, 4, 0, 0, 0)), + ("2019-10-23", datetime.datetime(2019, 10, 23, 0, 0, 0, 0)), + ("2019-10", datetime.datetime(2019, 10, 1, 0, 0, 0, 0)), + ("2019", datetime.datetime(2019, 1, 1, 0, 0, 0, 0)), + ("2019-10", datetime.datetime(2019, 10, 1, 0, 0, 0, 0)), + ("1571805872", test_date), # seconds + ("1571805872000", test_date), # millis + ) + + for i, (date_str, expected) in enumerate(cases): + actual = parse_date_as_epoch_timestamp(date_str) + self.assertEqual( + calendar.timegm(expected.utctimetuple()), + actual, + "case {}: failed, date_str={} expected={}".format( + i, date_str, expected + ), + ) + + # test invalid case + with self.assertRaises(DateParsingError): + parse_date_as_epoch_timestamp("foo") From f20c3db3712dce006357271bd9129cd331ac4458 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Tue, 22 Oct 2019 12:49:42 -0500 Subject: [PATCH 07/14] remove debug statement --- datadog/util/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datadog/util/cli.py b/datadog/util/cli.py index 58761e5be..bd16f403b 100644 --- a/datadog/util/cli.py +++ b/datadog/util/cli.py @@ -73,7 +73,6 @@ def parse_date_as_epoch_timestamp(date_str): def parse_date(date_str): - print("type: {} - {}".format(type(date_str), date_str)) if isinstance(date_str, datetime): return date_str elif isinstance(date_str, time.struct_time): From fef0049a5bec334bef25c2f37b6e4bbc1752271d Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Tue, 22 Oct 2019 16:38:35 -0500 Subject: [PATCH 08/14] add tox deps --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index 5612d31c3..1eae44714 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ passenv = DD_TEST_CLIENT* usedevelop = true deps = !integration: mock + freezegun pytest commands = !integration: pytest -v tests/unit {posargs} @@ -20,6 +21,7 @@ commands = passenv = DD_TEST_CLIENT* usedevelop = true deps = + freezegun pytest commands = pytest -v tests/integration -m "admin_needed" {posargs} @@ -27,6 +29,7 @@ commands = [testenv:flake8] skip_install = true deps = + freezegun flake8==3.4.1 commands = flake8 datadog From 2fd1e7cd3dd2b6af68e652c6b74c088f5b0689b8 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Wed, 23 Oct 2019 08:56:45 -0500 Subject: [PATCH 09/14] Update tox.ini Co-Authored-By: Hippolyte HENRY --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1eae44714..f63073e79 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ passenv = DD_TEST_CLIENT* usedevelop = true deps = !integration: mock - freezegun + !integration: freezegun pytest commands = !integration: pytest -v tests/unit {posargs} From 9b3f9895316d928140c94b14e82a797efd30c4e9 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Wed, 23 Oct 2019 08:56:53 -0500 Subject: [PATCH 10/14] Update tox.ini Co-Authored-By: Hippolyte HENRY --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index f63073e79..cc9e5a378 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,6 @@ commands = [testenv:flake8] skip_install = true deps = - freezegun flake8==3.4.1 commands = flake8 datadog From 2abf8e69e86f4dcbe89882ad4998134123b1c6f2 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Wed, 23 Oct 2019 08:57:00 -0500 Subject: [PATCH 11/14] Update tox.ini Co-Authored-By: Hippolyte HENRY --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index cc9e5a378..cf75f2a2a 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,6 @@ commands = passenv = DD_TEST_CLIENT* usedevelop = true deps = - freezegun pytest commands = pytest -v tests/integration -m "admin_needed" {posargs} From a1cc2ddc01d37dfbee3a2fb87e7bb26bbb885498 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Wed, 23 Oct 2019 13:37:08 -0500 Subject: [PATCH 12/14] add more logs for this failure, not able to reproduce locally --- tests/unit/util/test_cli.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/unit/util/test_cli.py b/tests/unit/util/test_cli.py index b521aadf8..143b60f33 100644 --- a/tests/unit/util/test_cli.py +++ b/tests/unit/util/test_cli.py @@ -151,8 +151,8 @@ def test_parse_date(self): self.assertEqual( expected, actual, - "case {}: failed, date_str={} expected={}".format( - i, date_str, expected + "case {}: failed, date_str={} expected={} actual={}".format( + i, date_str, expected, actual ), ) @@ -190,11 +190,12 @@ def test_parse_date_as_epoch_timestamp(self): for i, (date_str, expected) in enumerate(cases): actual = parse_date_as_epoch_timestamp(date_str) + expected_timestamp = calendar.timegm(expected.utctimetuple()) self.assertEqual( - calendar.timegm(expected.utctimetuple()), + expected_timestamp, actual, - "case {}: failed, date_str={} expected={}".format( - i, date_str, expected + "case {}: failed, date_str={} expected={} actual={}".format( + i, date_str, expected_timestamp, actual ), ) From 2018572e0d8e58bc47ecc97ded4c856217850a33 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Wed, 23 Oct 2019 14:29:09 -0500 Subject: [PATCH 13/14] adjust tests to account for pypy --- datadog/util/cli.py | 27 +++++++++++++++++---------- datadog/util/compat.py | 7 +++++++ tests/unit/util/test_cli.py | 29 ++++++++++++++++++----------- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/datadog/util/cli.py b/datadog/util/cli.py index bd16f403b..6fc15ad5c 100644 --- a/datadog/util/cli.py +++ b/datadog/util/cli.py @@ -69,22 +69,26 @@ def _midnight(): def parse_date_as_epoch_timestamp(date_str): - return force_to_epoch_seconds(parse_date(date_str)) + return parse_date(date_str, to_epoch_ts=True) -def parse_date(date_str): +def parse_date(date_str, to_epoch_ts=False): + formatter = lambda d: d + if to_epoch_ts: + formatter = lambda d: force_to_epoch_seconds(d) + if isinstance(date_str, datetime): - return date_str + return formatter(date_str) elif isinstance(date_str, time.struct_time): - return datetime.fromtimestamp(time.mktime(date_str)) + return formatter(datetime.fromtimestamp(time.mktime(date_str))) # Parse relative dates. if date_str == "today": - return _midnight() + return formatter(_midnight()) elif date_str == "yesterday": - return _midnight() - timedelta(days=1) + return formatter(_midnight() - timedelta(days=1)) elif date_str == "tomorrow": - return _midnight() + timedelta(days=1) + return formatter(_midnight() + timedelta(days=1)) elif date_str.endswith(("ago", "ahead")): m = _date_fieldre.match(date_str) if m: @@ -103,15 +107,18 @@ def parse_date(date_str): if unit[-1] != "s": unit += "s" # tolerate 1 hour assert unit in units, "'%s' not in %s" % (unit, units) - return datetime.utcnow() + time_direction * timedelta(**{unit: num}) + return formatter(datetime.utcnow() + time_direction * timedelta(**{unit: num})) elif date_str == "now": - return datetime.utcnow() + return formatter(datetime.utcnow()) def _from_epoch_timestamp(seconds): + print("_from_epoch_timestamp({})".format(seconds)) return datetime.utcfromtimestamp(float(seconds)) def _from_epoch_ms_timestamp(millis): + print("_from_epoch_ms_timestamp({})".format(millis)) in_sec = float(millis) / 1000.0 + print("_from_epoch_ms_timestamp({}) -> {}".format(millis, in_sec)) return _from_epoch_timestamp(in_sec) # Or parse date formats (most specific to least specific) @@ -131,7 +138,7 @@ def _from_epoch_ms_timestamp(millis): for parse_func in parse_funcs: try: - return parse_func(date_str) + return formatter(parse_func(date_str)) except Exception: pass raise DateParsingError(u"Could not parse {0} as date".format(date_str)) diff --git a/datadog/util/compat.py b/datadog/util/compat.py index e15890c42..890b51eff 100644 --- a/datadog/util/compat.py +++ b/datadog/util/compat.py @@ -34,6 +34,13 @@ def is_higher_py35(): return _is_py_version_higher_than(3, 5) +def is_pypy(): + """ + Assert that PyPy is being used (regardless of 2 or 3) + """ + return '__pypy__' in sys.builtin_module_names + + get_input = input # Python 3.x diff --git a/tests/unit/util/test_cli.py b/tests/unit/util/test_cli.py index 143b60f33..d5717dbcf 100644 --- a/tests/unit/util/test_cli.py +++ b/tests/unit/util/test_cli.py @@ -1,6 +1,5 @@ from argparse import ArgumentTypeError from freezegun import freeze_time -import calendar import datetime import unittest @@ -16,6 +15,8 @@ parse_date_as_epoch_timestamp, parse_date, ) +from datadog.util.compat import is_pypy +from datadog.util.format import force_to_epoch_seconds class TestCLI(unittest.TestCase): @@ -122,7 +123,7 @@ def test_midnight(self): @freeze_time("2019-10-23 04:44:32", tz_offset=0) def test_parse_date(self): test_date = datetime.datetime(2019, 10, 23, 4, 44, 32, 0) - cases = ( + cases = [ (test_date, test_date), # already an instance, return ("today", datetime.datetime(2019, 10, 23, 0, 0, 0)), ("yesterday", datetime.datetime(2019, 10, 22, 0, 0, 0)), @@ -143,8 +144,11 @@ def test_parse_date(self): ("2019", datetime.datetime(2019, 1, 1, 0, 0, 0, 0)), ("2019-10", datetime.datetime(2019, 10, 1, 0, 0, 0, 0)), ("1571805872", test_date), # seconds - ("1571805872000", test_date), # millis - ) + ] + if not is_pypy(): + cases.append( + ("1571805872000", test_date) + ) # millis, pypy does not work (known) for i, (date_str, expected) in enumerate(cases): actual = parse_date(date_str) @@ -164,7 +168,7 @@ def test_parse_date(self): def test_parse_date_as_epoch_timestamp(self): # this applies the same rules but always returns epoch seconds test_date = datetime.datetime(2019, 10, 23, 4, 44, 32, 0) - cases = ( + cases = [ (test_date, test_date), # already an instance, return ("today", datetime.datetime(2019, 10, 23, 0, 0, 0)), ("yesterday", datetime.datetime(2019, 10, 22, 0, 0, 0)), @@ -185,17 +189,20 @@ def test_parse_date_as_epoch_timestamp(self): ("2019", datetime.datetime(2019, 1, 1, 0, 0, 0, 0)), ("2019-10", datetime.datetime(2019, 10, 1, 0, 0, 0, 0)), ("1571805872", test_date), # seconds - ("1571805872000", test_date), # millis - ) + ] + if not is_pypy(): + cases.append( + ("1571805872000", test_date) + ) # millis, pypy does not work (known) for i, (date_str, expected) in enumerate(cases): - actual = parse_date_as_epoch_timestamp(date_str) - expected_timestamp = calendar.timegm(expected.utctimetuple()) + actual_timestamp = parse_date_as_epoch_timestamp(date_str) + expected_timestamp = force_to_epoch_seconds(expected) self.assertEqual( expected_timestamp, - actual, + actual_timestamp, "case {}: failed, date_str={} expected={} actual={}".format( - i, date_str, expected_timestamp, actual + i, date_str, expected_timestamp, actual_timestamp ), ) From 48454bb5fdb940b4f62d844b68ef53d3277c76b2 Mon Sep 17 00:00:00 2001 From: Cody Lee Date: Wed, 23 Oct 2019 14:52:04 -0500 Subject: [PATCH 14/14] make pylint happy --- datadog/util/cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/datadog/util/cli.py b/datadog/util/cli.py index 6fc15ad5c..fb3f7eb8e 100644 --- a/datadog/util/cli.py +++ b/datadog/util/cli.py @@ -72,10 +72,15 @@ def parse_date_as_epoch_timestamp(date_str): return parse_date(date_str, to_epoch_ts=True) +def _parse_date_noop_formatter(d): + """ NOOP - only here for pylint """ + return d + + def parse_date(date_str, to_epoch_ts=False): - formatter = lambda d: d + formatter = _parse_date_noop_formatter if to_epoch_ts: - formatter = lambda d: force_to_epoch_seconds(d) + formatter = force_to_epoch_seconds if isinstance(date_str, datetime): return formatter(date_str)