diff --git a/datadog_sync/cli.py b/datadog_sync/cli.py index dc73f8b3..5846f738 100644 --- a/datadog_sync/cli.py +++ b/datadog_sync/cli.py @@ -2,7 +2,6 @@ # under the 3-clause BSD style license (see LICENSE). # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. - import sys from click import group diff --git a/datadog_sync/commands/shared/options.py b/datadog_sync/commands/shared/options.py index be6f30ff..6233a608 100644 --- a/datadog_sync/commands/shared/options.py +++ b/datadog_sync/commands/shared/options.py @@ -2,16 +2,21 @@ # under the 3-clause BSD style license (see LICENSE). # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import configobj from sys import exit from click import Choice, Option, option, File from datadog_sync import constants +from typing import TYPE_CHECKING, Any, Callable, Dict, List + +if TYPE_CHECKING: + from click.core import Context class CustomOptionClass(Option): - def handle_parse_result(self, ctx, opts, args): + def handle_parse_result(self, ctx: Context, opts: Dict[Any, Any], args: List[Any]) -> Any: try: return super(Option, self).handle_parse_result(ctx, opts, args) except Exception as e: @@ -75,7 +80,7 @@ def handle_parse_result(self, ctx, opts, args): ] -def click_config_file_provider(ctx, opts, value): +def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: None) -> None: config = configobj.ConfigObj(value, unrepr=True) ctx.default_map = ctx.default_map or {} ctx.default_map.update(config) @@ -174,23 +179,23 @@ def click_config_file_provider(ctx, opts, value): ] -def source_auth_options(func): +def source_auth_options(func: Callable) -> Callable: return _build_options_helper(func, _source_auth_options) -def destination_auth_options(func): +def destination_auth_options(func: Callable) -> Callable: return _build_options_helper(func, _destination_auth_options) -def common_options(func): +def common_options(func: Callable) -> Callable: return _build_options_helper(func, _common_options) -def non_import_common_options(func): +def non_import_common_options(func: Callable) -> Callable: return _build_options_helper(func, _non_import_common_options) -def _build_options_helper(func, options): +def _build_options_helper(func: Callable, options: List[Callable]) -> Callable: for _option in options: func = _option(func) return func diff --git a/datadog_sync/model/dashboard_lists.py b/datadog_sync/model/dashboard_lists.py index 205b7474..18908a77 100644 --- a/datadog_sync/model/dashboard_lists.py +++ b/datadog_sync/model/dashboard_lists.py @@ -3,13 +3,16 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import copy -from typing import Optional, List, Dict +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient from datadog_sync.utils.resource_utils import CustomClientHTTPError, check_diff +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient + class DashboardLists(BaseResource): resource_type = "dashboard_lists" @@ -19,7 +22,7 @@ class DashboardLists(BaseResource): excluded_attributes=["id", "type", "author", "created", "modified", "is_favorite", "dashboard_count"], ) # Additional Dashboards specific attributes - dash_list_items_path = "/api/v2/dashboard/lists/manual/{}/dashboards" + dash_list_items_path: str = "/api/v2/dashboard/lists/manual/{}/dashboards" def get_resources(self, client: CustomClient) -> List[Dict]: resp = client.get(self.resource_config.base_path).json() @@ -32,6 +35,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = if _id: resource = source_client.get(self.resource_config.base_path + f"/{_id}").json() + resource = cast(dict, resource) _id = str(resource["id"]) resp = None try: diff --git a/datadog_sync/model/dashboards.py b/datadog_sync/model/dashboards.py index cfbbe22c..e02cd0da 100644 --- a/datadog_sync/model/dashboards.py +++ b/datadog_sync/model/dashboards.py @@ -3,10 +3,13 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class Dashboards(BaseResource): @@ -31,8 +34,9 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client import_id = _id or resource["id"] - dashboard = source_client.get(self.resource_config.base_path + f"/{import_id}").json() - self.resource_config.source_resources[import_id] = dashboard + resource = source_client.get(self.resource_config.base_path + f"/{import_id}").json() + resource = cast(dict, resource) + self.resource_config.source_resources[import_id] = resource def pre_resource_action_hook(self, _id, resource: Dict) -> None: pass diff --git a/datadog_sync/model/downtimes.py b/datadog_sync/model/downtimes.py index f896a6bb..f6cf8482 100644 --- a/datadog_sync/model/downtimes.py +++ b/datadog_sync/model/downtimes.py @@ -2,12 +2,15 @@ # under the 3-clause BSD style license (see LICENSE). # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import math -from typing import Optional, List, Dict +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datetime import datetime from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient RECURRING_TIMES = { @@ -48,6 +51,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json() + resource = cast(dict, resource) if resource["canceled"]: return # Dispose the recurring child downtimes and only retain the parent diff --git a/datadog_sync/model/host_tags.py b/datadog_sync/model/host_tags.py index fdfb5562..24f9ce5c 100644 --- a/datadog_sync/model/host_tags.py +++ b/datadog_sync/model/host_tags.py @@ -3,10 +3,13 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class HostTags(BaseResource): @@ -24,6 +27,8 @@ def get_resources(self, client: CustomClient) -> List[Dict]: def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> None: if _id: return # This should never occur. No resource depends on it. + + resource = cast(dict, resource) tag = resource[0] hosts = resource[1] for host in hosts: diff --git a/datadog_sync/model/logs_custom_pipelines.py b/datadog_sync/model/logs_custom_pipelines.py index 83fa34e2..2de00115 100644 --- a/datadog_sync/model/logs_custom_pipelines.py +++ b/datadog_sync/model/logs_custom_pipelines.py @@ -3,10 +3,13 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class LogsCustomPipelines(BaseResource): @@ -28,6 +31,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json() + resource = cast(dict, resource) if resource["is_read_only"]: return self.resource_config.source_resources[resource["id"]] = resource diff --git a/datadog_sync/model/logs_indexes.py b/datadog_sync/model/logs_indexes.py index c814ffa0..c239d8cc 100644 --- a/datadog_sync/model/logs_indexes.py +++ b/datadog_sync/model/logs_indexes.py @@ -3,10 +3,13 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class LogsIndexes(BaseResource): @@ -31,6 +34,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json() + resource = cast(dict, resource) if not resource.get("daily_limit"): resource["disable_daily_limit"] = True self.resource_config.source_resources[resource["name"]] = resource diff --git a/datadog_sync/model/logs_metrics.py b/datadog_sync/model/logs_metrics.py index 7247bea7..7df9ccc1 100644 --- a/datadog_sync/model/logs_metrics.py +++ b/datadog_sync/model/logs_metrics.py @@ -3,10 +3,13 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class LogsMetrics(BaseResource): @@ -26,6 +29,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json()["data"] + resource = cast(dict, resource) self.resource_config.source_resources[resource["id"]] = resource def pre_resource_action_hook(self, _id, resource: Dict) -> None: diff --git a/datadog_sync/model/logs_restriction_queries.py b/datadog_sync/model/logs_restriction_queries.py index dd7c928d..7d3fd6fb 100644 --- a/datadog_sync/model/logs_restriction_queries.py +++ b/datadog_sync/model/logs_restriction_queries.py @@ -3,12 +3,16 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict, Tuple +from __future__ import annotations +from typing import TYPE_CHECKING, Any, Optional, List, Dict, Tuple, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient, PaginationConfig +from datadog_sync.utils.custom_client import PaginationConfig from datadog_sync.utils.resource_utils import CustomClientHTTPError, check_diff +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient + class LogsRestrictionQueries(BaseResource): resource_type = "logs_restriction_queries" @@ -35,7 +39,7 @@ def get_resources(self, client: CustomClient) -> List[Dict]: ) return resp - def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> None: + def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict[str, Any]] = None) -> None: source_client = self.config.source_client import_id = _id or resource["id"] diff --git a/datadog_sync/model/metric_tag_configurations.py b/datadog_sync/model/metric_tag_configurations.py index d846390c..eb035aec 100644 --- a/datadog_sync/model/metric_tag_configurations.py +++ b/datadog_sync/model/metric_tag_configurations.py @@ -3,10 +3,13 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class MetricTagConfigurations(BaseResource): @@ -28,6 +31,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}/tags").json()["data"] + resource = cast(dict, resource) self.resource_config.source_resources[resource["id"]] = resource def pre_resource_action_hook(self, _id, resource: Dict) -> None: diff --git a/datadog_sync/model/monitors.py b/datadog_sync/model/monitors.py index 0925f3a1..07a9bcbd 100644 --- a/datadog_sync/model/monitors.py +++ b/datadog_sync/model/monitors.py @@ -3,12 +3,14 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import re -from typing import Optional, List, Dict +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient -from datadog_sync.utils.resource_utils import ResourceConnectionError + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class Monitors(BaseResource): @@ -41,6 +43,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json() + resource = cast(dict, resource) if resource["type"] in ("synthetics alert", "slo alert"): return @@ -97,6 +100,8 @@ def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optiona failed_connections.append(_id) r_obj[key] = (r_obj[key].replace("#", "")).strip() return failed_connections - elif key != "query": + elif key == "query": + return None + else: # Use default connect_id method in base class when not handling special case for `query` return super(Monitors, self).connect_id(key, r_obj, resource_to_connect) diff --git a/datadog_sync/model/notebooks.py b/datadog_sync/model/notebooks.py index b1786206..49ee59b3 100644 --- a/datadog_sync/model/notebooks.py +++ b/datadog_sync/model/notebooks.py @@ -3,10 +3,14 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient, PaginationConfig +from datadog_sync.utils.custom_client import PaginationConfig + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class Notebooks(BaseResource): @@ -42,6 +46,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json()["data"] + resource = cast(dict, resource) self.handle_special_case_attr(resource) self.resource_config.source_resources[resource["id"]] = resource diff --git a/datadog_sync/model/roles.py b/datadog_sync/model/roles.py index 726f8571..e2fce5b4 100644 --- a/datadog_sync/model/roles.py +++ b/datadog_sync/model/roles.py @@ -3,12 +3,15 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import copy -from typing import Optional, List, Dict +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig from datadog_sync.utils.resource_utils import CustomClientHTTPError, check_diff -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class Roles(BaseResource): @@ -18,10 +21,10 @@ class Roles(BaseResource): excluded_attributes=["id", "attributes.created_at", "attributes.modified_at", "attributes.user_count"], ) # Additional Roles specific attributes - source_permissions = {} - destination_permissions = {} - destination_roles_mapping = None - permissions_base_path = "/api/v2/permissions" + source_permissions: Dict = {} + destination_permissions: Dict = {} + destination_roles_mapping: Optional[Dict] = None + permissions_base_path: str = "/api/v2/permissions" def get_resources(self, client: CustomClient) -> List[Dict]: resp = client.paginated_request(client.get)(self.resource_config.base_path) @@ -40,6 +43,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json()["data"] + resource = cast(dict, resource) if self.source_permissions and "permissions" in resource["relationships"]: for permission in resource["relationships"]["permissions"]["data"]: if permission["id"] in self.source_permissions: diff --git a/datadog_sync/model/service_level_objectives.py b/datadog_sync/model/service_level_objectives.py index 547f774c..032c7e37 100644 --- a/datadog_sync/model/service_level_objectives.py +++ b/datadog_sync/model/service_level_objectives.py @@ -2,11 +2,13 @@ # under the 3-clause BSD style license (see LICENSE). # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient -from datadog_sync.utils.resource_utils import ResourceConnectionError + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class ServiceLevelObjectives(BaseResource): @@ -27,7 +29,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = if _id: source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json()["data"] - + resource = cast(dict, resource) self.resource_config.source_resources[resource["id"]] = resource def pre_resource_action_hook(self, _id, resource: Dict) -> None: diff --git a/datadog_sync/model/slo_corrections.py b/datadog_sync/model/slo_corrections.py index 3ce7021d..8127e174 100644 --- a/datadog_sync/model/slo_corrections.py +++ b/datadog_sync/model/slo_corrections.py @@ -3,11 +3,14 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datetime import datetime from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class SLOCorrections(BaseResource): @@ -30,6 +33,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json()["data"] + resource = cast(dict, resource) if resource["attributes"].get("end", False): if (round(datetime.now().timestamp()) - int(resource["attributes"]["end"])) / 86400 > 90: return diff --git a/datadog_sync/model/synthetics_global_variables.py b/datadog_sync/model/synthetics_global_variables.py index 0ad68161..862e44d5 100644 --- a/datadog_sync/model/synthetics_global_variables.py +++ b/datadog_sync/model/synthetics_global_variables.py @@ -3,11 +3,13 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient -from datadog_sync.utils.resource_utils import ResourceConnectionError + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class SyntheticsGlobalVariables(BaseResource): @@ -41,6 +43,7 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json() + resource = cast(dict, resource) self.resource_config.source_resources[resource["id"]] = resource def pre_resource_action_hook(self, _id, resource: Dict) -> None: diff --git a/datadog_sync/model/synthetics_private_locations.py b/datadog_sync/model/synthetics_private_locations.py index 09ffffc4..bacfd4e4 100644 --- a/datadog_sync/model/synthetics_private_locations.py +++ b/datadog_sync/model/synthetics_private_locations.py @@ -3,12 +3,15 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import re -from typing import List, Dict, Optional +from typing import TYPE_CHECKING, List, Dict, Optional from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient class SyntheticsPrivateLocations(BaseResource): @@ -18,8 +21,8 @@ class SyntheticsPrivateLocations(BaseResource): excluded_attributes=["id", "modifiedAt", "createdAt", "createdBy", "metadata", "secrets", "config"], ) # Additional SyntheticsPrivateLocations specific attributes - base_locations_path = "/api/v1/synthetics/locations" - pl_id_regex = re.compile("^pl:.*") + base_locations_path: str = "/api/v1/synthetics/locations" + pl_id_regex: re.Pattern = re.compile("^pl:.*") def get_resources(self, client: CustomClient) -> List[Dict]: resp = client.get(self.base_locations_path).json() @@ -32,7 +35,6 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = if self.pl_id_regex.match(import_id): pl = source_client.get(self.resource_config.base_path + f"/{import_id}").json() - self.resource_config.source_resources[import_id] = pl def pre_resource_action_hook(self, _id, resource: Dict) -> None: diff --git a/datadog_sync/model/synthetics_tests.py b/datadog_sync/model/synthetics_tests.py index 38765773..c52c8bdf 100644 --- a/datadog_sync/model/synthetics_tests.py +++ b/datadog_sync/model/synthetics_tests.py @@ -3,11 +3,14 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Any, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient -from datadog_sync.utils.resource_utils import ResourceConnectionError + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient + from datadog_sync.model.synthetics_private_locations import SyntheticsPrivateLocations class SyntheticsTests(BaseResource): @@ -30,8 +33,8 @@ class SyntheticsTests(BaseResource): ], ) # Additional SyntheticsTests specific attributes - browser_test_path = "/api/v1/synthetics/tests/browser/{}" - api_test_path = "/api/v1/synthetics/tests/api/{}" + browser_test_path: str = "/api/v1/synthetics/tests/browser/{}" + api_test_path: str = "/api/v1/synthetics/tests/api/{}" def get_resources(self, client: CustomClient) -> List[Dict]: resp = client.get(self.resource_config.base_path).json() @@ -46,12 +49,14 @@ def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = except Exception: resource = source_client.get(self.api_test_path.format(_id)).json() + resource = cast(dict, resource) _id = resource["public_id"] if resource.get("type") == "browser": resource = source_client.get(self.browser_test_path.format(_id)).json() elif resource.get("type") == "api": resource = source_client.get(self.api_test_path.format(_id)).json() + resource = cast(dict, resource) self.resource_config.source_resources[f"{resource['public_id']}#{resource['monitor_id']}"] = resource def pre_resource_action_hook(self, _id, resource: Dict) -> None: @@ -81,7 +86,7 @@ def delete_resource(self, _id: str) -> None: destination_client.post(self.resource_config.base_path + "/delete", body) def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]: - failed_connections = [] + failed_connections: List[str] = [] if resource_to_connect == "synthetics_private_locations": pl = self.config.resources["synthetics_private_locations"] resources = self.config.resources[resource_to_connect].resource_config.destination_resources @@ -109,7 +114,7 @@ def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optiona return super(SyntheticsTests, self).connect_id(key, r_obj, resource_to_connect) @staticmethod - def remove_global_variables_from_config(resource: Dict) -> Dict: + def remove_global_variables_from_config(resource: Dict[str, Any]) -> Dict[str, Any]: if "config" in resource and "configVariables" in resource["config"]: for variables in resource["config"]["configVariables"]: if variables["type"] == "global": diff --git a/datadog_sync/model/users.py b/datadog_sync/model/users.py index b217b42d..e9997c42 100644 --- a/datadog_sync/model/users.py +++ b/datadog_sync/model/users.py @@ -3,12 +3,15 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. -from typing import Optional, List, Dict +from __future__ import annotations +from typing import TYPE_CHECKING, Any, Optional, List, Dict, cast from datadog_sync.utils.base_resource import BaseResource, ResourceConfig -from datadog_sync.utils.custom_client import CustomClient from datadog_sync.utils.resource_utils import CustomClientHTTPError, check_diff +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient + class Users(BaseResource): resource_type = "users" @@ -38,11 +41,12 @@ def get_resources(self, client: CustomClient) -> List[Dict]: return resp - def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> None: + def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict[str, Any]] = None) -> None: if _id: source_client = self.config.source_client resource = source_client.get(self.resource_config.base_path + f"/{_id}").json()["data"] + resource = cast(dict, resource) if resource["attributes"]["disabled"]: return diff --git a/datadog_sync/utils/base_resource.py b/datadog_sync/utils/base_resource.py index 979e7c67..5ad2fcc2 100644 --- a/datadog_sync/utils/base_resource.py +++ b/datadog_sync/utils/base_resource.py @@ -3,18 +3,18 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import abc from collections import defaultdict from dataclasses import dataclass, field from pprint import pformat -from typing import Optional, Dict, List +from typing import TYPE_CHECKING, Optional, Dict, List from datadog_sync.utils.custom_client import CustomClient -from datadog_sync.utils.resource_utils import ( - open_resources, - find_attr, - ResourceConnectionError, -) +from datadog_sync.utils.resource_utils import open_resources, find_attr, ResourceConnectionError + +if TYPE_CHECKING: + from datadog_sync.utils.configuration import Configuration @dataclass @@ -28,10 +28,10 @@ class ResourceConfig: source_resources: dict = field(default_factory=dict) destination_resources: dict = field(default_factory=dict) - def __post_init__(self): + def __post_init__(self) -> None: self.build_excluded_attributes() - def build_excluded_attributes(self): + def build_excluded_attributes(self) -> None: if self.excluded_attributes: for i, attr in enumerate(self.excluded_attributes): self.excluded_attributes[i] = "root" + "".join(["['{}']".format(v) for v in attr.split(".")]) @@ -41,7 +41,7 @@ class BaseResource(abc.ABC): resource_type: str resource_config: ResourceConfig - def __init__(self, config): + def __init__(self, config: Configuration) -> None: self.config = config self.resource_config.source_resources, self.resource_config.destination_resources = open_resources( self.resource_type diff --git a/datadog_sync/utils/configuration.py b/datadog_sync/utils/configuration.py index 0d93d680..0390b98b 100644 --- a/datadog_sync/utils/configuration.py +++ b/datadog_sync/utils/configuration.py @@ -3,16 +3,10 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import logging -from sys import exit -from dataclasses import dataclass -from typing import ( - Any, - Union, - Dict, - List, - Optional, -) +from dataclasses import dataclass, field +from typing import Any, Optional, Union, Dict, List from datadog_sync import models from datadog_sync.utils.custom_client import CustomClient @@ -25,20 +19,20 @@ @dataclass class Configuration(object): - logger: Union[Log, logging.Logger, None] = None - source_client: Optional[CustomClient] = None - destination_client: Optional[CustomClient] = None - resources: Optional[Dict[str, BaseResource]] = None - resources_arg: Optional[List[str]] = None - filters: Optional[Dict[str, Filter]] = None - filter_operator: Optional[str] = None - force_missing_dependencies: Optional[bool] = None - skip_failed_resource_connections: Optional[bool] = None - max_workers: Optional[int] = None - cleanup: Optional[int] = None - - -def build_config(cmd, **kwargs: Any) -> Configuration: + logger: Union[Log, logging.Logger] + source_client: CustomClient + destination_client: CustomClient + filters: Dict[str, List[Filter]] + filter_operator: str + force_missing_dependencies: bool + skip_failed_resource_connections: bool + max_workers: int + cleanup: int + resources: Dict[str, BaseResource] = field(default_factory=dict) + resources_arg: List[str] = field(default_factory=list) + + +def build_config(cmd: str, **kwargs: Optional[Any]) -> Configuration: # configure logger logger = Log(kwargs.get("verbose")) @@ -130,7 +124,7 @@ def init_resources(cfg: Configuration) -> Dict[str, BaseResource]: return resources -def _validate_client(client: CustomClient): +def _validate_client(client: CustomClient) -> None: try: client.get(VALIDATE_ENDPOINT).json() except CustomClientHTTPError as e: diff --git a/datadog_sync/utils/custom_client.py b/datadog_sync/utils/custom_client.py index 404a5550..fb96ebd8 100644 --- a/datadog_sync/utils/custom_client.py +++ b/datadog_sync/utils/custom_client.py @@ -2,12 +2,11 @@ # under the 3-clause BSD style license (see LICENSE). # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. - import time import logging import platform from dataclasses import dataclass -from typing import Optional, Callable +from typing import Dict, Optional, Callable import requests @@ -17,7 +16,7 @@ log = logging.getLogger(LOGGER_NAME) -def request_with_retry(func): +def request_with_retry(func: Callable) -> Callable: def wrapper(*args, **kwargs): retry = True default_backoff = 5 @@ -58,7 +57,7 @@ def wrapper(*args, **kwargs): class CustomClient: - def __init__(self, host, auth, retry_timeout): + def __init__(self, host: Optional[str], auth: Dict[str, str], retry_timeout: int) -> None: self.host = host self.timeout = 30 self.session = requests.Session() @@ -91,7 +90,7 @@ def delete(self, path, body=None, **kwargs): url = self.host + path return self.session.delete(url, json=body, timeout=self.timeout, **kwargs) - def paginated_request(self, func): + def paginated_request(self, func: Callable) -> Callable: def wrapper(*args, **kwargs): pagination_config = kwargs.pop("pagination_config", self.default_pagination) @@ -124,7 +123,7 @@ def wrapper(*args, **kwargs): return wrapper -def build_default_headers(auth_obj): +def build_default_headers(auth_obj: Dict[str, str]) -> Dict[str, str]: headers = { "DD-API-KEY": auth_obj["apiKeyAuth"], "DD-APPLICATION-KEY": auth_obj["appKeyAuth"], @@ -134,7 +133,7 @@ def build_default_headers(auth_obj): return headers -def _get_user_agent(): +def _get_user_agent() -> str: from datadog_sync._version import __version__ as version return "datadog-sync-cli/{version} (python {pyver}; os {os}; arch {arch})".format( diff --git a/datadog_sync/utils/filter.py b/datadog_sync/utils/filter.py index 784e06b4..fd71ecaf 100644 --- a/datadog_sync/utils/filter.py +++ b/datadog_sync/utils/filter.py @@ -3,10 +3,12 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import logging from re import match from datadog_sync.constants import LOGGER_NAME +from typing import TYPE_CHECKING, Any, Dict, List, Tuple FILTER_TYPE = "Type" @@ -20,7 +22,7 @@ class Filter: - def __init__(self, resource_type, attr_name, attr_re): + def __init__(self, resource_type: str, attr_name: str, attr_re: str): self.resource_type = resource_type self.attr_name = attr_name.split(".") self.attr_re = attr_re @@ -53,8 +55,8 @@ def _is_match(self, value): return match(self.attr_re, str(value)) is not None -def process_filters(filter_list): - filters = {} +def process_filters(filter_list: List[str]) -> Dict[str, List[Filter]]: + filters: Dict[str, List[Filter]] = {} if not filter_list: return filters @@ -63,18 +65,25 @@ def process_filters(filter_list): f_dict = {} f_list = _filter.strip("; ").split(";") + invalid_filter = False for option in f_list: try: f_dict.update(dict([option.split("=", 1)])) except ValueError: log.warning("invalid filter option: %s, filter: %s", option, _filter) - return + invalid_filter = True + break + if invalid_filter: + continue # Check if required keys are present: for k in REQUIRED_KEYS: if k not in f_dict: log.warning("invalid filter missing key %s in filter: %s", k, _filter) - return + invalid_filter = True + break + if invalid_filter: + continue # Build and assign regex matcher to VALUE key f_dict[FILTER_VALUE] = build_regex(f_dict) diff --git a/datadog_sync/utils/log.py b/datadog_sync/utils/log.py index ddab0792..9062f3d0 100644 --- a/datadog_sync/utils/log.py +++ b/datadog_sync/utils/log.py @@ -3,12 +3,14 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import logging from datadog_sync.constants import LOGGER_NAME +from typing import TYPE_CHECKING -def _configure_logging(verbose): +def _configure_logging(verbose: bool) -> None: # Set logging level and format _format = "%(asctime)s - %(levelname)s - %(message)s" if verbose: @@ -18,7 +20,7 @@ def _configure_logging(verbose): class Log: - def __init__(self, verbose): + def __init__(self, verbose: bool) -> None: _configure_logging(verbose) self.exception_logged = False @@ -36,7 +38,7 @@ def error(self, msg, *arg): self._exception_logged() self.logger.error(msg, *arg) - def info(self, msg, *arg): + def info(self, msg: str, *arg) -> None: self.logger.info(msg, *arg) def warning(self, msg, *arg): diff --git a/datadog_sync/utils/resource_utils.py b/datadog_sync/utils/resource_utils.py index 8699463b..04588675 100644 --- a/datadog_sync/utils/resource_utils.py +++ b/datadog_sync/utils/resource_utils.py @@ -3,6 +3,7 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations import os import re import json @@ -14,6 +15,10 @@ from datadog_sync.constants import RESOURCE_FILE_PATH, LOGGER_NAME from datadog_sync.constants import SOURCE_ORIGIN, DESTINATION_ORIGIN +from typing import Callable, List, Optional, Set, TYPE_CHECKING, Any, Dict, Tuple + +if TYPE_CHECKING: + from datadog_sync.utils.configuration import Configuration log = logging.getLogger(LOGGER_NAME) @@ -34,25 +39,26 @@ class LoggedException(Exception): """Raise this when an error was already logged.""" -def find_attr(keys_list, resource_to_connect, r_obj, connect_func): +def find_attr(keys_list_str: str, resource_to_connect: str, r_obj: Any, connect_func: Callable) -> Optional[List[str]]: if isinstance(r_obj, list): failed_connections = [] for k in r_obj: - failed = find_attr(keys_list, resource_to_connect, k, connect_func) + failed = find_attr(keys_list_str, resource_to_connect, k, connect_func) if failed: failed_connections.extend(failed) return failed_connections else: - keys_list = keys_list.split(".", 1) + keys_list = keys_list_str.split(".", 1) if len(keys_list) == 1 and keys_list[0] in r_obj: if not r_obj[keys_list[0]]: - return + return None return connect_func(keys_list[0], r_obj, resource_to_connect) if isinstance(r_obj, dict): if keys_list[0] in r_obj: return find_attr(keys_list[1], resource_to_connect, r_obj[keys_list[0]], connect_func) + return None def prep_resource(resource_config, resource): @@ -100,7 +106,7 @@ def check_diff(resource_config, resource, state): ) -def open_resources(resource_type): +def open_resources(resource_type: str) -> Tuple[Dict[Any, Any], Dict[Any, Any]]: source_resources = dict() destination_resources = dict() @@ -124,7 +130,7 @@ def open_resources(resource_type): return source_resources, destination_resources -def dump_resources(config, resource_types, origin): +def dump_resources(config: Configuration, resource_types: Set[str], origin: str) -> None: for resource_type in resource_types: if origin == SOURCE_ORIGIN: resources = config.resources[resource_type].resource_config.source_resources @@ -134,18 +140,18 @@ def dump_resources(config, resource_types, origin): write_resources_file(resource_type, origin, resources) -def write_resources_file(resource_type, origin, resources): +def write_resources_file(resource_type: str, origin: str, resources: Any) -> None: resource_path = RESOURCE_FILE_PATH.format(origin, resource_type) with open(resource_path, "w") as f: json.dump(resources, f, indent=2) -def thread_pool_executor(max_workers=None): +def thread_pool_executor(max_workers: Optional[int] = None) -> ThreadPoolExecutor: return ThreadPoolExecutor(max_workers=max_workers) -def init_topological_sorter(graph): +def init_topological_sorter(graph: Dict[str, Set[str]]) -> TopologicalSorter: sorter = TopologicalSorter(graph) sorter.prepare() return sorter diff --git a/datadog_sync/utils/resources_handler.py b/datadog_sync/utils/resources_handler.py index 7b2b954d..1f5e3e77 100644 --- a/datadog_sync/utils/resources_handler.py +++ b/datadog_sync/utils/resources_handler.py @@ -3,6 +3,7 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations from collections import deque from concurrent.futures import wait @@ -23,19 +24,24 @@ init_topological_sorter, write_resources_file, ) +from typing import Dict, TYPE_CHECKING, Optional, Tuple + +if TYPE_CHECKING: + from datadog_sync.utils.configuration import Configuration + from graphlib import TopologicalSorter class ResourcesHandler: - def __init__(self, config, init_manager=True) -> None: + def __init__(self, config: Configuration, init_manager: bool = True) -> None: self.config = config # Additional config for resource manager if init_manager: - self.resources_manager = ResourcesManager(config) - self.resource_done_queue = deque() - self.sorter = None + self.resources_manager: ResourcesManager = ResourcesManager(config) + self.resource_done_queue: deque = deque() + self.sorter: Optional[TopologicalSorter] = None - def apply_resources(self): + def apply_resources(self) -> Tuple[int, int]: # Init executors parralel_executor = thread_pool_executor(self.config.max_workers) serial_executor = thread_pool_executor(1) @@ -147,13 +153,13 @@ def apply_resources(self): return successes, errors - def import_resources(self): + def import_resources(self) -> None: for resource_type in self.config.resources_arg: self.config.logger.info("Importing %s", resource_type) successes, errors = self._import_resources_helper(resource_type) self.config.logger.info(f"Finished importing {resource_type}: {successes} successes, {errors} errors") - def diffs(self): + def diffs(self) -> None: executor = thread_pool_executor(self.config.max_workers) futures = [] for _id, resource_type in self.resources_manager.all_resources.items(): @@ -163,7 +169,7 @@ def diffs(self): futures.append(executor.submit(self._diffs_worker, _id, resource_type, delete=True)) wait(futures) - def _diffs_worker(self, _id, resource_type, delete=False): + def _diffs_worker(self, _id, resource_type, delete=False) -> None: r_class = self.config.resources[resource_type] if delete: @@ -193,14 +199,14 @@ def _diffs_worker(self, _id, resource_type, delete=False): else: print("Resource to be added {} source ID {}: \n {}".format(resource_type, _id, pformat(resource))) - def _import_resources_helper(self, resource_type): + def _import_resources_helper(self, resource_type: str) -> Tuple[int, int]: r_class = self.config.resources[resource_type] r_class.resource_config.source_resources.clear() try: get_resp = r_class.get_resources(self.config.source_client) except Exception as e: - self.config.logger.error(f"Error while importing resources {self.resource_type}: {str(e)}") + self.config.logger.error(f"Error while importing resources {resource_type}: {str(e)}") return 0, 0 futures = [] @@ -223,7 +229,7 @@ def _import_resources_helper(self, resource_type): write_resources_file(resource_type, SOURCE_ORIGIN, r_class.resource_config.source_resources) return successes, errors - def _apply_resource_worker(self, _id, resource_type): + def _apply_resource_worker(self, _id: str, resource_type: str) -> None: try: r_class = self.config.resources[resource_type] resource = self.config.resources[resource_type].resource_config.source_resources[_id] @@ -265,7 +271,7 @@ def _apply_resource_worker(self, _id, resource_type): # always place in done queue regardless of exception thrown self.resource_done_queue.append(_id) - def _force_missing_dep_import_worker(self, _id, resource_type): + def _force_missing_dep_import_worker(self, _id: str, resource_type: str): try: self.config.resources[resource_type].import_resource(_id=_id) except CustomClientHTTPError as e: @@ -277,24 +283,24 @@ def _force_missing_dep_import_worker(self, _id, resource_type): _id, resource_type ) - def _cleanup_worker(self, _id, resource_type): + def _cleanup_worker(self, _id: str, resource_type: str) -> None: self.config.logger.info(f"deleting resource type {resource_type} with id: {_id}") try: self.config.resources[resource_type].delete_resource(_id) self.config.resources[resource_type].resource_config.destination_resources.pop(_id, None) self.config.logger.info(f"succesffully deleted resource type {resource_type} with id: {_id}") - except Exception as e: + except CustomClientHTTPError as e: if e.status_code == 404: self.config.resources[resource_type].resource_config.destination_resources.pop(_id, None) - return + return None self.config.logger.error( - f"Error while deleting resource {self.resource_type}. source ID: {_id} - Error: {str(e)}" + f"Error while deleting resource {resource_type}. source ID: {_id} - Error: {str(e)}" ) raise LoggedException(e) -def _cleanup_prompt(config, resources_to_cleanup, prompt=True): +def _cleanup_prompt(config: Configuration, resources_to_cleanup: Dict[str, str], prompt: bool = True) -> bool: if config.cleanup == FORCE or not prompt: return True elif config.cleanup == TRUE: @@ -305,3 +311,5 @@ def _cleanup_prompt(config, resources_to_cleanup, prompt=True): ) return confirm("Delete above resources from destination org?") + else: + return False diff --git a/datadog_sync/utils/resources_manager.py b/datadog_sync/utils/resources_manager.py index 2065f11d..067d31da 100644 --- a/datadog_sync/utils/resources_manager.py +++ b/datadog_sync/utils/resources_manager.py @@ -3,21 +3,25 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2019 Datadog, Inc. +from __future__ import annotations from copy import deepcopy from collections import deque -from typing import Set +from typing import TYPE_CHECKING, Dict, List, Set from datadog_sync.constants import FALSE from datadog_sync.utils.resource_utils import find_attr +if TYPE_CHECKING: + from datadog_sync.utils.configuration import Configuration + class ResourcesManager: - def __init__(self, config): - self.config = config - self.all_resources = {} # mapping of all resources to its resource_type - self.all_cleanup_resources = {} # mapping of all resources to cleanup - self.dependencies_graph = {} # dependency graph - self.missing_resources_queue = deque() # queue for missing resources + def __init__(self, config: Configuration) -> None: + self.config: Configuration = config + self.all_resources: Dict[str, str] = {} # mapping of all resources to its resource_type + self.all_cleanup_resources: Dict[str, str] = {} # mapping of all resources to cleanup + self.dependencies_graph: Dict[str, Set[str]] = {} # dependency graph + self.missing_resources_queue: deque = deque() # queue for missing resources for resource_type in config.resources_arg: for _id, _ in config.resources[resource_type].resource_config.source_resources.items(): @@ -36,27 +40,29 @@ def __init__(self, config): self.all_cleanup_resources[cleanup_id] = resource_type def _resource_connections(self, _id: str, resource_type: str) -> Set[str]: - failed_connections = [] + failed_connections: List[str] = [] if not self.config.resources[resource_type].resource_config.resource_connections: return set(failed_connections) resource = deepcopy(self.config.resources[resource_type].resource_config.source_resources[_id]) - for resource_to_connect, v in self.config.resources[resource_type].resource_config.resource_connections.items(): - for attr_connection in v: - failed = find_attr( - attr_connection, - resource_to_connect, - resource, - self.config.resources[resource_type].connect_id, - ) - if failed: - # After retrieving all of the failed connections, we check if - # the resources are imported. Otherwise append to missing with its type. - for f_id in failed: - if f_id not in self.config.resources[resource_to_connect].resource_config.source_resources: - self.missing_resources_queue.append((f_id, resource_to_connect)) - - failed_connections.extend(failed) + if self.config.resources[resource_type].resource_config.resource_connections: + for resource_to_connect, v in self.config.resources[ + resource_type + ].resource_config.resource_connections.items(): + for attr_connection in v: + failed = find_attr( + attr_connection, + resource_to_connect, + resource, + self.config.resources[resource_type].connect_id, + ) + if failed: + # After retrieving all of the failed connections, we check if + # the resources are imported. Otherwise append to missing with its type. + for f_id in failed: + if f_id not in self.config.resources[resource_to_connect].resource_config.source_resources: + self.missing_resources_queue.append((f_id, resource_to_connect)) + failed_connections.extend(failed) return set(failed_connections) diff --git a/setup.cfg b/setup.cfg index 78f6f86f..b2039a43 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,3 +51,6 @@ tests = pytest-console-scripts pytest-recording python-dateutil + +[mypy] +ignore_missing_imports = true diff --git a/tests/conftest.py b/tests/conftest.py index f75d46c4..114e2f29 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ from datadog_sync.utils.configuration import Configuration from datadog_sync import constants from datadog_sync.utils.configuration import init_resources +from datadog_sync.utils.custom_client import CustomClient PATTERN_DOUBLE_UNDERSCORE = re.compile(r"__+") @@ -116,10 +117,18 @@ def vcr_config(): @pytest.fixture(scope="module") def config(): max_workers = os.getenv(constants.MAX_WORKERS) + custom_client = CustomClient(None, {"apiKeyAuth": "123", "appKeyAuth": "123"}, None) cfg = Configuration( logger=logging.getLogger(__name__), max_workers=int(max_workers), + source_client=custom_client, + destination_client=custom_client, + filters={}, + filter_operator="OR", + force_missing_dependencies=False, + skip_failed_resource_connections=True, + cleanup=False, ) resources = init_resources(cfg)