Add support for writing to an S3 bucket

datadog_sync/commands/_import.py

@@ -9,6 +9,7 @@
     common_options,
     destination_auth_options,
     source_auth_options,
+    storage_options,
 )
 from datadog_sync.commands.shared.utils import run_cmd
 from datadog_sync.constants import Command
@@ -18,6 +19,7 @@
 @source_auth_options
 @destination_auth_options
 @common_options
+@storage_options
 def _import(**kwargs):
     """Import Datadog resources."""
     run_cmd(Command.IMPORT, **kwargs)

datadog_sync/commands/diffs.py

@@ -10,6 +10,7 @@
     destination_auth_options,
     diffs_common_options,
     source_auth_options,
+    storage_options,
 )
 from datadog_sync.commands.shared.utils import run_cmd
 from datadog_sync.constants import Command
@@ -20,6 +21,7 @@
 @destination_auth_options
 @common_options
 @diffs_common_options
+@storage_options
 def diffs(**kwargs):
     """Log Datadog resources diffs."""
     run_cmd(Command.DIFFS, **kwargs)

datadog_sync/commands/migrate.py

@@ -11,6 +11,7 @@
     diffs_common_options,
     source_auth_options,
     sync_common_options,
+    storage_options,
 )
 from datadog_sync.commands.shared.utils import run_cmd
 from datadog_sync.constants import Command
@@ -22,6 +23,7 @@
 @common_options
 @diffs_common_options
 @sync_common_options
+@storage_options
 def migrate(**kwargs):
     """Migrate Datadog resources from one datacenter to another."""
     run_cmd(Command.MIGRATE, **kwargs)

datadog_sync/commands/reset.py

@@ -9,6 +9,7 @@
     CustomOptionClass,
     common_options,
     destination_auth_options,
+    storage_options,
 )
 from datadog_sync.commands.shared.utils import run_cmd
 from datadog_sync.constants import Command
@@ -17,6 +18,7 @@
 @command(Command.RESET.value, short_help="WARNING: Reset Datadog resources by deleting them.")
 @destination_auth_options
 @common_options
+@storage_options
 @option(
     "--do-not-backup",
     required=False,

datadog_sync/commands/shared/options.py

@@ -187,30 +187,99 @@ def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: Non
         help="Enables sync-cli metrics being sent to both source and destination",
         cls=CustomOptionClass,
     ),
+]
+
+_storage_options = [
+    option(
+        "--storage-type",
+        default=constants.LOCAL_STORAGE_TYPE,
+        show_default=True,
+        type=Choice(
+            constants.STORAGE_TYPES,
+            case_sensitive=False,
+        ),
+        help=f"Set to one of {constants.STORAGE_TYPES} to specify which type of storage to use",
+        cls=CustomOptionClass,
+    ),
     option(
         "--source-resources-path",
+        default=constants.SOURCE_PATH_DEFAULT,
+        show_default=True,
         envvar=constants.DD_SOURCE_RESOURCES_PATH,
         type=Path(
             file_okay=False,
             dir_okay=True,
             resolve_path=True,
         ),
         required=False,
-        help="Specify the local path to the source resources",
-        default=constants.SOURCE_PATH_DEFAULT,
+        help=f"Path to the source resources, only used if --storage-type is '{constants.LOCAL_STORAGE_TYPE}'",
         cls=CustomOptionClass,
     ),
     option(
         "--destination-resources-path",
+        default=constants.DESTINATION_PATH_DEFAULT,
+        show_default=True,
         envvar=constants.DD_DESTINATION_RESOURCES_PATH,
         type=Path(
             file_okay=False,
             dir_okay=True,
             resolve_path=True,
         ),
         required=False,
-        help="Specify the local path to the destination resources",
+        help=f"Path to the destination resources, only used if --storage-type is '{constants.LOCAL_STORAGE_TYPE}'",
+        cls=CustomOptionClass,
+    ),
+    option(
+        "--aws-access-key-id",
+        envvar=constants.AWS_ACCESS_KEY_ID,
+        required=False,
+        help=f"AWS access key id, only used if --storage-type is '{constants.S3_STORAGE_TYPE}'",
+        cls=CustomOptionClass,
+    ),
+    option(
+        "--aws-bucket-name",
+        envvar=constants.AWS_BUCKET_NAME,
+        required=False,
+        help=f"AWS bucket name, only used if --storage-type is '{constants.S3_STORAGE_TYPE}'",
+        cls=CustomOptionClass,
+    ),
+    option(
+        "--aws-bucket-key-prefix-source",
+        default=constants.SOURCE_PATH_DEFAULT,
+        show_default=True,
+        envvar=constants.AWS_BUCKET_KEY_PREFIX_SOURCE,
+        required=False,
+        help=f"AWS S3 bucket source key prefix, only used if --storage-type is '{constants.S3_STORAGE_TYPE}'",
+        cls=CustomOptionClass,
+    ),
+    option(
+        "--aws-bucket-key-prefix-destination",
         default=constants.DESTINATION_PATH_DEFAULT,
+        show_default=True,
+        envvar=constants.AWS_BUCKET_KEY_PREFIX_DESTINATION,
+        required=False,
+        help=f"AWS S3 bucket destination key prefix, only used if --storage-type is '{constants.S3_STORAGE_TYPE}'",
+        cls=CustomOptionClass,
+    ),
+    option(
+        "--aws-region-name",
+        envvar=constants.AWS_REGION_NAME,
+        required=False,
+        help=f"AWS region name, only used if --storage-type is '{constants.S3_STORAGE_TYPE}'",
+        cls=CustomOptionClass,
+    ),
+    option(
+        "--aws-secret-access-key",
+        envvar=constants.AWS_SECRET_ACCESS_KEY,
+        required=False,
+        help=f"AWS secret access key, only used if --storage-type is '{constants.S3_STORAGE_TYPE}'",
+        cls=CustomOptionClass,
+    ),
+    option(
+        "--aws-session-token",
+        envvar=constants.AWS_SESSION_TOKEN,
+        required=False,
+        help=f"AWS session token, only used if --storage-type is '{constants.S3_STORAGE_TYPE}'",
         cls=CustomOptionClass,
     ),
 ]
@@ -275,6 +344,10 @@ def common_options(func: Callable) -> Callable:
     return _build_options_helper(func, _common_options)
 
 
+def storage_options(func: Callable) -> Callable:
+    return _build_options_helper(func, _storage_options)
+
+
 def diffs_common_options(func: Callable) -> Callable:
     return _build_options_helper(func, _diffs_common_options)
 

datadog_sync/commands/sync.py

@@ -10,6 +10,7 @@
     destination_auth_options,
     diffs_common_options,
     source_auth_options,
+    storage_options,
     sync_common_options,
 )
 from datadog_sync.commands.shared.utils import run_cmd
@@ -22,6 +23,7 @@
 @common_options
 @diffs_common_options
 @sync_common_options
+@storage_options
 def sync(**kwargs):
     """Sync Datadog resources to destination."""
     run_cmd(Command.SYNC, **kwargs)

datadog_sync/constants.py

@@ -21,9 +21,33 @@
 DD_CLEANUP = "DD_CLEANUP"
 DD_VALIDATE = "DD_VALIDATE"
 DD_VERIFY_DDR_STATUS = "DD_VERIFY_DDR_STATUS"
+
+LOCAL_STORAGE_TYPE = "local"
+S3_STORAGE_TYPE = "s3"
+STORAGE_TYPES = [
+    LOCAL_STORAGE_TYPE,
+    S3_STORAGE_TYPE,
+]
+
 DD_DESTINATION_RESOURCES_PATH = "DD_DESTINATION_RESOURCES_PATH"
 DD_SOURCE_RESOURCES_PATH = "DD_SOURCE_RESOURCES_PATH"
 
+# S3 env parameter names
+AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"
+AWS_BUCKET_NAME = "AWS_BUCKET_NAME"
+AWS_BUCKET_KEY_PREFIX_SOURCE = "AWS_BUCKET_KEY_PREFIX_SOURCE"
+AWS_BUCKET_KEY_PREFIX_DESTINATION = "AWS_BUCKET_KEY_PREFIX_DESTINATION"
+AWS_REGION_NAME = "AWS_REGION_NAME"
+AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
+AWS_SESSION_TOKEN = "AWS_SESSION_TOKEN"
+AWS_CONFIG_PROPERTIES = [
+    "aws_bucket_name",
+    "aws_region_name",
+    "aws_access_key_id",
+    "aws_secret_access_key",
+    "aws_session_token",
+]
+
 # Default variables
 DEFAULT_API_URL = "https://api.datadoghq.com"
 

datadog_sync/model/security_monitoring_rules.py

@@ -44,13 +44,7 @@ class SecurityMonitoringRules(BaseResource):
             "options.impossibleTravelOptions",
             "cases.condition",
         ],
-        non_nullable_attr=[
-            "queries.additionalFilters",
-            "blocking",
-            "metadata",
-            "creator",
-            "updater"
-        ],
+        non_nullable_attr=["queries.additionalFilters", "blocking", "metadata", "creator", "updater"],
         null_values={
             "additionalFilters": [""],
             "blocking": [False],

datadog_sync/utils/configuration.py

@@ -12,11 +12,14 @@
 
 from datadog_sync.constants import (
     Command,
+    AWS_CONFIG_PROPERTIES,
     DESTINATION_PATH_DEFAULT,
     DESTINATION_PATH_PARAM,
     FALSE,
     FORCE,
+    LOCAL_STORAGE_TYPE,
     LOGGER_NAME,
+    S3_STORAGE_TYPE,
     SOURCE_PATH_DEFAULT,
     SOURCE_PATH_PARAM,
     TRUE,
@@ -34,6 +37,7 @@
 from datadog_sync.utils.filter import Filter, process_filters
 from datadog_sync.utils.resource_utils import CustomClientHTTPError
 from datadog_sync.utils.state import State
+from datadog_sync.utils.storage.storage_types import StorageType
 
 
 @dataclass
@@ -164,9 +168,27 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration:
             "force": FORCE,
         }[cleanup.lower()]
 
-    # Set resource paths
-    source_resources_path = kwargs.get(SOURCE_PATH_PARAM, SOURCE_PATH_DEFAULT)
-    destination_resources_path = kwargs.get(DESTINATION_PATH_PARAM, DESTINATION_PATH_DEFAULT)
+    # determine where the states are stored
+    storage_type = kwargs.get("storage_type", "local").lower()
+    config = {}
+
+    if storage_type == S3_STORAGE_TYPE:
+        logger.info("Using AWS S3 to store state files")
+        storage_type = StorageType.AWS_S3_BUCKET
+        source_resources_path = kwargs.get("aws_bucket_key_prefix_source", SOURCE_PATH_DEFAULT)
+        destination_resources_path = kwargs.get("aws_bucket_key_prefix_destination", DESTINATION_PATH_DEFAULT)
+        for aws_config_property in AWS_CONFIG_PROPERTIES:
+            property_value = kwargs.get(aws_config_property, None)
+            if not property_value:
+                raise ValueError(f"Missing AWS configuration parameter: {aws_config_property}")
+            config[aws_config_property] = property_value
+    elif storage_type == LOCAL_STORAGE_TYPE:
+        logger.info("Using local filesystem to store state files")
+        storage_type = StorageType.LOCAL_FILE
+        source_resources_path = kwargs.get(SOURCE_PATH_PARAM, SOURCE_PATH_DEFAULT)
+        destination_resources_path = kwargs.get(DESTINATION_PATH_PARAM, DESTINATION_PATH_DEFAULT)
+    else:
+        raise ValueError("Unsupported storage type")
 
     # Confusing, but the source for the import needs to be the destination of the reset
     # If a destination is going to be reset then a backup needs to be preformed. A back up
@@ -177,7 +199,12 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration:
         source_resources_path = f"{destination_resources_path}/.backup/{str(time.time())}"
 
     # Initialize state
-    state = State(source_resources_path=source_resources_path, destination_resources_path=destination_resources_path)
+    state = State(
+        type_=storage_type,
+        source_resources_path=source_resources_path,
+        destination_resources_path=destination_resources_path,
+        config=config,
+    )
 
     # Initialize Configuration
     config = Configuration(

datadog_sync/utils/state.py

@@ -6,12 +6,15 @@
 
 from datadog_sync.constants import (
     Origin,
-    DESTINATION_PATH_PARAM,
+    AWS_BUCKET_KEY_PREFIX_DESTINATION,
+    AWS_BUCKET_KEY_PREFIX_SOURCE,
     DESTINATION_PATH_DEFAULT,
-    SOURCE_PATH_PARAM,
+    DESTINATION_PATH_PARAM,
     SOURCE_PATH_DEFAULT,
+    SOURCE_PATH_PARAM,
 )
 from datadog_sync.utils.storage._base_storage import BaseStorage, StorageData
+from datadog_sync.utils.storage.aws_s3_bucket import AWSS3Bucket
 from datadog_sync.utils.storage.local_file import LocalFile
 from datadog_sync.utils.storage.storage_types import StorageType
 
@@ -25,6 +28,17 @@ def __init__(self, type_: StorageType = StorageType.LOCAL_FILE, **kwargs: object
                 source_resources_path=source_resources_path,
                 destination_resources_path=destination_resources_path,
             )
+        elif type_ == StorageType.AWS_S3_BUCKET:
+            source_resources_path = kwargs.get(AWS_BUCKET_KEY_PREFIX_SOURCE, SOURCE_PATH_DEFAULT)
+            destination_resources_path = kwargs.get(AWS_BUCKET_KEY_PREFIX_DESTINATION, DESTINATION_PATH_DEFAULT)
+            config = kwargs.get("config", {})
+            if not config:
+                raise ValueError("AWS configuration not found")
+            self._storage: BaseStorage = AWSS3Bucket(
+                source_resources_path=source_resources_path,
+                destination_resources_path=destination_resources_path,
+                config=config,
+            )
         else:
             raise NotImplementedError(f"Storage type {type_} not implemented")