From be3b685db1e3b0b9c8ca4ba653c615b332c49bf6 Mon Sep 17 00:00:00 2001 From: Cyril Fait Date: Thu, 8 Sep 2022 19:28:32 +0000 Subject: [PATCH] Redshift Serverless test infrastructure --- test_infra/app.py | 20 ++++- test_infra/poetry.lock | 123 ++++++++++++++++----------- test_infra/pyproject.toml | 10 +-- test_infra/stacks/base_stack.py | 6 ++ test_infra/stacks/databases_stack.py | 44 +++++++++- 5 files changed, 144 insertions(+), 59 deletions(-) diff --git a/test_infra/app.py b/test_infra/app.py index 8c5416ecc..2da2f51e7 100644 --- a/test_infra/app.py +++ b/test_infra/app.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 -from aws_cdk import App +import os + +from aws_cdk import App, Environment from stacks.base_stack import BaseStack from stacks.databases_stack import DatabasesStack from stacks.lakeformation_stack import LakeFormationStack @@ -7,7 +9,13 @@ app = App() -base = BaseStack(app, "aws-sdk-pandas-base") +env = {"env": Environment(account=os.environ["CDK_DEFAULT_ACCOUNT"], region=os.environ["CDK_DEFAULT_REGION"])} + +base = BaseStack( + app, + "aws-sdk-pandas-base", + **env, +) DatabasesStack( app, @@ -15,9 +23,14 @@ base.get_vpc, base.get_bucket, base.get_key, + **env, ) -LakeFormationStack(app, "aws-sdk-pandas-lakeformation") +LakeFormationStack( + app, + "aws-sdk-pandas-lakeformation", + **env, +) OpenSearchStack( app, @@ -25,6 +38,7 @@ base.get_vpc, base.get_bucket, base.get_key, + **env, ) app.synth() diff --git a/test_infra/poetry.lock b/test_infra/poetry.lock index 7d8b5e911..bc7b6b0d9 100644 --- a/test_infra/poetry.lock +++ b/test_infra/poetry.lock @@ -7,65 +7,69 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] -name = "aws-cdk-lib" -version = "2.20.0" -description = "Version 2 of the AWS Cloud Development Kit library" +name = "aws-cdk.aws-glue-alpha" +version = "2.41.0a0" +description = "The CDK Construct Library for AWS::Glue" category = "main" optional = false -python-versions = ">=3.6" +python-versions = "~=3.7" [package.dependencies] +aws-cdk-lib = ">=2.41.0,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.55.1,<2.0.0" +jsii = ">=1.67.0,<2.0.0" publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" [[package]] -name = "aws-cdk.aws-glue-alpha" -version = "2.20.0a0" -description = "The CDK Construct Library for AWS::Glue" +name = "aws-cdk.aws-neptune-alpha" +version = "2.41.0a0" +description = "The CDK Construct Library for AWS::Neptune" category = "main" optional = false -python-versions = ">=3.6" +python-versions = "~=3.7" [package.dependencies] -aws-cdk-lib = ">=2.20.0,<3.0.0" +aws-cdk-lib = ">=2.41.0,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.55.1,<2.0.0" +jsii = ">=1.67.0,<2.0.0" publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" [[package]] -name = "aws-cdk.aws-neptune-alpha" -version = "2.20.0a0" -description = "The CDK Construct Library for AWS::Neptune" +name = "aws-cdk.aws-redshift-alpha" +version = "2.41.0a0" +description = "The CDK Construct Library for AWS::Redshift" category = "main" optional = false -python-versions = ">=3.6" +python-versions = "~=3.7" [package.dependencies] -aws-cdk-lib = ">=2.20.0,<3.0.0" +aws-cdk-lib = ">=2.41.0,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.55.1,<2.0.0" +jsii = ">=1.67.0,<2.0.0" publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" [[package]] -name = "aws-cdk.aws-redshift-alpha" -version = "2.20.0a0" -description = "The CDK Construct Library for AWS::Redshift" +name = "aws-cdk-lib" +version = "2.41.0" +description = "Version 2 of the AWS Cloud Development Kit library" category = "main" optional = false -python-versions = ">=3.6" +python-versions = "~=3.7" [package.dependencies] -aws-cdk-lib = ">=2.20.0,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.55.1,<2.0.0" +jsii = ">=1.67.0,<2.0.0" publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" [[package]] name = "cattrs" @@ -82,15 +86,16 @@ typing_extensions = {version = "*", markers = "python_version >= \"3.7\" and pyt [[package]] name = "constructs" -version = "10.0.120" +version = "10.1.96" description = "A programming model for software-defined state" category = "main" optional = false -python-versions = ">=3.6" +python-versions = "~=3.7" [package.dependencies] -jsii = ">=1.57.0,<2.0.0" +jsii = ">=1.67.0,<2.0.0" publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" [[package]] name = "exceptiongroup" @@ -101,20 +106,22 @@ optional = false python-versions = ">=3.7" [package.extras] -test = ["pytest (>=6)", "coverage (>=6)"] +test = ["coverage (>=6)", "pytest (>=6)"] [[package]] name = "jsii" -version = "1.57.0" +version = "1.67.0" description = "Python client for jsii runtime" category = "main" optional = false -python-versions = "~=3.6" +python-versions = "~=3.7" [package.dependencies] -attrs = ">=21.2,<22.0" -cattrs = {version = ">=1.8,<22.2", markers = "python_version >= \"3.7\""} +attrs = ">=21.2,<23.0" +cattrs = ">=1.8,<22.2" +publication = ">=0.0.3" python-dateutil = "*" +typeguard = ">=2.13.3,<2.14.0" typing-extensions = ">=3.7,<5.0" [[package]] @@ -144,6 +151,18 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "typeguard" +version = "2.13.3" +description = "Run-time type checker for Python" +category = "main" +optional = false +python-versions = ">=3.5.3" + +[package.extras] +doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["mypy", "pytest", "typing-extensions"] + [[package]] name = "typing-extensions" version = "4.2.0" @@ -155,44 +174,44 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = ">=3.7.1, <3.11" -content-hash = "b669f7595bc0a85be13d396707830ef9aa5dec3a9d089ca61ca7c5433b61ada8" +content-hash = "341a367c0389e288f61dd06225f8a2ce1c2221e33dd1d16473e5089977b4a4b0" [metadata.files] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] -aws-cdk-lib = [ - {file = "aws-cdk-lib-2.20.0.tar.gz", hash = "sha256:0014fa03ccaee987ead9eb6bad9329f5801bf34bb2322a509ec034b7c7d2e859"}, - {file = "aws_cdk_lib-2.20.0-py3-none-any.whl", hash = "sha256:ff32bd82f17f512bde6731e917675974934626ddac420a7cec0010b4a20320c6"}, -] "aws-cdk.aws-glue-alpha" = [ - {file = "aws-cdk.aws-glue-alpha-2.20.0a0.tar.gz", hash = "sha256:8b803bab56eca7bb71deaecbc13586d00987639920a476b8e81f7b385985f6cd"}, - {file = "aws_cdk.aws_glue_alpha-2.20.0a0-py3-none-any.whl", hash = "sha256:803749210586a8886b782d214412ad0580f7762cab65a751fe11ba45d478afc5"}, + {file = "aws-cdk.aws-glue-alpha-2.41.0a0.tar.gz", hash = "sha256:c631eaa007388147536616fbf5994a4967a945093b5e270f5ced2e914d121268"}, + {file = "aws_cdk.aws_glue_alpha-2.41.0a0-py3-none-any.whl", hash = "sha256:cc1d15b0f0559366f2e74ba86832be6d23271f1d5050ce68a4bf1286ae50c883"}, ] "aws-cdk.aws-neptune-alpha" = [ - {file = "aws-cdk.aws-neptune-alpha-2.20.0a0.tar.gz", hash = "sha256:15cfaf7de5adac8333f25e889908740df6d43dbb734b71c292d9564de557a78a"}, - {file = "aws_cdk.aws_neptune_alpha-2.20.0a0-py3-none-any.whl", hash = "sha256:f4b74871bb111d14f73b4c7153180f70a148c30ea2cb2b4a78ba6524e646c494"}, + {file = "aws-cdk.aws-neptune-alpha-2.41.0a0.tar.gz", hash = "sha256:d68538ec5a68a912874c5c712d75a4d4dfe6f9f6b85f28b693f1fc55bf6096af"}, + {file = "aws_cdk.aws_neptune_alpha-2.41.0a0-py3-none-any.whl", hash = "sha256:7c3b20951ec605cd95d0e364bdb1cfde5000f479845459bc459da0848d1b977b"}, ] "aws-cdk.aws-redshift-alpha" = [ - {file = "aws-cdk.aws-redshift-alpha-2.20.0a0.tar.gz", hash = "sha256:7ec2469451c17804e90dcfa6f4bb9610ba5f31f4f31afdfd4b848ad08a43e36e"}, - {file = "aws_cdk.aws_redshift_alpha-2.20.0a0-py3-none-any.whl", hash = "sha256:cd1a9731261f1a4911989f6194fd075fc7a743b39ed0f8310da2bed6671a3359"}, + {file = "aws-cdk.aws-redshift-alpha-2.41.0a0.tar.gz", hash = "sha256:befe318762c019de1881532a33c5b8e18caef56329042b26ffaa04a4cd05d1e9"}, + {file = "aws_cdk.aws_redshift_alpha-2.41.0a0-py3-none-any.whl", hash = "sha256:0f7cbdd97e9677219e625b8bbcdc809b335aa8fa99a73a2547e78e4dcab18694"}, +] +aws-cdk-lib = [ + {file = "aws-cdk-lib-2.41.0.tar.gz", hash = "sha256:fec5d44c17ce3d59a4aa43b7978de87a8a2bbfa8768ba2620524c84d66d317d6"}, + {file = "aws_cdk_lib-2.41.0-py3-none-any.whl", hash = "sha256:cbb6822a1e08ae6720d9117862ec9605c5a5a0e9d4e8ac8792da33ccd2cbf14b"}, ] cattrs = [ {file = "cattrs-22.1.0-py3-none-any.whl", hash = "sha256:d55c477b4672f93606e992049f15d526dc7867e6c756cd6256d4af92e2b1e364"}, {file = "cattrs-22.1.0.tar.gz", hash = "sha256:94b67b64cf92c994f8784c40c082177dc916e0489a73a9a36b24eb18a9db40c6"}, ] constructs = [ - {file = "constructs-10.0.120-py3-none-any.whl", hash = "sha256:eacb32725afb7ec03a7dbc9a0574a6bef1714ad1681b870d7ec5839d5ab7a281"}, - {file = "constructs-10.0.120.tar.gz", hash = "sha256:dfb0bcf540705b6798351727671b3f607a60c3c070e7818834f0f87ccf2aa2a7"}, + {file = "constructs-10.1.96-py3-none-any.whl", hash = "sha256:bc0367bf9e24e42ef45d4e5dd9f733ba4fd8182914402684fd49bafddec9577f"}, + {file = "constructs-10.1.96.tar.gz", hash = "sha256:b91d633068da730020d5537d40d99fcd6e752eab043142f293fee600b0ceb60a"}, ] exceptiongroup = [ {file = "exceptiongroup-1.0.0rc3-py3-none-any.whl", hash = "sha256:f734ede30b0d3f21f91d17bb83216e3ee780df0738a293e15d08925279623782"}, {file = "exceptiongroup-1.0.0rc3.tar.gz", hash = "sha256:ed5799dc1260d2421564c21567f7ec84cffb39f149cf0e5e7e0aa6548e1f7288"}, ] jsii = [ - {file = "jsii-1.57.0-py3-none-any.whl", hash = "sha256:4888091986a9ed8d50b042cc9c35a9564dd54c19e78adb890bf06d9ffac1b325"}, - {file = "jsii-1.57.0.tar.gz", hash = "sha256:ff7a3c51c1a653dd8a4342043b5f8e40b928bc617e3141e0d5d66175d22a754b"}, + {file = "jsii-1.67.0-py3-none-any.whl", hash = "sha256:a4868f8ae05ff62fef328dd197d5834c0f3c291948a1768ad931f1fc05935cb2"}, + {file = "jsii-1.67.0.tar.gz", hash = "sha256:ca16eb9c15377b77d10942439b089a10eb9657bffc63559c7a08bb7141cacc0c"}, ] publication = [ {file = "publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6"}, @@ -206,6 +225,10 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +typeguard = [ + {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"}, + {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"}, +] typing-extensions = [ {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, diff --git a/test_infra/pyproject.toml b/test_infra/pyproject.toml index 45219a570..05def9a3e 100644 --- a/test_infra/pyproject.toml +++ b/test_infra/pyproject.toml @@ -7,8 +7,8 @@ license = "Apache License 2.0" [tool.poetry.dependencies] python = ">=3.7.1, <3.11" -"aws-cdk-lib" = "^2.20.0" -"constructs" = "^10.0.119" -"aws-cdk.aws-glue-alpha" = "^2.20.0a0" -"aws-cdk.aws-redshift-alpha" = "^2.20.0a0" -"aws-cdk.aws-neptune-alpha" = "^2.20.0a0" +"aws-cdk-lib" = "^2.35.0" +"constructs" = "^10.1.67" +"aws-cdk.aws-glue-alpha" = "^2.35.0a0" +"aws-cdk.aws-redshift-alpha" = "^2.35.0a0" +"aws-cdk.aws-neptune-alpha" = "^2.35.0a0" diff --git a/test_infra/stacks/base_stack.py b/test_infra/stacks/base_stack.py index 694baca47..312b31529 100644 --- a/test_infra/stacks/base_stack.py +++ b/test_infra/stacks/base_stack.py @@ -96,6 +96,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs: str) -> None: value=self.vpc.public_subnets[1].subnet_id, export_name="aws-sdk-pandas-base-PublicSubnet2", ) + CfnOutput( + self, + "PublicSubnet3", + value=self.vpc.public_subnets[2].subnet_id, + export_name="aws-sdk-pandas-base-PublicSubnet3", + ) CfnOutput( self, "PrivateSubnet", diff --git a/test_infra/stacks/databases_stack.py b/test_infra/stacks/databases_stack.py index c54bd7579..56efc2e15 100644 --- a/test_infra/stacks/databases_stack.py +++ b/test_infra/stacks/databases_stack.py @@ -9,6 +9,7 @@ from aws_cdk import aws_neptune_alpha as neptune from aws_cdk import aws_rds as rds from aws_cdk import aws_redshift_alpha as redshift +from aws_cdk import aws_redshiftserverless as redshiftserverless from aws_cdk import aws_s3 as s3 from aws_cdk import aws_secretsmanager as secrets from aws_cdk import aws_ssm as ssm @@ -42,6 +43,7 @@ def __init__( self._set_catalog_encryption() if databases_context["redshift"]: self._setup_redshift() + self._setup_redshift_serverless() if databases_context["postgresql"]: self._setup_postgresql() if databases_context["mysql"]: @@ -297,6 +299,46 @@ def _setup_redshift(self) -> None: CfnOutput(self, "RedshiftSchema", value=schema) CfnOutput(self, "RedshiftRole", value=redshift_role.role_arn) + def _setup_redshift_serverless(self) -> None: + database = "test" + redshift_cfn_namespace = redshiftserverless.CfnNamespace( + self, + "aws-sdk-pandas-redshift-serverless-namespace", + namespace_name="aws-sdk-pandas-redshift-serverless-namespace", + admin_username=self.db_username, + admin_user_password=self.db_password, + db_name=database, + ) + redshift_cfn_workgroup = redshiftserverless.CfnWorkgroup( + self, + "aws-sdk-pandas-redshift-serverless-workgroup", + workgroup_name="aws-sdk-pandas-redshift-serverless-workgroup", + namespace_name=redshift_cfn_namespace.namespace_name, + publicly_accessible=True, + security_group_ids=[self.db_security_group.security_group_id], + subnet_ids=[subnet.subnet_id for subnet in self.vpc.public_subnets], + ) + redshift_cfn_workgroup.node.add_dependency(redshift_cfn_namespace) + secret = secrets.Secret( + self, + "aws-sdk-pandas-redshift-serverless-secret", + secret_name="aws-sdk-pandas/redshift-serverless", + description="Redshift Serverless credentials", + generate_secret_string=secrets.SecretStringGenerator( + generate_string_key="dummy", + secret_string_template=json.dumps( + { + "username": self.db_username, + "password": self.db_password, + "engine": "redshift-serverless", + } + ), + ), + ) + CfnOutput(self, "RedshiftServerlessSecretArn", value=secret.secret_arn) + CfnOutput(self, "RedshiftServerlessWorkgroup", value=redshift_cfn_workgroup.workgroup_name) + CfnOutput(self, "RedshiftServerlessDatabase", value=database) + def _setup_postgresql(self) -> None: port = 3306 database = "postgres" @@ -483,7 +525,7 @@ def _setup_mysql_serverless(self) -> None: ), backup_retention=Duration.days(1), vpc=self.vpc, - vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_NAT), + vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS), subnet_group=self.rds_subnet_group, security_groups=[self.db_security_group], enable_data_api=True,