From 74ec178421eb65f9c29eba524c4752f88c22d71f Mon Sep 17 00:00:00 2001 From: Jay McConnell Date: Thu, 31 Oct 2019 15:40:21 -0500 Subject: [PATCH] config docs generation --- generate_config_docs.py | 78 ++++++++++ taskcat/_dataclasses.py | 160 ++++++++++++++++---- taskcat/cfg/config_schema.json | 38 ++++- tests/data/config_full_example/.taskcat.yml | 66 ++++++++ 4 files changed, 310 insertions(+), 32 deletions(-) create mode 100644 generate_config_docs.py create mode 100644 tests/data/config_full_example/.taskcat.yml diff --git a/generate_config_docs.py b/generate_config_docs.py new file mode 100644 index 000000000..7f0387bbc --- /dev/null +++ b/generate_config_docs.py @@ -0,0 +1,78 @@ +# flake8: noqa +import json + +if __name__ == "__main__": + + schema = json.load(open("./taskcat/cfg/config_schema.json", "r")) + + def resolve_ref(props): + if "$ref" in props: + ref = props["$ref"].split("/")[-1] + del props["$ref"] + props.update(schema["definitions"][ref]) + return props + + for k, v in schema["properties"].items(): + item_str = f"* `{k}` " + v = resolve_ref(v) + item_str += f"*type:* `{v['type']}` " + if "description" in v: + item_str += f'{v["description"]}' + print(item_str) + if "properties" in v: + for ik, iv in v["properties"].items(): + item_str = f" * `{ik}` " + iv = resolve_ref(iv) + item_str += f"*type:* `{iv['type']}` " + if "description" in iv: + item_str += f'{iv["description"]}' + print(item_str) + if iv["type"] == "object": + if "properties" in iv: + for iik, iiv in iv["properties"].items(): + item_str = f" * `{iik}` " + iiv = resolve_ref(iiv) + item_str += f"*type:* `{iiv['type']}` " + if "description" in iiv: + item_str += f'{iiv["description"]}' + elif "additionalProperties" in iv: + name = ik[:-1] if ik.endswith("s") else ik + item_str = f" * `<{name.upper()}_NAME>` " + props = resolve_ref(iv["additionalProperties"]) + item_str += f"*type:* `{iv['type']}` " + if "description" in props: + item_str += f'{props["description"]}' + print(item_str) + elif "additionalProperties" in v: + name = k[:-1] if k.endswith("s") else k + item_str = f" * `<{name.upper()}_NAME>` " + props = resolve_ref(v["additionalProperties"]) + item_str += f"*type:* `{v['type']}` " + if "description" in props: + item_str += f'{props["description"]}' + if "properties" in props: + for ik, iv in props["properties"].items(): + item_str = f" * `{ik}` " + iv = resolve_ref(iv) + item_str += f"*type:* `{iv['type']}` " + if "description" in iv: + item_str += f'{iv["description"]}' + print(item_str) + if iv["type"] == "object": + if "properties" in iv: + for iik, iiv in iv["properties"].items(): + item_str = f" * `{iik}` " + iiv = resolve_ref(iiv) + item_str += f"*type:* `{iiv['type']}` " + if "description" in iiv: + item_str += f'{iiv["description"]}' + elif "additionalProperties" in iv: + name = ik[:-1] if ik.endswith("s") else ik + item_str = f" * `<{name.upper()}_NAME>` " + iprops = resolve_ref(iv["additionalProperties"]) + item_str += f"*type:* `{iv['type']}` " + if "description" in iprops: + item_str += f'{iprops["description"]}' + print(item_str) + else: + print(v) diff --git a/taskcat/_dataclasses.py b/taskcat/_dataclasses.py index 737b35209..51a0589fd 100644 --- a/taskcat/_dataclasses.py +++ b/taskcat/_dataclasses.py @@ -15,6 +15,57 @@ LOG = logging.getLogger(__name__) +# property descriptions + +METADATA = { + "project__name": { + "description": "Project name, used as s3 key prefix when " "uploading objects" + }, + "auth": {"description": "AWS authentication section"}, + "project__owner": { + "description": "email address for project owner (not used at present)" + }, + "regions": {"description": "List of AWS regions"}, + "az_ids": { + "description": "List of Availablilty Zones ID's to exclude when generating " + "availability zones" + }, + "package_lambda": { + "description": "Package Lambda functions into zips before uploading to s3, " + "set to false to disable" + }, + "lambda_zip_path": { + "description": "Path relative to the project root to place Lambda zip " + "files, default is 'lambda_functions/zips'" + }, + "lambda_source_path": { + "description": "Path relative to the project root containing Lambda zip " + "files, default is 'lambda_functions/source'" + }, + "s3_bucket": { + "description": "Name of S3 bucket to upload project to, if left out " + "a bucket will be auto-generated" + }, + "parameters": { + "description": "Parameter key-values to pass to CloudFormation, " + "parameters provided in global config take precedence" + }, + "build_submodules": { + "description": "Build Lambda zips recursively for submodules, " + "set to false to disable" + }, + "template": { + "description": "path to template file relative to the project " + "config file path" + }, + "tags": {"description": "Tags to apply to CloudFormation template"}, + "enable_sig_v2": { + "description": "Enable (deprecated) sigv2 access to auto-generated buckets" + }, + "s3_object_acl": { + "description": "ACL for uploaded s3 objects, defaults to 'private'" + }, +} # types @@ -36,7 +87,12 @@ class ParameterKeyField(FieldEncoder): @property def json_schema(self): - return {"type": "string", "pattern": r"[a-zA-Z0-9]*^$"} + return { + "type": "string", + "pattern": r"[a-zA-Z0-9]*^$", + "Description": "CloudFormation parameter name, can contain letters and " + "numbers only", + } JsonSchemaMixin.register_field_encoders({ParameterKey: ParameterKeyField()}) @@ -49,6 +105,7 @@ def json_schema(self): "type": "string", "pattern": r"^(ap|eu|us|sa|ca|cn|af|me|us-gov)-(central|south|north|east|" r"west|southeast|southwest|northeast|northwest)-[0-9]$", + "description": "AWS Region name eg.: 'us-east-1'", } @@ -58,7 +115,11 @@ def json_schema(self): class AlNumDashField(FieldEncoder): @property def json_schema(self): - return {"type": "string", "pattern": r"^[a-z0-9-]*$"} + return { + "type": "string", + "pattern": r"^[a-z0-9-]*$", + "description": "accepts lower case letters, numbers and -", + } JsonSchemaMixin.register_field_encoders({AlNumDash: AlNumDashField()}) @@ -71,6 +132,7 @@ def json_schema(self): "type": "string", "pattern": r"^(ap|eu|us|sa|ca|cn|af|me)(n|s|e|w|c|ne|se|nw|sw)[0-9]-az[0-9]" r"$", + "description": "Availability Zone ID, eg.: 'use1-az1'", } @@ -247,41 +309,81 @@ class TestObj: @dataclass class GeneralConfig(JsonSchemaMixin, allow_additional_props=False): - parameters: Optional[Dict[ParameterKey, ParameterValue]] = field(default=None) - tags: Optional[Dict[TagKey, TagValue]] = field(default=None) - auth: Optional[Dict[Region, str]] = field(default=None) - s3_bucket: Optional[str] = field(default=None) + """General configuration settings.""" + + parameters: Optional[Dict[ParameterKey, ParameterValue]] = field( + default=None, metadata=METADATA["parameters"] + ) + tags: Optional[Dict[TagKey, TagValue]] = field( + default=None, metadata=METADATA["tags"] + ) + auth: Optional[Dict[Region, str]] = field(default=None, metadata=METADATA["auth"]) + s3_bucket: Optional[str] = field(default=None, metadata=METADATA["s3_bucket"]) @dataclass class TestConfig(JsonSchemaMixin, allow_additional_props=False): - template: Optional[str] = field(default=None) - parameters: Optional[Dict[ParameterKey, ParameterValue]] = field(default=None) - regions: Optional[List[Region]] = field(default=None) - tags: Optional[Dict[TagKey, TagValue]] = field(default=None) - auth: Optional[Dict[Region, str]] = field(default=None) - s3_bucket: Optional[S3BucketName] = field(default=None) - az_blacklist: Optional[List[AzId]] = field(default=None) + """Test specific configuration section.""" + + template: Optional[str] = field(default=None, metadata=METADATA["template"]) + parameters: Optional[Dict[ParameterKey, ParameterValue]] = field( + default=None, metadata=METADATA["parameters"] + ) + regions: Optional[List[Region]] = field(default=None, metadata=METADATA["regions"]) + tags: Optional[Dict[TagKey, TagValue]] = field( + default=None, metadata=METADATA["tags"] + ) + auth: Optional[Dict[Region, str]] = field(default=None, metadata=METADATA["auth"]) + s3_bucket: Optional[S3BucketName] = field( + default=None, metadata=METADATA["s3_bucket"] + ) + az_blacklist: Optional[List[AzId]] = field( + default=None, metadata=METADATA["az_ids"] + ) # pylint: disable=too-many-instance-attributes @dataclass class ProjectConfig(JsonSchemaMixin, allow_additional_props=False): - name: Optional[ProjectName] = field(default=None) - auth: Optional[Dict[Region, str]] = field(default=None) - owner: Optional[str] = field(default=None) - regions: Optional[List[Region]] = field(default=None) - az_blacklist: Optional[List[AzId]] = field(default=None) - package_lambda: Optional[bool] = field(default=None) - lambda_zip_path: Optional[str] = field(default=None) - lambda_source_path: Optional[str] = field(default=None) - s3_bucket: Optional[S3BucketName] = field(default=None) - parameters: Optional[Dict[ParameterKey, ParameterValue]] = field(default=None) - build_submodules: Optional[bool] = field(default=None) - template: Optional[str] = field(default=None) - tags: Optional[Dict[TagKey, TagValue]] = field(default=None) - s3_enable_sig_v2: Optional[bool] = field(default=None) - s3_object_acl: Optional[str] = field(default=None) + """Project specific configuration section""" + + name: Optional[ProjectName] = field( + default=None, metadata=METADATA["project__name"] + ) + auth: Optional[Dict[Region, str]] = field(default=None, metadata=METADATA["auth"]) + owner: Optional[str] = field(default=None, metadata=METADATA["project__owner"]) + regions: Optional[List[Region]] = field(default=None, metadata=METADATA["regions"]) + az_blacklist: Optional[List[AzId]] = field( + default=None, metadata=METADATA["az_ids"] + ) + package_lambda: Optional[bool] = field( + default=None, metadata=METADATA["package_lambda"] + ) + lambda_zip_path: Optional[str] = field( + default=None, metadata=METADATA["lambda_zip_path"] + ) + lambda_source_path: Optional[str] = field( + default=None, metadata=METADATA["lambda_source_path"] + ) + s3_bucket: Optional[S3BucketName] = field( + default=None, metadata=METADATA["s3_bucket"] + ) + parameters: Optional[Dict[ParameterKey, ParameterValue]] = field( + default=None, metadata=METADATA["parameters"] + ) + build_submodules: Optional[bool] = field( + default=None, metadata=METADATA["build_submodules"] + ) + template: Optional[str] = field(default=None, metadata=METADATA["template"]) + tags: Optional[Dict[TagKey, TagValue]] = field( + default=None, metadata=METADATA["tags"] + ) + s3_enable_sig_v2: Optional[bool] = field( + default=None, metadata=METADATA["enable_sig_v2"] + ) + s3_object_acl: Optional[str] = field( + default=None, metadata=METADATA["s3_object_acl"] + ) PROPAGATE_KEYS = ["tags", "parameters", "auth"] @@ -292,6 +394,8 @@ class ProjectConfig(JsonSchemaMixin, allow_additional_props=False): # pylint: disable=no-member @dataclass class BaseConfig(JsonSchemaMixin, allow_additional_props=False): + """Taskcat configuration file""" + general: GeneralConfig = field(default_factory=GeneralConfig) project: ProjectConfig = field(default_factory=ProjectConfig) tests: Dict[TestName, TestConfig] = field(default_factory=dict) diff --git a/taskcat/cfg/config_schema.json b/taskcat/cfg/config_schema.json index df9a41dd7..03ed3aed2 100644 --- a/taskcat/cfg/config_schema.json +++ b/taskcat/cfg/config_schema.json @@ -4,12 +4,13 @@ "definitions": { "GeneralConfig": { "additionalProperties": false, - "description": "GeneralConfig(parameters: Union[Dict[ParameterKey, Union[str, int, bool, List[Union[int, str]]]], NoneType] = None, tags: Union[Dict[TagKey, TagValue], NoneType] = None, auth: Union[Dict[Region, str], NoneType] = None, s3_bucket: Union[str, NoneType] = None)", + "description": "General configuration settings.", "properties": { "auth": { "additionalProperties": { "type": "string" }, + "description": "AWS authentication section", "type": "object" }, "parameters": { @@ -39,15 +40,18 @@ } ] }, + "description": "Parameter key-values to pass to CloudFormation, parameters provided in global config take precedence", "type": "object" }, "s3_bucket": { + "description": "Name of S3 bucket to upload project to, if left out a bucket will be auto-generated", "type": "string" }, "tags": { "additionalProperties": { "type": "string" }, + "description": "Tags to apply to CloudFormation template", "type": "object" } }, @@ -55,38 +59,47 @@ }, "ProjectConfig": { "additionalProperties": false, - "description": "ProjectConfig(name: Union[ProjectName, NoneType] = None, auth: Union[Dict[Region, str], NoneType] = None, owner: Union[str, NoneType] = None, regions: Union[List[Region], NoneType] = None, az_blacklist: Union[List[AzId], NoneType] = None, package_lambda: Union[bool, NoneType] = None, lambda_zip_path: Union[str, NoneType] = None, lambda_source_path: Union[str, NoneType] = None, s3_bucket: Union[S3BucketName, NoneType] = None, parameters: Union[Dict[ParameterKey, Union[str, int, bool, List[Union[int, str]]]], NoneType] = None, build_submodules: Union[bool, NoneType] = None, template: Union[str, NoneType] = None, tags: Union[Dict[TagKey, TagValue], NoneType] = None, s3_enable_sig_v2: Union[bool, NoneType] = None, s3_object_acl: Union[str, NoneType] = None)", + "description": "Project specific configuration section", "properties": { "auth": { "additionalProperties": { "type": "string" }, + "description": "AWS authentication section", "type": "object" }, "az_blacklist": { + "description": "List of Availablilty Zones ID's to exclude when generating availability zones", "items": { + "description": "Availability Zone ID, eg.: 'use1-az1'", "pattern": "^(ap|eu|us|sa|ca|cn|af|me)(n|s|e|w|c|ne|se|nw|sw)[0-9]-az[0-9]$", "type": "string" }, "type": "array" }, "build_submodules": { + "description": "Build Lambda zips recursively for submodules, set to false to disable", "type": "boolean" }, "lambda_source_path": { + "description": "Path relative to the project root containing Lambda zip files, default is 'lambda_functions/source'", "type": "string" }, "lambda_zip_path": { + "description": "Path relative to the project root to place Lambda zip files, default is 'lambda_functions/zips'", "type": "string" }, "name": { + "description": "Project name, used as s3 key prefix when uploading objects", "pattern": "^[a-z0-9-]*$", "type": "string" }, "owner": { + "description": "email address for project owner (not used at present)", "type": "string" }, "package_lambda": { + "description": "Package Lambda functions into zips before uploading to s3, set to false to disable", "type": "boolean" }, "parameters": { @@ -116,32 +129,40 @@ } ] }, + "description": "Parameter key-values to pass to CloudFormation, parameters provided in global config take precedence", "type": "object" }, "regions": { + "description": "List of AWS regions", "items": { + "description": "AWS Region name eg.: 'us-east-1'", "pattern": "^(ap|eu|us|sa|ca|cn|af|me|us-gov)-(central|south|north|east|west|southeast|southwest|northeast|northwest)-[0-9]$", "type": "string" }, "type": "array" }, "s3_bucket": { + "description": "Name of S3 bucket to upload project to, if left out a bucket will be auto-generated", "pattern": "^[a-z0-9-]*$", "type": "string" }, "s3_enable_sig_v2": { + "description": "Enable (deprecated) sigv2 access to auto-generated buckets", "type": "boolean" }, "s3_object_acl": { + "description": "ACL for uploaded s3 objects, defaults to 'private'", "type": "string" }, "tags": { "additionalProperties": { "type": "string" }, + "description": "Tags to apply to CloudFormation template", "type": "object" }, "template": { + "description": "path to template file relative to the project config file path", "type": "string" } }, @@ -149,16 +170,19 @@ }, "TestConfig": { "additionalProperties": false, - "description": "TestConfig(template: Union[str, NoneType] = None, parameters: Union[Dict[ParameterKey, Union[str, int, bool, List[Union[int, str]]]], NoneType] = None, regions: Union[List[Region], NoneType] = None, tags: Union[Dict[TagKey, TagValue], NoneType] = None, auth: Union[Dict[Region, str], NoneType] = None, s3_bucket: Union[S3BucketName, NoneType] = None, az_blacklist: Union[List[AzId], NoneType] = None)", + "description": "Test specific configuration section.", "properties": { "auth": { "additionalProperties": { "type": "string" }, + "description": "AWS authentication section", "type": "object" }, "az_blacklist": { + "description": "List of Availablilty Zones ID's to exclude when generating availability zones", "items": { + "description": "Availability Zone ID, eg.: 'use1-az1'", "pattern": "^(ap|eu|us|sa|ca|cn|af|me)(n|s|e|w|c|ne|se|nw|sw)[0-9]-az[0-9]$", "type": "string" }, @@ -191,16 +215,20 @@ } ] }, + "description": "Parameter key-values to pass to CloudFormation, parameters provided in global config take precedence", "type": "object" }, "regions": { + "description": "List of AWS regions", "items": { + "description": "AWS Region name eg.: 'us-east-1'", "pattern": "^(ap|eu|us|sa|ca|cn|af|me|us-gov)-(central|south|north|east|west|southeast|southwest|northeast|northwest)-[0-9]$", "type": "string" }, "type": "array" }, "s3_bucket": { + "description": "Name of S3 bucket to upload project to, if left out a bucket will be auto-generated", "pattern": "^[a-z0-9-]*$", "type": "string" }, @@ -208,16 +236,18 @@ "additionalProperties": { "type": "string" }, + "description": "Tags to apply to CloudFormation template", "type": "object" }, "template": { + "description": "path to template file relative to the project config file path", "type": "string" } }, "type": "object" } }, - "description": "BaseConfig(general: taskcat._dataclasses.GeneralConfig = , project: taskcat._dataclasses.ProjectConfig = , tests: Dict[TestName, taskcat._dataclasses.TestConfig] = )", + "description": "Taskcat configuration file", "properties": { "general": { "$ref": "#/definitions/GeneralConfig", diff --git a/tests/data/config_full_example/.taskcat.yml b/tests/data/config_full_example/.taskcat.yml new file mode 100644 index 000000000..75013b208 --- /dev/null +++ b/tests/data/config_full_example/.taskcat.yml @@ -0,0 +1,66 @@ +general: # [OPTIONAL] General configuration settings. + # [OPTIONAL] Parameter key-values to pass to CloudFormation, parameters provided in + # global config take precedence + parameters: + MyParam: my-param-value + # [OPTIONAL] Name of S3 bucket to upload project to, if left out a bucket will be + # auto-generated + s3_bucket: my-bucket + tags: # [OPTIONAL] Tags to apply to CloudFormation template + MyTagKey: MyTagValue +project: # [REQUIRED] Project specific configuration section + # [REQUIRED] Project name, used as s3 key prefix when uploading objects + name: my-cfn-project + # [OPTIONAL] List of Availablilty Zones ID's to exclude when generating azs + az_blacklist: + - use1-az1 + # build Lambda zips recursively for submodules, set to false to disable + build_submodules: false + # [OPTIONAL] Path relative to the project root containing Lambda zip files, default + # is 'lambda_functions/source' + lambda_source_path: functions/source + # [OPTIONAL] Path relative to the project root to place Lambda zip files, default is + # 'lambda_functions/zips' + lambda_zip_path: functions/packages + owner: # [OPTIONAL] email address for project owner (not used at present) + # Package Lambda functions into zips before uploading to s3, set to false to disable + package_lambda: false + # [OPTIONAL] Parameter key-values to pass to CloudFormation, parameters provided in + # general section take precedence + parameters: + MyOtherParam: OtherValue + regions: # List of AWS regions + - us-east-1 + - us-west-2 + # [OPTIONAL] Name of S3 bucket to upload project to, if left out a bucket will be + # auto-generated + s3_bucket: my-s3-bucket + # [OPTIONAL] Enable (deprecated) sigv2 access to auto-generated buckets + s3_enable_sig_v2: true + # [OPTIONAL] ACL for uploaded s3 objects, defaults to 'private' + s3_object_acl: public-read + tags: # [OPTIONAL] Tags to apply to CloudFormation template + MyTagKey: MyTagValue + # [REQUIRED] path to template file relative to the project config file path. If + # defined, template is optional in tests + template: ./relative-path/to/template.yaml +tests: # [REQUIRED] Test definition section + my-test: + # [OPTIONAL] List of Availablilty Zones ID's to exclude when generating azs + az_blacklist: + - use1-az1 + # [OPTIONAL] Parameter key-values to pass to CloudFormation, parameters provided in + # general section take precedence + parameters: + MyOtherParam: OtherValue + # [OPTIONAL] List of AWS regions, will override any regions defined in project + regions: + - eu-west-1 + # [OPTIONAL] Name of S3 bucket to upload project to, if left out a bucket will be + # auto-generated + s3_bucket: my-s3-bucket + tags: # [OPTIONAL] Tags to apply to CloudFormation template + MyTagKey: MyTagValue + # [REQUIRED] path to template file relative to the project config file path. If + # template is defined in project section this can be ommitted + template: ./relative-path/to/template.yaml