diff --git a/cli/README.md b/cli/README.md index 35d36edddcf..fea8c3c0305 100644 --- a/cli/README.md +++ b/cli/README.md @@ -17,7 +17,7 @@ After build find binary at bin/cft location Follow cft --help instructions -Google Cloud Formation Toolkit CLI +Google Cloud Foundation Toolkit CLI ```bash Usage: diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 0aa84957c6e..d15e647116c 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -11,8 +11,8 @@ import ( // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "cft", - Short: "Google Cloud Formation Toolkit CLI", - Long: "Google Cloud Formation Toolkit CLI", + Short: "Google Cloud Foundation Toolkit CLI", + Long: "Google Cloud Foundation Toolkit CLI", Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { // no params means same as -h flag diff --git a/dm/CHANGELOG.md b/dm/CHANGELOG.md index b50c3f8b031..3caad756935 100644 --- a/dm/CHANGELOG.md +++ b/dm/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## CFT Templates + +### 23.08.2019 + +- Adding container images for test automation +- Finalizing 'cft-dm-dev' branch for merge to master + ### 21.03.2019 - *Templates/iam_member*: The template is now using virtual.projects.iamMemberBinding which is and advanced diff --git a/dm/CI/cft_base_contianer/Dockerfile b/dm/CI/cft_base_contianer/Dockerfile new file mode 100644 index 00000000000..ecbf9c56a74 --- /dev/null +++ b/dm/CI/cft_base_contianer/Dockerfile @@ -0,0 +1,24 @@ +FROM gcr.io/cloud-builders/gcloud + +RUN apt-get update && apt-get -y install make \ + && apt-get -y install gettext-base \ + && pip install --upgrade pip \ + && pip install setuptools \ + && git clone https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit \ + && cd cloud-foundation-toolkit/dm \ + && rm -rf templates \ + && make prerequisites \ + && make build \ + && make install \ + && pip install tox \ + && pip install pytest \ + && make cft-venv \ + && make template-prerequisites \ + && make cft-prerequisites \ + && . venv/bin/activate \ + && ./src/cftenv \ + && pwd \ + && cft --version \ + && bats -v + +WORKDIR /cloud-foundation-toolkit/dm diff --git a/dm/CI/cft_base_contianer/cloudbuild.yaml b/dm/CI/cft_base_contianer/cloudbuild.yaml new file mode 100644 index 00000000000..521a0005c04 --- /dev/null +++ b/dm/CI/cft_base_contianer/cloudbuild.yaml @@ -0,0 +1,13 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/$PROJECT_ID/cft:${_CFT_VERSION}', + '-t', 'gcr.io/$PROJECT_ID/cft', + '--build-arg', 'CFT_VERSION=${_CFT_VERSION}', + '.'] +substitutions: + _CFT_VERSION: 0.0.4 + +images: +- 'gcr.io/$PROJECT_ID/cft:latest' +- 'gcr.io/$PROJECT_ID/cft:$_CFT_VERSION' +tags: ['cft-test-dm'] diff --git a/dm/CI/cft_schema_runner/Dockerfile b/dm/CI/cft_schema_runner/Dockerfile new file mode 100644 index 00000000000..1053fabc63f --- /dev/null +++ b/dm/CI/cft_schema_runner/Dockerfile @@ -0,0 +1,16 @@ +FROM gcr.io/cloud-builders/gcloud + +RUN apt-get update +RUN apt-get install python-setuptools -y +RUN apt-get install npm -y +RUN apt-get install jq -y +RUN pip install yq +RUN npm install -g ajv-cli +RUN ln -s /usr/bin/nodejs /usr/bin/node + +COPY docker-entrypoint.sh /root/ +RUN chmod 777 /root/docker-entrypoint.sh + +ENTRYPOINT ["/root/docker-entrypoint.sh"] + +CMD [] diff --git a/dm/CI/cft_schema_runner/cloudbuild-test.yaml b/dm/CI/cft_schema_runner/cloudbuild-test.yaml new file mode 100644 index 00000000000..362cd929115 --- /dev/null +++ b/dm/CI/cft_schema_runner/cloudbuild-test.yaml @@ -0,0 +1,177 @@ +# find . -name "*.yam"l | grep examples| sort -n | awk '{print "- name: 'gcr.io/\$PROJECT_ID/cft-schema'\n args: [`"$1"`]"}' | sed "s/\`/'/g" + +steps: +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/autoscaler/examples/autoscaler_regional.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/autoscaler/examples/autoscaler_zonal.yaml'] +# bug +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/backend_service/examples/backend_service_global.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/backend_service/examples/backend_service_regional.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/bastion/examples/bastion.yaml'] +# Skip, complex example +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/bigquery/examples/bigquery.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloudbuild/examples/cloudbuild_reposource.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloudbuild/examples/cloudbuild_storagesource.yaml'] +# FAILING +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/cloudbuild/examples/cloudbuild_trigger.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloudbuild/examples/cloudbuild.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloud_function/examples/cloud_function_upload.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloud_function/examples/cloud_function.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloud_router/examples/cloud_router.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloud_spanner/examples/cloud_spanner.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloud_sql/examples/cloud_sql_read_replica.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloud_sql/examples/cloud_sql.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloud_tasks/examples/cloud_tasks_queue.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/cloud_tasks/examples/cloud_tasks_task.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/dataproc/examples/dataproc.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/dns_managed_zone/examples/dns_managed_zone_legacy.yaml'] +# FIXME +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/dns_managed_zone/examples/dns_managed_zone_private_visibility_config.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/dns_managed_zone/examples/dns_managed_zone_private.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/dns_managed_zone/examples/dns_managed_zone_public.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/dns_managed_zone/examples/dns_managed_zone.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/dns_records/examples/dns_records.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/external_load_balancer/examples/external_load_balancer_https.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/external_load_balancer/examples/external_load_balancer_http.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/external_load_balancer/examples/external_load_balancer_ssl.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/external_load_balancer/examples/external_load_balancer_tcp.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/firewall/examples/firewall.yaml'] +# FIXME +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/folder/examples/folder.yaml'] +# FIXME +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/forseti/examples/forseti.yaml'] +# FIXME +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/forwarding_rule/examples/forwarding_rule_global.yaml'] +# FIXME +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/forwarding_rule/examples/forwarding_rule_regional.yaml'] +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/gcs_bucket/examples/gcs_bucket_iam_bindings.yaml'] +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/gcs_bucket/examples/gcs_bucket_lifecycle.yaml'] +# SCHEMA version issue +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/gcs_bucket/examples/gcs_bucket.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/gke/examples/gke_regional_private.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/gke/examples/gke_regional.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/gke/examples/gke_zonal.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/haproxy/examples/haproxy.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/healthcheck/examples/healthcheck.yaml'] +# FIXME +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/iam_custom_role/examples/iam_custom_role.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/iam_member/examples/iam_member.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/instance/examples/instance.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/instance_template/examples/instance_template.yaml'] +# Schema faulty? +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/interconnect_attachment/examples/interconnect_attachment_dedicated.yaml'] +# Schema faulty? +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/interconnect_attachment/examples/interconnect_attachment_partner.yaml'] +# - name: gcr.io/$PROJECT_ID/cft-schema +# FIXME +# args: ['./templates/interconnect/examples/interconnect_dedicated.yaml'] +# FIXME +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/internal_load_balancer/examples/internal_load_balancer.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/ip_reservation/examples/ip_reservation.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/kms/examples/kms_signkey.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/kms/examples/kms.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/logsink/examples/billingaccount_logsink_bucket_destination.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/logsink/examples/folder_logsink_bq_destination.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/logsink/examples/org_logsink_pubsub_destination.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/logsink/examples/project_logsink_bucket_destination.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/managed_instance_group/examples/managed_instance_group_healthcheck.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/managed_instance_group/examples/managed_instance_group.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/nat_gateway/examples/nat_gateway.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/network/examples/network.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/network_peering/examples/network_peering.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/org_policy/examples/org_policy.yaml'] +# FIXME +# - name: gcr.io/$PROJECT_ID/cft-schema +# args: ['./templates/project/examples/project.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/pubsub/examples/pubsub_push.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/pubsub/examples/pubsub.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/route/examples/route.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/runtime_config/examples/runtime_config.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_bindings.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_legacy.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_policy.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/ssl_certificate/examples/ssl_certificate.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/stackdriver_metric_descriptor/examples/stackdriver_metric_descriptor.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/target_proxy/examples/target_proxy_https.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/target_proxy/examples/target_proxy_http.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/target_proxy/examples/target_proxy_ssl.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/target_proxy/examples/target_proxy_tcp.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/url_map/examples/url_map.yaml'] +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/vpn/examples/vpn.yaml'] +tags: ['cft-dm-schema-runner'] diff --git a/dm/CI/cft_schema_runner/cloudbuild.yaml b/dm/CI/cft_schema_runner/cloudbuild.yaml new file mode 100644 index 00000000000..d967a6435bb --- /dev/null +++ b/dm/CI/cft_schema_runner/cloudbuild.yaml @@ -0,0 +1,13 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/$PROJECT_ID/cft-schema:${_CFT_VERSION}', + '-t', 'gcr.io/$PROJECT_ID/cft-schema', + '--build-arg', 'CFT_VERSION=${_CFT_VERSION}', + '.'] +substitutions: + _CFT_VERSION: 0.0.4 + +images: +- 'gcr.io/$PROJECT_ID/cft-schema:latest' +- 'gcr.io/$PROJECT_ID/cft-schema:$_CFT_VERSION' +tags: ['cft-test-dm'] diff --git a/dm/CI/cft_schema_runner/docker-entrypoint.sh b/dm/CI/cft_schema_runner/docker-entrypoint.sh new file mode 100644 index 00000000000..8692d98e2ce --- /dev/null +++ b/dm/CI/cft_schema_runner/docker-entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -eu + +readonly GIT_URL='https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit' +readonly CLONE_DIRNAME="$(mktemp -d)" +readonly BRANCH_NAME="cft-dm-dev" + +readonly COLOR_RESET='\033[0m' +readonly COLOR_BOLD='\033[1m' +readonly COLOR_BG_BLUE='\033[44m' + +echo_color() { + echo -e "${COLOR_BOLD}${COLOR_BG_BLUE}$@${COLOR_RESET}" +} + +echo_color "Cloning repo" +git clone "${GIT_URL}" "${CLONE_DIRNAME}" +cd "${CLONE_DIRNAME}" +git checkout "${BRANCH_NAME}" + +echo_color 'Initializing CFT DM templates' + +cd dm/templates + +# cat healthcheck/examples/healthcheck.yaml | yq .resources[0].properties > project.json; cat healthcheck/healthcheck.py.schema | yq . > project.py.schema.json; ajv validate -s project.py.schema.json -d project.json + +EXAMPLE_COUNT=`cat $@ | yq '.resources | length'` +EXAMPLE_COUNT=$(($EXAMPLE_COUNT-1)) + +while [ $EXAMPLE_COUNT -ge 0 ]; +do + echo_color "Example $EXAMPLE_COUNT" + cat $@ | yq .resources[$EXAMPLE_COUNT].properties > example.json; + cat example.json + export SCHEMA_PATH=`cat $@ | yq -r .imports[0].path | awk '{print $1".schema"}'` + echo_color $SCHEMA_PATH + cat $SCHEMA_PATH | yq . > example.py.schema.json; + echo_color "Schema validation" + ajv validate -s example.py.schema.json -d example.json + EXAMPLE_COUNT=$(($EXAMPLE_COUNT-1)) + +done diff --git a/dm/CI/cft_test_runner/Dockerfile b/dm/CI/cft_test_runner/Dockerfile new file mode 100644 index 00000000000..fc4044f120c --- /dev/null +++ b/dm/CI/cft_test_runner/Dockerfile @@ -0,0 +1,14 @@ +FROM gcr.io/cft-test-workspace-221111/cft:latest + +COPY cloud-foundation-tests.conf /etc/cloud-foundation-tests.conf +RUN cat /etc/cloud-foundation-tests.conf +RUN chmod 666 /etc/cloud-foundation-tests.conf +COPY docker-entrypoint.sh /root/ +RUN chmod 777 /root/docker-entrypoint.sh + + +WORKDIR /cloud-foundation-toolkit/dm + +ENTRYPOINT ["/root/docker-entrypoint.sh"] + +CMD [] diff --git a/dm/CI/cft_test_runner/cloud-foundation-tests.conf b/dm/CI/cft_test_runner/cloud-foundation-tests.conf new file mode 100644 index 00000000000..9ecd0be0ae6 --- /dev/null +++ b/dm/CI/cft_test_runner/cloud-foundation-tests.conf @@ -0,0 +1,5 @@ +export CLOUD_FOUNDATION_ORGANIZATION_ID="12345678 +export CLOUD_FOUNDATION_PROJECT_ID="project_ID" +export CLOUDDNS_CROSS_PROJECT_ID="project_ID2" +export CLOUD_FOUNDATION_BILLING_ACCOUNT_ID="123456-789ABCD-000111" +export CLOUD_FOUNDATION_USER_ACCOUNT="user@cft.tips" diff --git a/dm/CI/cft_test_runner/cloudbuild-test.yaml b/dm/CI/cft_test_runner/cloudbuild-test.yaml new file mode 100644 index 00000000000..23459becab0 --- /dev/null +++ b/dm/CI/cft_test_runner/cloudbuild-test.yaml @@ -0,0 +1,9 @@ +steps: +- name: 'gcr.io/$PROJECT_ID/cft-ci-test' + args: ['${_BATS_TEST_FILE}'] + +substitutions: + _BATS_TEST_FILE: ./templates/autoscaler/tests/integration/autoscaler.bats # default value + +tags: ['cft-dm-test-runner'] +timeout: '7200s' diff --git a/dm/CI/cft_test_runner/cloudbuild.yaml b/dm/CI/cft_test_runner/cloudbuild.yaml new file mode 100644 index 00000000000..85861c89e58 --- /dev/null +++ b/dm/CI/cft_test_runner/cloudbuild.yaml @@ -0,0 +1,13 @@ +steps: +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/$PROJECT_ID/cft-ci-test:${_CFT_VERSION}', + '-t', 'gcr.io/$PROJECT_ID/cft-ci-test', + '--build-arg', 'CFT_VERSION=${_CFT_VERSION}', + '.'] +substitutions: + _CFT_VERSION: 0.0.4 + +images: +- 'gcr.io/$PROJECT_ID/cft-ci-test:latest' +- 'gcr.io/$PROJECT_ID/cft-ci-test:$_CFT_VERSION' +tags: ['cft-test-dm'] diff --git a/dm/CI/cft_test_runner/docker-entrypoint.sh b/dm/CI/cft_test_runner/docker-entrypoint.sh new file mode 100644 index 00000000000..013253fa6fc --- /dev/null +++ b/dm/CI/cft_test_runner/docker-entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -eu + +readonly GIT_URL='https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit' +readonly CLONE_DIRNAME="$(mktemp -d)" +readonly BRANCH_NAME="cft-dm-dev" +readonly DM_root="/cloud-foundation-toolkit/dm" + +readonly COLOR_RESET='\033[0m' +readonly COLOR_BOLD='\033[1m' +readonly COLOR_BG_BLUE='\033[44m' + +echo_color() { + echo -e "${COLOR_BOLD}${COLOR_BG_BLUE}$@${COLOR_RESET}" +} + +echo_color 'Activating venv for testing' + +cd "${DM_root}" + +set +u # Turn off because virtualenv uses undefined variables +. venv/bin/activate \ +./src/cftenv +set -u + +export CLOUD_FOUNDATION_CONF=/etc/cloud-foundation-tests.conf + +echo_color "Cloning repo" + +git clone "${GIT_URL}" "${CLONE_DIRNAME}" +cd "${CLONE_DIRNAME}" +git checkout "${BRANCH_NAME}" + +mv "${CLONE_DIRNAME}/dm/templates" "${DM_root}" + +echo_color "Welcome your Majesty, ready to run some tests!" + +# Running bats tests relative to dm folder for example: "./templates/project/tests/integration/project.bats" + +cd "${DM_root}" + +chmod 777 $@ +exec bats $@ diff --git a/dm/Makefile b/dm/Makefile index 7605515ba92..be78ad89d96 100644 --- a/dm/Makefile +++ b/dm/Makefile @@ -18,7 +18,7 @@ help: .ONESHELL: .PHONY: cft-prerequisites cft-venv cft-clean-venv cft-test cft-test-venv template-prerequisites -prerequisites: +cft-prerequisites: ${PYTHON} -m pip install -r requirements/prerequisites.txt cft-venv: diff --git a/dm/docs/userguide.md b/dm/docs/userguide.md index bed80b9352c..966cc5db81b 100644 --- a/dm/docs/userguide.md +++ b/dm/docs/userguide.md @@ -370,7 +370,7 @@ Proceed as follows: ```shell cd dm -sudo make prerequisites # Installs prerequisites in system python +sudo make cft-prerequisites # Installs prerequisites in system python make build # builds the package sudo make install # installs the package in /usr/local ``` @@ -390,7 +390,7 @@ To update CFT to a newer version, proceed as follows: ```shell cd dm make clean -sudo make prerequisites +sudo make cft-prerequisites make build sudo make uninstall sudo make install diff --git a/dm/templates/autoscaler/autoscaler.py b/dm/templates/autoscaler/autoscaler.py index 5a1e06d6092..713c3452153 100644 --- a/dm/templates/autoscaler/autoscaler.py +++ b/dm/templates/autoscaler/autoscaler.py @@ -14,8 +14,10 @@ """ This template creates an autoscaler. """ REGIONAL_LOCAL_AUTOSCALER_TYPES = { - True: 'compute.v1.regionAutoscaler', - False: 'compute.v1.autoscaler' + # https://cloud.google.com/compute/docs/reference/rest/v1/regionAutoscalers + True: 'gcp-types/compute-v1:regionAutoscalers', + # https://cloud.google.com/compute/docs/reference/rest/v1/autoscalers + False: 'gcp-types/compute-v1:autoscalers' } def set_optional_property(receiver, source, property_name): @@ -44,14 +46,17 @@ def generate_config(context): properties = context.properties name = properties.get('name', context.env['name']) + project = properties.get('project', context.env['project']) target = properties['target'] policy = {} autoscaler = { 'type': None, # Will be set up at a later stage. - 'name': name, + 'name': context.env['name'], 'properties': { + 'name': name, + 'project': project, 'autoscalingPolicy': policy, 'target': target } @@ -82,7 +87,7 @@ def generate_config(context): }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) } ] + [location_output] } diff --git a/dm/templates/autoscaler/autoscaler.py.schema b/dm/templates/autoscaler/autoscaler.py.schema index f81ec46b0e7..6c83b269e27 100644 --- a/dm/templates/autoscaler/autoscaler.py.schema +++ b/dm/templates/autoscaler/autoscaler.py.schema @@ -15,9 +15,19 @@ info: title: Autoscaler author: Sourced Group Inc. + version: 1.0.0 description: | Creates an autoscaler. - See https://cloud.google.com/compute/docs/autoscaler/ for more details. + + For more information on this resource: + https://cloud.google.com/compute/docs/autoscaler/ + + APIs endpoints used by this template: + - gcp-types/compute-v1:autoscalers => + https://cloud.google.com/compute/docs/reference/rest/v1/autoscalers + - gcp-types/compute-v1:regionAutoscalers => + https://cloud.google.com/compute/docs/reference/rest/v1/regionAutoscalers + additionalProperties: false @@ -42,7 +52,11 @@ oneOf: properties: name: type: string - description: The resource name. + description: The function name. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the autoscaler. description: type: string description: The resource description. @@ -78,6 +92,7 @@ properties: The maximum number of replicas the autoscaler can scale up to. cpuUtilization: type: object + additionalProperties: false description: | Defines the CPU utilization policy that allows the autoscaler to scale based on the average CPU utilization of a managed instance group. @@ -92,6 +107,7 @@ properties: The CPU utilization the autoscaler must maintain (as a target value). loadBalancingUtilization: type: object + additionalProperties: false required: - utilizationTarget description: | @@ -108,6 +124,7 @@ properties: Configuration parameters for autoscaling based on a custom metric. items: type: object + additionalProperties: false required: - metric - utilizationTarget diff --git a/dm/templates/backend_service/backend_service.py b/dm/templates/backend_service/backend_service.py index db9c8cef682..b3df580582b 100644 --- a/dm/templates/backend_service/backend_service.py +++ b/dm/templates/backend_service/backend_service.py @@ -14,8 +14,10 @@ """ This template creates a backend service. """ REGIONAL_GLOBAL_TYPE_NAMES = { - True: 'compute.v1.regionBackendService', - False: 'compute.v1.backendService' + # https://cloud.google.com/compute/docs/reference/rest/v1/regionBackendServices + True: 'gcp-types/compute-v1:regionBackendServices', + # https://cloud.google.com/compute/docs/reference/rest/v1/backendServices + False: 'gcp-types/compute-v1:backendServices' } @@ -52,19 +54,24 @@ def generate_config(context): properties = context.properties res_name = context.env['name'] name = properties.get('name', res_name) + project_id = properties.get('project', context.env['project']) is_regional = 'region' in properties region = properties.get('region') - backend_properties = {'name': name} + backend_properties = { + 'name': name, + 'project': project_id, + } resource = { 'name': res_name, 'type': REGIONAL_GLOBAL_TYPE_NAMES[is_regional], - 'properties': backend_properties + 'properties': backend_properties, } optional_properties = [ 'description', 'backends', + 'iap', 'timeoutSec', 'protocol', 'region', @@ -74,6 +81,7 @@ def generate_config(context): 'affinityCookieTtlSec', 'loadBalancingScheme', 'connectionDraining', + 'healthChecks', 'cdnPolicy' ] diff --git a/dm/templates/backend_service/backend_service.py.schema b/dm/templates/backend_service/backend_service.py.schema index 09087d3ff32..1ddb5bbcd3d 100644 --- a/dm/templates/backend_service/backend_service.py.schema +++ b/dm/templates/backend_service/backend_service.py.schema @@ -15,16 +15,91 @@ info: title: Backend Service author: Sourced Group Inc. + version: 1.0.0 description: | - Creates a backend service. For details, visit + Creates a backend service. + + For more information on this resource: https://cloud.google.com/load-balancing/docs/backend-service. + APIs endpoints used by this template: + - gcp-types/compute-v1:backendServices => + https://cloud.google.com/compute/docs/reference/rest/v1/backendServices + - gcp-types/compute-v1:regionBackendServices => + https://cloud.google.com/compute/docs/reference/rest/v1/regionBackendServices + additionalProperties: false + +allOf: + - oneOf: + - required: + - healthCheck + - required: + - healthChecks + - allOf: + - not: + required: + - healthCheck + - not: + required: + - healthChecks + - oneOf: + - allOf: + - properties: + loadBalancingScheme: + enum: ["INTERNAL"] + sessionAffinity: + enum: + - NONE + - CLIENT_IP + - CLIENT_IP_PROTO + - CLIENT_IP_PORT_PROTO + protocol: + default: TCP + enum: + - UDP + - TCP + backends: + items: + balancingMode: + enum: ["CONNECTION"] + - not: + required: + - affinityCookieTtlSec + - not: + required: + - enableCDN + - not: + required: + - portName + - allOf: + - properties: + loadBalancingScheme: + enum: ["EXTERNAL"] + sessionAffinity: + enum: + - NONE + - CLIENT_IP + - GENERATED_COOKIE + protocol: + default: HTTP + enum: + - HTTP + - HTTPS + - TCP + - SSL + - required: + - portName + properties: name: type: string - description: The backend service name. + description: The backend service name. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the service. description: type: string description: An optional description of the resource. @@ -34,13 +109,45 @@ properties: The URL of the region where the regional backend service resides. backends: type: array + uniqueItems: true description: | The list of backends (instance groups) to which the backend service distributes traffic. items: type: object + additionalProperties: false required: - group + oneOf: + - allOf: + - properties: + balancingMode: + enum: ["RATE"] + - not: + required: + - maxUtilization + - not: + required: + - maxConnections + - not: + required: + - maxConnectionsPerInstance + - not: + required: + - maxConnectionsPerEndpoint + - allOf: + - properties: + balancingMode: + enum: ["CONNECTION"] + - not: + required: + - maxUtilization + - not: + required: + - maxRate + - properties: + balancingMode: + enum: ["UTILIZATION"] properties: description: type: string @@ -83,6 +190,14 @@ properties: Can be used with any balancing mode. For the RATE mode, either maxRate or maxRatePerInstance must be set. Cannot be used for INTERNAL load balancing. + maxRatePerEndpoint: + type: number + description: | + The max requests per second (RPS) that a single backend network endpoint can handle. + This is used to calculate the capacity of the group. Can be used in either balancing mode. + For RATE mode, either maxRate or maxRatePerEndpoint must be set. + + This cannot be used for internal load balancing. maxConnections: type: number description: | @@ -98,6 +213,15 @@ properties: group. Can be used in either CONNECTION or UTILIZATION balancing modes. For the CONNECTION mode, either maxConnections or maxConnectionsPerInstance must be set. Cannot be used for INTERNAL load balancing. + maxConnectionsPerEndpoint: + type: number + description: | + The max number of simultaneous connections that a single backend network endpoint can handle. + This is used to calculate the capacity of the group. Can be used in either + CONNECTION or UTILIZATION balancing modes. + For CONNECTION mode, either maxConnections or maxConnectionsPerEndpoint must be set. + + This cannot be used for internal load balancing. capacityScaler: type: number minimum: 0 @@ -111,6 +235,15 @@ properties: description: | The URL of the HealthCheck, HttpHealthCheck, or HttpsHealthCheck resource for healthchecking the backend service. + healthChecks: + type: array + uniqueItems: true + maxItems: 1 + description: | + The URL of the HealthCheck, HttpHealthCheck, or HttpsHealthCheck resource + for healthchecking the backend service. + items: + type: string timeoutSec: type: number default: 30 @@ -170,11 +303,13 @@ properties: Defines whether the backend service is used with INTERNAL or EXTERNAL load balancing schema. Backend service created for one type of load balancing cannot be used with the other. + default: EXTERNAL enum: - INTERNAL - EXTERNAL connectionDraining: type: object + additionalProperties: false description: the connection draining settings. properties: drainingTimeoutSec: @@ -182,12 +317,31 @@ properties: description: | The time period during which the instance is drained (not accepting new connections but still procedding the ones accepted earlier). + customRequestHeaders: + type: array + uniqueItems: true + description: | + Headers that the HTTP/S load balancer should add to proxied requests. + items: + type: string + iap: + type: object + additionalProperties: false + properties: + enabled: + type: boolean + oauth2ClientId: + type: string + oauth2ClientSecret: + type: string cdnPolicy: type: object + additionalProperties: false description: The cloud CDN configuration for the backend service. properties: cacheKeyPolicy: type: object + additionalProperties: false description: The CacheKeyPolicy for the CdnPolicy. properties: includeProtocol: @@ -207,6 +361,7 @@ properties: False, the query string is excluded from the cache key entirely. queryStringWhitelist: type: array + uniqueItems: true description: | The names of the query string parameters to include in cache keys. All other parameters are excluded. Specify either @@ -216,6 +371,7 @@ properties: type: string queryStringBlacklist: type: array + uniqueItems: true description: | The names of query string parameters to exclude from cache keys. All other parameters are included. Specify either diff --git a/dm/templates/backend_service/examples/backend_service_global.yaml b/dm/templates/backend_service/examples/backend_service_global.yaml index 8e9ddb8d7e1..24f212859b9 100644 --- a/dm/templates/backend_service/examples/backend_service_global.yaml +++ b/dm/templates/backend_service/examples/backend_service_global.yaml @@ -20,4 +20,5 @@ resources: - group: balancingMode: RATE maxRate: 10000 - healthCheck: + healthChecks: + - diff --git a/dm/templates/backend_service/examples/backend_service_regional.yaml b/dm/templates/backend_service/examples/backend_service_regional.yaml index 2eebdd5603f..3dd10cee253 100644 --- a/dm/templates/backend_service/examples/backend_service_regional.yaml +++ b/dm/templates/backend_service/examples/backend_service_regional.yaml @@ -21,4 +21,6 @@ resources: loadBalancingScheme: INTERNAL backends: - group: - healthCheck: + balancingMode: CONNECTION + healthChecks: + - diff --git a/dm/templates/backend_service/tests/integration/backend_service.yaml b/dm/templates/backend_service/tests/integration/backend_service.yaml index 17b9e8e91a8..d7bf314045d 100644 --- a/dm/templates/backend_service/tests/integration/backend_service.yaml +++ b/dm/templates/backend_service/tests/integration/backend_service.yaml @@ -8,7 +8,7 @@ resources: - name: regional-internal-backend-service-${RAND} type: backend_service.py properties: - name: regional-backend-${RAND} + name: regional-internal-backend-service-${RAND} region: ${REGION} protocol: ${REGIONAL_BALANCING_PROTOCOL} description: ${RES_DESCRIPTION} diff --git a/dm/templates/bastion/bastion.py b/dm/templates/bastion/bastion.py index 26ff2f99dc5..9d8f6372600 100644 --- a/dm/templates/bastion/bastion.py +++ b/dm/templates/bastion/bastion.py @@ -65,7 +65,8 @@ def get_ssh_firewall_rule( ssh_rule = { 'name': name, - 'type': 'compute.v1.firewall', + # https://cloud.google.com/compute/docs/reference/rest/v1/firewalls + 'type': 'gcp-types/compute-v1:firewalls', 'properties': ssh_props } @@ -85,7 +86,7 @@ def get_ssh_firewall_rule( ] -def create_bastion_in_ssh_rule(bastion, firewall_settings): +def create_bastion_in_ssh_rule(bastion, network, firewall_settings): """ Creates a firewall rule for inbound SSH traffic. """ to_bastion_rule = firewall_settings.get('sshToBastion') @@ -100,15 +101,16 @@ def create_bastion_in_ssh_rule(bastion, firewall_settings): bastion['properties']['tags'] = {'items': existing_tags} rule_setup = { + 'name': to_bastion_rule['name'], 'sourceTags': to_bastion_rule.get('sourceTags'), 'targetTags': [bastion_host_tag], 'sourceRanges': to_bastion_rule.get('sourceRanges'), 'priority': to_bastion_rule.get('priority'), - 'network': bastion['properties']['network'] + 'network': network } return get_ssh_firewall_rule( - to_bastion_rule['name'], + '{}-to'.format(bastion['name']), rule_setup, 'sshToBastionRuleName', 'sshToBastionRuleSelfLink' @@ -117,7 +119,7 @@ def create_bastion_in_ssh_rule(bastion, firewall_settings): return [], [] -def create_bastion_out_ssh_rule(bastion, firewall_settings): +def create_bastion_out_ssh_rule(bastion, network, firewall_settings): """ Creates a firewall rule for the Bastion outbound SSH traffic. """ from_bastion_rule = firewall_settings.get('sshFromBastion') @@ -138,14 +140,15 @@ def create_bastion_out_ssh_rule(bastion, firewall_settings): raise ValueError(msg) rule_setup = { + 'name': from_bastion_rule['name'], 'sourceTags': bastion_host_tags, 'targetTags': [bastion_target_tag], 'priority': from_bastion_rule.get('priority'), - 'network': bastion['properties']['network'], + 'network': network, } return get_ssh_firewall_rule( - from_bastion_rule['name'], + '{}-from'.format(bastion['name']), rule_setup, 'sshFromBastionRuleName', 'sshFromBastionRuleSelfLink' @@ -154,16 +157,18 @@ def create_bastion_out_ssh_rule(bastion, firewall_settings): return [], [] -def create_firewall_rules(bastion, firewall_settings): +def create_firewall_rules(bastion, network, firewall_settings): """ Creates in/out SSH rules for the Bastion host. """ ssh_in_resources, ssh_in_outputs = create_bastion_in_ssh_rule( bastion, + network, firewall_settings ) ssh_out_resources, ssh_out_outputs = create_bastion_out_ssh_rule( bastion, + network, firewall_settings ) @@ -177,16 +182,29 @@ def generate_config(context): """ Entry point for the deployment resources. """ properties = context.properties + project_id = properties.get('project', context.env['project']) name = properties.get('name', context.env['name']) + network = properties['network'] + if not '$(ref' in network: + if not 'global/' in network: + network = 'global/networks/{}'.format(network) + if not 'project/' in network: + network = 'projects/{}/{}'.format(project_id, network) + bastion_props = { + 'project': project_id, + 'name': name, 'zone': properties['zone'], - 'network': properties['network'], + 'networks': [{ + 'network': network, + 'accessConfigs': [{'type': 'ONE_TO_ONE_NAT'}] + }], 'machineType': properties['machineType'], - 'diskImage': IMAGE + 'diskImage': IMAGE, } - bastion = {'name': name, 'type': 'instance.py', 'properties': bastion_props} + bastion = {'name': context.env['name'], 'type': 'instance.py', 'properties': bastion_props} optional_props = ['diskSizeGb', 'metadata', 'tags'] @@ -200,12 +218,17 @@ def generate_config(context): if firewall_settings: extra_resources, extra_outputs = create_firewall_rules( bastion, + network, firewall_settings ) else: extra_resources = [] extra_outputs = [] + resources = [bastion] + extra_resources + for resource in resources: + resource['properties']['project'] = project_id + outputs = [ { 'name': 'name', @@ -213,19 +236,19 @@ def generate_config(context): }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) }, { 'name': 'internalIp', - 'value': '$(ref.{}.internalIp)'.format(name) + 'value': '$(ref.{}.internalIp)'.format(context.env['name']) }, { 'name': 'externalIp', - 'value': '$(ref.{}.externalIp)'.format(name) + 'value': '$(ref.{}.externalIp)'.format(context.env['name']) } ] return { - 'resources': [bastion] + extra_resources, + 'resources': resources, 'outputs': outputs + extra_outputs } diff --git a/dm/templates/bastion/bastion.py.schema b/dm/templates/bastion/bastion.py.schema index b6c8193188c..d960e981877 100644 --- a/dm/templates/bastion/bastion.py.schema +++ b/dm/templates/bastion/bastion.py.schema @@ -15,10 +15,20 @@ info: title: Bastion Host author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of a Bastion host - a jump host for SSHing into those instances that have no external IP address. + For more information on this resource: + https://cloud.google.com/solutions/connecting-securely + + APIs endpoints used by this template: + - gcp-types/compute-v1:firewalls => + https://cloud.google.com/compute/docs/reference/rest/v1/firewalls + - gcp-types/compute-v1:instances => + https://cloud.google.com/compute/docs/reference/rest/v1/instances + imports: - path: ../instance/instance.py name: instance.py @@ -32,7 +42,12 @@ required: properties: name: type: string - description: The name of the Bastion host. + description: | + The name of the Bastion host. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the instance. network: type: string description: | @@ -57,10 +72,14 @@ properties: type: boolean default: true description: | + !!! WARNING !!! This feature does not stop the GCP deamon addig + Cloud Identity users to sudoers based on Compute OS Login Admin IAM. + When Trues (default), disables `sudo` on the Bastion host for tighter security. metadata: type: object + additionalProperties: false description: | The Bastion host metadata. If the 'disableSudo' property is True, this is the only place where you can configure the Bastion host using @@ -73,9 +92,11 @@ properties: properties: items: type: array + uniqueItems: True description: A collection of metadata key-value pairs. items: type: object + additionalProperties: false properties: key: type: string @@ -83,6 +104,7 @@ properties: type: [string, number, boolean] createFirewallRules: type: object + additionalProperties: false description: | If set, creates the firewall rules for the SSH traffic coming in an out of the Bastion host. @@ -94,6 +116,7 @@ properties: properties: sshToBastion: type: object + additionalProperties: false description: | Configures the firewall rule that controls the SSH traffic flow to the Bastion host. If none of the other SSH firewall rules exist, this @@ -118,6 +141,7 @@ properties: description: The name of the firewall rule. sourceTags: type: array + uniqueItems: True description: | If source tags are specified, the firewall rule applies only to the traffic with source IPs that match the primary network @@ -127,6 +151,7 @@ properties: type: string sourceRanges: type: array + uniqueItems: True description: | If source ranges are specified, the firewall applies only to the traffic that has source IP address in these ranges. These ranges @@ -140,6 +165,7 @@ properties: maximum: 65535 sshFromBastion: type: object + additionalProperties: false description: | Creates a firewall rule that allows the SSH traffic from the Bastion host to any instance within the same network with a particular tag. @@ -166,12 +192,14 @@ properties: maximum: 65535 tags: type: object + additionalProperties: false required: - items description: Tags to apply to the instance. properties: items: type: array + uniqueItems: True description: An array of tags. items: type: string diff --git a/dm/templates/bastion/tests/integration/bastion.bats b/dm/templates/bastion/tests/integration/bastion.bats index c49fde3784f..7dec17b3dc2 100755 --- a/dm/templates/bastion/tests/integration/bastion.bats +++ b/dm/templates/bastion/tests/integration/bastion.bats @@ -28,7 +28,7 @@ if [[ -e "${RANDOM_FILE}" ]]; then export ZONE="us-central1-c" export BASTION1_DISABLE_SUDO="false" export BASTION2_DISABLE_SUDO="true" - export BASTION2_DISK_SIZE="20" + export BASTION2_DISK_SIZE="10" export NETWORK_NAME="test-network-${RAND}" export PROVISION_COMPLETED_MARKER="provision-completed-marker" export BASTION2_STARTUP="echo '${PROVISION_COMPLETED_MARKER}'" @@ -78,6 +78,8 @@ function teardown() { run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --config ${CONFIG} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] } @@ -85,6 +87,8 @@ function teardown() { run gcloud compute instances describe ${BASTION1_RES_NAME} \ --zone ${ZONE} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "machineTypes/${BASTION1_MACHINE_TYPE}" ]] [[ "$output" =~ "zones/$(ZONE)" ]] @@ -93,17 +97,23 @@ function teardown() { @test "Verifying the first Bastion's sudo is ON" { # Wait until VM provisioning finishes + i=0 until gcloud compute instances get-serial-port-output \ ${BASTION1_RES_NAME} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" \ --zone ${ZONE} | grep ${PROVISION_COMPLETED_MARKER}; do - sleep 10; + sleep 5; + i=$(($i+1)) + + if [[ $i > 10 ]]; then break; fi done run gcloud compute ssh ${BASTION1_RES_NAME} --command "sudo whoami" \ --zone ${ZONE} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "root" ]] } @@ -112,6 +122,8 @@ function teardown() { run gcloud compute instances describe ${BASTION2_NAME} \ --zone ${ZONE} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "machineTypes/${DEFAULT_MACHINE_TYPE}" ]] [[ "$output" =~ "zones/${ZONE}" ]] @@ -124,30 +136,43 @@ function teardown() { run gcloud compute disks describe ${BASTION2_NAME} \ --zone ${ZONE} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "sizeGb: '${BASTION2_DISK_SIZE}'" ]] } -@test "Verifying the second Bastion's sudo is OFF" { - # Wait until VM provisioning finishes - until gcloud compute instances get-serial-port-output ${BASTION2_NAME} \ - --project "${CLOUD_FOUNDATION_PROJECT_ID}" \ - --zone ${ZONE} | grep ${PROVISION_COMPLETED_MARKER}; do - sleep 10; - done - - run gcloud compute ssh ${BASTION2_NAME} --command "sudo -n whoami" \ - --zone ${ZONE} \ - --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$status" -eq 0 ]] -} +### Invalid test because Compute OS Login Admin IAM role adds sudoers ### +# +#@test "Verifying the second Bastion's sudo is OFF" { +# # Wait until VM provisioning finishes +# i=0 +# until gcloud compute instances get-serial-port-output ${BASTION2_NAME} \ +# --project "${CLOUD_FOUNDATION_PROJECT_ID}" \ +# --zone ${ZONE} | grep ${PROVISION_COMPLETED_MARKER}; do +# +# sleep 5; +# i=$(($i+1)) +# +# if [[ $i > 10 ]]; then break; fi +# done +# +# run gcloud compute ssh ${BASTION2_NAME} --command "sudo -n whoami" \ +# --zone ${ZONE} \ +# --project "${CLOUD_FOUNDATION_PROJECT_ID}" +# echo "status = ${status}" +# echo "output = ${output}" +# [[ ! "$status" -eq 0 ]] +#} @test "Verifying the second Bastion's tags" { run gcloud compute instances describe ${BASTION2_NAME} \ --format "yaml(tags)" \ --zone ${ZONE} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "- ${BASTION2_EXTRA_TAG}" ]] [[ "$output" =~ "- ${BASTION2_TAG}" ]] @@ -156,6 +181,8 @@ function teardown() { @test "Verifying Bastion's inbound firewall rule" { run gcloud compute firewall-rules describe "${SSH_TO_BASTION_RULE_NAME}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "IPProtocol: tcp" ]] [[ "$output" =~ "- '22'" ]] @@ -169,6 +196,8 @@ function teardown() { run gcloud compute firewall-rules describe "${SSH_TO_BASTION_RULE_NAME}" \ --format="yaml(sourceRanges)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "${SSH_TO_BASTION_SOURCE_RANGE}" ]] } @@ -177,6 +206,8 @@ function teardown() { run gcloud compute firewall-rules describe "${SSH_TO_BASTION_RULE_NAME}" \ --format="yaml(sourceTags)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "${SSH_TO_BASTION_SOURCE_TAG}" ]] } @@ -185,6 +216,8 @@ function teardown() { run gcloud compute firewall-rules describe "${SSH_TO_BASTION_RULE_NAME}" \ --format="yaml(targetTags)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "${BASTION2_TAG}" ]] } @@ -193,6 +226,8 @@ function teardown() { run gcloud compute firewall-rules describe \ "${SSH_FROM_BASTION_DEFAULT_RULE_NAME}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "IPProtocol: tcp" ]] [[ "$output" =~ "- '22'" ]] @@ -207,6 +242,8 @@ function teardown() { "${SSH_FROM_BASTION_DEFAULT_RULE_NAME}" \ --format="yaml(sourceTags)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "${BASTION2_TAG}" ]] } @@ -216,6 +253,8 @@ function teardown() { "${SSH_FROM_BASTION_DEFAULT_RULE_NAME}" \ --format="yaml(targetTags)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "status = ${status}" + echo "output = ${output}" [[ "$status" -eq 0 ]] [[ "$output" =~ "${SSH_FROM_BASTION_SOURCE_TAG}" ]] } diff --git a/dm/templates/bigquery/bigquery_dataset.py b/dm/templates/bigquery/bigquery_dataset.py index 11d3849e1f2..e2725c7d18e 100644 --- a/dm/templates/bigquery/bigquery_dataset.py +++ b/dm/templates/bigquery/bigquery_dataset.py @@ -20,15 +20,18 @@ def generate_config(context): # You can modify the roles you wish to whitelist. whitelisted_roles = ['READER', 'WRITER', 'OWNER'] - name = context.properties['name'] + properties = context.properties + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) properties = { 'datasetReference': { 'datasetId': name, - 'projectId': context.env['project'] + 'projectId': project_id }, - 'location': context.properties['location'] + 'location': context.properties['location'], + 'projectId': project_id, } optional_properties = ['description', 'defaultTableExpirationMs'] @@ -68,8 +71,9 @@ def generate_config(context): resources = [ { - 'type': 'bigquery.v2.dataset', - 'name': name, + # https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets + 'type': 'gcp-types/bigquery-v2:datasets', + 'name': context.env['name'], 'properties': properties } ] @@ -77,7 +81,7 @@ def generate_config(context): outputs = [ { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) }, { 'name': 'datasetId', @@ -85,15 +89,15 @@ def generate_config(context): }, { 'name': 'etag', - 'value': '$(ref.{}.etag)'.format(name) + 'value': '$(ref.{}.etag)'.format(context.env['name']) }, { 'name': 'creationTime', - 'value': '$(ref.{}.creationTime)'.format(name) + 'value': '$(ref.{}.creationTime)'.format(context.env['name']) }, { 'name': 'lastModifiedTime', - 'value': '$(ref.{}.lastModifiedTime)'.format(name) + 'value': '$(ref.{}.lastModifiedTime)'.format(context.env['name']) } ] diff --git a/dm/templates/bigquery/bigquery_dataset.py.schema b/dm/templates/bigquery/bigquery_dataset.py.schema index 1e07c2fcd37..d53678ffdd5 100644 --- a/dm/templates/bigquery/bigquery_dataset.py.schema +++ b/dm/templates/bigquery/bigquery_dataset.py.schema @@ -15,11 +15,17 @@ info: title: BigQuery Dataset author: Sourced Group Inc. + version: 1.0.0 description: | Creates a BigQuery dataset. + For information on this resource: https://cloud.google.com/bigquery/docs/. + APIs endpoints used by this template: + - gcp-types/bigquery-v2:datasets => + https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets + imports: - path: bigquery_dataset.py @@ -31,7 +37,21 @@ required: properties: name: type: string - description: The resource name. + description: | + The table dataset name. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the dataset. The + Google apps domain is prefixed if applicable. + friendlyName: + type: string + description: | + A descriptive name for the dataset. + description: + type: string + description: | + A user-friendly description of the dataset. location: type: string description: | @@ -45,6 +65,7 @@ properties: - US access: type: array + uniqueItems: true description: | An array of objects that define dataset access for one or more entities. You can set this property when inserting or updating @@ -56,33 +77,34 @@ properties: access.specialGroup: projectOwners; access.role: OWNER access.userByEmail: [dataset creator email]; access.role: OWNER items: - role: - type: string - description: | - The role (rights) granted to the user specified by the other - member of the access object. The following string values are - supported: READER, WRITER, OWNER. See details at - https://cloud.google.com/bigquery/docs/access-control. - enum: - - READER - - WRITER - - OWNER - oneOf: - - domain: + type: object + additionalProperties: false + required: + - role + properties: + role: + type: string + description: | + An IAM role ID that should be granted to the user, group, or domain specified in this access entry. + The following legacy mappings will be applied: OWNER <=> roles/bigquery.dataOwner + WRITER <=> roles/bigquery.dataEditor READER <=> roles/bigquery.dataViewer This field will accept any of + the above formats, but will return only the legacy format. For example, if you set this field to + "roles/bigquery.dataOwner", it will be returned back as "OWNER". @mutable bigquery.datasets.update + domain: type: string description: | The domain to grant access to. All users signed in with the specified domain are granted the corresponding access. Example: "example.com". - - userByEmail: + userByEmail: type: string description: | The email address of a user to grant access to. For example: fred@example.com. - - groupByEmail: + groupByEmail: type: string description: The email address of a Google Group to grant access to. - - specialGroup: + specialGroup: type: string description: | The special group to grant access to. Possible values include: @@ -90,8 +112,9 @@ properties: projectReaders: readers of the enclosing project projectWriters: writers of the enclosing project allAuthenticatedUsers: all authenticated BigQuery users - - view: + view: type: object + additionalProperties: false description: | A view from a different dataset to grant access to. Queries executed against that view have the Read access to tables in that @@ -112,9 +135,6 @@ properties: The table ID. The ID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_). The maximum length is 1,024 characters. - description: - type: string - description: A user-friendly description of the dataset. setDefaultOwner: type: boolean default: False @@ -136,6 +156,26 @@ properties: expirationTime while creating the table, that value takes precedence over the default expiration time indicated by this property. minimum: 3600000 + defaultPartitionExpirationMs: + type: string + format: int64 + description: | + The default partition expiration for all partitioned tables in the dataset, in milliseconds. + Once this property is set, all newly-created partitioned tables in the dataset will have an expirationMs + property in the timePartitioning settings set to this value, and changing the value will only affect new tables, + not existing ones. The storage in a partition will have an expiration time of its partition time plus this value. + Setting this property overrides the use of defaultTableExpirationMs for partitioned tables: only one of + defaultTableExpirationMs and defaultPartitionExpirationMs will be used for any new partitioned table. + If you provide an explicit timePartitioning.expirationMs when creating or updating a partitioned table, + that value takes precedence over the default partition expiration time indicated by this property. + labels: + type: object + description: | + Map labels associated with this dataset. + Example: + name: wrench + mass: 1.3kg + count: 3 outputs: properties: diff --git a/dm/templates/bigquery/bigquery_table.py b/dm/templates/bigquery/bigquery_table.py index ea527d3366f..b129d8329de 100644 --- a/dm/templates/bigquery/bigquery_table.py +++ b/dm/templates/bigquery/bigquery_table.py @@ -18,16 +18,19 @@ def generate_config(context): """ Entry point for the deployment resources. """ - name = context.properties['name'] + properties = context.properties + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) properties = { 'tableReference': { 'tableId': name, 'datasetId': context.properties['datasetId'], - 'projectId': context.env['project'] + 'projectId': project_id }, - 'datasetId': context.properties['datasetId'] + 'datasetId': context.properties['datasetId'], + 'projectId': project_id, } optional_properties = [ @@ -48,51 +51,52 @@ def generate_config(context): resources = [ { - 'type': 'bigquery.v2.table', - 'name': name, - 'properties': properties, - 'metadata': { - 'dependsOn': [context.properties['datasetId']] - } + # https://cloud.google.com/bigquery/docs/reference/rest/v2/tables + 'type': 'gcp-types/bigquery-v2:tables', + 'name': context.env['name'], + 'properties': properties } ] + if 'dependsOn' in context.properties: + resources[0]['metadata'] = {'dependsOn': context.properties['dependsOn']} + outputs = [ { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) }, { 'name': 'etag', - 'value': '$(ref.{}.etag)'.format(name) + 'value': '$(ref.{}.etag)'.format(context.env['name']) }, { 'name': 'creationTime', - 'value': '$(ref.{}.creationTime)'.format(name) + 'value': '$(ref.{}.creationTime)'.format(context.env['name']) }, { 'name': 'lastModifiedTime', - 'value': '$(ref.{}.lastModifiedTime)'.format(name) + 'value': '$(ref.{}.lastModifiedTime)'.format(context.env['name']) }, { 'name': 'location', - 'value': '$(ref.{}.location)'.format(name) + 'value': '$(ref.{}.location)'.format(context.env['name']) }, { 'name': 'numBytes', - 'value': '$(ref.{}.numBytes)'.format(name) + 'value': '$(ref.{}.numBytes)'.format(context.env['name']) }, { 'name': 'numLongTermBytes', - 'value': '$(ref.{}.numLongTermBytes)'.format(name) + 'value': '$(ref.{}.numLongTermBytes)'.format(context.env['name']) }, { 'name': 'numRows', - 'value': '$(ref.{}.numRows)'.format(name) + 'value': '$(ref.{}.numRows)'.format(context.env['name']) }, { 'name': 'type', - 'value': '$(ref.{}.type)'.format(name) + 'value': '$(ref.{}.type)'.format(context.env['name']) } ] diff --git a/dm/templates/bigquery/bigquery_table.py.schema b/dm/templates/bigquery/bigquery_table.py.schema index ad0cbb8865c..e36cf31a921 100644 --- a/dm/templates/bigquery/bigquery_table.py.schema +++ b/dm/templates/bigquery/bigquery_table.py.schema @@ -15,11 +15,17 @@ info: title: BigQuery Table author: Sourced Group Inc. + version: 1.0.0 description: | Creates a BigQuery table. - For more information on this resource: + + For information on this resource: https://cloud.google.com/bigquery/docs/. + APIs endpoints used by this template: + - gcp-types/bigquery-v2:tables => + https://cloud.google.com/bigquery/docs/reference/rest/v2/tables + imports: - path: bigquery_table.py @@ -31,14 +37,25 @@ required: properties: name: type: string - description: The resource name. + description: | + The table name name. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the table. The + Google apps domain is prefixed if applicable. datasetId: type: string description: | The ID of the dataset the table belongs to. friendlyName: type: string - description: A descriptive name for the table. + description: | + A descriptive name for the table. + description: + type: string + description: | + A user-friendly description of the dataset. expirationTime: type: string description: | @@ -47,9 +64,314 @@ properties: deleted, and their storage is reclaimed. The defaultTableExpirationMs property of the encapsulating dataset can be used to set a default expirationTime on newly created tables. For example, 1535739430. + encryptionConfiguration: + type: object + additionalProperties: false + description: | + Custom encryption configuration (e.g., Cloud KMS keys). + properties: + kmsKeyName: + type: string + description: | + Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. + The BigQuery Service Account associated with your project requires access to this encryption key. + externalDataConfiguration: + type: object + additionalProperties: false + description: | + Describes the data format, location, and other properties of a table stored outside of BigQuery. + By defining these properties, the data source can then be queried as if it were a standard BigQuery table. + required: + - sourceUris + - sourceFormat + properties: + sourceUris: + type: array + minItems: 1 + uniqueItems: true + description: | + The fully-qualified URIs that point to your data in Google Cloud. For Google Cloud Storage URIs: + Each URI can contain one '*' wildcard character and it must come after the 'bucket' name. + Size limits related to load jobs apply to external data sources. For Google Cloud Bigtable URIs: + Exactly one URI can be specified and it has be a fully specified and valid HTTPS URL for a + Google Cloud Bigtable table. For Google Cloud Datastore backups, exactly one URI can be specified. + Also, the '*' wildcard character is not allowed. + items: + type: object + schema: + type: object + description: | + The schema for the data. Schema is required for CSV and JSON formats. Schema is disallowed for + Google Cloud Bigtable, Cloud Datastore backups, and Avro formats. + sourceFormat: + type: string + description: | + The data format. For CSV files, specify "CSV". For Google sheets, specify "GOOGLE_SHEETS". + For newline-delimited JSON, specify "NEWLINE_DELIMITED_JSON". For Avro files, specify "AVRO". + For Google Cloud Datastore backups, specify "DATASTORE_BACKUP". + [Beta] For Google Cloud Bigtable, specify "BIGTABLE". + enum: + - CSV + - GOOGLE_SHEETS + - NEWLINE_DELIMITED_JSON + - AVRO + - DATASTORE_BACKUP + - BIGTABLE + maxBadRecords: + type: number + description: | + The maximum number of bad records that BigQuery can ignore when reading data. If the number of + bad records exceeds this value, an invalid error is returned in the job result. + The default value is 0, which requires that all records are valid. This setting is ignored + for Google Cloud Bigtable, Google Cloud Datastore backups and Avro formats. + autodetect: + type: boolean + description: | + Indicates if BigQuery should allow extra values that are not represented in the table schema. + If true, the extra values are ignored. If false, records with extra columns are treated as bad records, + and if there are too many bad records, an invalid error is returned in the job result. + The default value is false. The sourceFormat property determines what BigQuery treats as an extra value: + CSV: Trailing columns JSON: Named values that don't match any column names + Google Cloud Bigtable: This setting is ignored + Google Cloud Datastore backups: This setting is ignored + Avro: This setting is ignored. + compression: + type: string + description: | + The compression type of the data source. Possible values include GZIP and NONE. The default value is NONE. + This setting is ignored for Google Cloud Bigtable, Google Cloud Datastore backups and Avro formats. + An empty string is an invalid value. + enum: + - NONE + - GZIP + csvOptions: + type: object + additionalProperties: false + description: | + Additional properties to set if sourceFormat is set to CSV. + properties: + fieldDelimiter: + type: string + description: | + The separator for fields in a CSV file. BigQuery converts the string to ISO-8859-1 encoding, + and then uses the first byte of the encoded string to split the data in its raw, binary state. + BigQuery also supports the escape sequence "\t" to specify a tab separator. + The default value is a comma (','). + skipLeadingRows: + type: number + description: | + The number of rows at the top of a CSV file that BigQuery will skip when reading the data. + The default value is 0. This property is useful if you have header rows in the file that should be skipped. + quote: + type: string + description: | + The value that is used to quote data sections in a CSV file. BigQuery converts the string to + ISO-8859-1 encoding, and then uses the first byte of the encoded string to split the data in its raw, + binary state. The default value is a double-quote ('"'). If your data does not contain quoted sections, + set the property value to an empty string. If your data contains quoted newline characters, + you must also set the allowQuotedNewlines property to true. @default '"' + allowQuotedNewlines: + type: boolean + description: | + Indicates if BigQuery should allow quoted data sections that contain newline characters in a CSV file. + The default value is false. + allowJaggedRows: + type: boolean + description: | + Indicates if BigQuery should accept rows that are missing trailing optional columns. + If true, BigQuery treats missing trailing columns as null values. + If false, records with missing trailing columns are treated as bad records, and if there are + too many bad records, an invalid error is returned in the job result. The default value is false. + encoding: + type: string + description: | + The character encoding of the data. The supported values are UTF-8 or ISO-8859-1. + The default value is UTF-8. BigQuery decodes the data after the raw, binary data has + been split using the values of the quote and fieldDelimiter properties. + enum: + - UTF-8 + - ISO-8859-1 + bigtableOptions: + type: object + additionalProperties: false + description: | + Additional options if sourceFormat is set to BIGTABLE. + properties: + columnFamilies: + type: array + uniqueItems: true + description: | + tabledata.list of column families to expose in the table schema along with their types. + This list restricts the column families that can be referenced in queries and specifies their value types. + You can use this list to do type conversions - see the 'type' field for more details. + If you leave this list empty, all column families are present in the table schema and their values + are read as BYTES. During a query only the column families referenced in that query are read from Bigtable. + items: + type: object + additionalProperties: false + properties: + familyId: + type: string + description: | + Identifier of the column family. + type: + type: string + description: | + The type to convert the value in cells of this column family. The values are expected to be + encoded using HBase Bytes.toBytes function when using the BINARY encoding value. + Following BigQuery types are allowed (case-sensitive) - BYTES STRING INTEGER FLOAT BOOLEAN + Default type is BYTES. This can be overridden for a specific column by listing that + column in 'columns' and specifying a type for it. + enum: + - BYTES + - STRING + - INTEGER + - FLOAT + - BOOLEAN + encoding: + type: string + description: | + The encoding of the values when the type is not STRING. Acceptable encoding values are: + - TEXT - indicates values are alphanumeric text strings. + - BINARY - indicates values are encoded using HBase Bytes.toBytes family of functions. + This can be overridden for a specific column by listing that column in + 'columns' and specifying an encoding for it. + enum: + - TEXT + - BINARY + columns: + type: array + uniqueItems: true + description: | + Lists of columns that should be exposed as individual fields as opposed to a list of + (column name, value) pairs. All columns whose qualifier matches a qualifier in this list + can be accessed as .. Other columns can be accessed as a list through .Column field. + items: + type: object + additionalProperties: false + required: + - qualifierEncoded + properties: + qualifierEncoded: + type: string + description: | + Qualifier of the column. Columns in the parent column family that has this exact qualifier + are exposed as . field. If the qualifier is valid UTF-8 string, it can be specified in + the qualifierString field. Otherwise, a base-64 encoded value must be set to qualifierEncoded. + The column field name is the same as the column qualifier. However, if the qualifier is not a + valid BigQuery field identifier i.e. does not match [a-zA-Z][a-zA-Z0-9_]*, a valid identifier + must be provided as fieldName. + qualifierString: + type: string + fieldName: + type: string + description: | + If the qualifier is not a valid BigQuery field identifier i.e. does not match + [a-zA-Z][a-zA-Z0-9_]*, a valid identifier must be provided as the column field name + and is used as field name in queries. + type: + type: string + description: | + The type to convert the value in cells of this column. The values are expected to be + encoded using HBase Bytes.toBytes function when using the BINARY encoding value. + Following BigQuery types are allowed (case-sensitive) - BYTES STRING INTEGER FLOAT BOOLEAN + Default type is BYTES. 'type' can also be set at the column family level. + However, the setting at this level takes precedence if 'type' is set at both levels. + enum: + - BYTES + - STRING + - INTEGER + - FLOAT + - BOOLEAN + encoding: + type: string + description: | + The encoding of the values when the type is not STRING. Acceptable encoding values are: + - TEXT - indicates values are alphanumeric text strings. + - BINARY - indicates values are encoded using HBase Bytes.toBytes family of functions. + 'encoding' can also be set at the column family level. However, the setting at this level + takes precedence if 'encoding' is set at both levels. + enum: + - TEXT + - BINARY + onlyReadLatest: + type: boolean + description: | + If this is set, only the latest version of value in this column are exposed. + 'onlyReadLatest' can also be set at the column family level. However, the setting at + this level takes precedence if 'onlyReadLatest' is set at both levels. + ignoreUnspecifiedColumnFamilies: + type: boolean + description: | + If field is true, then the column families that are not specified in columnFamilies list are not + exposed in the table schema. Otherwise, they are read with BYTES type values. The default value is false. + readRowkeyAsString: + type: boolean + description: | + If field is true, then the rowkey column families will be read and converted to string. + Otherwise they are read with BYTES type values and users need to manually cast them with CAST if necessary. + The default value is false. + googleSheetsOptions: + type: object + additionalProperties: false + description: | + Additional options if sourceFormat is set to GOOGLE_SHEETS. + properties: + skipLeadingRows: + type: number + description: | + The number of rows at the top of a sheet that BigQuery will skip when reading the data. + The default value is 0. This property is useful if you have header rows that should be skipped. + When autodetect is on, behavior is the following: * skipLeadingRows unspecified - Autodetect tries to + detect headers in the first row. If they are not detected, the row is read as data. Otherwise data + is read starting from the second row. * skipLeadingRows is 0 - Instructs autodetect that there are + no headers and data should be read starting from the first row. * skipLeadingRows = N > 0 - Autodetect + skips N-1 rows and tries to detect headers in row N. If headers are not detected, row N is just skipped. + Otherwise row N is used to extract column names for the detected schema. + range: + type: string + description: | + [Beta] Range of a sheet to query from. Only used when non-empty. + hivePartitioningMode: + type: string + description: | + [Experimental] When set, what mode of hive partitioning to use when reading data. + Two modes are supported: + - AUTO: automatically infer partition key name(s) and type(s). + - STRINGS: automatically infer partition key name(s). All types are strings. + Not all storage formats support hive partitioning -- requesting hive partitioning + on an unsupported format will lead to an error. + enum: + - AUTO + - STRINGS + clustering: + type: object + additionalProperties: false + description: | + Clustering specification for the table. Must be specified with time-based partitioning, data in the table + will be first partitioned and subsequently clustered. + required: + - fields + properties: + fields: + type: array + minItems: 1 + uniqueItems: true + description: | + One or more fields on which data should be clustered. Only top-level, non-repeated, simple-type fields + are supported. The order of the fields will determine how clusters will be generated, so it is important. + items: + type: string + requirePartitionFilter: + type: boolean + description: | + [Beta] If set to true, queries over this table require a partition filter that can be used for + partition elimination to be specified. timePartitioning: type: object - description: The time-based partitioning specification for this table. + additionalProperties: false + description: | + The time-based partitioning specification for this table. properties: expirationMs: type: string @@ -69,7 +391,7 @@ properties: requirePartitionFilter: type: boolean description: | - If True, queries over the table require a partition filter + [Beta] If True, queries over the table require a partition filter (that can be used for partition elimination) to be specified. type: type: string @@ -78,6 +400,7 @@ properties: per day. view: type: object + additionalProperties: false description: The view definintion. properties: query: @@ -94,6 +417,7 @@ properties: value. userDefinedFunctionResources: type: array + uniqueItems: true description: | User-defined function resources used in the query. items: @@ -111,12 +435,14 @@ properties: (gs://bucket/path). schema: type: array + uniqueItems: true description: | The schema for the data. Required for the CSV and JSON formats. Disallowed for the Google Cloud Bigtable, Cloud Datastore backups, and Avro formats. items: type: object + additionalProperties: false description: Defines the table fields. required: - name @@ -167,6 +493,14 @@ properties: type: string description: | The field description. The maximum length is 1,024 characters. + labels: + type: object + description: | + Map labels associated with this table. + Example: + name: wrench + mass: 1.3kg + count: 3 outputs: properties: diff --git a/dm/templates/bigquery/tests/integration/bigquery.yaml b/dm/templates/bigquery/tests/integration/bigquery.yaml index 90db0f86d13..557227f2c24 100644 --- a/dm/templates/bigquery/tests/integration/bigquery.yaml +++ b/dm/templates/bigquery/tests/integration/bigquery.yaml @@ -28,6 +28,8 @@ resources: properties: name: test_bq_table_${RAND} datasetId: $(ref.test-bq-dataset-${RAND}.datasetId) + dependsOn: + - test-bq-dataset-${RAND} schema: - name: firstname type: STRING diff --git a/dm/templates/cloud_function/cloud_function.py b/dm/templates/cloud_function/cloud_function.py index 82bbaee725d..965ba0c0c5a 100644 --- a/dm/templates/cloud_function/cloud_function.py +++ b/dm/templates/cloud_function/cloud_function.py @@ -19,16 +19,16 @@ NO_RESOURCES_OR_OUTPUTS = [], [] -def get_source_url_output(function_name): +def get_source_url_output(function_name, context): """ Generates the Cloud Function output with a link to the source archive. """ return { 'name': 'sourceArchiveUrl', - 'value': '$(ref.{}.sourceArchiveUrl)'.format(function_name) + 'value': '$(ref.{}.sourceArchiveUrl)'.format(function_name, context.env['name']) } -def append_cloud_storage_sources(function, context): +def append_cloud_storage_sources(function, project, context): """ Adds source code from the Cloud Storage. """ properties = context.properties @@ -36,14 +36,14 @@ def append_cloud_storage_sources(function, context): local_path = properties.get('localUploadPath') resources = [] - outputs = [get_source_url_output(function['name'])] + outputs = [get_source_url_output(function['name'], context)] if local_path: # The 'upload.py' file must be imported into the YAML file first. from upload import generate_upload_path, upload_source upload_path = upload_path or generate_upload_path() - res = upload_source(function, context.imports, local_path, upload_path) + res = upload_source(context.env['name'], project, function, context.imports, local_path, upload_path) source_resources, source_outputs = res resources += source_resources outputs += source_outputs @@ -58,32 +58,35 @@ def append_cloud_storage_sources(function, context): def append_cloud_repository_sources(function, context): """ Adds the source code from the cloud repository. """ - append_optional_property(function, - context.properties, - 'sourceRepositoryUrl') + repo = context.properties.get('sourceRepository', { + 'url': context.properties.get('sourceRepositoryUrl') + }) + function['properties']['sourceRepository'] = repo name = function['name'] output = { 'name': 'sourceRepositoryUrl', - 'value': '$(ref.{}.sourceRepository.repositoryUrl)'.format(name) + 'value': '$(ref.{}.sourceRepository.deployedUrl)'.format(context.env['name']) } return [], [output] -def append_source_code(function, context): +def append_source_code(function, project, context): """ Append a reference to the Cloud Function's source code. """ properties = context.properties - if 'sourceArchiveUrl' in properties or 'localUploadPath' in properties: - return append_cloud_storage_sources(function, context) - elif 'sourceRepositoryUrl' in properties: + + if 'sourceRepository' in properties or 'sourceRepositoryUrl' in properties: return append_cloud_repository_sources(function, context) - msg = """At least one of the following properties must be provided: - - sourceRepositoryUrl - - localUploadPath - - sourceArchiveUrl""" - raise ValueError(msg) + if 'sourceUploadUrl' in properties: + append_optional_property(function, properties, 'sourceUploadUrl') + return [], [] + + if 'sourceArchiveUrl' in properties or 'localUploadPath' in properties: + return append_cloud_storage_sources(function, project, context) + + raise ValueError('At least one of source properties must be provided') def append_trigger_topic(function, properties): """ Appends the Pub/Sub event trigger. """ @@ -97,13 +100,13 @@ def append_trigger_topic(function, properties): return NO_RESOURCES_OR_OUTPUTS -def append_trigger_http(function): +def append_trigger_http(function, context): """ Appends the HTTPS trigger and returns the generated URL. """ function['properties']['httpsTrigger'] = {} output = { 'name': 'httpsTriggerUrl', - 'value': '$(ref.{}.httpsTrigger.url)'.format(function['name']) + 'value': '$(ref.{}.httpsTrigger.url)'.format(context.env['name']) } return [], [output] @@ -132,7 +135,7 @@ def append_trigger(function, context): elif 'triggerStorage' in context.properties: return append_trigger_storage(function, context) - return append_trigger_http(function) + return append_trigger_http(function, context) def append_optional_property(function, properties, prop_name): """ If the property is set, it is added to the function body. """ @@ -142,26 +145,32 @@ def append_optional_property(function, properties, prop_name): function['properties'][prop_name] = val return -def create_function_resource(resource_name, context): +def create_function_resource(context): """ Creates the Cloud Function resource. """ properties = context.properties - region = properties['region'] - function_name = properties.get('name', resource_name) + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) + location = properties.get('location', properties.get('region')) function = { - 'type': 'cloudfunctions.v1beta2.function', - 'name': function_name, + # https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions + 'type': 'gcp-types/cloudfunctions-v1:projects.locations.functions', + 'name': context.env['name'], 'properties': { - 'location': region, - 'function': function_name, + 'parent': 'projects/{}/locations/{}'.format(project_id, location), + 'function': name, + # 'name': 'projects/{}/locations/{}/functions/{}'.format(project_id, location, name), }, } optional_properties = ['entryPoint', + 'labels', + 'environmentVariables', 'timeout', 'runtime', + 'maxInstances', 'availableMemoryMb', 'description'] @@ -169,7 +178,7 @@ def create_function_resource(resource_name, context): append_optional_property(function, properties, prop) trigger_resources, trigger_outputs = append_trigger(function, context) - code_resources, code_outputs = append_source_code(function, context) + code_resources, code_outputs = append_source_code(function, project_id, context) if code_resources: function['metadata'] = { @@ -184,15 +193,14 @@ def create_function_resource(resource_name, context): }, { 'name': 'name', - 'value': '$(ref.{}.name)'.format(function_name) + 'value': '$(ref.{}.name)'.format(context.env['name']) } ]) def generate_config(context): """ Entry point for the deployment resources. """ - resource_name = context.env['name'] - resources, outputs = create_function_resource(resource_name, context) + resources, outputs = create_function_resource(context) return { 'resources': resources, diff --git a/dm/templates/cloud_function/cloud_function.py.schema b/dm/templates/cloud_function/cloud_function.py.schema index 95ae3e791a8..25f97e98e2d 100644 --- a/dm/templates/cloud_function/cloud_function.py.schema +++ b/dm/templates/cloud_function/cloud_function.py.schema @@ -15,23 +15,64 @@ info: title: Cloud Function author: Sourced Group Inc. + version: 1.0.0 description: | Creates a Cloud Function from a local file system, a Cloud Storage bucket, or a cloud source repository, and then assigns HTTPS, Storage, or Pub/Sub trigger to that Cloud Function. + For more information on this resource: + https://cloud.google.com/functions/ + + APIs endpoints used by this template: + - gcp-types/cloudfunctions-v1:projects.locations.functions => + https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions + - gcp-types/cloudbuild-v1:cloudbuild.projects.builds.create => + https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.builds/create + + APIs that should be enabled (on the seed project as well): + - cloudfunctions.googleapis.com + - cloudbuild.googleapis.com + + Additionally, ID@cloudbuild.gserviceaccount.com service account of the seed project should have + storage.buckets.create on the target project. + additionalProperties: false -required: - - region +allOf: + - oneOf: + - required: + - region + - required: + - location + - oneOf: + - required: + - sourceRepository + - required: + - sourceRepositoryUrl + - required: + - sourceUploadUrl + - anyOf: + - required: + - sourceArchiveUrl + - required: + - localUploadPath properties: name: type: string - description: The function name. + description: The function name. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the Cloud function. The + Google apps domain is prefixed if applicable. region: type: string - description: The region where the function is deployed. + description: The region where the function is deployed. Deprecated, use "location" field + location: + type: string + description: The location where the function is deployed. timeout: type: string description: The timeout for the function, in seconds; e.g., '120s'. @@ -42,10 +83,12 @@ properties: The runtime in which the function is going to run. See https://cloud.google.com/functions/docs/concepts/exec#runtimes. enum: - - nodejs6 + - go111 + - nodejs6 # deprecated! - nodejs8 + - nodejs10 - python37 - default: nodejs6 + default: nodejs10 availableMemoryMb: type: integer description: The amount of memory available for the function, MB. @@ -55,27 +98,53 @@ properties: description: | The function name (as defined in the source code) to be executed. Defaults to the resource name's suffix. + sourceUploadUrl: + type: string + description: | + The Google Cloud Storage signed URL used for source uploading, generated by + [google.cloud.functions.v1.GenerateUploadUrl][] localUploadPath: type: string description: | - The partial or complete path. If defined, all imported files that - start with this path are treated as function source code, archived - and then uploaded to Google Storage. The destination URL is either + The partial or complete path. If defined, all imported files that + start with this path are treated as function source code, archived + and then uploaded to Google Storage. The destination URL is either taken from the sourceArchiveUrl property or auto-generated. If the target bucket does not exist, it is created. sourceArchiveUrl: type: string description: | The URL of the archive containing the Cloud Function's source code - in Google Storage. When used along with localUploadPath, - this is the path to which the source code is uploaded. If the URL points + in Google Storage, starting with gs://, pointing to the zip archive which contains the function. + When used along with localUploadPath, this is the path to which the source code is uploaded. If the URL points to a non-existing bucket, the bucket is created automatically. + sourceRepository: + type: object + additionalProperties: false + description: | + The source repository where a function is hosted. + required: + - url + properties: + url: + type: string + description: | + The URL pointing to the hosted repository where the function is defined. There are supported Cloud Source + Repository URLs in the following formats: + + To refer to a specific commit: + https://source.developers.google.com/projects/*/repos/*/revisions/*/paths/* + To refer to a moveable alias (branch): + https://source.developers.google.com/projects/*/repos/*/moveable-aliases/*/paths/* + In particular, to refer to HEAD use master moveable alias. + To refer to a specific fixed alias (tag): + https://source.developers.google.com/projects/*/repos/*/fixed-aliases/*/paths/* + + You may omit paths/* if you want to use the main directory. sourceRepositoryUrl: type: string description: | - The URL of the cloud source repository pointing to the function source code. - E.g., https://source.developers.google.com/projects/PROJET_NAME/repos/REPO_NAME/moveable-aliases/BRANCH_NAME/paths/SUBPATH. - See more at https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions. + DEPRECATED, alias for sourceRepository->url triggerTopic: type: string description: | @@ -85,6 +154,7 @@ properties: https://cloud.google.com/functions/docs/concepts/events-triggers#events. triggerStorage: type: object + additionalProperties: false description: | Configures the Cloud Storage trigger for the function. If neither triggerTopic nor triggerStorage are provided, the function is triggered by an HTTPS call. @@ -107,6 +177,25 @@ properties: - delete - archive - metadataUpdate + labels: + type: object + description: | + Map labels associated with this Cloud Function. + Example: + name: wrench + mass: 1.3kg + count: 3 + environmentVariables: + type: object + description: | + Map of environment variables that shall be available during function execution. + Example: + FOO: BAR + maxInstances: + type: number + description: | + The limit on the maximum number of function instances that may coexist at a given time. + This feature is currently in alpha, available only for whitelisted users. outputs: properties: diff --git a/dm/templates/cloud_function/upload.py b/dm/templates/cloud_function/upload.py index 7d9c079b48e..c2ec919cc8f 100644 --- a/dm/templates/cloud_function/upload.py +++ b/dm/templates/cloud_function/upload.py @@ -30,7 +30,7 @@ def extract_source_files(imports, local_upload_path): if imported_file.startswith(local_upload_path): file_name = imported_file[len(local_upload_path):] file_content = imports[imported_file] - imported_files.append((file_name, file_content)) + imported_files.append((file_name.lstrip('/'), file_content)) return imported_files @@ -48,7 +48,7 @@ def archive_files(files): sources_zip.close() return output_file.getvalue() -def upload_source(function, imports, local_path, source_archive_url): +def upload_source(context_name, project, function, imports, local_path, source_archive_url): """ Uploads the Cloud Function source code from the local machine to a Cloud Storage bucket. If the bucket does not exist, creates it. """ @@ -80,7 +80,8 @@ def upload_source(function, imports, local_path, source_archive_url): volume_archive_path) build_action = { - 'name': 'upload-task', + 'name': '{}-upload-task'.format(context_name), + # https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.builds/create 'action': 'gcp-types/cloudbuild-v1:cloudbuild.projects.builds.create', 'metadata': { @@ -88,6 +89,7 @@ def upload_source(function, imports, local_path, source_archive_url): }, 'properties': { + 'projectId': project, 'steps': [ { # Saves a ZIP to a file. @@ -97,6 +99,7 @@ def upload_source(function, imports, local_path, source_archive_url): }, { # Creates a bucket if one does not exist. 'name': 'gcr.io/cloud-builders/gsutil', + 'env': ['CLOUDSDK_CORE_PROJECT={}'.format(project)], 'args': [ '-c', 'gsutil mb {} || true'.format(bucket_name) @@ -105,6 +108,7 @@ def upload_source(function, imports, local_path, source_archive_url): }, { # Uploads the ZIP to the bucket. 'name': 'gcr.io/cloud-builders/gsutil', + 'env': ['CLOUDSDK_CORE_PROJECT={}'.format(project)], 'args': [ 'cp', volume_archive_path, source_archive_url diff --git a/dm/templates/cloud_router/cloud_router.py b/dm/templates/cloud_router/cloud_router.py index 253a4294af6..d351b472d5c 100644 --- a/dm/templates/cloud_router/cloud_router.py +++ b/dm/templates/cloud_router/cloud_router.py @@ -14,36 +14,55 @@ """ This template creates a Cloud Router. """ +def append_optional_property(res, properties, prop_name): + """ If the property is set, it is added to the resource. """ + + val = properties.get(prop_name) + if val: + res['properties'][prop_name] = val + return + def generate_config(context): """ Entry point for the deployment resources. """ - name = context.properties.get('name', context.env['name']) + properties = context.properties + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) - resources = [ - { - 'name': context.env['name'], - 'type': 'compute.v1.router', - 'properties': - { - 'name': - name, - 'bgp': { - 'asn': context.properties['asn'] - }, - 'network': - generate_network_url( - context, - context.properties['network'] - ), - 'region': - context.properties['region'] - } - } + bgp = properties.get('bgp', {'asn': properties.get('asn')}) + + router = { + 'name': context.env['name'], + # https://cloud.google.com/compute/docs/reference/rest/v1/routers + 'type': 'gcp-types/compute-v1:routers', + 'properties': + { + 'name': + name, + 'project': + project_id, + 'region': + properties['region'], + 'bgp': bgp, + 'network': + properties.get('networkURL', generate_network_uri( + project_id, + properties.get('network', ''))), + } + } + + optional_properties = [ + 'description', + 'bgpPeers', + 'interfaces', + 'nats', ] + for prop in optional_properties: + append_optional_property(router, properties, prop) + return { - 'resources': - resources, + 'resources': [router], 'outputs': [ { @@ -64,10 +83,10 @@ def generate_config(context): } -def generate_network_url(context, network): - """Format the resource name as a resource URI.""" +def generate_network_uri(project_id, network): + """Format the network name as a network URI.""" return 'projects/{}/global/networks/{}'.format( - context.env['project'], + project_id, network ) diff --git a/dm/templates/cloud_router/cloud_router.py.schema b/dm/templates/cloud_router/cloud_router.py.schema index 43408ba0989..4776f31fa0e 100644 --- a/dm/templates/cloud_router/cloud_router.py.schema +++ b/dm/templates/cloud_router/cloud_router.py.schema @@ -15,39 +15,373 @@ info: title: Cloud Router author: Sourced Group Inc. + version: 1.0.1 description: | Deploys a Cloud Router. For more information on this resource: https://cloud.google.com/router/docs/ + APIs endpoints used by this template: + - gcp-types/compute-v1:routers => + https://cloud.google.com/compute/docs/reference/rest/v1/routers + imports: - path: cloud_router.py additionalProperties: false -required: - - network - - region - - asn +allOf: + - required: + - region + - oneOf: + - required: + - networkURL + - required: + - network + - oneOf: + - required: + - asn + - required: + - bgp properties: name: type: string - description: The name of Cloud Router the resource. - network: + description: | + Must comply with RFC1035. Specifically, the name must be 1-63 characters long and match + the regular expression [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a lowercase letter, + and all following characters must be a dash, lowercase letter, or digit, except the last character, + which cannot be a dash. + Resource name would be used if omitted. + description: + type: string + description: | + An optional description of this resource. Provide this property when you create the resource. + project: type: string - description: The name of the network to which the Cloud Router belongs. + description: | + The project ID of the project containing the Cloud Router instance. The + Google apps domain is prefixed if applicable. region: type: string description: The URI of the region where the Cloud Router resides. + networkURL: + type: string + description: The URL (or URI) of the network to which the Cloud Router belongs. + network: + type: string + description: The name of the network to which the Cloud Router belongs (without project prefix). + bgp: + type: object + additionalProperties: false + description: | + BGP information specific to this router. + required: + - asn + properties: + asn: + type: integer + description: | + The local BGP Autonomous System Number (ASN). Must be an RFC6996 private ASN, + either 16-bit or 32-bit. The value will be fixed for this router. + All VPN tunnels that link to this router will have the same + local ASN. + advertiseMode: + type: string + description: | + User-specified flag to indicate which mode to use for advertisement. The options are DEFAULT or CUSTOM. + enum: + - DEFAULT + - CUSTOM + advertisedGroups: + type: array + description: | + User-specified list of prefix groups to advertise in custom mode. This field can only be populated if + advertiseMode is CUSTOM and is advertised to all peers of the router. These groups will be advertised + in addition to any specified prefixes. Leave this field blank to advertise no custom groups. + uniqueItems: True + items: + type: string + enum: + - ALL_SUBNETS + advertisedIpRanges: + type: array + description: | + User-specified list of individual IP ranges to advertise in custom mode. This field can only be populated + if advertiseMode is CUSTOM and is advertised to all peers of the router. These IP ranges will be advertised + in addition to any specified groups. Leave this field blank to advertise no custom IP ranges. + uniqueItems: True + items: + type: object + additionalProperties: false + required: + - range + properties: + range: + type: string + description: | + The IP range to advertise. The value must be a CIDR-formatted string. + description: + type: string + description: | + User-specified description for the IP range. + bgpPeers: + type: array + description: | + BGP information that must be configured into the routing stack to establish BGP peering. This information + must specify the peer ASN and either the interface name, IP address, or peer IP address. Please refer to RFC4273. + uniqueItems: True + items: + type: object + additionalProperties: false + required: + - name + - interfaceName + - ipAddress + - peerIpAddress + properties: + name: + type: string + description: | + Name of this BGP peer. The name must be 1-63 characters long and comply with RFC1035. + interfaceName: + type: string + description: | + Name of the interface the BGP peer is associated with. + ipAddress: + type: string + description: | + IP address of the interface inside Google Cloud Platform. Only IPv4 is supported. + peerIpAddress: + type: string + description: | + IP address of the BGP interface outside Google Cloud Platform. Only IPv4 is supported. + peerAsn: + type: string + description: | + Peer BGP Autonomous System Number (ASN). Each BGP interface may use a different value. + advertisedRoutePriority: + type: string + description: | + The priority of routes advertised to this BGP peer. Where there is more than one matching + route of maximum length, the routes with the lowest priority value win. + advertiseMode: + type: string + description: | + User-specified flag to indicate which mode to use for advertisement. + advertisedGroups: + type: array + description: | + User-specified list of prefix groups to advertise in custom mode, which can take + one of the following options: + + - ALL_SUBNETS: Advertises all available subnets, including peer VPC subnets. + - ALL_VPC_SUBNETS: Advertises the router's own VPC subnets. + - ALL_PEER_VPC_SUBNETS: Advertises peer subnets of the router's VPC network. + Note that this field can only be populated if advertiseMode is CUSTOM and overrides the list + defined for the router (in the "bgp" message). These groups are advertised in addition + to any specified prefixes. Leave this field blank to advertise no custom groups. + uniqueItems: True + items: + type: string + enum: + - ALL_SUBNETS + - ALL_VPC_SUBNETS + - ALL_PEER_VPC_SUBNETS + advertisedIpRanges: + type: array + description: | + User-specified list of individual IP ranges to advertise in custom mode. This field can only + be populated if advertiseMode is CUSTOM and overrides the list defined for + the router (in the "bgp" message). These IP ranges are advertised in addition to any specified groups. + Leave this field blank to advertise no custom IP ranges. + uniqueItems: True + items: + type: object + additionalProperties: false + required: + - range + properties: + range: + type: string + description: | + The IP range to advertise. The value must be a CIDR-formatted string. + randescriptionge: + type: string + description: | + User-specified description for the IP range. + interfaces: + type: array + description: | + Router interfaces. Each interface requires either one linked resource, (for example, linkedVpnTunnel), + or IP address and IP address range (for example, ipRange), or both. + uniqueItems: True + items: + type: object + additionalProperties: false + required: + - name + oneOf: + - allOf: + - required: + - linkedVpnTunnel + - not: + required: + - linkedInterconnectAttachment + - allOf: + - required: + - linkedInterconnectAttachment + - not: + required: + - linkedVpnTunnel + properties: + name: + type: string + description: | + Name of this interface entry. The name must be 1-63 characters long and comply with RFC1035. + linkedVpnTunnel: + type: string + description: | + URI of the linked VPN tunnel, which must be in the same region as the router. Each interface can have + one linked resource, which can be either a VPN tunnel or an Interconnect attachment. + linkedInterconnectAttachment: + type: string + description: | + URI of the linked Interconnect attachment. It must be in the same region as the router. Each interface + can have one linked resource, which can be either be a VPN tunnel or an Interconnect attachment. + ipRange: + type: string + description: | + IP address and range of the interface. The IP range must be in the RFC3927 link-local IP address space. + The value must be a CIDR-formatted string, for example: 169.254.0.1/30. NOTE: Do not truncate the address + as it represents the IP address of the interface. + nats: + type: array + description: | + A list of NAT services created in this router. + uniqueItems: True + items: + type: object + additionalProperties: false + required: + - name + - sourceSubnetworkIpRangesToNat + properties: + name: + type: string + description: | + Unique name of this Nat service. The name must be 1-63 characters long and comply with RFC1035. + sourceSubnetworkIpRangesToNat: + type: string + description: | + Specify the Nat option, which can take one of the following values: + + - ALL_SUBNETWORKS_ALL_IP_RANGES: All of the IP ranges in every Subnetwork are allowed to Nat. + - ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES: All of the primary IP ranges in every Subnetwork are allowed to Nat. + - LIST_OF_SUBNETWORKS: A list of Subnetworks are allowed to Nat (specified in the field subnetwork below) + The default is SUBNETWORK_IP_RANGE_TO_NAT_OPTION_UNSPECIFIED. Note that if this field contains + ALL_SUBNETWORKS_ALL_IP_RANGES or ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, then there should not be any + other Router.Nat section in any Router for this network in this region. + enum: + - ALL_SUBNETWORKS_ALL_IP_RANGES + - ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES + - LIST_OF_SUBNETWORKS + subnetworks: + type: array + description: | + A list of Subnetwork resources whose traffic should be translated by NAT Gateway. It is used only + when LIST_OF_SUBNETWORKS is selected for the SubnetworkIpRangeToNatOption above. + uniqueItems: True + items: + type: object + additionalProperties: false + required: + - name + properties: + name: + type: string + description: | + URL for the subnetwork resource that will use NAT. + sourceIpRangesToNat: + type: array + description: | + Specify the options for NAT ranges in the Subnetwork. All options of a single value + are valid except NAT_IP_RANGE_OPTION_UNSPECIFIED. The only valid option with + multiple values is: ["PRIMARY_IP_RANGE", "LIST_OF_SECONDARY_IP_RANGES"] Default: [ALL_IP_RANGES] + uniqueItems: True + items: + type: string + secondaryIpRangeNames: + type: array + description: | + A list of the secondary ranges of the Subnetwork that are allowed to use NAT. + This can be populated only if "LIST_OF_SECONDARY_IP_RANGES" is + one of the values in sourceIpRangesToNat. + uniqueItems: True + items: + type: string + natIps: + type: array + description: | + A list of URLs of the IP resources used for this Nat service. These IP addresses must + be valid static external IP addresses assigned to the project. + uniqueItems: True + items: + type: string + natIpAllocateOption: + type: string + description: | + Specify the NatIpAllocateOption, which can take one of the following values: + + - MANUAL_ONLY: Uses only Nat IP addresses provided by customers. + When there are not enough specified Nat IPs, the Nat service fails for new VMs. + - AUTO_ONLY: Nat IPs are allocated by Google Cloud Platform; customers can't specify any Nat IPs. + When choosing AUTO_ONLY, then natIp should be empty. + enum: + - MANUAL_ONLY + - AUTO_ONLY + minPortsPerVm: + type: integer + description: | + Minimum number of ports allocated to a VM from this NAT config. If not set, a default + number of ports is allocated to a VM. This is rounded up to the nearest power of 2. + For example, if the value of this field is 50, at least 64 ports are allocated to a VM. + udpIdleTimeoutSec: + type: integer + description: | + Timeout (in seconds) for UDP connections. Defaults to 30s if not set. + icmpIdleTimeoutSec: + type: integer + description: | + Timeout (in seconds) for ICMP connections. Defaults to 30s if not set. + tcpEstablishedIdleTimeoutSec: + type: integer + description: | + Timeout (in seconds) for TCP established connections. Defaults to 1200s if not set. + tcpTransitoryIdleTimeoutSec: + type: integer + description: | + Timeout (in seconds) for TCP transitory connections. Defaults to 30s if not set. + logConfig: + type: object + additionalProperties: false + description: | + Configure logging on this NAT. + properties: + enable: + type: boolean + description: | + Indicates whether or not to export logs. This is false by default. + filter: + type: string + description: | + Specifies the desired filtering of logs on this NAT. If unspecified, logs are exported + for all connections handled by this NAT. asn: type: integer description: | - The local BGP Autonomous System Number (ASN). Must be an RFC6996 private ASN, - either 16-bit or 32-bit. The value will be fixed for this router. - All VPN tunnels that link to this router will have the same - local ASN. + DEPRECATED. Alias for bgp->asn outputs: properties: diff --git a/dm/templates/cloud_router/tests/schemas/invalid_additional_options.yaml b/dm/templates/cloud_router/tests/schemas/invalid_additional_options.yaml new file mode 100644 index 00000000000..41c7594ae79 --- /dev/null +++ b/dm/templates/cloud_router/tests/schemas/invalid_additional_options.yaml @@ -0,0 +1,4 @@ +network: asd +region: us-east1 +asn: 65001 +foo: bar diff --git a/dm/templates/cloud_router/tests/schemas/valid_basic.yaml b/dm/templates/cloud_router/tests/schemas/valid_basic.yaml new file mode 100644 index 00000000000..c4e9529f7c0 --- /dev/null +++ b/dm/templates/cloud_router/tests/schemas/valid_basic.yaml @@ -0,0 +1,5 @@ +network: asd +region: us-east1 +asn: 65001 +name: foo +project: foo diff --git a/dm/templates/cloud_spanner/cloud_spanner.py b/dm/templates/cloud_spanner/cloud_spanner.py index 80af4abbe06..2304b77a7b4 100644 --- a/dm/templates/cloud_spanner/cloud_spanner.py +++ b/dm/templates/cloud_spanner/cloud_spanner.py @@ -14,6 +14,14 @@ """ This template creates a Cloud Spanner instance and database. """ +def append_optional_property(res, properties, prop_name): + """ If the property is set, it is added to the resource. """ + + val = properties.get(prop_name) + if val: + res['properties'][prop_name] = val + return + def get_spanner_instance_id(project_id, base_name): """ Generate the instance URL """ @@ -36,21 +44,24 @@ def generate_config(context): """ resources_list = [] - name = context.env['name'] + properties = context.properties + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) - instance_id = get_spanner_instance_id(context.env['project'], name) + instance_id = get_spanner_instance_id(project_id, name) instance_config = get_spanner_instance_config( - context.env['project'], + project_id, context.properties['instanceConfig'] ) resource = { 'name': name, - 'type': 'spanner.v1.instance', + # https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances + 'type': 'gcp-types/spanner-v1:projects.instances', 'properties': { 'instanceId': name, - 'parent': 'projects/{}'.format(context.env['project']), + 'parent': 'projects/{}'.format(project_id), 'instance': { 'name': instance_id, @@ -60,11 +71,18 @@ def generate_config(context): } } } + + optional_properties = [ + 'labels', + ] + for prop in optional_properties: + append_optional_property(resource, properties, prop) resources_list.append(resource) if context.properties.get('bindings'): policy = { 'name': "{}{}".format(name, '-setIamPolicy'), + # https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances/setIamPolicy 'action': 'gcp-types/spanner-v1:spanner.projects.instances.setIamPolicy', # pylint: disable=line-too-long 'properties': { @@ -88,6 +106,7 @@ def generate_config(context): ) database_resource = { 'name': database_resource_name, + # https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases 'type': 'gcp-types/spanner-v1:projects.instances.databases', 'properties': { @@ -105,6 +124,7 @@ def generate_config(context): 'name': "{}{}".format(database_resource_name, "-setIamPolicy"), + # https://cloud.google.com/spanner/docs/reference/rest/v1/projects.instances.databases/setIamPolicy 'action': 'gcp-types/spanner-v1:spanner.projects.instances.databases.setIamPolicy', # pylint: disable=line-too-long 'properties': { diff --git a/dm/templates/cloud_spanner/cloud_spanner.py.schema b/dm/templates/cloud_spanner/cloud_spanner.py.schema index 950ce39c3e1..a5613867bc9 100644 --- a/dm/templates/cloud_spanner/cloud_spanner.py.schema +++ b/dm/templates/cloud_spanner/cloud_spanner.py.schema @@ -15,6 +15,7 @@ info: title: Cloud Spanner author: Sourced Group Inc. + version: 1.0.0 description: Creates a Cloud Spanner instance and database. additionalProperties: false @@ -25,11 +26,24 @@ required: - instanceConfig properties: + name: + type: string + description: | + A unique identifier for the instance, which cannot be changed after the instance is created.Values are + of the form projects//instances/[a-z][-a-z0-9]*[a-z0-9]. The final segment of the name must be + between 2 and 64 characters in length. + This does not include the project ID. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the Cloud Spanner instance. The + Google apps domain is prefixed if applicable. displayName: type: string description: The cluster display name in GCP Console. nodeCount: type: integer + minimum: 1 description: The number of instances allocated to your node. instanceConfig: type: string @@ -42,6 +56,7 @@ properties: - regional-asia-east1 - regional-asia-east2 - regional-asia-northeast1 + - regional-asia-northeast2 - regional-asia-south1 - regional-asia-southeast1 - regional-australia-southeast1 @@ -49,17 +64,27 @@ properties: - regional-europe-west1 - regional-europe-west2 - regional-europe-west4 + - regional-europe-west6 - regional-northamerica-northeast1 - regional-us-central1 - regional-us-east1 - regional-us-east4 - regional-us-west1 - regional-us-west2 + labels: + type: object + description: | + Map labels associated with this Cloud spanner instance. + Example: + name: wrench + mass: 1.3kg + count: 3 bindings: type: array description: IAM policies applied at the CLUSTER level. items: type: object + additionalProperties: false required: - role - members @@ -82,6 +107,7 @@ properties: description: A list of databases created under the instance cluster. items: type: object + additionalProperties: false required: - name properties: @@ -93,6 +119,7 @@ properties: description: A list of IAM policies applied at the DATABASE level. items: type: object + additionalProperties: false required: - role - members @@ -131,6 +158,7 @@ outputs: patternProperties: ".*": type: object + additionalProperties: false description: Details for an address resource. properties: state: diff --git a/dm/templates/cloud_spanner/tests/schemas/invalid_additional_options.yaml b/dm/templates/cloud_spanner/tests/schemas/invalid_additional_options.yaml new file mode 100644 index 00000000000..4f26d755b10 --- /dev/null +++ b/dm/templates/cloud_spanner/tests/schemas/invalid_additional_options.yaml @@ -0,0 +1,4 @@ +displayName: "Spanner Cluster 1" +nodeCount: 2 +instanceConfig: nam3 +foo: bar diff --git a/dm/templates/cloud_spanner/tests/schemas/valid_basic.yaml b/dm/templates/cloud_spanner/tests/schemas/valid_basic.yaml new file mode 100644 index 00000000000..feba6ca8c69 --- /dev/null +++ b/dm/templates/cloud_spanner/tests/schemas/valid_basic.yaml @@ -0,0 +1,5 @@ +displayName: "Spanner Cluster 1" +nodeCount: 2 +instanceConfig: nam3 +name: foo +project: foo diff --git a/dm/templates/cloud_spanner/tests/schemas/valid_complex.yaml b/dm/templates/cloud_spanner/tests/schemas/valid_complex.yaml new file mode 100644 index 00000000000..50ec0b84146 --- /dev/null +++ b/dm/templates/cloud_spanner/tests/schemas/valid_complex.yaml @@ -0,0 +1,13 @@ +displayName: "Spanner Cluster 1" +nodeCount: 2 +instanceConfig: nam3 +bindings: + - role: "roles/spanner.admin" + members: + - "user:myuser@mycompany.com" +databases: + - name: "spannerdb1" + bindings: + - role: "roles/spanner.databaseAdmin" + members: + - "user:myotheruser@mycompany.com" diff --git a/dm/templates/cloud_sql/README.md b/dm/templates/cloud_sql/README.md index 3c04723163a..a29fefce59f 100644 --- a/dm/templates/cloud_sql/README.md +++ b/dm/templates/cloud_sql/README.md @@ -43,7 +43,7 @@ See the `properties` section in the schema file(s): case, [examples/cloud\_sql.yaml](examples/cloud_sql.yaml): ```shell - cp templates/dns_managed_zone/examples/cloud_sql.yaml my_cloud_sql.yaml + cp templates/cloud_sql/examples/cloud_sql.yaml my_cloud_sql.yaml ``` 4. Change the values in the config file to match your specific GCP setup (for @@ -61,16 +61,30 @@ See the `properties` section in the schema file(s): --config my_cloud_sql.yaml ``` + To deploy with CFT: + + ```shell + cft apply my_cloud_sql.yaml + ``` + 6. In case you need to delete your deployment: ```shell gcloud deployment-manager deployments delete ``` + To delete deployment with CFT: + + ```shell + cft delete my_cloud_sql.yaml + ``` + `Notes:` After a Cloud SQL instance is deleted, its name cannot be reused for up to 7 days. ## Examples -- [Cloud SQL](examples/cloud_sql_.yaml) +- [Cloud SQL](examples/cloud_sql.yaml) - [Cloud SQL with Read Replica](examples/cloud_sql_read_replica.yaml) +- [Cloud SQL Postgres](examples/cloud_sql_postgres.yaml) +- [Cloud SQL Private Networking](examples/cloud_sql_private_network.yaml) diff --git a/dm/templates/cloud_sql/cloud_sql.py b/dm/templates/cloud_sql/cloud_sql.py index 2a8ab06bb93..cd5258ffff2 100644 --- a/dm/templates/cloud_sql/cloud_sql.py +++ b/dm/templates/cloud_sql/cloud_sql.py @@ -64,7 +64,8 @@ def get_instance(res_name, project_id, properties): instance = { 'name': name, - 'type': 'sqladmin.v1beta4.instance', + # https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances + 'type': 'gcp-types/sqladmin-v1beta4:instances', 'properties': instance_properties } @@ -88,6 +89,10 @@ def get_instance(res_name, project_id, properties): 'name': 'connectionName', 'value': '$(ref.{}.connectionName)'.format(name) }, + { + 'name': 'backendType', + 'value': '$(ref.{}.backendType)'.format(name) + }, ] return DMBundle(instance, outputs) @@ -116,7 +121,8 @@ def get_database(instance_name, project_id, properties): database = { 'name': res_name, - 'type': 'sqladmin.v1beta4.database', + # https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases + 'type': 'gcp-types/sqladmin-v1beta4:databases', 'properties': db_properties } @@ -160,7 +166,8 @@ def get_user(instance_name, project_id, properties): user = { 'name': res_name, - 'type': 'sqladmin.v1beta4.user', + # https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/users + 'type': 'gcp-types/sqladmin-v1beta4:users', 'properties': user_properties } diff --git a/dm/templates/cloud_sql/cloud_sql.py.schema b/dm/templates/cloud_sql/cloud_sql.py.schema index f6bd8b43bdd..bf38a7cd124 100644 --- a/dm/templates/cloud_sql/cloud_sql.py.schema +++ b/dm/templates/cloud_sql/cloud_sql.py.schema @@ -15,6 +15,7 @@ info: title: Cloud SQL author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of a Cloud SQL instance with database and user resources. For more information, see https://cloud.google.com/sql/docs/. @@ -29,16 +30,32 @@ required: - settings properties: + name: + type: string + description: | + The name of the Cloud SQL instance. This does not include the project ID. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the Cloud SQL instance. The + Google apps domain is prefixed if applicable. databaseVersion: type: string description: | The database engine type and version. - The value of this field cannot be changed after instance creation. - Supports MySQL Second Generation instances MYSQL_5_7 | MYSQL_5_6, - First Generation instances MYSQL_5_6 | MYSQL_5_5, and - PostgreSQL instances POSTGRES_9_6. + The databaseVersion field can not be changed after instance creation. + MySQL Second Generation instances: MYSQL_5_7 (default) or MYSQL_5_6. + PostgreSQL instances: POSTGRES_9_6 + MySQL First Generation instances: MYSQL_5_6 (default) or MYSQL_5_5 + enum: + - MYSQL_5_5 # Deprecated + - MYSQL_5_6 + - MYSQL_5_7 + - POSTGRES_9_6 failoverReplica: type: object + additionalProperties: false description: | The name and status of the failover replica. Applicable only to Second Generation instances. @@ -71,27 +88,28 @@ properties: maxDiskSize: type: number description: The maximum disk size of the instance in bytes. - name: - type: string - description: | - The name of the Cloud SQL instance. This does not include the project ID. onPremisesConfiguration: type: object + additionalProperties: false description: | Configuration specific to on-premises instances. + requires: + - hostPort + - kind properties: hostPort: type: string description: | The host and port of the on-premises instance in the host:port format. - project: - type: string - description: | - The project ID of the project containing the Cloud SQL instance. The - Google apps domain is prefixed if applicable. + kind: + type: string + const: "sql#onPremisesConfiguration" + description: | + This is always sql#onPremisesConfiguration. replicaConfiguration: type: object + additionalProperties: false description: | Configuration specific to failover replicas and read replicas. properties: @@ -106,6 +124,7 @@ properties: instance. mysqlReplicaConfiguration: type: object + additionalProperties: false description: | MySQL-specific configuration when replicating from an MySQL on-premises master. Replication configuration information such as the @@ -166,12 +185,13 @@ properties: type: object description: SSL configuration. serviceAccountEmailAddress: - type: object + type: string description: | - The service account's email address assigned to the instance. Applicable - only to Second Generation instances. + The service account email address assigned to the instance. This property is applicable only + to Second Generation instances. settings: type: object + additionalProperties: false description: User settings. required: - tier @@ -214,6 +234,7 @@ properties: - REGIONAL backupConfiguration: type: object + additionalProperties: false description: The daily backup configuration for the instance. properties: binaryLogEnabled: @@ -257,6 +278,7 @@ properties: Database flags passed to the instance at startup. items: type: object + additionalProperties: false properties: name: type: string @@ -279,6 +301,7 @@ properties: specific to read replica instances. Writable. ipConfiguration: type: object + additionalProperties: false description: | Settings for IP Management. This allows to enable or disable the instance IP and to define which external networks can connect to the @@ -293,6 +316,7 @@ properties: notation (e.g., 192.168.100.0/24). Writable. items: type: object + additionalProperties: false properties: expirationTime: type: string @@ -327,6 +351,7 @@ properties: Defines whether SSL connections over IP must be enforced. locationPreference: type: object + additionalProperties: false description: | Location preference settings. This allows the instance to be located as near as possible to either an App Engine app or Compute @@ -345,6 +370,7 @@ properties: us-central1-b, etc.). maintenanceWindow: type: object + additionalProperties: false description: | The maintenance window for the instance. Specifies when the instance can be restarted for maintenance purposes. Not used for First @@ -365,11 +391,16 @@ properties: description: | The pricing plan for the instance: PER_USE or PACKAGE. Only PER_USE is supported for Second Generation instances. + enum: + - PER_USE + - PACKAGE replicationType: type: string description: | - The type of replication the instance uses: ASYNCHRONOUS or - SYNCHRONOUS. + The type of replication the instance uses: ASYNCHRONOUS or SYNCHRONOUS. + enum: + - ASYNCHRONOUS + - SYNCHRONOUS settingsVersion: type: number description: | @@ -409,6 +440,29 @@ properties: on the instance type (First Generation or Second Generation/PostgreSQL). For a complete list of valid values, see Instance Locations. The region cannot be changed after instance creation. + See https://cloud.google.com/sql/docs/mysql/locations + enum: + - northamerica-northeast1 + - us-central + - us-central1 + - us-east1 + - us-east4 + - us-west1 + - us-west2 + - southamerica-east1 + - europe-north1 + - europe-west1 + - europe-west2 + - europe-west3 + - europe-west4 + - europe-west6 + - asia-east1 + - asia-east2 + - asia-northeast1 + - asia-northeast2 + - asia-south1 + - asia-southeast1 + - australia-southeast1 databases: type: array description: SQL Databases to create in the new instance. @@ -416,6 +470,7 @@ properties: - name items: type: object + additionalProperties: false properties: name: type: string @@ -430,10 +485,11 @@ properties: users: type: array description: Cloud SQL users to create in the new instance. - required: - - name items: type: object + additionalProperties: false + required: + - name properties: name: type: string @@ -493,6 +549,9 @@ outputs: - databaseNames: type: array description: The names of the created databases. + - backendType: + type: string + description: Database generation. - databaseSelfLinks: type: array description: | diff --git a/dm/templates/cloud_sql/examples/cloud_sql_postgres.yaml b/dm/templates/cloud_sql/examples/cloud_sql_postgres.yaml new file mode 100644 index 00000000000..a99268ae00f --- /dev/null +++ b/dm/templates/cloud_sql/examples/cloud_sql_postgres.yaml @@ -0,0 +1,29 @@ +# Example of the Cloud SQL template usage. +# Note that Postgres instances use different instance types then MySQL instances + +imports: + - path: templates/cloud_sql/cloud_sql.py + name: cloud_sql.py + +resources: + - name: cloud-sql-postgres-instance + type: cloud_sql.py + properties: + region: us-central1 + databaseVersion: POSTGRES_9_6 + instanceType: CLOUD_SQL_INSTANCE + settings: + tier: db-f1-micro + backupConfiguration: + startTime: '02:00' + enabled: true + locationPreference: + zone: us-central1-c + users: + - name: user-1 + host: 10.1.1.1 + - name: user-2 + host: 10.1.1.2 + databases: + - name: db-1 + - name: db-2 diff --git a/dm/templates/cloud_sql/examples/cloud_sql_private_network.yaml b/dm/templates/cloud_sql/examples/cloud_sql_private_network.yaml new file mode 100644 index 00000000000..372c35010d9 --- /dev/null +++ b/dm/templates/cloud_sql/examples/cloud_sql_private_network.yaml @@ -0,0 +1,32 @@ +# Example of the Cloud SQL template usage. +# Replace 'your-vpc-network-name' with the name of the vpc +# in which the Cloud SQL instance will be deployed. + +imports: + - path: templates/cloud_sql/cloud_sql.py + name: cloud_sql.py + +resources: + - name: cloud-sql-instance + type: cloud_sql.py + properties: + region: us-central1 + settings: + tier: db-n1-standard-1 + backupConfiguration: + startTime: '02:00' + enabled: true + binaryLogEnabled: true + locationPreference: + zone: us-central1-c + ipConfiguration: + ipv4Enabled: false + privateNetwork: 'your-vpc-network-name' + users: + - name: user-1 + host: 10.1.1.1 + - name: user-2 + host: 10.1.1.2 + databases: + - name: db-1 + - name: db-2 diff --git a/dm/templates/cloud_sql/examples/cloud_sql_read_replica.yaml b/dm/templates/cloud_sql/examples/cloud_sql_read_replica.yaml index 1542a76fd72..bc0ce55f4d0 100644 --- a/dm/templates/cloud_sql/examples/cloud_sql_read_replica.yaml +++ b/dm/templates/cloud_sql/examples/cloud_sql_read_replica.yaml @@ -37,4 +37,5 @@ resources: masterInstanceName: $(ref.cloud-sql-master-instance.name) # Wait until all the resources required by the master instance had been # created. - dependsOn: $(ref.cloud-sql-master-instance.resources) + dependsOn: + - $(ref.cloud-sql-master-instance.resources) diff --git a/dm/templates/cloud_sql/tests/schemas/invalid_additional_options.yaml b/dm/templates/cloud_sql/tests/schemas/invalid_additional_options.yaml new file mode 100644 index 00000000000..7c8d53098d7 --- /dev/null +++ b/dm/templates/cloud_sql/tests/schemas/invalid_additional_options.yaml @@ -0,0 +1,4 @@ +region: us-central1 +settings: + tier: db-n1-standard-1 +foo: bar diff --git a/dm/templates/cloud_sql/tests/schemas/invalid_additional_options_nested.yaml b/dm/templates/cloud_sql/tests/schemas/invalid_additional_options_nested.yaml new file mode 100644 index 00000000000..25f56ace31c --- /dev/null +++ b/dm/templates/cloud_sql/tests/schemas/invalid_additional_options_nested.yaml @@ -0,0 +1,4 @@ +region: us-central1 +settings: + tier: db-n1-standard-1 + foo: bar diff --git a/dm/templates/cloud_sql/tests/schemas/invalid_missing_region.yaml b/dm/templates/cloud_sql/tests/schemas/invalid_missing_region.yaml new file mode 100644 index 00000000000..f169277a46f --- /dev/null +++ b/dm/templates/cloud_sql/tests/schemas/invalid_missing_region.yaml @@ -0,0 +1,2 @@ +settings: + tier: db-n1-standard-1 diff --git a/dm/templates/cloud_sql/tests/schemas/invalid_missing_tier.yaml b/dm/templates/cloud_sql/tests/schemas/invalid_missing_tier.yaml new file mode 100644 index 00000000000..aa3e8a6672c --- /dev/null +++ b/dm/templates/cloud_sql/tests/schemas/invalid_missing_tier.yaml @@ -0,0 +1 @@ +region: us-central1 diff --git a/dm/templates/cloud_sql/tests/schemas/valid_basic.yaml b/dm/templates/cloud_sql/tests/schemas/valid_basic.yaml new file mode 100644 index 00000000000..5693db2b0fb --- /dev/null +++ b/dm/templates/cloud_sql/tests/schemas/valid_basic.yaml @@ -0,0 +1,5 @@ +region: us-central1 +settings: + tier: db-n1-standard-1 +name: foo +project: foo diff --git a/dm/templates/cloud_sql/tests/schemas/valid_complex.yaml b/dm/templates/cloud_sql/tests/schemas/valid_complex.yaml new file mode 100644 index 00000000000..f71571802a5 --- /dev/null +++ b/dm/templates/cloud_sql/tests/schemas/valid_complex.yaml @@ -0,0 +1,17 @@ +region: us-central1 +settings: + tier: db-n1-standard-1 + backupConfiguration: + startTime: '02:00' + enabled: true + binaryLogEnabled: true + locationPreference: + zone: us-central1-c +users: + - name: user-1 + host: 10.1.1.1 + - name: user-2 + host: 10.1.1.2 +databases: + - name: db-1 + - name: db-2 diff --git a/dm/templates/cloud_tasks/queue.py b/dm/templates/cloud_tasks/queue.py index 1238d6fa2dc..30799a2b332 100644 --- a/dm/templates/cloud_tasks/queue.py +++ b/dm/templates/cloud_tasks/queue.py @@ -25,7 +25,7 @@ def generate_config(context): parent = 'projects/{}/locations/{}'.format(project_id, location) queue = { - 'name': name, + 'name': context.env['name'], 'type': '{}/cloudtasks:projects.locations.queues'.format(project_id), 'properties': { 'name': '{}/queues/{}'.format(parent, name), @@ -45,11 +45,11 @@ def generate_config(context): outputs = [ { 'name': 'name', - 'value': '$(ref.{}.name)'.format(name) + 'value': '$(ref.{}.name)'.format(context.env['name']) }, { 'name': 'state', - 'value': '$(ref.{}.state)'.format(name) + 'value': '$(ref.{}.state)'.format(context.env['name']) } ] diff --git a/dm/templates/cloud_tasks/queue.py.schema b/dm/templates/cloud_tasks/queue.py.schema index d952f5f07a5..107f1d30aa3 100644 --- a/dm/templates/cloud_tasks/queue.py.schema +++ b/dm/templates/cloud_tasks/queue.py.schema @@ -15,8 +15,10 @@ info: title: Cloud Tasks Queue author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of a Cloud Tasks Queue resource. + For more information on this resource, see https://cloud.google.com/tasks/docs/dual-overview @@ -51,6 +53,7 @@ properties: https://cloud.google.com/about/locations/. rateLimits: type: object + additionalProperties: false description: | Rate limits for task dispatches. Controls the total rate of dispatches from a queue (i.e., all traffic dispatched from the queue, @@ -78,6 +81,7 @@ properties: maximum: 5000 retryConfig: type: object + additionalProperties: false description: | Settings that determine the retry behavior. For tasks created using Cloud Tasks: the queue-level retry settings apply to all tasks in the @@ -153,10 +157,12 @@ properties: Tasks will pick the default. appEngineHttpQueue: type: object + additionalProperties: false description: The App Engine HTTP queue. properties: appEngineRoutingOverride: type: object + additionalProperties: false description: | Overrides for the task-level appEngineRouting. If set, appEngineRoutingOverride is used for all tasks in the queue, diff --git a/dm/templates/cloud_tasks/task.py b/dm/templates/cloud_tasks/task.py index 39825b49797..cddbe4c6885 100644 --- a/dm/templates/cloud_tasks/task.py +++ b/dm/templates/cloud_tasks/task.py @@ -44,7 +44,7 @@ def generate_config(context): task = { 'name': - name, + context.env['name'], 'type': '{}/cloudtasks:projects.locations.queues.tasks'.format(project_id), 'properties': @@ -74,19 +74,19 @@ def generate_config(context): 'outputs': [ { 'name':'name', - 'value': '$(ref.{}.name)'.format(name) + 'value': '$(ref.{}.name)'.format(context.env['name']) }, { 'name':'createTime', - 'value': '$(ref.{}.createTime)'.format(name) + 'value': '$(ref.{}.createTime)'.format(context.env['name']) }, { 'name':'view', - 'value': '$(ref.{}.view)'.format(name) + 'value': '$(ref.{}.view)'.format(context.env['name']) }, { 'name':'scheduleTime', - 'value': '$(ref.{}.scheduleTime)'.format(name) + 'value': '$(ref.{}.scheduleTime)'.format(context.env['name']) } ] } diff --git a/dm/templates/cloud_tasks/task.py.schema b/dm/templates/cloud_tasks/task.py.schema index 296f7592583..5fa4844a7cb 100644 --- a/dm/templates/cloud_tasks/task.py.schema +++ b/dm/templates/cloud_tasks/task.py.schema @@ -15,8 +15,10 @@ info: title: Cloud Tasks author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of a Cloud Task resource. + For more information on this resource, see https://cloud.google.com/tasks/docs/dual-overview. @@ -26,7 +28,6 @@ imports: required: - task - queueId - - appEngineHttpRequest properties: queueId: @@ -55,6 +56,8 @@ properties: If `queueId` is the full name of the queue, this field is ignored. task: type: object + required: + - appEngineHttpRequest description: | The task to add. Task names have the following format: projects/PROJECT_ID/locations/LOCATION_ID/queues/QUEUE_ID/tasks/TASK_ID. @@ -79,6 +82,7 @@ properties: For example, "2014-10-02T15:01:23.045123456Z". appEngineHttpRequest: type: object + additionalProperties: false description: | Defines the HTTP request that is sent to an App Engine app when the task is dispatched. This can only be used for tasks in a queue with @@ -101,6 +105,7 @@ properties: - DELETE appEngineRouting: type: object + additionalProperties: false description: Task-level settings for App Engine routing. properties: service: diff --git a/dm/templates/cloudbuild/cloudbuild.py b/dm/templates/cloudbuild/cloudbuild.py index 767eae9963c..4a5b93ce472 100644 --- a/dm/templates/cloudbuild/cloudbuild.py +++ b/dm/templates/cloudbuild/cloudbuild.py @@ -20,12 +20,15 @@ def generate_config(context): resources = [] outputs = [] properties = context.properties + project_id = properties.get('project', context.env['project']) name = context.env['name'] build_steps = properties['steps'] cloud_build = { 'name': name, + # https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.builds/create 'action': 'gcp-types/cloudbuild-v1:cloudbuild.projects.builds.create', 'properties': { + 'projectId': project_id, 'steps': build_steps }, 'metadata': { diff --git a/dm/templates/cloudbuild/cloudbuild.py.schema b/dm/templates/cloudbuild/cloudbuild.py.schema index 25f9d791a65..b8a90a39109 100644 --- a/dm/templates/cloudbuild/cloudbuild.py.schema +++ b/dm/templates/cloudbuild/cloudbuild.py.schema @@ -17,8 +17,13 @@ info: author: Sourced Group Inc. description: | Creates a Cloud Build resource. + For more information on this resource, see - https://cloud.google.com/cloud-build/docs/. + https://cloud.google.com/cloud-build/docs/ + + APIs endpoints used by this template: + - gcp-types/cloudbuild-v1:cloudbuild.projects.builds.create => + https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.builds/create imports: - path: cloudbuild.py @@ -29,6 +34,11 @@ required: - steps properties: + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. source: description: The location of the source files to build. oneOf: @@ -36,9 +46,11 @@ properties: - "$ref": "#/definitions/repoSource" steps: type: array + uniqItems: true description: The list of steps in the build pipeline. items: type: object + additionalProperties: false description: A step in the build pipeline. required: - name @@ -46,10 +58,21 @@ properties: name: type: string description: | - The name of the container image that runs this particular build - step. + The name of the container image that will run this particular build step. + + If the image is available in the host's Docker daemon's cache, it will be run directly. If not, the host + will attempt to pull the image first, using the builder service account's credentials if necessary. + + The Docker daemon's cache will already have the latest versions of all of the officially supported + build steps (https://github.com/GoogleCloudPlatform/cloud-builders). The Docker daemon will also have + cached many of the layers for some popular images, like "ubuntu", "debian", but they will be refreshed + at the time you attempt to use them. + + If you built an image in a previous build step, it will be stored in the host's Docker daemon's cache + and is available to use as the name for a later build step. env: type: array + uniqItems: true description: | The list of environment variable definitions to be used when running a step. The elements are in the "KEY=VALUE" form, @@ -86,6 +109,7 @@ properties: reference this build step as a dependency. waitFor: type: array + uniqItems: true description: | The ID(s) of the step(s) that the build step depends on. This build step will not start until all the build steps in waitFor @@ -98,10 +122,11 @@ properties: type: string description: | The entry point to be used instead of the build step image's - default entry point. If not set, the image's default entry + default entry point. If not set, the image's default entry point is used. secretEnv: type: array + uniqItems: true description: | The list of environment variables which are encrypted using the Cloud Key Management Service crypto key. These values must be @@ -110,9 +135,11 @@ properties: type: string volumes: type: array + uniqItems: true description: The list of volumes to mount into the build step. items: type: object + additionalProperties: false properties: name: type: string @@ -144,6 +171,7 @@ properties: terminated by 's'; for example 3.5s. images: type: array + uniqItems: true description: | The list of images to be pushed upon successful completion of all build steps. The images are pushed using the Builder service account's @@ -154,12 +182,14 @@ properties: type: string artifacts: type: object + additionalProperties: false description: | Artifacts produced by the build, to be uploaded upon successful completion of all build steps. properties: images: type: array + uniqItems: true description: | The list of images to be pushed upon successful completion of all build steps. The images are pushed using the Builder serviceaccount's @@ -170,31 +200,27 @@ properties: type: string objects: type: object + additionalProperties: false description: | The list of objects to be uploaded to Cloud Storage upon successful completion of all build steps. Files in the workspace matching the specified paths globs are uploaded using the Builder serviceaccount's credentials. properties: - artifactObjects: - type: object + location: + type: string description: | - Files in the workspace to upload to Cloud Storage upon successful - completion of all build steps. - properties: - location: - type: string - description: | - The Cloud Storage bucket, with an optional object path, in - the "gs://bucket/path/to/somewhere/" form. Files in the - workspace matching any pattern specified uder that path are - uploaded to Cloud Storage with this location as a prefix. - paths: - type: array - description: | - Path globs used to match files in the build's workspace. - items: - type: string + The Cloud Storage bucket, with an optional object path, in + the "gs://bucket/path/to/somewhere/" form. Files in the + workspace matching any pattern specified uder that path are + uploaded to Cloud Storage with this location as a prefix. + paths: + type: array + uniqItems: true + description: | + Path globs used to match files in the build's workspace. + items: + type: string logsBucket: type: string description: | @@ -202,6 +228,7 @@ properties: Log file names are in the ${logsBucket}/log-${build_id}.txt format. options: type: object + additionalProperties: false description: Special options for the build. properties: sourceProvenanceHash: @@ -249,6 +276,11 @@ properties: - STREAM_DEFAULT - STREAM_ON - STREAM_OFF + workerPool: + type: string + description: | + Option to specify a WorkerPool for the build. User specifies the pool with the format + "[WORKERPOOL_PROJECT_ID]/[WORKERPOOL_NAME]". This is an experimental field. logging: type: string description: | @@ -258,6 +290,54 @@ properties: - LOGGING_UNSPECIFIED - LEGACY - GCS_ONLY + point is used. + env: + type: array + uniqItems: true + description: | + A list of global environment variable definitions that will exist for all build steps in this build. + If a variable is defined in both globally and in a build step, the variable will use the build step value. + + The elements are of the form "KEY=VALUE" for the environment variable "KEY" being given the value "VALUE". + items: + type: string + secretEnv: + type: array + uniqItems: true + description: | + A list of global environment variables, which are encrypted using a Cloud Key Management Service crypto key. + These values must be specified in the build's Secret. These variables will be available to + all build steps in this build. + items: + type: string + volumes: + type: array + uniqItems: true + description: | + Global list of volumes to mount for ALL build steps + + Each volume is created as an empty volume prior to starting the build process. Upon completion of the build, + volumes and their contents are discarded. Global volume names and paths cannot conflict with the volumes + defined a build step. + + Using a global volume in a build with only one step is not valid as it is indicative of a build request + with an incorrect configuration. + items: + type: object + additionalProperties: false + properties: + name: + type: string + description: | + The name of the volume to mount. Volume names must be unique + per build step, and must be valid names for Docker volumes. + Each named volume must be used by at least two build steps. + path: + type: string + description: | + The path to mount the volume at. Paths must be absolute. + They cannot conflict with other volume paths on the same + build step or with certain reserved volume paths. substitutions: type: object description: | @@ -265,15 +345,19 @@ properties: Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. tags: type: array - description: Build annotation tags. These are NOT Docker tags. + uniqItems: true + description: | + Tags for annotation of a Build. These are not docker tags. items: type: string secrets: type: object + additionalProperties: false description: Secrets to decrypt using Cloud Key Management Service. properties: secret: type: object + additionalProperties: false properties: kmsKeyName: type: string diff --git a/dm/templates/cloudbuild/trigger.py b/dm/templates/cloudbuild/trigger.py index ea1b1a080fb..e7416a4c860 100644 --- a/dm/templates/cloudbuild/trigger.py +++ b/dm/templates/cloudbuild/trigger.py @@ -20,7 +20,7 @@ def generate_config(context): resources = [] properties = context.properties name = context.env['name'] - project_id = context.env['project'] + project_id = properties.get('project', context.env['project']) # set projectId in triggerTemplate properties['triggerTemplate']['projectId'] = project_id build_def = properties.get('build') @@ -29,28 +29,28 @@ def generate_config(context): # build trigger create action build_trigger_create = { - 'name': - name, - 'action': - 'gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.create', + 'name': name, + # https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.triggers/create + 'action': 'gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.create', 'metadata': { 'runtimePolicy': ['CREATE'], }, 'properties': { + 'projectId': project_id, 'triggerTemplate': properties['triggerTemplate'] } } # build trigger update action build_trigger_update = { - 'name': - name + '-update', - 'action': - 'gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.patch', + 'name': name + '-update', + # https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.triggers/patch + 'action': 'gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.patch', 'metadata': { 'runtimePolicy': ['UPDATE_ON_CHANGE'], }, 'properties': { + 'projectId': project_id, 'id': build_trigger_id, 'triggerId': build_trigger_id, 'triggerTemplate': properties['triggerTemplate'] @@ -62,7 +62,8 @@ def generate_config(context): 'disabled', 'substitutions', 'ignoredFiles', - 'includedFiles' + 'includedFiles', + 'tags' ] for prop in optional_properties: @@ -82,10 +83,9 @@ def generate_config(context): # build trigger delete action build_trigger_delete = { - 'name': - name + '-delete', - 'action': - 'gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.delete', + 'name': name + '-delete', + # https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.triggers/delete + 'action': 'gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.delete', 'metadata': { 'runtimePolicy': ['DELETE'], }, diff --git a/dm/templates/cloudbuild/trigger.py.schema b/dm/templates/cloudbuild/trigger.py.schema index dc281952fe8..ded1aacd4a9 100644 --- a/dm/templates/cloudbuild/trigger.py.schema +++ b/dm/templates/cloudbuild/trigger.py.schema @@ -17,21 +17,38 @@ info: author: Sourced Group Inc. description: | Supports creation of an automated Cloud Build trigger. + For more information on this resource, see - https://cloud.google.com/cloud-build/docs/running-builds/automate-builds. + https://cloud.google.com/cloud-build/docs/running-builds/automate-builds + + APIs endpoints used by this template: + - gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.create => + https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.triggers/create + - gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.patch => + https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.triggers/patch + - gcp-types/cloudbuild-v1:cloudbuild.projects.triggers.delete => + https://cloud.google.com/cloud-build/docs/api/reference/rest/v1/projects.triggers/delete imports: - path: trigger.py +additionalProperties: false + required: - triggerTemplate properties: + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. description: type: string description: The human-readable trigger description. triggerTemplate: type: object + additionalProperties: false description: | The template describing the types of source changes that trigger a build. Branch and tag names in the trigger templates are interpreted as regular @@ -59,6 +76,7 @@ properties: oneOf: - "$ref": "#/definitions/branchName" - "$ref": "#/definitions/tagName" + - "$ref": "#/definitions/commitSha" disabled: type: boolean default: False @@ -70,6 +88,7 @@ properties: Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. ignoredFiles: type: array + uniqItems: true description: | A file glob match using http://godoc/pkg/path/filepath#Match extended with support for "**". If both ignoredFiles and changed files are empty, @@ -81,6 +100,7 @@ properties: type: string includedFiles: type: array + uniqItems: true description: | A file glob match using http://godoc/pkg/path/filepath#Match extended with support for "**". If any of the files altered in the commit pass @@ -95,10 +115,18 @@ properties: oneOf: - "$ref": "#/definitions/build" - "$ref": "#/definitions/filename" + tags: + type: array + uniqItems: true + description: | + Tags for annotation of a BuildTrigger + items: + type: string definitions: storageSource: type: object + additionalProperties: false description: | The location of the source in an archive file on Google Cloud Storage. properties: @@ -117,6 +145,7 @@ definitions: the latest generation is used. repoSource: type: object + additionalProperties: false description: The source location in the Google Cloud Source repository. properties: projectId: @@ -140,6 +169,9 @@ definitions: - "$ref": "#/definitions/branchName" - "$ref": "#/definitions/tagName" - "$ref": "#/definitions/commitSha" + commitSha: + type: string + description: Explicit commit SHA to build. branchName: type: string description: The name of the branch to build. @@ -153,19 +185,23 @@ definitions: the template. build: type: object + additionalProperties: false description: The contents of the build template. properties: source: type: object + additionalProperties: false description: The location of the source files to build. oneOf: - "$ref": "#/definitions/storageSource" - "$ref": "#/definitions/repoSource" steps: type: array + uniqItems: true description: The list of the steps in the build pipeline. items: type: object + additionalProperties: false description: A step in the build pipeline. required: - name @@ -176,6 +212,7 @@ definitions: The name of the container image that runs the build step. env: type: array + uniqItems: true description: | The list of environment variable definitions to be used when running a step. The elements are in the "KEY=VALUE" form, @@ -213,6 +250,7 @@ definitions: reference this build step as a dependency. waitFor: type: array + uniqItems: true description: | The ID(s) of the step(s) this build step depends on. The build step will not start until all the build steps in @@ -229,6 +267,7 @@ definitions: point is used. secretEnv: type: array + uniqItems: true description: | The list of environment variables are encrypted using the Cloud Key Management Service crypto key. These values must be @@ -237,9 +276,11 @@ definitions: type: string volumes: type: array + uniqItems: true description: The list of volumes to mount into the build step. items: type: object + additionalProperties: false properties: name: type: string @@ -273,6 +314,7 @@ definitions: by 's'. Example: "3.5s". images: type: array + uniqItems: true description: | The list of images to be pushed upon successful completion of all build steps. The images are pushed using the Builder service @@ -283,12 +325,14 @@ definitions: type: string artifacts: type: object + additionalProperties: false description: | Artifacts produced by the build that must be uploaded upon successful completion of all build steps. properties: images: type: array + uniqItems: true description: | The list of images to be pushed upon successful completion of all build steps. The images are pushed using the Builder service @@ -299,31 +343,27 @@ definitions: type: string objects: type: object + additionalProperties: false description: | The list of objects to be uploaded to Cloud Storage upon successful completion of all build steps. Files in the workspace matching specified paths globs are uploaded using the Builder service account's credentials. properties: - artifactObjects: - type: object + location: + type: string description: | - Files in the workspace to upload to Cloud Storage upon - successful completion of all build steps. - properties: - location: - type: string - description: | - The Cloud Storage bucket, with an optional object path, - in the "gs://bucket/path/to/somewhere/" form. Files in - the workspace matching any pattern under that path are - uploaded to Cloud Storage with this location as a prefix. - paths: - type: array - description: | - Path globs used to match files in the build's workspace. - items: - type: string + The Cloud Storage bucket, with an optional object path, + in the "gs://bucket/path/to/somewhere/" form. Files in + the workspace matching any pattern under that path are + uploaded to Cloud Storage with this location as a prefix. + paths: + type: array + uniqItems: true + description: | + Path globs used to match files in the build's workspace. + items: + type: string logsBucket: type: string description: | @@ -332,10 +372,12 @@ definitions: format. options: type: object + additionalProperties: false description: Special options for this build. properties: sourceProvenanceHash: type: array + uniqItems: true items: type: string enum: @@ -388,22 +430,73 @@ definitions: - LOGGING_UNSPECIFIED - LEGACY - GCS_ONLY + env: + type: array + uniqItems: true + description: | + A list of global environment variable definitions that will exist for all build steps in this build. + If a variable is defined in both globally and in a build step, the variable will use the build step value. + + The elements are of the form "KEY=VALUE" for the environment variable "KEY" being given the value "VALUE". + items: + type: string + secretEnv: + type: array + uniqItems: true + description: | + A list of global environment variables, which are encrypted using a Cloud Key Management Service crypto key. + These values must be specified in the build's Secret. These variables will be available to + all build steps in this build. + items: + type: string + volumes: + type: array + uniqItems: true + description: | + Global list of volumes to mount for ALL build steps + + Each volume is created as an empty volume prior to starting the build process. Upon completion of the build, + volumes and their contents are discarded. Global volume names and paths cannot conflict with the volumes + defined a build step. + + Using a global volume in a build with only one step is not valid as it is indicative of a build request + with an incorrect configuration. + items: + type: object + additionalProperties: false + properties: + name: + type: string + description: | + The name of the volume to mount. Volume names must be unique + per build step, and must be valid names for Docker volumes. + Each named volume must be used by at least two build steps. + path: + type: string + description: | + The path to mount the volume at. Paths must be absolute. + They cannot conflict with other volume paths on the same + build step or with certain reserved volume paths. substitutions: type: object + additionalProperties: false description: | Substitution data for the build resource. A list of key-value items. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. tags: type: array + uniqItems: true description: Build annotation tags. These are NOT Docker tags. items: type: string secrets: type: object + additionalProperties: false description: Secrets to decrypt using Cloud Key Management Service. properties: secret: type: object + additionalProperties: false properties: kmsKeyName: type: string @@ -412,6 +505,7 @@ definitions: variables. secretEnv: type: object + additionalProperties: false description: | Maps of the environment variable names to their encrypted values. diff --git a/dm/templates/dataproc/dataproc.py b/dm/templates/dataproc/dataproc.py index ec08754ae51..359c2d19408 100644 --- a/dm/templates/dataproc/dataproc.py +++ b/dm/templates/dataproc/dataproc.py @@ -13,17 +13,13 @@ # limitations under the License. """ This template creates a Dataproc cluster. """ -PRIMARY_GROUP_SCHEMA = {'numInstances': None, 'machineType': 'machineTypeUri'} - -SECONDARY_GROUP_SCHEMA = {'numInstances': None, 'isPreemptible': None} - -GROUP_SCHEMAS = { - 'master': PRIMARY_GROUP_SCHEMA, - 'worker': PRIMARY_GROUP_SCHEMA, - 'secondaryWorker': SECONDARY_GROUP_SCHEMA +NODES_SCHEMA = { + 'numInstances': None, + 'isPreemptible': None, + 'machineType': 'machineTypeUri', + 'accelerators': None, } - def get_disk_config(properties): """ If any disk property is specified, creates the diskConfig section. """ @@ -100,7 +96,7 @@ def set_instance_group_config(properties, cluster, image, instance_group): """ Assign instance group config to the cluster. """ group_spec = properties.get(instance_group) - group_schema = GROUP_SCHEMAS[instance_group] + group_schema = NODES_SCHEMA group_config = get_instance_group_config(group_spec, image, group_schema) config_name = instance_group + 'Config' cluster['properties']['config'][config_name] = group_config @@ -117,15 +113,16 @@ def generate_config(context): properties = context.properties name = properties.get('name', context.env['name']) - project_id = context.env['project'] + project_id = properties.get('project', context.env['project']) image = context.properties.get('image') region = properties['region'] cluster_config = get_gce_cluster_config(properties) cluster = { - 'name': name, - 'type': 'dataproc.v1.cluster', + 'name': context.env['name'], + # https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.clusters + 'type': 'gcp-types/dataproc-v1:projects.regions.clusters', 'properties': { 'clusterName': name, @@ -137,8 +134,9 @@ def generate_config(context): } } - for prop in ['configBucket', 'softwareConfig', 'initializationActions']: + for prop in ['configBucket', 'softwareConfig', 'initializationActions', 'encryptionConfig']: add_optional_property(cluster['properties']['config'], properties, prop) + add_optional_property(cluster['properties'], properties, 'labels') outputs = [ { @@ -147,7 +145,7 @@ def generate_config(context): }, { 'name': 'configBucket', - 'value': '$(ref.{}.config.configBucket)'.format(name) + 'value': '$(ref.{}.config.configBucket)'.format(context.env['name']) } ] diff --git a/dm/templates/dataproc/dataproc.py.schema b/dm/templates/dataproc/dataproc.py.schema index e98ae093105..66c7c06c140 100644 --- a/dm/templates/dataproc/dataproc.py.schema +++ b/dm/templates/dataproc/dataproc.py.schema @@ -15,19 +15,109 @@ info: title: Dataproc author: Sourced Group Inc. + version: 1.0.0 description: | Creates a Dataproc cluster. + For more information on this resource: + https://cloud.google.com/compute/ + + APIs endpoints used by this template: + - gcp-types/dataproc-v1:projects.regions.clusters => + https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.clusters + imports: - path: dataproc.py additionalProperties: false +definitions: + nodeConfig: + properties: + numInstances: + type: integer + description: The number of VM instances in the instance group. + isPreemptible: + type: boolean + description: | + If True, specifies that the instance group consists of preemptible + instances. + imageUri: + type: string + description: | + The Compute Engine image resource used for cluster instances. + It can be specified or may be inferred from SoftwareConfig.image_version. + machineType: + type: string + description: | + The Compute Engine machine type used for the cluster instances. + A full URL, partial URI, or short name are valid. Examples: + - https://www.googleapis.com/compute/v1/projects/[projectId]/zones/us-east1-a/machineTypes/n1-standard-2 + - projects/[projectId]/zones/us-east1-a/machineTypes/n1-standard-2 + - n1-standard-2 + diskType: + type: string + default: pd-standard + description: The boot disk type. + enum: + - pd-standard + - pd-ssd + diskSizeGb: + type: integer + default: 500 + description: The boot disk size in GB. + numLocalSsds: + type: integer + default: 0 + description: The number of attached SSDs. + minimum: 0 + maximum: 4 + accelerators: + type: array + uniqItems: true + description: | + The Compute Engine accelerator configuration for these instances. + + Beta Feature: This feature is still under development. It may be changed before final release + items: + type: object + additionalProperties: false + properties: + acceleratorTypeUri: + type: string + description: | + Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. + See Compute Engine AcceleratorTypes. + + Examples: + + https://www.googleapis.com/compute/beta/projects/[projectId]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80 + projects/[projectId]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80 + nvidia-tesla-k80 + Auto Zone Exception: If you are using the Cloud Dataproc Auto Zone Placement feature, + you must use the short name of the accelerator type resource, for example, nvidia-tesla-k80. + acceleratorCount: + type: number + description: | + The number of the accelerator cards of this type exposed to this instance. + properties: name: type: string description: | - The cluster name. If not provided, the resource name is used. + The cluster name. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the service. + labels: + type: object + description: | + Optional. The labels to associate with this cluster. Label keys must contain 1 to 63 characters, + and must conform to RFC 1035. Label values may be empty, but, if present, must contain 1 to 63 characters, + and must conform to RFC 1035. No more than 32 labels can be associated with a cluster. + + An object containing a list of "key": value pairs. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. region: type: string default: global @@ -56,6 +146,7 @@ properties: Engine service account. serviceAccountScopes: type: array + uniqItems: true description: | A list of URIs of service account scopes to be included in the Compute Engine instances. @@ -94,6 +185,7 @@ properties: and config. softwareConfig: type: object + additionalProperties: false description: | The selection and config of software inside the cluster. properties: @@ -107,12 +199,27 @@ properties: type: object description: | The key-value pairs for properties to set on the daemon config files. + optionalComponents: + type: array + uniqItems: true + description: | + The set of optional components to activate on the cluster. + items: + type: string + enum: + - COMPONENT_UNSPECIFIED + - ANACONDA + - HIVE_WEBHCAT + - JUPYTER + - ZEPPELIN initializationActions: type: array + uniqItems: true description: | A list of commands to execute on each node after the config is completed. items: type: object + additionalProperties: false description: | The executable to run on a fully configured node + the timeout period for the executable completion. @@ -125,87 +232,38 @@ properties: description: | The executable completion timeout, e.g. "3.5s". The default value is 10 minutes. + encryptionConfig: + type: object + additionalProperties: false + description: | + Encryption settings for the cluster. + required: + - gcePdKmsKeyName + properties: + gcePdKmsKeyName: + type: string + descritption: | + The Cloud KMS key name to use for PD disk encryption for all instances in the cluster. master: type: object + additionalProperties: false description: | The Compute Engine config settings for the master instance in the cluster. - properties: - numInstances: - type: integer - description: The number of VM instances in the instance group. - machineType: - type: string - description: | - The Compute Engine machine type used for the cluster instances. - A full URL, partial URI, or short name are valid. Examples: - - https://www.googleapis.com/compute/v1/projects/[projectId]/zones/us-east1-a/machineTypes/n1-standard-2 - - projects/[projectId]/zones/us-east1-a/machineTypes/n1-standard-2 - - n1-standard-2 - diskType: - type: string - default: pd-standard - description: The boot disk type. - enum: - - pd-standard - - pd-ssd - diskSizeGb: - type: integer - default: 500 - description: The boot disk size in GB. - numLocalSsds: - type: integer - default: 0 - description: The number of attached SSDs. - minimum: 0 - maximum: 4 + $ref: '#/definitions/nodeConfig' worker: type: object + additionalProperties: false description: | The Compute Engine config settings for worker instances in the cluster. - properties: - numInstances: - type: integer - description: The number of VM instances in the instance group. - machineType: - type: string - description: | - The Compute Engine machine type used for cluster instances. - A full URL, partial URI, or short name are valid. Examples: - - https://www.googleapis.com/compute/v1/projects/[projectId]/zones/us-east1-a/machineTypes/n1-standard-2 - - projects/[projectId]/zones/us-east1-a/machineTypes/n1-standard-2 - - n1-standard-2 - diskType: - type: string - default: pd-standard - description: The boot disk type. - enum: - - pd-standard - - pd-ssd - diskSizeGb: - type: integer - default: 500 - description: The boot disk size in GB. - numLocalSsds: - type: integer - default: 0 - description: The number of attached SSDs. - minimum: 0 - maximum: 4 + $ref: '#/definitions/nodeConfig' secondaryWorker: type: object + additionalProperties: false description: | The Compute Engine config settings for additional worker instances in the cluster. - properties: - numInstances: - type: integer - description: The number of VM instances in the instance group. - isPreemptible: - type: boolean - description: | - If True, specifies that the instance group consists of preemptible - instances. + $ref: '#/definitions/nodeConfig' outputs: properties: - masterInstanceNames: diff --git a/dm/templates/dns_managed_zone/README.md b/dm/templates/dns_managed_zone/README.md index 19b4b9b96ca..ac80d5cc690 100644 --- a/dm/templates/dns_managed_zone/README.md +++ b/dm/templates/dns_managed_zone/README.md @@ -12,7 +12,7 @@ This template creates a managed zone in the Cloud DNS (Domain Name System). ### Resources -- [dns.v1.managedZone](https://cloud.google.com/dns/docs/) +- [gcp-types/dns-v1:managedZones](https://cloud.google.com/dns/docs/reference/v1/managedZones) ### Properties @@ -59,5 +59,16 @@ See the `properties` section in the schema file(s): ``` ## Examples - - [Cloud DNS Managed Zone](examples/dns_managed_zone.yaml) +- [Cloud DNS Managed Zone with legacy property](examples/dns_managed_zone_legacy.yaml) +- [Managed Zone with `public visibility`](examples/dns_managed_zone_public.yaml) +- [Managed Zone with `private visibility`](examples/dns_managed_zone_private.yaml) +- [Managed Zone with `private visibility config`](examples/dns_managed_zone_private_visibility_config.yaml) + +## Tests Cases +- [Simple Managed Zone Test](tests/integration/dns_mz_simple.bats) +- [Backward Compatibility Test](tests/integration/dns_mz_bkwrd_cmptb.bats) +- [Managed Zone with `public visibility`](tests/integration/dns_mz_public.bats) +- [Managed Zone with `private visibility`](tests/integration/dns_mz_private.bats) +- [Managed Zone with `private visibility config`](tests/integration/dns_mz_prvt_vsblt_cfg.bats) +- [Managed Zone with `cross-project reference`](tests/integration/dns_mz_cross_project.bats) \ No newline at end of file diff --git a/dm/templates/dns_managed_zone/dns_managed_zone.py b/dm/templates/dns_managed_zone/dns_managed_zone.py index cacc2487c70..abc203b5d16 100644 --- a/dm/templates/dns_managed_zone/dns_managed_zone.py +++ b/dm/templates/dns_managed_zone/dns_managed_zone.py @@ -11,52 +11,63 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" This template creates a managed zone resource in the Cloud DNS. """ +"""This template creates a managed zone resource in the Cloud DNS.""" def generate_config(context): - """ Entry point for the deployment resources. """ - - resources = [] - - managed_zone_name = context.properties.get('zoneName') + """Entry point for the deployment resources.""" + # Backward Compatibility with the old property `zoneName` + try: + managed_zone_name = context.properties['zoneName'] + except KeyError: + managed_zone_name = context.properties.get('name', context.env['name']) dnsname = context.properties['dnsName'] managed_zone_description = context.properties['description'] name_servers = '$(ref.' + context.env['name'] + '.nameServers)' + project_id = context.properties.get('project', context.env['project']) + + resources = [] + outputs = [ + { + 'name': 'dnsName', + 'value': dnsname + }, + { + 'name': 'managedZoneDescription', + 'value': managed_zone_description + }, + { + 'name': 'nameServers', + 'value': name_servers + }, + { + 'name': 'managedZoneName', + 'value': managed_zone_name + } + ] managed_zone = { 'name': context.env['name'], - 'type': 'dns.v1.managedZone', - 'properties': - { - 'name': managed_zone_name, - 'dnsName': dnsname, - 'description': managed_zone_description - } + # https://cloud.google.com/dns/docs/reference/v1/managedZones + 'type': 'gcp-types/dns-v1:managedZones', + 'properties': { + 'name': managed_zone_name, + 'dnsName': dnsname, + 'description': managed_zone_description, + 'project_id': project_id + } } - resources.append(managed_zone) - - return { - 'resources': - resources, - 'outputs': - [ + # making resources and outputs additional properties + for prop in context.properties: + if prop not in managed_zone['properties']: + managed_zone['properties'][prop] = context.properties[prop] + outputs.append( { - 'name': 'dnsName', - 'value': dnsname - }, - { - 'name': 'managedZoneDescription', - 'value': managed_zone_description - }, - { - 'name': 'nameServers', - 'value': name_servers - }, - { - 'name': 'managedZoneName', - 'value': managed_zone_name + 'name': prop, + 'value': context.properties[prop] } - ] - } + ) + resources.append(managed_zone) + + return {'resources': resources, 'outputs': outputs} diff --git a/dm/templates/dns_managed_zone/dns_managed_zone.py.schema b/dm/templates/dns_managed_zone/dns_managed_zone.py.schema index 6c8c817b67c..028d4a8199d 100644 --- a/dm/templates/dns_managed_zone/dns_managed_zone.py.schema +++ b/dm/templates/dns_managed_zone/dns_managed_zone.py.schema @@ -15,42 +15,215 @@ info: title: Cloud DNS Managed Zone author: Source Group Inc. + version: 1.0.0 description: | Creates a managed zone in the Cloud DNS. - For more information on this resource + + For more information on this resource: - https://cloud.google.com/dns/zones/ + APIs endpoints used by this template: + - gcp-types/dns-v1:managedZones => + https://cloud.google.com/dns/docs/reference/v1/managedZones + imports: - path: dns_managed_zone.py -additionalProperties: false +# Note: Supported Backward Compatibility with the old property `zoneName` +oneOf: + - required: + - dnsName + - zoneName + - required: + - dnsName + - name -required: - - zoneName - - dnsName +additionalProperties: false properties: zoneName: type: string - pattern: ^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$ + pattern: ^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$ description: | - A user-assigned name for the managed zone. - This is required by the Cloud DNS. - Must be 1-63 characters long, must begin with a letter, - end with a letter or digit, and only contain lowercase letters, digits or dashes. - dnsName: + Old resource name to support backward compatablility. + Value is rescricted by API pattern for `resource.name` + The name must be 1-63 characters long, must begin with a letter, end + with a letter or digit, and only contain lowercase letters, digits or dashes. + project: type: string - pattern: \.$ description: | - The DNS name of the managed zone; for example, "example.com." - Make sure that the value ends with a period "." + The Project ID for Cross-Project Reference. + zoneName: + type: string + pattern: ^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$ + description: | + Old resource name to support backward compatablility. + Value is rescricted by API pattern for `resource.name` + The name must be 1-63 characters long, must begin with a letter, end + with a letter or digit, and only contain lowercase letters, digits or dashes. description: type: string pattern: ^.{0,1023}$ description: | - A description of the managed zone. A mutable string, max 1024 characters - long. Associated with the resource for users' convenience; does not affect + A description of the managed zone. A mutable string, max 1024 characters + long. Associated with the resource for users' convenience; does not affect managed zone's function. + dnsName: + type: string + pattern: ^([(a-z)\d\-]{1,62}\.){1,3}([(a-z)\d\-]{1,61}){0,1}\.$ + description: | + The DNS name of the managed zone; for example, "example.com." + Make sure that the value ends with a period "." + dnssecConfig: + type: object + description: DNSSEC configuration. + additionalProperties: false + required: + - kind + - state + - defaultKeySpecs + proeprties: + defaultKeySpecs: + type: array + uniqueItems: true + description: | + Specifies parameters that will be used for generating initial DnsKeys + for this ManagedZone. Output only while state is not OFF. + items: + type: object + additionalProperties: false + required: + - kind + - algorithm + - keyType + - keyLength + properties: + algorithm: + oneOf: + - type: string + pattern: ^ecdsap(256|384)sha(256|384)$ + - type: string + pattern: ^rsasha(1|256|512)$ + description: | + String mnemonic specifying the DNSSEC algorithm of this key. + Acceptable values are: + - "ecdsap256sha256" + - "ecdsap384sha384" + - "rsasha1" + - "rsasha256" + - "rsasha512" + keyLength: + type: integer + description: Length of the keys in bits. + keyType: + type: string + pattern: ^(key|zone)Signing$ + description: | + Specifies whether this is a key signing key (KSK) or a zone + signing key (ZSK). Key signing keys have the Secure Entry Point + flag set and, when active, will only be used to sign resource + record sets of type DNSKEY. Zone signing keys do not have the + Secure Entry Point flag set and will be used to sign all other + types of resource record sets. + Acceptable values are: + - "keySigning" + - "zoneSigning" + kind: + type: string + pattern: ^dns#managedZoneDnsSecConfig$ + default: "dns#managedZoneDnsSecConfig" + description: | + Identifies what kind of resource this is. + Value: the fixed string "dns#managedZoneDnsSecConfig". + nonExistence: + type: string + description: | + Specifies the mechanism used to provide authenticated + denial-of-existence responses. Output only while state is not OFF. + Acceptable values are: + - "nsec" + - "nsec3" + pattern: ^nsec3?$ + state: + type: string + pattern: ^(on|off|transfer)$ + description: | + Specifies whether DNSSEC is enabled, and what mode it is in. + Acceptable values are: + - "off" + - "on" + - "transfer" + kind: + type: string + pattern: ^dns#managedZone$ + default: "dns#managedZone" + description: | + Identifies what kind of resource this is. + Value is the fixed string "dns#managedZone". + labels: + type: object + description: User labels. + propertyNames: + type: string + name: + type: string + pattern: ^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$ + description: | + User assigned name for this resource. Must be unique within the project. + The name must be 1-63 characters long, must begin with a letter, end + with a letter or digit, and only contain lowercase letters, digits or dashes. + nameServerSet: + type: string + description: | + Optionally specifies the NameServerSet for this ManagedZone. A + NameServerSet is a set of DNS name servers that all host the same + ManagedZones. Most users will leave this field unset. + nameServers: + type: array + description: | + Delegate your managed_zone to these virtual name servers; defined by the + server (output only) + privateVisibilityConfig: + type: object + description: | + For privately visible zones, the set of Virtual Private Cloud resources + that the zone is visible from. + additionalProperties: false + properties: + kind: + type: string + pattern: ^dns#managedZonePrivateVisibilityConfig$ + description: | + Identifies what kind of resource this is. + Value: the fixed string "dns#managedZonePrivateVisibilityConfig" + networks: + type: array + items: + type: object + additionalProperties: false + required: + - kind + - networkUrl + properties: + kind: + type: string + pattern: ^dns#managedZonePrivateVisibilityConfigNetwork$ + description: | + Identifies what kind of resource this is. + Value: the fixed string "dns#managedZonePrivateVisibilityConfigNetwork". + networkUrl: + type: string + pattern: ^https:\/\/www.googleapis.com\/compute\/v1\/projects\/[a-zA-Z0-9_-]+\/global\/networks\/[a-zA-Z0-9_-]+$ + description: | + The fully qualified URL of the VPC network to bind to. This should be formatted + like https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network} + visibility: + type: string + pattern: ^(public|private)$ + description: | + The zone's visibility. Public zones are exposed to the Internet, while + private zones are visible only to Virtual Private Cloud resources. + Acceptable values are "private" and "public". outputs: properties: @@ -67,9 +240,25 @@ outputs: - managedZoneName: type: string description: The managed zone's resource name. - + - visibility: + type: string + description: | + The zone's visibility. Public zones are exposed to the Internet, + while private zones are visible only to Virtual Private Cloud + resources. + - privateVisibilityConfig: + type: object + description: | + For privately visible zones, the set of Virtual Private Cloud + resources that the zone is visible from. + - dnssecConfig: + type: object + description: DNSSEC configuration. documentation: - templates/dns_managed_zone/README.md examples: - templates/dns_managed_zone/examples/dns_managed_zone.yaml + - templates/dns_managed_zone/examples/dns_managed_zone_private.yaml + - templates/dns_managed_zone/examples/dns_managed_zone_private_visibility_config.yaml + - templates/dns_managed_zone/examples/dns_managed_zone_public.yaml diff --git a/dm/templates/dns_managed_zone/examples/dns_managed_zone.yaml b/dm/templates/dns_managed_zone/examples/dns_managed_zone.yaml index b67eb24c295..5755fdb2a61 100644 --- a/dm/templates/dns_managed_zone/examples/dns_managed_zone.yaml +++ b/dm/templates/dns_managed_zone/examples/dns_managed_zone.yaml @@ -1,7 +1,7 @@ # Example of the DNS managed zone template usage. # # In this example, a DNS managed zone is created with the use of -# the `zoneName` and `dnsName` properties. +# the `name` and `dnsName` properties. imports: - path: templates/dns_managed_zone/dns_managed_zone.py @@ -11,6 +11,6 @@ resources: - name: test-managed-zone type: dns_managed_zone.py properties: - zoneName: test-managed-zone + name: test-managed-zone dnsName: foobar.local. description: 'My foobar DNS Managed Zone' diff --git a/dm/templates/dns_managed_zone/examples/dns_managed_zone_legacy.yaml b/dm/templates/dns_managed_zone/examples/dns_managed_zone_legacy.yaml new file mode 100644 index 00000000000..0f2dcb6e104 --- /dev/null +++ b/dm/templates/dns_managed_zone/examples/dns_managed_zone_legacy.yaml @@ -0,0 +1,16 @@ +# Example of the DNS managed zone template usage. +# +# In this example, a DNS managed zone is created with the use of +# the old `zoneName` and `dnsName` properties. + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: test-managed-zone + type: dns_managed_zone.py + properties: + zoneName: test-managed-zone + dnsName: foobar.local. + description: 'My foobar DNS Managed Zone' diff --git a/dm/templates/dns_managed_zone/examples/dns_managed_zone_private.yaml b/dm/templates/dns_managed_zone/examples/dns_managed_zone_private.yaml new file mode 100644 index 00000000000..49caf906e28 --- /dev/null +++ b/dm/templates/dns_managed_zone/examples/dns_managed_zone_private.yaml @@ -0,0 +1,17 @@ +# Example of the DNS managed zone template usage. +# +# In this example, a private DNS managed zone is created with the use of +# the `visibility` and `dnsName` properties. + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: private-mz + type: dns_managed_zone.py + properties: + name: private-mz + dnsName: private-mz.local. + description: "Private DNS Managed Zone" + visibility: private diff --git a/dm/templates/dns_managed_zone/examples/dns_managed_zone_private_visibility_config.yaml b/dm/templates/dns_managed_zone/examples/dns_managed_zone_private_visibility_config.yaml new file mode 100644 index 00000000000..98e3e42e80e --- /dev/null +++ b/dm/templates/dns_managed_zone/examples/dns_managed_zone_private_visibility_config.yaml @@ -0,0 +1,22 @@ +# Example of the DNS managed zone template usage. +# +# In this example, a private DNS managed zone is created with the use of +# the `visibility` and `privateVisibilityConfig` properties. + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: private-mz-with-visibility + type: dns_managed_zone.py + properties: + name: private-mz-with-visibility + dnsName: private-visibility.local. + description: "Private DNS Managed Zone with visibility config" + visibility: private + privateVisibilityConfig: + kind: "dns#managedZonePrivateVisibilityConfig" + networks: + - kind: "dns#managedZonePrivateVisibilityConfigNetwork" + networkUrl: "https://www.googleapis.com/compute/v1/projects//global/networks/" diff --git a/dm/templates/dns_managed_zone/examples/dns_managed_zone_public.yaml b/dm/templates/dns_managed_zone/examples/dns_managed_zone_public.yaml new file mode 100644 index 00000000000..6fa68ebd94e --- /dev/null +++ b/dm/templates/dns_managed_zone/examples/dns_managed_zone_public.yaml @@ -0,0 +1,17 @@ +# Example of the DNS managed zone template usage. +# +# In this example, a Public DNS managed zone is created with the use of +# the `zoneName`, `dnsName` and `visibility` properties. + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: public-mz + type: dns_managed_zone.py + properties: + name: public-mz + dnsName: public-test.local. + description: "Public DNS Managed Zone" + visibility: public diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_bkwrd_cmptb.bats b/dm/templates/dns_managed_zone/tests/integration/dns_mz_bkwrd_cmptb.bats new file mode 100755 index 00000000000..83bb5088792 --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_bkwrd_cmptb.bats @@ -0,0 +1,86 @@ +#!/usr/bin/env bats + +source tests/helpers.bash + +TEST_NAME=$(basename "${BATS_TEST_FILENAME}" | cut -d '.' -f 1) + +# Create a random 10-char string and save it in a file. +RANDOM_FILE="/tmp/${CLOUD_FOUNDATION_ORGANIZATION_ID}-${TEST_NAME}.txt" +if [[ ! -e "${RANDOM_FILE}" ]]; then + RAND=$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 10) + echo ${RAND} > "${RANDOM_FILE}" +fi + +# Set variables based on the random string saved in the file. +# envsubst requires all variables used in the example/config to be exported. +if [[ -e "${RANDOM_FILE}" ]]; then + export RAND=$(cat "${RANDOM_FILE}") + DEPLOYMENT_NAME="${CLOUD_FOUNDATION_PROJECT_ID}-${TEST_NAME}-${RAND}" + # Replace underscores in the deployment name with dashes. + DEPLOYMENT_NAME=${DEPLOYMENT_NAME//_/-} + CONFIG=".${DEPLOYMENT_NAME}.yaml" + export CLOUDDNS_ZONE_NAME="test-managed-zone-${RAND}" + export CLOUDDNS_DNS_NAME="${RAND}.com." + export CLOUDDNS_DESCRIPTION="Managed DNS Zone for Testing" +fi + +########## HELPER FUNCTIONS ########## + +function create_config() { + echo "Creating ${CONFIG}" + envsubst < templates/dns_managed_zone/tests/integration/${TEST_NAME}.yaml > "${CONFIG}" +} + +function delete_config() { + echo "Deleting ${CONFIG}" + rm -f "${CONFIG}" +} + +function setup() { + # Global setup; this is executed once per test file. + if [ ${BATS_TEST_NUMBER} -eq 1 ]; then + create_config + fi + + # Per-test setup steps. +} + +function teardown() { + # Global teardown; this is executed once per test file. + if [[ "$BATS_TEST_NUMBER" -eq "${#BATS_TEST_NAMES[@]}" ]]; then + delete_config + rm -f "${RANDOM_FILE}" + fi + + # Per-test teardown steps. +} + + +########## TESTS ########## + +@test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { + gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ + --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] +} + +@test "Verify if a managed zone with name $CLOUDDNS_ZONE_NAME was created" { + run gcloud dns managed-zones list --format=flattened \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} + +@test "Verify if a DNS named ${CLOUDDNS_DNS_NAME} was created" { + run gcloud dns managed-zones list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_DNS_NAME}" ]] +} + +@test "Deleting deployment ${DEPLOYMENT_NAME}" { + gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" \ + -q --project "${CLOUD_FOUNDATION_PROJECT_ID}" + run gcloud dns managed-zones list + [[ "$status" -eq 0 ]] + [[ ! "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} \ No newline at end of file diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_bkwrd_cmptb.yaml b/dm/templates/dns_managed_zone/tests/integration/dns_mz_bkwrd_cmptb.yaml new file mode 100644 index 00000000000..48e35ae5680 --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_bkwrd_cmptb.yaml @@ -0,0 +1,19 @@ +# Test Case: Backward Compatibility +# Use Case: +# You have updated CFT code base up to the latest version and now it works +# with your old-style written templates in a slightly different way. +# +# F.e.: `zoneName` property is now replaced by `name` to align syntax with +# the naming convention of the API. + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: ${CLOUDDNS_ZONE_NAME}-resource + type: dns_managed_zone.py + properties: + zoneName: ${CLOUDDNS_ZONE_NAME} + dnsName: ${CLOUDDNS_DNS_NAME} + description: ${CLOUDDNS_DESCRIPTION} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_cross_project.bats b/dm/templates/dns_managed_zone/tests/integration/dns_mz_cross_project.bats new file mode 100755 index 00000000000..2d1b32274c1 --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_cross_project.bats @@ -0,0 +1,91 @@ +#!/usr/bin/env bats + +source tests/helpers.bash + +TEST_NAME=$(basename "${BATS_TEST_FILENAME}" | cut -d '.' -f 1) + +# Create a random 10-char string and save it in a file. +RANDOM_FILE="/tmp/${CLOUD_FOUNDATION_ORGANIZATION_ID}-${TEST_NAME}.txt" +if [[ ! -e "${RANDOM_FILE}" ]]; then + RAND=$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 10) + echo ${RAND} > "${RANDOM_FILE}" +fi + +# Set variables based on the random string saved in the file. +# envsubst requires all variables used in the example/config to be exported. +if [[ -e "${RANDOM_FILE}" ]]; then + export RAND=$(cat "${RANDOM_FILE}") + DEPLOYMENT_NAME="${CLOUD_FOUNDATION_PROJECT_ID}-${TEST_NAME}-${RAND}" + # Replace underscores in the deployment name with dashes. + DEPLOYMENT_NAME=${DEPLOYMENT_NAME//_/-} + CONFIG=".${DEPLOYMENT_NAME}.yaml" + export CLOUDDNS_ZONE_NAME="test-managed-zone-${RAND}" + export CLOUDDNS_DNS_NAME="${RAND}.com." + export CLOUDDNS_DESCRIPTION="Managed DNS Zone for Testing" +fi + +if [ -z "${CLOUDDNS_CROSS_PROJECT_ID}" ]; then + echo "CLOUDDNS_CROSS_PROJECT_ID is not set, nothing to test." >&2 + exit 1 +fi + +########## HELPER FUNCTIONS ########## + +function create_config() { + echo "Creating ${CONFIG}" + envsubst < templates/dns_managed_zone/tests/integration/${TEST_NAME}.yaml > "${CONFIG}" +} + +function delete_config() { + echo "Deleting ${CONFIG}" + rm -f "${CONFIG}" +} + +function setup() { + # Global setup; this is executed once per test file. + if [ ${BATS_TEST_NUMBER} -eq 1 ]; then + create_config + fi + + # Per-test setup steps. +} + +function teardown() { + # Global teardown; this is executed once per test file. + if [[ "$BATS_TEST_NUMBER" -eq "${#BATS_TEST_NAMES[@]}" ]]; then + delete_config + rm -f "${RANDOM_FILE}" + fi + + # Per-test teardown steps. +} + + +########## TESTS ########## + +@test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { + gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ + --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] +} + +@test "Verify if a managed zone with name $CLOUDDNS_ZONE_NAME was created" { + run gcloud dns managed-zones list --format=flattened \ + --project "${CLOUDDNS_CROSS_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} + +@test "Verify if a DNS named ${CLOUDDNS_DNS_NAME} was created" { + run gcloud dns managed-zones list --project "${CLOUDDNS_CROSS_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_DNS_NAME}" ]] +} + +@test "Deleting deployment ${DEPLOYMENT_NAME}" { + gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" \ + -q --project "${CLOUD_FOUNDATION_PROJECT_ID}" + run gcloud dns managed-zones list + [[ "$status" -eq 0 ]] + [[ ! "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_cross_project.yaml b/dm/templates/dns_managed_zone/tests/integration/dns_mz_cross_project.yaml new file mode 100644 index 00000000000..5e3b394057e --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_cross_project.yaml @@ -0,0 +1,22 @@ +# Test Case: Cross-Project Reference +# Use Case: +# You have multiple projects with dependancies on each other. +# Like one project assumes a presence of DNS Zone in another in order +# to use it as an endpoint. So within your agregated pipe-line you may want +# to provision resources in both of the projets. +# +# Please note: you should grant Editor permission on the cross-referenced +# Project to your current Google APIs account @cloudservices.gserviceaccount.com + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: ${CLOUDDNS_ZONE_NAME}-resource + type: dns_managed_zone.py + properties: + name: ${CLOUDDNS_ZONE_NAME} + dnsName: ${CLOUDDNS_DNS_NAME} + description: ${CLOUDDNS_DESCRIPTION} + project: ${CLOUDDNS_CROSS_PROJECT_ID} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_private.bats b/dm/templates/dns_managed_zone/tests/integration/dns_mz_private.bats new file mode 100755 index 00000000000..a4c6f5ac45c --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_private.bats @@ -0,0 +1,94 @@ +#!/usr/bin/env bats + +source tests/helpers.bash + +TEST_NAME=$(basename "${BATS_TEST_FILENAME}" | cut -d '.' -f 1) + +# Create a random 10-char string and save it in a file. +RANDOM_FILE="/tmp/${CLOUD_FOUNDATION_ORGANIZATION_ID}-${TEST_NAME}.txt" +if [[ ! -e "${RANDOM_FILE}" ]]; then + RAND=$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 10) + echo ${RAND} > "${RANDOM_FILE}" +fi + +# Set variables based on the random string saved in the file. +# envsubst requires all variables used in the example/config to be exported. +if [[ -e "${RANDOM_FILE}" ]]; then + export RAND=$(cat "${RANDOM_FILE}") + DEPLOYMENT_NAME="${CLOUD_FOUNDATION_PROJECT_ID}-${TEST_NAME}-${RAND}" + # Replace underscores in the deployment name with dashes. + DEPLOYMENT_NAME=${DEPLOYMENT_NAME//_/-} + CONFIG=".${DEPLOYMENT_NAME}.yaml" + export CLOUDDNS_ZONE_NAME="test-managed-zone-${RAND}" + export CLOUDDNS_DNS_NAME="${RAND}.com." + export CLOUDDNS_DESCRIPTION="Managed DNS Zone for Testing" + export CLOUDDNS_VISIBILITY="private" +fi + +########## HELPER FUNCTIONS ########## + +function create_config() { + echo "Creating ${CONFIG}" + envsubst < templates/dns_managed_zone/tests/integration/${TEST_NAME}.yaml > "${CONFIG}" +} + +function delete_config() { + echo "Deleting ${CONFIG}" + rm -f "${CONFIG}" +} + +function setup() { + # Global setup; this is executed once per test file. + if [ ${BATS_TEST_NUMBER} -eq 1 ]; then + create_config + fi + + # Per-test setup steps. +} + +function teardown() { + # Global teardown; this is executed once per test file. + if [[ "$BATS_TEST_NUMBER" -eq "${#BATS_TEST_NAMES[@]}" ]]; then + delete_config + rm -f "${RANDOM_FILE}" + fi + + # Per-test teardown steps. +} + + +########## TESTS ########## + +@test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { + gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ + --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] +} + +@test "Verify if a managed zone with name $CLOUDDNS_ZONE_NAME was created" { + run gcloud dns managed-zones list --format=flattened \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} + +@test "Verify if a DNS named ${CLOUDDNS_DNS_NAME} was created" { + run gcloud dns managed-zones list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_DNS_NAME}" ]] +} + +@test "Verify if visibility is ${CLOUDDNS_VISIBILITY}" { + run gcloud dns managed-zones describe ${CLOUDDNS_ZONE_NAME} \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "visibility: ${CLOUDDNS_VISIBILITY}" ]] +} + +@test "Deleting deployment ${DEPLOYMENT_NAME}" { + gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" \ + -q --project "${CLOUD_FOUNDATION_PROJECT_ID}" + run gcloud dns managed-zones list + [[ "$status" -eq 0 ]] + [[ ! "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_private.yaml b/dm/templates/dns_managed_zone/tests/integration/dns_mz_private.yaml new file mode 100644 index 00000000000..2edaa029f93 --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_private.yaml @@ -0,0 +1,17 @@ +# Test Case: Private Visibility +# Use Case: +# You want to create a Private Managed Zone, which is not exposed to Internet +# and visible only to Virtual Private Cloud resources. + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: ${CLOUDDNS_ZONE_NAME}-resource + type: dns_managed_zone.py + properties: + name: ${CLOUDDNS_ZONE_NAME} + dnsName: ${CLOUDDNS_DNS_NAME} + description: ${CLOUDDNS_DESCRIPTION} + visibility: ${CLOUDDNS_VISIBILITY} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_prvt_vsblt_cfg.bats b/dm/templates/dns_managed_zone/tests/integration/dns_mz_prvt_vsblt_cfg.bats new file mode 100755 index 00000000000..51dae4a169e --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_prvt_vsblt_cfg.bats @@ -0,0 +1,102 @@ +#!/usr/bin/env bats + +source tests/helpers.bash + +TEST_NAME=$(basename "${BATS_TEST_FILENAME}" | cut -d '.' -f 1) + +# Create a random 10-char string and save it in a file. +RANDOM_FILE="/tmp/${CLOUD_FOUNDATION_ORGANIZATION_ID}-${TEST_NAME}.txt" +if [[ ! -e "${RANDOM_FILE}" ]]; then + RAND=$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 10) + echo ${RAND} > "${RANDOM_FILE}" +fi + +# Set variables based on the random string saved in the file. +# envsubst requires all variables used in the example/config to be exported. +if [[ -e "${RANDOM_FILE}" ]]; then + export RAND=$(cat "${RANDOM_FILE}") + DEPLOYMENT_NAME="${CLOUD_FOUNDATION_PROJECT_ID}-${TEST_NAME}-${RAND}" + # Replace underscores in the deployment name with dashes. + DEPLOYMENT_NAME=${DEPLOYMENT_NAME//_/-} + CONFIG=".${DEPLOYMENT_NAME}.yaml" + export CLOUDDNS_ZONE_NAME="test-managed-zone-${RAND}" + export CLOUDDNS_DNS_NAME="${RAND}.com." + export CLOUDDNS_DESCRIPTION="Managed DNS Zone for Testing" + export CLOUDDNS_VISIBILITY="private" + export CLOUDDNS_NETWORK_URL="https://www.googleapis.com/compute/v1/projects/${CLOUD_FOUNDATION_PROJECT_ID}/global/networks/default" +fi + +########## HELPER FUNCTIONS ########## + +function create_config() { + echo "Creating ${CONFIG}" + envsubst < templates/dns_managed_zone/tests/integration/${TEST_NAME}.yaml > "${CONFIG}" +} + +function delete_config() { + echo "Deleting ${CONFIG}" + rm -f "${CONFIG}" +} + +function setup() { + # Global setup; this is executed once per test file. + if [ ${BATS_TEST_NUMBER} -eq 1 ]; then + create_config + fi + + # Per-test setup steps. +} + +function teardown() { + # Global teardown; this is executed once per test file. + if [[ "$BATS_TEST_NUMBER" -eq "${#BATS_TEST_NAMES[@]}" ]]; then + delete_config + rm -f "${RANDOM_FILE}" + fi + + # Per-test teardown steps. +} + + +########## TESTS ########## + +@test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { + gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ + --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] +} + +@test "Verify if a managed zone with name $CLOUDDNS_ZONE_NAME was created" { + run gcloud dns managed-zones list --format=flattened \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} + +@test "Verify if a DNS named ${CLOUDDNS_DNS_NAME} was created" { + run gcloud dns managed-zones list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_DNS_NAME}" ]] +} + +@test "Verify if visibility is ${CLOUDDNS_VISIBILITY}" { + run gcloud dns managed-zones describe ${CLOUDDNS_ZONE_NAME} \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "visibility: ${CLOUDDNS_VISIBILITY}" ]] +} + +@test "Verify if networkUrl is ${CLOUDDNS_NETWORK_URL}" { + run gcloud dns managed-zones describe ${CLOUDDNS_ZONE_NAME} \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "networkUrl: ${CLOUDDNS_NETWORK_URL}" ]] +} + +@test "Deleting deployment ${DEPLOYMENT_NAME}" { + gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" \ + -q --project "${CLOUD_FOUNDATION_PROJECT_ID}" + run gcloud dns managed-zones list + [[ "$status" -eq 0 ]] + [[ ! "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_prvt_vsblt_cfg.yaml b/dm/templates/dns_managed_zone/tests/integration/dns_mz_prvt_vsblt_cfg.yaml new file mode 100644 index 00000000000..642d22e70d0 --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_prvt_vsblt_cfg.yaml @@ -0,0 +1,22 @@ +# Test Case: Private Visibility +# Use Case: +# You want to create a Private Managed Zone visible only for specific +# networks of your Virtual Private Cloud. + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: ${CLOUDDNS_ZONE_NAME}-resource + type: dns_managed_zone.py + properties: + name: ${CLOUDDNS_ZONE_NAME} + dnsName: ${CLOUDDNS_DNS_NAME} + description: ${CLOUDDNS_DESCRIPTION} + visibility: ${CLOUDDNS_VISIBILITY} + privateVisibilityConfig: + kind: "dns#managedZonePrivateVisibilityConfig" + networks: + - kind: "dns#managedZonePrivateVisibilityConfigNetwork" + networkUrl: ${CLOUDDNS_NETWORK_URL} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_public.bats b/dm/templates/dns_managed_zone/tests/integration/dns_mz_public.bats new file mode 100755 index 00000000000..e48411694bd --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_public.bats @@ -0,0 +1,95 @@ +#!/usr/bin/env bats + +source tests/helpers.bash + +TEST_NAME=$(basename "${BATS_TEST_FILENAME}" | cut -d '.' -f 1) + +# Create a random 10-char string and save it in a file. +RANDOM_FILE="/tmp/${CLOUD_FOUNDATION_ORGANIZATION_ID}-${TEST_NAME}.txt" +if [[ ! -e "${RANDOM_FILE}" ]]; then + RAND=$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 10) + echo ${RAND} > "${RANDOM_FILE}" +fi + +# Set variables based on the random string saved in the file. +# envsubst requires all variables used in the example/config to be exported. +if [[ -e "${RANDOM_FILE}" ]]; then + export RAND=$(cat "${RANDOM_FILE}") + DEPLOYMENT_NAME="${CLOUD_FOUNDATION_PROJECT_ID}-${TEST_NAME}-${RAND}" + # Replace underscores in the deployment name with dashes. + DEPLOYMENT_NAME=${DEPLOYMENT_NAME//_/-} + CONFIG=".${DEPLOYMENT_NAME}.yaml" + export CLOUDDNS_ZONE_NAME="test-managed-zone-${RAND}" + export CLOUDDNS_DNS_NAME="${RAND}.com." + export CLOUDDNS_DESCRIPTION="Managed DNS Zone for Testing" + export CLOUDDNS_VISIBILITY="public" + export CLOUDDNS_NETWORKS="default" +fi + +########## HELPER FUNCTIONS ########## + +function create_config() { + echo "Creating ${CONFIG}" + envsubst < templates/dns_managed_zone/tests/integration/${TEST_NAME}.yaml > "${CONFIG}" +} + +function delete_config() { + echo "Deleting ${CONFIG}" + rm -f "${CONFIG}" +} + +function setup() { + # Global setup; this is executed once per test file. + if [ ${BATS_TEST_NUMBER} -eq 1 ]; then + create_config + fi + + # Per-test setup steps. +} + +function teardown() { + # Global teardown; this is executed once per test file. + if [[ "$BATS_TEST_NUMBER" -eq "${#BATS_TEST_NAMES[@]}" ]]; then + delete_config + rm -f "${RANDOM_FILE}" + fi + + # Per-test teardown steps. +} + + +########## TESTS ########## + +@test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { + gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ + --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] +} + +@test "Verify if a managed zone with name $CLOUDDNS_ZONE_NAME was created" { + run gcloud dns managed-zones list --format=flattened \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} + +@test "Verify if a DNS named ${CLOUDDNS_DNS_NAME} was created" { + run gcloud dns managed-zones list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_DNS_NAME}" ]] +} + +@test "Verify if visibility is ${CLOUDDNS_VISIBILITY}" { + run gcloud dns managed-zones describe ${CLOUDDNS_ZONE_NAME} \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "visibility: ${CLOUDDNS_VISIBILITY}" ]] +} + +@test "Deleting deployment ${DEPLOYMENT_NAME}" { + gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" \ + -q --project "${CLOUD_FOUNDATION_PROJECT_ID}" + run gcloud dns managed-zones list + [[ "$status" -eq 0 ]] + [[ ! "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] +} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_mz_public.yaml b/dm/templates/dns_managed_zone/tests/integration/dns_mz_public.yaml new file mode 100644 index 00000000000..328e7dc7278 --- /dev/null +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_public.yaml @@ -0,0 +1,17 @@ +# Test Case: Public Visibility +# Use Case: +# You want to create a Managed Zone with Public Visibility, +# which makes it exposed to Internet + +imports: + - path: templates/dns_managed_zone/dns_managed_zone.py + name: dns_managed_zone.py + +resources: + - name: ${CLOUDDNS_ZONE_NAME}-resource + type: dns_managed_zone.py + properties: + name: ${CLOUDDNS_ZONE_NAME} + dnsName: ${CLOUDDNS_DNS_NAME} + description: ${CLOUDDNS_DESCRIPTION} + visibility: ${CLOUDDNS_VISIBILITY} diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_managed_zone.bats b/dm/templates/dns_managed_zone/tests/integration/dns_mz_simple.bats old mode 100644 new mode 100755 similarity index 84% rename from dm/templates/dns_managed_zone/tests/integration/dns_managed_zone.bats rename to dm/templates/dns_managed_zone/tests/integration/dns_mz_simple.bats index 5b03dcb6626..ab1bf5af83c --- a/dm/templates/dns_managed_zone/tests/integration/dns_managed_zone.bats +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_simple.bats @@ -59,19 +59,20 @@ function teardown() { ########## TESTS ########## @test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { - gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ - --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ "$status" -eq 0 ]] + gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ + --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] } @test "Verify if a managed zone with name $CLOUDDNS_ZONE_NAME was created" { - run gcloud dns managed-zones list --format=flattened - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] + run gcloud dns managed-zones list --format=flattened \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + [[ "$status" -eq 0 ]] + [[ "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] } @test "Verify if a DNS named ${CLOUDDNS_DNS_NAME} was created" { - run gcloud dns managed-zones list + run gcloud dns managed-zones list --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ "$status" -eq 0 ]] [[ "$output" =~ "${CLOUDDNS_DNS_NAME}" ]] } @@ -84,4 +85,3 @@ function teardown() { [[ "$status" -eq 0 ]] [[ ! "$output" =~ "${CLOUDDNS_ZONE_NAME}" ]] } - diff --git a/dm/templates/dns_managed_zone/tests/integration/dns_managed_zone.yaml b/dm/templates/dns_managed_zone/tests/integration/dns_mz_simple.yaml similarity index 76% rename from dm/templates/dns_managed_zone/tests/integration/dns_managed_zone.yaml rename to dm/templates/dns_managed_zone/tests/integration/dns_mz_simple.yaml index 0ed353ad347..6a1a7956d9a 100644 --- a/dm/templates/dns_managed_zone/tests/integration/dns_managed_zone.yaml +++ b/dm/templates/dns_managed_zone/tests/integration/dns_mz_simple.yaml @@ -1,5 +1,4 @@ -# Test of the DNS managed zone template. -# +# Test of the simplest DNS managed zone template. imports: - path: templates/dns_managed_zone/dns_managed_zone.py @@ -9,6 +8,6 @@ resources: - name: ${CLOUDDNS_ZONE_NAME}-resource type: dns_managed_zone.py properties: - zoneName: ${CLOUDDNS_ZONE_NAME} + name: ${CLOUDDNS_ZONE_NAME} dnsName: ${CLOUDDNS_DNS_NAME} description: ${CLOUDDNS_DESCRIPTION} diff --git a/dm/templates/dns_records/dns_records.py b/dm/templates/dns_records/dns_records.py index 304e32137d0..393416bcf8d 100644 --- a/dm/templates/dns_records/dns_records.py +++ b/dm/templates/dns_records/dns_records.py @@ -15,20 +15,21 @@ def generate_config(context): - """ - Entry point for the deployment resources. - DNS RecordSet is natively supported since 2019. + """ + Entry point for the deployment resources. + + DNS RecordSet is natively supported since 2019. """ recordset = { 'name': context.env['name'], + # https://cloud.google.com/dns/docs/reference/v1/resourceRecordSets 'type': 'gcp-types/dns-v1:resourceRecordSets', - 'properties': - { - 'name': context.properties['dnsName'], - 'managedZone': context.properties['zoneName'], - 'records': context.properties['resourceRecordSets'] - } + 'properties': { + 'name': context.properties['dnsName'], + 'managedZone': context.properties['zoneName'], + 'records': context.properties['resourceRecordSets'], + } } return {'resources': [recordset]} diff --git a/dm/templates/dns_records/dns_records.py.schema b/dm/templates/dns_records/dns_records.py.schema index e943b107dcf..6b34dadbc8e 100644 --- a/dm/templates/dns_records/dns_records.py.schema +++ b/dm/templates/dns_records/dns_records.py.schema @@ -15,11 +15,16 @@ info: title: Cloud DNS Records author: Sourced Group Inc. + version: 1.0.0 description: | Creates DNS resource recordsets. + For more information on this resource: - https://cloud.google.com/dns/records/ - - https://cloud.google.com/dns/api/v1/managedZones + + APIs endpoints used by this template: + - gcp-types/dns-v1:resourceRecordSets => + https://cloud.google.com/dns/docs/reference/v1/resourceRecordSets imports: - path: dns_records.py @@ -40,10 +45,11 @@ properties: This is required by the Cloud DNS. dnsName: type: string - pattern: \.$ + pattern: ^([(a-z)\d\-]{1,62}\.){1,3}([(a-z)\d\-]{1,61}){0,1}\.$ description: | The DNS name of the managed zone; for example, "example.com." A fully qualified domain name (FQDN) must end with a period "." + Must be fully compliant with RFC 1035. resourceRecordSets: type: array description: | @@ -61,13 +67,43 @@ properties: name: type: string description: The name of the DNS record. Must end with dnsName. + pattern: ([(a-z)\d\-]{1,62}\.){1,3}([(a-z)\d\-]{1,61}){0,1}\.$ + kind: + type: string + description: | + Identifies what kind of resource this is. + Value: the fixed string "dns#resourceRecordSet". + pattern: ^dns#resourceRecordSet$ + default: "dns#resourceRecordSet" + rrdatas: + type: array + description: | + A list of resourceRecordSets as defined in + RFC 1035 (section 5) and RFC 1034 (section 3.6.1). + Examples - https://cloud.google.com/dns/records/json-record + items: + type: string + signatureRrdatas: + type: array + description: As defined in RFC 4034 (section 3.2). + items: + type: string + ttl: + type: integer + description: | + Number of seconds that this ResourceRecordSet can be cached by resolvers. + minimum: 0 type: type: string - description: The type of the record. + description: | + The identifier of a supported record type. + https://cloud.google.com/dns/docs/overview#supported_dns_record_types enum: - A - AAAA + - CAA - CNAME + - IPSECKEY - MX - NAPTR - NS @@ -75,15 +111,6 @@ properties: - SOA - SPF - SRV + - SSHFP + - TLSA - TXT - ttl: - type: integer - description: The time-to-live cache, in seconds. - minimum: 0 - rrdatas: - type: array - description: | - A list of resourceRecordSets as defined in - RFC 1035 (section 5) and RFC 1034 (section 3.6.1). - items: - type: string diff --git a/dm/templates/dns_records/tests/integration/dns_records.bats b/dm/templates/dns_records/tests/integration/dns_records.bats index e1c82197fcd..09bd3340d5c 100755 --- a/dm/templates/dns_records/tests/integration/dns_records.bats +++ b/dm/templates/dns_records/tests/integration/dns_records.bats @@ -58,7 +58,8 @@ function setup() { create_config # Create DNS Managed Zone gcloud dns managed-zones create --dns-name="${CLOUDDNS_DNS_NAME}" \ - --description="Test managed zone" "${CLOUDDNS_ZONE_NAME}" + --description="Test managed zone" "${CLOUDDNS_ZONE_NAME}" \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" fi # Per-test setup steps. @@ -71,7 +72,8 @@ function teardown() { rm -f "${RANDOM_FILE}" # Delete DNS Managed Zone echo "Deleting cloud DNS managed zone: ${CLOUDDNS_ZONE_NAME}" - gcloud dns managed-zones delete "${CLOUDDNS_ZONE_NAME}" + gcloud dns managed-zones delete "${CLOUDDNS_ZONE_NAME}" \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" fi # Per-test teardown steps. diff --git a/dm/templates/external_load_balancer/examples/external_load_balancer_http.yaml b/dm/templates/external_load_balancer/examples/external_load_balancer_http.yaml index f473653daee..5ea0bbde1e9 100644 --- a/dm/templates/external_load_balancer/examples/external_load_balancer_http.yaml +++ b/dm/templates/external_load_balancer/examples/external_load_balancer_http.yaml @@ -17,7 +17,7 @@ resources: properties: portRange: 80 backendServices: - - name: default-backend-service + - resourceName: default-backend-service sessionAffinity: GENERATED_COOKIE affinityCookieTtlSec: 1000 portName: http diff --git a/dm/templates/external_load_balancer/examples/external_load_balancer_https.yaml b/dm/templates/external_load_balancer/examples/external_load_balancer_https.yaml index f5fc6bb5ce1..6dcecd90e7d 100644 --- a/dm/templates/external_load_balancer/examples/external_load_balancer_https.yaml +++ b/dm/templates/external_load_balancer/examples/external_load_balancer_https.yaml @@ -24,12 +24,12 @@ resources: properties: portRange: 443 backendServices: - - name: static-backend-service + - resourceName: static-backend-service portName: https healthCheck: backends: - group: - - name: media-backend-service + - resourceName: media-backend-service portName: https healthCheck: backends: diff --git a/dm/templates/external_load_balancer/examples/external_load_balancer_ssl.yaml b/dm/templates/external_load_balancer/examples/external_load_balancer_ssl.yaml index a8a1595eb1a..6f7387d746e 100644 --- a/dm/templates/external_load_balancer/examples/external_load_balancer_ssl.yaml +++ b/dm/templates/external_load_balancer/examples/external_load_balancer_ssl.yaml @@ -18,7 +18,7 @@ resources: properties: portRange: 443 backendServices: - - name: backend-service + - resourceName: backend-service portName: https healthCheck: backends: diff --git a/dm/templates/external_load_balancer/examples/external_load_balancer_tcp.yaml b/dm/templates/external_load_balancer/examples/external_load_balancer_tcp.yaml index 668076ec4f6..dd9c7640984 100644 --- a/dm/templates/external_load_balancer/examples/external_load_balancer_tcp.yaml +++ b/dm/templates/external_load_balancer/examples/external_load_balancer_tcp.yaml @@ -20,7 +20,7 @@ resources: properties: portRange: 80 backendServices: - - name: backend-service + - resourceName: backend-service portName: http healthCheck: backends: diff --git a/dm/templates/external_load_balancer/external_load_balancer.py b/dm/templates/external_load_balancer/external_load_balancer.py index c02572d1264..2e50f333412 100644 --- a/dm/templates/external_load_balancer/external_load_balancer.py +++ b/dm/templates/external_load_balancer/external_load_balancer.py @@ -14,6 +14,8 @@ """ This template creates an external load balancer. """ import copy +from hashlib import sha1 +import json def set_optional_property(destination, source, prop_name): @@ -23,17 +25,20 @@ def set_optional_property(destination, source, prop_name): destination[prop_name] = source[prop_name] -def get_backend_service(properties, backend_spec): +def get_backend_service(properties, backend_spec, res_name, project_id): """ Creates the backend service. """ + name = backend_spec.get('resourceName', res_name) + backend_name = backend_spec.get('name', name) backend_properties = { + 'name': backend_name, + 'project': project_id, 'loadBalancingScheme': 'EXTERNAL', - 'protocol': get_protocol(properties) + 'protocol': get_protocol(properties), } - backend_name = backend_spec['name'] backend_resource = { - 'name': backend_name, + 'name': name, 'type': 'backend_service.py', 'properties': backend_properties } @@ -61,24 +66,30 @@ def get_backend_service(properties, backend_spec): }, { 'name': 'backendServiceSelfLink', - 'value': '$(ref.{}.selfLink)'.format(backend_name), + 'value': '$(ref.{}.selfLink)'.format(name), }, ] -def get_forwarding_rule(properties, target, name): +def get_forwarding_rule(properties, target, res_name, project_id): """ Creates the forwarding rule. """ + name = '{}-forwarding-rule'.format(res_name) rule_properties = { + 'name': properties.get('name', res_name), + 'project': project_id, 'loadBalancingScheme': 'EXTERNAL', 'target': '$(ref.{}.selfLink)'.format(target['name']), - 'IPProtocol': 'TCP' + 'IPProtocol': 'TCP', } rule_resource = { 'name': name, 'type': 'forwarding_rule.py', - 'properties': rule_properties + 'properties': rule_properties, + 'metadata': { + 'dependsOn': [target['name']], + }, } optional_properties = [ @@ -94,7 +105,7 @@ def get_forwarding_rule(properties, target, name): return [rule_resource], [ { 'name': 'forwardingRuleName', - 'value': name, + 'value': rule_properties['name'], }, { 'name': 'forwardingRuleSelfLink', @@ -107,7 +118,7 @@ def get_forwarding_rule(properties, target, name): ] -def get_backend_services(properties): +def get_backend_services(properties, res_name, project_id): """ Creates all backend services to be used by the load balancer. """ backend_resources = [] @@ -118,7 +129,8 @@ def get_backend_services(properties): backend_specs = properties['backendServices'] for backend_spec in backend_specs: - resources, outputs = get_backend_service(properties, backend_spec) + backend_res_name = '{}-backend-service-{}'.format(res_name, sha1(json.dumps(backend_spec)).hexdigest()[:10]) + resources, outputs = get_backend_service(properties, backend_spec, backend_res_name, project_id) backend_resources += resources # Merge outputs with the same name. for output in outputs: @@ -154,21 +166,26 @@ def update_refs_recursively(properties): update_refs_recursively(item) -def get_url_map(properties, res_name): +def get_url_map(properties, res_name, project_id): """ Creates a UrlMap resource. """ - name = properties.get('name', res_name + '-url-map') spec = copy.deepcopy(properties) + spec['project'] = project_id + spec['name'] = properties.get('name', res_name) update_refs_recursively(spec) - resource = {'name': name, 'type': 'url_map.py', 'properties': spec} + resource = { + 'name': res_name, + 'type': 'url_map.py', + 'properties': spec, + } - self_link = '$(ref.{}.selfLink)'.format(name) + self_link = '$(ref.{}.selfLink)'.format(res_name) return self_link, [resource], [ { 'name': 'urlMapName', - 'value': '$(ref.{}.name)'.format(name) + 'value': '$(ref.{}.name)'.format(res_name) }, { 'name': 'urlMapSelfLink', @@ -177,20 +194,25 @@ def get_url_map(properties, res_name): ] -def get_target_proxy(properties, res_name): +def get_target_proxy(properties, res_name, project_id, bs_resources): """ Creates a target proxy resource. """ protocol = get_protocol(properties) + depends = [] if 'HTTP' in protocol: + urlMap = copy.deepcopy(properties['urlMap']) + if 'name' not in urlMap and 'name' in properties: + urlMap['name'] = '{}-url-map'.format(properties['name']) target, resources, outputs = get_url_map( - properties['urlMap'], - res_name + urlMap, + '{}-url-map'.format(res_name), + project_id ) + depends.append(resources[0]['name']) else: - backend_services = properties['backendServices'] - service_name = backend_services[0]['name'] - target = get_ref(service_name) + depends.append(bs_resources[0]['name']) + target = get_ref(bs_resources[0]['name']) resources = [] outputs = [] @@ -199,9 +221,14 @@ def get_target_proxy(properties, res_name): 'name': name, 'type': 'target_proxy.py', 'properties': { + 'name': '{}-target'.format(properties.get('name', res_name)), + 'project': project_id, 'protocol': protocol, - 'target': target - } + 'target': target, + }, + 'metadata': { + 'dependsOn': [depends], + }, } for prop in ['proxyHeader', 'quicOverride']: @@ -265,18 +292,19 @@ def generate_config(context): """ Entry point for the deployment resources. """ properties = context.properties - name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) # Forwarding rule + target proxy + backend service = ELB - bs_resources, bs_outputs = get_backend_services(properties) - target_resources, target_outputs = get_target_proxy(properties, name) + bs_resources, bs_outputs = get_backend_services(properties, context.env['name'], project_id) + target_resources, target_outputs = get_target_proxy(properties, context.env['name'], project_id, bs_resources) rule_resources, rule_outputs = get_forwarding_rule( properties, target_resources[0], - name + context.env['name'], + project_id ) return { 'resources': bs_resources + target_resources + rule_resources, - 'outputs': bs_outputs + target_outputs + rule_outputs + 'outputs': bs_outputs + target_outputs + rule_outputs, } diff --git a/dm/templates/external_load_balancer/external_load_balancer.py.schema b/dm/templates/external_load_balancer/external_load_balancer.py.schema index bf2225c5106..168be7eff7a 100644 --- a/dm/templates/external_load_balancer/external_load_balancer.py.schema +++ b/dm/templates/external_load_balancer/external_load_balancer.py.schema @@ -15,6 +15,7 @@ info: title: External Load Balancer author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of an HTTP(S), SSL Proxy, or TCP Proxy external load balancer. For details, visit https://cloud.google.com/load-balancing/docs/. @@ -39,6 +40,11 @@ properties: description: | The external load balancer name. This name is assigned to the underlying forwarding rule resource. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the service. description: type: string description: | @@ -61,30 +67,194 @@ properties: enum: - IPV4 - IPV6 + urlMap: + type: object + additionalProperties: false + properties: + name: + type: string + description: | + Must comply with RFC1035. Specifically, the name must be 1-63 characters long and match + the regular expression [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a lowercase letter, + and all following characters must be a dash, lowercase letter, or digit, except the last character, + which cannot be a dash. + ELB name would be used if omitted. + description: + type: string + description: The resource description (optional). + defaultService: + type: string + description: | + The full or partial URL of the defaultService resource to which traffic is directed if none of the + hostRules match. If defaultRouteAction is additionally specified, advanced routing actions like URL Rewrites, + etc. take effect prior to sending the request to the backend. However, if defaultService is specified, + defaultRouteAction cannot contain any weightedBackendServices. Conversely, if routeAction specifies any + weightedBackendServices, service must not be specified. + + Only one of defaultService, defaultUrlRedirect or defaultRouteAction.weightedBackendService must be set. + + Authorization requires one or more of the following Google IAM permissions on the specified resource defaultService: + - compute.backendBuckets.use + - compute.backendServices.use + hostRules: + type: array + uniqItems: true + description: | + The list of HostRules to use against the URL. + items: + type: object + additionalProperties: false + properties: + description: + type: string + description: | + The resource description (optional). + hosts: + type: array + description: | + The list of host patterns to match. They must be valid hostnames, except * will match any string of + ([a-z0-9-.]*). In that case, * must be the first character and must be followed + in the pattern by either - or .. + items: + type: string + pathMatcher: + type: string + description: | + The name of the PathMatcher to use for matching the path portion of + the URL if the hostRule matches the URL's host portion. + pathMatchers: + type: array + uniqItems: true + description: | + The list of the named PathMatchers to use against the URL. + items: + type: object + additionalProperties: false + properties: + name: + type: string + description: | + The name to which the PathMatcher is referred by the HostRule. + description: + type: string + description: | + The resource description (optional). + defaultService: + type: string + description: | + The full or partial URL to the BackendService resource. This will be used if none of the pathRules or + routeRules defined by this PathMatcher are matched. For example, the following are + all valid URLs to a BackendService resource: + - https://www.googleapis.com/compute/v1/projects/project/global/backendServices/backendService + - compute/v1/projects/project/global/backendServices/backendService + - global/backendServices/backendService + + If defaultRouteAction is additionally specified, advanced routing actions like URL Rewrites, etc. take + effect prior to sending the request to the backend. However, if defaultService is specified, + defaultRouteAction cannot contain any weightedBackendServices. + Conversely, if defaultRouteAction specifies any weightedBackendServices, defaultService must not be specified. + Only one of defaultService, defaultUrlRedirect or defaultRouteAction.weightedBackendService must be set. + + Authorization requires one or more of the following Google IAM permissions on the specified resource defaultService: + - compute.backendBuckets.use + - compute.backendServices.use + + Authorization requires one or more of the following Google IAM permissions on the specified resource defaultService: + - compute.backendBuckets.use + - compute.backendServices.use + pathRules: + type: array + uniqItems: true + description: | + The list of path rules. + items: + type: object + additionalProperties: false + properties: + service: + type: string + description: | + The full or partial URL of the backend service resource to which traffic is directed if this + rule is matched. If routeAction is additionally specified, advanced routing actions like + URL Rewrites, etc. take effect prior to sending the request to the backend. However, if service + is specified, routeAction cannot contain any weightedBackendService s. Conversely, if routeAction + specifies any weightedBackendServices, service must not be specified. + + Only one of urlRedirect, service or routeAction.weightedBackendService must be set. + + Authorization requires one or more of the following Google IAM permissions on the specified resource service: + - compute.backendBuckets.use + - compute.backendServices.use + paths: + type: array + uniqItems: true + description: | + The list of the path patterns to match. Each pattern must + start with /. Asterisks (*) are allowed only at the end, + following the /. The string fed to the path matcher does not + include any text after the first ? or #, and those characters + are not allowed here. + items: + type: string + tests: + type: array + uniqItems: true + description: | + The list of the expected URL mapping tests. Request to update this UrlMap + succeed only if all of the test cases pass. You can specify a maximum of + 100 tests per UrlMap. + items: + type: object + additionalProperties: false + properties: + description: + type: string + description: | + The test case description. + host: + type: string + description: | + The host portion of the URL. + path: + type: string + description: | + The path portion of the URL. + service: + type: string + description: | + The BackendService resource the given URL is expected to be mapped + to. backendServices: type: array + uniqItems: true description: | Backend services to create. These services will deliver traffic to the instance groups. items: type: object + additionalProperties: false required: - backends - healthCheck properties: + resourceName: + type: string + description: Overrides resource name. name: type: string - description: The backend service name. + description: The backend service name. Resource name is used if omitted. description: type: string description: The resource description (optional). backends: type: array + uniqItems: true description: | The list of backends (instance groups) to which the backend service distributes traffic. items: type: object + additionalProperties: false required: - group properties: @@ -186,6 +356,7 @@ properties: last only until the end of the browser session (or equivalent). connectionDraining: type: object + additionalProperties: false description: The connection draining settings. properties: drainingTimeoutSec: @@ -196,10 +367,12 @@ properties: accepted earlier). cdnPolicy: type: object + additionalProperties: false description: The cloud CDN configuration for the backend service. properties: cacheKeyPolicy: type: object + additionalProperties: false description: The CacheKeyPolicy for the CdnPolicy. properties: includeProtocol: @@ -221,6 +394,7 @@ properties: cache key entirely. queryStringWhitelist: type: array + uniqItems: true description: | The names of the query string parameters to include in cache keys. All other parameters are excluded. Specify @@ -231,6 +405,7 @@ properties: type: string queryStringBlacklist: type: array + uniqItems: true description: | The names of the query string parameters to exclude from the cache keys. All other parameters are included. Specify @@ -279,6 +454,7 @@ properties: - PROXY_V1 ssl: type: object + additionalProperties: false description: | Encryption settings for connections processed by the resource. required: @@ -286,6 +462,7 @@ properties: properties: certificate: type: object + additionalProperties: false description: SSL certificate settings. oneOf: - required: diff --git a/dm/templates/external_load_balancer/tests/integration/external_load_balancer.bats b/dm/templates/external_load_balancer/tests/integration/external_load_balancer.bats index bfc00e09bcf..db060da3b00 100755 --- a/dm/templates/external_load_balancer/tests/integration/external_load_balancer.bats +++ b/dm/templates/external_load_balancer/tests/integration/external_load_balancer.bats @@ -91,6 +91,7 @@ function teardown() { run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --config ${CONFIG} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "$output" [[ "$status" -eq 0 ]] } diff --git a/dm/templates/external_load_balancer/tests/integration/external_load_balancer.yaml b/dm/templates/external_load_balancer/tests/integration/external_load_balancer.yaml index 3179086708e..bb8a093d582 100644 --- a/dm/templates/external_load_balancer/tests/integration/external_load_balancer.yaml +++ b/dm/templates/external_load_balancer/tests/integration/external_load_balancer.yaml @@ -27,7 +27,7 @@ resources: description: ${HTTP_DESCRIPTION} portRange: ${HTTP_PORT_RANGE} backendServices: - - name: ${HTTP_FIRST_BACKEND_NAME} + - resourceName: ${HTTP_FIRST_BACKEND_NAME} description: ${HTTP_FIRST_BACKEND_DESC} timeoutSec: ${TIMEOUT_SEC} sessionAffinity: ${SESSION_AFFINITY} @@ -39,7 +39,7 @@ resources: healthCheck: $(ref.${HTTP_HEALTHCHECK_NAME}.selfLink) backends: - group: $(ref.${HTTP_IGM_NAME}.instanceGroup) - - name: ${HTTP_SECOND_BACKEND_NAME} + - resourceName: ${HTTP_SECOND_BACKEND_NAME} healthCheck: $(ref.${HTTP_HEALTHCHECK_NAME}.selfLink) backends: - group: $(ref.${HTTP_IGM_NAME}.instanceGroup) @@ -59,7 +59,7 @@ resources: portRange: ${HTTPS_PORT_RANGE} quicOverride: ${QUIC_OVERRIDE} backendServices: - - name: ${HTTPS_FIRST_BACKEND_NAME} + - resourceName: ${HTTPS_FIRST_BACKEND_NAME} healthCheck: $(ref.${HTTPS_HEALTHCHECK_NAME}.selfLink) portName: ${HTTPS_PORT_NAME} backends: diff --git a/dm/templates/firewall/firewall.py b/dm/templates/firewall/firewall.py index 47f0afff5bb..3143e34a128 100644 --- a/dm/templates/firewall/firewall.py +++ b/dm/templates/firewall/firewall.py @@ -13,48 +13,43 @@ # limitations under the License. """ This template creates firewall rules for a network. """ - -def get_network(properties): - """ Gets a network name. """ - - network_name = properties.get('network') - if network_name: - is_self_link = '/' in network_name or '.' in network_name - - if is_self_link: - network_url = network_name - else: - network_url = 'global/networks/{}'.format(network_name) - - return network_url +from hashlib import sha1 def generate_config(context): """ Entry point for the deployment resources. """ - network = context.properties.get('network') + properties = context.properties + project_id = properties.get('project', context.env['project']) + network = properties.get('network') + if network: + if not ('/' in network or '.' in network): + network = 'global/networks/{}'.format(network) + else: + network = 'projects/{}/global/networks/{}'.format( + project_id, + properties.get('networkName', 'default') + ) resources = [] out = {} - for i, rule in enumerate(context.properties['rules'], 1000): - # Use VPC if specified in the properties. Otherwise, specify - # the network URL in the config. If the network is not specified in - # the config, the API defaults to 'global/networks/default'. - if network and not rule.get('network'): - rule['network'] = get_network(context.properties) + for i, rule in enumerate(properties['rules'], 1000): + res_name = sha1(rule['name']).hexdigest()[:10] + rule['network'] = network rule['priority'] = rule.get('priority', i) + rule['project'] = project_id resources.append( { - 'name': rule['name'], - 'type': 'compute.beta.firewall', + 'name': res_name, + 'type': 'gcp-types/compute-v1:firewalls', 'properties': rule } ) - out[rule['name']] = { - 'selfLink': '$(ref.' + rule['name'] + '.selfLink)', - 'creationTimestamp': '$(ref.' + rule['name'] + out[res_name] = { + 'selfLink': '$(ref.' + res_name + '.selfLink)', + 'creationTimestamp': '$(ref.' + res_name + '.creationTimestamp)', } diff --git a/dm/templates/firewall/firewall.py.schema b/dm/templates/firewall/firewall.py.schema index b16856dac65..1ff4636872f 100644 --- a/dm/templates/firewall/firewall.py.schema +++ b/dm/templates/firewall/firewall.py.schema @@ -15,56 +15,243 @@ info: title: Firewall author: Sourced Group Inc. - description: Deploys firewall rules + version: 1.0.0 + description: | + Deploys firewall rules. + + For more information on this resource: + https://cloud.google.com/vpc/docs/firewalls + + APIs endpoints used by this template: + - gcp-types/compute-v1:firewalls => + https://cloud.google.com/compute/docs/reference/rest/v1/firewalls additionalProperties: false required: - rules +allOf: + - oneOf: + - required: + - project + - required: + - network + - allOf: + - not: + required: + - project + - not: + required: + - network + - oneOf: + - required: + - networkName + - required: + - network + - allOf: + - not: + required: + - networkName + - not: + required: + - network + properties: + project: + type: string + description: | + The project ID of the project containing firewall rules. network: type: string description: | - The network name. Defaults to 'global/networks/default'. + URL of the network resource for this firewall rule. If not specified when creating a firewall rule, + the default network is used. + networkName: + type: string + description: | + The name of network to create firewalls in. rules: type: array + uniqueItems: True description: | - An array of firewall rules as defined in the documentation: - https://cloud.google.com/compute/docs/reference/rest/beta/firewalls. - - If the 'priority' field value is set in a rule, that value is used "as is". - If the 'priority' field value is not set in the rule, the template sets - the priority to the same value as the rule's index in the array +1000. - For example, the priority for the first rule in the array becomes '1000', - for the second rule '1001', and so on. If the 'priority' field is not set in - any of the rules in the array, the ruleset is sorted by priority automatically. - We strongly advise being consistent in your use of the 'priority' field: - either provide or skip values in all instances throughout the ruleset. + An array of firewall rules. + items: + type: object + additionalProperties: false + properties: + name: + type: string + description: | + Name of the resource; provided by the client when the resource is created. The name must be 1-63 + characters long, and comply with RFC1035. Specifically, the name must be 1-63 characters long and match + the regular expression `a-z?. The first character must be a lowercase letter, and all following + characters (except for the last character) must be a dash, lowercase letter, or digit. The last character + must be a lowercase letter or digit. + Resource name would be used if omitted. + description: + type: string + description: | + An optional description of this resource. Provide this field when you create the resource. + priority: + type: integer + description: | + Priority for this rule. This is an integer between 0 and 65535, both inclusive. + Relative priorities determine which rule takes effect if multiple rules apply. Lower values indicate + higher priority. For example, a rule with priority 0 has higher precedence than a rule with priority 1. + DENY rules take precedence over ALLOW rules if they have equal priority. Note that VPC networks have + implied rules with a priority of 65535. To avoid conflicts with the implied rules, use a priority + number less than 65535. - Example: - - name: allow-proxy-from-inside - allowed: - - IPProtocol: tcp - ports: - - "80" - - "443" - description: This rule allows connectivity to HTTP proxies. - direction: INGRESS - sourceRanges: - - 10.0.0.0/8 - - name: allow-dns-from-inside - allowed: - - IPProtocol: udp + If the 'priority' field value is not set in the rule, the template sets + the priority to the same value as the rule's index in the array +1000. + For example, the priority for the first rule in the array becomes '1000', + for the second rule '1001', and so on. If the 'priority' field is not set in + any of the rules in the array, the ruleset is sorted by priority automatically. + We strongly advise being consistent in your use of the 'priority' field: + either provide or skip values in all instances throughout the ruleset. + sourceRanges: + type: array + uniqueItems: True + description: | + If source ranges are specified, the firewall rule applies only to traffic that has a source IP address in + these ranges. These ranges must be expressed in CIDR format. One or both of sourceRanges and sourceTags + may be set. If both fields are set, the rule applies to traffic that has a source IP address within + sourceRanges OR a source IP from a resource with a matching tag listed in the sourceTags field. + The connection does not need to match both fields for the rule to apply. Only IPv4 is supported. + items: + type: string + destinationRanges: + type: array + uniqueItems: True + description: | + If destination ranges are specified, the firewall rule applies only to traffic that has destination IP + address in these ranges. These ranges must be expressed in CIDR format. Only IPv4 is supported. + items: + type: string + sourceTags: + type: array + uniqueItems: True + description: | + If source tags are specified, the firewall rule applies only to traffic with source IPs that match the + primary network interfaces of VM instances that have the tag and are in the same VPC network. Source tags + cannot be used to control traffic to an instance's external IP address, it only applies to traffic between + instances in the same virtual network. Because tags are associated with instances, not IP addresses. + One or both of sourceRanges and sourceTags may be set. If both fields are set, the firewall applies to + traffic that has a source IP address within sourceRanges OR a source IP from a resource with a matching + tag listed in the sourceTags field. The connection does not need to match both fields + for the firewall to apply. + items: + type: string + targetTags: + type: array + uniqueItems: True + description: | + A list of tags that controls which instances the firewall rule applies to. If targetTags are specified, + then the firewall rule applies only to instances in the VPC network that have one of those tags. + If no targetTags are specified, the firewall rule applies to all instances on the specified network. + items: + type: string + sourceServiceAccounts: + type: array + uniqueItems: True + description: | + If source service accounts are specified, the firewall rules apply only to traffic originating from + an instance with a service account in this list. Source service accounts cannot be used to control + traffic to an instance's external IP address because service accounts are associated with an instance, + not an IP address. sourceRanges can be set at the same time as sourceServiceAccounts. If both are set, + the firewall applies to traffic that has a source IP address within the sourceRanges OR a source IP + that belongs to an instance with service account listed in sourceServiceAccount. The connection does + not need to match both fields for the firewall to apply. sourceServiceAccounts cannot be used + at the same time as sourceTags or targetTags. + items: + type: string + allowed: + type: array + uniqueItems: True + description: | + The list of ALLOW rules specified by this firewall. Each rule specifies a protocol and + port-range tuple that describes a permitted connection. + items: + type: object + additionalProperties: false + required: + - IPProtocol + properties: + IPProtocol: + type: string + description: | + The IP protocol to which this rule applies. The protocol type is required when creating + a firewall rule. This value can either be one of the following well known protocol strings + (tcp, udp, icmp, esp, ah, ipip, sctp) or the IP protocol number. ports: - - "53" - - IPProtocol: tcp + type: array + uniqueItems: True + description: | + An optional list of ports to which this rule applies. This field is only applicable for + the UDP or TCP protocol. Each entry must be either an integer or a range. + If not specified, this rule applies to connections through any port. + + Example inputs include: ["22"], ["80","443"], and ["12345-12349"]. + items: + type: string + denied: + type: array + uniqueItems: True + description: | + The list of DENY rules specified by this firewall. Each rule specifies a protocol and port-range + tuple that describes a denied connection. + items: + type: object + additionalProperties: false + required: + - IPProtocol + properties: + IPProtocol: + type: string + description: | + The IP protocol to which this rule applies. The protocol type is required when creating + a firewall rule. This value can either be one of the following well known protocol strings + (tcp, udp, icmp, esp, ah, ipip, sctp) or the IP protocol number. ports: - - "53" - description: This rule allows DNS queries to Google's 8.8.8.8 - direction: EGRESS - destinationRanges: - - 8.8.8.8/32 + type: array + uniqueItems: True + description: | + An optional list of ports to which this rule applies. This field is only applicable for + the UDP or TCP protocol. Each entry must be either an integer or a range. + If not specified, this rule applies to connections through any port. + + Example inputs include: ["22"], ["80","443"], and ["12345-12349"]. + items: + type: string + direction: + type: string + description: | + Direction of traffic to which this firewall applies, either INGRESS or EGRESS. + The default is INGRESS. For INGRESS traffic, you cannot specify the destinationRanges field, + and for EGRESS traffic, you cannot specify the sourceRanges or sourceTags fields. + enum: + - INGRESS + - EGRESS + logConfig: + type: object + description: | + This field denotes the logging options for a particular firewall rule. + If logging is enabled, logs will be exported to Stackdriver. + required: + - enable + properties: + enable: + type: boolean + description: | + This field denotes whether to enable logging for a particular firewall rule. + disabled: + type: boolean + description: | + Denotes whether the firewall rule is disabled. When set to true, the firewall rule is not + enforced and the network behaves as if it did not exist. If this is unspecified, the firewall rule will be enabled. + + outputs: properties: diff --git a/dm/templates/folder/folder.py b/dm/templates/folder/folder.py index bc16e2bf0c0..4a57129d9f6 100644 --- a/dm/templates/folder/folder.py +++ b/dm/templates/folder/folder.py @@ -16,6 +16,8 @@ parent folder. """ +from hashlib import sha1 + def generate_config(context): """ Entry point for the deployment resources. """ @@ -23,14 +25,20 @@ def generate_config(context): resources = [] out = {} for folder in context.properties.get('folders', []): - - create_folder = folder['name'] - - parent = folder.get('orgId', folder.get('folderId')) - + if folder.get('parent'): + parent = '{}s/{}'.format(folder['parent']['type'], folder['parent']['id']) + else: + parent = folder.get('orgId', folder.get('folderId')) + + suffix = folder.get( + 'resourceNameSuffix', + sha1('{}/folders/{}'.format(parent, folder.get('displayName'))).hexdigest()[:10] + ) + create_folder = '{}-{}'.format(context.env['name'], suffix) resources.append( { 'name': create_folder, + # https://cloud.google.com/resource-manager/reference/rest/v2/folders 'type': 'gcp-types/cloudresourcemanager-v2:folders', 'properties': { diff --git a/dm/templates/folder/folder.py.schema b/dm/templates/folder/folder.py.schema index 17b58fcae23..8b44e55bb89 100644 --- a/dm/templates/folder/folder.py.schema +++ b/dm/templates/folder/folder.py.schema @@ -14,44 +14,96 @@ info: title: Folder + author: Sourced Group Inc. + version: 1.0.0 description: | Creates a folder under an organization or under a parent folder. + For more information on this resource: + https://cloud.google.com/resource-manager/ + + APIs endpoints used by this template: + - gcp-types/cloudresourcemanager-v2:folders => + https://cloud.google.com/resource-manager/reference/rest/v2/folders + imports: - path: folder.py -additionalProperties: false - +additionalProperties: false + required: - folders properties: folders: type: array - description: List of folders to create. + description: | + List of folders to create. + minItems: 1 + uniqueItems: True items: - orgId: - type: string - pattern: ^organization\/[0-9]{8,25}$ - description: | - The organization ID. If this field is set, the folder is - created under the organization. The value must conform to the - format `organizations/`. For example, - `organizations/111122223333`. - folderId: - type: string - pattern: ^folder\/[0-9]{8,25}$ - description: | - The folder ID. If this field is set, the folder is created - under the folder specified by the ID. The value must conform - to the format `folders/`. For example, - `folders/1234567890`. - displayName: - type: string - description: The display name of the folder. - pattern: | - [\p{L}\p{N}]({\p{L}\p{N}_- ]{0,28}[\p{L}\p{N}])? + type: object + oneOf: + - required: + - orgId + - required: + - folderId + - required: + - parent + required: + - displayName + properties: + resourceNameSuffix: + type: string + description: | + Optional resource name suffix + orgId: + type: string + pattern: ^organizations\/[0-9]{8,25}$ + description: | + The organization ID. If this field is set, the folder is + created under the organization. The value must conform to the + format `organizations/`. For example, + `organizations/111122223333`. + DEPRECATED. Please use "parent" + folderId: + type: string + pattern: ^folders\/[0-9]{8,25}$ + description: | + The folder ID. If this field is set, the folder is created + under the folder specified by the ID. The value must conform + to the format `folders/`. For example, + `folders/1234567890`. + DEPRECATED. Please use "parent" + parent: + type: object + additionalProperties: false + description: The parent of the folder. + required: + - type + - id + properties: + type: + type: string + decription: The parent type (organization or folder). + enum: + - organization + - folder + default: organization + id: + type: [integer, string] + description: | + The ID of the folder's parent. + pattern: ^[0-9]{8,25}$ + displayName: + type: string + pattern: ^[a-zA-Z0-9]([a-zA-Z0-9_\- ]{0,28}[a-zA-Z0-9])?$ + description: | + The folder’s display name. A folder’s display name must be unique amongst its siblings, e.g. no two folders + with the same parent can share the same display name. The display name must start and end with + a letter or digit, may contain letters, digits, spaces, hyphens and underscores and + can be no longer than 30 characters. outputs: properties: diff --git a/dm/templates/forseti/README.md b/dm/templates/forseti/README.md deleted file mode 100644 index 1b608ea156e..00000000000 --- a/dm/templates/forseti/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# Forseti Security - -This template creates a new project (or reuses an existing one) and deploys -the [Forseti Security](https://forsetisecurity.org/) solution in it. The -solution consists of two instances (the client and the server), two service -accounts for them, a Cloud SQL instance, and the firewall rules with the IAM -policies for all of the above to function properly. See the Forseti Security -[manual](https://forsetisecurity.org/docs/v2.0/setup/manual.html) -installation page for specific configuration changes. - -## Prerequisites - -- Install [gcloud](https://cloud.google.com/sdk) -- Create a [GCP project, set up billing, enable requisite APIs](../project/README.md) -- Grant the following roles to Deployment Manager's service account at the - organization level: - - Billing Account Administrator - - Org Admin - - Storage Admin - - Cloud SQL Admin -- Enable the [Cloud SQL Admin API](https://cloud.google.com/sql/docs/mysql/admin-api/) -- When reusing an existing project, enable the following APIs: - - api-admin.googleapis.com - - api-appengine.googleapis.com - - api-bigquery-json.googleapis.com - - api-cloudbilling.googleapis.com - - api-cloudresourcemanager.googleapis.com - - api-compute.googleapis.com - - api-deploymentmanager.googleapis.com - - api-iam.googleapis.com - - api-sql-component.googleapis.com - - api-sqladmin.googleapis.com -- Create a Cloud Storage bucket containing the `forseti_conf_server.yaml` file - in the configs folder; see for [example](https://github.com/GoogleCloudPlatform/forseti-security/blob/stable/configs/server/forseti_conf_server.yaml.sample). - -## Deployment - -### Resources - -- [compute.v1.instance](https://cloud.google.com/compute/docs/reference/rest/v1/instances) -- [compute.v1.network](https://cloud.google.com/compute/docs/reference/rest/v1/networks) -- [compute.v1.firewall](https://cloud.google.com/compute/docs/reference/rest/v1/firewalls) -- [cloudresourcemanager.v1.project](https://cloud.google.com/resource-manager/reference/rest/v1/projects) -- [sqladmin.v1beta4.instance](https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/databases) -- [sqladmin.v1beta4.database](https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances) -- [iam.v1.serviceAccount](https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts) - -### Properties - -See the `properties` section in the schema file(s): - -- [Forseti](forseti.py.schema) - -### Usage - -1. Clone the [Deployment Manager Samples repository](https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit): - - ```shell - git clone https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit - ``` - -2. Go to the [dm](../../) directory: - - ```shell - cd dm - ``` - -3. Copy the example DM config to be used as a model for the deployment; in this - case, [examples/forseti.yaml](examples/forseti.yaml): - - ```shell - cp templates/forseti/examples/forseti.yaml my_forseti.yaml - ``` - -4. Change the values in the config file to match your specific GCP setup (for - properties, refer to the schema files listed above): - - ```shell - vim my_forseti.yaml # <== change values to match your GCP setup - ``` - -5. Create your deployment (replace \ with the relevant - deployment name): - - ```shell - gcloud deployment-manager deployments create \ - --config my_forseti.yaml - ``` - -6. In case you need to delete your deployment: - - ```shell - gcloud deployment-manager deployments delete - ``` - -## Examples - -- [Forseti](examples/forseti.yaml) diff --git a/dm/templates/forseti/client.py b/dm/templates/forseti/client.py deleted file mode 100644 index e4e5ce8806e..00000000000 --- a/dm/templates/forseti/client.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright 2017 The Forseti Security Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Creates a GCE client instance for Forseti Security. """ - -FORSETI_HOME = '$USER_HOME/forseti-security' - -FORSETI_CLIENT_CONFIG = '$FORSETI_HOME/forseti_conf_client.yaml' - -EXPORT_VARS = ('export FORSETI_HOME={}\n' - 'export FORSETI_CLIENT_CONFIG={}\n' - ).format(FORSETI_HOME, - FORSETI_CLIENT_CONFIG) - - -STARTUP_SCRIPT_TEMPLATE = """#!/bin/bash -exec > /tmp/deployment.log -exec 2>&1 -# Ubuntu update. -sudo apt-get update -y -sudo apt-get upgrade -y -# Forseti setup. -sudo apt-get install -y git unzip -# Forseti dependencies -sudo apt-get install -y libffi-dev libssl-dev libmysqlclient-dev python-pip python-dev build-essential -USER=ubuntu -USER_HOME=/home/ubuntu -# Install fluentd if necessary. -FLUENTD=$(ls /usr/sbin/google-fluentd) -if [ -z "$FLUENTD" ]; then - cd $USER_HOME - curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh - bash install-logging-agent.sh -fi -# Install Forseti Security. -cd $USER_HOME -rm -rf *forseti* -# Download Forseti source code -{download_forseti} -cd forseti-security -git fetch --all -{checkout_forseti_version} -# Forseti dependencies -pip install --upgrade pip==9.0.3 -pip install -q --upgrade setuptools wheel -pip install -q --upgrade -r requirements.txt -# Install Forseti -python setup.py install -# Set ownership of the forseti project to $USER -chown -R $USER {forseti_home} -# Export variables -{persist_forseti_vars} -# Store the variables in /etc/profile.d/forseti_environment.sh -# so all the users will have access to them -echo "echo '{persist_forseti_vars}' >> /etc/profile.d/forseti_environment.sh" | sudo sh -echo "server_ip: {server_ip}" > $FORSETI_CLIENT_CONFIG -chmod ugo+r $FORSETI_CLIENT_CONFIG -echo "Execution of startup script finished" -""" - -def get_full_machine_type(project_id, zone, machine_type): - """ Gets a full URL to the specified machine type. """ - - prefix = 'https://www.googleapis.com/compute/v1' - - return '{}/projects/{}/zones/{}/machineTypes/{}'.format( - prefix, - project_id, - zone, - machine_type - ) - -def generate_config(context): - """ Entry point for the deployment resources. """ - - properties = context.properties - instance_name = properties.get('name', context.env['name']) - project_id = properties.get('project', context.env['project']) - source_image = properties['sourceImage'] - source_path = properties['srcPath'] - version = properties['srcVersion'] - zone = properties['zone'] - machine_type = properties['machineType'] - machine_type_uri = get_full_machine_type(project_id, zone, machine_type) - server_ip = properties['serverIp'] - startup_script = STARTUP_SCRIPT_TEMPLATE.format( - download_forseti='git clone {}.git'.format(source_path), - checkout_forseti_version='git checkout {}'.format(version), - server_ip=server_ip, - forseti_home=FORSETI_HOME, - persist_forseti_vars=EXPORT_VARS, - ) - - resources = [ - { - 'name': instance_name, - 'type': 'gcp-types/compute-v1:instances', - 'properties': - { - 'project': - project_id, - 'zone': - zone, - 'machineType': - machine_type_uri, - 'disks': - [ - { - 'deviceName': 'boot', - 'type': 'PERSISTENT', - 'boot': True, - 'autoDelete': True, - 'initializeParams': - { - 'sourceImage': source_image, - } - } - ], - 'networkInterfaces': - [ - { - 'network': - properties['network'], - 'accessConfigs': - [ - { - 'name': 'External NAT', - 'type': 'ONE_TO_ONE_NAT' - } - ] - } - ], - 'serviceAccounts': - [ - { - 'email': properties['serviceAccountEmail'], - 'scopes': properties['serviceAccountScopes'], - } - ], - 'tags': { - 'items': properties.get('tags', - []) - }, - 'metadata': - { - 'items': - [ - { - 'key': 'startup-script', - 'value': startup_script - } - ] - } - } - } - ] - - outputs = [ - { - 'name': 'name', - 'value': '$(ref.{}.name)'.format(instance_name) - }, - { - 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(instance_name) - } - ] - - return {'resources': resources, 'outputs': outputs} diff --git a/dm/templates/forseti/client.py.schema b/dm/templates/forseti/client.py.schema deleted file mode 100644 index 80a5cfa5c17..00000000000 --- a/dm/templates/forseti/client.py.schema +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2018 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -info: - title: Forseti Security - author: Sourced Group Inc. - description: | - Supports creation of a Forseti Security client instance. - -imports: - - path: client.py - -additionalProperties: false - -required: - - project - - zone - - network - - serverIp - - serviceAccountEmail - -properties: - project: - type: string - description: | - The project ID of the project where the instance must be placed. - name: - type: string - default: forseti-client - description: The name of the Forseti client instance. - sourceImage: - type: string - default: projects/ubuntu-os-cloud/global/images/family/ubuntu-1804-lts - description: | - The source image for the disk. To create a disk with one of the - public operating system images, specify the image by its family name. - For example, specify family/debian-9 to use the latest Debian 9 image - projects/debian-cloud/global/images/family/debian-9. - To create a disk with a custom (yor own) image, specify the - image name in the following format: global/images/my-custom-image. - See https://cloud.google.com/compute/docs/images for details. - srcPath: - type: string - default: https://github.com/GoogleCloudPlatform/forseti-security - description: | - The URL of the git repository containing the Forseti source code. - srcVersion: - type: string - default: 'v2.0' - description: The version (git tag name) of the source code to use. - serverIp: - type: string - description: The Forseti server's IP address. - zone: - type: string - description: The Availability zone. E.g., 'us-central1-a'. - machineType: - type: string - default: n1-standard-1 - description: | - The Compute Instance type; e.g., 'n1-standard-1'. - See https://cloud.google.com/compute/docs/machine-types for details. - network: - type: string - description: The URL of the network. - serviceAccountEmail: - type: string - description: The email address of the service account. - serviceAccountScopes: - type: array - default: [] - description: | - The list of scopes to be made available for the service account. - items: - type: string - description: | - The access scope, e.g., 'https://www.googleapis.com/auth/compute.readonly'. - See https://cloud.google.com/compute/docs/access/service-accounts#accesscopesiam - for more details. - tags: - type: array - default: [] - description: Network tags for the instance. - -outputs: - properties: - - name: - type: string - description: The name of the Forseti client instance. - - selfLink: - type: string - description: The URI (SelfLink) of the Forseti client resource. diff --git a/dm/templates/forseti/examples/forseti.yaml b/dm/templates/forseti/examples/forseti.yaml deleted file mode 100644 index 4a665aeffbb..00000000000 --- a/dm/templates/forseti/examples/forseti.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Example of the Forseti template usage. -# -# Replace the following placeholders with valid values: -# : an organization ID -# : a zone where the server instance must be created -# : a zone where the client instance must be created -# : an ID of the project to create -# : a type of the project's parent (folder or org) -# : a project parent's ID -# : a billing account ID -# : a name of the bucket that contains the Forseti server -# configuration -# : a region where the SQL Instance is created -# : a name of the SQL Instance to create - -imports: - - path: templates/forseti/forseti.py - name: forseti.py - -resources: - - name: forseti - type: forseti.py - properties: - organizationId: - server: - zone: - client: - zone: - project: - create: true - id: - parent: - type: - id: - billingAccountId: - bucket: - name: - cloudSql: - region: - instanceName: diff --git a/dm/templates/forseti/forseti.py b/dm/templates/forseti/forseti.py deleted file mode 100644 index 78bd4f5ed58..00000000000 --- a/dm/templates/forseti/forseti.py +++ /dev/null @@ -1,808 +0,0 @@ -# Copyright 2018 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" This template creates Forseti Security tools and resources. """ - -import collections -import random -import string -import copy - -# The helper tuple for handling resources and their outputs. -DMResource = collections.namedtuple( - 'DMResource', - 'self_link resources outputs' -) - -SUFFIX_LENGTH = 10 -CHAR_CHOICE = string.digits + string.ascii_lowercase -FORSETI_APIS = [ - 'admin.googleapis.com', - 'appengine.googleapis.com', - 'bigquery-json.googleapis.com', - 'cloudbilling.googleapis.com', - 'iam.googleapis.com', - 'cloudresourcemanager.googleapis.com', - 'sqladmin.googleapis.com', - 'sql-component.googleapis.com', - 'compute.googleapis.com', - 'deploymentmanager.googleapis.com' -] -PROJECT_REMOVE_SA = True -PROJECT_REMOVE_VPC = True -CLOUD_MAN = 'gcp-types/cloudresourcemanager-v1:cloudresourcemanager' -IAM = 'gcp-types/iam-v1:iam.projects' -STORAGE = 'gcp-types/storage-v1:storage' - -# If True, the organization and project policies that were previously added for -# the Forseti service accounts are removed on deployment deletion. -# However, due to the nature of the IAM patches, this step fails if the -# underlying policy has changed after the deployment creation. - -CLEANUP_POLICY_ON_DELETE = True - -def get_random_string(length): - """ Generates a random string of a given length. """ - - return ''.join([random.choice(CHAR_CHOICE) for _ in range(length)]) - -def generate_project_id(prefix): - """ Generates a new project ID. """ - - return prefix + '-' + get_random_string(SUFFIX_LENGTH) - -def create_forseti_project(deployment_name, properties): - """ Generates a new project for the Forseti tools. """ - - project_id = properties.get('id', generate_project_id(deployment_name)) - project_name = properties.get('name', project_id) - - project = { - 'type': 'project.py', - 'name': project_id, - 'properties': { - 'name': project_name, - 'parent': copy.deepcopy(properties['parent']), - 'billingAccountId': properties['billingAccountId'], - 'activateApis': FORSETI_APIS, - 'removeDefaultSA': PROJECT_REMOVE_SA, - 'removeDefaultVPC': PROJECT_REMOVE_VPC, - 'serviceAccounts': [], - 'groups': [] - } - } - - return DMResource( - self_link=get_ref(project_id, 'projectId'), - resources=[project], - outputs=[ - { - 'name': 'projectId', - 'value': project_id - }, - { - 'name': 'resources', - 'value': get_ref(project_id, 'resources') - } - ] - ) - -def wait_for_init_complete(project, *deps): - """ Adds explicit dependsOn metadata to the project dependencies. """ - - resources_output = find_output_value('resources', project.outputs) - for dependency in deps: - for resource in dependency.resources: - if 'type' in resource or 'getIamPolicy' in resource['action']: - resource['metadata'] = {'dependsOn': resources_output} - -def get_forseti_project(deployment_name, properties): - """ Gets a reference to a project for the Forseti resources. """ - - create = properties['create'] - - if create: - return create_forseti_project(deployment_name, properties) - - return DMResource(self_link=properties['id'], resources=[], outputs=[]) - -def create_policy_bindings(member, roles): - """ Converts member+roles args to a proper policy bindings object. """ - - bindings = [] - - for role in roles: - bindings.append({'role': role, 'members': [member]}) - - return bindings - -def get_action_path(res_type): - """ - Gets a proper type provider path for assigning the IAM policy for a given - resource type. - """ - - if res_type == 'serviceAccount': - type_provider = IAM - elif res_type == 'bucket': - type_provider = STORAGE - else: - type_provider = CLOUD_MAN - - return '{}.{}s'.format(type_provider, res_type) - -def set_member_roles(member, roles, res_type, res_id, project_id): - """ Sets the IAM policy of a given resource. """ - - random_suffix = get_random_string(SUFFIX_LENGTH) - - bindings = create_policy_bindings(member, roles) - - action_path = get_action_path(res_type) - - if res_type == 'bucket': - properties = { - 'bucket': res_id, - 'project': project_id, - 'bindings': bindings - } - else: - properties = { - 'resource': res_id, - 'policy': {'bindings': bindings} - } - - resources = [ - { - 'name': 'set-iam-policy-' + random_suffix, - 'action': '{}.setIamPolicy'.format(action_path), - 'properties': properties - }, - ] - - return DMResource(None, resources, []) - -def patch_member_roles(member, roles, res_type, res_id): - """ Patches the IAM policy of a given resource. """ - - random_suffix = get_random_string(SUFFIX_LENGTH) - get_iam_policy_name = 'get-iam-policy-' + random_suffix - set_iam_policy_name = 'set-iam-policy-' + random_suffix - rem_iam_policy_name = 'rem-iam-policy-' + random_suffix - - bindings = create_policy_bindings(member, roles) - - action_path = get_action_path(res_type) - - resources = [ - { - # Get the existing policy. - 'name': get_iam_policy_name, - 'action': '{}.getIamPolicy'.format(action_path), - 'properties': { - 'resource': res_id - } - }, - { - # Patch the existing policy. - 'name': set_iam_policy_name, - 'action': '{}.setIamPolicy'.format(action_path), - 'properties': { - 'resource': res_id, - 'policy': '$(ref.{})'.format(get_iam_policy_name), - 'gcpIamPolicyPatch': {'add': bindings} - } - }, - ] - - if CLEANUP_POLICY_ON_DELETE: - resources.append({ - 'name': rem_iam_policy_name, - 'action': '{}.setIamPolicy'.format(action_path), - 'metadata': {'runtimePolicy': ['DELETE']}, - 'properties': - { - 'resource': res_id, - 'policy': '$(ref.' + set_iam_policy_name + ')', - 'gcpIamPolicyPatch': {'remove': copy.deepcopy(bindings)} - } - }) - - return DMResource(None, resources, []) - -def get_service_account( - default_id, - properties, - project_roles, - project_id, - org_roles, - org_id, - sa_roles -): - """ Creates a new service account. """ - - account_id = properties.get('accountId', default_id) - display_name = properties.get('displayName', account_id) - sa_res_name = account_id - sa_res = { - 'name': sa_res_name, - 'type': 'iam.v1.serviceAccount', - 'properties': - { - 'accountId': account_id, - 'displayName': display_name, - 'projectId': project_id - } - } - - self_link = '$(ref.{}.email)'.format(sa_res_name) - sa_name = 'serviceAccount:{}'.format(self_link) - - sa_bundle = DMResource(self_link, [sa_res], []) - - if project_roles: - project_policy = patch_member_roles( - sa_name, - project_roles, - 'project', - project_id - ) - sa_bundle = merge_dm_resources(sa_bundle, project_policy) - - if org_roles: - org_policy = patch_member_roles( - sa_name, - org_roles, - 'organization', - 'organizations/{}'.format(org_id) - ) - sa_bundle = merge_dm_resources(sa_bundle, org_policy) - - if sa_roles: - sa_policy = set_member_roles( - sa_name, - sa_roles, - 'serviceAccount', - 'projects/{}/serviceAccounts/{}'.format(project_id, self_link), - project_id - ) - sa_bundle = merge_dm_resources(sa_bundle, sa_policy) - - return sa_bundle - -def get_client_service_account(project_id, properties): - """ Creates a new service account for the client instance. """ - - client_sa_settings = properties.get('serviceAccount', {}) - default_sa_id = 'forseti-client-gcp-' + get_random_string(SUFFIX_LENGTH) - project_roles = [ - 'roles/logging.logWriter', - 'roles/storage.objectViewer' - ] - - return get_service_account( - default_sa_id, - client_sa_settings, - project_roles, - project_id, - None, - None, - None - ) - -def merge_dm_resources(first, *argv): - """ Merges an arbitrary number of DM resources into one. """ - - if argv: - merged_resource = merge_dm_resources(argv[0], *argv[1:]) - return DMResource( - first.self_link, - first.resources + merged_resource.resources, - first.outputs + merged_resource.outputs - ) - - return first - -def get_server_service_account(project_id, properties, org_id): - """ Creates a new service account for the server instance. """ - - server_sa_settings = properties.get('serviceAccount', {}) - default_sa_id = 'forseti-server-gcp-' + get_random_string(SUFFIX_LENGTH) - project_roles = [ - 'roles/cloudsql.client', - 'roles/logging.logWriter', - 'roles/storage.objectViewer', - 'roles/storage.objectCreator' - ] - org_roles = [ - 'roles/appengine.appViewer', - 'roles/bigquery.dataViewer', - 'roles/browser', - 'roles/cloudasset.viewer', - 'roles/cloudsql.viewer', - 'roles/compute.networkViewer', - 'roles/compute.securityAdmin', - 'roles/iam.securityReviewer', - 'roles/servicemanagement.quotaViewer', - 'roles/serviceusage.serviceUsageConsumer' - ] - sa_roles = [ - 'roles/iam.serviceAccountTokenCreator' - ] - sa_bundle = get_service_account( - default_sa_id, - server_sa_settings, - project_roles, - project_id, - org_roles, - org_id, - sa_roles - ) - - return sa_bundle - -def keep_first(collection): - """ Removes from the collection all elements except for the first one. """ - - while len(collection) > 1: - collection.pop(1) - -def squash_patch_policies(set_policies, policy_ref): - """ - Optimizes the policy assignments, so that we can achieve the same - result with a smaller number of steps. - """ - - if set_policies: - first_set_policy = set_policies[0] - source_bindings = first_set_policy['properties']['gcpIamPolicyPatch'] - # Merge the remaining set- policies into the first one. - for other in set_policies[1:]: - other_bindings = other['properties']['gcpIamPolicyPatch'] - for action in other_bindings: - if not action in source_bindings: - source_bindings[action] = other_bindings[action] - else: - source_bindings[action] += other_bindings[action] - - # Update the reference to the get- policy. - first_set_policy['properties']['policy'] = policy_ref - return DMResource(None, [first_set_policy], []) - - return DMResource(None, [], []) - -def group_iam_policies_by_targets(policies): - """ Groups the collection of IAM actions by target. """ - - policies_by_targets = {} - for policy in policies: - target = policy['properties']['resource'] - if not target in policies_by_targets: - policies_by_targets[target] = [] - policies_by_targets[target].append(policy) - - return policies_by_targets - -def is_get_policy(policy): - """ Checks if the current policy action is a get action. """ - - return 'getIamPolicy' in policy['action'] - -def is_set_policy(policy): - """ Checks if the current policy action is a set action. """ - - return 'setIamPolicy' in policy['action'] and not 'metadata' in policy - -def is_rem_policy(policy): - """ Checks if the current policy action is a patch-on-delete action. """ - - return 'setIamPolicy' in policy['action'] and 'metadata' in policy - -def optimize_policies_creation(first_sa_bundle, second_sa_bundle): - """ - Reorganizes the policy patches so they don't affect the same resource - more than once, thus avoiding the race condition. - """ - - all_resources = first_sa_bundle.resources + second_sa_bundle.resources - policies = [policy for policy in all_resources if 'action' in policy] - - # Group policies by target (organization, project). - policies_by_targets = group_iam_policies_by_targets(policies) - - # A placeholder for the result. - extracted_policies = DMResource(self_link=None, resources=[], outputs=[]) - - # Leave only one get- and one set- IamPolicy (add, remove) for each target. - for _, target_policies in policies_by_targets.items(): - get_policy = next( - (p for p in target_policies if is_get_policy(p)), - None - ) - if get_policy: - extracted_policies.resources.append(get_policy) - - set_policies = [p for p in target_policies if is_set_policy(p)] - set_policy_bundle = squash_patch_policies( - set_policies, - '$(ref.{})'.format(get_policy['name']) - ) - extracted_policies = merge_dm_resources( - extracted_policies, - set_policy_bundle - ) - - rem_policies = [p for p in target_policies if is_rem_policy(p)] - - rem_policy_bundle = squash_patch_policies( - rem_policies, - '$(ref.{})'.format(set_policy_bundle.resources[0]['name']) - ) - extracted_policies = merge_dm_resources( - extracted_policies, - rem_policy_bundle - ) - else: - extracted_policies.resources.extend(target_policies) - - keep_first(first_sa_bundle.resources) - keep_first(second_sa_bundle.resources) - - return extracted_policies - -def get_ref(res_name, prop='selfLink'): - """ Gets a Deployment Manager reference link. """ - - return '$(ref.{}.{})'.format(res_name, prop) - -def get_server_bucket(properties, project_id, server_sa_email): - """ Configures and gets a link to the server configuration bucket. """ - - name = properties['name'] - - bucket = DMResource(name, [], []) - - roles = set_member_roles( - 'serviceAccount:' + server_sa_email, - ['roles/storage.objectAdmin'], - 'bucket', - name, - project_id - ) - - return merge_dm_resources(bucket, roles) - -def find_output_value(name, outputs): - """ Finds a specific output within a collection. """ - - return next( - output['value'] for output in outputs if output['name'] == name - ) - -def get_cloud_sql(properties, project_id): - """ Creates a Cloud SQL instance with a database. """ - - instance_name = properties.get('instanceName', 'forseti-sql-' + project_id) - # Add random suffix when creating/recreating instances. - # GCP keeps the names for up to a week. - # instance_name += '-' + get_random_string(SUFFIX_LENGTH) - - self_link = '$(ref.{}.name)'.format(instance_name) - - sql = { - 'name': instance_name, - 'type': 'gcp-types/sqladmin-v1beta4:instances', - 'properties': { - 'name': instance_name, - 'project': project_id, - 'backendType': 'SECOND_GEN', - 'databaseVersion': 'MYSQL_5_7', - 'region': properties['region'], - 'settings': { - 'tier': 'db-n1-standard-1', - 'backupConfiguration': { - 'enabled': True, - 'binaryLogEnabled': True - }, - 'activationPolicy': 'ALWAYS', - 'ipConfiguration': { - 'ipv4Enabled': True, - 'authorizedNetworks': [], - 'requireSsl': True - }, - 'dataDiskSizeGb': '25', - 'dataDiskType': 'PD_SSD', - }, - 'instanceType': 'CLOUD_SQL_INSTANCE', - } - } - - db_name = instance_name + '-db' - database = { - 'name': db_name, - 'type': 'gcp-types/sqladmin-v1beta4:databases', - 'properties': - { - 'name': db_name, - 'project': project_id, - 'instance': self_link - } - } - - return DMResource(self_link, [sql, database], [ - { - 'name': 'connectionName', - 'value': get_ref(instance_name, 'connectionName') - }, - { - 'name': 'databaseName', - 'value': '$(ref.{}.name)'.format(db_name) - }, - ]) - -def get_firewall_rule(name, properties, project_id, network): - """ Creates a firewall rule. """ - - resource = { - 'name': name, - 'type': 'gcp-types/compute-v1:firewalls', - 'properties': copy.deepcopy(properties) - } - - resource['properties']['project'] = project_id - resource['properties']['network'] = network - - return DMResource(get_ref(name), [resource], []) - -def get_firewall_rules(prefix, project_id, network): - """ Creates firewall rules required by the Forseti network. """ - - icmp_desc = { - 'sourceRanges': ['0.0.0.0/0'], - 'allowed': [{'IPProtocol': 'icmp'}] - } - icmp_rule = get_firewall_rule( - prefix + '-allow-icmp', - icmp_desc, - project_id, - network - ) - - ssh_desc = { - 'sourceRanges': ['0.0.0.0/0'], - 'allowed': [{'IPProtocol': 'tcp', 'ports': [22]}] - } - ssh_rule = get_firewall_rule( - prefix + '-allow-ssh', - ssh_desc, - project_id, - network - ) - - client_to_server_desc = { - 'sourceTags': ['forseti-client'], - 'targetTags': ['forseti-server'], - 'allowed': [{'IPProtocol': 'all'}] - } - - cts_name = prefix + '-allow-client-to-server' - cts_bundle = get_firewall_rule( - cts_name, - client_to_server_desc, - project_id, - network - ) - - return merge_dm_resources(icmp_rule, ssh_rule, cts_bundle) - -def get_network(project_id): - """ Creates a Forseti VPC. """ - - name = 'forseti-network' - - network = { - 'name': name, - 'type': 'gcp-types/compute-v1:networks', - 'properties': { - 'name': name, - 'project': project_id, - 'autoCreateSubnetworks': True - } - } - - self_link = get_ref(name) - - firewall_bundle = get_firewall_rules(name, project_id, self_link) - - network_bundle = DMResource(self_link, [network], [ - { - 'name': 'networkName', - 'value': get_ref(name, 'name') - }, - { - 'name': 'networkSelfLink', - 'value': self_link - }, - ]) - - return merge_dm_resources(network_bundle, firewall_bundle) - -def get_server(properties, project, network, sql, service_account, bucket): - """ Creates a Forseti server instance. """ - - name = properties['name'] - - instance_properties = copy.deepcopy(properties) - instance = { - 'name': name, - 'type': 'server.py', - 'properties': instance_properties - } - - if 'serviceAccount' in instance_properties: - del instance_properties['serviceAccount'] - - instance_properties['name'] = name - instance_properties['tags'] = ['forseti-server'] - - for sql_prop in ['databaseName', 'connectionName']: - instance_properties[sql_prop] = find_output_value( - sql_prop, - sql.outputs - ) - - instance_properties['bucket'] = bucket.self_link - instance_properties['project'] = project.self_link - instance_properties['serviceAccountEmail'] = service_account.self_link - instance_properties['serviceAccountScopes'] = [ - 'https://www.googleapis.com/auth/cloud-platform' - ] - instance_properties['network'] = network.self_link - - self_link = get_ref(name) - - return DMResource(self_link, [instance], [ - { - 'name': 'serverName', - 'value': get_ref(name, 'name') - }, - { - 'name': 'serverSelfLink', - 'value': self_link - }, - { - 'name': 'serverInternalIp', - 'value': get_ref(name, 'internalIp') - } - ]) - -def get_client(properties, project, network, service_account, server): - """ Creates a Forseti client instance. """ - - name = properties['name'] - - instance_properties = copy.deepcopy(properties) - instance = { - 'name': name, - 'type': 'client.py', - 'properties': instance_properties - } - - if 'serviceAccount' in instance_properties: - del instance_properties['serviceAccount'] - - instance_properties['name'] = name - instance_properties['tags'] = ['forseti-client'] - - instance_properties['serverIp'] = find_output_value( - 'serverInternalIp', - server.outputs - ) - instance_properties['project'] = project.self_link - instance_properties['serviceAccountEmail'] = service_account.self_link - instance_properties['serviceAccountScopes'] = [ - 'https://www.googleapis.com/auth/cloud-platform' - ] - instance_properties['network'] = network.self_link - - self_link = get_ref(name) - - return DMResource(self_link, [instance], [ - { - 'name': 'clientName', - 'value': get_ref(name, 'name') - }, - { - 'name': 'clientSelfLink', - 'value': self_link - } - ]) - -def generate_config(context): - """ Entry point for the deployment resources. """ - - properties = context.properties - org_id = properties['organizationId'] - project = get_forseti_project(context.env['name'], properties['project']) - - # Service accounts - client_sa = get_client_service_account( - project.self_link, - properties['client'] - ) - - server_sa = get_server_service_account( - project.self_link, - properties['server'], - org_id - ) - - # Avoid race conditions at policy creation. - policies = optimize_policies_creation(server_sa, client_sa) - - # Network + firewalls - network = get_network(project.self_link) - - # Configure the server config bucket. - bucket = get_server_bucket( - properties['bucket'], - project.self_link, - server_sa.self_link - ) - - # Cloud SQL. - cloud_sql = get_cloud_sql(properties['cloudSql'], project.self_link) - - if project.outputs: # creates a project - wait_for_init_complete( - project, - client_sa, - server_sa, - network, - cloud_sql, - policies - ) - - # Client/Server instances. - server = get_server( - properties['server'], - project, - network, - cloud_sql, - server_sa, - bucket - ) - client = get_client( - properties['client'], - project, - network, - client_sa, server - ) - - # Join all resources into one final collection. - result = merge_dm_resources( - project, - client_sa, - server_sa, - policies, - bucket, - cloud_sql, - network, - server, - client - ) - - return { - 'resources': result.resources, - 'outputs': result.outputs - } diff --git a/dm/templates/forseti/forseti.py.schema b/dm/templates/forseti/forseti.py.schema deleted file mode 100644 index 66e2b6aa8be..00000000000 --- a/dm/templates/forseti/forseti.py.schema +++ /dev/null @@ -1,256 +0,0 @@ -# Copyright 2018 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -info: - title: Forseti Security - author: Sourced Group Inc. - description: | - Supports creation of a Forseti Security project, client, and server - instances. - -imports: - - path: ../project/project.py - name: project.py - - path: server.py - - path: client.py - -required: - - project - - organizationId - - server - - client - - bucket - - cloudSql - -properties: - project: - type: object - description: Forseti project settings. - properties: - create: - type: boolean - default: false - description: Defines whether or not to create the project. - id: - type: string - description: | - The Forseti project ID. If you are creating a new project and the - value is missing, the project ID is auto-generated. - name: - type: string - description: | - The optional project name. Ignored when using an existing project. - parent: - type: object - description: The parent of the project. - properties: - type: - type: string - description: The parent type (organization or folder). - enum: - - organization - - folder - default: organization - id: - type: [integer, string] - description: The ID of the projects' parent. - billingAccountId: - type: string - description: | - The ID of the billing account to attach to the projects. - For example, '00E12A-0AB8B2-078CE8'. - organizationId: - type: number - description: | - ID of the organization under which the Forseti project is going to be - deployed. - server: - type: object - description: The Forseti server config. - required: - - zone - properties: - serviceAccount: - type: object - description: Forseti server's new service account settings. - properties: - accountId: - type: string - description: The optional ID of the service account. - displayName: - type: string - description: The optional name of the service account. - name: - type: string - default: forseti-server - description: The name of the Forseti server instance. - sourceImage: - type: string - default: projects/ubuntu-os-cloud/global/images/family/ubuntu-1804-lts - description: | - The source image for the disk. To create a disk with one of the - public operating system images, specify the image by its family name. - For example, specify family/debian-9 to use the latest Debian 9 image - projects/debian-cloud/global/images/family/debian-9. - To create a disk with a custom (your own) image, specify the - image name in the following format: global/images/my-custom-image. - See https://cloud.google.com/compute/docs/images for details. - srcPath: - type: string - default: https://github.com/GoogleCloudPlatform/forseti-security - description: | - The URL of the git repository containing the Forseti source code. - srcVersion: - type: string - default: 'v2.0' - description: The version (git tag name) of the source code to use. - zone: - type: string - description: The availability zone. E.g., 'us-central1-a'. - port: - type: number - default: 3306 - description: The Cloud SQL port number. - frequency: - type: string - default: '0 */2 * * *' - description: The CRON string specifying how often Forseti should run. - machineType: - type: string - default: n1-standard-1 - description: | - The Compute Instance type; e.g., 'n1-standard-1'. - See https://cloud.google.com/compute/docs/machine-types for details. - sqlOsArch: - type: string - default: linux.amd64 - description: The architecture of the Cloud SQL proxy to use. - client: - type: object - description: The Forseti client instance configuration. - required: - - zone - properties: - serviceAccount: - type: object - description: | - Forseti client's new service account settings. - properties: - accountId: - type: string - description: The optional ID of the service account. - displayName: - type: string - description: The optional name of the service account. - name: - type: string - default: forseti-client - description: The name of the Forseti client instance. - sourceImage: - type: string - default: projects/ubuntu-os-cloud/global/images/family/ubuntu-1804-lts - description: | - The source image for the disk. To create a disk with one of the - public operating system images, specify the image by its family name. - For example, specify family/debian-9 to use the latest Debian 9 image - projects/debian-cloud/global/images/family/debian-9. - To create a disk with a custom (your own) image, specify the image - name in the following format: global/images/my-custom-image. - See https://cloud.google.com/compute/docs/images for details. - srcPath: - type: string - default: https://github.com/GoogleCloudPlatform/forseti-security - description: | - The URL of the git repository containing the Forseti source code. - srcVersion: - type: string - default: 'v2.0' - description: The version (git tag name) of the source code to use. - zone: - type: string - description: The availability zone. E.g., 'us-central1-a'. - machineType: - type: string - default: n1-standard-1 - description: | - The Compute Instance type; e.g., 'n1-standard-1'. - See https://cloud.google.com/compute/docs/machine-types for details. - bucket: - type: object - description: The Forseti server's configuration bucket. - required: - - name - properties: - name: - type: string - description: | - The name of the bucket containing the configuration files. As a bare - minimum, the bucket must contain the Forseti server configuration - file. - E.g. - configs/ - forseti_conf_server.yaml - Additionally, it can contain the `rules` folder. - cloudSql: - type: object - description: The Forseti Cloud SQL instance configuration. - required: - - region - - instanceName - properties: - region: - type: string - description: The region where the instance is created. - instanceName: - type: string - description: The Cloud SQL Instance name. - -outputs: - properties: - - projectId: - type: string - description: The ID of created project. - - connectionName: - type: string - description: The connection string to the Cloud SQL instance. - - databaseName: - type: string - description: The name of the database created for Forseti. - - networkName: - type: string - description: The name of the network created for Forseti. - - networkSelfLink: - type: string - description: The URI (SelfLink) of the network resource. - - serverName: - type: string - description: The name of the Forseti server instance. - - serverSelfLink: - type: string - description: The URI (SelfLink) of the Forseti server resource. - - serverInternalIp: - type: string - description: The internal IP address of the server resource. - - clientName: - type: string - description: The name of the Forseti client instance. - - clientSelfLink: - type: string - description: The URI (SelfLink) of the Forseti client resource. - -documentation: - - templates/forseti/README.md - -examples: - - templates/forseti/examples/forseti.yaml diff --git a/dm/templates/forseti/server.py b/dm/templates/forseti/server.py deleted file mode 100644 index d025fb86d15..00000000000 --- a/dm/templates/forseti/server.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright 2017 The Forseti Security Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Creates a GCE server instance for Forseti Security. """ - -FORSETI_HOME = '$USER_HOME/forseti-security' - -FORSETI_SERVER_CONF = '{}/configs/forseti_conf_server.yaml'.format(FORSETI_HOME) - -EXPORT_FORSETI_VARS = ( - 'export FORSETI_HOME={}\n' - 'export FORSETI_SERVER_CONF={}\n' -).format(FORSETI_HOME, - FORSETI_SERVER_CONF) - -STARTUP_SCRIPT_TEMPLATE = """#!/bin/bash -exec > /tmp/deployment.log -exec 2>&1 -# Ubuntu update. -sudo apt-get update -y -sudo apt-get upgrade -y -sudo apt-get update && sudo apt-get --assume-yes install google-cloud-sdk -USER_HOME=/home/ubuntu -# Install fluentd if necessary. -FLUENTD=$(ls /usr/sbin/google-fluentd) -if [ -z "$FLUENTD" ]; then - cd $USER_HOME - curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh - bash install-logging-agent.sh -fi -# Check whether Cloud SQL proxy is installed. -CLOUD_SQL_PROXY=$(which cloud_sql_proxy) -if [ -z "$CLOUD_SQL_PROXY" ]; then - cd $USER_HOME - wget https://dl.google.com/cloudsql/cloud_sql_proxy.{cloudsql_arch} - sudo mv cloud_sql_proxy.{cloudsql_arch} /usr/local/bin/cloud_sql_proxy - chmod +x /usr/local/bin/cloud_sql_proxy -fi -# Install Forseti Security. -cd $USER_HOME -rm -rf *forseti* -# Download Forseti source code -{download_forseti} -cd forseti-security -git fetch --all -{checkout_forseti_version} -# Forseti Host Setup -sudo apt-get install -y git unzip -# Forseti host dependencies -sudo apt-get install -y $(cat install/dependencies/apt_packages.txt | grep -v "#" | xargs) -# Forseti dependencies -pip install --upgrade pip==9.0.3 -pip install -q --upgrade setuptools wheel -pip install -q --upgrade -r requirements.txt -# Setup Forseti logging -touch /var/log/forseti.log -chown ubuntu:root /var/log/forseti.log -cp {forseti_home}/configs/logging/fluentd/forseti.conf /etc/google-fluentd/config.d/forseti.conf -cp {forseti_home}/configs/logging/logrotate/forseti /etc/logrotate.d/forseti -chmod 644 /etc/logrotate.d/forseti -service google-fluentd restart -logrotate /etc/logrotate.conf -# Change the access level of configs/ rules/ and run_forseti.sh -chmod -R ug+rwx {forseti_home}/configs {forseti_home}/rules {forseti_home}/install/gcp/scripts/run_forseti.sh -# Install Forseti -python setup.py install -# Export variables required by initialize_forseti_services.sh. -{export_initialize_vars} -# Export variables required by run_forseti.sh -{export_forseti_vars} -# Store the variables in /etc/profile.d/forseti_environment.sh -# so all the users will have access to them -echo "echo '{export_forseti_vars}' >> /etc/profile.d/forseti_environment.sh" | sudo sh -# Download server configuration from GCS -gsutil cp gs://{scanner_bucket}/configs/forseti_conf_server.yaml {forseti_server_conf} -gsutil cp -r gs://{scanner_bucket}/rules {forseti_home}/ -# Start Forseti service depends on vars defined above. -bash ./install/gcp/scripts/initialize_forseti_services.sh -echo "Starting services." -systemctl start cloudsqlproxy -sleep 5 -systemctl start forseti -echo "Success! The Forseti API server has been started." -# Create a Forseti env script -FORSETI_ENV="$(cat < $USER_HOME/forseti_env.sh -USER=ubuntu -# Use flock to prevent rerun of the same cron job when the previous job is still running. -# If the lock file does not exist under the tmp directory, it will create the file and put a lock on top of the file. -# When the previous cron job is not finished and the new one is trying to run, it will attempt to acquire the lock -# to the lock file and fail because the file is already locked by the previous process. -# The -n flag in flock will fail the process right away when the process is not able to acquire the lock so we won't -# queue up the jobs. -# If the cron job failed the acquire lock on the process, it will log a warning message to syslog. -(echo "{run_frequency} (/usr/bin/flock -n /home/ubuntu/forseti-security/forseti_cron_runner.lock $FORSETI_HOME/install/gcp/scripts/run_forseti.sh || echo '[forseti-security] Warning: New Forseti cron job will not be started, because previous Forseti job is still running.') 2>&1 | logger") | crontab -u $USER - -echo "Added the run_forseti.sh to crontab under user $USER" -echo "Execution of startup script finished" -""" - -def get_export_initialize_vars(database, port, connection_string): - """ Gets the shell script that persists the Forseti env variables. """ - - template = """ - export SQL_PORT={}\n - export SQL_INSTANCE_CONN_STRING="{}"\n - export FORSETI_DB_NAME="{}"\n - """ - return template.format(port, connection_string, database) - - -def get_full_machine_type(project_id, zone, machine_type): - """ Gets a full URL to the specified machine type. """ - - prefix = 'https://www.googleapis.com/compute/v1' - - return '{}/projects/{}/zones/{}/machineTypes/{}'.format( - prefix, - project_id, - zone, - machine_type - ) - - -def generate_config(context): - """ Generates a configuration. """ - - properties = context.properties - instance_name = properties.get('name', context.env['name']) - project_id = properties.get('project', context.env['project']) - source_image = properties['sourceImage'] - source_path = properties['srcPath'] - version = properties['srcVersion'] - zone = properties['zone'] - bucket = properties['bucket'] - run_frequency = properties['frequency'] - machine_type = properties['machineType'] - machine_type_uri = get_full_machine_type(project_id, zone, machine_type) - - startup_script = STARTUP_SCRIPT_TEMPLATE.format( - cloudsql_arch=properties['sqlOsArch'], - download_forseti='git clone {}.git'.format(source_path), - checkout_forseti_version='git checkout {}'.format(version), - forseti_home=FORSETI_HOME, - scanner_bucket=bucket, - forseti_server_conf=FORSETI_SERVER_CONF, - export_initialize_vars=get_export_initialize_vars( - properties['databaseName'], - properties['port'], - properties['connectionName'] - ), - export_forseti_vars=EXPORT_FORSETI_VARS, - run_frequency=run_frequency, - ) - - resources = [] - - resources.append( - { - 'name': instance_name, - 'type': 'gcp-types/compute-v1:instances', - 'properties': - { - 'project': - project_id, - 'zone': - zone, - 'machineType': - machine_type_uri, - 'disks': - [ - { - 'deviceName': 'boot', - 'type': 'PERSISTENT', - 'boot': True, - 'autoDelete': True, - 'initializeParams': - { - 'sourceImage': source_image - } - } - ], - 'networkInterfaces': - [ - { - 'network': - properties['network'], - 'accessConfigs': - [ - { - 'name': 'External NAT', - 'type': 'ONE_TO_ONE_NAT' - } - ] - } - ], - 'serviceAccounts': - [ - { - 'email': properties['serviceAccountEmail'], - 'scopes': properties['serviceAccountScopes'], - } - ], - 'tags': { - 'items': properties.get('tags', - []) - }, - 'metadata': - { - 'items': - [ - { - 'key': 'startup-script', - 'value': startup_script - } - ] - } - } - } - ) - return { - 'resources': - resources, - 'outputs': - [ - { - 'name': 'name', - 'value': '$(ref.{}.name)'.format(instance_name) - }, - { - 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(instance_name) - }, - { - 'name': - 'internalIp', - 'value': - '$(ref.{}.networkInterfaces[0].networkIP)'. - format(instance_name) - } - ] - } diff --git a/dm/templates/forseti/server.py.schema b/dm/templates/forseti/server.py.schema deleted file mode 100644 index 76b73c149af..00000000000 --- a/dm/templates/forseti/server.py.schema +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2018 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -info: - title: Forseti Security - author: Sourced Group Inc. - description: | - Supports creation of a Forseti Security server instance. - -imports: - - path: server.py - -required: - - project - - zone - - bucket - - connectionName - - databaseName - - network - - serviceAccountEmail - -properties: - project: - type: string - description: | - The project ID of the project where the instance must be placed. - name: - type: string - default: forseti-server - description: The name of the Forseti server instance. - sourceImage: - type: string - default: projects/ubuntu-os-cloud/global/images/family/ubuntu-1804-lts - description: | - The source image for the disk. To create a disk with one of the - public operating system images, specify the image by its family name. - For example, specify family/debian-9 to use the latest Debian 9 image - projects/debian-cloud/global/images/family/debian-9. - To create a disk with a custom (your own) image, specify the - image name in the following format: global/images/my-custom-image. - See https://cloud.google.com/compute/docs/images for details. - srcPath: - type: string - default: https://github.com/GoogleCloudPlatform/forseti-security - description: | - The URL of the git repository containing the Forseti source code. - srcVersion: - type: string - default: 'v2.0' - description: The version (git tag name) of the source code to use. - connectionName: - type: string - description: The connection string to the Cloud SQL Instance. - zone: - type: string - description: The availability zone. E.g., 'us-central1-a'. - bucket: - type: string - description: | - The name of the bucket containing the configuration files. As a bare - minimum, the bucket must contain the Forseti server configuration file. - E.g. - configs/ - forseti_conf_server.yaml - Additionally, it can contain the `rules` folder. - databaseName: - type: string - description: The name of the database created for Forseti. - port: - type: number - default: 3306 - description: The Cloud SQL port number. - frequency: - type: string - default: '0 */2 * * *' - description: The CRON string specifying how often Forseti should run. - machineType: - type: string - default: n1-standard-1 - description: | - The Compute Instance type; e.g., 'n1-standard-1'. - See https://cloud.google.com/compute/docs/machine-types for details. - network: - type: string - description: The URL of the network. - sqlOsArch: - type: string - default: linux.amd64 - description: The architecture of the Cloud SQL proxy to use. - serviceAccountEmail: - type: string - description: The email address of the service account - serviceAccountScopes: - type: array - default: [] - description: | - The list of scopes to be made available for the service account. - items: - type: string - description: | - The access scope; e.g., 'https://www.googleapis.com/auth/compute.readonly'. - See https://cloud.google.com/compute/docs/access/service-accounts#accesscopesiam - for more details. - tags: - type: array - default: [] - description: Network tags for the instance. - -outputs: - properties: - - name: - type: string - description: The name of the Forseti server instance. - - selfLink: - type: string - description: The URI (SelfLink) of the Forseti server resource. - - internalIp: - type: string - description: The internal IP address of the server resource. diff --git a/dm/templates/forseti/tests/integration/forseti.bats b/dm/templates/forseti/tests/integration/forseti.bats deleted file mode 100755 index aeca8ae1c15..00000000000 --- a/dm/templates/forseti/tests/integration/forseti.bats +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env bats - -source tests/helpers.bash - -TEST_NAME=$(basename "${BATS_TEST_FILENAME}" | cut -d '.' -f 1) - -# Create a random 10-char string and save it in a file. -RANDOM_FILE="/tmp/${CLOUD_FOUNDATION_ORGANIZATION_ID}-${TEST_NAME}.txt" -if [[ ! -e "${RANDOM_FILE}" ]]; then - RAND=$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 10) - echo ${RAND} > "${RANDOM_FILE}" -fi - -# Set variables based on the random string saved in the file. -# envsubst requires all variables used in the example/config to be exported. -if [[ -e "${RANDOM_FILE}" ]]; then - export RAND=$(cat "${RANDOM_FILE}") - DEPLOYMENT_NAME="${CLOUD_FOUNDATION_PROJECT_ID}-${TEST_NAME}-${RAND}" - # Replace underscores in the deployment name with dashes. - DEPLOYMENT_NAME=${DEPLOYMENT_NAME//_/-} - CONFIG=".${DEPLOYMENT_NAME}.yaml" - export BUCKET_NAME="forseti-security-bucket-${RAND}" - export SERVER_ZONE="us-central1-a" - export CLIENT_ZONE="us-central1-a" - export PROJECT_ID="forseti-project-${RAND}" - export PROJECT_NAME="Forseti Security-${RAND}" - export SQL_NAME="forseti-sql-instance-${RAND}" - export SQL_DB_NAME="${SQL_NAME}-db" - export SQL_REGION="us-central1" - export SERVER_SA_PREFIX="forseti-server-gcp" - export CLIENT_SA_PREFIX="forseti-client-gcp" - export SERVER_NAME="forseti-server" - export CLIENT_NAME="forseti-client" - export CLOUD_FOUNDATION_FOLDER_NAME="test-forseti-folder-${RAND}" -fi - -########## HELPER FUNCTIONS ########## - -function create_config() { - echo "Creating ${CONFIG}" - envsubst < ${BATS_TEST_DIRNAME}/${TEST_NAME}.yaml > "${CONFIG}" -} - -function delete_config() { - echo "Deleting ${CONFIG}" - rm -f "${CONFIG}" -} - -function setup() { - # Global setup; executed once per test file. - if [ ${BATS_TEST_NUMBER} -eq 1 ]; then - # Set up instance groups to be load-balanced via HAProxy. - gsutil mb gs://${BUCKET_NAME} - gcloud alpha resource-manager folders create \ - --display-name="${CLOUD_FOUNDATION_FOLDER_NAME}" \ - --organization="${CLOUD_FOUNDATION_ORGANIZATION_ID}" > ~/output.txt - export CLOUD_FOUNDATION_FOLDER_ID=$(gcloud alpha resource-manager folders list \ - --project "${CLOUD_FOUNDATION_PROJECT_ID}" \ - --organization "${CLOUD_FOUNDATION_ORGANIZATION_ID}" | \ - grep "${CLOUD_FOUNDATION_FOLDER_NAME}" | \ - awk '{print $3}') - gcloud alpha resource-manager folders list --organization="${CLOUD_FOUNDATION_ORGANIZATION_ID}" >> ~/output.txt - create_config - fi - - # Per-test setup steps. -} - -function teardown() { - # Global teardown; executed once per test file. - if [[ "$BATS_TEST_NUMBER" -eq "${#BATS_TEST_NAMES[@]}" ]]; then - rm -f "${RANDOM_FILE}" - gcloud alpha resource-manager folders delete \ - "${CLOUD_FOUNDATION_FOLDER_ID}" - gsutil rb gs://${BUCKET_NAME} - delete_config - fi - - # Per-test teardown steps. -} - - -@test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { - run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ - --config "${CONFIG}" \ - --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ "$status" -eq 0 ]] -} - -@test "Verifying that new project exists" { - run gcloud projects list - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${PROJECT_ID}" ]] - [[ "$output" =~ "${PROJECT_NAME}" ]] -} - -@test "Verifying server's service account" { - run gcloud iam service-accounts list --project ${PROJECT_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${SERVER_SA_PREFIX}" ]] - - run gcloud projects get-iam-policy ${PROJECT_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "roles/cloudsql.client" ]] - [[ "$output" =~ "roles/logging.logWriter" ]] - [[ "$output" =~ "roles/storage.objectCreator" ]] - [[ "$output" =~ "roles/storage.objectViewer" ]] - - run gcloud organizations get-iam-policy \ - ${CLOUD_FOUNDATION_ORGANIZATION_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "roles/appengine.appViewer" ]] - [[ "$output" =~ "roles/bigquery.dataViewer" ]] - [[ "$output" =~ "roles/browser" ]] - [[ "$output" =~ "roles/cloudasset.viewer" ]] - [[ "$output" =~ "roles/cloudsql.viewer" ]] - [[ "$output" =~ "roles/compute.networkViewer" ]] - [[ "$output" =~ "roles/compute.securityAdmin" ]] - [[ "$output" =~ "roles/iam.securityReviewer" ]] - [[ "$output" =~ "roles/servicemanagement.quotaViewer" ]] - [[ "$output" =~ "roles/serviceusage.serviceUsageConsumer" ]] -} - -@test "Verifying client's service account" { - run gcloud iam service-accounts list --project ${PROJECT_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${CLIENT_SA_PREFIX}" ]] - - run gcloud projects get-iam-policy ${PROJECT_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "roles/logging.logWriter" ]] - [[ "$output" =~ "roles/storage.objectViewer" ]] -} - -@test "Verifying sql instance" { - run gcloud sql instances list \ - --project ${PROJECT_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${SQL_NAME}" ]] - [[ "$output" =~ "MYSQL_5_7" ]] -} - -@test "Verifying sql database" { - run gcloud sql databases list \ - --instance ${SQL_NAME} \ - --project ${PROJECT_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${SQL_DB_NAME}" ]] -} - -@test "Verifying Forseti server" { - run gcloud compute instances list \ - --project ${PROJECT_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${SERVER_NAME}" ]] - - run gcloud compute instances describe ${SERVER_NAME} \ - --project ${PROJECT_ID} \ - --zone ${SERVER_ZONE} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${SQL_DB_NAME}" ]] - [[ "$output" =~ "${SQL_NAME}" ]] - [[ "$output" =~ "${BUCKET_NAME}" ]] -} - -@test "Verifying Forseti client" { - run gcloud compute instances list \ - --project ${PROJECT_ID} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "${CLIENT_NAME}" ]] - - run gcloud compute instances describe ${CLIENT_NAME} \ - --project ${PROJECT_ID} \ - --zone ${SERVER_ZONE} - [[ "$status" -eq 0 ]] - [[ "$output" =~ "forseti_conf_client.yaml" ]] - [[ "$output" =~ "server_ip" ]] -} - -@test "Deleting deployment" { - run gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" -q \ - --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ "$status" -eq 0 ]] -} diff --git a/dm/templates/forseti/tests/integration/forseti.yaml b/dm/templates/forseti/tests/integration/forseti.yaml deleted file mode 100644 index 56ee570a584..00000000000 --- a/dm/templates/forseti/tests/integration/forseti.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# The test of the Forseti template. - -imports: - - path: templates/forseti/forseti.py - name: forseti.py - -resources: - - name: forseti - type: forseti.py - properties: - organizationId: ${CLOUD_FOUNDATION_ORGANIZATION_ID} - server: - zone: ${SERVER_ZONE} - client: - zone: ${CLIENT_ZONE} - project: - create: true - id: ${PROJECT_ID} - name: ${PROJECT_NAME} - parent: - type: folder - id: ${CLOUD_FOUNDATION_FOLDER_ID} - billingAccountId: ${CLOUD_FOUNDATION_BILLING_ACCOUNT_ID} - bucket: - name: ${BUCKET_NAME} - cloudSql: - region: ${SQL_REGION} - instanceName: ${SQL_NAME} diff --git a/dm/templates/forwarding_rule/forwarding_rule.py b/dm/templates/forwarding_rule/forwarding_rule.py index 5527e931d5b..fb7f6584a54 100644 --- a/dm/templates/forwarding_rule/forwarding_rule.py +++ b/dm/templates/forwarding_rule/forwarding_rule.py @@ -14,8 +14,10 @@ """ This template creates a forwarding rule. """ REGIONAL_GLOBAL_TYPE_NAMES = { - True: 'compute.v1.forwardingRule', - False: 'compute.v1.globalForwardingRule' + # https://cloud.google.com/compute/docs/reference/rest/v1/forwardingRules + True: 'gcp-types/compute-v1:forwardingRules', + # https://cloud.google.com/compute/docs/reference/rest/v1/globalForwardingRules + False: 'gcp-types/compute-v1:globalForwardingRules' } @@ -55,12 +57,16 @@ def generate_config(context): properties = context.properties name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) is_regional = 'region' in properties region = properties.get('region') - rule_properties = {'name': name} + rule_properties = { + 'name': name, + 'project': project_id, + } resource = { - 'name': name, + 'name': context.env['name'], 'type': REGIONAL_GLOBAL_TYPE_NAMES[is_regional], 'properties': rule_properties } @@ -77,12 +83,15 @@ def generate_config(context): 'subnetwork', 'network', 'backendService', - 'ipVersion' + 'ipVersion', + 'serviceLabel', + 'networkTier', + 'allPorts', ] for prop in optional_properties: set_optional_property(rule_properties, properties, prop) - outputs = get_forwarding_rule_outputs(name, region) + outputs = get_forwarding_rule_outputs(context.env['name'], region) return {'resources': [resource], 'outputs': outputs} diff --git a/dm/templates/forwarding_rule/forwarding_rule.py.schema b/dm/templates/forwarding_rule/forwarding_rule.py.schema index b5e93372890..0a4635c3e86 100644 --- a/dm/templates/forwarding_rule/forwarding_rule.py.schema +++ b/dm/templates/forwarding_rule/forwarding_rule.py.schema @@ -15,17 +15,147 @@ info: title: Forwarding Rule author: Sourced Group Inc. + version: 1.0.0 description: | - Creates a forwrding rule. - See https://cloud.google.com/load-balancing/docs/forwarding-rules for - details. + Creates a forwarding rule. + + For more information on this resource: + https://cloud.google.com/load-balancing/docs/forwarding-rules + + APIs endpoints used by this template: + - gcp-types/compute-v1:forwardingRules => + https://cloud.google.com/compute/docs/reference/rest/v1/forwardingRules + - gcp-types/compute-v1:globalForwardingRules => + https://cloud.google.com/compute/docs/reference/rest/v1/globalForwardingRules additionalProperties: false +allOf: + - oneOf: + - properties: + loadBalancingScheme: + enum: + - INTERNAL + IPProtocol: + enum: + - TCP + - UDP + - not: + properties: + loadBalancingScheme: + enum: + - INTERNAL + - oneOf: + - allOf: + - required: + - region + - properties: + networkTier: + enum: + - PREMIUM + - STANDARD + - allOf: + - not: + required: + - region + - properties: + networkTier: + enum: + - PREMIUM + - oneOf: + - properties: + loadBalancingScheme: + enum: + - INTERNAL_SELF_MANAGED + IPProtocol: + enum: + - TCP + - not: + properties: + loadBalancingScheme: + enum: + - INTERNAL_SELF_MANAGED + - oneOf: + - allOf: + - properties: + IPProtocol: + enum: + - TCP + - UDP + - SCTP + - required: + - portRange + - not: + required: + - portRange + - oneOf: + - allOf: + - properties: + loadBalancingScheme: + enum: + - INTERNAL + - anyOf: + - required: + - ports + - required: + - backendService + - required: + - subnetwork + - required: + - serviceLabel + - allOf: + - not: + required: + - ports + - not: + required: + - backendService + - not: + required: + - subnetwork + - not: + required: + - serviceLabel + - oneOf: + - allOf: + - properties: + loadBalancingScheme: + enum: + - INTERNAL + - INTERNAL_SELF_MANAGED + - required: + - network + - not: + required: + - network + - oneOf: + - allOf: + - loadBalancingScheme: + enum: + - EXTERNAL + - not: + required: + - region + - required: + - ipVersion + - not: + required: + - ipVersion + properties: name: type: string - description: The resource name. + description: | + Must comply with RFC1035. Specifically, the name must be 1-63 characters long and match + the regular expression [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a lowercase letter, + and all following characters must be a dash, lowercase letter, or digit, except the last character, + which cannot be a dash. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the forwarding rule. The + Google apps domain is prefixed if applicable. description: type: string description: The resource description (optional). @@ -37,15 +167,39 @@ properties: IPAddress: type: string description: | - The IP address on behalf of which the forwarding rule serves. The - address can be specified either by a literal IP address or by a URL - reference to an existing Address resource. + The IP address that this forwarding rule is serving on behalf of. + + Addresses are restricted based on the forwarding rule's load balancing scheme + (EXTERNAL or INTERNAL) and scope (global or regional). + + When the load balancing scheme is EXTERNAL, for global forwarding rules, the address must be a global IP, + and for regional forwarding rules, the address must live in the same region as the forwarding rule. + If this field is empty, an ephemeral IPv4 address from the same scope (global or regional) will be assigned. + A regional forwarding rule supports IPv4 only. A global forwarding rule supports either IPv4 or IPv6. + + When the load balancing scheme is INTERNAL_SELF_MANAGED, this must be a URL reference to an existing Address + resource ( internal regional static IP address), with a purpose of GCE_END_POINT and addressType of INTERNAL. + + When the load balancing scheme is INTERNAL, this can only be an RFC 1918 IP address belonging to the + network/subnet configured for the forwarding rule. By default, if this field is empty, an ephemeral + internal IP address will be automatically allocated from the IP range of the subnet or network + configured for this forwarding rule. + + An address can be specified either by a literal IP address or a URL reference to an existing Address resource. + The following examples are all valid: + - 100.1.2.3 + - https://www.googleapis.com/compute/v1/projects/project/regions/region/addresses/address + - projects/project/regions/region/addresses/address + - regions/region/addresses/address + - global/addresses/address + - address IPProtocol: type: string description: | - The IP protocol to which the rule applies. If the load balancing scheme - is INTERNAL, the valid valuse are TCP and UDP. For the INTERNAL_SELF_MANAGED - load balancing scheme, only TCP is valid. + The IP protocol to which this rule applies. Valid options are TCP, UDP, ESP, AH, SCTP or ICMP. + + When the load balancing scheme is INTERNAL, only TCP and UDP are valid. + When the load balancing scheme is INTERNAL_SELF_MANAGED, only TCPis valid. enum: - TCP - UDP @@ -63,10 +217,16 @@ properties: when IPProtocol is TCP, UDP, or SCTP. ports: type: array + uniqItems: true description: | - The list of ports; only packets addressed to these ports are forwarded - to the backends configured with the forwarding rule. Used in conjunction with - the backendService field for INTERNAL load balancing. + This field is used along with the backendService field for internal load balancing. + + When the load balancing scheme is INTERNAL, a list of ports can be configured, for example, + ['80'], ['8000','9000'] etc. Only packets addressed to these ports will be forwarded to the + backends configured with this forwarding rule. + + You may specify a maximum of up to 5 ports. + maxItems: 5 items: type: integer minimum: 1 @@ -74,11 +234,19 @@ properties: target: type: string description: | - The URL of the target resource to receive the matched traffic. For - regional forwarding rules, this target must be located in the same region - as the forwarding rule. For global forwarding rules, this target must be a - global load balancing resource. - For example: https://www.googleapis.com/compute/v1/projects/{project}/global/{targetType}/{targetName} + The URL of the target resource to receive the matched traffic. For regional forwarding rules, + this target must live in the same region as the forwarding rule. For global forwarding rules, this + target must be a global load balancing resource. The forwarded traffic must be of a type appropriate + to the target object. For INTERNAL_SELF_MANAGED load balancing, only HTTP and HTTPS targets are valid. + + Authorization requires one or more of the following Google IAM permissions on the specified resource target: + - compute.targetHttpProxies.use + - compute.targetHttpsProxies.use + - compute.targetInstances.use + - compute.targetPools.use + - compute.targetSslProxies.use + - compute.targetTcpProxies.use + - compute.targetVpnGateways.use loadBalancingScheme: type: string description: | @@ -97,6 +265,32 @@ properties: description: | The subnetwork the load-balanced IP must belong to for the forwarding rule. Used only for INTERNAL load balancing. + serviceLabel: + type: string + description: | + An optional prefix to the service name for this Forwarding Rule. If specified, will be the first label + of the fully qualified service name. + + The label must be 1-63 characters long, and comply with RFC1035. Specifically, the label must be 1-63 characters + long and match the regular expression [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a + lowercase letter, and all following characters must be a dash, lowercase letter, or digit, except + the last character, which cannot be a dash. + + This field is only used for internal load balancing. + networkTier: + type: string + description: | + This signifies the networking tier used for configuring this load balancer and can only + take the following values: PREMIUM , STANDARD. + + For regional ForwardingRule, the valid values are PREMIUM and STANDARD. For GlobalForwardingRule, + the valid value is PREMIUM. + + If this field is not specified, it is assumed to be PREMIUM. If IPAddress is specified, + this value must be equal to the networkTier of the Address. + enum: + - STANDARD + - PREMIUM network: type: string description: | @@ -117,6 +311,14 @@ properties: enum: - IPV4 - IPV6 + allPorts: + type: boolean + description: | + This field is used along with the backendService field for internal load balancing or with the target + field for internal TargetInstance. This field cannot be used with port or portRange fields. + + When the load balancing scheme is INTERNAL and protocol is TCP/UDP, specify this field to allow packets + addressed to any ports will be forwarded to the backends configured with this forwarding rule. outputs: properties: diff --git a/dm/templates/gcs_bucket/gcs_bucket.py b/dm/templates/gcs_bucket/gcs_bucket.py index 22850365a66..a10330f94b8 100644 --- a/dm/templates/gcs_bucket/gcs_bucket.py +++ b/dm/templates/gcs_bucket/gcs_bucket.py @@ -18,16 +18,18 @@ def generate_config(context): """ Entry point for the deployment resources. """ resources = [] - project_id = context.env['project'] - bucket_name = context.properties.get('name', context.env['name']) + properties = context.properties + project_id = properties.get('project', context.env['project']) + bucket_name = properties.get('name', context.env['name']) # output variables - bucket_selflink = '$(ref.{}.selfLink)'.format(bucket_name) + bucket_selflink = '$(ref.{}.selfLink)'.format(context.env['name']) bucket_uri = 'gs://' + bucket_name + '/' bucket = { - 'name': bucket_name, - 'type': 'storage.v1.bucket', + 'name': context.env['name'], + # https://cloud.google.com/storage/docs/json_api/v1/buckets + 'type': 'gcp-types/storage-v1:buckets', 'properties': { 'project': project_id, 'name': bucket_name @@ -39,6 +41,14 @@ def generate_config(context): bucket['properties']['billing'] = {'requesterPays': requesterPays} optional_props = [ + 'acl', + 'iamConfiguration', + 'retentionPolicy', + 'encryption', + 'defaultEventBasedHold', + 'cors', + 'defaultObjectAcl', + 'billing', 'location', 'versioning', 'storageClass', @@ -51,27 +61,32 @@ def generate_config(context): ] for prop in optional_props: - if prop in context.properties: - bucket['properties'][prop] = context.properties[prop] + if prop in properties: + bucket['properties'][prop] = properties[prop] resources.append(bucket) # If IAM policy bindings are defined, apply these bindings. + # https://cloud.google.com/storage/docs/json_api/v1/buckets/setIamPolicy storage_provider_type = 'gcp-types/storage-v1:storage.buckets.setIamPolicy' - bindings = context.properties.get('bindings', []) + bindings = properties.get('bindings', []) if bindings: iam_policy = { - 'name': bucket_name + '-iampolicy', + 'name': '{}-iampolicy'.format(context.env['name']), 'action': (storage_provider_type), 'properties': { - 'bucket': '$(ref.' + bucket_name + '.name)', + 'bucket': '$(ref.{}.name)'.format(context.env['name']), 'project': project_id, 'bindings': bindings } } resources.append(iam_policy) + if properties.get('billing', {}).get('requesterPays'): + for resource in resources: + resource['properties']['userProject'] = properties.get('userProject', context.env['project']) + return { 'resources': resources, diff --git a/dm/templates/gcs_bucket/gcs_bucket.py.schema b/dm/templates/gcs_bucket/gcs_bucket.py.schema index 1f96e6cbcce..8d4356ff554 100644 --- a/dm/templates/gcs_bucket/gcs_bucket.py.schema +++ b/dm/templates/gcs_bucket/gcs_bucket.py.schema @@ -15,27 +15,87 @@ info: title: Google Cloud Storage Bucket author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of a Google Cloud Storage bucket. + For more information on this resource: https://cloud.google.com/storage/docs/json_api/. + APIs endpoints used by this template: + - gcp-types/storage-v1:buckets => + https://cloud.google.com/storage/docs/json_api/v1/buckets + - gcp-types/storage-v1:storage.buckets.setIamPolicy => + https://cloud.google.com/storage/docs/json_api/v1/buckets/setIamPolicy + imports: - path: gcs_bucket.py additionalProperties: false -required: - - name +allOf: + - oneOf: + - properties: + iamConfiguration: + properties: + bucketPolicyOnly: + properties: + enabled: + enum: [False] + - allOf: + - not: + required: + - acl + - not: + required: + - defaultObjectAcl + - required: + - iamConfiguration + properties: + iamConfiguration: + properties: + bucketPolicyOnly: + properties: + enabled: + enum: [True] + - oneOf: + - allOf: + - not: + required: + - userProject + - properties: + billing: + properties: + requesterPays: + enum: [False] + - allOf: + - required: + - billing + - properties: + billing: + properties: + requesterPays: + enum: [True] properties: name: type: string - description: The name of the bucket. + description: The name of the bucket. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the bucket. + userProject: + type: string + description: | + The project to be billed for this request. Current project is used if not set. location: type: string default: us-east1 description: The region name where the bucket is deployed. + defaultEventBasedHold: + type: boolean + description: Whether or not to automatically apply an eventBasedHold to new objects added to the bucket. storageClass: type: string default: STANDARD @@ -52,6 +112,7 @@ properties: - DURABLE_REDUCED_AVAILABILITY versioning: type: object + additionalProperties: false description: Enables/disables object versioning. required: - enabled @@ -88,56 +149,257 @@ properties: an alias for a set of specific ACL entries that you can use to quickly apply multiple ACL entries to a bucket or object in a single operation. Ref: https://cloud.google.com/storage/docs/access-control/lists. + encryption: + type: object + additionalProperties: false + description: | + Encryption configuration for a bucket. + required: + - defaultKmsKeyName + properties: + defaultKmsKeyName: + type: string + description: | + A Cloud KMS key that will be used to encrypt objects inserted into this bucket, + if no encryption method is specified. + retentionPolicy: + type: object + additionalProperties: false + description: | + The bucket's retention policy, which defines the minimum age an object in the bucket + must have to be deleted or overwritten. + required: + - retentionPeriod + properties: + retentionPeriod: + type: integer + maximum: 3155760000 + exclusiveMaximum: True + description: | + The period of time, in seconds, that objects in the bucket must be retained and cannot be deleted, + overwritten, or archived. The value must be less than 3,155,760,000 seconds. + iamConfiguration: + type: object + additionalProperties: false + description: | + The bucket's IAM configuration. + required: + - bucketPolicyOnly + properties: + bucketPolicyOnly: + type: object + additionalProperties: false + description: | + The bucket's Bucket Policy Only configuration. + required: + - enabled + properties: + enabled: + type: boolean + default: False + description: | + Whether or not the bucket uses Bucket Policy Only. + If set, access checks only use bucket-level IAM policies or above. + billing: + type: object + additionalProperties: false + description: | + The bucket's billing configuration. + required: + - requesterPays + properties: + requesterPays: + type: boolean + default: False + description: | + When set to true, Requester Pays is enabled for this bucket. logging: type: object + additionalProperties: false required: - logBucket properties: logBucket: type: string description: | - The destination bucket where the current bucket's logs + The destination bucket where the current bucket's logs must be placed. logObjectPrefix: type: string description: The prefix for log object names. + cors: + type: array + uniqueItems: true + description: | + The bucket's Cross-Origin Resource Sharing (CORS) configuration. + items: + type: object + additionalProperties: false + required: + - method + - origin + properties: + maxAgeSeconds: + type: integer + description: | + The value, in seconds, to return in the Access-Control-Max-Age header used in preflight responses. + method: + type: array + uniqueItems: true + description: | + The list of HTTP methods on which to include CORS response headers, (GET, OPTIONS, POST, etc) + Note: "*" is permitted in the list of methods, and means "any method". + items: + type: string + origin: + type: array + uniqueItems: true + description: | + The list of Origins eligible to receive CORS response headers. + Note: "*" is permitted in the list of origins, and means "any Origin". + items: + type: string + responseHeader: + type: array + uniqueItems: true + description: | + The list of HTTP headers other than the simple response headers to give permission + for the user-agent to share across domains. + items: + type: string + defaultObjectAcl: + type: array + uniqueItems: true + description: | + Default access controls to apply to new objects when no ACL is provided. + This list defines an entity and role for one or more defaultObjectAccessControls Resources. + items: + type: object + additionalProperties: false + required: + - role + - entity + properties: + role: + type: string + description: | + The access permission for the entity. + + Acceptable values are: + "OWNER" + "READER" + enum: + - OWNER + - READER + entity: + type: string + description: | + The entity holding the permission, in one of the following forms: + - user-userId + - user-email + - group-groupId + - group-email + - domain-domain + - project-team-projectId + - allUsers + - allAuthenticatedUsers + + Examples: + - The user liz@example.com would be user-liz@example.com. + - The group example@googlegroups.com would be group-example@googlegroups.com. + - To refer to all members of the G Suite for Business domain example.com, the entity would be domain-example.com. + acl: + type: array + uniqueItems: true + description: | + Access controls on the bucket, containing one or more bucketAccessControls Resources. + items: + type: object + additionalProperties: false + required: + - role + - entity + properties: + role: + type: string + description: | + The access permission for the entity. + enum: + - OWNER + - READER + - WRITER + entity: + type: string + description: | + The entity holding the permission, in one of the following forms: + - user-userId + - user-email + - group-groupId + - group-email + - domain-domain + - project-team-projectId + - allUsers + - allAuthenticatedUsers + + Examples: + - The user liz@example.com would be user-liz@example.com. + - The group example@googlegroups.com would be group-example@googlegroups.com. + - To refer to all members of the G Suite for Business domain example.com, the entity would be domain-example.com. bindings: type: array + uniqueItems: true description: IAM bindings for the bucket. items: type: object + additionalProperties: false required: - role - members properties: role: type: string - pattern: ^roles\/ + pattern: ^roles\/storage\. description: The role to assign to members. members: type: array + uniqueItems: true items: type: string description: | - The member to add the binding for. Must be in the form user| - group|serviceAccount:email or domain:domain. - Can also be one of the following special values: allUsers, - allAuthenticatedUsers. + A collection of identifiers for members who may assume the provided role. + Recognized identifiers are as follows: + allUsers — A special identifier that represents anyone on the internet; with or without a Google account. + allAuthenticatedUsers — A special identifier that represents anyone who is authenticated with + a Google account or a service account. + user:emailid — An email address that represents a specific account. For example, + user:alice@gmail.com or user:joe@example.com. + serviceAccount:emailid — An email address that represents a service account. For example, + serviceAccount:my-other-app@appspot.gserviceaccount.com . + group:emailid — An email address that represents a Google group. For example, group:admins@example.com. + domain:domain — A G Suite domain name that represents all the users of that domain. + For example, domain:google.com or domain:example.com. + projectOwner:projectid — Owners of the given project. For example, projectOwner:my-example-project + projectEditor:projectid — Editors of the given project. For example, projectEditor:my-example-project + projectViewer:projectid — Viewers of the given project. For example, projectViewer:my-example-project lifecycle: type: object + additionalProperties: false description: The storage object's lifecycle actions and conditions. properties: rule: type: array + uniqueItems: true description: The lifecycle action and condition. items: type: object + additionalProperties: false required: - action - condition properties: action: type: object + additionalProperties: false description: The action to be taken if the condition is met. required: - type @@ -157,6 +419,7 @@ properties: - Delete condition: type: object + additionalProperties: false description: The lifecycle condition. properties: age: @@ -170,6 +433,7 @@ properties: For example, "2013-01-15". matchesStorageClass: type: array + uniqueItems: true description: | All objects with any of the selected storage classes. items: @@ -201,6 +465,7 @@ properties: When set to true, Requester Pays is enabled for this bucket. website: type: object + additionalProperties: false description: | The bucket's website configuration, controlling how the service behaves when accessing the bucket contents as a web site. diff --git a/dm/templates/gke/examples/gke_regional.yaml b/dm/templates/gke/examples/gke_regional.yaml index e3fa95d28f8..b101eb60c40 100644 --- a/dm/templates/gke/examples/gke_regional.yaml +++ b/dm/templates/gke/examples/gke_regional.yaml @@ -13,19 +13,20 @@ resources: - name: myk8sregional type: gke.py properties: - clusterLocationType: Regional region: us-east1 cluster: name: myk8sregional description: my awesome k8s cluster network: subnetwork: - nodeConfig: - oauthScopes: - - https://www.googleapis.com/auth/compute - - https://www.googleapis.com/auth/devstorage.read_only - - https://www.googleapis.com/auth/logging.write - - https://www.googleapis.com/auth/monitoring + nodePools: + - name: default + config: + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring locations: - us-east1-c - us-east1-b diff --git a/dm/templates/gke/examples/gke_regional_private.yaml b/dm/templates/gke/examples/gke_regional_private.yaml index c534380c790..8394df52095 100644 --- a/dm/templates/gke/examples/gke_regional_private.yaml +++ b/dm/templates/gke/examples/gke_regional_private.yaml @@ -18,46 +18,47 @@ resources: - name: myk8sregional type: gke.py properties: - clusterLocationType: Regional region: us-east1 cluster: name: myk8sregional description: my awesome k8s cluster network: subnetwork: - intialNodeCount: 1 - initialClusterVersion: 1.10.6-gke.3 - nodeConfig: - localSsdCount: 1 - oauthScopes: - - https://www.googleapis.com/auth/compute - - https://www.googleapis.com/auth/devstorage.read_only - - https://www.googleapis.com/auth/logging.write - - https://www.googleapis.com/auth/monitoring - taints: - - key: mykey1 - value: value1 - effect: NO_SCHEDULE - - key: mykey2 - value: value2 - effect: NO_EXECUTE + nodePools: + - name: default + initialNodeCount: 1 + autoscaling: + enabled: True + minNodeCount: 1 + maxNodeCount: 2 + management: + autoUpgrade: True + autoRepair: True + config: + localSsdCount: 1 + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring + taints: + - key: mykey1 + value: value1 + effect: NO_SCHEDULE + - key: mykey2 + value: value2 + effect: NO_EXECUTE locations: - us-east1-c - us-east1-b - autoScaling: - enabled: True - minNodeCount: 1 - maxNodeCount: 2 - management: - autoUpgrade: True - autoRepair: True masterAuth: username: password: loggingService: logging.googleapis.com monitoringService: monitoring.googleapis.com - privateCluster: True - masterIpv4CidrBlock: 172.16.0.0/28 + privateClusterConfig: + enablePrivateNodes: True + masterIpv4CidrBlock: 172.16.0.0/28 clusterIpv4Cidr: 10.0.0.0/11 ipAllocationPolicy: useIpAliases: True diff --git a/dm/templates/gke/examples/gke_zonal.yaml b/dm/templates/gke/examples/gke_zonal.yaml index 928e154146e..ab5e60c11af 100644 --- a/dm/templates/gke/examples/gke_zonal.yaml +++ b/dm/templates/gke/examples/gke_zonal.yaml @@ -21,9 +21,11 @@ resources: description: my awesome k8s cluster network: subnetwork: - nodeConfig: - oauthScopes: - - https://www.googleapis.com/auth/compute - - https://www.googleapis.com/auth/devstorage.read_only - - https://www.googleapis.com/auth/logging.write - - https://www.googleapis.com/auth/monitoring + nodePools: + - name: default + config: + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring diff --git a/dm/templates/gke/gke.py b/dm/templates/gke/gke.py index c03f65dfd48..bc79345e43a 100644 --- a/dm/templates/gke/gke.py +++ b/dm/templates/gke/gke.py @@ -20,57 +20,55 @@ def generate_config(context): resources = [] outputs = [] - project_id = context.env['project'] properties = context.properties - cluster_type = properties.get('clusterLocationType') + name = properties['cluster'].get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) propc = properties['cluster'] - name = propc.get('name') or context.env['name'] gke_cluster = { - 'name': name, + 'name': context.env['name'], 'type': '', 'properties': { + 'parent': 'projects/{}/locations/{}'.format( + project_id, + properties.get('zone', properties.get('location', properties.get('region'))) + ), 'cluster': { - 'name': - name + '-cluster', - 'initialNodeCount': - propc.get('initialNodeCount'), - 'initialClusterVersion': - propc.get('initialClusterVersion') + 'name': name, } } } - if cluster_type == 'Regional': - provider = 'gcp-types/container-v1beta1:projects.locations.clusters' - if not properties.get('region'): - raise KeyError( - "region is a required property for a {} Cluster." - .format(cluster_type) - ) - parent = 'projects/{}/locations/{}'.format( - project_id, - properties.get('region') - ) - gke_cluster['properties']['parent'] = parent - - elif cluster_type == 'Zonal': - provider = 'container.v1.cluster' - if not properties.get('zone'): - raise KeyError( - "zone is a required property for a {} Cluster." - .format(cluster_type) - ) + if properties.get('zone'): + # https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.zones.clusters + gke_cluster['type'] = 'gcp-types/container-v1beta1:projects.zones.clusters' + # TODO: remove, this is a bug gke_cluster['properties']['zone'] = properties.get('zone') - - gke_cluster['type'] = provider + else: + # https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters + gke_cluster['type'] = 'gcp-types/container-v1beta1:projects.locations.clusters' req_props = ['network', 'subnetwork'] optional_props = [ + 'initialNodeCount', + 'initialClusterVersion', 'description', 'nodeConfig', + 'nodePools', + 'privateClusterConfig', + 'binaryAuthorization', + 'binaryAuthorization', + 'networkConfig', + 'defaultMaxPodsConstraint', + 'resourceUsageExportConfig', + 'authenticatorGroupsConfig', + 'verticalPodAutoscaling', + 'tierSettings', + 'enableTpu', + 'databaseEncryption', + 'workloadIdentityConfig', 'masterAuth', 'loggingService', 'monitoringService', @@ -118,28 +116,33 @@ def generate_config(context): 'servicesIpv4Cidr' ] + initial_cluster_version = propc.get('initialClusterVersion') + less_than_112 = ( + initial_cluster_version.lower() != 'latest' and + version.parse(initial_cluster_version.split('-')[0]) < version.parse("1.12") + ) + if ( - version.parse(propc.get('initialClusterVersion').split('-')[0]) < version.parse("1.12") or - propc.get('masterAuth', {}).get('clientCertificateConfig', False) + # https://github.com/GoogleCloudPlatform/deploymentmanager-samples/issues/463 + propc.get('enableDefaultAuthOutput', False) and ( + less_than_112 or propc.get('masterAuth', {}).get('clientCertificateConfig', False) + ) ): output_props.append('clientCertificate') output_props.append('clientKey') - - if not propc.get('ipAllocationPolicy', {}).get('useIpAliases', False): - output_props.append('nodeIpv4CidrSize') for outprop in output_props: output_obj = {} output_obj['name'] = outprop ma_props = ['clusterCaCertificate', 'clientCertificate', 'clientKey'] if outprop in ma_props: - output_obj['value'] = '$(ref.' + name + \ - '.masterAuth.' + outprop + ')' + output_obj['value'] = '$(ref.' + context.env['name'] + \ + '.masterAuth.' + outprop + ')' elif outprop == 'instanceGroupUrls': - output_obj['value'] = '$(ref.' + name + \ + output_obj['value'] = '$(ref.' + context.env['name'] + \ '.nodePools[0].' + outprop + ')' else: - output_obj['value'] = '$(ref.' + name + '.' + outprop + ')' + output_obj['value'] = '$(ref.' + context.env['name'] + '.' + outprop + ')' outputs.append(output_obj) diff --git a/dm/templates/gke/gke.py.schema b/dm/templates/gke/gke.py.schema index 7ab5b8c6125..8caa93d4788 100644 --- a/dm/templates/gke/gke.py.schema +++ b/dm/templates/gke/gke.py.schema @@ -15,11 +15,19 @@ info: title: Google Kubernetes Engine (GKE) author: Sourced Group Inc. + version: 1.0.0 description: | Schema for deploying a GKE cluster. - For more information on this resource + + For more information on this resource: https://cloud.google.com/kubernetes-engine/docs + APIs endpoints used by this template: + - gcp-types/container-v1beta1:projects.locations.clusters => + https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters + - gcp-types/container-v1beta1:projects.zones.clusters => + https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.zones.clusters + imports: - path: gke.py @@ -28,30 +36,293 @@ additionalProperties: false required: - cluster +definitions: + locations: + type: array + uniqueItems: True + description: | + The list of the Google Compute Engine locations in which the cluster's + nodes should be located. + items: + type: string + initialNodeCount: + type: number + description: | + The number of nodes to create in this cluster. You must ensure that + your Compute Engine resource quota is sufficient for this number of + instances. You must also have available firewall and routes quota. + minimum: 1 + nodeConfig: + type: object + additionalProperties: false + description: Parameters used in creating the cluster's nodes. + required: + - oauthScopes + properties: + machineType: + type: string + description: | + The name of a Google Compute Engine machine type (e.g. n1-standard-1). + + If unspecified, the default machine type is n1-standard-1. + diskSizeGb: + type: number + minimum: 10 + description: | + Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. + + If unspecified, the default disk size is 100GB. + imageType: + type: string + default: cos + description: | + The image type to use for this node. Note that for a given image type, the latest version of it will be used. + enum: + - cos + - Ubuntu + oauthScopes: + type: array + uniqueItems: True + description: | + The set of Google API scopes to be made available on all of the node VMs under the "default" service account. + + The following scopes are recommended, but not required, and by default are not included: + + https://www.googleapis.com/auth/compute is required for mounting persistent storage on your nodes. + https://www.googleapis.com/auth/devstorage.read_only is required for communicating with gcr.io. + If unspecified, no scopes are added, unless Cloud Logging or Cloud Monitoring are enabled, + in which case their required scopes will be added. + items: + type: string + serviceAccount: + type: string + description: | + The Google Cloud Platform Service Account to be used by the node VMs. + If no Service Account is specified, the "default" service account is used. + metadata: + type: object + pattern: "[a-zA-Z0-9-_]+" + description: | + The metadata key/value pairs assigned to instances in the cluster. + + Keys must conform to the regexp [a-zA-Z0-9-_]+ and be less than 128 bytes in length. + These are reflected as part of a URL in the metadata server. Additionally, to avoid ambiguity, + keys must not conflict with any other metadata keys for the project or be one of the reserved keys: + "cluster-location" "cluster-name" "cluster-uid" "configure-sh" "containerd-configure-sh" "enable-os-login" + "gci-update-strategy" "gci-ensure-gke-docker" "instance-template" "kube-env" "startup-script" "user-data" + "disable-address-manager" "windows-startup-script-ps1" "common-psm1" "k8s-node-setup-psm1" "install-ssh-psm1" + "user-profile-psm1" "serial-port-logging-enable" + + Values are free-form strings, and only have meaning as interpreted by the image running in the instance. + The only restriction placed on them is that each value's size must be less than or equal to 32 KB. + + The total size of all keys and values must be less than 512 KB. + + An object containing a list of "key": value pairs. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + labels: + type: object + description: | + The map of Kubernetes labels (key/value pairs) to be applied to each node. + These will added in addition to any default label(s) that Kubernetes may apply to the node. + In case of conflict in label keys, the applied set may differ depending on the Kubernetes version -- + it's best to assume the behavior is undefined and conflicts should be avoided. For more information, + including usage and the valid values, see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + + An object containing a list of "key": value pairs. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + localSsdCount: + type: number + description: | + The number of local SSD disks to be attached to the node. + + The limit for this value is dependant upon the maximum number of disks available on a machine per zone. + See: https://cloud.google.com/compute/docs/disks/local-ssd#local_ssd_limits for more information. + tags: + type: array + uniqueItems: True + description: | + The list of instance tags applied to all nodes. Tags are used to identify valid sources or targets for + network firewalls and are specified by the client during cluster or node pool creation. + Each tag within the list must comply with RFC1035. + items: + type: string + preemptible: + type: boolean + description: | + Whether the nodes are created as preemptible VM instances. + See: https://cloud.google.com/compute/docs/instances/preemptible for more information about preemptible VM instances. + sandboxConfig: + type: object + additionalProperties: false + description: | + Sandbox configuration for this node. + required: + - sandboxType + properties: + sandboxType: + type: string + description: | + Type of the sandbox to use for the node (e.g. 'gvisor') + diskType: + type: string + description: | + Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') + + If unspecified, the default disk type is 'pd-standard' + enum: + - pd-standard + - pd-ssd + accelerators: + type: array + uniqueItems: True + description: | + A list of hardware accelerators to be attached to each node. + See https://cloud.google.com/compute/docs/gpus for more information about support for GPUs. + items: + type: object + additionalProperties: false + description: The Hardware Accelerator request object. + required: + - acceleratorCount + - acceleratorType + properties: + acceleratorCount: + type: string + description: | + The number of the accelerator cards exposed to an instance. + acceleratorType: + type: string + description: | + The accelerator type resource name. The list of supported + accelerator types can be found here + https://cloud.google.com/compute/docs/gpus/#Introduction + minCpuPlatform: + type: string + description: | + Specifies a minimum CPU platform for the VM instance. Applicable values are the friendly names of CPU platforms, + such as minCpuPlatform: "Intel Haswell" or minCpuPlatform: "Intel Sandy Bridge". + enum: + - Intel Sandy Bridge + - Intel Ivy Bridge + - Intel Haswell + - Intel Broadwell + - Intel Skylake + workloadMetadataConfig: + type: object + additionalProperties: false + description: | + The workload metadata configuration for the node. + items: + type: object + additionalProperties: false + required: + - nodeMetadata + properties: + nodeMetadata: + type: array + uniqueItems: True + description: | + Configuration that defines how to expose the node metadata to the workload running on the node. + items: + type: string + enum: + - UNSPECIFIED + - SECURE + - EXPOSE + - GKE_METADATA_SERVER + shieldedInstanceConfig: + type: object + additionalProperties: false + description: | + Shielded Instance options. + items: + type: object + additionalProperties: false + properties: + enableSecureBoot: + type: boolean + description: | + Defines whether the instance has Secure Boot enabled. + + Secure Boot helps ensure that the system only runs authentic software by verifying the digital + signature of all boot components, and halting the boot process if signature verification fails. + enableIntegrityMonitoring: + type: boolean + description: | + Defines whether the instance has integrity monitoring enabled. + + Enables monitoring and attestation of the boot integrity of the instance. The attestation is + performed against the integrity policy baseline. This baseline is initially derived from the + implicitly trusted boot image when the instance is created. + taints: + type: array + uniqueItems: True + description: | + A list of Kubernetes taints to be applied to each node. + items: + type: object + additionalProperties: false + description: The taint object's key, value, and effect. + required: + - key + - value + - effect + properties: + key: + type: string + description: | + The taint object's key. + value: + type: string + description: | + The taint object's value. + effect: + type: string + enum: + - EFFECT_UNSPECIFIED + - NO_SCHEDULE + - PREFER_NO_SCHEDULE + - NO_EXECUTE + properties: + project: + type: string + description: | + The project ID of the project containing the cluster. The + Google apps domain is prefixed if applicable. clusterLocationType: type: string - default: Zonal - description: Location type for the cluster Zonal or Regional. + description: | + Location type for the cluster Zonal or Regional. DEPRECATED enum: - Regional - Zonal region: type: string - default: us-east1 description: | - The region the cluster belongs to. Should be set when clusterLocationType - is set to Regional + The region the cluster belongs to. Should be set when clusterLocationType is set to Regional. DEPRECATED + location: + type: string + description: | + The location the cluster belongs to. zone: type: string - default: us-east1-b description: The zone the cluster belongs to. cluster: type: object + additionalProperties: false description: The cluster configuration. required: - network - subnetwork + oneOf: + - allOf: + - required: + - nodePools + - not: + required: + - initialNodeCount + - required: + - nodeConfig properties: name: type: string @@ -60,227 +331,171 @@ properties: type: string description: An optional description of the cluster. initialNodeCount: - type: number - default: 1 - description: | - The number of nodes to create in this cluster. You must ensure that - your Compute Engine resource quota is sufficient for this number of - instances. You must also have available firewall and routes quota. - minimum: 1 + $ref: '#/definitions/initialNodeCount' nodeConfig: - type: object - description: Parameters used in creating the cluster's nodes. + $ref: '#/definitions/nodeConfig' + nodePools: + type: array + uniqueItems: True + minItems: 1 + description: | + The node pools associated with this cluster. This field should not be set if "nodeConfig" or + "initialNodeCount" are specified. required: - - oauthScopes - properties: - machineType: - type: string - default: n1-standard-1 - description: | - The name of the Google Compute Engine machine type. - diskSizeGb: - type: number - default: 100 - minimum: 10 - description: | - Size of the disk attached to each node, specified in GB. - The smallest allowed disk size is 10GB. - imageType: - type: string - default: cos - description: The image type to use for the node. - enum: - - cos - - Ubuntu - oauthScopes: - type: array - description: | - The set of Google API scopes to be made available on all - of the node VMs under the "default" service account. - E.g., scopes - https://www.googleapis.com/auth/compute - https://www.googleapis.com/auth/devstorage.read_only - https://www.googleapis.com/auth/logging.write - https://www.googleapis.com/auth/monitoring - items: + - name + - config + items: + type: object + properties: + name: type: string - serviceAccount: - type: string - description: | - The GCP Service Account to be used by the node VMs. - metadata: - type: object - pattern: "[a-zA-Z0-9-_]+" - description: | - The metadata key/value pairs assigned to instances in the - cluster. Keys must conform to the regexp [a-zA-Z0-9-_]+ and be - less than 128 bytes in length. Additionally, to avoid ambiguity, - keys must neiter conflict with any other metadata keys for the - project nor be one of the reserved keys "cluster-location", - "cluster-name", "cluster-uid", "configure-sh", - "gci-update-strategy", "gci-ensure-gke-docker", - "instance-template", "kube-env", "startup-script", or - "user-data". The total size of all keys and values must be less - than 512 KB. - labels: - type: object - description: | - The map of Kubernetes labels (key/value pairs) to be applied to each - node. These are added to the default label(s) that - Kubernetes may apply to the nodes. - localSsdCount: - type: number - description: The number of local SSD disks to be attached to the node. - tags: - type: array - description: | - A list of instance tags applied to all nodes. Tags are used to - identify valid sources or targets for network firewalls, and are - specified by the client during the cluster or node pool creation. - All tags must comply with RFC1035. - items: + description: | + The name of the node pool. + config: + $ref: '#/definitions/nodeConfig' + initialNodeCount: + $ref: '#/definitions/initialNodeCount' + locations: + $ref: '#/definitions/locations' + version: type: string - preemptible: - type: boolean - default: False - description: | - Defines whether the nodes are created as preemptible VM instances. - https://cloud.google.com/compute/docs/instances/preemptible - accelerators: - type: array - description: | - A list of hardware accelerators to be attached to each node. - See https://cloud.google.com/compute/docs/gpus for more - information about support for GPUs. - items: + description: | + The version of the Kubernetes of this node. + autoscaling: type: object - description: The Hardware Accelerator request object. - required: - - acceleratorCount - - acceleratorType + additionalProperties: false + description: | + Autoscaler configuration for this NodePool. + Autoscaler is enabled only if a valid configuration is present. properties: - acceleratorCount: - type: string + enabled: + type: boolean description: | - The number of the accelerator cards exposed to an instance. - acceleratorType: - type: string + Is autoscaling enabled for this node pool. + minNodeCount: + type: integer description: | - The accelerator type resource name. The list of supported - accelerator types can be found here - https://cloud.google.com/compute/docs/gpus/#Introduction - minCpuPlatform: - type: string - description: | - The minimum CPU platform to be used by the instance. - The instance may be scheduled on the specified or newer CPU - platform. Applicable values are the friendly names of CPU - platforms, such as "Intel Haswell" or "Intel Sandy Bridge". - workloadMetadataConfig: - type: object - description: The workload metadata configuration for the node. - items: + Minimum number of nodes in the NodePool. Must be >= 1 and <= maxNodeCount. + maxNodeCount: + type: integer + description: | + Maximum number of nodes in the NodePool. Must be >= minNodeCount. There has to enough quota to scale up the cluster. + autoprovisioned: + type: boolean + description: | + Can this node pool be deleted automatically. + management: type: object - required: - - nodeMetadata + additionalProperties: false + description: | + NodeManagement configuration for this NodePool. properties: - nodeMetadata: - type: array + autoUpgrade: + type: boolean description: | - Configuration that defines how to expose the node - metadata to the workload running on the node. - items: - type: string - enum: - - UNSPECIFIED - - SECURE - - EXPOSE - taints: - type: array - description: | - A list of Kubernetes taints to be applied to each node. - items: + Whether the nodes will be automatically upgraded. + autoRepair: + type: boolean + description: | + Whether the nodes will be automatically repaired. + maxPodsConstraint: type: object - description: The taint object's key, value, and effect. - required: - - key - - value - - effect + additionalProperties: false + description: | + The constraint on the maximum number of pods that can be run simultaneously on a node in the node pool. properties: - key: - type: string - description: The taint object's key. - value: - type: string - description: The taint object's value. - effect: - type: string - enum: - - EFFECT_UNSPECIFIED - - NO_SCHEDULE - - PREFER_NO_SCHEDULE - - NO_EXECUTE + maxPodsPerNode: + type: integer + description: | + Constraint enforced on the max num of pods per node. + enableDefaultAuthOutput: + type: boolean + default: False + description: | + If clientKey/clientCertificate should be returned. masterAuth: type: object + additionalProperties: false description: | - The authentication information for accessing the master endpoint. + The authentication information for accessing the master endpoint. If unspecified, the + defaults are used: For clusters before v1.12, if masterAuth is unspecified, username will be set to "admin", + a random password will be generated, and a client certificate will be issued. properties: username: type: string description: | - The username for HTTP basic authentication to the master - endpoint. For clusters v1.6.0 and later, you can disable basic - authentication by providing an empty username. + The username to use for HTTP basic authentication to the master endpoint. For clusters v1.6.0 and later, + basic authentication can be disabled by leaving username unspecified (or setting it to the empty string). password: type: string description: | - The password to use for HTTP basic authentication to the master - endpoint. Because the master endpoint is open to the Internet, - you must create a strong password. If a password is provided, - 'username' must be also provided (non-empty). + The password to use for HTTP basic authentication to the master endpoint. Because the master endpoint + is open to the Internet, you should create a strong password. If a password is provided for cluster + creation, username must be non-empty. minLength: 16 clientCertificateConfig: type: object - description: The configuration for client certificates on the cluster. + additionalProperties: false + description: | + The configuration for client certificates on the cluster. + require: + - issueClientCertificate properties: issueClientCertificate: type: boolean + description: | + Issue a client certificate. initialClusterVersion: type: string - default: 1.9.7-gke.6 + default: latest description: | - The initial Kubernetes version for the cluster. - The version can be upgraded later; the upgrades are reflected by the - currentMasterVersion and currentNodeVersion values. + The initial Kubernetes version for this cluster. Valid versions are those found in validMasterVersions + returned by getServerConfig. The version can be upgraded over time; such upgrades are reflected + in currentMasterVersion and currentNodeVersion. + + Users may specify either explicit versions offered by Kubernetes Engine or version aliases, + which have the following behavior: + + "latest": picks the highest valid Kubernetes version + "1.X": picks the highest valid patch+gke.N patch in the 1.X version + "1.X.Y": picks the highest valid gke.N patch in the 1.X.Y version + "1.X.Y-gke.N": picks an explicit Kubernetes version + "","-": picks the default Kubernetes version loggingService: type: string default: logging.googleapis.com description: | - The logging service the cluster uses. Currently - available options - logging.googleapis.com (default) - the Google Cloud Logging service - none - no logs - If left empty, the default option is used. + The logging service the cluster should use to write logs. Currently available options: + + logging.googleapis.com - the Google Cloud Logging service. + none - no logs will be exported from the cluster. + if left as an empty string,logging.googleapis.com will be used. + enum: + - none + - logging.googleapis.com monitoringService: type: string default: monitoring.googleapis.com description: | - The monitoring service the cluster uses. - The currently available options are - monitoring.googleapis.com (default) - the Google Cloud monitoring service - none - no metrics are exported from the cluster - If left empty, the default option is used. + The monitoring service the cluster should use to write metrics. Currently available options: + + monitoring.googleapis.com - the Google Cloud Monitoring service. + none - no metrics will be exported from the cluster. + if left as an empty string, monitoring.googleapis.com will be used. + enum: + - none + - monitoring.googleapis.com network: type: string default: default description: | - The name of the Google Compute Engine network to which the cluster is - connected. If left unspecified, the default network is used. + The name of the Google Compute Engine network to which the cluster is connected. If left unspecified, + the default network will be used. On output this shows the network ID instead of the name. subnetwork: type: string description: | - The name of the Google Compute Engine subnetwork to which the - cluster is connected. + The name of the Google Compute Engine subnetwork to which the cluster is connected. + On output this shows the subnetwork ID instead of the name. clusterIpv4Cidr: type: string description: | @@ -288,28 +503,27 @@ properties: in the CIDR notation (e.g. 10.96.0.0/14). Leave blank to have one automatically chosen or specify a /14 block in 10.0.0.0/8. locations: - type: array - description: | - The list of the Google Compute Engine locations in which the cluster's - nodes should be located. - items: - type: string + $ref: '#/definitions/locations' enableKubernetesAlpha: type: boolean description: | - Specifies whether Kubernetes alpha features are enabled on the - cluster, including alpha API groups (e.g., v1beta1) and features - that may not be production-ready. + Kubernetes alpha features are enabled on this cluster. This includes alpha API groups (e.g. v1beta1) + and features that may not be production ready in the kubernetes version of the master and nodes. + The cluster has no SLA for uptime and master/node upgrades are disabled. + Alpha enabled clusters are automatically deleted thirty days after creation. resourceLabels: type: object description: | - The resource labels for the cluster to use to annotate any related GCE - resources. + The resource labels for the cluster to use to annotate any related Google Compute Engine resources. + + An object containing a list of "key": value pairs. + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. labelFingerprint: type: string description: The fingerprint of the set of labels for the cluster. legacyAbac: type: object + additionalProperties: false description: The configuration for the legacy ABAC authorization mode. required: - enabled @@ -318,33 +532,32 @@ properties: type: boolean default: False description: | - Defines whether the ABAC authorizer is enabled for this cluster. - When enabled, it identities wheter the system, including its - service accounts, nodes, and controllers, has statically granted - permissions beyond those provided by the RBAC configuration - or IAM. + Whether the ABAC authorizer is enabled for this cluster. When enabled, identities in the system, + including service accounts, nodes, and controllers, will have statically granted permissions + beyond those provided by the RBAC configuration or IAM. networkPolicy: type: object + additionalProperties: false description: | The configuration options for the NetworkPolicy feature https://kubernetes.io/docs/concepts/services-networking/networkpolicies/ properties: provider: type: array + uniqueItems: True description: The selected network policy provider. items: type: string - default: PROVIDER_UNSPECIFIED enum: - PROVIDER_UNSPECIFIED - CALICO enabled: type: boolean - default: False description: | Defines whether the network policy is enabled on the cluster. ipAllocationPolicy: type: object + additionalProperties: false description: The configuration for the cluster IP allocation. properties: useIpAliases: @@ -419,8 +632,23 @@ properties: certain kinds of network routes (with CIDR ranges that are larger than the cluster CIDR range). By default, we do not allow cluster CIDR ranges to intersect with any user-declared routes. + tpuIpv4CidrBlock: + type: string + description: | + The IP address range of the Cloud TPUs in this cluster. + If unspecified, a range will be automatically chosen with the default size. + + This field is only applicable when useIpAliases is true. + + If unspecified, the range will use the default size. + + Set to /netmask (e.g. /14) to have a range chosen with a specific netmask. + + Set to a CIDR notation (e.g. 10.96.0.0/14) from the RFC-1918 private networks + (e.g. 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) to pick a specific range to use. masterAuthorizedNetworksConfig: type: object + additionalProperties: false description: | The configuration for the master authorized networks feature. required: @@ -433,10 +661,12 @@ properties: Defines whether the master authorized networks feature is enabled. cidrBlocks: type: array + uniqueItems: True description: | A list of cidrBlocks in the CIDR notation. items: type: object + additionalProperties: false description: The CIDR block object. required: - cidrBlock @@ -449,72 +679,182 @@ properties: description: The cidrBlock in the CIDR notation. addonsConfig: type: object + additionalProperties: false description: | - Configurations for the various add-ons available to run in the cluster. + Configurations for the various addons available to run in the cluster. properties: httpLoadBalancing: type: object + additionalProperties: false description: | - Configuration options for the HTTP (L7) Load Balancing Controller - add-on, which simplifies setting up HTTP load balancers for - services in the cluster. + Configuration for the HTTP (L7) load balancing controller addon, which makes it easy to set up + HTTP load balancers for services in a cluster. + required: + - disabled properties: disabled: type: boolean - default: False description: | - Specifies whether the HTTP Load Balancing controller is - enabled in the cluster. If enabled, it runs a small pod - in the cluster that manages the load balancers. + Whether the HTTP Load Balancing controller is enabled in the cluster. + When enabled, it runs a small pod in the cluster that manages the load balancers. horizontalPodAutoscaling: type: object + additionalProperties: false description: | - Configuration options for the Horizontal Pod Autoscaling feature, - which increases or decreases the number of replica pods the - replication controller has, based on the resource usage of the - existing pods. + Configuration for the horizontal pod autoscaling feature, which increases or decreases the number + of replica pods a replication controller has based on the resource usage of the existing pods. + required: + - disabled properties: disabled: type: boolean description: | - Specifies whether the Horizontal Pod Autoscaling feature is - enabled in the cluster. When enabled, it ensures that a - Heapster pod is running in the cluster, which is also used - by the Cloud Monitoring service. + Whether the Horizontal Pod Autoscaling feature is enabled in the cluster. When enabled, it ensures + that a Heapster pod is running in the cluster, which is also used by the Cloud Monitoring service. kubernetesDashboard: type: object - description: The configuration for the Kubernetes Dashboard. + additionalProperties: false + description: | + Configuration for the Kubernetes Dashboard. This addon is deprecated, and will be disabled in 1.15. + It is recommended to use the Cloud Console to manage and monitor your Kubernetes clusters, workloads + and applications. + For more information, see: https://cloud.google.com/kubernetes-engine/docs/concepts/dashboards + required: + - disabled properties: disabled: type: boolean - default: False description: | - Defines whether the Kubernetes Dashboard is enabled for - the cluster. + Whether the Kubernetes Dashboard is enabled for this cluster. networkPolicyConfig: type: object + additionalProperties: false description: | The configuration for the NetworkPolicy add-on. This only tracks whether the add-on is enabled on the Master. It does not track whether network policy is enabled for the nodes. + required: + - disabled properties: disabled: type: boolean description: | - Defines whether the NetworkPolicy add-on is enabled for - the cluster. + Whether NetworkPolicy is enabled for this cluster. + istioConfig: + type: object + additionalProperties: false + description: | + Configuration for Istio, an open platform to connect, manage, and secure microservices. + required: + - disabled + properties: + disabled: + type: boolean + description: | + Whether Istio is enabled for this cluster. + auth: + type: string + description: | + The specified Istio auth mode, either none, or mutual TLS. + enum: + - AUTH_NONE + - AUTH_MUTUAL_TLS + cloudRunConfig: + type: object + additionalProperties: false + description: | + Configuration for the Cloud Run addon. The IstioConfig addon must be enabled in order to enable + Cloud Run addon. This option can only be enabled at cluster creation time. + required: + - disabled + properties: + disabled: + type: boolean + description: | + Whether Cloud Run addon is enabled for this cluster. + autoscaling: + type: object + additionalProperties: false + description: | + Cluster-level autoscaling configuration. + properties: + enableNodeAutoprovisioning: + type: boolean + description: | + Enables automatic node pool creation and deletion. + resourceLimits: + type: array + uniqueItems: True + description: | + Contains global constraints regarding minimum and maximum amount of resources in the cluster. + items: + type: object + additionalProperties: false + properties: + resourceType: + type: string + description: | + Resource name "cpu", "memory" or gpu-specific string. + minimum: + type: integer + description: | + Minimum amount of the resource in the cluster. + maximum: + type: integer + description: | + Maximum amount of the resource in the cluster. + autoprovisioningNodePoolDefaults: + type: object + additionalProperties: false + description: | + AutoprovisioningNodePoolDefaults contains defaults for a node pool created by NAP. + properties: + oauthScopes: + type: array + uniqueItems: True + description: | + Scopes that are used by NAP when creating node pools. + If oauthScopes are specified, serviceAccount should be empty. + items: + type: string + serviceAccount: + type: string + description: | + The Google Cloud Platform Service Account to be used by the node VMs. + If serviceAccount is specified, scopes should be empty. + autoprovisioningLocations: + type: array + uniqueItems: True + description: | + The list of Google Compute Engine zones in which the NodePool's nodes can be created by NAP. + items: + type: string + binaryAuthorization: + type: object + additionalProperties: false + description: | + Configuration for Binary Authorization. + properties: + enabled: + type: boolean + description: | + Enable Binary Authorization for this cluster. + If enabled, all container images will be validated by Google Binauthz. maintenancePolicy: type: object + additionalProperties: false description: | The configuration of the maintenance policy for the cluster. properties: window: type: object + additionalProperties: false description: | The time window within which maintenance may be performed. properties: dailyMaintenanceWindow: type: object + additionalProperties: false description: The daily maintenance operation window. properties: startTime: @@ -523,30 +863,182 @@ properties: Time within the maintenance window to start the maintenance operations. It must be in the HH:MM format, where HH 00-23 and MM 00-59 GMT. + defaultMaxPodsConstraint: + type: object + additionalProperties: false + description: | + The default constraint on the maximum number of pods that can be run simultaneously on a node in the node pool of this cluster. Only honored if cluster created with IP Alias support. + required: + - maxPodsPerNode + properties: + maxPodsPerNode: + type: integer + description: | + Constraint enforced on the max num of pods per node. + resourceUsageExportConfig: + type: object + additionalProperties: false + description: | + Configuration for exporting resource usages. Resource usage export is disabled when this config unspecified. + properties: + bigqueryDestination: + type: object + additionalProperties: false + description: | + Configuration to use BigQuery as usage export destination. + required: + - datasetId + properties: + datasetId: + type: string + description: | + The ID of a BigQuery Dataset. + enableNetworkEgressMetering: + type: boolean + description: | + Whether to enable network egress metering for this cluster. + If enabled, a daemonset will be created in the cluster to meter network egress traffic. + consumptionMeteringConfig: + type: object + additionalProperties: false + description: | + Configuration to enable resource consumption metering. + required: + - enabled + properties: + enabled: + type: boolean + description: | + Whether to enable consumption metering for this cluster. + If enabled, a second BigQuery table will be created to hold resource consumption records. podSecurityPolicyConfig: type: object - description: The configuration for the PodSecurityPolicy feature. + additionalProperties: false + description: | + The configuration for the PodSecurityPolicy feature. required: - enabled properties: enabled: type: boolean description: | - If True, enables the PodSecurityPolicy controller for the - cluster. If enabled, pods must be valid under PodSecurityPolicy - to be created. + Enable the PodSecurityPolicy controller for this cluster. + If enabled, pods must be valid under a PodSecurityPolicy to be created. + authenticatorGroupsConfig: + type: object + additionalProperties: false + description: | + Configuration controlling RBAC group membership information. + required: + - enabled + properties: + enabled: + type: boolean + description: | + Whether this cluster should return group membership lookups during authentication + using a group of security groups. + securityGroup: + type: string + description: | + The name of the security group-of-groups to be used. Only relevant if enabled = true. + privateClusterConfig: + type: object + additionalProperties: false + description: | + Configuration for private cluster. + properties: + enablePrivateNodes: + type: boolean + description: | + Whether nodes have internal IP addresses only. If enabled, all nodes are given only RFC 1918 + private addresses and communicate with the master via private networking. + enablePrivateEndpoint: + type: boolean + description: | + Whether the master's internal IP address is used as the cluster endpoint. + enablePeeringRouteSharing: + type: boolean + description: | + Whether to enable route sharing over the network peering. + masterIpv4CidrBlock: + type: string + description: | + The IP range in CIDR notation to use for the hosted master network. This range will be used + for assigning internal IP addresses to the master or set of masters, as well as the ILB VIP. + This range must not overlap with any other ranges in use within the cluster's network. + verticalPodAutoscaling: + type: object + additionalProperties: false + description: | + Cluster-level Vertical Pod Autoscaling configuration. + required: + - enabled + properties: + enabled: + type: boolean + description: | + Enables vertical pod autoscaling. + tierSettings: + type: object + additionalProperties: false + description: | + Cluster tier settings. + required: + - tier + properties: + tier: + type: string + description: | + Cluster tier. + enum: + - UNSPECIFIED + - STANDARD + - ADVANCED + workloadIdentityConfig: + type: object + additionalProperties: false + description: | + Configuration for the use of Kubernetes Service Accounts in GCP IAM policies. + properties: + identityNamespace: + type: string + description: | + IAM Identity Namespace to attach all Kubernetes Service Accounts to. + databaseEncryption: + type: object + additionalProperties: false + description: | + Configuration of etcd encryption. + properties: + state: + type: string + description: | + Denotes the state of etcd encryption. + enum: + - UNKNOWN + - ENCRYPTED + - DECRYPTED + keyName: + type: string + description: | + Name of CloudKMS key to use for the encryption of secrets in etcd. + Ex. projects/my-project/locations/global/keyRings/my-ring/cryptoKeys/my-key + enableTpu: + type: boolean + description: | + Enable the ability to use Cloud TPUs in this cluster. privateCluster: type: boolean description: | - Defines whether the cluster is private. Private clusters, - by default, have no external IP addresses on the nodes. The nodes - and the master communicate over private IP addresses. + If this is a private cluster setup. Private clusters are clusters that, by default have no + external IP addresses on the nodes and where nodes and the master communicate over private IP addresses. + This field is deprecated, use privateClusterConfig.enable_private_nodes instead. masterIpv4CidrBlock: type: string description: | - The IP prefix in the CIDR notation to use for the hosted - master network. This prefix is used for assigning private IP - addresses to the master or set of masters, as well as the ILB VIP. + The IP prefix in CIDR notation to use for the hosted master network. + This prefix will be used for assigning private IP addresses to the master or set of masters, as well as + the ILB VIP. This field is deprecated, use privateClusterConfig.master_ipv4_cidr_block instead. outputs: properties: - selfLink: diff --git a/dm/templates/gke/tests/integration/gke.yaml b/dm/templates/gke/tests/integration/gke.yaml index cc44fb4295b..8f700ad3b42 100644 --- a/dm/templates/gke/tests/integration/gke.yaml +++ b/dm/templates/gke/tests/integration/gke.yaml @@ -16,15 +16,17 @@ resources: network: ${NETWORK_NAME} subnetwork: ${SUBNET_NAME} initialClusterVersion: ${CLUSTER_VERSION} - initialNodeCount: ${NODE_COUNT} - nodeConfig: - machineType: ${MACHINE_TYPE} - oauthScopes: - - https://www.googleapis.com/auth/compute - - https://www.googleapis.com/auth/devstorage.read_only - - https://www.googleapis.com/auth/logging.write - - https://www.googleapis.com/auth/monitoring - localSsdCount: ${LOCALSSD_COUNT} + nodePools: + - name: default + initialNodeCount: ${NODE_COUNT} + config: + machineType: ${MACHINE_TYPE} + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring + localSsdCount: ${LOCALSSD_COUNT} locations: - us-east1-b - us-east1-d diff --git a/dm/templates/haproxy/examples/haproxy.yaml b/dm/templates/haproxy/examples/haproxy.yaml index 16496a9d413..5262e3aa229 100644 --- a/dm/templates/haproxy/examples/haproxy.yaml +++ b/dm/templates/haproxy/examples/haproxy.yaml @@ -6,8 +6,6 @@ # * zones/us-east1-b/instanceGroups/instance-group-3 imports: - - path: templates/instance/instance.py - name: instance.py - path: templates/haproxy/haproxy.py name: haproxy.py diff --git a/dm/templates/haproxy/haproxy.py b/dm/templates/haproxy/haproxy.py index 930f48485c1..5c3cf9c2193 100644 --- a/dm/templates/haproxy/haproxy.py +++ b/dm/templates/haproxy/haproxy.py @@ -156,7 +156,7 @@ def generate_config(context): """ Entry point for the deployment resources. """ properties = context.properties - lb_name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) zone = properties['zone'] metadata = properties.get('metadata', {'items':[]}) @@ -167,10 +167,12 @@ def generate_config(context): service_account = properties['serviceAccountEmail'] load_balancer = { - 'name': lb_name, + 'name': context.env['name'], 'type': 'instance.py', 'properties': { + 'name': properties.get('name', context.env['name']), + 'project': project_id, 'machineType': properties['machineType'], 'diskImage': DISK_IMAGE, 'zone': zone, @@ -192,19 +194,19 @@ def generate_config(context): 'outputs': [ { 'name': 'internalIp', - 'value': '$(ref.{}.internalIp)'.format(lb_name) + 'value': '$(ref.{}.internalIp)'.format(context.env['name']) }, { 'name': 'externalIp', - 'value': '$(ref.{}.externalIp)'.format(lb_name) + 'value': '$(ref.{}.externalIp)'.format(context.env['name']) }, { 'name': 'name', - 'value': '$(ref.{}.name)'.format(lb_name) + 'value': '$(ref.{}.name)'.format(context.env['name']) }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(lb_name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) }, { 'name': 'port', diff --git a/dm/templates/haproxy/haproxy.py.schema b/dm/templates/haproxy/haproxy.py.schema index 2337f292aa0..4b2f2969977 100644 --- a/dm/templates/haproxy/haproxy.py.schema +++ b/dm/templates/haproxy/haproxy.py.schema @@ -15,11 +15,14 @@ info: title: HAProxy load balancer template author: Sourced Group Inc. + version: 1.0.0 description: | Deploys a Compute instance, with an HAProxy installed and configured to load-balance traffic between instance groups. imports: + - path: ../instance/instance.py + name: instance.py - path: haproxy.py additionalProperties: false @@ -33,7 +36,11 @@ required: properties: name: type: string - description: A name of a load balancer instance + description: A name of a load balancer instance. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the service. network: type: string description: | @@ -56,6 +63,7 @@ properties: instanceGroups instances. loadBalancer: type: object + additionalProperties: false description: Front-end settings for the HAProxy load balancer. required: - port @@ -81,6 +89,7 @@ properties: - tcp instances: type: object + additionalProperties: false description: Back-end settings of the HAProxy load balancer. required: - port @@ -92,6 +101,7 @@ properties: Number of the port at which instances will accept requests. groups: type: array + uniqItems: true description: A list of instanceGroups that will be load-balanced. items: type: string diff --git a/dm/templates/haproxy/tests/integration/haproxy.bats b/dm/templates/haproxy/tests/integration/haproxy.bats index 49d5fa08540..5eeb5f15ef4 100755 --- a/dm/templates/haproxy/tests/integration/haproxy.bats +++ b/dm/templates/haproxy/tests/integration/haproxy.bats @@ -80,28 +80,58 @@ function teardown() { run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --config "${CONFIG}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "Status: $status" + echo "Output: $output" [[ "$status" -eq 0 ]] } @test "Verifying that the HAProxy instance was created in deployment ${DEPLOYMENT_NAME}" { run gcloud compute instances list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "Status: $status" + echo "Output: $output" [[ "$status" -eq 0 ]] [[ "$output" =~ "ilb-proxy-${RAND}" ]] + + # Enabling OS login for the next tests + run gcloud compute instances add-metadata "ilb-proxy-${RAND}" \ + --metadata enable-oslogin=TRUE \ + --zone us-central1-a \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "Pre-run Status: $status" + echo "Pre-run Output: $output" + + [[ "$status" -eq 0 ]] + + run gcloud compute ssh "ilb-proxy-${RAND}" --zone us-central1-a --tunnel-through-iap \ + --command "echo 'OK' " \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "SSH Status: $status" + echo "SSH Output: $output" + + echo "sleeping 30" + sleep 30 + + [[ "$status" -eq 0 ]] } @test "Verifying that haproxy.cfg was populated with instances and had all properties set" { + # Wait for the HAProxy instance to be configured. until gcloud compute instances get-serial-port-output "ilb-proxy-${RAND}" \ --zone us-central1-a \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" | grep /etc/haproxy/haproxy.cfg; do + echo "sleeping 10" sleep 10; done # Verify VM serial output - run gcloud compute ssh "ilb-proxy-${RAND}" --zone us-central1-a \ + run gcloud compute ssh "ilb-proxy-${RAND}" --zone us-central1-a --tunnel-through-iap \ --command "sudo tail -n 15 /etc/haproxy/haproxy.cfg" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "SSH Status: $status" + echo "SSH Output: $output" [[ "$status" -eq 0 ]] [[ "$output" =~ "group-${RAND}-1" ]] # has instances from group 1 [[ "$output" =~ "group-${RAND}-2" ]] # has instances from group 2 @@ -112,10 +142,12 @@ function teardown() { } @test "Verifying that update interval was set" { - run gcloud compute ssh "ilb-proxy-${RAND}" --zone us-central1-a \ + run gcloud compute ssh "ilb-proxy-${RAND}" --zone us-central1-a --tunnel-through-iap \ --command "sudo crontab -l" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "Status: $status" + echo "Output: $output" [[ "$status" -eq 0 ]] [[ "$output" = "*/15 * * * * /sbin/haproxy-conf-updater" ]] } @@ -123,5 +155,7 @@ function teardown() { @test "Deleting deployment" { run gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" -q \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "Status: $status" + echo "Output: $output" [[ "$status" -eq 0 ]] } diff --git a/dm/templates/healthcheck/examples/healthcheck.yaml b/dm/templates/healthcheck/examples/healthcheck.yaml index b2ea0c6fd5d..c44e3ee9284 100644 --- a/dm/templates/healthcheck/examples/healthcheck.yaml +++ b/dm/templates/healthcheck/examples/healthcheck.yaml @@ -15,7 +15,7 @@ resources: healthyThreshold: 2 host: my-host.testing port: 80 - healthcheck_type: HTTP + healthcheckType: HTTP - name: my-legacy-https-healthcheck-local type: healthcheck.py properties: @@ -25,7 +25,7 @@ resources: unhealthyThreshold: 2 healthyThreshold: 2 port: 80 - healthcheck_type: HTTPS + healthcheckType: HTTPS - name: my-beta-http-healthcheck-local type: healthcheck.py properties: @@ -36,7 +36,7 @@ resources: healthyThreshold: 2 host: my-host.testing port: 80 - healthcheck_type: HTTP + healthcheckType: HTTP response: my-response version: beta - name: my-beta-https-healthcheck-local @@ -49,7 +49,7 @@ resources: healthyThreshold: 2 host: my-host.testing port: 80 - healthcheck_type: HTTPS + healthcheckType: HTTPS response: my-response version: beta - name: my-beta-http2-healthcheck-local @@ -61,7 +61,7 @@ resources: unhealthyThreshold: 2 healthyThreshold: 2 port: 80 - healthcheck_type: HTTP2 + healthcheckType: HTTP2 version: beta - name: my-tcp-healthcheck-local type: healthcheck.py @@ -72,7 +72,7 @@ resources: unhealthyThreshold: 2 healthyThreshold: 2 port: 80 - healthcheck_type: TCP + healthcheckType: TCP - name: my-ssl-healthcheck-local type: healthcheck.py properties: @@ -82,7 +82,7 @@ resources: unhealthyThreshold: 2 healthyThreshold: 2 port: 80 - healthcheck_type: SSL + healthcheckType: SSL - name: my-beta-tcp-healthcheck type: healthcheck.py properties: @@ -92,7 +92,7 @@ resources: unhealthyThreshold: 2 healthyThreshold: 2 port: 80 - healthcheck_type: TCP + healthcheckType: TCP - name: my-beta-ssl-healthcheck type: healthcheck.py properties: @@ -102,7 +102,7 @@ resources: unhealthyThreshold: 2 healthyThreshold: 2 port: 80 - healthcheck_type: SSL + healthcheckType: SSL - name: my-requestpath-healthcheck-local type: healthcheck.py properties: @@ -113,7 +113,7 @@ resources: healthyThreshold: 2 proxyHeader: PROXY_V1 requestPath: /health.html - healthcheck_type: HTTPS + healthcheckType: HTTPS - name: my-response-healthcheck-local type: healthcheck.py properties: @@ -123,6 +123,6 @@ resources: unhealthyThreshold: 2 healthyThreshold: 2 proxyHeader: PROXY_V1 - healthcheck_type: TCP + healthcheckType: TCP request: request-data response: response-data diff --git a/dm/templates/healthcheck/healthcheck.py b/dm/templates/healthcheck/healthcheck.py index 661afcedda7..62d3d5d91f4 100644 --- a/dm/templates/healthcheck/healthcheck.py +++ b/dm/templates/healthcheck/healthcheck.py @@ -29,36 +29,37 @@ def generate_config(context): """ Entry point for the deployment resources. """ resources = [] - outputs = [] - healthcheck = {} properties = context.properties - healthcheck_name = context.env['name'] + healthcheck_name = properties.get('name', context.env['name']) healthcheck_type = properties['healthcheckType'] healthcheck_version = properties.get('version', 'v1') + + project_id = properties.get('project', context.env['project']) + # Deployment Manager resource types per healthcheck type. healthcheck_type_dictionary = { 'HTTP': { - 'v1': 'compute.v1.httpHealthCheck', - 'beta': 'compute.beta.httpHealthCheck' + 'v1': 'gcp-types/compute-v1:httpHealthChecks', + 'beta': 'gcp-types/compute-beta:httpHealthChecks' }, 'HTTPS': { - 'v1': 'compute.v1.httpsHealthCheck', - 'beta': 'compute.beta.httpsHealthCheck' + 'v1': 'gcp-types/compute-v1:httpsHealthChecks', + 'beta': 'gcp-types/compute-beta:httpsHealthChecks' }, 'SSL': { - 'v1': 'compute.v1.healthCheck', - 'beta': 'compute.beta.healthCheck' + 'v1': 'gcp-types/compute-v1:healthChecks', + 'beta': 'gcp-types/compute-beta:healthChecks' }, 'TCP': { - 'v1': 'compute.v1.healthCheck', - 'beta': 'compute.beta.healthCheck' + 'v1': 'gcp-types/compute-v1:healthChecks', + 'beta': 'gcp-types/compute-beta:healthChecks' }, 'HTTP2': { - 'beta': 'compute.beta.healthCheck' + 'beta': 'gcp-types/compute-beta:healthChecks' } } @@ -74,23 +75,25 @@ def generate_config(context): # Create a generic healthcheck object. healthcheck = { 'name': - healthcheck_name, + context.env['name'], 'type': healthcheck_type_dictionary[healthcheck_type][healthcheck_version] } # Create the generic healthcheck properties separately. healthcheck_properties = { - 'description': properties.get('description', - ''), 'checkIntervalSec': properties['checkIntervalSec'], 'timeoutSec': properties['timeoutSec'], 'unhealthyThreshold': properties['unhealthyThreshold'], 'healthyThreshold': properties['healthyThreshold'], 'kind': 'compute#healthCheck', - 'type': healthcheck_type + 'type': healthcheck_type, + 'project': project_id, + 'name': healthcheck_name, } + set_if_exists(healthcheck_properties, properties, 'description') + # Create a specific healthcheck object. specific_healthcheck_type = healthcheck_object_dictionary[healthcheck_type] specific_healthcheck = { @@ -125,15 +128,15 @@ def generate_config(context): outputs = [ { 'name': 'name', - 'value': '$(ref.{}.name)'.format(healthcheck_name) + 'value': '$(ref.{}.name)'.format(context.env['name']) }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(healthcheck_name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) }, { 'name': 'creationTimestamp', - 'value': '$(ref.{}.creationTimestamp)'.format(healthcheck_name) + 'value': '$(ref.{}.creationTimestamp)'.format(context.env['name']) } ] diff --git a/dm/templates/healthcheck/healthcheck.py.schema b/dm/templates/healthcheck/healthcheck.py.schema index b64bf2600cc..d5507c236a7 100644 --- a/dm/templates/healthcheck/healthcheck.py.schema +++ b/dm/templates/healthcheck/healthcheck.py.schema @@ -15,12 +15,27 @@ info: title: Healthcheck author: Sourced Group Inc. + version: 1.0.0 description: | Creates a Healthcheck resource. For more information on this resource: https://cloud.google.com/load-balancing/docs/health-checks. + APIs endpoints used by this template: + - gcp-types/compute-v1:httpHealthChecks => + https://cloud.google.com/compute/docs/reference/rest/v1/httpHealthChecks + - gcp-types/compute-v1:httpsHealthChecks => + https://cloud.google.com/compute/docs/reference/rest/v1/httpsHealthChecks + - gcp-types/compute-v1:healthChecks => + https://cloud.google.com/compute/docs/reference/rest/v1/healthChecks + - gcp-types/compute-beta:httpHealthChecks => + https://cloud.google.com/compute/docs/reference/rest/beta/httpHealthChecks + - gcp-types/compute-beta:httpsHealthChecks => + https://cloud.google.com/compute/docs/reference/rest/beta/httpsHealthChecks + - gcp-types/compute-beta:healthChecks => + https://cloud.google.com/compute/docs/reference/rest/beta/healthChecks + imports: - path: healthcheck.py @@ -30,6 +45,22 @@ required: - healthcheckType properties: + name: + type: string + description: | + Name of the resource. Provided by the client when the resource is created. The name must be 1-63 characters long, + and comply with RFC1035. Specifically, the name must be 1-63 characters long and match the regular + expression [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a lowercase letter, and all + following characters must be a dash, lowercase letter, or digit, except the last character, which cannot be a dash. + project: + type: string + description: | + The project ID of the project containing the Cloud Router instance. The + Google apps domain is prefixed if applicable. + description: + type: string + description: | + An optional description of this resource. Provide this property when you create the resource. checkIntervalSec: type: integer default: 5 diff --git a/dm/templates/healthcheck/tests/integration/cloudbuild-schema.yaml b/dm/templates/healthcheck/tests/integration/cloudbuild-schema.yaml new file mode 100644 index 00000000000..6f3bd22cd91 --- /dev/null +++ b/dm/templates/healthcheck/tests/integration/cloudbuild-schema.yaml @@ -0,0 +1,4 @@ +steps: +- name: gcr.io/$PROJECT_ID/cft-schema + args: ['./templates/healthcheck/examples/healthcheck.yaml'] +tags: ['cft-dm-schema-runner'] diff --git a/dm/templates/healthcheck/tests/integration/healthcheck.bats b/dm/templates/healthcheck/tests/integration/healthcheck.bats index f1f31203d31..23db155c9e3 100644 --- a/dm/templates/healthcheck/tests/integration/healthcheck.bats +++ b/dm/templates/healthcheck/tests/integration/healthcheck.bats @@ -142,7 +142,7 @@ function teardown() { [[ "$output" =~ "response: response-data" ]] } -@test "HTTP healthcheck was created" { +@test "HTTP beta healthcheck was created" { RESOURCE_NAME=${RESOURCE_NAME_PREFIX}-beta-http run gcloud beta compute http-health-checks describe ${RESOURCE_NAME}\ --project "${CLOUD_FOUNDATION_PROJECT_ID}" @@ -154,7 +154,7 @@ function teardown() { [[ "$output" =~ "port: ${PORT_80}" ]] } -@test "HTTPS healthcheck was created" { +@test "HTTPS beta healthcheck was created" { RESOURCE_NAME=${RESOURCE_NAME_PREFIX}-beta-https run gcloud beta compute https-health-checks describe ${RESOURCE_NAME}\ --project "${CLOUD_FOUNDATION_PROJECT_ID}" @@ -166,9 +166,9 @@ function teardown() { [[ "$output" =~ "port: 443" ]] } -@test "HTTPS healthcheck was created" { +@test "HTTPS beta http2 healthcheck was created" { RESOURCE_NAME=${RESOURCE_NAME_PREFIX}-beta-http2 - run gcloud beta compute health-checks describe ${RESOURCE_NAME}\ + run gcloud beta compute health-checks describe --global ${RESOURCE_NAME}\ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ "$status" -eq 0 ]] [[ "$output" =~ "checkIntervalSec: ${CHECK_INTERVAL_SEC}" ]] @@ -178,9 +178,9 @@ function teardown() { [[ "$output" =~ "port: ${PORT_80}" ]] } -@test "TCP healthcheck was created" { +@test "TCP beta healthcheck was created" { RESOURCE_NAME=${RESOURCE_NAME_PREFIX}-beta-tcp - run gcloud beta compute health-checks describe ${RESOURCE_NAME} \ + run gcloud beta compute health-checks describe --global ${RESOURCE_NAME} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ "$status" -eq 0 ]] [[ "$output" =~ "checkIntervalSec: ${CHECK_INTERVAL_SEC}" ]] @@ -191,9 +191,9 @@ function teardown() { [[ "$output" =~ "type: TCP" ]] } -@test "SSL healthcheck was created" { +@test "SSL beta healthcheck was created" { RESOURCE_NAME=${RESOURCE_NAME_PREFIX}-beta-ssl - run gcloud beta compute health-checks describe ${RESOURCE_NAME} \ + run gcloud beta compute health-checks describe --global ${RESOURCE_NAME} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ "$status" -eq 0 ]] [[ "$output" =~ "checkIntervalSec: ${CHECK_INTERVAL_SEC}" ]] diff --git a/dm/templates/iam_custom_role/organization_custom_role.py b/dm/templates/iam_custom_role/organization_custom_role.py index 1c954ff17ae..3062da90962 100644 --- a/dm/templates/iam_custom_role/organization_custom_role.py +++ b/dm/templates/iam_custom_role/organization_custom_role.py @@ -17,31 +17,33 @@ def generate_config(context): """ Entry point for the deployment resources. """ - org_id = str(context.properties['orgId']) - included_permissions = context.properties['includedPermissions'] + properties = context.properties + org_id = str(properties['orgId']) + included_permissions = properties['includedPermissions'] role = { 'includedPermissions': included_permissions, # Default the stage to General Availability. - 'stage': 'GA' + 'stage': properties.get('stage') } - title = context.properties.get('title') + title = properties.get('title') if title: role['title'] = title - description = context.properties.get('description') + description = properties.get('description') if description: role['description'] = description resources = [ { 'name': context.env['name'], + # https://cloud.google.com/iam/reference/rest/v1/organizations.roles 'type': 'gcp-types/iam-v1:organizations.roles', 'properties': { 'parent': 'organizations/' + org_id, - 'roleId': context.properties['roleId'], + 'roleId': properties['roleId'], 'role': role } } diff --git a/dm/templates/iam_custom_role/organization_custom_role.py.schema b/dm/templates/iam_custom_role/organization_custom_role.py.schema index a83d3363f20..5f4a8df33d2 100644 --- a/dm/templates/iam_custom_role/organization_custom_role.py.schema +++ b/dm/templates/iam_custom_role/organization_custom_role.py.schema @@ -15,6 +15,7 @@ info: title: Custom IAM Role (Organization Level) author: Sourced Group Inc. + version: 1.0.0 description: | Creates a custom organization-level IAM role under the specified organization ID. For more information on this resource: @@ -33,6 +34,10 @@ info: takes 37 days from the initial deletion request -- a new role can be created using the deleted role's ID. + APIs endpoints used by this template: + - gcp-types/iam-v1:organizations.roles => + https://cloud.google.com/iam/reference/rest/v1/organizations.roles + imports: - path: organization_custom_role.py @@ -53,17 +58,35 @@ properties: roleId: type: string pattern: ^[a-zA-Z][0-9a-zA-Z]{7,63}$ - description: A unique ID of the custom role. + description: | + A unique ID of the custom role. title: type: string - description: The title of the custom role. + description: | + The title of the custom role. description: type: string - description: Description of the custom role. + description: | + Description of the custom role. + stage: + type: string + description: | + The current launch stage of the role. If the ALPHA launch stage has been selected for a role, + the stage field will not be included in the returned definition for the role. + default: GA + enum: + - ALPHA + - BETA + - GA + - DEPRECATED + - DISABLED + - EAP includedPermissions: type: array + uniqueItems: true default: [] - description: Permissions that the custom role includes. + description: | + Permissions that the custom role includes. documentation: - templates/iam_custom_role/README.md diff --git a/dm/templates/iam_custom_role/project_custom_role.py b/dm/templates/iam_custom_role/project_custom_role.py index 465e92ce76f..4a75f2f8b2f 100644 --- a/dm/templates/iam_custom_role/project_custom_role.py +++ b/dm/templates/iam_custom_role/project_custom_role.py @@ -17,31 +17,33 @@ def generate_config(context): """ Entry point for the deployment resources. """ - project_id = context.env['project'] - included_permissions = context.properties['includedPermissions'] + properties = context.properties + included_permissions = properties['includedPermissions'] + project_id = properties.get('project', context.env['project']) role = { 'includedPermissions': included_permissions, # Default the stage to General Availability. - 'stage': 'GA' + 'stage': properties.get('stage') } - title = context.properties.get('title') + title = properties.get('title') if title: role['title'] = title - description = context.properties.get('description') + description = properties.get('description') if description: role['description'] = description resources = [ { 'name': context.env['name'], + # https://cloud.google.com/iam/reference/rest/v1/projects.roles 'type': 'gcp-types/iam-v1:projects.roles', 'properties': { 'parent': 'projects/' + project_id, - 'roleId': context.properties['roleId'], + 'roleId': properties['roleId'], 'role': role } } diff --git a/dm/templates/iam_custom_role/project_custom_role.py.schema b/dm/templates/iam_custom_role/project_custom_role.py.schema index 3e3081ae8c3..0bb82537499 100644 --- a/dm/templates/iam_custom_role/project_custom_role.py.schema +++ b/dm/templates/iam_custom_role/project_custom_role.py.schema @@ -15,6 +15,7 @@ info: title: Custom IAM Role (Project Level) author: Sourced Group Inc. + version: 1.0.0 description: | Creates a custom project-level IAM role under the specified organization ID. For more information on this resource: @@ -33,6 +34,10 @@ info: takes 37 days from the initial deletion request -- a new role can be created using the deleted role's ID. + APIs endpoints used by this template: + - gcp-types/iam-v1:projects.roles => + https://cloud.google.com/iam/reference/rest/v1/projects.roles + imports: - path: project_custom_role.py @@ -43,20 +48,42 @@ required: - includedPermissions properties: + project: + type: string + description: | + The project ID of the project to modify. roleId: type: string pattern: ^[a-zA-Z][0-9a-zA-Z]{7,63}$ - description: A unique ID of the custom role. + description: | + A unique ID of the custom role. title: type: string - description: The title of the custom role. + description: | + The title of the custom role. description: type: string - description: Description of the custom role. + description: | + Description of the custom role. + stage: + type: string + description: | + The current launch stage of the role. If the ALPHA launch stage has been selected for a role, + the stage field will not be included in the returned definition for the role. + default: GA + enum: + - ALPHA + - BETA + - GA + - DEPRECATED + - DISABLED + - EAP includedPermissions: type: array + uniqueItems: true default: [] - description: Permissions that the custom role includes. + description: | + Permissions that the custom role includes. documentation: - templates/iam_custom_role/README.md diff --git a/dm/templates/iam_member/examples/iam_member.yaml b/dm/templates/iam_member/examples/iam_member.yaml index afc581421b9..58ffa31fda5 100644 --- a/dm/templates/iam_member/examples/iam_member.yaml +++ b/dm/templates/iam_member/examples/iam_member.yaml @@ -7,6 +7,7 @@ # Replace `service-account` with a valid service account. # Replace `group-address` with a valid group. # Replace `domain-name` with a valid domain. +# Replace `folderId` with a folder ID to assign roles to. imports: @@ -14,7 +15,7 @@ imports: name: iam_member.py resources: - - name: iam-member-test + - name: iam-member-project type: iam_member.py properties: roles: @@ -26,3 +27,17 @@ resources: members: - group: - domain: + + - name: iam-member-folder + type: iam_member.py + properties: + folderId: "" + roles: + - role: roles/editor + members: + - user: + - serviceAccount: + - role: roles/viewer + members: + - group: + - domain: diff --git a/dm/templates/iam_member/iam_member.py b/dm/templates/iam_member/iam_member.py index f3bec0ca62a..d2e495c7fdf 100644 --- a/dm/templates/iam_member/iam_member.py +++ b/dm/templates/iam_member/iam_member.py @@ -13,28 +13,60 @@ # limitations under the License. """ This template creates an IAM policy member. """ +from hashlib import sha1 + def generate_config(context): """ Entry point for the deployment resources. """ - project_id = context.properties.get('projectId', context.env['project']) + properties = context.properties + folder_id = properties.get('folderId') + org_id = properties.get('organizationId') + project_id = properties.get('projectId', context.env['project']) resources = [] - for ii, role in enumerate(context.properties['roles']): - for i, member in enumerate(role['members']): - policy_get_name = 'get-iam-policy-{}-{}-{}'.format(context.env['name'], ii, i) + for role in properties['roles']: + for member in role['members']: + suffix = sha1('{}-{}'.format(role['role'], member)).hexdigest()[:10] + policy_get_name = '{}-{}'.format(context.env['name'], suffix) - resources.append( - { - 'name': policy_get_name, + if org_id: + resources.append({ + 'name': '{}-organization'.format(policy_get_name), + # TODO - Virtual type documentation needed + 'type': 'gcp-types/cloudresourcemanager-v1:virtual.organizations.iamMemberBinding', + 'properties': { + 'resource': org_id, + 'role': role['role'], + 'member': member, + } + }) + elif folder_id: + resources.append({ + 'name': '{}-folder'.format(policy_get_name), + # TODO - Virtual type documentation needed + 'type': 'gcp-types/cloudresourcemanager-v2:virtual.folders.iamMemberBinding', + 'properties': { + 'resource': folder_id, + 'role': role['role'], + 'member': member, + } + }) + else: + resources.append({ + 'name': '{}-project'.format(policy_get_name), + # TODO - Virtual type documentation needed 'type': 'gcp-types/cloudresourcemanager-v1:virtual.projects.iamMemberBinding', - 'properties': - { + 'properties': { 'resource': project_id, 'role': role['role'], - 'member': member + 'member': member, } - } - ) + }) + + + if 'dependsOn' in properties: + for resource in resources: + resource['metadata'] = {'dependsOn': properties['dependsOn']} return {"resources": resources} diff --git a/dm/templates/iam_member/iam_member.py.schema b/dm/templates/iam_member/iam_member.py.schema index 721e5e77c09..6bcb1d76dc0 100644 --- a/dm/templates/iam_member/iam_member.py.schema +++ b/dm/templates/iam_member/iam_member.py.schema @@ -15,7 +15,20 @@ info: title: IAM policy member author: Sourced Group Inc. - description: Manages an IAM policy member + version: 1.0.0 + description: | + Manages an IAM policy member + + For more information on this resource: + https://cloud.google.com/iam/docs/overview + + APIs endpoints used by this template: + - gcp-types/cloudresourcemanager-v1:virtual.projects.iamMemberBinding => + TODO - Virtual type documentation needed + - gcp-types/cloudresourcemanager-v2:virtual.folders.iamMemberBinding => + TODO - Virtual type documentation needed + - gcp-types/cloudresourcemanager-v1:virtual.organizations.iamMemberBinding => + TODO - Virtual type documentation needed imports: - path: iam_member.py @@ -25,37 +38,75 @@ additionalProperties: false required: - roles +oneOf: + - required: + - folderId + - required: + - organizationId + - required: + - projectId + - allOf: + - not: + required: + - folderId + - not: + required: + - organizationId + properties: + folderId: + type: string + description: | + Folder ID to assign members to. + organizationId: + type: string + description: | + Organization ID to assign members to. projectId: type: string description: | Overwrite of project ID in case IAM bindings are referencing to - a different project. + a different project or if you need to assign members to folders/organizations as well. roles: type: array - description: An array of roles and members. + uniqueItems: true + minItems: 1 + description: | + An array of roles and members. items: - role: - type: string - description: The role to grant to members. - members: - type: array - description: A list of identities. - items: + type: object + additionalProperties: false + properties: + role: type: string description: | - Specifies the identity requesting access to a Cloud Platform - resource. Can have the following values: - - user:{emailid} - An email address that represents a specific - IAM User account. For example, user:name@example.com - - serviceAccount:{emailid} - An email address that represents a - Service Account. For example, - serviceAccount:my-other-app@appspot.gserviceaccount.com - - group:{emailid} - An email address that represents a Google group. - For example, group:admins@example.com - - domain:{domain} - A Cloud Identity or G Suite domain name that - represents all the users of that domain. For example, acme.com - or example.com. + The role to grant to members. + members: + type: array + description: | + A list of identities. + items: + type: string + description: | + Specifies the identity requesting access to a Cloud Platform + resource. Can have the following values: + - user:{emailid} - An email address that represents a specific + IAM User account. For example, user:name@example.com + - serviceAccount:{emailid} - An email address that represents a + Service Account. For example, + serviceAccount:my-other-app@appspot.gserviceaccount.com + - group:{emailid} - An email address that represents a Google group. + For example, group:admins@example.com + - domain:{domain} - A Cloud Identity or G Suite domain name that + represents all the users of that domain. For example, acme.com + or example.com. + dependsOn: + type: array + description: | + The list of the resources that must be created before this template is applied. + items: + type: string + description: The resource name. documentation: - templates/iam_member/README.md diff --git a/dm/templates/iam_member/tests/integration/iam_member.bats b/dm/templates/iam_member/tests/integration/iam_member.bats index 1ce77a908a6..576468f9896 100755 --- a/dm/templates/iam_member/tests/integration/iam_member.bats +++ b/dm/templates/iam_member/tests/integration/iam_member.bats @@ -50,7 +50,7 @@ function teardown() { # Global teardown; this is executed once per test file. if [[ "$BATS_TEST_NUMBER" -eq "${#BATS_TEST_NAMES[@]}" ]]; then gcloud iam service-accounts delete "${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com" \ - --project "${CLOUD_FOUNDATION_PROJECT_ID}" + --quiet --project "${CLOUD_FOUNDATION_PROJECT_ID}" delete_config rm -f "${RANDOM_FILE}" fi @@ -65,7 +65,7 @@ function teardown() { --project "${CLOUD_FOUNDATION_PROJECT_ID}" } -@test "Verifying that roles were assigned in deployment ${DEPLOYMENT_NAME}" { +@test "Verifying that roles were assigned to project in deployment ${DEPLOYMENT_NAME}" { run gcloud projects get-iam-policy "${CLOUD_FOUNDATION_PROJECT_ID}" \ --flatten="bindings[].members" \ --format='table(bindings.role)' \ @@ -74,6 +74,21 @@ function teardown() { [[ "$output" =~ "roles/viewer" ]] } +@test "Verifying that roles were assigned to folder in deployment ${DEPLOYMENT_NAME}" { + # Get the test folder ID and make it available. + TEST_ORG_FOLDER_NAME=$(gcloud alpha resource-manager folders list \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" \ + --organization "${CLOUD_FOUNDATION_ORGANIZATION_ID}" | \ + grep "org-folder-${RAND}" | awk '{print $3}') + run gcloud alpha resource-manager folders get-iam-policy "folders/${TEST_ORG_FOLDER_NAME}" \ + --flatten="bindings[].members" \ + --format='table(bindings.role)' \ + --filter="bindings.members:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com" + + [[ "$output" =~ "roles/editor" ]] + [[ "$output" =~ "roles/viewer" ]] +} + @test "Deleting deployment" { gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" -q diff --git a/dm/templates/iam_member/tests/integration/iam_member.yaml b/dm/templates/iam_member/tests/integration/iam_member.yaml index 3ffcc81ac21..2284e8ceaf0 100644 --- a/dm/templates/iam_member/tests/integration/iam_member.yaml +++ b/dm/templates/iam_member/tests/integration/iam_member.yaml @@ -10,7 +10,7 @@ imports: name: iam_member.py resources: - - name: iam-member-test-${RAND} + - name: iam-member-test-project-${RAND} type: iam_member.py properties: roles: @@ -20,3 +20,21 @@ resources: - role: roles/viewer members: - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com + - name: iam-member-test-folder-${RAND} + type: iam_member.py + properties: + folderId: $(ref.test-folder-${RAND}.name) + roles: + - role: roles/editor + members: + - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com + - role: roles/viewer + members: + - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com + + - name: test-folder-${RAND} + type: gcp-types/cloudresourcemanager-v2:folders + properties: + name: org-folder-${RAND} + parent: organizations/${CLOUD_FOUNDATION_ORGANIZATION_ID} + displayName: org-folder-${RAND} diff --git a/dm/templates/instance/examples/instance.yaml b/dm/templates/instance/examples/instance.yaml index 3c684f806db..12ce0881245 100644 --- a/dm/templates/instance/examples/instance.yaml +++ b/dm/templates/instance/examples/instance.yaml @@ -15,7 +15,9 @@ resources: machineType: f1-micro diskType: pd-ssd networks: - - name: default + - network: default + accessConfigs: + - type: ONE_TO_ONE_NAT metadata: items: - key: startup-script diff --git a/dm/templates/instance/instance.py b/dm/templates/instance/instance.py index bce8c8dd6ee..4ea5124bf2e 100644 --- a/dm/templates/instance/instance.py +++ b/dm/templates/instance/instance.py @@ -49,36 +49,32 @@ def get_network_interfaces(properties): """ network_interfaces = [] - networks = properties.get('networks', [{ - "name": properties.get('network'), - "hasExternalIp": properties.get('hasExternalIp'), - "natIP": properties.get('natIP'), - "subnetwork": properties.get('subnetwork'), - "networkIP": properties.get('networkIP'), - }]) + networks = properties.get('networks', []) + if len(networks) == 0 and properties.get('network'): + network = { + "network": properties.get('network'), + "subnetwork": properties.get('subnetwork'), + "networkIP": properties.get('networkIP'), + } + networks.append(network) + if (properties.get('hasExternalIp')): + network['accessConfigs'] = [{ + "type": "ONE_TO_ONE_NAT", + }] + if properties.get('natIP'): + network['accessConfigs'][0]['natIP'] = properties.get('natIP') for network in networks: - if not '.' in network['name'] and not '/' in network['name']: - network_name = 'global/networks/{}'.format(network['name']) + if not '.' in network['network'] and not '/' in network['network']: + network_name = 'global/networks/{}'.format(network['network']) else: - network_name = network['name'] + network_name = network['network'] network_interface = { 'network': network_name, } - if network['hasExternalIp']: - access_configs = { - 'name': 'External NAT', - 'type': 'ONE_TO_ONE_NAT' - } - - if network.get('natIP'): - access_configs['natIP'] = network['natIP'] - - network_interface['accessConfigs'] = [access_configs] - - netif_optional_props = ['subnetwork', 'networkIP'] + netif_optional_props = ['subnetwork', 'networkIP', 'aliasIpRanges', 'accessConfigs'] for prop in netif_optional_props: if network.get(prop): network_interface[prop] = network[prop] @@ -90,40 +86,79 @@ def get_network_interfaces(properties): def generate_config(context): """ Entry point for the deployment resources. """ - zone = context.properties['zone'] - vm_name = context.properties.get('name', context.env['name']) - machine_type = context.properties['machineType'] + properties = context.properties + zone = properties['zone'] + vm_name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) + machine_type = properties['machineType'] - boot_disk = create_boot_disk(context.properties, zone, vm_name) - network_interfaces = get_network_interfaces(context.properties) + network_interfaces = get_network_interfaces(properties) instance = { - 'name': vm_name, - 'type': 'compute.v1.instance', + 'name': context.env['name'], + # https://cloud.google.com/compute/docs/reference/rest/v1/instances + 'type': 'gcp-types/compute-v1:instances', 'properties':{ + 'name': vm_name, 'zone': zone, + 'project': project_id, 'machineType': 'zones/{}/machineTypes/{}'.format(zone, machine_type), - 'disks': [boot_disk], 'networkInterfaces': network_interfaces } } - for name in ['metadata', 'serviceAccounts', 'canIpForward', 'tags']: - set_optional_property(instance['properties'], context.properties, name) + optionalProperties = [ + 'description', + 'scheduling', + 'disks', + 'minCpuPlatform', + 'guestAccelerators', + 'deletionProtection', + 'hostname', + 'shieldedInstanceConfig', + 'shieldedInstanceIntegrityPolicy', + 'labels', + 'metadata', + 'serviceAccounts', + 'canIpForward', + 'tags', + ] + for name in optionalProperties: + set_optional_property(instance['properties'], properties, name) + + if not properties.get('disks'): + instance['properties']['disks'] = [create_boot_disk(properties, zone, vm_name)] outputs = [ { 'name': 'networkInterfaces', - 'value': '$(ref.{}.networkInterfaces)'.format(vm_name) + 'value': '$(ref.{}.networkInterfaces)'.format(context.env['name']) }, { 'name': 'name', - 'value': '$(ref.{}.name)'.format(vm_name) + 'value': '$(ref.{}.name)'.format(context.env['name']) }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(vm_name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) } ] + if len(network_interfaces) == 1: + outputs.append({ + 'name': 'internalIp', + 'value': '$(ref.{}.networkInterfaces[0].networkIP)'.format(context.env['name']) + }) + + if 'accessConfigs' in network_interfaces[0]: + accessConfigs = network_interfaces[0]['accessConfigs'] + for i, row in enumerate(accessConfigs, 0): + if row['type'] == 'ONE_TO_ONE_NAT': + outputs.append({ + 'name': 'externalIp', + 'value': '$(ref.{}.networkInterfaces[0].accessConfigs[{}].natIP)'.format(context.env['name'], i) + }) + break + + return {'resources': [instance], 'outputs': outputs} diff --git a/dm/templates/instance/instance.py.schema b/dm/templates/instance/instance.py.schema index 48737c20c11..818d7e55ba1 100644 --- a/dm/templates/instance/instance.py.schema +++ b/dm/templates/instance/instance.py.schema @@ -15,9 +15,17 @@ info: title: Compute Instance author: Sourced Group Inc. + version: 1.0.0 description: | Deploys a Compute Instance connected to a custom (or default) network. + For more information on this resource: + https://cloud.google.com/compute/ + + APIs endpoints used by this template: + - gcp-types/compute-v1:instances => + https://cloud.google.com/compute/docs/reference/rest/v1/instances + imports: - path: instance.py @@ -71,41 +79,65 @@ definitions: specify a static external IP address, it must live in the same region as the zone of the instance. If hasExternalIp is false this field is ignored. + network: + type: string + description: | + URL of the network resource for this instance. When creating an instance, if neither the network + nor the subnetwork is specified, the default network global/networks/default is used; + if the network is not specified but the subnetwork is specified, the network is inferred. + + If you specify this property, you can specify the network as a full or partial URL. + For example, the following are all valid URLs: + + - https://www.googleapis.com/compute/v1/projects/project/global/networks/network + - projects/project/global/networks/network + - global/networks/default + Authorization requires one or more of the following Google IAM permissions on the specified resource network: + + - compute.networks.use + - compute.networks.useExternalIp subnetwork: type: string description: | - The URL of the Subnetwork resource for this instance. If the network - resource is in legacy mode, do not provide this property. If the network - is in auto subnet mode, providing the subnetwork is optional. If the - network is in custom subnet mode, then this field should be specified. - If you specify this property, you can specify the subnetwork as a full - or partial URL. For example, the following are all valid URLs: - - https://www.googleapis.com/compute/v1/projects/project/regions/region/subnetworks/subnetwork - - regions/region/subnetworks/subnetwork + The URL of the Subnetwork resource for this instance. If the network resource is in legacy mode, + do not specify this field. If the network is in auto subnet mode, specifying the subnetwork is optional. + If the network is in custom subnet mode, specifying the subnetwork is required. + If you specify this field, you can specify the subnetwork as a full or partial URL. For example, the following are all valid URLs: + + - https://www.googleapis.com/compute/v1/projects/project/regions/region/subnetworks/subnetwork + - regions/region/subnetworks/subnetwork + Authorization requires one or more of the following Google IAM permissions on the specified resource subnetwork: + + - compute.subnetworks.use + - compute.subnetworks.useExternalIp networkIP: type: string description: | - An IPv4 internal network address to assign to the instance for this - network interface. If not specified by the user, an unused internal IP - is assigned by the system. + An IPv4 internal IP address to assign to the instance for this network interface. + If not specified by the user, an unused internal IP is assigned by the system. properties: name: type: string - description: The name of the Instance resource. - network: + description: The name of the Instance resource. Resource name would be used if omitted. + project: type: string description: | - Name of the network the instance will be connected to; - e.g., 'my-custom-network' or 'default'. - hasExternalIp: - $ref: '#/definitions/hasExternalIp' - natIP: - $ref: '#/definitions/natIP' + The project ID of the project containing the instance. + description: + type: string + description: | + An optional description of this resource. Provide this property when you create the resource. + network: + $ref: '#/definitions/network' subnetwork: $ref: '#/definitions/subnetwork' networkIP: $ref: '#/definitions/networkIP' + hasExternalIp: + $ref: '#/definitions/hasExternalIp' + natIP: + $ref: '#/definitions/natIP' networks: type: array description: | @@ -115,26 +147,89 @@ properties: type: object additionalProperties: false required: - - name + - network properties: - name: - type: string - description: | - Name of the network the instance will be connected to; - e.g., 'my-custom-network' or 'default'. - hasExternalIp: - $ref: '#/definitions/hasExternalIp' - natIP: - $ref: '#/definitions/natIP' + network: + $ref: '#/definitions/network' subnetwork: $ref: '#/definitions/subnetwork' networkIP: $ref: '#/definitions/networkIP' + aliasIpRanges: + type: array + uniqueItems: true + description: | + An array of alias IP ranges for this network interface. You can only specify this + field for network interfaces in VPC networks. + items: + type: object + additionalProperties: false + properties: + ipCidrRange: + type: string + description: | + The IP alias ranges to allocate for this interface. This IP CIDR range must belong + to the specified subnetwork and cannot contain IP addresses reserved by system or + used by other network interfaces. This range may be a single IP address (such as 10.2.3.4), + a netmask (such as /24) or a CIDR-formatted string (such as 10.1.2.0/24). + subnetworkRangeName: + type: string + description: | + The name of a subnetwork secondary IP range from which to allocate an IP alias range. + If not specified, the primary range of the subnetwork is used. + accessConfigs: + type: array + uniqueItems: true + description: | + An array of configurations for this interface. Currently, only one access config, ONE_TO_ONE_NAT, + is supported. If there are no accessConfigs specified, then this instance will have no external internet access. + items: + type: object + additionalProperties: false + properties: + type: + type: string + description: | + The type of configuration. The default and only option is ONE_TO_ONE_NAT. + enum: + - ONE_TO_ONE_NAT + name: + type: string + description: | + The name of this access configuration. The default and recommended name is External NAT, + but you can use any arbitrary string, such as My external IP or Network Access. + setPublicPtr: + type: boolean + description: | + Specifies whether a public DNS 'PTR' record should be created to map the external + IP address of the instance to a DNS domain name. + publicPtrDomainName: + type: string + description: | + The DNS domain name for the public PTR record. You can set this field only + if the setPublicPtr field is enabled. + networkTier: + type: string + description: | + This signifies the networking tier used for configuring this access configuration + and can only take the following values: PREMIUM, STANDARD. + + If an AccessConfig is specified without a valid external IP address, an + ephemeral IP will be created with this networkTier. + + If an AccessConfig with a valid external IP address is specified, it must match + that of the networkTier associated with the Address resource owning that IP. + enum: + - STANDARD + - PREMIUM + natIP: + $ref: '#/definitions/natIP' zone: type: string description: Availability zone. E.g. 'us-central1-a' tags: type: object + additionalProperties: false description: | Tags to apply to this instance. Tags are used to identify valid sources or targets for network firewalls and are specified by the client during @@ -144,6 +239,7 @@ properties: properties: items: type: array + uniqueItems: true description: | An array of tags. Each tag must be 1-63 characters long, and comply with RFC1035. @@ -154,6 +250,271 @@ properties: description: | The Compute Instance type; e.g., 'n1-standard-1'. See https://cloud.google.com/compute/docs/machine-types for details. + disks: + type: array + uniqueItems: true + description: | + Array of disks associated with this instance. Persistent disks must be created before you can assign them. + items: + type: object + additionalProperties: false + oneOf: + - required: + - source + - required: + - initializeParams + - allOf: + - not: + required: + - source + - not: + required: + - initializeParams + properties: + type: + type: string + description: | + Specifies the type of the disk, either SCRATCH or PERSISTENT. If not specified, the default is PERSISTENT. + enum: + - SCRATCH + - PERSISTENT + mode: + type: string + description: | + The mode in which to attach this disk, either READ_WRITE or READ_ONLY. + If not specified, the default is to attach the disk in READ_WRITE mode. + enum: + - READ_WRITE + - READ_ONLY + source: + type: string + description: | + Specifies a valid partial or full URL to an existing Persistent Disk resource. + When creating a new instance, one of initializeParams.sourceImage or + disks.source is required except for local SSD. + + If desired, you can also attach existing non-root persistent disks using this property. + This field is only applicable for persistent disks. + + Note that for InstanceTemplate, specify the disk name, not the URL for the disk. + + Authorization requires one or more of the following Google IAM permissions on the specified resource source: + + compute.disks.use + compute.disks.useReadOnly + deviceName: + type: string + description: | + Specifies a unique device name of your choice that is reflected into the /dev/disk/by-id/google-* + tree of a Linux operating system running within the instance. This name can be used to reference + the device for mounting, resizing, and so on, from within the instance. + + If not specified, the server chooses a default device name to apply to this disk, in the + form persistent-disk-x, where x is a number assigned by Google Compute Engine. + This field is only applicable for persistent disks. + boot: + type: boolean + description: | + Indicates that this is a boot disk. The virtual machine will use the first partition + of the disk for its root filesystem. + initializeParams: + type: object + additionalProperties: false + description: | + Specifies the parameters for a new disk that will be created alongside the new instance. + Use initialization parameters to create boot disks or local SSDs attached to the new instance. + + This property is mutually exclusive with the source property; you can only define one or the other, but not both. + properties: + labels: + type: object + description: | + Labels to apply to this disk. These can be later modified by the disks.setLabels method. + This field is only applicable for persistent disks. + + An object containing a list of "key": value pairs. + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + + Authorization requires the following Google IAM permission on the specified resource labels: + + compute.disks.setLabels + diskName: + type: string + description: | + Specifies the disk name. If not specified, the default is to use the name of the instance. + If the disk with the instance name exists already in the given zone/region, + a new name will be automatically generated. + sourceImage: + type: string + description: | + The source image to create this disk. When creating a new instance, one of + initializeParams.sourceImage or disks.source is required except for local SSD. + + To create a disk with one of the public operating system images, specify the image by its family name. + For example, specify family/debian-9 to use the latest Debian 9 image: + + projects/debian-cloud/global/images/family/debian-9 + + Alternatively, use a specific version of a public operating system image: + + projects/debian-cloud/global/images/debian-9-stretch-vYYYYMMDD + + To create a disk with a custom image that you created, specify the image name in the following format: + + global/images/my-custom-image + + You can also specify a custom image by its image family, which returns the latest version of the + image in that family. Replace the image name with family/family-name: + + global/images/family/my-image-family + + If the source image is deleted later, this field will not be set. + + Authorization requires the following Google IAM permission on the specified resource sourceImage: + + compute.images.useReadOnly + description: + type: string + description: | + An optional description. Provide this property when creating the disk. + diskSizeGb: + type: number + description: | + Specifies the size of the disk in base-2 GB. + diskType: + type: string + description: | + Specifies the disk type to use to create the instance. If not specified, the default is pd-standard, + specified using the full URL. For example: + + https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/pd-standard + + Other values include pd-ssd and local-ssd. If you define this field, you can provide either the full + or partial URL. For example, the following are valid values: + + https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/diskType + projects/project/zones/zone/diskTypes/diskType + zones/zone/diskTypes/diskType + Note that for InstanceTemplate, this is the name of the disk type, not URL. + enum: + - pd-standard + - pd-ssd + - local-ssd + sourceImageEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source image. Required if the source image is + protected by a customer-supplied encryption key. + + Instance templates do not store customer-supplied encryption keys, so you cannot create disks + for instances in a managed instance group if the source images are encrypted with your own keys. + properties: + rawKey: + type: string + description: | + Specifies a 256-bit customer-supplied encryption key, encoded in RFC 4648 base64 + to either encrypt or decrypt this resource. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. + sourceSnapshot: + type: string + description: | + The source snapshot to create this disk. When creating a new instance, one of + initializeParams.sourceSnapshot or disks.source is required except for local SSD. + + To create a disk with a snapshot that you created, specify the snapshot name in the following format: + + global/snapshots/my-backup + + If the source snapshot is deleted later, this field will not be set. + + Authorization requires the following Google IAM permission on the specified resource sourceSnapshot: + + compute.snapshots.useReadOnly + sourceSnapshotEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source snapshot. + properties: + rawKey: + type: string + description: | + Specifies a 256-bit customer-supplied encryption key, encoded in RFC 4648 base64 + to either encrypt or decrypt this resource. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. + autoDelete: + type: boolean + description: | + Specifies whether the disk will be auto-deleted when the instance is deleted + (but not when the disk is detached from the instance). + interface: + type: string + description: | + Specifies the disk interface to use for attaching this disk, which is either SCSI or NVME. + The default is SCSI. Persistent disks must always use SCSI and the request will fail if you + attempt to attach a persistent disk in any other format than SCSI. Local SSDs can use either NVME or SCSI. + For performance characteristics of SCSI over NVMe, see Local SSD performance. + enum: + - SCSI + - NVME + guestOsFeatures: + type: array + uniqueItems: true + description: | + A list of features to enable on the guest operating system. Applicable only for bootable images. + Read Enabling guest operating system features to see a list of available options. + items: + type: object + additionalProperties: false + properties: + type: + type: string + description: | + https://cloud.google.com/compute/docs/images/create-delete-deprecate-private-images#guest-os-features + The ID of a supported feature. Read Enabling guest operating system features + to see a list of available options. + enum: + - MULTI_IP_SUBNET + - SECURE_BOOT + - UEFI_COMPATIBLE + - VIRTIO_SCSI_MULTIQUEUE + - WINDOWS + diskEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source snapshot. + properties: + rawKey: + type: string + description: | + Encrypts or decrypts a disk using a customer-supplied encryption key. + + If you are creating a new disk, this field encrypts the new disk using an encryption + key that you provide. If you are attaching an existing disk that is already encrypted, + this field decrypts the disk using the customer-supplied encryption key. + + If you encrypt a disk using a customer-supplied key, you must provide the same key again when + you attempt to use this resource at a later time. For example, you must provide the key when + you create a snapshot or an image from the disk or when you attach the disk + to a virtual machine instance. + + If you do not provide an encryption key, then the disk will be encrypted using an automatically + generated key and you do not need to provide a key to use the disk later. + + Instance templates do not store customer-supplied encryption keys, so you cannot use your own keys + to encrypt disks in a managed instance group. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. canIpForward: type: boolean default: False @@ -181,8 +542,116 @@ properties: diskSizeGb: type: integer minimum: 10 + scheduling: + type: object + additionalProperties: false + description: | + Sets the scheduling options for this instance. + properties: + onHostMaintenance: + type: string + description: | + Defines the maintenance behavior for this instance. For standard instances, the default behavior is MIGRATE. + For preemptible instances, the default and only possible behavior is TERMINATE. + For more information, see Setting Instance Scheduling Options. + enum: + - MIGRATE + - TERMINATE + automaticRestart: + type: boolean + description: | + Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine + (not terminated by a user). You can only set the automatic restart option for standard instances. + Preemptible instances cannot be automatically restarted. + + By default, this is set to true so an instance is automatically restarted if it is terminated by Compute Engine. + preemptible: + type: boolean + description: | + Defines whether the instance is preemptible. This can only be set during instance creation, + it cannot be set or changed after the instance has been created. + nodeAffinities: + type: array + uniqueItems: true + description: | + A set of node affinity and anti-affinity. + items: + type: object + additionalProperties: false + properties: + key: + type: string + description: | + Corresponds to the label key of Node resource. + operator: + type: string + description: | + Defines the operation of node selection. + values: + type: array + uniqueItems: true + description: | + Corresponds to the label values of Node resource. + items: + type: string + deletionProtection: + type: boolean + description: | + Whether the resource should be protected against deletion. + + Authorization requires the following Google IAM permission on the specified resource deletionProtection: + + compute.instances.setDeletionProtection + hostname: + type: string + labels: + type: object + description: | + Labels to apply to this instance. These can be later modified by the setLabels method. + + An object containing a list of "key": value pairs. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + + Authorization requires the following Google IAM permission on the specified resource labels: + + compute.instances.setLabels + minCpuPlatform: + type: string + description: | + Specifies a minimum CPU platform for the VM instance. Applicable values are the friendly names of CPU platforms, + such as minCpuPlatform: "Intel Haswell" or minCpuPlatform: "Intel Sandy Bridge". + enum: + - Intel Sandy Bridge + - Intel Ivy Bridge + - Intel Haswell + - Intel Broadwell + - Intel Skylake + shieldedInstanceConfig: + type: object + additionalProperties: false + properties: + enableSecureBoot: + type: boolean + description: | + Defines whether the instance has Secure Boot enabled. + enableVtpm: + type: boolean + description: | + Defines whether the instance has the vTPM enabled. + enableIntegrityMonitoring: + type: boolean + description: | + Defines whether the instance has integrity monitoring enabled. + shieldedInstanceIntegrityPolicy: + type: object + additionalProperties: false + properties: + updateAutoLearnPolicy: + type: boolean + description: | + Updates the integrity policy baseline using the measurements from the VM instance's most recent boot. metadata: type: object + additionalProperties: false required: - items description: | @@ -194,7 +663,9 @@ properties: properties: items: type: array - description: A collection of metadata key-value pairs. + uniqueItems: true + description: | + A collection of metadata key-value pairs. items: type: object additionalProperties: false @@ -205,6 +676,7 @@ properties: type: [string, number, boolean] serviceAccounts: type: array + uniqueItems: true description: | A list of service accounts, with their specified scopes, authorized for this instance. Only one service account per VM instance is supported. @@ -214,16 +686,37 @@ properties: properties: email: type: string - description: Email address of the service account + description: | + Email address of the service account scopes: type: array - description: The list of scopes to be made available for this service account + description: | + The list of scopes to be made available for this service account items: type: string description: | Access scope, e.g. 'https://www.googleapis.com/auth/compute.readonly' Visit https://cloud.google.com/compute/docs/access/service-accounts#accesscopesiam for more details + guestAccelerators: + type: array + uniqueItems: true + description: | + A list of the type and count of accelerator cards attached to the instance. + items: + type: object + additionalProperties: false + properties: + acceleratorType: + type: string + description: | + Full or partial URL of the accelerator type resource to attach to this instance. For example: projects/my-project/zones/us-central1-c/acceleratorTypes/nvidia-tesla-p100 + If you are creating an instance template, specify only the accelerator name. + See GPUs on Compute Engine for a full list of accelerator types. + acceleratorCount: + type: integer + description: | + The number of the guest accelerator cards exposed to this instance. outputs: properties: diff --git a/dm/templates/instance/tests/integration/instance_1_nic.yaml b/dm/templates/instance/tests/integration/instance_1_nic.yaml index ffb660a3566..79f64593a03 100644 --- a/dm/templates/instance/tests/integration/instance_1_nic.yaml +++ b/dm/templates/instance/tests/integration/instance_1_nic.yaml @@ -18,7 +18,7 @@ resources: diskType: pd-ssd canIpForward: true networks: - - name: $(ref.test-network-0-${RAND}.selfLink) + - network: $(ref.test-network-0-${RAND}.selfLink) subnetwork: $(ref.test-subnetwork-0-${RAND}.selfLink) metadata: items: diff --git a/dm/templates/instance/tests/integration/instance_2_nics.yaml b/dm/templates/instance/tests/integration/instance_2_nics.yaml index ffbc4a88cd8..7f99a8d28a4 100644 --- a/dm/templates/instance/tests/integration/instance_2_nics.yaml +++ b/dm/templates/instance/tests/integration/instance_2_nics.yaml @@ -18,9 +18,9 @@ resources: diskType: pd-ssd canIpForward: true networks: - - name: $(ref.test-network-0-${RAND}.selfLink) + - network: $(ref.test-network-0-${RAND}.selfLink) subnetwork: $(ref.test-subnetwork-0-${RAND}.selfLink) - - name: $(ref.test-network-1-${RAND}.selfLink) + - network: $(ref.test-network-1-${RAND}.selfLink) subnetwork: $(ref.test-subnetwork-1-${RAND}.selfLink) metadata: items: diff --git a/dm/templates/instance_template/examples/instance_template.yaml b/dm/templates/instance_template/examples/instance_template.yaml index 72bc1291ab8..f7103769941 100644 --- a/dm/templates/instance_template/examples/instance_template.yaml +++ b/dm/templates/instance_template/examples/instance_template.yaml @@ -12,7 +12,9 @@ resources: properties: diskImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-1804-lts networks: - - default + - network: default + accessConfigs: + - type: ONE_TO_ONE_NAT machineType: f1-micro tags: items: diff --git a/dm/templates/instance_template/instance_template.py b/dm/templates/instance_template/instance_template.py index 9e1802c062a..f10c0393a3e 100644 --- a/dm/templates/instance_template/instance_template.py +++ b/dm/templates/instance_template/instance_template.py @@ -49,36 +49,32 @@ def get_network_interfaces(properties): """ network_interfaces = [] - networks = properties.get('networks', [{ - "name": properties.get('network'), - "hasExternalIp": properties.get('hasExternalIp'), - "natIP": properties.get('natIP'), - "subnetwork": properties.get('subnetwork'), - "networkIP": properties.get('networkIP'), - }]) + networks = properties.get('networks', []) + if len(networks) == 0 and properties.get('network'): + network = { + "network": properties.get('network'), + "subnetwork": properties.get('subnetwork'), + "networkIP": properties.get('networkIP'), + } + networks.append(network) + if (properties.get('hasExternalIp')): + network['accessConfigs'] = [{ + "type": "ONE_TO_ONE_NAT", + }] + if properties.get('natIP'): + network['accessConfigs'][0]["natIp"] = properties.get('natIP') for network in networks: - if not '.' in network['name'] and not '/' in network['name']: - network_name = 'global/networks/{}'.format(network['name']) + if not '.' in network['network'] and not '/' in network['network']: + network_name = 'global/networks/{}'.format(network['network']) else: - network_name = network['name'] + network_name = network['network'] network_interface = { 'network': network_name, } - if network['hasExternalIp']: - access_configs = { - 'name': 'External NAT', - 'type': 'ONE_TO_ONE_NAT' - } - - if network.get('natIP'): - access_configs['natIP'] = network['natIP'] - - network_interface['accessConfigs'] = [access_configs] - - netif_optional_props = ['subnetwork', 'networkIP'] + netif_optional_props = ['subnetwork', 'networkIP', 'aliasIpRanges', 'accessConfigs'] for prop in netif_optional_props: if network.get(prop): network_interface[prop] = network[prop] @@ -93,17 +89,19 @@ def generate_config(context): properties = context.properties name = properties.get('name', context.env['name']) machine_type = properties['machineType'] - boot_disk = create_boot_disk(properties) network_interfaces = get_network_interfaces(context.properties) + project_id = properties.get('project', context.env['project']) instance_template = { - 'name': name, - 'type': 'compute.v1.instanceTemplate', + 'name': context.env['name'], + # https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates + 'type': 'gcp-types/compute-v1:instanceTemplates', 'properties': { + 'name': name, + 'project': project_id, 'properties': { 'machineType': machine_type, - 'disks': [boot_disk], 'networkInterfaces': network_interfaces } } @@ -113,15 +111,22 @@ def generate_config(context): optional_props = [ 'metadata', + 'disks', + 'scheduling', 'tags', 'canIpForward', 'labels', 'serviceAccounts', - 'scheduling' + 'scheduling', + 'shieldedInstanceConfig', + 'minCpuPlatform', + 'guestAccelerators', ] for prop in optional_props: set_optional_property(template_spec, properties, prop) + if not template_spec.get('disks'): + template_spec['disks'] = [create_boot_disk(properties)] set_optional_property( template_spec, @@ -137,6 +142,18 @@ def generate_config(context): 'description' ) + set_optional_property( + instance_template['properties'], + properties, + 'sourceInstance' + ) + + set_optional_property( + instance_template['properties'], + properties, + 'sourceInstanceParams' + ) + return { 'resources': [instance_template], 'outputs': @@ -147,7 +164,7 @@ def generate_config(context): }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) } ] } diff --git a/dm/templates/instance_template/instance_template.py.schema b/dm/templates/instance_template/instance_template.py.schema index f9e4d37e350..38d535688c2 100644 --- a/dm/templates/instance_template/instance_template.py.schema +++ b/dm/templates/instance_template/instance_template.py.schema @@ -15,10 +15,16 @@ info: title: Instance Template author: Sourced Group Inc. + version: 1.0.0 description: | Creates an instance template. -additionalProperties: false + For more information on this resource: + https://cloud.google.com/compute/ + + APIs endpoints used by this template: + - gcp-types/compute-v1:instanceTemplates => + https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates required: - diskImage @@ -49,6 +55,8 @@ oneOf: required: - networks +additionalProperties: false + definitions: hasExternalIp: type: boolean @@ -66,49 +74,70 @@ definitions: specify a static external IP address, it must live in the same region as the zone of the instance. If hasExternalIp is false this field is ignored. + network: + type: string + description: | + URL of the network resource for this instance. When creating an instance, if neither the network + nor the subnetwork is specified, the default network global/networks/default is used; + if the network is not specified but the subnetwork is specified, the network is inferred. + + If you specify this property, you can specify the network as a full or partial URL. + For example, the following are all valid URLs: + + - https://www.googleapis.com/compute/v1/projects/project/global/networks/network + - projects/project/global/networks/network + - global/networks/default + Authorization requires one or more of the following Google IAM permissions on the specified resource network: + + - compute.networks.use + - compute.networks.useExternalIp subnetwork: type: string description: | - The URL of the Subnetwork resource for this instance. If the network - resource is in legacy mode, do not provide this property. If the network - is in auto subnet mode, providing the subnetwork is optional. If the - network is in custom subnet mode, then this field should be specified. - If you specify this property, you can specify the subnetwork as a full - or partial URL. For example, the following are all valid URLs: - - https://www.googleapis.com/compute/v1/projects/project/regions/region/subnetworks/subnetwork - - regions/region/subnetworks/subnetwork + The URL of the Subnetwork resource for this instance. If the network resource is in legacy mode, + do not specify this field. If the network is in auto subnet mode, specifying the subnetwork is optional. + If the network is in custom subnet mode, specifying the subnetwork is required. + If you specify this field, you can specify the subnetwork as a full or partial URL. For example, the following are all valid URLs: + + - https://www.googleapis.com/compute/v1/projects/project/regions/region/subnetworks/subnetwork + - regions/region/subnetworks/subnetwork + Authorization requires one or more of the following Google IAM permissions on the specified resource subnetwork: + + - compute.subnetworks.use + - compute.subnetworks.useExternalIp networkIP: type: string description: | - An IPv4 internal network address to assign to the instance for this - network interface. If not specified by the user, an unused internal IP - is assigned by the system. + An IPv4 internal IP address to assign to the instance for this network interface. + If not specified by the user, an unused internal IP is assigned by the system. properties: name: type: string - description: The name of the instance template resource. + description: The name of the instance template resource. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the instance. templateDescription: type: string - description: The resource description (optional). + description: | + The resource description (optional). instanceDescription: type: string description: | The description of the instance resource the instance template will create (optional). network: - type: string - description: | - Name of the network the instance will be connected to; - e.g., 'my-custom-network' or 'default'. - hasExternalIp: - $ref: '#/definitions/hasExternalIp' - natIP: - $ref: '#/definitions/natIP' + $ref: '#/definitions/network' subnetwork: $ref: '#/definitions/subnetwork' networkIP: $ref: '#/definitions/networkIP' + hasExternalIp: + $ref: '#/definitions/hasExternalIp' + natIP: + $ref: '#/definitions/natIP' networks: type: array description: | @@ -118,27 +147,512 @@ properties: type: object additionalProperties: false required: - - name + - network properties: - name: - type: string - description: | - Name of the network the instance will be connected to; - e.g., 'my-custom-network' or 'default'. - hasExternalIp: - $ref: '#/definitions/hasExternalIp' - natIP: - $ref: '#/definitions/natIP' + network: + $ref: '#/definitions/network' subnetwork: $ref: '#/definitions/subnetwork' networkIP: $ref: '#/definitions/networkIP' + aliasIpRanges: + type: array + uniqueItems: true + description: | + An array of alias IP ranges for this network interface. You can only specify this + field for network interfaces in VPC networks. + items: + type: object + additionalProperties: false + properties: + ipCidrRange: + type: string + description: | + The IP alias ranges to allocate for this interface. This IP CIDR range must belong + to the specified subnetwork and cannot contain IP addresses reserved by system or + used by other network interfaces. This range may be a single IP address (such as 10.2.3.4), + a netmask (such as /24) or a CIDR-formatted string (such as 10.1.2.0/24). + subnetworkRangeName: + type: string + description: | + The name of a subnetwork secondary IP range from which to allocate an IP alias range. + If not specified, the primary range of the subnetwork is used. + accessConfigs: + type: array + uniqueItems: true + description: | + An array of configurations for this interface. Currently, only one access config, ONE_TO_ONE_NAT, + is supported. If there are no accessConfigs specified, then this instance will have no external internet access. + items: + type: object + additionalProperties: false + properties: + type: + type: string + description: | + The type of configuration. The default and only option is ONE_TO_ONE_NAT. + enum: + - ONE_TO_ONE_NAT + name: + type: string + description: | + The name of this access configuration. The default and recommended name is External NAT, + but you can use any arbitrary string, such as My external IP or Network Access. + setPublicPtr: + type: boolean + description: | + Specifies whether a public DNS 'PTR' record should be created to map the external + IP address of the instance to a DNS domain name. + publicPtrDomainName: + type: string + description: | + The DNS domain name for the public PTR record. You can set this field only + if the setPublicPtr field is enabled. + networkTier: + type: string + description: | + This signifies the networking tier used for configuring this access configuration + and can only take the following values: PREMIUM, STANDARD. + + If an AccessConfig is specified without a valid external IP address, an + ephemeral IP will be created with this networkTier. + + If an AccessConfig with a valid external IP address is specified, it must match + that of the networkTier associated with the Address resource owning that IP. + enum: + - STANDARD + - PREMIUM + natIP: + $ref: '#/definitions/natIP' + disks: + type: array + uniqueItems: true + description: | + Array of disks associated with this instance. Persistent disks must be created before you can assign them. + items: + type: object + additionalProperties: false + oneOf: + - required: + - source + - required: + - initializeParams + - allOf: + - not: + required: + - source + - not: + required: + - initializeParams + properties: + type: + type: string + description: | + Specifies the type of the disk, either SCRATCH or PERSISTENT. If not specified, the default is PERSISTENT. + enum: + - SCRATCH + - PERSISTENT + mode: + type: string + description: | + The mode in which to attach this disk, either READ_WRITE or READ_ONLY. + If not specified, the default is to attach the disk in READ_WRITE mode. + enum: + - READ_WRITE + - READ_ONLY + source: + type: string + description: | + Specifies a valid partial or full URL to an existing Persistent Disk resource. + When creating a new instance, one of initializeParams.sourceImage or + disks.source is required except for local SSD. + + If desired, you can also attach existing non-root persistent disks using this property. + This field is only applicable for persistent disks. + + Note that for InstanceTemplate, specify the disk name, not the URL for the disk. + + Authorization requires one or more of the following Google IAM permissions on the specified resource source: + + compute.disks.use + compute.disks.useReadOnly + deviceName: + type: string + description: | + Specifies a unique device name of your choice that is reflected into the /dev/disk/by-id/google-* + tree of a Linux operating system running within the instance. This name can be used to reference + the device for mounting, resizing, and so on, from within the instance. + + If not specified, the server chooses a default device name to apply to this disk, in the + form persistent-disk-x, where x is a number assigned by Google Compute Engine. + This field is only applicable for persistent disks. + boot: + type: boolean + description: | + Indicates that this is a boot disk. The virtual machine will use the first partition + of the disk for its root filesystem. + initializeParams: + type: object + additionalProperties: false + description: | + Specifies the parameters for a new disk that will be created alongside the new instance. + Use initialization parameters to create boot disks or local SSDs attached to the new instance. + + This property is mutually exclusive with the source property; you can only define one or the other, but not both. + properties: + labels: + type: object + description: | + Labels to apply to this disk. These can be later modified by the disks.setLabels method. + This field is only applicable for persistent disks. + + An object containing a list of "key": value pairs. + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + + Authorization requires the following Google IAM permission on the specified resource labels: + + compute.disks.setLabels + diskName: + type: string + description: | + Specifies the disk name. If not specified, the default is to use the name of the instance. + If the disk with the instance name exists already in the given zone/region, + a new name will be automatically generated. + sourceImage: + type: string + description: | + The source image to create this disk. When creating a new instance, one of + initializeParams.sourceImage or disks.source is required except for local SSD. + + To create a disk with one of the public operating system images, specify the image by its family name. + For example, specify family/debian-9 to use the latest Debian 9 image: + + projects/debian-cloud/global/images/family/debian-9 + + Alternatively, use a specific version of a public operating system image: + + projects/debian-cloud/global/images/debian-9-stretch-vYYYYMMDD + + To create a disk with a custom image that you created, specify the image name in the following format: + + global/images/my-custom-image + + You can also specify a custom image by its image family, which returns the latest version of the + image in that family. Replace the image name with family/family-name: + + global/images/family/my-image-family + + If the source image is deleted later, this field will not be set. + + Authorization requires the following Google IAM permission on the specified resource sourceImage: + + compute.images.useReadOnly + description: + type: string + description: | + An optional description. Provide this property when creating the disk. + diskSizeGb: + type: number + description: | + Specifies the size of the disk in base-2 GB. + diskType: + type: string + description: | + Specifies the disk type to use to create the instance. If not specified, the default is pd-standard, + specified using the full URL. For example: + + https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/pd-standard + + Other values include pd-ssd and local-ssd. If you define this field, you can provide either the full + or partial URL. For example, the following are valid values: + + https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/diskType + projects/project/zones/zone/diskTypes/diskType + zones/zone/diskTypes/diskType + Note that for InstanceTemplate, this is the name of the disk type, not URL. + enum: + - pd-standard + - pd-ssd + - local-ssd + sourceImageEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source image. Required if the source image is + protected by a customer-supplied encryption key. + + Instance templates do not store customer-supplied encryption keys, so you cannot create disks + for instances in a managed instance group if the source images are encrypted with your own keys. + properties: + rawKey: + type: string + description: | + Specifies a 256-bit customer-supplied encryption key, encoded in RFC 4648 base64 + to either encrypt or decrypt this resource. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. + sourceSnapshot: + type: string + description: | + The source snapshot to create this disk. When creating a new instance, one of + initializeParams.sourceSnapshot or disks.source is required except for local SSD. + + To create a disk with a snapshot that you created, specify the snapshot name in the following format: + + global/snapshots/my-backup + + If the source snapshot is deleted later, this field will not be set. + + Authorization requires the following Google IAM permission on the specified resource sourceSnapshot: + + compute.snapshots.useReadOnly + sourceSnapshotEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source snapshot. + properties: + rawKey: + type: string + description: | + Specifies a 256-bit customer-supplied encryption key, encoded in RFC 4648 base64 + to either encrypt or decrypt this resource. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. + autoDelete: + type: boolean + description: | + Specifies whether the disk will be auto-deleted when the instance is deleted + (but not when the disk is detached from the instance). + interface: + type: string + description: | + Specifies the disk interface to use for attaching this disk, which is either SCSI or NVME. + The default is SCSI. Persistent disks must always use SCSI and the request will fail if you + attempt to attach a persistent disk in any other format than SCSI. Local SSDs can use either NVME or SCSI. + For performance characteristics of SCSI over NVMe, see Local SSD performance. + enum: + - SCSI + - NVME + guestOsFeatures: + type: array + uniqueItems: true + description: | + A list of features to enable on the guest operating system. Applicable only for bootable images. + Read Enabling guest operating system features to see a list of available options. + items: + type: object + additionalProperties: false + properties: + type: + type: string + description: | + https://cloud.google.com/compute/docs/images/create-delete-deprecate-private-images#guest-os-features + The ID of a supported feature. Read Enabling guest operating system features + to see a list of available options. + enum: + - MULTI_IP_SUBNET + - SECURE_BOOT + - UEFI_COMPATIBLE + - VIRTIO_SCSI_MULTIQUEUE + - WINDOWS + diskEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source snapshot. + properties: + rawKey: + type: string + description: | + Encrypts or decrypts a disk using a customer-supplied encryption key. + + If you are creating a new disk, this field encrypts the new disk using an encryption + key that you provide. If you are attaching an existing disk that is already encrypted, + this field decrypts the disk using the customer-supplied encryption key. + + If you encrypt a disk using a customer-supplied key, you must provide the same key again when + you attempt to use this resource at a later time. For example, you must provide the key when + you create a snapshot or an image from the disk or when you attach the disk + to a virtual machine instance. + + If you do not provide an encryption key, then the disk will be encrypted using an automatically + generated key and you do not need to provide a key to use the disk later. + + Instance templates do not store customer-supplied encryption keys, so you cannot use your own keys + to encrypt disks in a managed instance group. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. machineType: type: string default: n1-standard-1 description: | The Compute Instance type; e.g., 'n1-standard-1'. See https://cloud.google.com/compute/docs/machine-types for details. + scheduling: + type: object + additionalProperties: false + description: | + Sets the scheduling options for this instance. + properties: + onHostMaintenance: + type: string + description: | + Defines the maintenance behavior for this instance. For standard instances, the default behavior is MIGRATE. + For preemptible instances, the default and only possible behavior is TERMINATE. + For more information, see Setting Instance Scheduling Options. + enum: + - MIGRATE + - TERMINATE + automaticRestart: + type: boolean + description: | + Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine + (not terminated by a user). You can only set the automatic restart option for standard instances. + Preemptible instances cannot be automatically restarted. + + By default, this is set to true so an instance is automatically restarted if it is terminated by Compute Engine. + preemptible: + type: boolean + description: | + Defines whether the instance is preemptible. This can only be set during instance creation, + it cannot be set or changed after the instance has been created. + nodeAffinities: + type: array + uniqueItems: true + description: | + A set of node affinity and anti-affinity. + items: + type: object + additionalProperties: false + properties: + key: + type: string + description: | + Corresponds to the label key of Node resource. + operator: + type: string + description: | + Defines the operation of node selection. + values: + type: array + uniqueItems: true + description: | + Corresponds to the label values of Node resource. + items: + type: string + minCpuPlatform: + type: string + description: | + Specifies a minimum CPU platform for the VM instance. Applicable values are the friendly names of CPU platforms, + such as minCpuPlatform: "Intel Haswell" or minCpuPlatform: "Intel Sandy Bridge". + enum: + - Intel Sandy Bridge + - Intel Ivy Bridge + - Intel Haswell + - Intel Broadwell + - Intel Skylake + sourceInstance: + type: string + description: | + The source instance used to create the template. You can provide this as a partial or full URL to the resource. + For example, the following are valid values: + + - https://www.googleapis.com/compute/v1/projects/project/zones/zone/instances/instance + - projects/project/zones/zone/instances/instance + + Authorization requires the following Google IAM permission on the specified resource sourceInstance: + - compute.instances.get + sourceInstanceParams: + type: object + additionalProperties: false + description: | + The source instance params to use to create this instance template. + properties: + diskConfigs: + type: array + uniqueItems: true + description: | + Attached disks configuration. If not provided, defaults are applied: For boot disk and any other R/W disks, + new custom images will be created from each disk. For read-only disks, they will be attached + in read-only mode. Local SSD disks will be created as blank volumes. + items: + type: object + additionalProperties: false + properties: + deviceName: + type: string + description: | + Specifies the device name of the disk to which the configurations apply to. + instantiateFrom: + type: string + description: | + Specifies whether to include the disk and what image to use. Possible values are: + + - source-image: to use the same image that was used to create the source instance's corresponding disk. + Applicable to the boot disk and additional read-write disks. + - source-image-family: to use the same image family that was used to create the source instance's + corresponding disk. Applicable to the boot disk and additional read-write disks. + - custom-image: to use a user-provided image url for disk creation. Applicable to the boot disk and + additional read-write disks. + - attach-read-only: to attach a read-only disk. Applicable to read-only disks. + - do-not-include: to exclude a disk from the template. Applicable to additional read-write disks, + local SSDs, and read-only disks. + enum: + - source-image + - source-image-family + - custom-image + - attach-read-only + autoDelete: + type: boolean + description: | + Specifies whether the disk will be auto-deleted when the instance is deleted + (but not when the disk is detached from the instance). + customImage: + type: string + description: | + The custom source image to be used to restore this disk when instantiating this instance template.. + shieldedInstanceConfig: + type: object + additionalProperties: false + properties: + enableSecureBoot: + type: boolean + description: | + Defines whether the instance has Secure Boot enabled. + enableVtpm: + type: boolean + description: | + Defines whether the instance has the vTPM enabled. + enableIntegrityMonitoring: + type: boolean + description: | + Defines whether the instance has integrity monitoring enabled. + guestAccelerators: + type: array + uniqueItems: true + description: | + A list of the type and count of accelerator cards attached to the instance. + items: + type: object + additionalProperties: false + properties: + acceleratorType: + type: string + description: | + Full or partial URL of the accelerator type resource to attach to this instance. For example: projects/my-project/zones/us-central1-c/acceleratorTypes/nvidia-tesla-p100 + If you are creating an instance template, specify only the accelerator name. + See GPUs on Compute Engine for a full list of accelerator types. + acceleratorCount: + type: integer + description: | + The number of the guest accelerator cards exposed to this instance. canIpForward: type: boolean description: | @@ -167,6 +681,7 @@ properties: minimum: 10 metadata: type: object + additionalProperties: false description: | Instance metadata. For example: @@ -177,9 +692,14 @@ properties: properties: items: type: array + uniqueItems: true description: The metadata key-value pairs. items: type: object + additionalProperties: false + required: + - key + - value properties: key: type: string @@ -187,17 +707,20 @@ properties: type: string serviceAccounts: type: array + uniqueItems: true description: | The list of service accounts, with their specified scopes, authorized for this instance. Only one service account per VM instance is supported. items: type: object + additionalProperties: false properties: email: type: string description: The email address of the service account. scopes: type: array + uniqueItems: true description: | The list of scopes to be made available to the service account. items: @@ -209,6 +732,7 @@ properties: for details tags: type: object + additionalProperties: false description: | The list of tags to apply to the instances that are created from the template. The tags identify valid sources or targets for network @@ -216,6 +740,7 @@ properties: properties: items: type: array + uniqueItems: true description: The array of tags. items: type: string diff --git a/dm/templates/instance_template/tests/integration/instance_template.bats b/dm/templates/instance_template/tests/integration/instance_template.bats index cc22489cb40..c459295ca6b 100755 --- a/dm/templates/instance_template/tests/integration/instance_template.bats +++ b/dm/templates/instance_template/tests/integration/instance_template.bats @@ -111,7 +111,6 @@ function teardown() { --format "yaml(properties.networkInterfaces[0])" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ "$status" -eq 0 ]] - [[ "$output" =~ "name: External NAT" ]] [[ "$output" =~ "type: ONE_TO_ONE_NAT" ]] [[ "$output" =~ "network: ${NET}" ]] } diff --git a/dm/templates/instance_template/tests/integration/instance_template.yaml b/dm/templates/instance_template/tests/integration/instance_template.yaml index 847fbce2716..7ea71b774d7 100644 --- a/dm/templates/instance_template/tests/integration/instance_template.yaml +++ b/dm/templates/instance_template/tests/integration/instance_template.yaml @@ -15,7 +15,11 @@ resources: name: it-${RAND} instanceDescription: Instance description templateDescription: Template description - network: $(ref.test-network-${RAND}.selfLink) + networks: + - network: $(ref.test-network-${RAND}.selfLink) + accessConfigs: + - name: External NAT + type: ONE_TO_ONE_NAT diskImage: ${IMAGE} machineType: f1-micro canIpForward: true diff --git a/dm/templates/instance_template/tests/integration/instance_template_networks.bats b/dm/templates/instance_template/tests/integration/instance_template_networks.bats index 97c3a9f210a..a697b865226 100755 --- a/dm/templates/instance_template/tests/integration/instance_template_networks.bats +++ b/dm/templates/instance_template/tests/integration/instance_template_networks.bats @@ -15,7 +15,7 @@ fi # envsubst requires all variables used in the example/config to be exported. if [[ -e "${RANDOM_FILE}" ]]; then export RAND=$(cat "${RANDOM_FILE}") - DEPLOYMENT_NAME="${CLOUD_FOUNDATION_PROJECT_ID}-${TEST_NAME}-${RAND}" + DEPLOYMENT_NAME="$(echo ${CLOUD_FOUNDATION_PROJECT_ID}-${TEST_NAME}-${RAND} | head -c 63)" # Replace underscores in the deployment name with dashes. DEPLOYMENT_NAME=${DEPLOYMENT_NAME//_/-} CONFIG=".${DEPLOYMENT_NAME}.yaml" @@ -65,7 +65,7 @@ function teardown() { [[ "$status" -eq 0 ]] [[ "$output" =~ "diskType: pd-ssd" ]] [[ "$output" =~ "sourceImage: ${IMAGE}" ]] - [[ "$output" =~ "diskSizeGb: '50'" ]] +# [[ "$output" =~ "diskSizeGb: '50'" ]] } @test "Verifying instance spec properties" { diff --git a/dm/templates/instance_template/tests/integration/instance_template_networks.yaml b/dm/templates/instance_template/tests/integration/instance_template_networks.yaml index d753f9a84f0..5974fd46f01 100644 --- a/dm/templates/instance_template/tests/integration/instance_template_networks.yaml +++ b/dm/templates/instance_template/tests/integration/instance_template_networks.yaml @@ -9,17 +9,23 @@ imports: name: instance_template.py resources: - - name: instance-template-${RAND} + - name: it-${RAND} type: instance_template.py properties: name: it-${RAND} instanceDescription: Instance description templateDescription: Template description networks: - - name: $(ref.test-network-0-${RAND}.selfLink) + - network: $(ref.test-network-0-${RAND}.selfLink) subnetwork: $(ref.test-subnetwork-0-${RAND}.selfLink) - - name: $(ref.test-network-1-${RAND}.selfLink) + accessConfigs: + - name: External NAT + type: ONE_TO_ONE_NAT + - network: $(ref.test-network-1-${RAND}.selfLink) subnetwork: $(ref.test-subnetwork-1-${RAND}.selfLink) + accessConfigs: + - name: External NAT + type: ONE_TO_ONE_NAT diskImage: ${IMAGE} machineType: f1-micro canIpForward: true diff --git a/dm/templates/interconnect/interconnect.py b/dm/templates/interconnect/interconnect.py index 8c2a63ac08d..befca6d536d 100644 --- a/dm/templates/interconnect/interconnect.py +++ b/dm/templates/interconnect/interconnect.py @@ -17,14 +17,20 @@ def generate_config(context): """ Entry point for the deployment resources. """ + + properties = context.properties + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) + resources = [] intercon = { 'name': context.env['name'], - 'type': 'compute.v1.interconnects', + # https://cloud.google.com/compute/docs/reference/rest/v1/interconnects + 'type': 'gcp-types/compute-v1:interconnects', 'properties': { - 'name': - context.properties.get('name', context.env['name']), + 'project': project_id, + 'name': name, 'customerName': context.properties['customerName'], 'interconnectType': @@ -56,7 +62,7 @@ def generate_config(context): [ { 'name': 'name', - 'value': context.env['name'] + 'value': name }, { 'name': 'selfLink', diff --git a/dm/templates/interconnect/interconnect.py.schema b/dm/templates/interconnect/interconnect.py.schema index cc5ab16a527..eb9a59a926b 100644 --- a/dm/templates/interconnect/interconnect.py.schema +++ b/dm/templates/interconnect/interconnect.py.schema @@ -15,11 +15,16 @@ info: title: Interconnect (Dedicated) author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of an Interconnect resource. For more information on this resource: https://cloud.google.com/compute/docs/reference/rest/v1/interconnects. + APIs endpoints used by this template: + - gcp-types/compute-v1:interconnects => + https://cloud.google.com/compute/docs/reference/rest/v1/interconnects + imports: - path: interconnect.py @@ -32,6 +37,13 @@ required: - requestedLinkCount properties: + name: + type: string + description: The name of the Interconnect resource. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the instance. adminEnabled: type: boolean description: | @@ -55,7 +67,6 @@ properties: enum: - DEDICATED - PARTNER - - PARTNER_PROVIDER linkType: type: string description: | @@ -64,14 +75,11 @@ properties: allowed for a dedicated Interconnect. Options: Ethernet_10G_LR. enum: - LINK_TYPE_ETHERNET_10G_LR - location: - type: string + - LINK_TYPE_ETHERNET_100G_LR + requestedLinkCount: + type: number description: | - The URL of the InterconnectLocation object that defines where the - connection is to be provisioned. - name: - type: string - description: The Interconnect name. + Target number of physical links in the link bundle, as requested by the customer. location: type: string description: | diff --git a/dm/templates/interconnect_attachment/interconnect_attachment.py b/dm/templates/interconnect_attachment/interconnect_attachment.py index 6e0be470fc3..593cf820190 100644 --- a/dm/templates/interconnect_attachment/interconnect_attachment.py +++ b/dm/templates/interconnect_attachment/interconnect_attachment.py @@ -17,14 +17,20 @@ def generate_config(context): """ Entry point for the deployment resources. """ + + properties = context.properties + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) + resources = [] attach = { 'name': context.env['name'], - 'type': 'compute.v1.interconnectAttachments', + # https://cloud.google.com/compute/docs/reference/rest/v1/interconnectAttachments + 'type': 'gcp-types/compute-v1:interconnectAttachments', 'properties': { - 'name': - context.properties.get('name', context.env['name']), + 'project': project_id, + 'name': name, 'router': context.properties['router'], 'region': @@ -59,7 +65,7 @@ def generate_config(context): [ { 'name': 'name', - 'value': context.env['name'] + 'value': name }, { 'name': 'selfLink', diff --git a/dm/templates/interconnect_attachment/interconnect_attachment.py.schema b/dm/templates/interconnect_attachment/interconnect_attachment.py.schema index 23d66f2caf3..d45969626a6 100644 --- a/dm/templates/interconnect_attachment/interconnect_attachment.py.schema +++ b/dm/templates/interconnect_attachment/interconnect_attachment.py.schema @@ -15,6 +15,7 @@ info: title: Interconnect Attachment author: Sourced Group Inc. + version: 1.0.0 description: | Creates an Interconnect Attachment. @@ -22,6 +23,10 @@ info: https://cloud.google.com/interconnect/docs/how-to/dedicated/creating-vlan-attachments (Dedicated) https://cloud.google.com/interconnect/docs/how-to/partner/creating-vlan-attachments (Partner) + APIs endpoints used by this template: + - gcp-types/compute-v1:interconnectAttachments => + https://cloud.google.com/compute/docs/reference/rest/v1/interconnectAttachments + imports: - path: interconnect_attachment.py @@ -32,34 +37,164 @@ required: - region - type +oneOf: + - allOf: + - properties: + type: + enum: ["PARTNER"] + - not: + required: + - pairingKey + - not: + required: + - bandwidth + - not: + required: + - partnerMetadata + - not: + required: + - partnerAsn + - allOf: + - properties: + type: + enum: ["PARTNER_PROVIDER"] + - not: + required: + - adminEnabled + - not: + required: + - edgeAvailabilityDomain + - allOf: + - properties: + type: + enum: ["DEDICATED"] + - not: + required: + - pairingKey + - not: + required: + - edgeAvailabilityDomain + - not: + required: + - partnerAsn + properties: name: type: string - description: The name of the Interconnect Attachment. + description: | + The name of the Interconnect Attachment resource. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the instance. router: type: string - description: The URL of the cloud router that the attachment is being - attached to. + description: | + URL of the Cloud Router to be used for dynamic routing. This router must be in the same region as this + InterconnectAttachment. The InterconnectAttachment will automatically connect the Interconnect to the + network & region within which the Cloud Router is configured. + + Authorization requires the following Google IAM permission on the specified resource router: + - compute.routers.use region: type: string - description: The URL of the region where the router resides. + description: | + The URL of the region where the router resides. + pairingKey: + type: string + description: | + The opaque identifier of an PARTNER attachment used to initiate provisioning with a selected partner. + Of the form "XXXXX/region/domain" type: type: string - description: The type of Interconnect + description: | + The type of interconnect attachment this is. enum: - DEDICATED - PARTNER - PARTNER_PROVIDER + bandwidth: + type: string + description: | + Provisioned bandwidth capacity for the interconnect attachment. For attachments of type DEDICATED, + the user can set the bandwidth. For attachments of type PARTNER, the Google Partner that is operating + the interconnect must set the bandwidth. + Output only for PARTNER type, mutable for PARTNER_PROVIDER and DEDICATED. + enum: + - BPS_50M + - BPS_100M + - BPS_200M + - BPS_300M + - BPS_400M + - BPS_500M + - BPS_1G + - BPS_2G + - BPS_5G + - BPS_10G + adminEnabled: + type: boolean + description: | + Determines whether this Attachment will carry packets. Not present for PARTNER_PROVIDER. + partnerMetadata: + type: object + additionalProperties: false + description: | + Informational metadata about Partner attachments from Partners to display to customers. + Output only for for PARTNER type, mutable for PARTNER_PROVIDER, not available for DEDICATED. + properties: + partnerName: + type: string + description: | + Plain text name of the Partner providing this attachment. + This value may be validated to match approved Partner values. + interconnectName: + type: string + description: | + Plain text name of the Interconnect this attachment is connected to, as displayed in the Partner’s portal. + For instance "Chicago 1". This value may be validated to match approved Partner values. + portalUrl: + type: string + description: | + URL of the Partner’s portal for this Attachment. Partners may customise this to be a deep link to the + specific resource on the Partner portal. This value may be validated to match approved Partner values. + vlanTag8021q: + type: number + description: | + The IEEE 802.1Q VLAN tag for this attachment, in the range 2-4094. Only specified at creation time. interconnect: type: string description: | URL of the underlying Interconnect object that this attachment's traffic will traverse through. + + Authorization requires the following Google IAM permission on the specified resource interconnect: + - compute.interconnects.use + partnerAsn: + type: string + description: | + Optional BGP ASN for the router supplied by a Layer 3 Partner if they configured BGP on behalf of the customer. + Output only for PARTNER type, input only for PARTNER_PROVIDER, not available for DEDICATED. + candidateSubnets: + type: array + uniqItems: True + description: | + Up to 16 candidate prefixes that can be used to restrict the allocation of cloudRouterIpAddress and + customerRouterIpAddress for this attachment. All prefixes must be within link-local address space (169.254.0.0/16) + and must be /29 or shorter (/28, /27, etc). Google will attempt to select an unused /29 from the supplied + candidate prefix(es). The request will fail if all possible /29s are in use on Google’s edge. + If not supplied, Google will randomly select an unused /29 from all of link-local space. + maxItems: 16 + items: + type: string edgeAvailabilityDomain: type: string description: | Desired availability domain for the attachment. Only available for type - PARTNER, at creation time + PARTNER, at creation time. + + For improved reliability, customers should configure a pair of attachments, one per availability domain. + The selected availability domain will be provided to the Partner via the pairing key, so that the provisioned + circuit will lie in the specified domain. If not specified, the value will default to AVAILABILITY_DOMAIN_ANY. enum: - AVAILABILITY_DOMAIN_1 - AVAILABILITY_DOMAIN_2 diff --git a/dm/templates/internal_load_balancer/internal_load_balancer.py b/dm/templates/internal_load_balancer/internal_load_balancer.py index 216dbe3fa4c..e92a4ab68d5 100644 --- a/dm/templates/internal_load_balancer/internal_load_balancer.py +++ b/dm/templates/internal_load_balancer/internal_load_balancer.py @@ -21,19 +21,21 @@ def set_optional_property(destination, source, prop_name): destination[prop_name] = source[prop_name] -def get_backend_service(properties, name): +def get_backend_service(properties, project_id, res_name): """ Creates the backend service. """ + backend_spec = properties['backendService'] + name = '{}-bs'.format(res_name) backend_properties = { + 'name': backend_spec.get('name', properties.get('name', name)), + 'project': project_id, 'loadBalancingScheme': 'INTERNAL', 'protocol': properties['protocol'], - 'region': properties['region'] + 'region': properties['region'], } - backend_spec = properties['backendService'] - backend_name = backend_spec.get('name', name + '-bs') backend_resource = { - 'name': backend_name, + 'name': name, 'type': 'backend_service.py', 'properties': backend_properties } @@ -54,29 +56,31 @@ def get_backend_service(properties, name): return [backend_resource], [ { 'name': 'backendServiceName', - 'value': backend_name, + 'value': backend_resource['properties']['name'], }, { 'name': 'backendServiceSelfLink', - 'value': '$(ref.{}.selfLink)'.format(backend_name), + 'value': '$(ref.{}.selfLink)'.format(name), }, ] -def get_forwarding_rule(properties, backend, name): +def get_forwarding_rule(properties, backend, project_id, res_name): """ Creates the forwarding rule. """ rule_properties = { + 'name': properties.get('name', res_name), + 'project': project_id, 'loadBalancingScheme': 'INTERNAL', 'IPProtocol': properties['protocol'], 'backendService': '$(ref.{}.selfLink)'.format(backend['name']), - 'region': properties['region'] + 'region': properties['region'], } rule_resource = { - 'name': name, + 'name': res_name, 'type': 'forwarding_rule.py', - 'properties': rule_properties + 'properties': rule_properties, } optional_properties = [ @@ -94,15 +98,15 @@ def get_forwarding_rule(properties, backend, name): return [rule_resource], [ { 'name': 'forwardingRuleName', - 'value': name, + 'value': res_name, }, { 'name': 'forwardingRuleSelfLink', - 'value': '$(ref.{}.selfLink)'.format(name), + 'value': '$(ref.{}.selfLink)'.format(res_name), }, { 'name': 'IPAddress', - 'value': '$(ref.{}.IPAddress)'.format(name), + 'value': '$(ref.{}.IPAddress)'.format(res_name), }, { 'name': 'region', @@ -115,13 +119,14 @@ def generate_config(context): """ Entry point for the deployment resources. """ properties = context.properties - name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) - backend_resources, backend_outputs = get_backend_service(properties, name) + backend_resources, backend_outputs = get_backend_service(properties, project_id, context.env['name']) rule_resources, rule_outputs = get_forwarding_rule( properties, backend_resources[0], - name + project_id, + context.env['name'] ) return { diff --git a/dm/templates/internal_load_balancer/internal_load_balancer.py.schema b/dm/templates/internal_load_balancer/internal_load_balancer.py.schema index 4209557f7a1..d2dc2ee83cc 100644 --- a/dm/templates/internal_load_balancer/internal_load_balancer.py.schema +++ b/dm/templates/internal_load_balancer/internal_load_balancer.py.schema @@ -14,6 +14,7 @@ info: title: Internal Load Balancer + version: 1.0.0 author: Sourced Group Inc. description: | Supports the creation of an internal load balancing solution that consists @@ -38,6 +39,12 @@ properties: description: | The internal load balancer name. This name is assigned to the underlying forwarding rule resource. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. description: type: string description: | @@ -49,6 +56,7 @@ properties: The name of the region where the internal load balancer resides. ports: type: array + uniqItems: true description: | The list of ports; only packets addressed to these ports are forwarded to the backends configured with the load balancer. @@ -84,6 +92,7 @@ properties: backendService: type: object description: The backend service configuration. + additionalProperties: false required: - healthCheck - backends @@ -96,11 +105,13 @@ properties: description: An optional description of the backend service resource. backends: type: array + uniqItems: true description: | The list of backends (instance groups) to which the backend service distributes traffic. items: type: object + additionalProperties: false required: - group properties: @@ -137,6 +148,7 @@ properties: - CLIENT_IP_PORT_PROTO connectionDraining: type: object + additionalProperties: false description: The connection draining settings. properties: drainingTimeoutSec: diff --git a/dm/templates/ip_reservation/examples/ip_reservation.yaml b/dm/templates/ip_reservation/examples/ip_reservation.yaml index 2fb129a8623..722501cbb31 100644 --- a/dm/templates/ip_reservation/examples/ip_reservation.yaml +++ b/dm/templates/ip_reservation/examples/ip_reservation.yaml @@ -17,14 +17,14 @@ resources: properties: ipAddresses: - name: myglobal - ipType: global + ipType: GLOBAL description: 'my global ip' - name: myregionalexternal - ipType: regional + ipType: REGIONAL region: description: 'my static external ip' - name: myinternal - ipType: internal + ipType: INTERNAL # This IP address must be within the subnet range. address: 10.128.1.111 subnetwork: projects//regions//subnetworks/ diff --git a/dm/templates/ip_reservation/ip_address.py b/dm/templates/ip_reservation/ip_address.py index 359c009635e..3d6cf5d60c4 100644 --- a/dm/templates/ip_reservation/ip_address.py +++ b/dm/templates/ip_reservation/ip_address.py @@ -27,35 +27,48 @@ def get_resource_type(ip_type): """ Return the address resource type. """ if ip_type == 'GLOBAL': - return 'compute.v1.globalAddress' + # https://cloud.google.com/compute/docs/reference/rest/v1/globalAddresses + return 'gcp-types/compute-v1:globalAddresses' - return 'compute.v1.address' + # https://cloud.google.com/compute/docs/reference/rest/v1/addresses + return 'gcp-types/compute-v1:addresses' def generate_config(context): """ Entry point for the deployment resources. """ - + properties = context.properties resource_type = get_resource_type(context.properties['ipType']) address_type = get_address_type(context.properties['ipType']) name = context.properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) - properties = { + res_properties = { 'addressType': address_type, 'resourceType': 'addresses', + 'project': project_id, } - optional_properties = ['subnetwork', 'address', 'description', 'region'] + optional_properties = [ + 'subnetwork', + 'address', + 'description', + 'region', + 'networkTier', + 'prefixLength', + 'ipVersion', + 'purpose', + ] for prop in optional_properties: if prop in context.properties: - properties[prop] = str(context.properties[prop]) + res_properties[prop] = str(context.properties[prop]) resources = [ { 'name': name, 'type': resource_type, - 'properties': properties + 'properties': res_properties } ] diff --git a/dm/templates/ip_reservation/ip_address.py.schema b/dm/templates/ip_reservation/ip_address.py.schema index 681154c8670..7bfcbdfc7a9 100644 --- a/dm/templates/ip_reservation/ip_address.py.schema +++ b/dm/templates/ip_reservation/ip_address.py.schema @@ -15,7 +15,18 @@ info: title: IP Address author: Sourced Group Inc. - description: Creates an internal, external, or global IP address. + version: 1.0.0 + description: | + Creates an internal, external, or global IP address. + + For more information on this resource: + https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address + + APIs endpoints used by this template: + - gcp-types/compute-v1:globalAddresses => + https://cloud.google.com/compute/docs/reference/rest/v1/globalAddresses + - gcp-types/compute-v1:addresses => + https://cloud.google.com/compute/docs/reference/rest/v1/addresses additionalProperties: false @@ -23,11 +34,102 @@ required: - name - ipType +allOf: + - anyOf: + - allOf: + - properties: + purpose: + enum: ["GCE_ENDPOINT", "DNS_RESOLVER"] + - required: + - purpose + - allOf: + - properties: + ipType: + enum: ["INTERNAL"] + - required: + - ipType + - not: + required: + - subnetwork + - anyOf: + - allOf: + - properties: + purpose: + enum: ["VPC_PEERING"] + - required: + - purpose + - not: + required: + - network + - anyOf: + - allOf: + - properties: + ipType: + enum: ["REGIONAL", "INTERNAL"] + - required: + - ipType + - not: + required: + - region + - anyOf: + - allOf: + - properties: + ipType: + enum: ["GLOBAL"] + - required: + - ipType + - not: + required: + - ipVersion + + properties: name: type: string description: | Name of the reserved IP; unique within the context of the project. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the IP. The + Google apps domain is prefixed if applicable. + prefixLength: + type: number + description: | + The prefix length if the resource reprensents an IP range. + networkTier: + type: string + description: | + This signifies the networking tier used for configuring this address and can only take the following + values: PREMIUM or STANDARD. Global forwarding rules can only be Premium Tier. Regional forwarding rules + can be either Premium or Standard Tier. Standard Tier addresses applied to regional forwarding rules + can be used with any external load balancer. Regional forwarding rules in Premium Tier can only + be used with a network load balancer. + + If this field is not specified, it is assumed to be PREMIUM. + ipVersion: + type: string + description: | + The IP version that will be used by this address. Valid options are IPV4 or IPV6. + This can only be specified for a global address. + enum: + - IPV4 + - IPV6 + purpose: + type: string + description: | + The purpose of this resource, which can be one of the following values: + + GCE_ENDPOINT for addresses that are used by VM instances, alias IP ranges, internal load balancers, and similar resources. + DNS_RESOLVER for a DNS resolver address in a subnetwork + VPC_PEERING for addresses that are reserved for VPC peer networks. + NAT_AUTO for addresses that are external IP addresses automatically reserved for Cloud NAT. + enum: + - GCE_ENDPOINT + - DNS_RESOLVER + - VPC_PEERING + - NAT_AUTO ipType: type: string description: | @@ -44,24 +146,29 @@ properties: description: type: string description: | - Descriptive information for the reserved IP. + An optional description of this resource. Provide this field when you create the resource. address: type: string description: | - [OPTIONAL] If the field value (IP address) is provided, Deployment + If the field value (IP address) is provided, Deployment Manager tries to reserve the specified IP address. If the field is not set, Deployment Manager reserves an internal IP address that is part of the subnet definition. + network: + type: string + description: | + The URL of the network in which to reserve the address. + This field can only be used with INTERNAL type with the VPC_PEERING purpose. subnetwork: type: string description: | - [REQUIRED FOR INTERNAL ADDRESSES] The subnet from whose range the - IP address is reserved. + The URL of the subnetwork in which to reserve the address. If an IP address is specified, + it must be within the subnetwork's IP range. This field can only be used with INTERNAL type with + a GCE_ENDPOINT or DNS_RESOLVER purpose. region: type: string description: | - [REQUIRED FOR INTERNAL ADDRESSES] The region where the regional - address resides. + The region where the regional address resides. outputs: properties: diff --git a/dm/templates/ip_reservation/ip_reservation.py.schema b/dm/templates/ip_reservation/ip_reservation.py.schema index 2330c753117..7d4c7cef19b 100644 --- a/dm/templates/ip_reservation/ip_reservation.py.schema +++ b/dm/templates/ip_reservation/ip_reservation.py.schema @@ -15,7 +15,18 @@ info: title: IP Reservation author: Sourced Group Inc. - description: Reservers internal, external, or global IP address. + version: 1.0.0 + description: | + Reservers internal, external, or global IP address. + + For more information on this resource: + https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address + + APIs endpoints used by this template: + - gcp-types/compute-v1:globalAddresses => + https://cloud.google.com/compute/docs/reference/rest/v1/globalAddresses + - gcp-types/compute-v1:addresses => + https://cloud.google.com/compute/docs/reference/rest/v1/addresses imports: - path: ../ip_reservation/ip_address.py @@ -29,6 +40,7 @@ required: properties: ipAddresses: type: array + uniqueItems: true description: | An array of IPs to create as defined by the `ip_address.py` template. Example: @@ -36,6 +48,95 @@ properties: ipType: regional region: description: 'my static external ip' + items: + type: object + additionalProperties: false + properties: + name: + type: string + description: | + Name of the reserved IP; unique within the context of the project. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the IP. The + Google apps domain is prefixed if applicable. + prefixLength: + type: number + description: | + The prefix length if the resource reprensents an IP range. + networkTier: + type: string + description: | + This signifies the networking tier used for configuring this address and can only take the following + values: PREMIUM or STANDARD. Global forwarding rules can only be Premium Tier. Regional forwarding rules + can be either Premium or Standard Tier. Standard Tier addresses applied to regional forwarding rules + can be used with any external load balancer. Regional forwarding rules in Premium Tier can only + be used with a network load balancer. + + If this field is not specified, it is assumed to be PREMIUM. + ipVersion: + type: string + description: | + The IP version that will be used by this address. Valid options are IPV4 or IPV6. + This can only be specified for a global address. + enum: + - IPV4 + - IPV6 + purpose: + type: string + description: | + The purpose of this resource, which can be one of the following values: + + GCE_ENDPOINT for addresses that are used by VM instances, alias IP ranges, internal load balancers, and similar resources. + DNS_RESOLVER for a DNS resolver address in a subnetwork + VPC_PEERING for addresses that are reserved for VPC peer networks. + NAT_AUTO for addresses that are external IP addresses automatically reserved for Cloud NAT. + enum: + - GCE_ENDPOINT + - DNS_RESOLVER + - VPC_PEERING + - NAT_AUTO + ipType: + type: string + description: | + The IP types the user can reserve. + - GLOBAL - for global entities; the IPs can only be used with global + forwarding rules (GLB) + - REGIONAL - static external IPs that reside in a region + - INTERNAL - static internal (RFC1918) IPs that reside in a region on a + subnet + enum: + - GLOBAL + - REGIONAL + - INTERNAL + description: + type: string + description: | + An optional description of this resource. Provide this field when you create the resource. + address: + type: string + description: | + If the field value (IP address) is provided, Deployment + Manager tries to reserve the specified IP address. If the field is + not set, Deployment Manager reserves an internal IP address that + is part of the subnet definition. + network: + type: string + description: | + The URL of the network in which to reserve the address. + This field can only be used with INTERNAL type with the VPC_PEERING purpose. + subnetwork: + type: string + description: | + The URL of the subnetwork in which to reserve the address. If an IP address is specified, + it must be within the subnetwork's IP range. This field can only be used with INTERNAL type with + a GCE_ENDPOINT or DNS_RESOLVER purpose. + region: + type: string + description: | + The region where the regional address resides. outputs: properties: diff --git a/dm/templates/kms/kms.py b/dm/templates/kms/kms.py index 57ab03a2511..ab92f4ed250 100644 --- a/dm/templates/kms/kms.py +++ b/dm/templates/kms/kms.py @@ -21,16 +21,18 @@ def generate_config(context): resources = [] properties = context.properties + project_id = properties.get('project', context.env['project']) parent = 'projects/{}/locations/{}'.format( - context.env['project'], + project_id, properties.get('region') ) - keyring_name = properties.get('keyRingName') or context.env['name'].lower() + keyring_name = properties.get('keyRingName', context.env['name']) keyring_id = '{}/keyRings/{}'.format(parent, keyring_name) + # https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings provider = 'gcp-types/cloudkms-v1:projects.locations.keyRings' # keyring resource keyring = { - 'name': keyring_name, + 'name': context.env['name'], 'type': provider, 'properties': { 'parent': parent, @@ -42,8 +44,9 @@ def generate_config(context): # cryptographic key resources for key in properties.get('keys', []): key_name = key['cryptoKeyName'].lower() + key_resource = '{}-{}'.format(context.env['name'], key_name) crypto_key = { - 'name': key_name, + 'name': key_resource, 'type': provider + '.cryptoKeys', 'properties': { @@ -54,7 +57,7 @@ def generate_config(context): {}) }, 'metadata': { - 'dependsOn': [keyring_name] + 'dependsOn': [context.env['name']] } } @@ -66,11 +69,13 @@ def generate_config(context): # IAM policy bindings for the crypto key if 'iamPolicyBinding' in key: + provider = 'gcp-types/cloudkms-v1:cloudkms.projects.locations.keyRings' key_resource_name = '{}/cryptoKeys/{}'.format(keyring_id, key_name) - action_type = 'gcp-types/cloudkms-v1:cloudkms.projects.locations' crypto_key_iam = { - 'name': '{}-iamPolicy'.format(key_name), - 'action': action_type + '.keyRings.cryptoKeys.setIamPolicy', + 'name': '{}-iamPolicy'.format(key_resource), + # https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys/setIamPolicy + # https://cloudkms.googleapis.com/$discovery/rest?version=v1 + 'action': provider + '.cryptoKeys.setIamPolicy', 'properties': { 'resource': key_resource_name, @@ -79,7 +84,7 @@ def generate_config(context): } }, 'metadata': { - 'dependsOn': [key_name] + 'dependsOn': [key_resource] } } resources.append(crypto_key_iam) @@ -91,7 +96,7 @@ def generate_config(context): [ { 'name': 'keyRing', - 'value': '$(ref.{}.name)'.format(keyring_name) + 'value': '$(ref.{}.name)'.format(context.env['name']) } ] } diff --git a/dm/templates/kms/kms.py.schema b/dm/templates/kms/kms.py.schema index 818d417860e..f1be73e2d10 100644 --- a/dm/templates/kms/kms.py.schema +++ b/dm/templates/kms/kms.py.schema @@ -14,12 +14,20 @@ info: title: Google Cloud KMS KeyRing and Keys + version: 1.0.0 author: Sourced Group Inc. description: | Creates a Cloud KMS KeyRing and cryptographic keys. - For more information on this resource, + + For more information on this resource: https://cloud.google.com/kms/docs/reference/rest. + APIs endpoints used by this template: + - gcp-types/cloudkms-v1:projects.locations.keyRings => + https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings + - gcp-types/cloudkms-v1:cloudkms.projects.locations => + https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings + imports: - path: kms.py @@ -34,14 +42,20 @@ properties: pattern: ^[a-zA-Z0-9_-]{1,63} description: | The name for the KeyRing. Must be unique within a location. + project: + type: string + description: | + The project ID of the project containing the keyring. region: type: string default: global description: The KeyRing location. keys: type: array + uniqueItems: True items: type: object + additionalProperties: false description: The CryptoKey object. required: - cryptoKeyName @@ -82,6 +96,7 @@ properties: with up to nine fractional digits, terminated by 's'. Example '3.5s' versionTemplate: type: object + additionalProperties: false description: | The template that controls properties of new CryptoKeyVersion instances created by either cryptoKeyVersions.create or @@ -109,11 +124,15 @@ properties: - RSA_SIGN_PSS_2048_SHA256 - RSA_SIGN_PSS_3072_SHA256 - RSA_SIGN_PSS_4096_SHA256 + - RSA_SIGN_PSS_4096_SHA512 - RSA_SIGN_PKCS1_2048_SHA256 - RSA_SIGN_PKCS1_3072_SHA256 - RSA_SIGN_PKCS1_4096_SHA256 + - RSA_SIGN_PKCS1_4096_SHA512 - RSA_DECRYPT_OAEP_2048_SHA256 - RSA_DECRYPT_OAEP_3072_SHA256 + - RSA_DECRYPT_OAEP_4096_SHA256 + - RSA_DECRYPT_OAEP_4096_SHA512 - EC_SIGN_P256_SHA256 - EC_SIGN_P384_SHA384 labels: @@ -123,9 +142,11 @@ properties: https://cloud.google.com/kms/docs/labeling-keys. iamPolicyBinding: type: array + uniqueItems: True description: The IAM bindings for the CryptoKey. items: type: object + additionalProperties: false required: - role - members diff --git a/dm/templates/logsink/logsink.py b/dm/templates/logsink/logsink.py index 720b36b4a4a..1809568bf87 100644 --- a/dm/templates/logsink/logsink.py +++ b/dm/templates/logsink/logsink.py @@ -17,9 +17,14 @@ def create_pubsub(context, logsink_name): """ Create the pubsub destination. """ + properties = context.properties + project_id = properties.get('project', context.env['project']) + dest_properties = [] if 'pubsubProperties' in context.properties: dest_prop = context.properties['pubsubProperties'] + dest_prop['name'] = context.properties['destinationName'] + dest_prop['project'] = project_id access_control = dest_prop.get('accessControl', []) access_control.append( { @@ -31,7 +36,7 @@ def create_pubsub(context, logsink_name): dest_prop['accessControl'] = access_control dest_properties = [ { - 'name': context.properties['destinationName'], + 'name': '{}-pubsub'.format(context.env['name']), 'type': 'pubsub.py', 'properties': dest_prop } @@ -43,10 +48,14 @@ def create_pubsub(context, logsink_name): def create_bq_dataset(context, logsink_name): """ Create the BQ dataset destination. """ + properties = context.properties + project_id = properties.get('project', context.env['project']) + dest_properties = [] if 'bqProperties' in context.properties: dest_prop = context.properties['bqProperties'] dest_prop['name'] = context.properties['destinationName'] + dest_prop['project'] = project_id access = dest_prop.get('access', []) access.append( { @@ -58,7 +67,7 @@ def create_bq_dataset(context, logsink_name): dest_prop['access'] = access dest_properties = [ { - 'name': context.properties['destinationName'], + 'name': '{}-bigquery-dataset'.format(context.env['name']), 'type': 'bigquery_dataset.py', 'properties': dest_prop } @@ -70,11 +79,15 @@ def create_bq_dataset(context, logsink_name): def create_storage(context, logsink_name): """ Create the bucket destination. """ + properties = context.properties + project_id = properties.get('project', context.env['project']) + dest_properties = [] if 'storageProperties' in context.properties: bucket_name = context.properties['destinationName'] dest_prop = context.properties['storageProperties'] dest_prop['name'] = bucket_name + dest_prop['project'] = project_id bindings = dest_prop.get('bindings', []) bindings.append( { @@ -88,20 +101,22 @@ def create_storage(context, logsink_name): if 'bindings' in dest_prop: del dest_prop['bindings'] + name = '{}-bucket'.format(context.env['name']) dest_properties = [ { # Create the GCS Bucket - 'name': bucket_name, + 'name': name, 'type': 'gcs_bucket.py', 'properties': dest_prop }, { # Give the logsink writerIdentity permissions to the bucket - 'name': bucket_name + '-logging-storage-iampolicy', + 'name': '{}-iampolicy'.format(name), + # https://cloud.google.com/storage/docs/json_api/v1/buckets/setIamPolicy 'action': 'gcp-types/storage-v1:storage.buckets.setIamPolicy', 'properties': { - 'bucket': '$(ref.' + bucket_name + '.name)', + 'bucket': '$(ref.{}.name)'.format(name), 'project': context.env['project'], 'bindings': bindings } @@ -114,13 +129,14 @@ def create_storage(context, logsink_name): def generate_config(context): """ Entry point for the deployment resources. """ - project_id = context.env['project'] - name = context.properties.get('name', context.env['name']) + properties = context.properties + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) properties = { 'name': name, 'uniqueWriterIdentity': context.properties['uniqueWriterIdentity'], - 'sink': name + 'sink': name, } if 'orgId' in context.properties: @@ -169,9 +185,13 @@ def generate_config(context): if sink_filter: properties['filter'] = sink_filter + # https://cloud.google.com/logging/docs/reference/v2/rest/v2/folders.sinks + # https://cloud.google.com/logging/docs/reference/v2/rest/v2/billingAccounts.sinks + # https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.sinks + # https://cloud.google.com/logging/docs/reference/v2/rest/v2/organizations.sinks base_type = 'gcp-types/logging-v2:' resource = { - 'name': name, + 'name': context.env['name'], 'type': base_type + source_type + '.sinks', 'properties': properties } @@ -184,7 +204,7 @@ def generate_config(context): # GCS Bucket needs to be created first before the sink whereas # pub/sub and BQ do not. This might change in the future. resource['metadata'] = { - 'dependsOn': [context.properties['destinationName']] + 'dependsOn': [dest_properties[0]['name']] } return { @@ -194,7 +214,7 @@ def generate_config(context): [ { 'name': 'writerIdentity', - 'value': '$(ref.{}.writerIdentity)'.format(name) + 'value': '$(ref.{}.writerIdentity)'.format(context.env['name']) } ] } diff --git a/dm/templates/logsink/logsink.py.schema b/dm/templates/logsink/logsink.py.schema index 4714a6f61f0..4d283d0d8b3 100644 --- a/dm/templates/logsink/logsink.py.schema +++ b/dm/templates/logsink/logsink.py.schema @@ -15,9 +15,25 @@ info: title: Logging Sink author: Sourced Group Inc. + version: 1.0.0 description: | Creates a logging sink to export entries to a desired destination. + For more information on this resource: + - https://cloud.google.com/logging/docs/reference/v2/rest/ + + APIs endpoints used by this template: + - gcp-types/storage-v1:storage.buckets.setIamPolicy => + https://cloud.google.com/storage/docs/json_api/v1/buckets/setIamPolicy + - gcp-types/logging-v2:folders.sinks => + https://cloud.google.com/logging/docs/reference/v2/rest/v2/folders.sinks + - gcp-types/logging-v2:billingAccounts.sinks => + https://cloud.google.com/logging/docs/reference/v2/rest/v2/billingAccounts.sinks + - gcp-types/logging-v2:projects.sinks => + https://cloud.google.com/logging/docs/reference/v2/rest/v2/projects.sinks + - gcp-types/logging-v2:organizations.sinks => + https://cloud.google.com/logging/docs/reference/v2/rest/v2/organizations.sinks + imports: - path: ../pubsub/pubsub.py name: pubsub.py @@ -32,10 +48,26 @@ required: - destinationType - destinationName +oneOf: + - required: + - projectId + - required: + - orgId + - required: + - billingAccountId + - required: + - folderId + properties: name: type: string - description: Name of the sink resource. + description: | + Name of the sink. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. destinationName: type: string description: | @@ -123,6 +155,30 @@ properties: access: - role: OWNER userByEmail: my-email@email.com + projectId: + type: + - string + - number + description: | + Project ID to add sink to + orgId: + type: + - string + - number + description: | + Org ID to add sink to + billingAccountId: + type: + - string + - number + description: | + Billing account ID to add sink to + folderId: + type: + - string + - number + description: | + Folder ID to add sink to documentation: - templates/logsink/README.md diff --git a/dm/templates/logsink/tests/integration/logsink.bats b/dm/templates/logsink/tests/integration/logsink.bats index 9e739d712b5..0a4c1bc5f4b 100644 --- a/dm/templates/logsink/tests/integration/logsink.bats +++ b/dm/templates/logsink/tests/integration/logsink.bats @@ -51,7 +51,7 @@ function setup() { --organization="${CLOUD_FOUNDATION_ORGANIZATION_ID}" get_test_folder_id create_config - gcloud pubsub topics create test-topic-${RAND} + gcloud pubsub topics create test-${RAND} gsutil mb -l us-east1 gs://test-bucket-${RAND}/ bq mk test_dataset_${RAND} fi @@ -80,11 +80,19 @@ function teardown() { run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --config "${CONFIG}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] } @test "Verifying project sinks were created each with the requested destination in deployment ${DEPLOYMENT_NAME}" { run gcloud logging sinks list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "test-logsink-project-bq-${RAND}" ]] [[ "$output" =~ "test-logsink-project-pubsub-${RAND}" ]] @@ -94,6 +102,10 @@ function teardown() { @test "Verifying organization sinks were created each with a different as the destination in deployment ${DEPLOYMENT_NAME}" { run gcloud logging sinks list \ --organization "${CLOUD_FOUNDATION_ORGANIZATION_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "test-logsink-org-bq-${RAND}" ]] [[ "$output" =~ "test-logsink-org-pubsub-${RAND}" ]] @@ -103,6 +115,10 @@ function teardown() { @test "Verifying billing account sinks were created each with a different as the destination in deployment ${DEPLOYMENT_NAME}" { run gcloud logging sinks list --billing-account \ "${CLOUD_FOUNDATION_BILLING_ACCOUNT_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "test-logsink-billing-bq-${RAND}" ]] [[ "$output" =~ "test-logsink-billing-pubsub-${RAND}" ]] @@ -111,6 +127,10 @@ function teardown() { @test "Verifying folder sinks were created each with a different as the destination in deployment ${DEPLOYMENT_NAME}" { run gcloud logging sinks list --folder "${TEST_ORG_FOLDER_NAME}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "test-logsink-folder-bq-${RAND}" ]] [[ "$output" =~ "test-logsink-folder-pubsub-${RAND}" ]] @@ -119,19 +139,31 @@ function teardown() { @test "Verifying project sinks and the destination resource were created in deployment ${DEPLOYMENT_NAME}" { run gcloud logging sinks list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] #[[ "$output" =~ "test-logsink-project-bq-${RAND}" ]] [[ "$output" =~ "test-logsink-project-pubsub-create-${RAND}" ]] [[ "$output" =~ "test-logsink-project-storage-create-${RAND}" ]] run gcloud beta pubsub topics get-iam-policy \ - "test-logsink-project-pubsub-topic-dest-${RAND}" + "test-logsink-project-pubsub-dest-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "@gcp-sa-logging.iam.gserviceaccount.com" ]] [[ "$output" =~ "user:${CLOUD_FOUNDATION_USER_ACCOUNT}" ]] [[ "$output" =~ "role: roles/pubsub.admin" ]] run gsutil iam get "gs://test-logsink-project-storage-dest-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "@gcp-sa-logging.iam.gserviceaccount.com" ]] [[ "$output" =~ "roles/storage.admin" ]] @@ -144,18 +176,30 @@ function teardown() { @test "Verifying org sinks and the destination resource were created in deployment ${DEPLOYMENT_NAME}" { run gcloud logging sinks list \ --organization "${CLOUD_FOUNDATION_ORGANIZATION_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] #[[ "$output" =~ "test-logsink-org-bq-${RAND}" ]] [[ "$output" =~ "test-logsink-org-pubsub-create-${RAND}" ]] [[ "$output" =~ "test-logsink-org-storage-create-${RAND}" ]] - run gcloud beta pubsub topics get-iam-policy "test-logsink-org-pubsub-topic-dest-${RAND}" + run gcloud beta pubsub topics get-iam-policy "test-logsink-org-pubsub-dest-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "@gcp-sa-logging.iam.gserviceaccount.com" ]] [[ "$output" =~ "user:${CLOUD_FOUNDATION_USER_ACCOUNT}" ]] [[ "$output" =~ "role: roles/pubsub.admin" ]] run gsutil iam get "gs://test-logsink-org-storage-dest-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "@gcp-sa-logging.iam.gserviceaccount.com" ]] [[ "$output" =~ "roles/storage.admin" ]] @@ -168,19 +212,31 @@ function teardown() { @test "Verifying billing sinks and the destination resource were created in deployment ${DEPLOYMENT_NAME}" { run gcloud logging sinks list \ --billing-account "${CLOUD_FOUNDATION_BILLING_ACCOUNT_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] #[[ "$output" =~ "test-logsink-billing-bq-${RAND}" ]] [[ "$output" =~ "test-logsink-billing-pubsub-create-${RAND}" ]] [[ "$output" =~ "test-logsink-billing-storage-create-${RAND}" ]] run gcloud beta pubsub topics get-iam-policy \ - "test-logsink-billing-pubsub-topic-dest-${RAND}" + "test-logsink-billing-pubsub-dest-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "@gcp-sa-logging.iam.gserviceaccount.com" ]] [[ "$output" =~ "user:${CLOUD_FOUNDATION_USER_ACCOUNT}" ]] [[ "$output" =~ "role: roles/pubsub.admin" ]] run gsutil iam get "gs://test-logsink-billing-storage-dest-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "@gcp-sa-logging.iam.gserviceaccount.com" ]] [[ "$output" =~ "roles/storage.admin" ]] @@ -192,19 +248,31 @@ function teardown() { @test "Verifying folder sinks and the destination resource were created in deployment ${DEPLOYMENT_NAME}" { run gcloud logging sinks list --folder "${TEST_ORG_FOLDER_NAME}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] #[[ "$output" =~ "test-logsink-folder-bq-${RAND}" ]] [[ "$output" =~ "test-logsink-folder-pubsub-create-${RAND}" ]] [[ "$output" =~ "test-logsink-folder-storage-create-${RAND}" ]] run gcloud beta pubsub topics get-iam-policy \ - "test-logsink-folder-pubsub-topic-dest-${RAND}" + "test-logsink-folder-pubsub-dest-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "@gcp-sa-logging.iam.gserviceaccount.com" ]] [[ "$output" =~ "user:${CLOUD_FOUNDATION_USER_ACCOUNT}" ]] [[ "$output" =~ "role: roles/pubsub.admin" ]] run gsutil iam get "gs://test-logsink-folder-storage-dest-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] [[ "$output" =~ "@gcp-sa-logging.iam.gserviceaccount.com" ]] [[ "$output" =~ "roles/storage.admin" ]] @@ -217,16 +285,28 @@ function teardown() { @test "Deleting deployment" { run gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" -q \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] run gcloud logging sinks list --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ "$status" -eq 0 ]] + + echo "Status: $status" + echo "Output: $output" + #[[ ! "$output" =~ "test-logsink-project-bq-${RAND}" ]] [[ ! "$output" =~ "test-logsink-project-pubsub-${RAND}" ]] [[ ! "$output" =~ "test-logsink-project-storage-${RAND}" ]] run gcloud logging sinks list \ --organization "${CLOUD_FOUNDATION_ORGANIZATION_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] #[[ ! "$output" =~ "test-logsink-org-bq-${RAND}" ]] [[ ! "$output" =~ "test-logsink-org-pubsub-${RAND}" ]] @@ -242,6 +322,10 @@ function teardown() { #[[ ! "$output" =~ "test-logsink-billing-storage-${RAND}" ]] run gcloud logging sinks list --folder "${TEST_ORG_FOLDER_NAME}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] #[[ ! "$output" =~ "test-logsink-folder-bq-${RAND}" ]] [[ ! "$output" =~ "test-logsink-folder-pubsub-${RAND}" ]] diff --git a/dm/templates/managed_instance_group/examples/managed_instance_group.yaml b/dm/templates/managed_instance_group/examples/managed_instance_group.yaml index ea72fda9566..637890ff9b0 100644 --- a/dm/templates/managed_instance_group/examples/managed_instance_group.yaml +++ b/dm/templates/managed_instance_group/examples/managed_instance_group.yaml @@ -19,5 +19,7 @@ resources: instanceTemplate: diskImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-1804-lts networks: - - name: default + - network: default + accessConfigs: + - type: ONE_TO_ONE_NAT machineType: f1-micro diff --git a/dm/templates/managed_instance_group/examples/managed_instance_group_healthcheck.yaml b/dm/templates/managed_instance_group/examples/managed_instance_group_healthcheck.yaml index e9e21d90874..2c8833f0851 100644 --- a/dm/templates/managed_instance_group/examples/managed_instance_group_healthcheck.yaml +++ b/dm/templates/managed_instance_group/examples/managed_instance_group_healthcheck.yaml @@ -25,7 +25,10 @@ resources: targetSize: 3 instanceTemplate: diskImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-1804-lts - network: default + networks: + - network: default + accessConfigs: + - type: ONE_TO_ONE_NAT machineType: f1-micro healthChecks: - initialDelaySec: 500 diff --git a/dm/templates/managed_instance_group/managed_instance_group.py b/dm/templates/managed_instance_group/managed_instance_group.py index 24cbb4bd293..63683e59d69 100644 --- a/dm/templates/managed_instance_group/managed_instance_group.py +++ b/dm/templates/managed_instance_group/managed_instance_group.py @@ -16,8 +16,10 @@ import copy REGIONAL_LOCAL_IGM_TYPES = { - True: 'compute.v1.regionInstanceGroupManager', - False: 'compute.v1.instanceGroupManager' + # https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers + True: 'gcp-types/compute-v1:regionInstanceGroupManagers', + # https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers + False: 'gcp-types/compute-v1:instanceGroupManagers' } @@ -63,14 +65,15 @@ def get_instance_template(properties, name_prefix): return create_instance_template(properties, name_prefix) -def create_autoscaler(autoscaler_spec, igm): +def create_autoscaler(context, autoscaler_spec, igm): """ Creates an autoscaler. """ - igm_name = igm['name'] igm_properties = igm['properties'] autoscaler_properties = autoscaler_spec.copy() - name = autoscaler_properties.get('name', igm_name + '-autoscaler') + name = '{}-autoscaler'.format(context.env['name']) + + autoscaler_properties['project'] = context.properties.get('project', context.env['project']) autoscaler_resource = { 'type': 'autoscaler.py', @@ -85,7 +88,7 @@ def create_autoscaler(autoscaler_spec, igm): min_size = autoscaler_properties.pop('minSize') autoscaler_properties['minNumReplicas'] = min_size - autoscaler_properties['target'] = '$(ref.{}.selfLink)'.format(igm_name) + autoscaler_properties['target'] = '$(ref.{}.selfLink)'.format(context.env['name']) for location in ['zone', 'region']: set_optional_property(autoscaler_properties, igm_properties, location) @@ -98,12 +101,12 @@ def create_autoscaler(autoscaler_spec, igm): return [autoscaler_resource], [autoscaler_output] -def get_autoscaler(properties, igm): +def get_autoscaler(context, igm): """ Creates an autoscaler, if necessary. """ - autoscaler_spec = properties.get('autoscaler') + autoscaler_spec = context.properties.get('autoscaler') if autoscaler_spec: - return create_autoscaler(autoscaler_spec, igm) + return create_autoscaler(context, autoscaler_spec, igm) return [], [] @@ -150,8 +153,8 @@ def is_reference(candidate): def create_health_checks_assignment(healthchecks, igm_resource, project): """ Create resource for IGMs health checks assignment. """ - igm_name = igm_resource['name'] igm_properties = igm_resource['properties'] + igm_name = igm_properties['name'] properties = { 'instanceGroupManager': igm_name, @@ -162,6 +165,8 @@ def create_health_checks_assignment(healthchecks, igm_resource, project): dependencies = [] metadata = {'dependsOn': dependencies} # Have to use a type-provider for health checks assignment + # https://cloud.google.com/compute/docs/reference/rest/beta/regionInstanceGroupManagers/setAutoHealingPolicies + # https://cloud.google.com/compute/docs/reference/rest/beta/instanceGroupManagers/setAutoHealingPolicies type_provider = 'gcp-types/compute-beta' action = '{}:compute.{}GroupManagers.setAutoHealingPolicies'.format( type_provider, @@ -170,7 +175,7 @@ def create_health_checks_assignment(healthchecks, igm_resource, project): assign_healthcheck_resource = { 'action': action, - 'name': igm_name + '-set-hc', + 'name': igm_resource['name'] + '-set-hc', 'properties': properties, 'metadata': metadata } @@ -188,7 +193,7 @@ def create_health_checks_assignment(healthchecks, igm_resource, project): # setAutoHealingPolicies depends both on the health checks and IGM # resource - dependencies.append(igm_name) + dependencies.append(igm_resource['name']) for location in ['region', 'zone']: set_optional_property(properties, igm_properties, location) @@ -210,15 +215,22 @@ def get_health_checks(properties, igm_resource, project): return [] -def get_igm(properties, name, template_link): +def get_igm(context, template_link): """ Creates the IGM resource with its outputs. """ + properties = context.properties + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) is_regional = 'region' in properties - igm_properties = {'instanceTemplate': template_link} + igm_properties = { + 'name': name, + 'project': project_id, + 'instanceTemplate': template_link, + } igm = { - 'name': name, + 'name': context.env['name'], 'type': REGIONAL_LOCAL_IGM_TYPES[is_regional], 'properties': igm_properties } @@ -236,7 +248,7 @@ def get_igm(properties, name, template_link): for prop in known_properties: set_optional_property(igm_properties, properties, prop) - outputs = get_igm_outputs(name, igm_properties) + outputs = get_igm_outputs(context.env['name'], igm_properties) return [igm], outputs @@ -245,23 +257,23 @@ def generate_config(context): """ Entry point for the deployment resources. """ properties = context.properties - name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) # Instance template - template = get_instance_template(properties['instanceTemplate'], name) + properties['instanceTemplate']['project'] = project_id + template = get_instance_template(properties['instanceTemplate'], context.env['name']) template_link, template_resources, template_outputs = template # Instance group manager - igm_resources, igm_outputs = get_igm(properties, name, template_link) + igm_resources, igm_outputs = get_igm(context, template_link) igm = igm_resources[0] # Autoscaler - autoscaler = get_autoscaler(properties, igm) + autoscaler = get_autoscaler(context, igm) autoscaler_resources, autoscaler_outputs = autoscaler # Health checks - project = context.env['project'] - healthcheck_resources = get_health_checks(properties, igm, project) + healthcheck_resources = get_health_checks(properties, igm, project_id) return { 'resources': diff --git a/dm/templates/managed_instance_group/managed_instance_group.py.schema b/dm/templates/managed_instance_group/managed_instance_group.py.schema index 83cfdaf8d8f..6ef3c8a364f 100644 --- a/dm/templates/managed_instance_group/managed_instance_group.py.schema +++ b/dm/templates/managed_instance_group/managed_instance_group.py.schema @@ -15,9 +15,23 @@ info: title: Managed Instance Group author: Sourced Group Inc. + version: 1.0.0 description: | Creates a managed instance group with or without an autoscaler. + For more information on this resource: + https://cloud.google.com/compute/docs/instance-groups/ + + APIs endpoints used by this template: + - gcp-types/compute-v1:instanceGroupManagers => + https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers + - gcp-types/compute-v1:regionInstanceGroupManagers => + https://cloud.google.com/compute/docs/reference/rest/v1/regionInstanceGroupManagers + - gcp-types/compute-beta:compute.regionInstanceGroupManagers.setAutoHealingPolicies => + https://cloud.google.com/compute/docs/reference/rest/beta/regionInstanceGroupManagers/setAutoHealingPolicies + - gcp-types/compute-beta:compute.instanceGroupManagers.setAutoHealingPolicies => + https://cloud.google.com/compute/docs/reference/rest/beta/instanceGroupManagers/setAutoHealingPolicies + imports: - path: ../autoscaler/autoscaler.py name: autoscaler.py @@ -53,28 +67,51 @@ definitions: specify a static external IP address, it must live in the same region as the zone of the instance. If hasExternalIp is false this field is ignored. + network: + type: string + description: | + URL of the network resource for this instance. When creating an instance, if neither the network + nor the subnetwork is specified, the default network global/networks/default is used; + if the network is not specified but the subnetwork is specified, the network is inferred. + + If you specify this property, you can specify the network as a full or partial URL. + For example, the following are all valid URLs: + + - https://www.googleapis.com/compute/v1/projects/project/global/networks/network + - projects/project/global/networks/network + - global/networks/default + Authorization requires one or more of the following Google IAM permissions on the specified resource network: + + - compute.networks.use + - compute.networks.useExternalIp subnetwork: type: string description: | - The URL of the Subnetwork resource for this instance. If the network - resource is in legacy mode, do not provide this property. If the network - is in auto subnet mode, providing the subnetwork is optional. If the - network is in custom subnet mode, then this field should be specified. - If you specify this property, you can specify the subnetwork as a full - or partial URL. For example, the following are all valid URLs: - - https://www.googleapis.com/compute/v1/projects/project/regions/region/subnetworks/subnetwork - - regions/region/subnetworks/subnetwork + The URL of the Subnetwork resource for this instance. If the network resource is in legacy mode, + do not specify this field. If the network is in auto subnet mode, specifying the subnetwork is optional. + If the network is in custom subnet mode, specifying the subnetwork is required. + If you specify this field, you can specify the subnetwork as a full or partial URL. For example, the following are all valid URLs: + + - https://www.googleapis.com/compute/v1/projects/project/regions/region/subnetworks/subnetwork + - regions/region/subnetworks/subnetwork + Authorization requires one or more of the following Google IAM permissions on the specified resource subnetwork: + + - compute.subnetworks.use + - compute.subnetworks.useExternalIp networkIP: type: string description: | - An IPv4 internal network address to assign to the instance for this - network interface. If not specified by the user, an unused internal IP - is assigned by the system. + An IPv4 internal IP address to assign to the instance for this network interface. + If not specified by the user, an unused internal IP is assigned by the system. properties: name: type: string - description: The name of the managed instance group. + description: The name of the managed instance group. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the instance. description: type: string description: An optional description of the resource. @@ -99,11 +136,13 @@ properties: the group. namedPorts: type: array + uniqueItems: true description: | A list of the named ports configured for the instance groups complementary to the Instance Group Manager. items: type: object + additionalProperties: false required: - name - port @@ -188,20 +227,18 @@ properties: The description of the instance resource the instance template will create (optional). network: - type: string - description: | - Name of the network the instance will be connected to; - e.g., 'my-custom-network' or 'default'. - hasExternalIp: - $ref: '#/definitions/hasExternalIp' - natIP: - $ref: '#/definitions/natIP' + $ref: '#/definitions/network' subnetwork: $ref: '#/definitions/subnetwork' networkIP: $ref: '#/definitions/networkIP' + hasExternalIp: + $ref: '#/definitions/hasExternalIp' + natIP: + $ref: '#/definitions/natIP' networks: type: array + uniqueItems: true description: | Networks the instance will be connected to; e.g., 'my-custom-network' or 'default'. @@ -209,21 +246,506 @@ properties: type: object additionalProperties: false required: - - name + - network properties: - name: - type: string - description: | - Name of the network the instance will be connected to; - e.g., 'my-custom-network' or 'default'. - hasExternalIp: - $ref: '#/definitions/hasExternalIp' - natIP: - $ref: '#/definitions/natIP' + network: + $ref: '#/definitions/network' subnetwork: $ref: '#/definitions/subnetwork' networkIP: $ref: '#/definitions/networkIP' + aliasIpRanges: + type: array + uniqueItems: true + description: | + An array of alias IP ranges for this network interface. You can only specify this + field for network interfaces in VPC networks. + items: + type: object + additionalProperties: false + properties: + ipCidrRange: + type: string + description: | + The IP alias ranges to allocate for this interface. This IP CIDR range must belong + to the specified subnetwork and cannot contain IP addresses reserved by system or + used by other network interfaces. This range may be a single IP address (such as 10.2.3.4), + a netmask (such as /24) or a CIDR-formatted string (such as 10.1.2.0/24). + subnetworkRangeName: + type: string + description: | + The name of a subnetwork secondary IP range from which to allocate an IP alias range. + If not specified, the primary range of the subnetwork is used. + accessConfigs: + type: array + uniqueItems: true + description: | + An array of configurations for this interface. Currently, only one access config, ONE_TO_ONE_NAT, + is supported. If there are no accessConfigs specified, then this instance will have no external internet access. + items: + type: object + additionalProperties: false + properties: + type: + type: string + description: | + The type of configuration. The default and only option is ONE_TO_ONE_NAT. + enum: + - ONE_TO_ONE_NAT + name: + type: string + description: | + The name of this access configuration. The default and recommended name is External NAT, + but you can use any arbitrary string, such as My external IP or Network Access. + setPublicPtr: + type: boolean + description: | + Specifies whether a public DNS 'PTR' record should be created to map the external + IP address of the instance to a DNS domain name. + publicPtrDomainName: + type: string + description: | + The DNS domain name for the public PTR record. You can set this field only + if the setPublicPtr field is enabled. + networkTier: + type: string + description: | + This signifies the networking tier used for configuring this access configuration + and can only take the following values: PREMIUM, STANDARD. + + If an AccessConfig is specified without a valid external IP address, an + ephemeral IP will be created with this networkTier. + + If an AccessConfig with a valid external IP address is specified, it must match + that of the networkTier associated with the Address resource owning that IP. + enum: + - STANDARD + - PREMIUM + natIP: + $ref: '#/definitions/natIP' + disks: + type: array + uniqueItems: true + description: | + Array of disks associated with this instance. Persistent disks must be created before you can assign them. + items: + type: object + additionalProperties: false + oneOf: + - required: + - source + - required: + - initializeParams + - allOf: + - not: + required: + - source + - not: + required: + - initializeParams + properties: + type: + type: string + description: | + Specifies the type of the disk, either SCRATCH or PERSISTENT. If not specified, the default is PERSISTENT. + enum: + - SCRATCH + - PERSISTENT + mode: + type: string + description: | + The mode in which to attach this disk, either READ_WRITE or READ_ONLY. + If not specified, the default is to attach the disk in READ_WRITE mode. + enum: + - READ_WRITE + - READ_ONLY + source: + type: string + description: | + Specifies a valid partial or full URL to an existing Persistent Disk resource. + When creating a new instance, one of initializeParams.sourceImage or + disks.source is required except for local SSD. + + If desired, you can also attach existing non-root persistent disks using this property. + This field is only applicable for persistent disks. + + Note that for InstanceTemplate, specify the disk name, not the URL for the disk. + + Authorization requires one or more of the following Google IAM permissions on the specified resource source: + + compute.disks.use + compute.disks.useReadOnly + deviceName: + type: string + description: | + Specifies a unique device name of your choice that is reflected into the /dev/disk/by-id/google-* + tree of a Linux operating system running within the instance. This name can be used to reference + the device for mounting, resizing, and so on, from within the instance. + + If not specified, the server chooses a default device name to apply to this disk, in the + form persistent-disk-x, where x is a number assigned by Google Compute Engine. + This field is only applicable for persistent disks. + boot: + type: boolean + description: | + Indicates that this is a boot disk. The virtual machine will use the first partition + of the disk for its root filesystem. + initializeParams: + type: object + additionalProperties: false + description: | + Specifies the parameters for a new disk that will be created alongside the new instance. + Use initialization parameters to create boot disks or local SSDs attached to the new instance. + + This property is mutually exclusive with the source property; you can only define one or the other, but not both. + properties: + labels: + type: object + description: | + Labels to apply to this disk. These can be later modified by the disks.setLabels method. + This field is only applicable for persistent disks. + + An object containing a list of "key": value pairs. + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + + Authorization requires the following Google IAM permission on the specified resource labels: + + compute.disks.setLabels + diskName: + type: string + description: | + Specifies the disk name. If not specified, the default is to use the name of the instance. + If the disk with the instance name exists already in the given zone/region, + a new name will be automatically generated. + sourceImage: + type: string + description: | + The source image to create this disk. When creating a new instance, one of + initializeParams.sourceImage or disks.source is required except for local SSD. + + To create a disk with one of the public operating system images, specify the image by its family name. + For example, specify family/debian-9 to use the latest Debian 9 image: + + projects/debian-cloud/global/images/family/debian-9 + + Alternatively, use a specific version of a public operating system image: + + projects/debian-cloud/global/images/debian-9-stretch-vYYYYMMDD + + To create a disk with a custom image that you created, specify the image name in the following format: + + global/images/my-custom-image + + You can also specify a custom image by its image family, which returns the latest version of the + image in that family. Replace the image name with family/family-name: + + global/images/family/my-image-family + + If the source image is deleted later, this field will not be set. + + Authorization requires the following Google IAM permission on the specified resource sourceImage: + + compute.images.useReadOnly + description: + type: string + description: | + An optional description. Provide this property when creating the disk. + diskSizeGb: + type: number + description: | + Specifies the size of the disk in base-2 GB. + diskType: + type: string + description: | + Specifies the disk type to use to create the instance. If not specified, the default is pd-standard, + specified using the full URL. For example: + + https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/pd-standard + + Other values include pd-ssd and local-ssd. If you define this field, you can provide either the full + or partial URL. For example, the following are valid values: + + https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/diskType + projects/project/zones/zone/diskTypes/diskType + zones/zone/diskTypes/diskType + Note that for InstanceTemplate, this is the name of the disk type, not URL. + enum: + - pd-standard + - pd-ssd + - local-ssd + sourceImageEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source image. Required if the source image is + protected by a customer-supplied encryption key. + + Instance templates do not store customer-supplied encryption keys, so you cannot create disks + for instances in a managed instance group if the source images are encrypted with your own keys. + properties: + rawKey: + type: string + description: | + Specifies a 256-bit customer-supplied encryption key, encoded in RFC 4648 base64 + to either encrypt or decrypt this resource. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. + sourceSnapshot: + type: string + description: | + The source snapshot to create this disk. When creating a new instance, one of + initializeParams.sourceSnapshot or disks.source is required except for local SSD. + + To create a disk with a snapshot that you created, specify the snapshot name in the following format: + + global/snapshots/my-backup + + If the source snapshot is deleted later, this field will not be set. + + Authorization requires the following Google IAM permission on the specified resource sourceSnapshot: + + compute.snapshots.useReadOnly + sourceSnapshotEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source snapshot. + properties: + rawKey: + type: string + description: | + Specifies a 256-bit customer-supplied encryption key, encoded in RFC 4648 base64 + to either encrypt or decrypt this resource. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. + autoDelete: + type: boolean + description: | + Specifies whether the disk will be auto-deleted when the instance is deleted + (but not when the disk is detached from the instance). + interface: + type: string + description: | + Specifies the disk interface to use for attaching this disk, which is either SCSI or NVME. + The default is SCSI. Persistent disks must always use SCSI and the request will fail if you + attempt to attach a persistent disk in any other format than SCSI. Local SSDs can use either NVME or SCSI. + For performance characteristics of SCSI over NVMe, see Local SSD performance. + enum: + - SCSI + - NVME + guestOsFeatures: + type: array + uniqueItems: true + description: | + A list of features to enable on the guest operating system. Applicable only for bootable images. + Read Enabling guest operating system features to see a list of available options. + items: + type: object + additionalProperties: false + properties: + type: + type: string + description: | + https://cloud.google.com/compute/docs/images/create-delete-deprecate-private-images#guest-os-features + The ID of a supported feature. Read Enabling guest operating system features + to see a list of available options. + enum: + - MULTI_IP_SUBNET + - SECURE_BOOT + - UEFI_COMPATIBLE + - VIRTIO_SCSI_MULTIQUEUE + - WINDOWS + diskEncryptionKey: + type: object + additionalProperties: false + description: | + The customer-supplied encryption key of the source snapshot. + properties: + rawKey: + type: string + description: | + Encrypts or decrypts a disk using a customer-supplied encryption key. + + If you are creating a new disk, this field encrypts the new disk using an encryption + key that you provide. If you are attaching an existing disk that is already encrypted, + this field decrypts the disk using the customer-supplied encryption key. + + If you encrypt a disk using a customer-supplied key, you must provide the same key again when + you attempt to use this resource at a later time. For example, you must provide the key when + you create a snapshot or an image from the disk or when you attach the disk + to a virtual machine instance. + + If you do not provide an encryption key, then the disk will be encrypted using an automatically + generated key and you do not need to provide a key to use the disk later. + + Instance templates do not store customer-supplied encryption keys, so you cannot use your own keys + to encrypt disks in a managed instance group. + kmsKeyName: + type: string + description: | + The name of the encryption key that is stored in Google Cloud KMS. + scheduling: + type: object + additionalProperties: false + description: | + Sets the scheduling options for this instance. + properties: + onHostMaintenance: + type: string + description: | + Defines the maintenance behavior for this instance. For standard instances, the default behavior is MIGRATE. + For preemptible instances, the default and only possible behavior is TERMINATE. + For more information, see Setting Instance Scheduling Options. + enum: + - MIGRATE + - TERMINATE + automaticRestart: + type: boolean + description: | + Specifies whether the instance should be automatically restarted if it is terminated by Compute Engine + (not terminated by a user). You can only set the automatic restart option for standard instances. + Preemptible instances cannot be automatically restarted. + + By default, this is set to true so an instance is automatically restarted if it is terminated by Compute Engine. + preemptible: + type: boolean + description: | + Defines whether the instance is preemptible. This can only be set during instance creation, + it cannot be set or changed after the instance has been created. + nodeAffinities: + type: array + uniqueItems: true + description: | + A set of node affinity and anti-affinity. + items: + type: object + additionalProperties: false + properties: + key: + type: string + description: | + Corresponds to the label key of Node resource. + operator: + type: string + description: | + Defines the operation of node selection. + values: + type: array + uniqueItems: true + description: | + Corresponds to the label values of Node resource. + items: + type: string + minCpuPlatform: + type: string + description: | + Specifies a minimum CPU platform for the VM instance. Applicable values are the friendly names of CPU platforms, + such as minCpuPlatform: "Intel Haswell" or minCpuPlatform: "Intel Sandy Bridge". + enum: + - Intel Sandy Bridge + - Intel Ivy Bridge + - Intel Haswell + - Intel Broadwell + - Intel Skylake + sourceInstance: + type: string + description: | + The source instance used to create the template. You can provide this as a partial or full URL to the resource. + For example, the following are valid values: + + - https://www.googleapis.com/compute/v1/projects/project/zones/zone/instances/instance + - projects/project/zones/zone/instances/instance + + Authorization requires the following Google IAM permission on the specified resource sourceInstance: + - compute.instances.get + sourceInstanceParams: + type: object + additionalProperties: false + description: | + The source instance params to use to create this instance template. + properties: + diskConfigs: + type: array + uniqueItems: true + description: | + Attached disks configuration. If not provided, defaults are applied: For boot disk and any other R/W disks, + new custom images will be created from each disk. For read-only disks, they will be attached + in read-only mode. Local SSD disks will be created as blank volumes. + items: + type: object + additionalProperties: false + properties: + deviceName: + type: string + description: | + Specifies the device name of the disk to which the configurations apply to. + instantiateFrom: + type: string + description: | + Specifies whether to include the disk and what image to use. Possible values are: + + - source-image: to use the same image that was used to create the source instance's corresponding disk. + Applicable to the boot disk and additional read-write disks. + - source-image-family: to use the same image family that was used to create the source instance's + corresponding disk. Applicable to the boot disk and additional read-write disks. + - custom-image: to use a user-provided image url for disk creation. Applicable to the boot disk and + additional read-write disks. + - attach-read-only: to attach a read-only disk. Applicable to read-only disks. + - do-not-include: to exclude a disk from the template. Applicable to additional read-write disks, + local SSDs, and read-only disks. + enum: + - source-image + - source-image-family + - custom-image + - attach-read-only + autoDelete: + type: boolean + description: | + Specifies whether the disk will be auto-deleted when the instance is deleted + (but not when the disk is detached from the instance). + customImage: + type: string + description: | + The custom source image to be used to restore this disk when instantiating this instance template. + shieldedInstanceConfig: + type: object + additionalProperties: false + properties: + enableSecureBoot: + type: boolean + description: | + Defines whether the instance has Secure Boot enabled. + enableVtpm: + type: boolean + description: | + Defines whether the instance has the vTPM enabled. + enableIntegrityMonitoring: + type: boolean + description: | + Defines whether the instance has integrity monitoring enabled. + guestAccelerators: + type: array + uniqueItems: true + description: | + A list of the type and count of accelerator cards attached to the instance. + items: + type: object + additionalProperties: false + properties: + acceleratorType: + type: string + description: | + Full or partial URL of the accelerator type resource to attach to this instance. For example: projects/my-project/zones/us-central1-c/acceleratorTypes/nvidia-tesla-p100 + If you are creating an instance template, specify only the accelerator name. + See GPUs on Compute Engine for a full list of accelerator types. + acceleratorCount: + type: integer + description: | + The number of the guest accelerator cards exposed to this instance. machineType: type: string default: n1-standard-1 @@ -259,6 +781,7 @@ properties: minimum: 10 metadata: type: object + additionalProperties: false description: | Instance metadata. For example: @@ -269,9 +792,11 @@ properties: properties: items: type: array + uniqueItems: true description: The metadata key-value pairs. items: type: object + additionalProperties: false properties: key: type: string @@ -279,18 +804,21 @@ properties: type: string serviceAccounts: type: array + uniqueItems: true description: | The list of service accounts, with their specified scopes, authorized for this instance. Only one service account per VM instance is supported. items: type: object + additionalProperties: false properties: email: type: string description: The email address of the service account. scopes: type: array + uniqueItems: true description: | The list of scopes to be made available to the service account. items: @@ -302,6 +830,7 @@ properties: for details tags: type: object + additionalProperties: false description: | The list of tags to apply to the instances that are created from the template. The tags identify valid sources or targets for network @@ -309,6 +838,7 @@ properties: properties: items: type: array + uniqueItems: true description: The array of tags. items: type: string @@ -322,11 +852,13 @@ properties: count: 3 healthChecks: type: array + uniqueItems: true description: | An array that defines how individual instances must be checked for health. items: type: object + additionalProperties: false requried: - initialDelaySec - healthCheck @@ -341,17 +873,20 @@ properties: description: The healthcheck resource URL. distributionPolicy: type: object + additionalProperties: false description: | The policy that specifies the intended distribution of instances in a regional managed instance group. properties: zones: type: array + uniqueItems: true description: | A list of zones where the regional managed instance group creates and manages instances. items: type: object + additionalProperties: false properties: zone: type: string @@ -360,6 +895,7 @@ properties: the managed instance group is located. autoscaler: type: object + additionalProperties: false description: | The configuration of the autosaler - a mechanism that automatically adjusts the number of instances in a group based on the current load. @@ -391,6 +927,7 @@ properties: collecting information from a new instance. cpuUtilization: type: object + additionalProperties: false description: | Defines the CPU utilization policy that allows the autoscaler to scale based on the average CPU utilization of a managed instance @@ -407,6 +944,7 @@ properties: value). loadBalancingUtilization: type: object + additionalProperties: false required: - utilizationTarget description: | @@ -419,10 +957,12 @@ properties: description: The fraction of the back-end capacity utilization. customMetricUtilizations: type: array + uniqueItems: true description: | Configuration parameters for autoscaling based on a custom metric. items: type: object + additionalProperties: false required: - metric - utilizationTarget diff --git a/dm/templates/managed_instance_group/tests/integration/managed_instance_group.bats b/dm/templates/managed_instance_group/tests/integration/managed_instance_group.bats index d97f1778f17..9969933b3ab 100755 --- a/dm/templates/managed_instance_group/tests/integration/managed_instance_group.bats +++ b/dm/templates/managed_instance_group/tests/integration/managed_instance_group.bats @@ -79,12 +79,14 @@ function teardown() { @test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "$output" [[ "$status" -eq 0 ]] } @test "Verifying that a zonal intance group was created" { run gcloud compute instance-groups managed list \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "$output" [[ "$status" -eq 0 ]] [[ "$output" =~ "${ZONAL_MIG_NAME}" ]] [[ "$output" =~ "${ZONE}" ]] diff --git a/dm/templates/managed_instance_group/tests/integration/managed_instance_group.yaml b/dm/templates/managed_instance_group/tests/integration/managed_instance_group.yaml index b3bbe182d84..4ba09aa35f3 100644 --- a/dm/templates/managed_instance_group/tests/integration/managed_instance_group.yaml +++ b/dm/templates/managed_instance_group/tests/integration/managed_instance_group.yaml @@ -24,7 +24,10 @@ resources: instanceTemplate: name: ${INSTANCE_TEMPLATE_NAME} diskImage: ${IT_BASE_IMAGE} - network: ${IT_NETWORK} + networks: + - network: ${IT_NETWORK} + accessConfigs: + - type: ONE_TO_ONE_NAT healthChecks: - initialDelaySec: ${INITIAL_DELAY_SEC} healthCheck: projects/${CLOUD_FOUNDATION_PROJECT_ID}/global/httpHealthChecks/${SECOND_HEALTH_CHECK_NAME} diff --git a/dm/templates/nat_gateway/nat_gateway.py b/dm/templates/nat_gateway/nat_gateway.py index a454caa36e8..73b09203b68 100644 --- a/dm/templates/nat_gateway/nat_gateway.py +++ b/dm/templates/nat_gateway/nat_gateway.py @@ -58,7 +58,7 @@ def do_GET(self): gcloud beta runtime-config configs variables set $VARIABLE_NAME 1 --config-name $CONFIG_NAME """ -def get_network(properties): +def get_network(project_id, properties): """ Gets a network name. """ network_name = properties.get('network') @@ -67,12 +67,12 @@ def get_network(properties): if is_self_link: network_url = network_name else: - network_url = 'global/networks/{}'.format(network_name) + network_url = 'projects/{}/global/networks/{}'.format(project_id, network_name) return network_url -def get_subnetwork(context): +def get_subnetwork(project_id, context): """ Gets a subnetwork name. """ subnet_name = context.properties.get('subnetwork') @@ -83,7 +83,7 @@ def get_subnetwork(context): else: subnet_url = 'projects/{}/regions/{}/subnetworks/{}' subnet_url = subnet_url.format( - context.env['project'], + project_id, context.properties['region'], subnet_name ) @@ -91,7 +91,7 @@ def get_subnetwork(context): return subnet_url -def get_healthcheck(name): +def get_healthcheck(project_id, name): """ Generate a healthcheck resource. """ resource = { @@ -104,14 +104,15 @@ def get_healthcheck(name): 'requestPath': '/health-check', 'healthyThreshold': 1, 'unhealthyThreshold': 5, - 'checkIntervalSec': 30 + 'checkIntervalSec': 30, + 'project': project_id, } } return resource -def get_firewall(context, network): +def get_firewall(context, project_id, network): """ Generate a firewall rule for the healthcheck. """ # pylint: disable=line-too-long @@ -122,7 +123,8 @@ def get_firewall(context, network): 'type': 'firewall.py', 'properties': { - 'network': network, + 'project': project_id, + 'networkName': network, 'rules': [ { @@ -146,7 +148,8 @@ def get_firewall(context, network): return resource -def get_external_internal_ip(ip_name, +def get_external_internal_ip(project_id, + ip_name, external_ip_name, internal_ip_name, region, @@ -163,11 +166,13 @@ def get_external_internal_ip(ip_name, [ { 'name': external_ip_name, + 'project': project_id, 'ipType': 'REGIONAL', 'region': region }, { 'name': internal_ip_name, + 'project': project_id, 'ipType': 'INTERNAL', 'region': region, 'subnetwork': subnet @@ -179,7 +184,8 @@ def get_external_internal_ip(ip_name, return resource -def get_instance_template(context, +def get_instance_template(project_id, + context, instance_template_name, external_ip, internal_ip, @@ -193,6 +199,7 @@ def get_instance_template(context, 'type': 'instance_template.py', 'properties': { + 'project': project_id, 'natIP': external_ip, 'network': network, 'subnetwork': subnet, @@ -221,7 +228,7 @@ def get_instance_template(context, return resource -def get_route(context, route_name, internal_ip, network): +def get_route(project_id, context, route_name, internal_ip, network): """ Generate a route resource. """ resource = { @@ -229,6 +236,7 @@ def get_route(context, route_name, internal_ip, network): 'type': 'route.py', 'properties': { + 'project': project_id, 'network': network, 'routes': [ @@ -247,7 +255,8 @@ def get_route(context, route_name, internal_ip, network): return resource -def get_managed_instance_group(name, +def get_managed_instance_group(project_id, + name, healthcheck, instance_template_name, base_instance_name, @@ -256,9 +265,11 @@ def get_managed_instance_group(name, resource = { 'name': name, - 'type': 'compute.v1.instanceGroupManager', + # https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers + 'type': 'gcp-types/compute-v1:instanceGroupManagers', 'properties': { + 'project': project_id, 'instanceTemplate': '$(ref.' + instance_template_name + '.selfLink)', 'baseInstanceName': base_instance_name, @@ -285,14 +296,15 @@ def generate_config(context): prefix = context.env['name'] hc_name = prefix + '-healthcheck' region = context.properties['region'] - network_name = get_network(context.properties) - subnet_name = get_subnetwork(context) + project_id = context.properties.get('project', context.env['project']) + network_name = get_network(project_id, context.properties) + subnet_name = get_subnetwork(project_id, context) # Health check to be used by the managed instance groups. - resources.append(get_healthcheck(hc_name)) + resources.append(get_healthcheck(project_id, hc_name)) # Firewall rule that allows the healthcheck to work. - resources.append(get_firewall(context, network_name)) + resources.append(get_firewall(context, project_id, context.properties.get('network'))) # Outputs: out = {} @@ -306,6 +318,7 @@ def generate_config(context): internal_ip_name = prefix + '-ip-internal-' + zone resources.append( get_external_internal_ip( + project_id, ip_name, external_ip_name, internal_ip_name, @@ -328,6 +341,7 @@ def generate_config(context): instance_template_name = prefix + '-insttempl-' + zone resources.append( get_instance_template( + project_id, context, instance_template_name, external_ip, @@ -342,6 +356,7 @@ def generate_config(context): base_instance_name = prefix + '-gateway-' + zone resources.append( get_managed_instance_group( + project_id, instance_group_manager_name, hc_name, instance_template_name, @@ -354,7 +369,8 @@ def generate_config(context): # next hop. route_name = prefix + '-route-' + zone resources.append( - get_route(context, + get_route(project_id, + context, route_name, internal_ip, network_name) diff --git a/dm/templates/nat_gateway/nat_gateway.py.schema b/dm/templates/nat_gateway/nat_gateway.py.schema index 0554f73c92d..e2be3a4b062 100644 --- a/dm/templates/nat_gateway/nat_gateway.py.schema +++ b/dm/templates/nat_gateway/nat_gateway.py.schema @@ -15,6 +15,7 @@ info: title: Highly Available NAT Gateway author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of an HA NAT gateway. Internal network address translation (NAT) gateway instances can route traffic from internal-only @@ -23,6 +24,10 @@ info: instances while exposing a small set of NAT gateway virtual machines to the Internet. + APIs endpoints used by this template: + - gcp-types/compute-v1:instanceGroupManagers => + https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers + imports: - path: ../healthcheck/healthcheck.py name: healthcheck.py @@ -35,8 +40,8 @@ imports: - path: ../firewall/firewall.py name: firewall.py -additionalProperties: false - +additionalProperties: false + required: - network - subnetwork @@ -47,6 +52,11 @@ required: - nattedVmTag properties: + project: + type: string + description: | + The project ID of the project containing the NAT instance. The + Google apps domain is prefixed if applicable. network: type: string description: The VPC network to connect the NAT gateway VMs to. diff --git a/dm/templates/nat_gateway/tests/integration/nat_gateway.bats b/dm/templates/nat_gateway/tests/integration/nat_gateway.bats index 5ccd4049e96..4d522a4fd47 100755 --- a/dm/templates/nat_gateway/tests/integration/nat_gateway.bats +++ b/dm/templates/nat_gateway/tests/integration/nat_gateway.bats @@ -20,7 +20,7 @@ if [[ -e "${RANDOM_FILE}" ]]; then CONFIG=".${DEPLOYMENT_NAME}.yaml" fi -export PROJECT_NUMBER=$(gcloud projects list | grep "${CLOUD_FOUNDATION_PROJECT_ID}" | awk {'print $NF'}) +export PROJECT_NUMBER=$(gcloud projects describe ${CLOUD_FOUNDATION_PROJECT_ID} | grep projectNumber | sed 's/[^0-9]*//g') ########## HELPER FUNCTIONS ########## @@ -70,6 +70,29 @@ function teardown() { run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ "$status" -eq 0 ]] + + + # Enabling OS login for the next tests + run gcloud compute instances add-metadata "test-inst-has-ext-ip-${RAND}" \ + --metadata enable-oslogin=TRUE \ + --zone "us-east1-b" \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "Pre-run Status: $status" + echo "Pre-run Output: $output" + + [[ "$status" -eq 0 ]] + + run gcloud compute ssh "test-inst-has-ext-ip-${RAND}" --zone "us-east1-b" \ + --command "echo 'OK' " \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "SSH Status: $status" + echo "SSH Output: $output" + + echo "sleeping 30" + sleep 30 + + [[ "$status" -eq 0 ]] } @test "Verifying that resources were created in deployment ${DEPLOYMENT_NAME}" { @@ -129,6 +152,10 @@ function teardown() { --internal-ip --command 'wget google.com' --zone 'us-east1-b' \ --quiet" \ --quiet + + echo "status = ${status}" + echo "output = ${output}" + [[ "$status" -eq 0 ]] [[ "$output" =~ "HTTP request sent, awaiting response... 200 OK" ]] @@ -141,6 +168,10 @@ function teardown() { --command 'wget google.com --timeout=5' --zone 'us-east1-b' \ --quiet" \ --quiet + + echo "status = ${status}" + echo "output = ${output}" + [[ "$output" =~ "failed: Network is unreachable" ]] } diff --git a/dm/templates/network/network.py b/dm/templates/network/network.py index 1fd24a06428..1793a0251fa 100644 --- a/dm/templates/network/network.py +++ b/dm/templates/network/network.py @@ -14,48 +14,65 @@ """ This template creates a network, optionally with subnetworks. """ +def append_optional_property(res, properties, prop_name): + """ If the property is set, it is added to the resource. """ + + val = properties.get(prop_name) + if val: + res['properties'][prop_name] = val + return + + def generate_config(context): """ Entry point for the deployment resources. """ - name = context.properties.get('name') or context.env['name'] - network_self_link = '$(ref.{}.selfLink)'.format(name) - auto_create_subnetworks = context.properties.get( - 'autoCreateSubnetworks', - False - ) + properties = context.properties + name = properties.get('name', context.env['name']) + network_self_link = '$(ref.{}.selfLink)'.format(context.env['name']) - resources = [ - { - 'type': 'compute.v1.network', - 'name': name, - 'properties': - { - 'name': name, - 'autoCreateSubnetworks': auto_create_subnetworks - } - } + network_resource = { + # https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert + 'type': 'gcp-types/compute-v1:networks', + 'name': context.env['name'], + 'properties': + { + 'name': name, + 'autoCreateSubnetworks': properties.get('autoCreateSubnetworks', False) + } + } + optional_properties = [ + 'description', + 'routingConfig', + 'project', ] + for prop in optional_properties: + append_optional_property(network_resource, properties, prop) + resources = [network_resource] # Subnetworks: out = {} - for subnetwork in context.properties.get('subnetworks', []): + for i, subnetwork in enumerate( + properties.get('subnetworks', []), 1 + ): subnetwork['network'] = network_self_link + if properties.get('project'): + subnetwork['project'] = properties.get('project') + + subnetwork_name = 'subnetwork-{}'.format(i) resources.append( { - 'name': subnetwork['name'], + 'name': subnetwork_name, 'type': 'subnetwork.py', 'properties': subnetwork } ) - out[subnetwork['name']] = { - 'selfLink': '$(ref.{}.selfLink)'.format(subnetwork['name']), - 'ipCidrRange': '$(ref.{}.ipCidrRange)'.format(subnetwork['name']), - 'region': '$(ref.{}.region)'.format(subnetwork['name']), - 'network': '$(ref.{}.network)'.format(subnetwork['name']), - 'gatewayAddress': '$(ref.{}.gatewayAddress)'.format( - subnetwork['name'] - ) + out[subnetwork_name] = { + 'selfLink': '$(ref.{}.selfLink)'.format(subnetwork_name), + 'ipCidrRange': '$(ref.{}.ipCidrRange)'.format(subnetwork_name), + 'region': '$(ref.{}.region)'.format(subnetwork_name), + 'network': '$(ref.{}.network)'.format(subnetwork_name), + 'gatewayAddress': '$(ref.{}.gatewayAddress)'.format(subnetwork_name) } return { diff --git a/dm/templates/network/network.py.schema b/dm/templates/network/network.py.schema index 62df2c333fb..02fd343d4de 100644 --- a/dm/templates/network/network.py.schema +++ b/dm/templates/network/network.py.schema @@ -15,24 +15,65 @@ info: title: Network author: Sourced Group Inc. + version: 1.0.0 description: | Creates a network. For more information on this resource: - - https://cloud.google.com/compute/docs/reference/rest/v1/networks + - https://cloud.google.com/vpc/docs/vpc + + APIs endpoints used by this template: + - gcp-types/compute-v1:networks => + https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert imports: - path: subnetwork.py additionalProperties: false -# required: -# - name +oneOf: + - properties: + autoCreateSubnetworks: + enum: + - true + - properties: + subnetworks: + type: array + default: [] + minItems: 1 properties: name: type: string - description: Name of the network resource. + description: | + Name of the network resource. Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the Cloud Router instance. The + Google apps domain is prefixed if applicable. + description: + type: string + description: | + An optional description of this resource. Provide this property when you create the resource. + routingConfig: + type: object + additionalProperties: false + description: | + The network-level routing configuration for this network. Used by Cloud Router to determine what type + of network-wide routing behavior to enforce. + required: + - routingMode + properties: + routingMode: + type: string + description: | + The network-wide routing mode to use. If set to REGIONAL, this network's cloud routers will only advertise + routes with subnets of this network in the same region as the router. If set to GLOBAL, this network's + cloud routers will advertise routes with all subnets of this network, across regions. + enum: + - GLOBAL + - REGIONAL autoCreateSubnetworks: type: boolean default: false @@ -41,6 +82,7 @@ properties: 10.128.0.0/9; and (b) one subnetwork per region is created automatically. subnetworks: type: array + default: [] description: | An array of subnetworks, as defined in the `subnetwork.py` template. Example: @@ -54,6 +96,15 @@ properties: ipCidrRange: 172.16.0.0/24 - rangeName: my-secondary-range-2 ipCidrRange: 172.16.1.0/24 + items: + type: object + allOf: + - not: + required: + - project + - not: + required: + - network outputs: properties: diff --git a/dm/templates/network/subnetwork.py b/dm/templates/network/subnetwork.py index ea9cb2cbb7c..a2bc81ede03 100644 --- a/dm/templates/network/subnetwork.py +++ b/dm/templates/network/subnetwork.py @@ -17,28 +17,31 @@ def generate_config(context): """ Entry point for the deployment resources. """ - name = context.properties.get('name', context.env['name']) - required_properties = ['network', 'ipCidrRange', 'region'] + props = context.properties + props['name'] = props.get('name', context.env['name']) + required_properties = ['name', 'network', 'ipCidrRange', 'region'] optional_properties = [ + 'project', 'enableFlowLogs', 'privateIpGoogleAccess', 'secondaryIpRanges' ] # Load the mandatory properties, then the optional ones (if specified). - properties = {p: context.properties[p] for p in required_properties} + properties = {p: props[p] for p in required_properties} properties.update( { - p: context.properties[p] + p: props[p] for p in optional_properties - if p in context.properties + if p in props } ) resources = [ { - 'type': 'compute.v1.subnetwork', - 'name': name, + # https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks/insert + 'type': 'gcp-types/compute-v1:subnetworks', + 'name': context.env['name'], 'properties': properties } ] @@ -46,27 +49,27 @@ def generate_config(context): output = [ { 'name': 'name', - 'value': name + 'value': properties['name'] }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) }, { 'name': 'ipCidrRange', - 'value': '$(ref.{}.ipCidrRange)'.format(name) + 'value': '$(ref.{}.ipCidrRange)'.format(context.env['name']) }, { 'name': 'region', - 'value': '$(ref.{}.region)'.format(name) + 'value': '$(ref.{}.region)'.format(context.env['name']) }, { 'name': 'network', - 'value': '$(ref.{}.network)'.format(name) + 'value': '$(ref.{}.network)'.format(context.env['name']) }, { 'name': 'gatewayAddress', - 'value': '$(ref.{}.gatewayAddress)'.format(name) + 'value': '$(ref.{}.gatewayAddress)'.format(context.env['name']) } ] diff --git a/dm/templates/network/subnetwork.py.schema b/dm/templates/network/subnetwork.py.schema index 30c90b516ee..f0349f37a0d 100644 --- a/dm/templates/network/subnetwork.py.schema +++ b/dm/templates/network/subnetwork.py.schema @@ -15,7 +15,16 @@ info: title: Subnet author: Sourced Group Inc. - description: Creates a subnetwork. + version: 1.0.0 + description: | + Creates a subnetwork. + + For more information on this resource: + - https://cloud.google.com/vpc/docs/vpc + + APIs endpoints used by this template: + - gcp-types/compute-v1:subnetworks => + https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks/insert additionalProperties: false @@ -28,7 +37,17 @@ properties: name: type: string description: | - Name of the subnetwork. If not specified, the DM resource name is used. + The name of the resource, provided by the client when initially creating the resource. The name must + be 1-63 characters long, and comply with RFC1035. Specifically, the name must be 1-63 characters long and + match the regular expression [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a lowercase + letter, and all following characters must be a dash, lowercase letter, or digit, except the last character, + which cannot be a dash. + If not specified, the DM resource name is used. + project: + type: string + description: | + The project ID of the project containing the Cloud Router instance. The + Google apps domain is prefixed if applicable. network: type: string description: | @@ -62,6 +81,25 @@ properties: ipCidrRange: 172.16.0.0/24 - rangeName: my-secondary-range-2 ipCidrRange: 172.16.1.0/24 + items: + type: object + additionalProperties: false + required: + - rangeName + - ipCidrRange + properties: + rangeName: + type: string + description: | + The name associated with this subnetwork secondary range, used when adding an alias IP range + to a VM instance. The name must be 1-63 characters long, and comply with RFC1035. + The name must be unique within the subnetwork. + ipCidrRange: + type: string + description: | + The range of IP addresses belonging to this subnetwork secondary range. Provide this property + when you create the subnetwork. Ranges must be unique and non-overlapping with all primary + and secondary IP ranges within a network. Only IPv4 is supported. enableFlowLogs: type: boolean description: If "true", enables flow logging for the subnetwork. diff --git a/dm/templates/network/tests/schemas/invalid_subnets.yaml b/dm/templates/network/tests/schemas/invalid_subnets.yaml new file mode 100644 index 00000000000..d60702cf438 --- /dev/null +++ b/dm/templates/network/tests/schemas/invalid_subnets.yaml @@ -0,0 +1,15 @@ +autoCreateSubnetworks: true +subnetworks: + - name: test-subnetwork-1 + region: us-east1 + ipCidrRange: 10.0.0.0/24 + privateIpGoogleAccess: false + enableFlowLogs: true + secondaryIpRanges: + - rangeName: my-secondary-range-1 + ipCidrRange: 10.0.1.0/24 + - rangeName: my-secondary-range-2 + ipCidrRange: 10.0.2.0/24 + - name: test-subnetwork-2 + region: us-east1 + ipCidrRange: 192.168.0.0/24 diff --git a/dm/templates/network/tests/schemas/valid_auto.yaml b/dm/templates/network/tests/schemas/valid_auto.yaml new file mode 100644 index 00000000000..3de5b553e29 --- /dev/null +++ b/dm/templates/network/tests/schemas/valid_auto.yaml @@ -0,0 +1 @@ +autoCreateSubnetworks: true diff --git a/dm/templates/network/tests/schemas/valid_subnets.yaml b/dm/templates/network/tests/schemas/valid_subnets.yaml new file mode 100644 index 00000000000..0bcd684fc5f --- /dev/null +++ b/dm/templates/network/tests/schemas/valid_subnets.yaml @@ -0,0 +1,15 @@ +autoCreateSubnetworks: false +subnetworks: + - name: test-subnetwork-1 + region: us-east1 + ipCidrRange: 10.0.0.0/24 + privateIpGoogleAccess: false + enableFlowLogs: true + secondaryIpRanges: + - rangeName: my-secondary-range-1 + ipCidrRange: 10.0.1.0/24 + - rangeName: my-secondary-range-2 + ipCidrRange: 10.0.2.0/24 + - name: test-subnetwork-2 + region: us-east1 + ipCidrRange: 192.168.0.0/24 diff --git a/dm/templates/network_peering/network_peering.py b/dm/templates/network_peering/network_peering.py index 912b8e82e62..471cf87832b 100644 --- a/dm/templates/network_peering/network_peering.py +++ b/dm/templates/network_peering/network_peering.py @@ -19,10 +19,12 @@ def generate_config(context): resources = [] properties = context.properties - peer_name = properties['name'] or context.env['name'] + name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) peer_create = { - 'name': peer_name + '-createPeer', + 'name': context.env['name'] + '-create-peer', + # https://cloud.google.com/compute/docs/reference/rest/v1/networks/addPeering 'action': 'gcp-types/compute-v1:compute.networks.addPeering', 'metadata': { 'runtimePolicy': ['CREATE', @@ -30,14 +32,17 @@ def generate_config(context): }, 'properties': { - 'name': peer_name, + 'name': name, + 'project': project_id, 'network': properties['network'], 'peerNetwork': properties['peerNetwork'], 'autoCreateRoutes': properties.get('autoCreateRoutes') } } + peer_delete = { - 'name': peer_name + '-deletePeer', + 'name': context.env['name'] + '-delete-peer', + # https://cloud.google.com/compute/docs/reference/rest/v1/networks/removePeering 'action': 'gcp-types/compute-v1:compute.networks.removePeering', 'metadata': { 'runtimePolicy': ['DELETE', @@ -45,7 +50,8 @@ def generate_config(context): }, 'properties': { - 'name': peer_name, + 'name': name, + 'project': project_id, 'network': properties['network'], 'peerNetwork': properties['peerNetwork'] } diff --git a/dm/templates/network_peering/network_peering.py.schema b/dm/templates/network_peering/network_peering.py.schema index 7e0d4aecc51..0bf29001e84 100644 --- a/dm/templates/network_peering/network_peering.py.schema +++ b/dm/templates/network_peering/network_peering.py.schema @@ -14,12 +14,19 @@ info: title: VPC Network Peering + version: 1.0.0 author: Sourced Group Inc. description: | Creates peering between VPC networks. - For more information on this resource, see - https://cloud.google.com/compute/docs/reference/rest/beta/networks/addPeering. + For more information on this resource: + https://cloud.google.com/compute/docs/reference/rest/beta/networks/addPeering + + APIs endpoints used by this template: + - gcp-types/compute-v1:compute.networks.addPeering => + https://cloud.google.com/compute/docs/reference/rest/v1/networks/addPeering + - gcp-types/compute-v1:compute.networks.removePeering => + https://cloud.google.com/compute/docs/reference/rest/v1/networks/removePeering imports: - path: network_peering.py @@ -34,7 +41,14 @@ required: properties: name: type: string - description: The peering name. Must conform to RFC1035. + description: | + The peering name. Must conform to RFC1035. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. network: type: string description: | diff --git a/dm/templates/network_peering/tests/integration/network_peering.bats b/dm/templates/network_peering/tests/integration/network_peering.bats index f4a98f8e38d..16557fd22b5 100644 --- a/dm/templates/network_peering/tests/integration/network_peering.bats +++ b/dm/templates/network_peering/tests/integration/network_peering.bats @@ -67,6 +67,7 @@ function teardown() { run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --config "${CONFIG}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo $output [[ "$status" -eq 0 ]] } diff --git a/dm/templates/org_policy/org_policy.py b/dm/templates/org_policy/org_policy.py index 73711cebc7a..9f430679294 100644 --- a/dm/templates/org_policy/org_policy.py +++ b/dm/templates/org_policy/org_policy.py @@ -11,15 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -This template creates an organization policy to allow VMs to have public -IPs. -""" +"""This template creates an organization policy.""" def generate_config(context): - """ Entry point for the deployment resources. """ - + """Entry point for the deployment resources.""" project = context.properties['projectId'] resources = [] diff --git a/dm/templates/org_policy/org_policy.py.schema b/dm/templates/org_policy/org_policy.py.schema index 7265eb5aa3a..fda6f99f290 100644 --- a/dm/templates/org_policy/org_policy.py.schema +++ b/dm/templates/org_policy/org_policy.py.schema @@ -15,17 +15,28 @@ info: title: Organization Policy author: Sourced Group Inc. - description: Creates organizational policies. + version: 1.0.0 + description: | + Creates organizational policies. + + For more information on this resource: + https://cloud.google.com/resource-manager/reference/rest/v1/Policy + + APIs endpoints used by this template: + - gcp-types/cloudresourcemanager-v1:cloudresourcemanager.projects.setOrgPolicy => + https://cloud.google.com/resource-manager/reference/rest/v1/projects/setOrgPolicy + - gcp-types/cloudresourcemanager-v1:cloudresourcemanager.projects.clearOrgPolicy => + https://cloud.google.com/resource-manager/reference/rest/v1/projects/clearOrgPolicy imports: - path: org_policy.py -additionalProperties: false - required: - projectId - policies +additionalProperties: false + properties: projectId: type: string @@ -42,6 +53,74 @@ properties: - constraint: constraints/compute.disableNestedVirtualization booleanPolicy: enforced: true + uniqItems: true + items: + version: + type: integer + description: Version of the Policy. Default version is 0. + minimum: 1 + constraint: + type: string + description: | + The name of the Constraint the Policy is configuring. + For example, `constraints/serviceuser.services`. + Immutable after creation. + listPolicy: + type: object + description: List of values either allowed or disallowed. + additionalProperties: false + properties: + allowedValues: + type: array + description: | + List of values allowed at this resource. Can only be set if + `allValues` is set to `ALL_VALUES_UNSPECIFIED`. + uniqItems: true + items: + type: string + deniedValues: + type: array + description: | + List of values allowed at this resource. Can only be set if + `allValues` is set to `ALL_VALUES_UNSPECIFIED`. + uniqItems: true + items: + type: string + allValues: + type: string + description: | + The policy allValues state. + https://cloud.google.com/resource-manager/reference/rest/v1/Policy#AllValues + pattern: ^(ALL_VALUES_UNSPECIFIED|ALLOW|DENY)$ + suggestedValue: + type: string + description: | + Optional. The Google Cloud Console will try to default to a + configuration that matches the value specified in this Policy. If + suggestedValue is not set, it will inherit the value specified + higher in the hierarchy, unless inheritFromParent is false. + inheritFromParent: + type: boolean + description: | + Determines the inheritance behavior for this Policy. + For more details please look for `inheritFromParent` in the doc + https://cloud.google.com/resource-manager/reference/rest/v1/Policy + booleanPolicy: + type: object + description: | + For boolean Constraints, whether to enforce the Constraint or not. + additionalProperties: false + properties: + enforced: + type: boolean + description: | + If true, then the Policy is enforced. If false, then any + configuration is acceptable. + restoreDefault: + type: object + description: | + Restores the default behavior of the constraint. + Independent of Constraint type. documentation: - templates/org_policy/README.md diff --git a/dm/templates/org_policy/tests/integration/org_policy.bats b/dm/templates/org_policy/tests/integration/org_policy.bats index 7f7170f1d55..ebece2f04bf 100644 --- a/dm/templates/org_policy/tests/integration/org_policy.bats +++ b/dm/templates/org_policy/tests/integration/org_policy.bats @@ -54,19 +54,23 @@ function teardown() { @test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { - gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" --config "${CONFIG}" + gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ + --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" } @test "Verifying that resources were created in deployment ${DEPLOYMENT_NAME}" { - run gcloud beta resource-manager org-policies list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + run gcloud beta resource-manager org-policies list \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ "$output" =~ "compute.vmExternalIpAccess" ]] [[ "$output" =~ "compute.disableNestedVirtualization" ]] } @test "Deleting deployment" { - gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" -q + gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" -q - run gcloud beta resource-manager org-policies list --project "${CLOUD_FOUNDATION_PROJECT_ID}" + run gcloud beta resource-manager org-policies list \ + --project "${CLOUD_FOUNDATION_PROJECT_ID}" [[ ! "$output" =~ "compute.vmExternalIpAccess" ]] [[ ! "$output" =~ "compute.disableNestedVirtualization" ]] } diff --git a/dm/templates/project/README.md b/dm/templates/project/README.md index bbf8440eb36..4fe1a9800f9 100644 --- a/dm/templates/project/README.md +++ b/dm/templates/project/README.md @@ -53,7 +53,11 @@ Following are the prerequisites for creating a project via Deployment Manager. Y projects, create that node following [these instructions](https://cloud.google.com/resource-manager/docs/creating-managing-organization). 6. Grant the *DM Service Account* the following permissions on the Organization node: -`roles/resourcemanager.projectCreator`. This is visible in the Cloud Console's IAM permissions in *Resource Manager -> Project Creator*. See https://cloud.google.com/resource-manager/docs/access-control-proj. + + - `roles/resourcemanager.projectCreator` + - `roles/serviceusage.serviceUsageAdmin` + + This is visible in the Cloud Console's IAM permissions in *Resource Manager -> Project Creator* and *Resource Manager -> Service Usage Admin*. See https://cloud.google.com/resource-manager/docs/access-control-proj. 7. Create/find the *Billing Account* associated with the Organization. See: https://cloud.google.com/support/billing/. Take note of the *Billing Account*'s ID, which is formatted as follows:`00E12A-0AB8B2-078CE8`. diff --git a/dm/templates/project/project.py b/dm/templates/project/project.py index 0b2bf323c88..583604393c4 100644 --- a/dm/templates/project/project.py +++ b/dm/templates/project/project.py @@ -21,60 +21,68 @@ def generate_config(context): """ Entry point for the deployment resources. """ - project_id = context.properties.get('projectId', context.env['name']) - project_name = context.properties.get('name', context.env['name']) + properties = context.properties + project_name = properties.get('name', context.env['name']) + project_id = properties.get('projectId', project_name) # Ensure that the parent ID is a string. - context.properties['parent']['id'] = str(context.properties['parent']['id']) + properties['parent']['id'] = str(properties['parent']['id']) resources = [ { - 'name': 'project', - 'type': 'cloudresourcemanager.v1.project', + 'name': '{}-project'.format(context.env['name']), + # https://cloud.google.com/resource-manager/reference/rest/v1/projects/create + 'type': 'gcp-types/cloudresourcemanager-v1:projects', 'properties': { 'name': project_name, 'projectId': project_id, - 'parent': context.properties['parent'] + 'parent': properties['parent'], + 'labels' : properties.get('labels', {}) } }, { - 'name': 'billing', + 'name': '{}-billing'.format(context.env['name']), + # https://cloud.google.com/billing/reference/rest/v1/projects/updateBillingInfo 'type': 'deploymentmanager.v2.virtual.projectBillingInfo', 'properties': { 'name': - 'projects/$(ref.project.projectId)', + 'projects/$(ref.{}-project.projectId)'.format(context.env['name']), 'billingAccountName': 'billingAccounts/' + - context.properties['billingAccountId'] + properties['billingAccountId'] } } ] - api_resources, api_names_list = activate_apis(context.properties) + api_resources, api_names_list = activate_apis(context) resources.extend(api_resources) resources.extend(create_service_accounts(context, project_id)) + if isinstance(properties.get('usageExportBucket', True), bool): + properties['usageExportBucket'] = { + 'enabled': properties.get('usageExportBucket', True), + } if ( - context.properties.get('usageExportBucket', True) and - 'api-compute.googleapis.com' in api_names_list + properties.get('usageExportBucket', {}).get('enabled', True) and + "{}-api-compute.googleapis.com".format(context.env['name']) in api_names_list ): - resources.extend(create_bucket(context.properties)) + resources.extend(create_bucket(context)) - resources.extend(create_shared_vpc(project_id, context.properties)) + resources.extend(create_shared_vpc(context)) if ( - context.properties.get('removeDefaultVPC', True) and - 'api-compute.googleapis.com' in api_names_list + properties.get('removeDefaultVPC', True) and + "{}-api-compute.googleapis.com".format(context.env['name']) in api_names_list ): - resources.extend(delete_default_network(api_names_list)) + resources.extend(delete_default_network(context, api_names_list)) if ( - context.properties.get('removeDefaultSA', True) and - 'api-compute.googleapis.com' in api_names_list + properties.get('removeDefaultSA', True) and + "{}-api-compute.googleapis.com".format(context.env['name']) in api_names_list ): - resources.extend(delete_default_service_account(api_names_list)) + resources.extend(delete_default_service_account(context, api_names_list)) return { 'resources': @@ -83,17 +91,17 @@ def generate_config(context): [ { 'name': 'projectId', - 'value': '$(ref.project.projectId)' + 'value': '$(ref.{}-project.projectId)'.format(context.env['name']) }, { 'name': 'usageExportBucketName', - 'value': '$(ref.project.projectId)-usage-export' + 'value': '$(ref.{}-project.projectId)-usage-export'.format(context.env['name']) }, { 'name': 'serviceAccountDisplayName', 'value': - '$(ref.project.projectNumber)@cloudservices.gserviceaccount.com' # pylint: disable=line-too-long + '$(ref.{}-project.projectNumber)@cloudservices.gserviceaccount.com'.format(context.env['name']) # pylint: disable=line-too-long }, { 'name': @@ -105,9 +113,10 @@ def generate_config(context): } -def activate_apis(properties): +def activate_apis(context): """ Resources for API activation. """ + properties = context.properties concurrent_api_activation = properties.get('concurrentApiActivation') apis = properties.get('activateApis', []) @@ -127,26 +136,27 @@ def activate_apis(properties): apis.append('compute.googleapis.com') resources = [] - api_names_list = ['billing'] + api_names_list = ['{}-billing'.format(context.env['name'])] for api in apis: - depends_on = ['billing'] + depends_on = ['{}-billing'.format(context.env['name'])] # Serialize activation of all APIs by making apis[n] # depend on apis[n-1]. if resources and not concurrent_api_activation: depends_on.append(resources[-1]['name']) - api_name = 'api-' + api + api_name = '{}-api-{}'.format(context.env['name'], api) api_names_list.append(api_name) resources.append( { 'name': api_name, - 'type': 'deploymentmanager.v2.virtual.enableService', + # https://cloud.google.com/service-infrastructure/docs/service-management/reference/rest/v1/services/enable + 'type': 'gcp-types/servicemanagement-v1:servicemanagement.services.enable', 'metadata': { 'dependsOn': depends_on }, 'properties': { - 'consumerId': 'project:' + '$(ref.project.projectId)', + 'consumerId': 'project:$(ref.{}-project.projectId)'.format(context.env['name']), 'serviceName': api } } @@ -158,18 +168,19 @@ def activate_apis(properties): return resources, api_names_list -def create_project_iam(dependencies, role_member_list): +def create_project_iam(context, dependencies, role_member_list): """ Grant the shared project IAM permissions. """ resources = [ { # Get the IAM policy first, so as not to remove # any existing bindings. - 'name': 'project-iam-policy', + 'name': '{}-project-iam-policy'.format(context.env['name']), 'type': 'cft-iam_project_member.py', 'properties': { - 'projectId': '$(ref.project.projectId)', - 'roles': role_member_list + 'projectId': '$(ref.{}-project.projectId)'.format(context.env['name']), + 'roles': role_member_list, + 'dependsOn': dependencies, }, 'metadata': { @@ -195,8 +206,9 @@ def create_shared_vpc_subnet_iam(context, dependencies, members_list): ): resources.append( { - 'name': 'add-vpc-subnet-iam-policy-{}'.format(i), - 'type': 'gcp-types/compute-beta:compute.subnetworks.setIamPolicy', # pylint: disable=line-too-long + 'name': '{}-add-vpc-subnet-iam-policy-{}'.format(context.env['name'], i), + # https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks/setIamPolicy + 'type': 'gcp-types/compute-v1:compute.subnetworks.setIamPolicy', # pylint: disable=line-too-long 'metadata': { 'dependsOn': dependencies, @@ -206,12 +218,14 @@ def create_shared_vpc_subnet_iam(context, dependencies, members_list): 'name': subnet['subnetId'], 'project': context.properties['sharedVPC'], 'region': subnet['region'], - 'bindings': [ - { - 'role': 'roles/compute.networkUser', - 'members': members_list - } - ] + 'policy' : { + 'bindings': [ + { + 'role': 'roles/compute.networkUser', + 'members': members_list, + } + ], + }, } } ) @@ -223,13 +237,21 @@ def create_service_accounts(context, project_id): """ Create Service Accounts and grant project IAM permissions. """ resources = [] - network_list = ['serviceAccount:$(ref.project.projectNumber)@cloudservices.gserviceaccount.com'] # pylint: disable=line-too-long - service_account_dep = ["api-compute.googleapis.com"] + network_list = [ + 'serviceAccount:$(ref.{}-project.projectNumber)@cloudservices.gserviceaccount.com'.format(context.env['name']) + ] + service_account_dep = ["{}-api-compute.googleapis.com".format(context.env['name'])] policies_to_add = [] for service_account in context.properties['serviceAccounts']: account_id = service_account['accountId'] display_name = service_account.get('displayName', account_id) + + # Build a list of SA resources to be used as a dependency + # for permission granting. + name = '{}-service-account-{}'.format(context.env['name'], account_id) + service_account_dep.append(name) + sa_name = 'serviceAccount:{}@{}.iam.gserviceaccount.com'.format( account_id, project_id @@ -244,21 +266,27 @@ def create_service_accounts(context, project_id): for role in service_account['roles']: policies_to_add.append({'role': role, 'members': [sa_name]}) - # Build a list of SA resources to be used as a dependency - # for permission granting. - name = 'service-account-' + account_id - service_account_dep.append(name) - # Create the service account resource. resources.append( { + 'name': name, + # https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/create + 'type': 'gcp-types/iam-v1:projects.serviceAccounts', + 'properties': + { + 'accountId': account_id, + 'displayName': display_name, + 'name': 'projects/$(ref.{}-project.projectId)'.format(context.env['name']) + } + # There is a bug in gcp type for IAM that ignores "name" field + } if False else { 'name': name, 'type': 'iam.v1.serviceAccount', 'properties': { 'accountId': account_id, 'displayName': display_name, - 'projectId': '$(ref.project.projectId)' + 'projectId': '$(ref.{}-project.projectId)'.format(context.env['name']) } } ) @@ -276,7 +304,7 @@ def create_service_accounts(context, project_id): # Create the project IAM permissions. if policies_to_add: - iam = create_project_iam(service_account_dep, policies_to_add) + iam = create_project_iam(context, service_account_dep, policies_to_add) resources.extend(iam) if ( @@ -296,74 +324,79 @@ def create_service_accounts(context, project_id): return resources -def create_bucket(properties): +def create_bucket(context): """ Resources for the usage export bucket. """ + properties = context.properties resources = [] - bucket_name = '$(ref.project.projectId)-usage-export' + bucket_name = '$(ref.{}-project.projectId)-usage-export'.format(context.env['name']) # Create the bucket. - resources.append( - { - 'name': 'create-usage-export-bucket', - 'type': 'gcp-types/storage-v1:buckets', - 'properties': - { - 'project': '$(ref.project.projectId)', - 'name': bucket_name - }, - 'metadata': - { - 'dependsOn': ['api-storage-component.googleapis.com'] - } - } - ) + resources.append({ + 'name': '{}-create-usage-export-bucket'.format(context.env['name']), + # https://cloud.google.com/storage/docs/json_api/v1/buckets/insert + 'type': 'gcp-types/storage-v1:buckets', + 'properties': + { + 'project': '$(ref.{}-project.projectId)'.format(context.env['name']), + 'name': bucket_name + }, + 'metadata': + { + 'dependsOn': ['{}-api-storage-component.googleapis.com'.format(context.env['name'])] + } + }) # Set the project's usage export bucket. - resources.append( - { - 'name': - 'set-usage-export-bucket', - 'action': - 'gcp-types/compute-v1:compute.projects.setUsageExportBucket', # pylint: disable=line-too-long - 'properties': - { - 'project': '$(ref.project.projectId)', - 'bucketName': 'gs://' + bucket_name - }, - 'metadata': { - 'dependsOn': [ - 'create-usage-export-bucket', - 'api-compute.googleapis.com', - ] - } + usage_resource = { + 'name': + '{}-set-usage-export-bucket'.format(context.env['name']), + 'action': + # https://cloud.google.com/compute/docs/reference/rest/v1/projects/setUsageExportBucket + 'gcp-types/compute-v1:compute.projects.setUsageExportBucket', # pylint: disable=line-too-long + 'properties': + { + 'project': '$(ref.{}-project.projectId)'.format(context.env['name']), + 'bucketName': 'gs://' + bucket_name + }, + 'metadata': { + 'dependsOn': [ + '{}-create-usage-export-bucket'.format(context.env['name']), + '{}-api-compute.googleapis.com'.format(context.env['name']), + ] } - ) + } + if properties.get('usageExportBucket', {}).get('reportNamePrefix'): + usage_resource['properties']['reportNamePrefix'] = properties.get('usageExportBucket', {}).get('reportNamePrefix') + resources.append(usage_resource) + return resources -def create_shared_vpc(project_id, properties): +def create_shared_vpc(context): """ Configure the project Shared VPC properties. """ resources = [] + properties = context.properties service_project = properties.get('sharedVPC') if service_project: resources.append( { - 'name': project_id + '-attach-xpn-service-' + service_project, - 'type': 'compute.beta.xpnResource', + 'name': '{}-attach-xpn-service-{}'.format(context.env['name'], service_project), + # https://cloud.google.com/compute/docs/reference/rest/v1/projects/enableXpnResource + 'type': 'gcp-types/compute-v1:compute.projects.enableXpnResource', 'metadata': { - 'dependsOn': ['api-compute.googleapis.com'] + 'dependsOn': ['{}-api-compute.googleapis.com'.format(context.env['name'])] }, 'properties': { 'project': service_project, 'xpnResource': { - 'id': '$(ref.project.projectId)', + 'id': '$(ref.{}-project.projectId)'.format(context.env['name']), 'type': 'PROJECT', } } @@ -372,13 +405,14 @@ def create_shared_vpc(project_id, properties): elif properties.get('sharedVPCHost'): resources.append( { - 'name': project_id + '-xpn-host', - 'type': 'compute.beta.xpnHost', + 'name': '{}-xpn-host'.format(context.env['name']), + # https://cloud.google.com/compute/docs/reference/rest/v1/projects/enableXpnHost + 'type': 'gcp-types/compute-v1:compute.projects.enableXpnHost', 'metadata': { - 'dependsOn': ['api-compute.googleapis.com'] + 'dependsOn': ['{}-api-compute.googleapis.com'.format(context.env['name'])] }, 'properties': { - 'project': '$(ref.project.projectId)' + 'project': '$(ref.{}-project.projectId)'.format(context.env['name']) } } ) @@ -386,93 +420,62 @@ def create_shared_vpc(project_id, properties): return resources -def delete_default_network(api_names_list): +def delete_default_network(context, api_names_list): """ Delete the default network. """ - icmp_name = 'delete-default-allow-icmp' - internal_name = 'delete-default-allow-internal' - rdp_name = 'delete-default-allow-rdp' - ssh_name = 'delete-default-allow-ssh' + default_firewalls = [ + 'default-allow-icmp', + 'default-allow-internal', + 'default-allow-rdp', + 'default-allow-ssh', + ] - resource = [ - { - 'name': icmp_name, - 'action': 'gcp-types/compute-beta:compute.firewalls.delete', - 'metadata': { - 'dependsOn': api_names_list - }, - 'properties': - { - 'firewall': 'default-allow-icmp', - 'project': '$(ref.project.projectId)', - } - }, - { - 'name': internal_name, - 'action': 'gcp-types/compute-beta:compute.firewalls.delete', - 'metadata': { - 'dependsOn': api_names_list - }, - 'properties': - { - 'firewall': 'default-allow-internal', - 'project': '$(ref.project.projectId)', - } - }, - { - 'name': rdp_name, - 'action': 'gcp-types/compute-beta:compute.firewalls.delete', - 'metadata': { - 'dependsOn': api_names_list - }, - 'properties': - { - 'firewall': 'default-allow-rdp', - 'project': '$(ref.project.projectId)', - } - }, - { - 'name': ssh_name, - 'action': 'gcp-types/compute-beta:compute.firewalls.delete', + resources = [] + for firewall_name in default_firewalls: + resources.append({ + 'name': '{}-delete-{}'.format(context.env['name'], firewall_name), + # https://cloud.google.com/compute/docs/reference/rest/v1/firewalls/delete + 'action': 'gcp-types/compute-v1:compute.firewalls.delete', 'metadata': { 'dependsOn': api_names_list }, 'properties': { - 'firewall': 'default-allow-ssh', - 'project': '$(ref.project.projectId)', + 'firewall': firewall_name, + 'project': '$(ref.{}-project.projectId)'.format(context.env['name']), } - } - ] + }) # Ensure the firewall rules are removed before deleting the VPC. network_dependency = copy.copy(api_names_list) - network_dependency.extend([icmp_name, internal_name, rdp_name, ssh_name]) + network_dependency.extend([row['name'] for row in resources]) - resource.append( + resources.append( { - 'name': 'delete-default-network', - 'action': 'gcp-types/compute-beta:compute.networks.delete', + 'name': '{}-delete-default-network'.format(context.env['name']), + # https://cloud.google.com/compute/docs/reference/rest/v1/networks/delete + 'action': 'gcp-types/compute-v1:compute.networks.delete', 'metadata': { 'dependsOn': network_dependency }, 'properties': { 'network': 'default', - 'project': '$(ref.project.projectId)' + 'project': '$(ref.{}-project.projectId)'.format(context.env['name']) } } ) - return resource + return resources -def delete_default_service_account(api_names_list): +def delete_default_service_account(context, api_names_list): """ Delete the default service account. """ resource = [ { - 'name': 'delete-default-sa', + 'name': '{}-delete-default-sa'.format(context.env['name']), + # https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/delete 'action': 'gcp-types/iam-v1:iam.projects.serviceAccounts.delete', 'metadata': { @@ -482,7 +485,7 @@ def delete_default_service_account(api_names_list): 'properties': { 'name': - 'projects/$(ref.project.projectId)/serviceAccounts/$(ref.project.projectNumber)-compute@developer.gserviceaccount.com' # pylint: disable=line-too-long + 'projects/$(ref.{}-project.projectId)/serviceAccounts/$(ref.{}-project.projectNumber)-compute@developer.gserviceaccount.com'.format(context.env['name'], context.env['name']) # pylint: disable=line-too-long } } ] diff --git a/dm/templates/project/project.py.schema b/dm/templates/project/project.py.schema index 0eb032f6b92..439fc0633df 100644 --- a/dm/templates/project/project.py.schema +++ b/dm/templates/project/project.py.schema @@ -15,11 +15,41 @@ info: title: Project author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of a single project. The project is created with a - billing account attached, permissions altered, APIs activated, and + billing account attached, permissions altered, APIs activated, and service accounts created. + For more information on this resource: + https://cloud.google.com/resource-manager/ + + APIs endpoints used by this template: + - gcp-types/cloudresourcemanager-v1:projects => + https://cloud.google.com/resource-manager/reference/rest/v1/projects/create + - deploymentmanager.v2.virtual.projectBillingInfo => + https://cloud.google.com/billing/reference/rest/v1/projects/updateBillingInfo + - gcp-types/servicemanagement-v1:servicemanagement.services.enable => + https://cloud.google.com/service-infrastructure/docs/service-management/reference/rest/v1/services/enable + - gcp-types/compute-v1:compute.subnetworks.setIamPolicy => + https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks/setIamPolicy + - gcp-types/compute-v1:compute.projects.enableXpnHost => + https://cloud.google.com/compute/docs/reference/rest/v1/projects/enableXpnHost + - gcp-types/compute-v1:compute.projects.enableXpnResource => + https://cloud.google.com/compute/docs/reference/rest/v1/projects/enableXpnResource + - gcp-types/compute-v1:compute.networks.delete => + https://cloud.google.com/compute/docs/reference/rest/v1/networks/delete + - gcp-types/compute-v1:compute.firewalls.delete => + https://cloud.google.com/compute/docs/reference/rest/v1/firewalls/delete + - gcp-types/iam-v1:iam.projects.serviceAccounts.delete => + https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/delete + - gcp-types/compute-v1:compute.projects.setUsageExportBucket => + https://cloud.google.com/compute/docs/reference/rest/v1/projects/setUsageExportBucket + - gcp-types/storage-v1:buckets => + https://cloud.google.com/storage/docs/json_api/v1/buckets/insert + - gcp-types/iam-v1:projects.serviceAccounts => + https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/create + imports: - path: ../iam_member/iam_member.py name: cft-iam_project_member.py @@ -30,12 +60,15 @@ required: - billingAccountId oneOf: - - required: - - sharedVPCHost - not: - required: - - sharedVPC - - sharedVPCSubnets + - allOf: + - required: + - sharedVPCHost + - not: + required: + - sharedVPC + - not: + required: + - sharedVPCSubnets - required: - sharedVPC not: @@ -61,6 +94,13 @@ allOf: - $ref: '#/definitions/networkAccess-requires-sharedVPCSubnets' definitions: + usageExportBucket-enabled: + type: boolean + description: | + Defines whether a usage export bucket must be created. + False by default so as to not inadvertently incur + costs to the user. It is strongly suggested to be enabled + (set to True). networkAccess-requires-sharedVPCSubnets: oneOf: - $ref: '#/definitions/no-sharedVPCSubnets' @@ -119,7 +159,11 @@ properties: Example: tokyo-rain-123 parent: type: object + additionalProperties: false description: The parent of the project. + required: + - type + - id properties: type: type: string @@ -131,7 +175,16 @@ properties: id: type: [integer, string] description: | - The ID of the projects' parent. + The ID of the project's parent. + pattern: ^[0-9]{8,25}$ + labels: + type: object + description: | + Map of labels associated with this Project. + Example: + name: wrench + mass: 1.3kg + count: 3 billingAccountId: type: string description: | @@ -144,12 +197,25 @@ properties: type: string description: The list of APIs to enable for each project. usageExportBucket: - type: boolean - description: | - Defines whether a usage export bucket must be created. - False by default so as to not inadvertently incur - costs to the user. It is strongly suggested to be enabled - (set to True). + oneOf: + - $ref: '#/definitions/usageExportBucket-enabled' + - type: object + additionalProperties: false + description: | + Defines usage export bucket config. + required: + - enabled + properties: + enabled: + $ref: '#/definitions/usageExportBucket-enabled' + reportNamePrefix: + type: string + description: | + An optional prefix for the name of the usage report object stored in bucketName. If not supplied, + defaults to usage. The report is stored as a CSV file named report_name_prefix_gce_YYYYMMDD.csv + where YYYYMMDD is the day of the usage according to Pacific Time. If you supply a prefix, + it should conform to Cloud Storage object naming conventions. + serviceAccounts: type: array uniqueItems: True diff --git a/dm/templates/project/tests/integration/project.bats b/dm/templates/project/tests/integration/project.bats index d3a3db09104..f8e5c1d11ed 100644 --- a/dm/templates/project/tests/integration/project.bats +++ b/dm/templates/project/tests/integration/project.bats @@ -57,16 +57,29 @@ function teardown() { ########## TESTS ########## @test "Deploying project $DEPLOYMENT_NAME" { - gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" --config "${CONFIG}" + run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" --config "${CONFIG}" + + echo "Status: $status" + echo "Output: $output" + + [[ "$status" -eq 0 ]] } @test "Verifying that project $CLOUD_FOUNDATION_PROJECT_ID was created" { run gcloud projects list + + echo "Status: $status" + echo "Output: $output" + [[ "$output" =~ "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" ]] } @test "Verifying that APIs were activated for project ${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" { run gcloud services list --project "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$output" =~ "compute.googleapis.com" ]] [[ "$output" =~ "deploymentmanager.googleapis.com" ]] [[ "$output" =~ "pubsub.googleapis.com" ]] @@ -78,21 +91,37 @@ function teardown() { @test "Verifying that usage report export to the bucket was created for project ${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" { run gcloud compute project-info describe --project "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" \ --format="flattened[no-pad](usageExportLocation)" + + echo "Status: $status" + echo "Output: $output" + [[ "$output" =~ "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}-usage-export" ]] } @test "Verifying that the project is a shared vpc host project for project ${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" { run gcloud compute shared-vpc organizations list-host-projects "${CLOUD_FOUNDATION_ORGANIZATION_ID}" + + echo "Status: $status" + echo "Output: $output" + [[ "$output" =~ "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" ]] } @test "Verifying that the default VPC was deleted for project ${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" { run gcloud compute networks list --project "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ ! "$output" =~ "default" ]] } @test "Verifying that the default Compute Engine SA was removed for project ${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" { run gcloud iam service-accounts list --project "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ ! "$output" =~ "Compute Engine default service account" ]] } @@ -101,12 +130,40 @@ function teardown() { --flatten="bindings[].members" \ --format='table(bindings.role)' \ --filter="bindings.members:sa-${RAND}@${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}.iam.gserviceaccount.com" + + echo "Status: $status" + echo "Output: $output" + [[ "$output" =~ "roles/editor" ]] [[ "$output" =~ "roles/viewer" ]] } @test "Deleting deployment" { - gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" -q + ## TODO project creation should work without disabling XPN hosts. + + run gcloud alpha resource-manager liens list --project "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] + + run gcloud compute shared-vpc disable "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] + + run gcloud alpha resource-manager liens list --project "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] + + run gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" -q + + echo "Status: $status" + echo "Output: $output" + [[ "$status" -eq 0 ]] run gcloud projects list [[ ! "$output" =~ "${CLOUD_FOUNDATION_PROJECT_ID}-${RAND}" ]] diff --git a/dm/templates/project/tests/integration/project.yaml b/dm/templates/project/tests/integration/project.yaml index dab7e68d073..cc4808ef5c6 100644 --- a/dm/templates/project/tests/integration/project.yaml +++ b/dm/templates/project/tests/integration/project.yaml @@ -25,5 +25,5 @@ resources: roles: - roles/editor - roles/viewer - usageExportBucket: true - sharedVPCHost: true + usageExportBucket: True + sharedVPCHost: True diff --git a/dm/templates/pubsub/examples/pubsub.yaml b/dm/templates/pubsub/examples/pubsub.yaml index ad9ad41c34e..262cb93492b 100644 --- a/dm/templates/pubsub/examples/pubsub.yaml +++ b/dm/templates/pubsub/examples/pubsub.yaml @@ -11,7 +11,7 @@ resources: - name: test-pubsub type: pubsub.py properties: - topic: test-topic + name: test-topic accessControl: - role: roles/pubsub.subscriber members: diff --git a/dm/templates/pubsub/examples/pubsub_push.yaml b/dm/templates/pubsub/examples/pubsub_push.yaml index 92489b5cf2e..576b968e78f 100644 --- a/dm/templates/pubsub/examples/pubsub_push.yaml +++ b/dm/templates/pubsub/examples/pubsub_push.yaml @@ -12,7 +12,7 @@ resources: - name: test-push-pubsub type: pubsub.py properties: - topic: test-topic + name: test-topic subscriptions: - name: push-subscription pushEndpoint: diff --git a/dm/templates/pubsub/pubsub.py b/dm/templates/pubsub/pubsub.py index 3800f25dae3..47f066b6c2c 100644 --- a/dm/templates/pubsub/pubsub.py +++ b/dm/templates/pubsub/pubsub.py @@ -13,89 +13,126 @@ # limitations under the License. """ This template creates a Pub/Sub (publish-subscribe) service. """ -def create_subscription(resource_name, spec, topic_resource_name, spec_index): +from hashlib import sha1 +import json + + +def set_optional_property(destination, source, prop_name): + """ Copies the property value if present. """ + + if prop_name in source: + destination[prop_name] = source[prop_name] + +def create_subscription(resource_name, project_id, spec): """ Create a pull/push subscription from the simplified spec. """ + suffix = 'subscription-{}'.format(sha1(resource_name + json.dumps(spec)).hexdigest()[:10]) + subscription = { - 'name': '{}-subscription-{}'.format(resource_name, spec_index), - 'type': 'pubsub.v1.subscription', + 'name': '{}-{}'.format(resource_name, suffix), + # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions + 'type': 'gcp-types/pubsub-v1:projects.subscriptions', 'properties':{ - 'subscription': spec['name'], - 'topic': '$(ref.{}.name)'.format(topic_resource_name) + 'subscription': spec.get('name', suffix), + 'name': 'projects/{}/subscriptions/{}'.format(project_id, spec.get('name', suffix)), + 'topic': '$(ref.{}.name)'.format(resource_name) } } + resources_list = [subscription] + + optional_properties = [ + 'labels', + 'pushConfig', + 'ackDeadlineSeconds', + 'retainAckedMessages', + 'messageRetentionDuration', + 'expirationPolicy', + ] + + for prop in optional_properties: + set_optional_property(subscription['properties'], spec, prop) push_endpoint = spec.get('pushEndpoint') if push_endpoint is not None: subscription['properties']['pushConfig'] = { - 'pushEndpoint': push_endpoint + 'pushEndpoint': push_endpoint, } - ack_deadline_seconds = spec.get('ackDeadlineSeconds') - if ack_deadline_seconds is not None: - subscription['properties']['ackDeadlineSeconds'] = ack_deadline_seconds - - set_access_control(subscription, spec) - - return subscription - -def create_iam_policy(bindings_spec): - """ Create an IAM policy for the resource. """ - - return { - 'gcpIamPolicy': { - 'bindings': bindings_spec + if spec.get('accessControl'): + policy = { + 'name': "{}-{}".format(subscription['name'], 'setIamPolicy'), + # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/setIamPolicy + 'action': 'gcp-types/pubsub-v1:pubsub.projects.subscriptions.setIamPolicy', + 'properties': + { + 'resource': '$(ref.{}.name)'.format(subscription['name']), + 'policy': { + 'bindings': spec['accessControl'] + } + }, + 'metadata': { + 'dependsOn': [subscription['name']] + } } - } + resources_list.append(policy) -def set_access_control(resource, context): - """ If necessary, define access control for the resource """ + return resources_list - access_control = context.get('accessControl') - if access_control is not None: - resource['accessControl'] = create_iam_policy(access_control) +def generate_config(context): + """ Entry point for the deployment resources. """ -def create_pubsub(resource_name, pubsub_spec): - """ Create a topic with subscriptions. """ + properties = context.properties + name = properties.get('name', properties.get('topic', context.env['name'])) + project_id = properties.get('project', context.env['project']) - topic_name = pubsub_spec.get('topic', resource_name) - topic_resource_name = '{}-topic'.format(resource_name) topic = { - 'name': topic_resource_name, - 'type': 'pubsub.v1.topic', + 'name': context.env['name'], + # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics + 'type': 'gcp-types/pubsub-v1:projects.topics', 'properties':{ - 'topic': topic_name + 'topic': name, + 'name': 'projects/{}/topics/{}'.format(project_id, name), } } + resources_list = [topic] + + if properties.get('accessControl'): + policy = { + 'name': "{}-{}".format(context.env['name'], 'setIamPolicy'), + # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/setIamPolicy + 'action': 'gcp-types/pubsub-v1:pubsub.projects.topics.setIamPolicy', + 'properties': + { + 'resource': '$(ref.{}.name)'.format(context.env['name']), + 'policy': { + 'bindings': properties['accessControl'] + } + }, + 'metadata': { + 'dependsOn': [context.env['name']] + } + } + resources_list.append(policy) - set_access_control(topic, pubsub_spec) - - subscription_specs = pubsub_spec.get('subscriptions', []) - subscriptions = [create_subscription(resource_name, spec, - topic_resource_name, index) - for (index, spec) - in enumerate(subscription_specs, 1)] - - return [topic] + subscriptions + optional_properties = [ + 'labels', + ] -def create_topic_outputs(topic_resource): - """ Create outputs for the topic. """ + for prop in optional_properties: + set_optional_property(topic['properties'], properties, prop) - return [ - { - 'name': 'topicName', - 'value': '$(ref.{}.name)'.format(topic_resource['name']) - } - ] -def generate_config(context): - """ Entry point for the deployment resources. """ + subscription_specs = properties.get('subscriptions', []) - resource_name = context.env['name'] - pubsub_resources = create_pubsub(resource_name, context.properties) - pubsub_outputs = create_topic_outputs(pubsub_resources[0]) + for spec in subscription_specs: + resources_list = resources_list + create_subscription(context.env['name'], project_id, spec) return { - 'resources': pubsub_resources, - 'outputs': pubsub_outputs + 'resources': resources_list, + 'outputs': [ + { + 'name': 'topicName', + 'value': '$(ref.{}.name)'.format(context.env['name']) + } + ], } diff --git a/dm/templates/pubsub/pubsub.py.schema b/dm/templates/pubsub/pubsub.py.schema index 4f7d04030d3..345d8e53157 100644 --- a/dm/templates/pubsub/pubsub.py.schema +++ b/dm/templates/pubsub/pubsub.py.schema @@ -14,40 +14,189 @@ info: title: Pub/Sub (publish-subscribe) service + version: 1.0.0 author: Sourced Group Inc. description: | Creates a topic, optionally with multiple subscriptions. + For more information on this resource: + - https://cloud.google.com/pubsub/ + + APIs endpoints used by this template: + - gcp-types/pubsub-v1:projects.subscriptions => + https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions + - gcp-types/pubsub-v1:projects.topics => + https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics + additionalProperties: false +oneOf: + - required: + - name + - required: + - topic + properties: + name: + type: string + description: | + The name of the topic that will publish messages. Resource name would be used if omitted. topic: type: string description: | The name of the topic that will publish messages. If not specified, the deployment name is used. + DEPRECATED. + project: + type: string + description: | + The project ID of the project containing PubSub resources. The + Google apps domain is prefixed if applicable. + labels: + type: object + description: | + An object containing a list of "key": value pairs. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. subscriptions: type: array - description: A list of topic's subscriptions. + uniqueItems: True + description: | + A list of topic's subscriptions. item: type: object - description: The topic's subscription. + additionalProperties: false + description: | + The topic's subscription. + oneOf: + - required: + - pushEndpoint + - required: + - pushConfig properties: name: type: string - description: The subscription name. + description: | + The subscription name. Resource name would be used if omitted. pushEndpoint: type: string description: | The URL of the endpoint to push the messages to. + pushConfig: + type: object + additionalProperties: false + description: | + If push delivery is used with this subscription, this field is used to configure it. + An empty pushConfig signifies that the subscriber will pull and ack messages using API methods. + required: + - pushEndpoint + properties: + pushEndpoint: + type: string + description: | + A URL locating the endpoint to which messages should be pushed. + For example, a Webhook endpoint might use "https://example.com/push". + oidcToken: + type: object + description: | + If specified, Pub/Sub will generate and attach an OIDC JWT token as an Authorization header + in the HTTP request for every pushed message. + properties: + serviceAccountEmail: + type: string + description: | + Service account email to be used for generating the OIDC token. The caller + (for subscriptions.create, subscriptions.patch, and subscriptions.modifyPushConfig RPCs) + must have the iam.serviceAccounts.actAs permission for the service account. + audience: + type: string + description: | + Audience to be used when generating OIDC token. The audience claim identifies the recipients + that the JWT is intended for. The audience value is a single case-sensitive string. + Having multiple values (array) for the audience field is not supported. + More info about the OIDC JWT token audience here: https://tools.ietf.org/html/rfc7519#section-4.1.3 + Note: if not specified, the Push endpoint URL will be used. + attributes: + type: object + description: | + Endpoint configuration attributes. + + Every endpoint has a set of API supported attributes that can be used to control different + aspects of the message delivery. + + The currently supported attribute is x-goog-version, which you can use to change the format + of the pushed message. This attribute indicates the version of the data expected by the endpoint. + This controls the shape of the pushed message (i.e., its fields and metadata). + The endpoint version is based on the version of the Pub/Sub API. + + If not present during the subscriptions.create call, it will default to the version of the + API used to make such call. If not present during a subscriptions.modifyPushConfig call, + its value will not be changed. subscriptions.get calls will always return a valid version, + even if the subscription was created without this attribute. + + The possible values for this attribute are: + + v1beta1: uses the push format defined in the v1beta1 Pub/Sub API. + v1 or v1beta2: uses the push format defined in the v1 Pub/Sub API. + An object containing a list of "key": value pairs. + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. ackDeadlineSeconds: type: integer description: | - The maximum time to acknowledge a message receipt before retry. - minimum: 10 + The approximate amount of time (on a best-effort basis) Pub/Sub waits for the subscriber to acknowledge + receipt before resending the message. In the interval after the message is delivered and + before it is acknowledged, it is considered to be outstanding. + During that time period, the message will not be redelivered (on a best-effort basis). + + For pull subscriptions, this value is used as the initial value for the ack deadline. To override this + value for a given message, call subscriptions.modifyAckDeadline with the corresponding ackId if using + non-streaming pull or send the ackId in a StreamingModifyAckDeadlineRequest if using streaming pull. + The minimum custom deadline you can specify is 10 seconds. The maximum custom deadline you can specify + is 600 seconds (10 minutes). If this parameter is 0, a default value of 10 seconds is used. + + For push delivery, this value is also used to set the request timeout for the call to the push endpoint. + + If the subscriber never acknowledges the message, the Pub/Sub system will eventually redeliver the message. + minimum: 0 maximum: 600 + retainAckedMessages: + type: bool + description: | + Indicates whether to retain acknowledged messages. If true, then messages are not expunged from the + subscription's backlog, even if they are acknowledged, until they fall out of the + messageRetentionDuration window. This must be true if you would like to subscriptions.seek to a timestamp. + messageRetentionDuration: + type: string + description: | + How long to retain unacknowledged messages in the subscription's backlog, from the moment a message + is published. If retainAckedMessages is true, then this also configures the retention of + acknowledged messages, and thus configures how far back in time a subscriptions.seek can be done. + Defaults to 7 days. Cannot be more than 7 days or less than 10 minutes. + + A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". + expirationPolicy: + type: object + description: | + A policy that specifies the conditions for this subscription's expiration. A subscription is + considered active as long as any connected subscriber is successfully consuming messages from + the subscription or is issuing operations on the subscription. If expirationPolicy is not set, + a default policy with ttl of 31 days will be used. The minimum allowed value + for expirationPolicy.ttl is 1 day. + required: + - ttl + properties: + ttl: + type: string + description: | + Specifies the "time-to-live" duration for an associated resource. The resource expires if it is + not active for a period of ttl. The definition of "activity" depends on the type of + the associated resource. The minimum and maximum allowed values for ttl depend on the type + of the associated resource, as well. If ttl is not set, the associated resource never expires. + + A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". accessControl: type: array + uniqueItems: True description: | The subscription's IAM policy. For details, see https://cloud.google.com/pubsub/docs/reference/rest/v1/Policy. @@ -66,8 +215,9 @@ properties: type: string accessControl: type: array + uniqueItems: True description: | - The subscription's IAM policy. + The topic's IAM policy. For details, see https://cloud.google.com/pubsub/docs/reference/rest/v1/Policy item: type: object diff --git a/dm/templates/pubsub/tests/integration/pubsub.yaml b/dm/templates/pubsub/tests/integration/pubsub.yaml index 51a43593c43..c105bf7e74f 100644 --- a/dm/templates/pubsub/tests/integration/pubsub.yaml +++ b/dm/templates/pubsub/tests/integration/pubsub.yaml @@ -11,7 +11,7 @@ resources: - name: test-pubsub-${RAND} type: pubsub.py properties: - topic: test-topic-${RAND} + name: test-topic-${RAND} accessControl: - role: roles/pubsub.subscriber members: diff --git a/dm/templates/route/examples/route.yaml b/dm/templates/route/examples/route.yaml index f640b5c360e..2bbbd07f58b 100644 --- a/dm/templates/route/examples/route.yaml +++ b/dm/templates/route/examples/route.yaml @@ -35,7 +35,6 @@ resources: network: routes: - name: test-ip-route - routeType: ipaddress nextHopIp: priority: 20000 destRange: 0.0.0.0/0 diff --git a/dm/templates/route/route.py b/dm/templates/route/route.py index 7113081ffc1..1eb6690b5cb 100644 --- a/dm/templates/route/route.py +++ b/dm/templates/route/route.py @@ -14,61 +14,45 @@ """This template creates a custom route.""" +from hashlib import sha1 +import json + + def generate_config(context): """ Entry point for the deployment resources. """ - network_name = generate_network_url(context.properties) + properties = context.properties + project_id = properties.get('project', context.env['project']) + + network_name = generate_network_url(properties) resources = [] out = {} - for i, route in enumerate(context.properties['routes'], 1000): - - # Set the common route properties. - properties = { + for i, route in enumerate(properties['routes'], 1000): + name = route.get('name') + if not name: + name = '{}-{}'.format(context.env['name'], sha1(json.dumps(route)).hexdigest()[:10]) + + route_properties = { + 'name': name, 'network': network_name, - 'tags': route['tags'], - 'priority': route.get('priority', - i), - 'destRange': route['destRange'] + 'project': project_id, + 'priority': route.get('priority', i), } - - # Check the route type and fill out the following fields: - if route['routeType'] == 'ipaddress': - properties['nextHopIp'] = route.get('nextHopIp') - elif route['routeType'] == 'instance': - instance_name = route.get('instanceName') - zone = route.get('zone', '') - properties['nextHopInstance'] = generate_instance_url( - context.env['project'], - zone, - instance_name - ) - elif route['routeType'] == 'gateway': - gateway_name = route.get('gatewayName') - properties['nextHopGateway'] = generate_gateway_url( - context.env['project'], - gateway_name - ) - elif route['routeType'] == 'vpntunnel': - vpn_tunnel_name = route.get('vpnTunnelName') - region = route.get('region', '') - properties['nextHopVpnTunnel'] = generate_vpn_tunnel_url( - context.env['project'], - region, - vpn_tunnel_name - ) + for specified_properties in route: + route_properties[specified_properties] = route[specified_properties] resources.append( { - 'name': route['name'], - 'type': 'compute.v1.route', - 'properties': properties + 'name': name, + 'type': 'single_route.py', + 'properties': route_properties } ) - out[route['name']] = { - 'selfLink': '$(ref.' + route['name'] + '.selfLink)', - 'nextHopNetwork': network_name + out[name] = { + 'selfLink': '$(ref.' + name + '.selfLink)', + 'nextHopNetwork': '$(ref.' + name + '.nextHopNetwork)', } outputs = [{'name': 'routes', 'value': out}] @@ -88,31 +72,3 @@ def generate_network_url(properties): network_url = 'global/networks/{}'.format(network_name) return network_url - - -def generate_instance_url(project, zone, instance): - """ Format the resource name as a resource URI. """ - - is_self_link = '/' in instance or '.' in instance - - if is_self_link: - instance_url = instance - else: - instance_url = 'projects/{}/zones/{}/instances/{}' - instance_url = instance_url.format(project, zone, instance) - - return instance_url - - -def generate_gateway_url(project, gateway): - """ Format the resource name as a resource URI. """ - return 'projects/{}/global/gateways/{}'.format(project, gateway) - - -def generate_vpn_tunnel_url(project, region, vpn_tunnel): - """ Format the resource name as a resource URI. """ - return 'projects/{}/regions/{}/vpnTunnels/{}'.format( - project, - region, - vpn_tunnel - ) diff --git a/dm/templates/route/route.py.schema b/dm/templates/route/route.py.schema index 7ebd70d7079..48fabfeecd5 100644 --- a/dm/templates/route/route.py.schema +++ b/dm/templates/route/route.py.schema @@ -14,14 +14,22 @@ info: title: Route - author: Sourced Group - description: Creates a custom route. + author: Sourced Group Inc. + version: 1.0.0 + description: | + Creates a custom route. - For more information on this resource: - - https://cloud.google.com/compute/docs/reference/rest/v1/routes + For more information on this resource: + https://cloud.google.com/vpc/docs/routes + + APIs endpoints used by this template: + - gcp-types/compute-v1:instanceTemplates => + https://cloud.google.com/compute/docs/reference/rest/v1/routes imports: - path: route.py + - path: ../route/single_route.py + name: single_route.py additionalProperties: false @@ -30,79 +38,26 @@ required: - routes properties: - name: - type: string - description: Name of the route resource. network: type: string - description: Name of the network the route applies to. + description: | + Name of the network the route applies to. + project: + type: string + description: | + The project ID of the project containing the Route. routes: type: array + uniqueItems: True + minItems: 1 description: A list of routes. items: - name: - type: string - description: Name of the route. - routeType: - type: string - description: The resource type that will handle the matching packets. - enum: - - ipaddress - - instance - - gateway - - vpntunnel - tags: - type: array - description: A list of instance tags to which the route applies. - items: - type: string - description: An instance tag for the route. - priority: - type: number - description: The priority of the route. - default: 1000 - minimum: 0 - maximum: 65535 - destRange: - type: string - description: | - The destination range of outgoing packets the route applies - to. Example: 192.168.0.1/10. Only IPv4 is supported. - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}\/[0-9]{1,2}$ - nextHopIp: - type: string - description: | - Used when routeType is 'ipaddress'. - The network IP address of the instance that should handle the matching - packets. Example: 192.168.0.1. Only IPv4 is supported. - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ - instanceName: - type: string - description: | - Used when routeType is 'instance'. - The name of the instance that should handle the matching packets. - zone: - type: string - description: | - Used when routeType is 'instance'. - The zone where the instance resides. - gatewayName: - type: string - description: | - Used when routeType is 'gateway'. - The name of the gateway that will handle the matching packets. Only the - 'default-internet-gateway' value is supported. - default: default-internet-gateway - vpnTunnelName: - type: string - description: | - Used when routeType is 'vpntunnel'. - The name of the VPN tunnel that should handle the matching packets. - region: - type: string - description: | - Used when routeType is 'vpntunnel'. - The region where the VPN tunnel resides. + type: object + description: | + Please check the properties in single_route.py.schema for details. + required: + - tags + - destRange outputs: properties: @@ -116,14 +71,9 @@ outputs: patternProperties: ".*": type: object - description: Details for a route resource. - properties: - selfLink: - type: string - description: The URI (SelfLink) of the firewall rule resource. - nextHopNetwork: - type: string - description: URL to a Network that should handle matching packets. + description: | + Details for a route resource. Please check the outputs in + single_route.py.schema for details. documentation: - templates/route/README.md diff --git a/dm/templates/route/single_route.py b/dm/templates/route/single_route.py new file mode 100644 index 00000000000..9cb537714d1 --- /dev/null +++ b/dm/templates/route/single_route.py @@ -0,0 +1,119 @@ +# Copyright 2019 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This template creates a custom route.""" + + +from hashlib import sha1 +import json + + +def generate_config(context): + """ Entry point for the deployment resources. """ + + properties = context.properties + project_id = properties.get('project', context.env['project']) + name = context.env['name'] + + # Set the common route properties. + res_properties = { + 'name': properties.get('name', name), + 'network': properties['network'], + 'project': project_id, + 'tags': properties['tags'], + 'priority': properties['priority'], + 'destRange': properties['destRange'] + } + + # Check the route type and fill out the following fields: + if properties.get('routeType') == 'instance': + instance_name = properties.get('instanceName') + zone = properties.get('zone', '') + res_properties['nextHopInstance'] = generate_instance_url( + project_id, + zone, + instance_name + ) + elif properties.get('routeType') == 'gateway': + gateway_name = properties.get('gatewayName') + res_properties['nextHopGateway'] = generate_gateway_url( + project_id, + gateway_name + ) + elif properties.get('routeType') == 'vpntunnel': + vpn_tunnel_name = properties.get('vpnTunnelName') + region = properties.get('region', '') + res_properties['nextHopVpnTunnel'] = generate_vpn_tunnel_url( + project_id, + region, + vpn_tunnel_name + ) + + optional_properties = [ + 'nextHopIp', + 'nextHopInstance', + 'nextHopNetwork', + 'nextHopGateway', + 'nextHopVpnTunnel', + ] + + for prop in optional_properties: + if prop in properties: + res_properties[prop] = properties[prop] + + resources = [ + { + 'name': name, + # https://cloud.google.com/compute/docs/reference/rest/v1/routes + 'type': 'gcp-types/compute-v1:routes', + 'properties': res_properties + } + ] + + outputs = [ + {'name': 'selfLink', 'value': '$(ref.' + name + '.selfLink)'}, + {'name': 'nextHopNetwork', 'value': properties['network']}, + ] + + return {'resources': resources, 'outputs': outputs} + + +def generate_instance_url(project, zone, instance): + """ Format the resource name as a resource URI. """ + + is_self_link = '/' in instance or '.' in instance + + if is_self_link: + instance_url = instance + else: + instance_url = 'projects/{}/zones/{}/instances/{}' + instance_url = instance_url.format(project, zone, instance) + + return instance_url + + +def generate_gateway_url(project, gateway): + """ Format the resource name as a resource URI. """ + return 'projects/{}/global/gateways/{}'.format(project, gateway) + + +def generate_vpn_tunnel_url(project, region, vpn_tunnel): + """ Format the resource name as a resource URI. """ + is_self_link = '/' in vpn_tunnel or '.' in vpn_tunnel + + if is_self_link: + tunnel_url = vpn_tunnel + else: + tunnel_url = 'projects/{}/regions/{}/vpnTunnels/{}' + tunnel_url = tunnel_url.format(project, region, vpn_tunnel) + return tunnel_url diff --git a/dm/templates/route/single_route.py.schema b/dm/templates/route/single_route.py.schema new file mode 100644 index 00000000000..61720e6c992 --- /dev/null +++ b/dm/templates/route/single_route.py.schema @@ -0,0 +1,242 @@ +# Copyright 2019 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +info: + title: Route + author: Sourced Group Inc. + version: 1.0.0 + description: | + Creates a custom route. + + For more information on this resource: + https://cloud.google.com/vpc/docs/routes + + APIs endpoints used by this template: + - gcp-types/compute-v1:instanceTemplates => + https://cloud.google.com/compute/docs/reference/rest/v1/routes + +imports: + - path: single_route.py + +additionalProperties: false + +required: +- name +- network +- tags +- destRange + +allOf: + - oneOf: + - required: + - nextHopInstance + - required: + - nextHopNetwork + - required: + - nextHopGateway + - required: + - nextHopVpnTunnel + - anyOf: + - required: + - nextHopIp + - required: + - routeType + - oneOf: + - allOf: + - not: + required: + - routeType + - not: + required: + - nextHopIp + - required: + - nextHopIp + - required: + - instanceName + - required: + - gatewayName + - required: + - vpnTunnelName + - oneOf: + - not: + required: + - gatewayName + - allOf: + - required: + - gatewayName + - routeType + - properties: + routeType: + enum: ["gateway"] + - oneOf: + - not: + required: + - vpnTunnelName + - allOf: + - required: + - vpnTunnelName + - routeType + - region + - properties: + routeType: + enum: ["vpntunnel"] + - oneOf: + - not: + required: + - instanceName + - allOf: + - required: + - instanceName + - routeType + - zone + - properties: + routeType: + enum: ["instance"] + - oneOf: + - not: + required: + - nextHopIp + - allOf: + - required: + - nextHopIp + - routeType + - properties: + routeType: + enum: ["ipaddress"] + - allOf: + - required: + - nextHopIp + - not: + required: + - routeType + +properties: + name: + type: string + description: | + Name of the resource. Provided by the client when the resource is created. The name must be 1-63 characters long, + and comply with RFC1035. Specifically, the name must be 1-63 characters long and match the regular expression + [a-z]([-a-z0-9]*[a-z0-9])?. The first character must be a lowercase letter, and all following characters + (except for the last character) must be a dash, lowercase letter, or digit. + The last character must be a lowercase letter or digit. + Resource name would be used if omitted. + description: + type: string + description: | + An optional description of this resource. Provide this property when you create the resource. + network: + type: string + description: | + Name of the network the route applies to. + project: + type: string + description: | + The project ID of the project containing the Route. + routeType: + type: string + description: | + The resource type that will handle the matching packets. + Optionally you can use nextHop* attributes without specifying this field + enum: + - ipaddress + - instance + - gateway + - vpntunnel + tags: + type: array + uniqueItems: True + minItems: 1 + description: | + A list of instance tags to which the route applies. + items: + type: string + description: An instance tag for the route. + priority: + type: number + description: | + The priority of this route. Priority is used to break ties in cases where there is more than one + matching route of equal prefix length. In cases where multiple routes have equal prefix length, the one + with the lowest-numbered priority value wins. The default value is 1000. + The priority value must be from 0 to 65535, inclusive. + default: 1000 + minimum: 0 + maximum: 65535 + destRange: + type: string + description: | + The destination range of outgoing packets the route applies + to. Example: 192.168.0.1/10. Only IPv4 is supported. + pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}\/[0-9]{1,2}$ + nextHopInstance: + type: string + description: | + The URL to an instance that should handle matching packets. You can specify this as a full or partial URL. + For example: + https://www.googleapis.com/compute/v1/projects/project/zones/zone/instances/ + nextHopIp: + type: string + description: | + Used when routeType is 'ipaddress'. + The network IP address of the instance that should handle the matching + packets. Example: 192.168.0.1. Only IPv4 is supported. + pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ + nextHopNetwork: + type: string + description: | + The URL of the local network if it should handle matching packets. + nextHopGateway: + type: string + description: | + The URL to a gateway that should handle matching packets. You can only specify the internet gateway using + a full or partial valid URL: + projects/project/global/gateways/default-internet-gateway + nextHopVpnTunnel: + type: string + description: | + The URL to a VpnTunnel that should handle matching packets. + instanceName: + type: string + description: | + Used when routeType is 'instance'. + The name of the instance that should handle the matching packets. + zone: + type: string + description: | + Used when routeType is 'instance'. + The zone where the instance resides. + gatewayName: + type: string + description: | + Used when routeType is 'gateway'. + The name of the gateway that will handle the matching packets. Only the + 'default-internet-gateway' value is supported. + vpnTunnelName: + type: string + description: | + Used when routeType is 'vpntunnel'. + The name of the VPN tunnel that should handle the matching packets. + region: + type: string + description: | + Used when routeType is 'vpntunnel'. + The region where the VPN tunnel resides. + +outputs: + properties: + selfLink: + type: string + description: The URI (SelfLink) of the firewall rule resource. + nextHopNetwork: + type: string + description: URL to a Network that should handle matching packets. diff --git a/dm/templates/route/tests/integration/route.bats b/dm/templates/route/tests/integration/route.bats index 2a1760c7e48..732b036cabc 100644 --- a/dm/templates/route/tests/integration/route.bats +++ b/dm/templates/route/tests/integration/route.bats @@ -128,29 +128,54 @@ function teardown() { @test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { - gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ + run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" \ --config "${CONFIG}" + + echo "status = ${status}" + echo "output = ${output}" + + [ "$status" -eq 0 ] } @test "Verifying that resources were created in deployment ${DEPLOYMENT_NAME}" { run gcloud compute routes list --filter="name:gateway-route-${RAND} AND priority:1002" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "status = ${status}" + echo "output = ${output}" + echo "lines1 = ${lines[1]}" + [ "$status" -eq 0 ] [[ "${lines[1]}" =~ "gateway-route-${RAND}" ]] run gcloud compute routes list --filter="name:instance-route-${RAND} AND priority:1001" \ - --project "${CLOUD_FOUNDATION_PROJECT_ID}" + --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "status = ${status}" + echo "output = ${output}" + echo "lines1 = ${lines[1]}" + [ "$status" -eq 0 ] [[ "${lines[1]}" =~ "instance-route-${RAND}" ]] run gcloud compute routes list --filter="(name:ip-route-${RAND} AND priority:20000)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "status = ${status}" + echo "output = ${output}" + echo "lines1 = ${lines[1]}" + [ "$status" -eq 0 ] [[ "${lines[1]}" =~ "ip-route-${RAND}" ]] run gcloud compute routes list --filter="(name:vpn-tunnel-route-${RAND} AND priority:500)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" + + echo "status = ${status}" + echo "output = ${output}" + echo "lines1 = ${lines[1]}" + [ "$status" -eq 0 ] [[ "${lines[1]}" =~ "vpn-tunnel-route-${RAND}" ]] } diff --git a/dm/templates/route/tests/integration/route.yaml b/dm/templates/route/tests/integration/route.yaml index 378ca744466..196110c873d 100644 --- a/dm/templates/route/tests/integration/route.yaml +++ b/dm/templates/route/tests/integration/route.yaml @@ -15,7 +15,6 @@ resources: network: network-${RAND} routes: - name: ip-route-${RAND} - routeType: ipaddress nextHopIp: 10.118.8.12 priority: 20000 destRange: 0.0.0.0/0 @@ -42,3 +41,7 @@ resources: destRange: 0.0.0.0/0 tags: - my-vpntunnelroute-tag + - nextHopIp: 10.118.8.13 + destRange: 0.0.0.0/0 + tags: + - my-iproute-tag diff --git a/dm/templates/runtime_config/examples/runtime_config.yaml b/dm/templates/runtime_config/examples/runtime_config.yaml index c7fcc1da0c4..557087de7d8 100644 --- a/dm/templates/runtime_config/examples/runtime_config.yaml +++ b/dm/templates/runtime_config/examples/runtime_config.yaml @@ -8,15 +8,15 @@ resources: - name: my-test-config type: runtime_config.py properties: - config: my-test-config + name: my-test-config description: my config description variables: - - variable: myapp/dev/sql/connection_string + - name: myapp/dev/sql/connection_string text: super text value - - variable: myapp/dev/web/wildcardcert + - name: myapp/dev/web/wildcardcert value: c3VwZXJhd2Vzb21ldGV4dAo= waiters: - - waiter: my-test-waiter + - name: my-test-waiter timeout: 3.5s success: cardinality: diff --git a/dm/templates/runtime_config/runtime_config.py b/dm/templates/runtime_config/runtime_config.py index 82808c7a54d..50bdfaf5c19 100644 --- a/dm/templates/runtime_config/runtime_config.py +++ b/dm/templates/runtime_config/runtime_config.py @@ -16,21 +16,27 @@ """ +from hashlib import sha1 + + def generate_config(context): """ Entry point for the deployment resources. """ resources = [] properties = context.properties project_id = properties.get('projectId', context.env['project']) - name = properties.get('config', context.env['name']) + name = properties.get('name', properties.get('config', context.env['name'])) parent = 'projects/{}/configs/{}'.format(project_id, name) # The runtimeconfig resource. runtime_config = { 'name': name, - 'type': 'runtimeconfig.v1beta1.config', + # https://cloud.google.com/deployment-manager/runtime-configurator/reference/rest/v1beta1/projects.configs + 'type': 'gcp-types/runtimeconfig-v1beta1:projects.configs', 'properties': { 'config': name, + # TODO: uncomment after gcp type is fixed + # 'project': project_id, 'description': properties['description'] } } @@ -39,10 +45,12 @@ def generate_config(context): # The runtimeconfig variable resources. for variable in properties.get('variables', []): + suffix = sha1('{}-{}'.format(context.env['name'], variable.get('name', variable.get('variable')))).hexdigest()[:10] + variable['project'] = project_id variable['parent'] = parent variable['config'] = name variable_res = { - 'name': variable['variable'], + 'name': '{}-{}'.format(context.env['name'], suffix), 'type': 'variable.py', 'properties': variable } @@ -50,10 +58,12 @@ def generate_config(context): # The runtimeconfig waiter resources. for waiter in properties.get('waiters', []): + suffix = sha1('{}-{}'.format(context.env['name'], waiter.get('name', waiter.get('waiter')))).hexdigest()[:10] + waiter['project'] = project_id waiter['parent'] = parent waiter['config'] = name waiter_res = { - 'name': waiter['waiter'], + 'name': '{}-{}'.format(context.env['name'], suffix), 'type': 'waiter.py', 'properties': waiter } diff --git a/dm/templates/runtime_config/runtime_config.py.schema b/dm/templates/runtime_config/runtime_config.py.schema index 8ac593071da..8553c88dc08 100644 --- a/dm/templates/runtime_config/runtime_config.py.schema +++ b/dm/templates/runtime_config/runtime_config.py.schema @@ -15,10 +15,16 @@ info: title: Runtime Configurator author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of a Runtime Configurator. + For more information on this resource, see - https://cloud.google.com/deployment-manager/runtime-configurator/. + https://cloud.google.com/deployment-manager/runtime-configurator/ + + APIs endpoints used by this template: + - gcp-types/runtimeconfig-v1beta1:projects.configs => + https://cloud.google.com/deployment-manager/runtime-configurator/reference/rest/v1beta1/projects.configs imports: - path: variable.py @@ -26,13 +32,28 @@ imports: additionalProperties: false -required: - - config +oneOf: + - required: + - config + - required: + - name properties: config: type: string - description: The config resource name. + description: | + The config resource name. DEPRECATED, use "name" + Resource name would be used if omitted. + name: + type: string + description: | + The config resource name. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. projectId: type: string description: ProjectID of the project to create the config in. @@ -41,6 +62,7 @@ properties: description: The config resource description. variables: type: array + uniqItems: true description: | The list of variables as defined in the variable.py template. Example: @@ -48,6 +70,7 @@ properties: variableTextValue: "my variable value" waiters: type: array + uniqItems: true description: | The list of waiters as defined in the waiter.py template. Example: diff --git a/dm/templates/runtime_config/tests/integration/runtime_config.yaml b/dm/templates/runtime_config/tests/integration/runtime_config.yaml index 06cfa3efdf4..996720c33b1 100644 --- a/dm/templates/runtime_config/tests/integration/runtime_config.yaml +++ b/dm/templates/runtime_config/tests/integration/runtime_config.yaml @@ -8,15 +8,15 @@ resources: - name: ${CONFIG_NAME} type: runtime_config.py properties: - config: ${CONFIG_NAME} + name: ${CONFIG_NAME} description: my config description variables: - - variable: ${VARIABLE_1} + - name: ${VARIABLE_1} text: ${VARIABLE_1_VALUE} - - variable: ${VARIABLE_2} + - name: ${VARIABLE_2} value: ${VARIABLE_2_VALUE} waiters: - - waiter: ${WAITER_NAME} + - name: ${WAITER_NAME} timeout: ${WAITER_TIMEOUT} success: cardinality: diff --git a/dm/templates/runtime_config/variable.py b/dm/templates/runtime_config/variable.py index 74877acf121..21d0c207ecb 100644 --- a/dm/templates/runtime_config/variable.py +++ b/dm/templates/runtime_config/variable.py @@ -17,21 +17,28 @@ def generate_config(context): """ Entry point for the deployment resources. """ - name = context.properties.get('variable', context.env['name']) + properties = context.properties + project_id = properties.get('project', context.env['project']) config_name = context.properties.get('config') - required_properties = ['parent', 'variable'] + + props = { + 'variable': properties.get('name', properties.get('variable')), + 'parent': properties['parent'], + # TODO: uncomment after gcp type is fixed + # 'project': project_id, + } + optional_properties = ['text', 'value'] - # Load the required properties, then the optional ones if specified. - properties = {p: context.properties[p] for p in required_properties} - properties.update({ - p: context.properties[p] - for p in optional_properties if p in context.properties + props.update({ + p: properties[p] + for p in optional_properties if p in properties }) resources = [{ - 'name': name, - 'type': 'runtimeconfig.v1beta1.variable', - 'properties': properties, + 'name': context.env['name'], + # https://cloud.google.com/deployment-manager/runtime-configurator/reference/rest/v1beta1/projects.configs.variables + 'type': 'gcp-types/runtimeconfig-v1beta1:projects.configs.variables', + 'properties': props, 'metadata': { 'dependsOn': [config_name] } @@ -39,7 +46,7 @@ def generate_config(context): outputs = [{ 'name': 'updateTime', - 'value': '$(ref.{}.updateTime)'.format(name) + 'value': '$(ref.{}.updateTime)'.format(context.env['name']) }] return {'resources': resources, 'outputs': outputs} diff --git a/dm/templates/runtime_config/variable.py.schema b/dm/templates/runtime_config/variable.py.schema index c842c62934e..03a4bf01327 100644 --- a/dm/templates/runtime_config/variable.py.schema +++ b/dm/templates/runtime_config/variable.py.schema @@ -15,35 +15,60 @@ info: title: Variable author: Sourced Group Inc. - description: Creates a RuntimeConfig variable resource. + version: 1.0.0 + description: | + Creates a RuntimeConfig variable resource. + For more information on this resource, see - https://cloud.google.com/deployment-manager/runtime-configurator/. + https://cloud.google.com/deployment-manager/runtime-configurator/ + + APIs endpoints used by this template: + - gcp-types/runtimeconfig-v1beta1:projects.configs.variables => + https://cloud.google.com/deployment-manager/runtime-configurator/reference/rest/v1beta1/projects.configs.variables additionalProperties: false required: - parent - - variable -oneOf: - - required: - - text - - required: - - value +allOf: + - oneOf: + - required: + - text + - required: + - value + - oneOf: + - required: + - variable + - required: + - name properties: + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. parent: type: string description: | The path to the configuration that will own the waiter. The configuration must exist beforehand; the path must be in the projects/[PROJECT_ID]/configs/[CONFIG_NAME] format. - variable: + name: type: string description: | The key (name) of the variable. For example, status and users/jane-smith/favorite_color are valid keys. Can contain digits, letters, dashes, and slashes. The max length is 256 characters. + variable: + type: string + description: | + DEPRECATED, please use "name" + config: + type: string + description: | + Config resource name (for dependency) text: type: string description: | diff --git a/dm/templates/runtime_config/waiter.py b/dm/templates/runtime_config/waiter.py index ee611ef8a7c..192327d776a 100644 --- a/dm/templates/runtime_config/waiter.py +++ b/dm/templates/runtime_config/waiter.py @@ -17,21 +17,30 @@ def generate_config(context): """ Entry point for the deployment resources. """ - name = context.properties.get('name', context.env['name']) - config_name = context.properties.get('config') - required_properties = ['waiter', 'parent', 'timeout', 'success'] + properties = context.properties + project_id = properties.get('project', context.env['project']) + config_name = properties.get('config') + + props = { + 'waiter': properties.get('name', properties.get('waiter')), + 'parent': properties['parent'], + 'timeout': properties['timeout'], + 'success': properties['success'], + # TODO: uncomment after gcp type is fixed + # 'project': project_id, + } + optional_properties = ['failure'] - # Load the required properties, then the optional ones if specified. - properties = {p: context.properties[p] for p in required_properties} - properties.update({ - p: context.properties[p] - for p in optional_properties if p in context.properties + props.update({ + p: properties[p] + for p in optional_properties if p in properties }) resources = [{ - 'name': name, - 'type': 'runtimeconfig.v1beta1.waiter', - 'properties': properties, + 'name': context.env['name'], + # https://cloud.google.com/deployment-manager/runtime-configurator/reference/rest/v1beta1/projects.configs.waiters + 'type': 'gcp-types/runtimeconfig-v1beta1:projects.configs.waiters', + 'properties': props, 'metadata': { 'dependsOn': [config_name] } @@ -39,7 +48,7 @@ def generate_config(context): outputs = [{ 'name': 'createTime', - 'value': '$(ref.{}.createTime)'.format(name) + 'value': '$(ref.{}.createTime)'.format(context.env['name']) }] return {'resources': resources, 'outputs': outputs} diff --git a/dm/templates/runtime_config/waiter.py.schema b/dm/templates/runtime_config/waiter.py.schema index 9ad6b59dcff..cd335e78a6c 100644 --- a/dm/templates/runtime_config/waiter.py.schema +++ b/dm/templates/runtime_config/waiter.py.schema @@ -15,32 +15,57 @@ info: title: Waiter author: Sourced Group Inc. - description: Supports creation of a RuntimeConfig Waiter resource. + version: 1.0.0 + description: | + Supports creation of a RuntimeConfig Waiter resource. + For more information on this resource, see - https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter. + https://cloud.google.com/deployment-manager/runtime-configurator/creating-a-waiter + + APIs endpoints used by this template: + - gcp-types/runtimeconfig-v1beta1:projects.configs.waiters => + https://cloud.google.com/deployment-manager/runtime-configurator/reference/rest/v1beta1/projects.configs.waiters additionalProperties: false required: - parent - - waiter - timeout - success +oneOf: + - required: + - waiter + - required: + - name + properties: + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. parent: type: string description: | The path to the configuration that will own the waiter. The configuration must exist beforehand; the path must be in the projects/[PROJECT_ID]/configs/[CONFIG_NAME] format. - waiter: + name: type: string description: | The name of the waiter resource, in the projects/[PROJECT_ID]/configs/[CONFIG_NAME]/waiters/[WAITER_NAME] format. Must match the RFC 1035 segment specification. Maximum length is 64 bytes. + waiter: + type: string + description: | + DEPRECATED, please use "name" + config: + type: string + description: | + Config resource name (for dependency) timeout: type: string description: | @@ -50,6 +75,7 @@ properties: the waiter fails and sets the error code to DEADLINE_EXCEEDED. failure: type: object + additionalProperties: false description: | The failure condition for the waiter. If this condition is met, done is set to True, and the error code is set to ABORTED. The failure @@ -59,6 +85,7 @@ properties: properties: cardinality: type: object + additionalProperties: false description: The cardinality of the EndCondition. properties: path: @@ -73,6 +100,7 @@ properties: this condition. If not specified, defaults to 1. success: type: object + additionalProperties: false description: | The success condition. If this condition is met, done is set to True, and the error value remains unset. The failure condition takes @@ -81,6 +109,7 @@ properties: properties: cardinality: type: object + additionalProperties: false description: The cardinality of the EndCondition. properties: path: diff --git a/dm/templates/shared_vpc_subnet_iam/README.md b/dm/templates/shared_vpc_subnet_iam/README.md index 930458aa9f8..45e08730525 100644 --- a/dm/templates/shared_vpc_subnet_iam/README.md +++ b/dm/templates/shared_vpc_subnet_iam/README.md @@ -62,5 +62,11 @@ See `properties` section in the schema file(s): ``` ## Examples - -- [Shared VPC Subnet IAM](examples/shared_vpc_subnet_iam.yaml) +- [Shared VPC Subnet IAM Bindings syntax](examples/shared_vpc_subnet_iam_bindings.yaml) +- [Shared VPC Subnet IAM Policy syntax](examples/shared_vpc_subnet_iam_policy.yaml) +- [Shared VPC Subnet IAM Legacy](examples/shared_vpc_subnet_iam_legacy.yaml) + +## Tests Cases +- [Shared VPC Subnet IAM Bindings syntax](tests/integration/bindings.bats) +- [Shared VPC Subnet IAM Policy syntax](tests/integration/policy.bats) +- [Shared VPC Subnet IAM Legacy syntax](tests/integration/legacy.bats) diff --git a/dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_bindings.yaml b/dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_bindings.yaml new file mode 100644 index 00000000000..19bb2263f75 --- /dev/null +++ b/dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_bindings.yaml @@ -0,0 +1,29 @@ +# Example usage of the shared VPC subnet IAM template +# +# The `members` property is a list of members to +# grant IAM roles on a shared VPC subnetwork. +# A member can be a user, service account, group, or domain. +# +# Replace `resourceId` with a valid subnet ID. + +imports: + - path: templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py + name: shared_vpc_subnet_iam.py + +resources: + - name: test-shared-vpc-subnet-iam-policy + type: shared_vpc_subnet_iam.py + properties: + bindings: + - resourceId: test-subnet-1 + region: us-east1 + role: roles/compute.networkUser + members: + - user:name@example.com + - serviceAccount:example@myprojectname.gserviceaccount.com + - resourceId: + region: us-east1 + role: roles/compute.networkUser + members: + - group:admins@example.com + - domain:example.com diff --git a/dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam.yaml b/dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_legacy.yaml similarity index 100% rename from dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam.yaml rename to dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_legacy.yaml diff --git a/dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_policy.yaml b/dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_policy.yaml new file mode 100644 index 00000000000..ba71849d298 --- /dev/null +++ b/dm/templates/shared_vpc_subnet_iam/examples/shared_vpc_subnet_iam_policy.yaml @@ -0,0 +1,30 @@ +# Example usage of the shared VPC subnet IAM template +# +# The `members` property is a list of members to +# grant IAM roles on a shared VPC subnetwork. +# A member can be a user, service account, group, or domain. +# +# Replace `resourceId` with a valid subnet ID. + +imports: + - path: templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py + name: shared_vpc_subnet_iam.py + +resources: + - name: test-shared-vpc-subnet-iam-policy + type: shared_vpc_subnet_iam.py + properties: + policy: + bindings: + - resourceId: test-subnet-1 + region: us-east1 + role: roles/compute.networkUser + members: + - user:name@example.com + - serviceAccount:example@myprojectname.gserviceaccount.com + - resourceId: + region: us-east1 + role: roles/compute.networkUser + members: + - group:admins@example.com + - domain:example.com diff --git a/dm/templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py b/dm/templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py index 59693cbe477..38119687b95 100644 --- a/dm/templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py +++ b/dm/templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py @@ -11,43 +11,57 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" This template grants IAM roles to a user on a shared VPC subnetwork. """ +"""This template grants IAM roles to a user on a shared VPC subnetwork.""" -def generate_config(context): - """ Entry point for the deployment resources. """ - +def _append_resource(subnets, project, name_id): + """Append subnets to resources.""" resources = [] out = {} - for subnet in context.properties['subnets']: - subnet_id = subnet['subnetId'] - policy_name = 'iam-subnet-policy-{}'.format(subnet_id) - - policies_to_add = [ - { - 'role': subnet['role'], - 'members': subnet['members'] + for subnet in subnets: + policy_name = 'iam-subnet-policy-{}'.format(subnet[name_id]) + resources.append({ + 'name': policy_name, + # https://cloud.google.com/compute/docs/reference/rest/beta/subnetworks/setIamPolicy + 'type': 'gcp-types/compute-beta:compute.subnetworks.setIamPolicy', + 'properties': { + 'name': subnet[name_id], + 'project': project, + 'region': subnet['region'], + 'bindings': [{ + 'role': subnet['role'], + 'members': subnet['members'] + }] } - ] - - resources.append( - { - 'name': policy_name, - 'type': 'gcp-types/compute-beta:compute.subnetworks.setIamPolicy', # pylint: disable=line-too-long - 'properties': - { - 'name': subnet_id, - 'project': context.env['project'], - 'region': subnet['region'], - 'bindings': policies_to_add - } - } - ) + }) out[policy_name] = { 'etag': '$(ref.' + policy_name + '.etag)' } + return resources, out + +def generate_config(context): + """Entry point for the deployment resources.""" + try: + resources, out = _append_resource( + context.properties['subnets'], # Legacy syntax + context.env['project'], + 'subnetId' + ) + except KeyError: + try: + resources, out = _append_resource( + context.properties['policy']['bindings'], # Policy syntax + context.env['project'], + 'resourceId' + ) + except KeyError: + resources, out = _append_resource( + context.properties['bindings'], # Bindings syntax + context.env['project'], + 'resourceId' + ) outputs = [{'name': 'policies', 'value': out}] return {'resources': resources, 'outputs': outputs} diff --git a/dm/templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py.schema b/dm/templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py.schema index 9ccccaacf6d..cd4141e784b 100644 --- a/dm/templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py.schema +++ b/dm/templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py.schema @@ -15,44 +15,176 @@ info: title: Shared VPC Subnet IAM author: Sourced Group Inc. - description: Grants IAM roles to a user on a shared VPC subnetwork + version: 1.0.0 + description: | + Grants IAM roles to a user on a shared VPC subnetwork + + For more information on this resource: + https://cloud.google.com/compute/docs/reference/rest/beta/subnetworks + + APIs endpoints used by this template: + - gcp-types/compute-beta:compute.subnetworks.setIamPolicy => + https://cloud.google.com/compute/docs/reference/rest/beta/subnetworks/setIamPolicy + +imports: + - path: shared_vpc_subnet_iam.py + +oneOf: +- required: + - bindings +- required: + - policy +- required: + - subnets # Legacy additionalProperties: false -required: - - subnets +project: + type: string + description: The Project ID. -properties: - subnets: - type: array - description: An array of subnetworks and members to grant IAM roles to. - items: - subnetId: - type: string - description: The subnet ID. - region: - type: string - description: The region the subnetId resides in. - role: - type: string - description: The role to grant. - members: +definitions: + policy: + type: object + description: | + REQUIRED: The complete policy to be applied to the 'resource'. + The size of the policy is limited to a few 10s of KB. An empty policy is + in general a valid policy but certain services (like Projects) might + reject them. + additionalProperties: false + properties: + version: + type: integer + description: Deprecated + bindings: type: array - description: A list of member identities. + description: | + Associates a list of members to a role. bindings with no members will + result in an error. + uniqItems: true items: - type: string - description: | - The identity of a member requesting access for a Cloud Platform - resource. Can have the following values: - - user:{emailid} - An email address that represents a specific - Google account. For example, user:name@example.com - - serviceAccount:{emailid} - An email address that represents a - service account. For example, - serviceAccount:my-other-app@appspot.gserviceaccount.com - - group:{emailid} - An email address that represents a Google group. - For example, group:admins@example.com - - domain:{domain} - A Google Apps domain name that represents all - the users of that domain. For example, google.com or example.com. + subnetId: # legacy + type: string + description: The subnet ID. + requestId: + type: string + description: Name or id of the resource for this request. + role: + type: string + description: | + Role that is assigned to members. For example, + roles/viewer, roles/editor, or roles/owner. + members: + type: array + description: A list of member identities. + uniqItems: true + items: + type: string + description: | + Specifies the identities requesting access for a Cloud Platform + resource. `members` can have the following values: + - allUsers: A special identifier that represents anyone who + is on the internet; with or without a Google account. + - allAuthenticatedUsers: A special identifier that represents + anyone who is authenticated with a Google account or a + service account. + - user:{emailid} - An email address that represents a + specific Google account. For example, user:name@example.com + - serviceAccount:{emailid} - An email address that represents + a service account. For example, + serviceAccount:my-other-app@appspot.gserviceaccount.com + - group:{emailid} - An email address that represents a Google + group. For example, group:admins@example.com + - domain:{domain} - A Google Apps domain name that represents + all the users of that domain. For example, + google.com or example.com. + condition: + type: object + description: | + The condition that is associated with this binding. NOTE: An + unsatisfied condition will not allow user access via current + binding. Different bindings, including their conditions, are + examined independently. + additionalProperties: false + properties: + expression: + type: string + description: | + Textual representation of an expression in Common Expression + Language syntax. The application context of the containing + message determines which well-known feature set of CEL is + supported. + title: + type: string + description: | + An optional title for the expression, i.e. a short string + describing its purpose. This can be used e.g. in UIs which + allow to enter the expression. + description: + type: string + description: | + An optional description of the expression. This is a longer + text which describes the expression, e.g. when hovered over + it in a UI. + location: + type: string + description: | + An optional string indicating the location of the expression + for error reporting, e.g. a file name and a position in the + file. + auditConfigs: + type: object + description: | + Specifies cloud audit logging configuration for this policy. + additionalProperties: false + properties: + service: + type: string + description: | + Specifies a service that will be enabled for audit logging. + For example, storage.googleapis.com, cloudsql.googleapis.com. + allServices is a special value that covers all services. + auditLogConfigs: + type: object + description: | + The configuration for logging of each type of permission. + additionalProperties: false + properties: + logType: + type: string + description: The log type that this config enables. + exemptedMembers: + type: array + description: | + Specifies the identities that do not cause logging for this + type of permission. Follows the same format of + Binding.members. + uniqItems: true + items: + type: string + etag: + type: string + description: | + etag is used for optimistic concurrency control as a way to help + prevent simultaneous updates of a policy from overwriting each other. + It is strongly suggested that systems make use of the etag in the + read-modify-write cycle to perform policy updates in order to avoid + race conditions: An etag is returned in the response to getIamPolicy, + and systems are expected to put that etag in the request to + setIamPolicy to ensure that their change will be applied to the same + version of the policy. If no etag is provided in the call to + setIamPolicy, then the existing policy is overwritten blindly. + A base64-encoded string. + +properties: + policy: + $ref: '#/definitions/policy' + bindings: + $ref: '#/definitions/policy/properties/bindings' + subnets: # legacy + $ref: '#/definitions/policy/properties/bindings' + etag: + $ref: '#/definitions/policy/properties/etag' outputs: properties: diff --git a/dm/templates/shared_vpc_subnet_iam/tests/integration/bindings.bats b/dm/templates/shared_vpc_subnet_iam/tests/integration/bindings.bats new file mode 120000 index 00000000000..785c07d8fb8 --- /dev/null +++ b/dm/templates/shared_vpc_subnet_iam/tests/integration/bindings.bats @@ -0,0 +1 @@ +shared_vpc_subnet_iam.bats \ No newline at end of file diff --git a/dm/templates/shared_vpc_subnet_iam/tests/integration/bindings.yaml b/dm/templates/shared_vpc_subnet_iam/tests/integration/bindings.yaml new file mode 100644 index 00000000000..5ca33a8bc4a --- /dev/null +++ b/dm/templates/shared_vpc_subnet_iam/tests/integration/bindings.yaml @@ -0,0 +1,25 @@ +# Test of the shared VPC subnet IAM template. +# +# Variables: +# RAND: A random string used by the testing suite. +# + +imports: + - path: templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py + name: shared_vpc_subnet_iam.py + +resources: + - name: test-shared-vpc-subnet-iam-${RAND} + type: shared_vpc_subnet_iam.py + properties: + bindings: + - resourceId: subnet-${RAND}-1 + region: us-east1 + role: roles/compute.networkUser + members: + - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com + - resourceId: subnet-${RAND}-2 + region: us-east1 + role: roles/compute.networkUser + members: + - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com diff --git a/dm/templates/shared_vpc_subnet_iam/tests/integration/legacy.bats b/dm/templates/shared_vpc_subnet_iam/tests/integration/legacy.bats new file mode 120000 index 00000000000..785c07d8fb8 --- /dev/null +++ b/dm/templates/shared_vpc_subnet_iam/tests/integration/legacy.bats @@ -0,0 +1 @@ +shared_vpc_subnet_iam.bats \ No newline at end of file diff --git a/dm/templates/shared_vpc_subnet_iam/tests/integration/legacy.yaml b/dm/templates/shared_vpc_subnet_iam/tests/integration/legacy.yaml new file mode 100644 index 00000000000..da29a9bc14a --- /dev/null +++ b/dm/templates/shared_vpc_subnet_iam/tests/integration/legacy.yaml @@ -0,0 +1,25 @@ +# Test of the shared VPC subnet IAM template. +# +# Variables: +# RAND: A random string used by the testing suite. +# + +imports: + - path: templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py + name: shared_vpc_subnet_iam.py + +resources: + - name: test-shared-vpc-subnet-iam-${RAND} + type: shared_vpc_subnet_iam.py + properties: + subnets: + - subnetId: subnet-${RAND}-1 + region: us-east1 + role: roles/compute.networkUser + members: + - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com + - subnetId: subnet-${RAND}-2 + region: us-east1 + role: roles/compute.networkUser + members: + - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com diff --git a/dm/templates/shared_vpc_subnet_iam/tests/integration/policy.bats b/dm/templates/shared_vpc_subnet_iam/tests/integration/policy.bats new file mode 120000 index 00000000000..785c07d8fb8 --- /dev/null +++ b/dm/templates/shared_vpc_subnet_iam/tests/integration/policy.bats @@ -0,0 +1 @@ +shared_vpc_subnet_iam.bats \ No newline at end of file diff --git a/dm/templates/shared_vpc_subnet_iam/tests/integration/policy.yaml b/dm/templates/shared_vpc_subnet_iam/tests/integration/policy.yaml new file mode 100644 index 00000000000..b38fc74b15e --- /dev/null +++ b/dm/templates/shared_vpc_subnet_iam/tests/integration/policy.yaml @@ -0,0 +1,26 @@ +# Test of the shared VPC subnet IAM template. +# +# Variables: +# RAND: A random string used by the testing suite. +# + +imports: + - path: templates/shared_vpc_subnet_iam/shared_vpc_subnet_iam.py + name: shared_vpc_subnet_iam.py + +resources: + - name: test-shared-vpc-subnet-iam-${RAND} + type: shared_vpc_subnet_iam.py + properties: + policy: + bindings: + - resourceId: subnet-${RAND}-1 + region: us-east1 + role: roles/compute.networkUser + members: + - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com + - resourceId: subnet-${RAND}-2 + region: us-east1 + role: roles/compute.networkUser + members: + - serviceAccount:${TEST_SERVICE_ACCOUNT}@${CLOUD_FOUNDATION_PROJECT_ID}.iam.gserviceaccount.com diff --git a/dm/templates/shared_vpc_subnet_iam/tests/integration/shared_vpc_subnet_iam.bats b/dm/templates/shared_vpc_subnet_iam/tests/integration/shared_vpc_subnet_iam.bats old mode 100644 new mode 100755 diff --git a/dm/templates/ssl_certificate/examples/ssl_certificate.yaml b/dm/templates/ssl_certificate/examples/ssl_certificate.yaml index 8f6e2b4da3d..88ef96018c3 100644 --- a/dm/templates/ssl_certificate/examples/ssl_certificate.yaml +++ b/dm/templates/ssl_certificate/examples/ssl_certificate.yaml @@ -5,7 +5,7 @@ # : contents of the private key file imports: - - path: templates/ssl_cerficate/ssl_cerficate.py + - path: templates/ssl_certificate/ssl_certificate.py name: ssl_certificate.py resources: diff --git a/dm/templates/ssl_certificate/ssl_certificate.py b/dm/templates/ssl_certificate/ssl_certificate.py index 83114a46a8b..cb231f0d63b 100644 --- a/dm/templates/ssl_certificate/ssl_certificate.py +++ b/dm/templates/ssl_certificate/ssl_certificate.py @@ -26,12 +26,17 @@ def generate_config(context): properties = context.properties name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) - ssl_props = {} - resource = { + ssl_props = { 'name': name, - 'type': 'compute.v1.sslCertificate', - 'properties': ssl_props + 'project': project_id, + } + resource = { + 'name': context.env['name'], + # https://cloud.google.com/compute/docs/reference/rest/v1/sslCertificates + 'type': 'gcp-types/compute-v1:sslCertificates', + 'properties': ssl_props, } for prop in ['privateKey', 'certificate', 'description']: @@ -47,7 +52,7 @@ def generate_config(context): }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) } ] } diff --git a/dm/templates/ssl_certificate/ssl_certificate.py.schema b/dm/templates/ssl_certificate/ssl_certificate.py.schema index a8b51f46188..b87bb8f199c 100644 --- a/dm/templates/ssl_certificate/ssl_certificate.py.schema +++ b/dm/templates/ssl_certificate/ssl_certificate.py.schema @@ -15,7 +15,16 @@ info: title: SSL Certificate author: Sourced Group Inc. - description: Supports creation of the SSL certificate resource. + version: 1.0.0 + description: | + Supports creation of the SSL certificate resource. + + For more information on this resource: + https://cloud.google.com/load-balancing/docs/ssl-certificates + + APIs endpoints used by this template: + - gcp-types/compute-v1:sslCertificates => + https://cloud.google.com/compute/docs/reference/rest/v1/sslCertificates additionalProperties: false @@ -26,13 +35,25 @@ required: properties: name: type: string - description: The resource name. + description: | + Name of the resource. Provided by the client when the resource is created. The name must be 1-63 characters long, + and comply with RFC1035. Specifically, the name must be 1-63 characters long and match the regular expression + [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a lowercase letter, and all following + characters must be a dash, lowercase letter, or digit, except the last character, which cannot be a dash. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the SSL instance. The + Google apps domain is prefixed if applicable. description: type: string - description: The resource description (optional). + description: | + An optional description of this resource. Provide this property when you create the resource. privateKey: type: string - description: The write-only private key in the PEM format. + description: | + The write-only private key in the PEM format. certificate: type: string description: | diff --git a/dm/templates/ssl_certificate/tests/integration/ssl_certificate.yaml b/dm/templates/ssl_certificate/tests/integration/ssl_certificate.yaml index 598eb86888d..ae6e4cc83f9 100644 --- a/dm/templates/ssl_certificate/tests/integration/ssl_certificate.yaml +++ b/dm/templates/ssl_certificate/tests/integration/ssl_certificate.yaml @@ -5,9 +5,10 @@ imports: name: ssl_certificate.py resources: - - name: ${CERT_NAME} + - name: test-cert type: ssl_certificate.py properties: + name: ${CERT_NAME} description: ${CERT_DESCRIPTION} certificate: | -----BEGIN CERTIFICATE----- diff --git a/dm/templates/stackdriver_metric_descriptor/examples/stackdriver_metric_descriptor.yaml b/dm/templates/stackdriver_metric_descriptor/examples/stackdriver_metric_descriptor.yaml index 888803568e9..6ca8810c39f 100644 --- a/dm/templates/stackdriver_metric_descriptor/examples/stackdriver_metric_descriptor.yaml +++ b/dm/templates/stackdriver_metric_descriptor/examples/stackdriver_metric_descriptor.yaml @@ -17,7 +17,7 @@ resources: metricKind: CUMULATIVE valueType: INT64 unit: "1" + launchStage: ALPHA metadata: - launchStage: ALPHA samplePeriod: "10s" ingestDelay: "1s" diff --git a/dm/templates/stackdriver_metric_descriptor/stackdriver_metric_descriptor.py b/dm/templates/stackdriver_metric_descriptor/stackdriver_metric_descriptor.py index 916cc95a58f..ef7d22d1a0b 100644 --- a/dm/templates/stackdriver_metric_descriptor/stackdriver_metric_descriptor.py +++ b/dm/templates/stackdriver_metric_descriptor/stackdriver_metric_descriptor.py @@ -20,11 +20,15 @@ def generate_config(context): resources = [] outputs = [] properties = context.properties - name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) metric_descriptor = { - 'name': name, + 'name': context.env['name'], + # https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors 'type': 'gcp-types/monitoring-v3:projects.metricDescriptors', - 'properties': {} + 'properties': { + 'name': properties.get('name', context.env['name']), + # 'project': project_id, + } } required_properties = [ @@ -39,7 +43,7 @@ def generate_config(context): metric_descriptor['properties'][prop] = properties[prop] # Optional properties: - optional_properties = ['displayName', 'labels', 'description', 'metadata'] + optional_properties = ['displayName', 'labels', 'description', 'metadata', 'launchStage'] for prop in optional_properties: if prop in properties: @@ -64,7 +68,7 @@ def generate_config(context): output = {} if outprop in properties: output['name'] = outprop - output['value'] = '$(ref.{}.{})'.format(name, outprop) + output['value'] = '$(ref.{}.{})'.format(context.env['name'], outprop) outputs.append(output) return {'resources': resources, 'outputs': outputs} diff --git a/dm/templates/stackdriver_metric_descriptor/stackdriver_metric_descriptor.py.schema b/dm/templates/stackdriver_metric_descriptor/stackdriver_metric_descriptor.py.schema index a4ba1cd489f..3b992060b6d 100644 --- a/dm/templates/stackdriver_metric_descriptor/stackdriver_metric_descriptor.py.schema +++ b/dm/templates/stackdriver_metric_descriptor/stackdriver_metric_descriptor.py.schema @@ -15,26 +15,86 @@ info: title: Stackdriver Metric Descriptor author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of a Stackdriver Metric Descriptor. + For more information on this resource, see - https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors. + https://cloud.google.com/monitoring/ + + APIs endpoints used by this template: + - gcp-types/monitoring-v3:projects.metricDescriptors => + https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors imports: - path: stackdriver_metric_descriptor.py additionalProperties: false -required: - - type - - metricKind - - valueType - - unit +allOf: + - required: + - type + - metricKind + - valueType + - oneOf: + - required: + - launchStage + - allOf: + - required: + - metadata + - properties: + metadata: + required: + - launchStage + - allOf: + - not: + required: + - launchStage + - oneOf: + - not: + required: + - metadata + - properties: + metadata: + not: + required: + - launchStage + + - oneOf: + - valueType: + enum: + - INT64 + - DOUBLE + - DISTRIBUTION + - not: + required: + - unit + +definitions: + launchStage: + type: string + description: | + The launch stage as defined by Google Cloud Platform Launch Stages: + http://cloud.google.com/terms/launch-stages. + enum: + - LAUNCH_STAGE_UNSPECIFIED + - EARLY_ACCESS + - ALPHA + - BETA + - GA + - DEPRECATED properties: name: type: string - description: The resource name of the Metric Descriptor. + description: | + The name of the Metric Descriptor. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. type: type: string description: | @@ -120,23 +180,15 @@ properties: case without the ending period; for example, "Request count". Optional but recommended to be set for any metrics associated with user-visible concepts, such as Quota. + launchStage: + $ref: '#/definitions/launchStage' metadata: type: object description: | Additional annotations that can be used to guide the usage of a metric. properties: launchStage: - type: string - description: | - The launch stage as defined by Google Cloud Platform Launch Stages: - http://cloud.google.com/terms/launch-stages. - enum: - - LAUNCH_STAGE_UNSPECIFIED - - EARLY_ACCESS - - ALPHA - - BETA - - GA - - DEPRECATED + $ref: '#/definitions/launchStage' samplePeriod: type: string description: | diff --git a/dm/templates/stackdriver_metric_descriptor/tests/integration/stackdriver_metric_descriptor.yaml b/dm/templates/stackdriver_metric_descriptor/tests/integration/stackdriver_metric_descriptor.yaml index c1734a65767..2eb6eaf325f 100644 --- a/dm/templates/stackdriver_metric_descriptor/tests/integration/stackdriver_metric_descriptor.yaml +++ b/dm/templates/stackdriver_metric_descriptor/tests/integration/stackdriver_metric_descriptor.yaml @@ -16,7 +16,7 @@ imports: name: metric_descriptor.py resources: - - name: ${METRIC_NAME} + - name: metric type: metric_descriptor.py properties: name: ${METRIC_NAME} @@ -26,7 +26,7 @@ resources: metricKind: ${METRIC_KIND} valueType: ${VALUE_TYPE} unit: "${UNIT}" + launchStage: ${LAUNCH_STAGE} metadata: - launchStage: ${LAUNCH_STAGE} samplePeriod: "${SAMPLE_PERIOD}" - ingestDelay: "${INGEST_DELAY}" \ No newline at end of file + ingestDelay: "${INGEST_DELAY}" diff --git a/dm/templates/target_proxy/target_proxy.py b/dm/templates/target_proxy/target_proxy.py index 8f6b3227d4a..3f4c4c13a3d 100644 --- a/dm/templates/target_proxy/target_proxy.py +++ b/dm/templates/target_proxy/target_proxy.py @@ -26,7 +26,7 @@ def set_optional_property(destination, source, prop_name): destination[prop_name] = source[prop_name] -def get_certificate(properties, res_name): +def get_certificate(properties, project_id, res_name): """ Gets a link to an existing or newly created SSL Certificate resource. @@ -35,13 +35,15 @@ def get_certificate(properties, res_name): if 'url' in properties: return properties['url'], [], [] - name = properties.get('name', '{}-ssl-cert'.format(res_name)) + name = '{}-ssl-cert'.format(res_name) resource = { 'name': name, 'type': 'ssl_certificate.py', 'properties': copy.copy(properties) } + resource['properties']['name'] = properties.get('name', name) + resource['properties']['project'] = project_id self_link = '$(ref.{}.selfLink)'.format(name) outputs = [ @@ -58,18 +60,23 @@ def get_certificate(properties, res_name): return self_link, [resource], outputs -def get_insecure_proxy(is_http, name, properties, optional_properties): +def get_insecure_proxy(is_http, res_name, project_id, properties, optional_properties): """ Creates a TCP or HTTP Proxy resource. """ if is_http: - type_name = 'compute.v1.targetHttpProxy' + # https://cloud.google.com/compute/docs/reference/rest/v1/targetHttpProxies + type_name = 'gcp-types/compute-v1:targetHttpProxies' target_prop = 'urlMap' else: - type_name = 'compute.alpha.targetTcpProxy' + # https://cloud.google.com/compute/docs/reference/rest/v1/targetTcpProxies + type_name = 'gcp-types/compute-v1:targetTcpProxies' target_prop = 'service' - resource_props = {} - resource = {'type': type_name, 'name': name, 'properties': resource_props} + resource_props = { + 'name': properties.get('name', res_name), + 'project': project_id, + } + resource = {'type': type_name, 'name': res_name, 'properties': resource_props} resource_props[target_prop] = properties['target'] @@ -79,18 +86,20 @@ def get_insecure_proxy(is_http, name, properties, optional_properties): return [resource], [] -def get_secure_proxy(is_http, name, properties, optional_properties): +def get_secure_proxy(is_http, res_name, project_id, properties, optional_properties): """ Creates an SSL or HTTPS Proxy resource. """ if is_http: create_base_proxy = get_http_proxy - target_type = 'compute.v1.targetHttpsProxy' + # https://cloud.google.com/compute/docs/reference/rest/v1/targetHttpsProxies + target_type = 'gcp-types/compute-v1:targetHttpsProxies' else: create_base_proxy = get_tcp_proxy - target_type = 'compute.v1.targetSslProxy' + # https://cloud.google.com/compute/docs/reference/rest/v1/targetSslProxies + target_type = 'gcp-types/compute-v1:targetSslProxies' # Base proxy settings: - resources, outputs = create_base_proxy(properties, name) + resources, outputs = create_base_proxy(properties, res_name, project_id) resource = resources[0] resource['type'] = target_type resource_prop = resource['properties'] @@ -98,37 +107,40 @@ def get_secure_proxy(is_http, name, properties, optional_properties): set_optional_property(resource_prop, properties, prop) # SSL settings: - ssl = properties['ssl'] - url, ssl_resources, ssl_outputs = get_certificate(ssl['certificate'], name) - resource_prop['sslCertificates'] = [url] - set_optional_property(resource_prop, ssl, 'sslPolicy') + ssl_resources = [] + ssl_outputs = [] + if 'sslCertificates' not in resource_prop: + ssl = properties['ssl'] + url, ssl_resources, ssl_outputs = get_certificate(ssl['certificate'], project_id, res_name) + resource_prop['sslCertificates'] = [url] + set_optional_property(resource_prop, ssl, 'sslPolicy') return resources + ssl_resources, outputs + ssl_outputs -def get_http_proxy(properties, name): +def get_http_proxy(properties, res_name, project_id): """ Creates the HTTP Proxy resource. """ - return get_insecure_proxy(HTTP_BASE, name, properties, ['description']) + return get_insecure_proxy(HTTP_BASE, res_name, project_id, properties, ['description']) -def get_tcp_proxy(properties, name): +def get_tcp_proxy(properties, res_name, project_id): """ Creates the TCP Proxy resource. """ optional_properties = ['description', 'proxyHeader'] - return get_insecure_proxy(TCP_BASE, name, properties, optional_properties) + return get_insecure_proxy(TCP_BASE, res_name, project_id, properties, optional_properties) -def get_https_proxy(properties, name): +def get_https_proxy(properties, res_name, project_id): """ Creates the HTTPS Proxy resource. """ - return get_secure_proxy(HTTP_BASE, name, properties, ['quicOverride']) + return get_secure_proxy(HTTP_BASE, res_name, project_id, properties, ['quicOverride']) -def get_ssl_proxy(properties, name): +def get_ssl_proxy(properties, res_name, project_id): """ Creates the SSL Proxy resource. """ - return get_secure_proxy(TCP_BASE, name, properties, []) + return get_secure_proxy(TCP_BASE, res_name, project_id, properties, []) def generate_config(context): @@ -136,16 +148,17 @@ def generate_config(context): properties = context.properties name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) protocol = properties['protocol'] if protocol == 'SSL': - resources, outputs = get_ssl_proxy(properties, name) + resources, outputs = get_ssl_proxy(properties, context.env['name'], project_id) elif protocol == 'TCP': - resources, outputs = get_tcp_proxy(properties, name) + resources, outputs = get_tcp_proxy(properties, context.env['name'], project_id) elif protocol == 'HTTPS': - resources, outputs = get_https_proxy(properties, name) + resources, outputs = get_https_proxy(properties, context.env['name'], project_id) else: - resources, outputs = get_http_proxy(properties, name) + resources, outputs = get_http_proxy(properties, context.env['name'], project_id) return { 'resources': @@ -158,11 +171,11 @@ def generate_config(context): }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) }, { 'name': 'kind', - 'value': '$(ref.{}.kind)'.format(name) + 'value': '$(ref.{}.kind)'.format(context.env['name']) }, ] } diff --git a/dm/templates/target_proxy/target_proxy.py.schema b/dm/templates/target_proxy/target_proxy.py.schema index fa25cb9e19d..9cb2d91945d 100644 --- a/dm/templates/target_proxy/target_proxy.py.schema +++ b/dm/templates/target_proxy/target_proxy.py.schema @@ -15,6 +15,7 @@ info: title: Target Proxy author: Sourced Group Inc. + version: 1.0.0 description: | Depending on the configuration, supports creation of one of these proxy resources: @@ -23,6 +24,19 @@ info: - targetTcpPProxy - targetSslProxy + For more information on this resource: + https://cloud.google.com/load-balancing/docs/target-proxies + + APIs endpoints used by this template: + - gcp-types/compute-v1:targetSslProxies => + https://cloud.google.com/compute/docs/reference/rest/v1/targetSslProxies + - gcp-types/compute-v1:targetHttpProxies => + https://cloud.google.com/compute/docs/reference/rest/v1/targetHttpProxies + - gcp-types/compute-v1:targetHttpsProxies => + https://cloud.google.com/compute/docs/reference/rest/v1/targetHttpsProxies + - gcp-types/compute-v1:targetTcpProxies => + https://cloud.google.com/compute/docs/reference/rest/v1/targetTcpProxies + imports: - path: ../ssl_certificate/ssl_certificate.py name: ssl_certificate.py @@ -36,7 +50,17 @@ required: properties: name: type: string - description: The resource name. + description: | + Must comply with RFC1035. Specifically, the name must be 1-63 characters long and match + the regular expression [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a lowercase letter, + and all following characters must be a dash, lowercase letter, or digit, except the last character, + which cannot be a dash. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. description: type: string description: The resource description (optional). @@ -59,9 +83,26 @@ properties: description: | Encryption settings for connections processed by the resource. Used for HTTPS and SSL proxies only. - required: - - certificate + oneOf: + - required: + - sslCertificates + - required: + - certificate properties: + sslCertificates: + type: array + uniqItems: true + description: | + URLs to SslCertificate resources that are used to authenticate connections to Backends. + At least one SSL certificate must be specified. Currently, you may specify up to 15 SSL certificates. + + Authorization requires the following Google IAM permission on the specified resource sslCertificates: + + compute.sslCertificates.get + minItems: 0 + maxItems: 15 + items: + type: string certificate: type: object description: SSL certificate settings. diff --git a/dm/templates/target_proxy/tests/integration/target_proxy.yaml b/dm/templates/target_proxy/tests/integration/target_proxy.yaml index 51ee5081c81..2a1555deea3 100644 --- a/dm/templates/target_proxy/tests/integration/target_proxy.yaml +++ b/dm/templates/target_proxy/tests/integration/target_proxy.yaml @@ -5,9 +5,10 @@ imports: name: target_proxy.py resources: - - name: ${HTTPS_RES_NAME} + - name: test-proxy type: target_proxy.py properties: + name: ${HTTPS_RES_NAME} protocol: HTTPS target: $(ref.${URL_MAP_RES_NAME}.selfLink) quicOverride: ${HTTPS_QUIC_OVERRIDE} diff --git a/dm/templates/url_map/url_map.py b/dm/templates/url_map/url_map.py index ae86bce0f37..6ca91957e97 100644 --- a/dm/templates/url_map/url_map.py +++ b/dm/templates/url_map/url_map.py @@ -26,8 +26,17 @@ def generate_config(context): properties = context.properties name = properties.get('name', context.env['name']) + project_id = properties.get('project', context.env['project']) - resource = {'name': name, 'type': 'compute.v1.urlMap', 'properties': {}} + resource = { + 'name': context.env['name'], + # https://cloud.google.com/compute/docs/reference/rest/v1/urlMaps + 'type': 'gcp-types/compute-v1:urlMaps', + 'properties': { + 'name': name, + 'project': project_id, + }, + } optional_properties = [ 'defaultService', @@ -50,7 +59,7 @@ def generate_config(context): }, { 'name': 'selfLink', - 'value': '$(ref.{}.selfLink)'.format(name) + 'value': '$(ref.{}.selfLink)'.format(context.env['name']) } ] } diff --git a/dm/templates/url_map/url_map.py.schema b/dm/templates/url_map/url_map.py.schema index 91c1551ff3e..8609c7b82b9 100644 --- a/dm/templates/url_map/url_map.py.schema +++ b/dm/templates/url_map/url_map.py.schema @@ -15,35 +15,69 @@ info: title: URL Map author: Sourced Group Inc. - description: Supports creation of the URL Map resource. + version: 1.0.0 + description: | + Supports creation of the URL Map resource. + + For more information on this resource: + https://cloud.google.com/load-balancing/docs/https/url-map-concepts + + APIs endpoints used by this template: + - gcp-types/compute-v1:urlMaps => + https://cloud.google.com/compute/docs/reference/rest/v1/urlMaps additionalProperties: false properties: name: type: string - description: The resource name. + description: | + Must comply with RFC1035. Specifically, the name must be 1-63 characters long and match + the regular expression [a-z]([-a-z0-9]*[a-z0-9])? which means the first character must be a lowercase letter, + and all following characters must be a dash, lowercase letter, or digit, except the last character, + which cannot be a dash. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing the Cloud Router instance. The + Google apps domain is prefixed if applicable. description: type: string description: The resource description (optional). defaultService: type: string description: | - The URL of the BackendService resource if none of the hostRules match. + The full or partial URL of the defaultService resource to which traffic is directed if none of the + hostRules match. If defaultRouteAction is additionally specified, advanced routing actions like URL Rewrites, + etc. take effect prior to sending the request to the backend. However, if defaultService is specified, + defaultRouteAction cannot contain any weightedBackendServices. Conversely, if routeAction specifies any + weightedBackendServices, service must not be specified. + + Only one of defaultService, defaultUrlRedirect or defaultRouteAction.weightedBackendService must be set. + + Authorization requires one or more of the following Google IAM permissions on the specified resource defaultService: + - compute.backendBuckets.use + - compute.backendServices.use hostRules: type: array - description: The list of HostRules to use against the URL. + uniqItems: true + description: | + The list of HostRules to use against the URL. items: type: object + additionalProperties: false properties: description: type: string - description: The resource description (optional). + description: | + The resource description (optional). hosts: type: array description: | - The list of host patterns to match. They must be valid hostnames; - asterisk (*) matches any string of ([a-z0-9-.]*). + The list of host patterns to match. They must be valid hostnames, except * will match any string of + ([a-z0-9-.]*). In that case, * must be the first character and must be followed + in the pattern by either - or .. items: type: string pathMatcher: @@ -53,9 +87,12 @@ properties: the URL if the hostRule matches the URL's host portion. pathMatchers: type: array - description: The list of the named PathMatchers to use against the URL. + uniqItems: true + description: | + The list of the named PathMatchers to use against the URL. items: type: object + additionalProperties: false properties: name: type: string @@ -63,30 +100,57 @@ properties: The name to which the PathMatcher is referred by the HostRule. description: type: string - description: The resource description (optional). + description: | + The resource description (optional). defaultService: type: string description: | - The full or partial URL to the BackendService resource. Used if - none of the pathRules defined by the PathMatcher are matched by - the URL's path portion. For example, the following are valid URLs - for the BackendService resource: - - https://www.googleapis.com/compute/v1/projects/PROJECT/global/backendServices/BACKENDSERVICE - - compute/v1/projects/project/global/backendServices/BACKENDSERVICE - - global/backendServices/BACKENDSERVICE + The full or partial URL to the BackendService resource. This will be used if none of the pathRules or + routeRules defined by this PathMatcher are matched. For example, the following are + all valid URLs to a BackendService resource: + - https://www.googleapis.com/compute/v1/projects/project/global/backendServices/backendService + - compute/v1/projects/project/global/backendServices/backendService + - global/backendServices/backendService + + If defaultRouteAction is additionally specified, advanced routing actions like URL Rewrites, etc. take + effect prior to sending the request to the backend. However, if defaultService is specified, + defaultRouteAction cannot contain any weightedBackendServices. + Conversely, if defaultRouteAction specifies any weightedBackendServices, defaultService must not be specified. + Only one of defaultService, defaultUrlRedirect or defaultRouteAction.weightedBackendService must be set. + + Authorization requires one or more of the following Google IAM permissions on the specified resource defaultService: + - compute.backendBuckets.use + - compute.backendServices.use + + Authorization requires one or more of the following Google IAM permissions on the specified resource defaultService: + - compute.backendBuckets.use + - compute.backendServices.use pathRules: type: array - description: The list of path rules. + uniqItems: true + description: | + The list of path rules. items: type: object + additionalProperties: false properties: service: type: string description: | - The URL of the BackendService resource if the rule is - matched. + The full or partial URL of the backend service resource to which traffic is directed if this + rule is matched. If routeAction is additionally specified, advanced routing actions like + URL Rewrites, etc. take effect prior to sending the request to the backend. However, if service + is specified, routeAction cannot contain any weightedBackendService s. Conversely, if routeAction + specifies any weightedBackendServices, service must not be specified. + + Only one of urlRedirect, service or routeAction.weightedBackendService must be set. + + Authorization requires one or more of the following Google IAM permissions on the specified resource service: + - compute.backendBuckets.use + - compute.backendServices.use paths: type: array + uniqItems: true description: | The list of the path patterns to match. Each pattern must start with /. Asterisks (*) are allowed only at the end, @@ -97,22 +161,27 @@ properties: type: string tests: type: array + uniqItems: true description: | The list of the expected URL mapping tests. Request to update this UrlMap succeed only if all of the test cases pass. You can specify a maximum of 100 tests per UrlMap. items: type: object + additionalProperties: false properties: description: type: string - description: The test case description. + description: | + The test case description. host: type: string - description: The host portion of the URL. + description: | + The host portion of the URL. path: type: string - description: The path portion of the URL. + description: | + The path portion of the URL. service: type: string description: | diff --git a/dm/templates/vpn/tests/integration/vpn.bats b/dm/templates/vpn/tests/integration/vpn.bats index 50ec1d1b7a5..db68770fc8f 100644 --- a/dm/templates/vpn/tests/integration/vpn.bats +++ b/dm/templates/vpn/tests/integration/vpn.bats @@ -94,57 +94,58 @@ function teardown() { @test "Verifying the the static address was created in deployment ${DEPLOYMENT_NAME}" { - run gcloud compute addresses list --filter="name:test-vpn-${RAND}-ip" \ + run gcloud compute addresses list --filter="name:test-vpn-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [ "$status" -eq 0 ] - [[ "$output" =~ "test-vpn-${RAND}-ip us-east1" ]] + [[ "$output" =~ "test-vpn-${RAND}" ]] + [[ "$output" =~ "us-east1" ]] } @test "Verifying that the target VPN gateway was created in deployment ${DEPLOYMENT_NAME}" { run gcloud compute target-vpn-gateways list \ - --filter="name:test-vpn-${RAND}-tvpng" \ + --filter="name:test-vpn-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [ "$status" -eq 0 ] - [[ "$output" =~ "test-vpn-${RAND}-tvpng network-${RAND} us-east1" ]] + [[ "$output" =~ "test-vpn-${RAND} network-${RAND} us-east1" ]] } @test "Verifying that the VPN tunnel was created in deployment ${DEPLOYMENT_NAME}" { - run gcloud compute vpn-tunnels list --filter="name:test-vpn-${RAND}-vpn" \ + run gcloud compute vpn-tunnels list --filter="name:test-vpn-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [ "$status" -eq 0 ] - [[ "$output" =~ "test-vpn-${RAND}-vpn us-east1 test-vpn-${RAND}-tvpng 1.2.3.4" ]] + [[ "$output" =~ "test-vpn-${RAND} us-east1 test-vpn-${RAND} 1.2.3.4" ]] } @test "Verifying that the forwarding rules were created in deployment ${DEPLOYMENT_NAME}" { run gcloud compute forwarding-rules list --project "${CLOUD_FOUNDATION_PROJECT_ID}" [ "$status" -eq 0 ] - [[ "$output" =~ "test-vpn-${RAND}-esp-rule us-east1" ]] - [[ "$output" =~ "test-vpn-${RAND}-udp-4500-rule us-east1" ]] - [[ "$output" =~ "test-vpn-${RAND}-udp-500-rule us-east1" ]] + [[ "$output" =~ "test-vpn-${RAND}-esp us-east1" ]] + [[ "$output" =~ "test-vpn-${RAND}-udp-4500 us-east1" ]] + [[ "$output" =~ "test-vpn-${RAND}-udp-500 us-east1" ]] } @test "Deleting deployment" { gcloud deployment-manager deployments delete ${DEPLOYMENT_NAME} \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" -q - run gcloud compute addresses list --filter="name:test-vpn-${RAND}-ip" \ + run gcloud compute addresses list --filter="name:test-vpn-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$output" =~ "test-vpn-${RAND}-ip" ]] + [[ ! "$output" =~ "test-vpn-${RAND}" ]] run gcloud compute target-vpn-gateways list \ - --filter="name:test-vpn-${RAND}-tvpng" \ + --filter="name:test-vpn-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$output" =~ "test-vpn-${RAND}-tvpng" ]] + [[ ! "$output" =~ "test-vpn-${RAND}" ]] - run gcloud compute vpn-tunnels list --filter="name:test-vpn-${RAND}-vpn" \ + run gcloud compute vpn-tunnels list --filter="name:test-vpn-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$output" =~ "test-vpn-${RAND}-vpn" ]] + [[ ! "$output" =~ "test-vpn-${RAND}" ]] run gcloud compute forwarding-rules list --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$output" =~ "test-vpn-${RAND}-esp-rule" ]] - [[ ! "$output" =~ "test-vpn-${RAND}-udp-4500-rule" ]] - [[ ! "$output" =~ "test-vpn-${RAND}-udp-500-rule" ]] + [[ ! "$output" =~ "test-vpn-${RAND}-esp" ]] + [[ ! "$output" =~ "test-vpn-${RAND}-udp-4500" ]] + [[ ! "$output" =~ "test-vpn-${RAND}-udp-500" ]] } diff --git a/dm/templates/vpn/tests/integration/vpn.yaml b/dm/templates/vpn/tests/integration/vpn.yaml index 44b03457d3a..39f41d221c2 100644 --- a/dm/templates/vpn/tests/integration/vpn.yaml +++ b/dm/templates/vpn/tests/integration/vpn.yaml @@ -9,9 +9,10 @@ imports: name: vpn.py resources: - - name: test-vpn-${RAND} + - name: vpn-${RAND} type: vpn.py properties: + name: test-vpn-${RAND} region: us-east1 network: network-${RAND} peerAddress: 1.2.3.4 diff --git a/dm/templates/vpn/vpn.py b/dm/templates/vpn/vpn.py index 0371fab4933..38d51120f15 100644 --- a/dm/templates/vpn/vpn.py +++ b/dm/templates/vpn/vpn.py @@ -17,81 +17,180 @@ def generate_config(context): """ Entry point for the deployment resources. """ - network = generate_network_url( - context.env['project'], - context.properties['network'] - ) + + properties = context.properties + project_id = properties.get('project', context.env['project']) + + network = context.properties.get('networkURL', generate_network_uri( + project_id, + context.properties.get('network','') + )) target_vpn_gateway = context.env['name'] + '-tvpng' - static_ip = context.env['name'] + '-ip' esp_rule = context.env['name'] + '-esp-rule' udp_500_rule = context.env['name'] + '-udp-500-rule' udp_4500_rule = context.env['name'] + '-udp-4500-rule' vpn_tunnel = context.env['name'] + '-vpn' router_vpn_binding = context.env['name'] + '-router-vpn-binding' + resources = [] + if 'ipAddress' in context.properties: + ip_address = context.properties['ipAddress'] + static_ip = '' + else: + static_ip = context.env['name'] + '-ip' + resources.append({ + # The reserved address resource. + 'name': static_ip, + # https://cloud.google.com/compute/docs/reference/rest/v1/addresses + 'type': 'gcp-types/compute-v1:addresses', + 'properties': { + 'name': properties.get('name', static_ip), + 'project': project_id, + 'region': context.properties['region'] + } + }) + ip_address = '$(ref.' + static_ip + '.address)' - resources = [ + resources.extend([ { # The target VPN gateway resource. 'name': target_vpn_gateway, - 'type': 'compute.v1.targetVpnGateway', + # https://cloud.google.com/compute/docs/reference/rest/v1/targetVpnGateways + 'type': 'gcp-types/compute-v1:targetVpnGateways', 'properties': { + 'name': properties.get('name', target_vpn_gateway), + 'project': project_id, 'network': network, - 'region': context.properties['region'] + 'region': context.properties['region'], } }, - { - # The reserved address resource. - 'name': static_ip, - 'type': 'compute.v1.address', - 'properties': { - 'region': context.properties['region'] - } - }, { # The forwarding rule resource for the ESP traffic. 'name': esp_rule, - 'type': 'compute.v1.forwardingRule', + # https://cloud.google.com/compute/docs/reference/rest/v1/forwardingRules + 'type': 'gcp-types/compute-v1:forwardingRules', 'properties': { - 'IPAddress': '$(ref.' + static_ip + '.address)', + 'name': '{}-esp'.format(properties.get('name')) if 'name' in properties else esp_rule, + 'project': project_id, + 'IPAddress': ip_address, 'IPProtocol': 'ESP', 'region': context.properties['region'], - 'target': '$(ref.' + target_vpn_gateway + '.selfLink)' + 'target': '$(ref.' + target_vpn_gateway + '.selfLink)', } }, { # The forwarding rule resource for the UDP traffic on port 4500. 'name': udp_4500_rule, - 'type': 'compute.v1.forwardingRule', + # https://cloud.google.com/compute/docs/reference/rest/v1/forwardingRules + 'type': 'gcp-types/compute-v1:forwardingRules', 'properties': { - 'IPAddress': '$(ref.' + static_ip + '.address)', + 'name': '{}-udp-4500'.format(properties.get('name')) if 'name' in properties else udp_4500_rule, + 'project': project_id, + 'IPAddress': ip_address, 'IPProtocol': 'UDP', 'portRange': 4500, 'region': context.properties['region'], - 'target': '$(ref.' + target_vpn_gateway + '.selfLink)' + 'target': '$(ref.' + target_vpn_gateway + '.selfLink)', } }, { # The forwarding rule resource for the UDP traffic on port 500 'name': udp_500_rule, - 'type': 'compute.v1.forwardingRule', + # https://cloud.google.com/compute/docs/reference/rest/v1/forwardingRules + 'type': 'gcp-types/compute-v1:forwardingRules', 'properties': { - 'IPAddress': '$(ref.' + static_ip + '.address)', + 'name': '{}-udp-500'.format(properties.get('name')) if 'name' in properties else udp_500_rule, + 'project': project_id, + 'IPAddress': ip_address, 'IPProtocol': 'UDP', 'portRange': 500, 'region': context.properties['region'], - 'target': '$(ref.' + target_vpn_gateway + '.selfLink)' + 'target': '$(ref.' + target_vpn_gateway + '.selfLink)', } }, - { - # The VPN tunnel resource. - 'name': vpn_tunnel, - 'type': 'compute.v1.vpnTunnel', - 'properties': - { + + ]) + router_url_tag = 'routerURL' + router_name_tag = 'router' + + if router_name_tag in context.properties: + router_url = context.properties.get(router_url_tag, generate_router_uri( + context.env['project'], + context.properties['region'], + context.properties[router_name_tag])) + # Create dynamic routing VPN + resources.extend([ + { + # The VPN tunnel resource. + 'name': vpn_tunnel, + # https://cloud.google.com/compute/docs/reference/rest/v1/vpnTunnels + 'type': 'gcp-types/compute-v1:vpnTunnels', + 'properties': + { + 'name': properties.get('name', vpn_tunnel), + 'project': project_id, + 'description': + 'A vpn tunnel', + 'ikeVersion': + 2, + 'peerIp': + context.properties['peerAddress'], + 'region': + context.properties['region'], + 'router': router_url, + 'sharedSecret': + context.properties['sharedSecret'], + 'targetVpnGateway': + '$(ref.' + target_vpn_gateway + '.selfLink)' + }, + 'metadata': { + 'dependsOn': [esp_rule, + udp_500_rule, + udp_4500_rule] + } + }, + { + # An action that is executed after the vpn_tunnel function. + # It calls the method patch by ID on the descriptor document + # https://cloud.google.com/compute/docs/reference/rest/v1/routers/patch + 'name': router_vpn_binding, + 'action': 'gcp-types/compute-v1:compute.routers.patch', + 'properties': + { + 'project': project_id, + 'router': + context.properties[router_name_tag], + 'region': + context.properties['region'], + 'name': + context.properties[router_name_tag], + 'asn': + context.properties['asn'], + 'interfaces': + [ + { + 'ipRange': + '169.254.1.1/31', + 'linkedVpnTunnel': + '$(ref.' + vpn_tunnel + '.selfLink)', + 'name': + 'if-1' + } + ] + } + }]) + else: + # Create static routing VPN + resources.append( + { + # The VPN tunnel resource. + 'name': vpn_tunnel, + 'type': 'gcp-types/compute-v1:vpnTunnels', + 'properties': { + 'name': vpn_tunnel, 'description': 'A vpn tunnel', 'ikeVersion': @@ -100,55 +199,21 @@ def generate_config(context): context.properties['peerAddress'], 'region': context.properties['region'], - 'router': - generate_router_url( - context.env['project'], - context.properties['region'], - context.properties['router'] - ), 'sharedSecret': context.properties['sharedSecret'], 'targetVpnGateway': - '$(ref.' + target_vpn_gateway + '.selfLink)' + '$(ref.' + target_vpn_gateway + '.selfLink)', + 'localTrafficSelector': + context.properties['localTrafficSelector'], + 'remoteTrafficSelector': + context.properties['remoteTrafficSelector'], + }, - 'metadata': { - 'dependsOn': [esp_rule, - udp_500_rule, - udp_4500_rule] - } - }, - { - # An action that is executed after the vpn_tunnel function. - # It calls the method patch by ID on the descriptor document - # https://www.googleapis.com/discovery/v1/apis/compute/v1/rest. - 'name': router_vpn_binding, - 'action': 'gcp-types/compute-v1:compute.routers.patch', - 'properties': - { - 'router': - context.properties['router'], - 'region': - context.properties['region'], - 'project': - context.env['project'], - 'name': - context.properties['router'], - 'asn': - context.properties['asn'], - 'interfaces': - [ - { - 'ipRange': - '169.254.1.1/31', - 'linkedVpnTunnel': - '$(ref.' + vpn_tunnel + '.selfLink)', - 'name': - 'if-1' - } - ] + 'metadata': { + 'dependsOn': [esp_rule, udp_500_rule, udp_4500_rule] } - } - ] + }, + ) return { 'resources': @@ -178,20 +243,22 @@ def generate_config(context): { 'name': 'vpnTunnel', 'value': vpn_tunnel + }, + { + 'name': 'vpnTunnelUri', + 'value': '$(ref.'+vpn_tunnel+'.selfLink)' } ] } - -def generate_network_url(project_id, network): +def generate_network_uri(project_id, network): """Format the resource name as a resource URI.""" return 'projects/{}/global/networks/{}'.format(project_id, network) - -def generate_router_url(project_id, region, router): - """Format the resource name as a resource URI.""" +def generate_router_uri(project_id, region, router_name): + """Format the router name as a router URI.""" return 'projects/{}/regions/{}/routers/{}'.format( project_id, region, - router + router_name ) diff --git a/dm/templates/vpn/vpn.py.schema b/dm/templates/vpn/vpn.py.schema index c2f5f3a9442..fca34f3ac49 100644 --- a/dm/templates/vpn/vpn.py.schema +++ b/dm/templates/vpn/vpn.py.schema @@ -15,21 +15,80 @@ info: title: VPN author: Sourced Group - description: Creates a VPN tunnel, gateway, and fowarding rules. + version: 1.0.0 + description: | + Creates a VPN tunnel, gateway, and fowarding rules. + + For more information on this resource: + https://cloud.google.com/vpn/docs/concepts/overview + + APIs endpoints used by this template: + - gcp-types/compute-v1:instances => + https://cloud.google.com/compute/docs/reference/rest/v1/instances + - gcp-types/compute-v1:forwardingRules => + https://cloud.google.com/compute/docs/reference/rest/v1/forwardingRules + - gcp-types/compute-v1:addresses => + https://cloud.google.com/compute/docs/reference/rest/v1/addresses + - gcp-types/compute-v1:targetVpnGateways => + https://cloud.google.com/compute/docs/reference/rest/v1/targetVpnGateways + - gcp-types/compute-v1:vpnTunnels => + https://cloud.google.com/compute/docs/reference/rest/v1/vpnTunnels + - gcp-types/compute-v1:compute.routers.patch => + https://cloud.google.com/compute/docs/reference/rest/v1/routers/patch additionalProperties: false -required: - - network - - region - - peerAddress - - asn - - sharedSecret +allOf: + - required: + - region + - peerAddress + - sharedSecret + - oneOf: + - required: + - networkURL + - required: + - network + - oneOf: + # Use dynamic routing. + - required: + - asn + - router + # Use static routing. + - allOf: + - not: + required: + - router + - required: + - localTrafficSelector + - remoteTrafficSelector properties: + name: + type: string + description: | + Common name for all provisioned resources. + Resource name would be used if omitted. + project: + type: string + description: | + The project ID of the project containing resources. The + Google apps domain is prefixed if applicable. + routerURL: + type: string + description: URL (or URI) of the Router resource. Used by vpnTunnels. + router: + type: string + description: | + Name of the Router resource. + networkURL: + type: string + description: | + The URL (or URI) of the network to which the VPN belongs. network: type: string - description: The URI of the network to which the VPN belongs. + description: | + The name of the network to which the VPN belongs. Only use "networkName" + when it is impossible to get "networkURL". region: type: string description: The URI of the region where the VPN resides. @@ -48,6 +107,29 @@ properties: description: | The value is used to set the secure session between the Cloud VPN gateway and the peer VPN gateway. + localTrafficSelector: + type: array + description: | + Used when establishing the VPN tunnel with the peer VPN gateway. + default: ["0.0.0.0/0"] + uniqItems: true + items: + type: string + description: "CIDR formatted string, for example: 192.168.0.0/16." + remoteTrafficSelector: + type: array + description: | + Used when establishing the VPN tunnel with the peer VPN gateway. + default: ["0.0.0.0/0"] + uniqItems: true + items: + type: string + description: "CIDR formatted string, for example: 192.168.0.0/16." + ipAddress: + type: string + description: | + Static IP address used by forwarding rules. When not specified, a new static + IP address will be created. outputs: properties: @@ -68,10 +150,13 @@ outputs: description: The name of the ForwardingRule resource for the UDP 500 traffic. - vpnTunnel: type: string - description: The mame of the VPN tunnel resource. + description: The name of the VPN tunnel resource. + - vpnTunnelUri: + type: string + description: The URI of the VPN tunnel resource. documentation: - templates/vpn/README.md examples: - - templates/vpn/examples/vpn.yaml \ No newline at end of file + - templates/vpn/examples/vpn.yaml