From a45f1d1c0022dc848b726b2e57ebb7bd63f07df1 Mon Sep 17 00:00:00 2001 From: Andriy Kopachevskyy Date: Mon, 9 Sep 2019 13:05:28 +0300 Subject: [PATCH 1/2] Event function: update testing to new approach To support zone and region params of examples added custom code to main.tf to get random available Fixes #26 --- .gitignore | 3 + CONTRIBUTING.md | 119 ++++++++ Makefile | 147 +++------ build/int.cloudbuild.yaml | 44 +++ .../lint.cloudbuild.yaml | 11 + test/.gitignore | 1 + test/boilerplate/boilerplate.Makefile.txt | 13 - test/boilerplate/boilerplate.go.txt | 15 - test/boilerplate/boilerplate.py.txt | 13 - test/boilerplate/boilerplate.xml.txt | 15 - test/boilerplate/boilerplate.yaml.txt | 13 - test/integration.sh | 71 ----- test/make.sh | 161 ---------- test/setup/.gitignore | 2 + test/setup/iam.tf | 40 +++ test/setup/main.tf | 68 +++++ .../make_source.sh} | 21 +- test/setup/outputs.tf | 33 ++ test/setup/variables.tf | 26 ++ .../boilerplate.tf.txt => setup/versions.tf} | 12 + test/test_verify_boilerplate.py | 140 --------- test/verify_boilerplate.py | 281 ------------------ 22 files changed, 428 insertions(+), 821 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 build/int.cloudbuild.yaml rename test/boilerplate/boilerplate.Dockerfile.txt => build/lint.cloudbuild.yaml (65%) create mode 100644 test/.gitignore delete mode 100644 test/boilerplate/boilerplate.Makefile.txt delete mode 100644 test/boilerplate/boilerplate.go.txt delete mode 100644 test/boilerplate/boilerplate.py.txt delete mode 100644 test/boilerplate/boilerplate.xml.txt delete mode 100644 test/boilerplate/boilerplate.yaml.txt delete mode 100755 test/integration.sh delete mode 100755 test/make.sh create mode 100644 test/setup/.gitignore create mode 100644 test/setup/iam.tf create mode 100644 test/setup/main.tf rename test/{boilerplate/boilerplate.sh.txt => setup/make_source.sh} (53%) mode change 100644 => 100755 create mode 100644 test/setup/outputs.tf create mode 100644 test/setup/variables.tf rename test/{boilerplate/boilerplate.tf.txt => setup/versions.tf} (80%) delete mode 100755 test/test_verify_boilerplate.py delete mode 100644 test/verify_boilerplate.py diff --git a/.gitignore b/.gitignore index a04824e..c5fb326 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ \#*\# .\#* +# IDE-related fiels +.idea + # Vim-related files [._]*.s[a-w][a-z] [._]s[a-w][a-z] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..638e685 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,119 @@ +CONTRIBUTING# Contributing + +This document provides guidelines for contributing to the module. + +## Dependencies + +The following dependencies must be installed on the development system: + +- [Terraform][terraform-site] v0.12.Z +- [Docker Engine][docker-engine] +- [Google Cloud SDK][google-cloud-sdk] +- [make] + +### IAM Roles + +The Service Account which will be used to invoke this module must have +the following IAM roles: + +- Cloud Functions Developer: `roles/cloudfunctions.developer` +- Storage Admin: `roles/storage.admin` + +### APIs + +The project against which this module will be invoked must have the +following APIs enabled: + +- Cloud Functions API: `cloudfunctions.googleapis.com` +- Cloud Storage API: `storage-component.googleapis.com` + +The [Project Factory module][project-factory-module-site] can be used to +provision projects with specific APIs activated. + +## Generating Documentation for Inputs and Outputs + +The Inputs and Outputs tables in the READMEs of the root module, +submodules, and example modules are automatically generated based on +the `variables` and `outputs` of the respective modules. These tables +must be refreshed if the module interfaces are changed. + +### Execution + +Run `make generate_docs` to generate new Inputs and Outputs tables. + +## Integration Testing + +Integration tests are used to verify the behaviour of the root module, +submodules, and example modules. Additions, changes, and fixes should +be accompanied with tests. + +The integration tests are run using [Kitchen][kitchen], +[Kitchen-Terraform][kitchen-terraform], and [InSpec][inspec]. These +tools are packaged within a Docker image for convenience. + +The general strategy for these tests is to verify the behaviour of the +[example modules](./examples/), thus ensuring that the root module, +submodules, and example modules are all functionally correct. + +### Test Environment +The easiest way to test the module is in an isolated test project. The setup for such a project is defined in [test/setup](./test/setup/) directory. + +To use this setup, you need a service account with Project Creator access on a folder. Export the Service Account credentials to your environment like so: + +``` +export SERVICE_ACCOUNT_JSON=$(< credentials.json) +``` + +You will also need to set a few environment variables: +``` +export TF_VAR_org_id="your_org_id" +export TF_VAR_folder_id="your_folder_id" +export TF_VAR_billing_account="your_billing_account_id" +``` + +With these settings in place, you can prepare a test project using Docker: +``` +make docker_test_prepare +``` + +### Noninteractive Execution + +Run `make docker_test_integration` to test all of the example modules +noninteractively, using the prepared test project. + +### Interactive Execution + +1. Run `make docker_run` to start the testing Docker container in + interactive mode. + +1. Run `kitchen_do create ` to initialize the working + directory for an example module. + +1. Run `kitchen_do converge ` to apply the example module. + +1. Run `kitchen_do verify ` to test the example module. + +1. Run `kitchen_do destroy ` to destroy the example module + state. + +## Linting and Formatting + +Many of the files in the repository can be linted or formatted to +maintain a standard of quality. + +### Execution + +Run `make docker_test_lint`. + +[docker-engine]: https://www.docker.com/products/docker-engine +[flake8]: http://flake8.pycqa.org/en/latest/ +[gofmt]: https://golang.org/cmd/gofmt/ +[google-cloud-sdk]: https://cloud.google.com/sdk/install +[hadolint]: https://github.com/hadolint/hadolint +[inspec]: https://inspec.io/ +[kitchen-terraform]: https://github.com/newcontext-oss/kitchen-terraform +[kitchen]: https://kitchen.ci/ +[make]: https://en.wikipedia.org/wiki/Make_(software) +[shellcheck]: https://www.shellcheck.net/ +[terraform-docs]: https://github.com/segmentio/terraform-docs +[terraform]: https://terraform.io/ diff --git a/Makefile b/Makefile index e39f260..6a10795 100644 --- a/Makefile +++ b/Makefile @@ -18,117 +18,68 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -# Docker build config variables -CREDENTIALS_PATH ?= /cft/workdir/credentials.json -DOCKER_ORG := gcr.io/cloud-foundation-cicd -DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 2.3.0 -DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM} +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0 +DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools +REGISTRY_URL := gcr.io/cloud-foundation-cicd -all: check generate_docs - -.PHONY: check -check: check_shell check_python check_terraform check_base_files test_check_headers check_headers check_trailing_whitespace - -.PHONY: check_shell -check_shell: - @source test/make.sh && check_shell - -.PHONY: check_python -check_python: - @source test/make.sh && check_python - -.PHONY: check_terraform -check_terraform: - @source test/make.sh && check_terraform - -.PHONY: check_base_files -check_base_files: - @source test/make.sh && basefiles - -.PHONY: check_trailing_whitespace -check_trailing_whitespace: - @source test/make.sh && check_trailing_whitespace - -.PHONY: test_check_headers -test_check_headers: - @echo "Testing the validity of the header check" - @python test/test_verify_boilerplate.py - -.PHONY: check_headers -check_headers: - @source test/make.sh && check_headers - -# Integration tests -.PHONY: test_integration -test_integration: - test/integration.sh - -.PHONY: generate_docs -generate_docs: - @source test/make.sh && generate_docs - -# Versioning -.PHONY: version -version: - @source helpers/version-repo.sh - -# Run docker +# Enter docker container for local development .PHONY: docker_run docker_run: docker run --rm -it \ - -e PROJECT_ID \ -e SERVICE_ACCOUNT_JSON \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "source test/integration.sh && setup_environment && exec /bin/bash" + -v $(CURDIR):/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /bin/bash -.PHONY: docker_create -docker_create: +# Execute prepare tests within the docker container +.PHONY: docker_test_prepare +docker_test_prepare: docker run --rm -it \ - -e PROJECT_ID \ -e SERVICE_ACCOUNT_JSON \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "source test/integration.sh && setup_environment && kitchen create" - -.PHONY: docker_converge -docker_converge: + -e TF_VAR_org_id \ + -e TF_VAR_folder_id \ + -e TF_VAR_billing_account \ + -v $(CURDIR):/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /usr/local/bin/execute_with_credentials.sh prepare_environment + +# Clean up test environment within the docker container +.PHONY: docker_test_cleanup +docker_test_cleanup: docker run --rm -it \ - -e PROJECT_ID \ -e SERVICE_ACCOUNT_JSON \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "source test/integration.sh && setup_environment && kitchen converge" - -.PHONY: docker_verify -docker_verify: + -e TF_VAR_org_id \ + -e TF_VAR_folder_id \ + -e TF_VAR_billing_account \ + -v $(CURDIR):/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /usr/local/bin/execute_with_credentials.sh cleanup_environment + +# Execute integration tests within the docker container +.PHONY: docker_test_integration +docker_test_integration: docker run --rm -it \ - -e PROJECT_ID \ -e SERVICE_ACCOUNT_JSON \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "source test/integration.sh && setup_environment && kitchen verify" + -v $(CURDIR):/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /usr/local/bin/test_integration.sh -.PHONY: docker_destroy -docker_destroy: +# Execute lint tests within the docker container +.PHONY: docker_test_lint +docker_test_lint: docker run --rm -it \ - -e PROJECT_ID \ - -e SERVICE_ACCOUNT_JSON \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "source test/integration.sh && setup_environment && kitchen destroy" + -v $(CURDIR):/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /usr/local/bin/test_lint.sh -.PHONY: test_integration_docker -test_integration_docker: +# Generate documentation +.PHONY: docker_generate_docs +docker_generate_docs: docker run --rm -it \ - -e PROJECT_ID \ - -e SERVICE_ACCOUNT_JSON \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - make test_integration + -v $(CURDIR):/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs' + +# Alias for backwards compatibility +.PHONY: generate_docs +generate_docs: docker_generate_docs diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml new file mode 100644 index 0000000..998da2b --- /dev/null +++ b/build/int.cloudbuild.yaml @@ -0,0 +1,44 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +timeout: 3600s +steps: +- id: prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment'] + env: + - 'TF_VAR_org_id=$_ORG_ID' + - 'TF_VAR_folder_id=$_FOLDER_ID' + - 'TF_VAR_billing_account=$_BILLING_ACCOUNT' +- id: echo + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cat test/source.sh'] +- id: create + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] +- id: converge + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge'] +- id: verify + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify'] +- id: destroy + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy'] +tags: +- 'ci' +- 'integration' +substitutions: + _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' diff --git a/test/boilerplate/boilerplate.Dockerfile.txt b/build/lint.cloudbuild.yaml similarity index 65% rename from test/boilerplate/boilerplate.Dockerfile.txt rename to build/lint.cloudbuild.yaml index 086a24e..779febf 100644 --- a/test/boilerplate/boilerplate.Dockerfile.txt +++ b/build/lint.cloudbuild.yaml @@ -11,3 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +steps: +- name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + id: 'lint' + args: ['/usr/local/bin/test_lint.sh'] +tags: +- 'ci' +- 'lint' +substitutions: + _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..d69ba0d --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +source.sh diff --git a/test/boilerplate/boilerplate.Makefile.txt b/test/boilerplate/boilerplate.Makefile.txt deleted file mode 100644 index 086a24e..0000000 --- a/test/boilerplate/boilerplate.Makefile.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/test/boilerplate/boilerplate.go.txt b/test/boilerplate/boilerplate.go.txt deleted file mode 100644 index 0fb04be..0000000 --- a/test/boilerplate/boilerplate.go.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright 2019 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ diff --git a/test/boilerplate/boilerplate.py.txt b/test/boilerplate/boilerplate.py.txt deleted file mode 100644 index 086a24e..0000000 --- a/test/boilerplate/boilerplate.py.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/test/boilerplate/boilerplate.xml.txt b/test/boilerplate/boilerplate.xml.txt deleted file mode 100644 index cd0c8f4..0000000 --- a/test/boilerplate/boilerplate.xml.txt +++ /dev/null @@ -1,15 +0,0 @@ - diff --git a/test/boilerplate/boilerplate.yaml.txt b/test/boilerplate/boilerplate.yaml.txt deleted file mode 100644 index 086a24e..0000000 --- a/test/boilerplate/boilerplate.yaml.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/test/integration.sh b/test/integration.sh deleted file mode 100755 index 049534b..0000000 --- a/test/integration.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Always clean up. -DELETE_AT_EXIT="$(mktemp -d)" -finish() { - echo 'BEGIN: finish() trap handler' >&2 - kitchen destroy - [[ -d "${DELETE_AT_EXIT}" ]] && rm -rf "${DELETE_AT_EXIT}" - echo 'END: finish() trap handler' >&2 -} - -# Map the input parameters provided by Concourse CI, or whatever mechanism is -# running the tests to Terraform input variables. Also setup credentials for -# use with kitchen-terraform, inspec, and gcloud. -setup_environment() { - local tmpfile - tmpfile="$(mktemp)" - echo "${SERVICE_ACCOUNT_JSON}" > "${tmpfile}" - - # gcloud variables - export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${tmpfile}" - # Application default credentials (Terraform google provider and inspec-gcp) - export GOOGLE_APPLICATION_CREDENTIALS="${tmpfile}" - - # Terraform variables - export TF_VAR_project_id="${PROJECT_ID}" - export TF_VAR_region="${REGION:-us-east1}" - export TF_VAR_zone="${ZONE:-us-east1-b}" - - # Stubs for module/*. Use for terraform linter - export TF_VAR_entry_point="" - export TF_VAR_event_trigger="{event_type=\"\",resource=\"\"}" - export TF_VAR_name="test" - export TF_VAR_runtime="" - export TF_VAR_source_directory="" - export TF_VAR_source_repository_url="" - export TF_VAR_filter="" -} - -main() { - set -eu - # Setup trap handler to auto-cleanup - export TMPDIR="${DELETE_AT_EXIT}" - trap finish EXIT - - # Setup environment variables - setup_environment - set -x - - # Execute the test lifecycle - kitchen verify -} - -# if script is being executed and not sourced. -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - main "$@" -fi diff --git a/test/make.sh b/test/make.sh deleted file mode 100755 index 5fc65eb..0000000 --- a/test/make.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Please note that this file was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). -# Please make sure to contribute relevant changes upstream! - -# Create a temporary directory that's auto-cleaned, even if the process aborts. -DELETE_AT_EXIT="$(mktemp -d)" -finish() { - [[ -d "${DELETE_AT_EXIT}" ]] && rm -rf "${DELETE_AT_EXIT}" -} -trap finish EXIT -# Create a temporary file in the auto-cleaned up directory while avoiding -# overwriting TMPDIR for other processes. -# shellcheck disable=SC2120 -# (Arguments may be passed, e.g. maketemp -d) -maketemp() { - TMPDIR="${DELETE_AT_EXIT}" mktemp "$@" -} - -# find_files is a helper to exclude .git directories and match only regular -# files to avoid double-processing symlinks. -find_files() { - local pth="$1" - shift - find "${pth}" '(' \ - -path '*/.git' \ - -o -path '*/.terraform' \ - -o -path '*/.kitchen' \ - ')' \ - -prune -o -type f "$@" -} - -# Compatibility with both GNU and BSD style xargs. -compat_xargs() { - local compat=() - # Test if xargs is GNU or BSD style. GNU xargs will succeed with status 0 - # when given --no-run-if-empty and no input on STDIN. BSD xargs will fail and - # exit status non-zero If xargs fails, assume it is BSD style and proceed. - # stderr is silently redirected to avoid console log spam. - if xargs --no-run-if-empty /dev/null; then - compat=("--no-run-if-empty") - fi - xargs "${compat[@]}" "$@" -} - -# This function makes sure that the required files for -# releasing to OSS are present -function basefiles() { - local fn required_files="LICENSE README.md" - echo "Checking for required files ${required_files}" - for fn in ${required_files}; do - test -f "${fn}" || echo "Missing required file ${fn}" - done -} - -# This function runs 'terraform validate' against all -# directory paths which contain *.tf files. -function check_terraform() { - local rval=125 - # fmt is before validate for faster feedback, validate requires terraform - # init which takes time. - echo "Running terraform fmt" - find_files . -name "*.tf" -print0 \ - | compat_xargs -0 -n1 dirname \ - | sort -u \ - | compat_xargs -t -n1 echo #terraform fmt -diff -check=true -write=false - rval="$?" - if [[ "${rval}" -gt 0 ]]; then - echo "Error: terraform fmt failed with exit code ${rval}" >&2 - echo "Check the output for diffs and correct using terraform fmt " >&2 - return "${rval}" - fi - echo "Running terraform validate" - find_files . -not -path "./test/fixtures/shared/*" -name "*.tf" -print0 \ - | compat_xargs -0 -n1 dirname \ - | sort -u \ - | compat_xargs -t -n1 helpers/terraform_validate -} - -# This function runs the flake8 linter on every file -# ending in '.py' -function check_python() { - echo "Running flake8" - find_files . -name "*.py" -print0 | compat_xargs -0 flake8 - return 0 -} - -# This function runs the shellcheck linter on every -# file ending in '.sh' -function check_shell() { - echo "Running shellcheck" - find_files . -name "*.sh" -print0 | compat_xargs -0 shellcheck -x -} - -# This function makes sure that there is no trailing whitespace -# in any files in the project. -# There are some exclusions -function check_trailing_whitespace() { - local rc - echo "Checking for trailing whitespace" - find_files . -print \ - | grep -v -E '\.(pyc|png)$' \ - | compat_xargs grep -H -n '[[:blank:]]$' - rc=$? - if [[ ${rc} -eq 0 ]]; then - return 1 - fi -} - -function generate_docs() { - echo "Generating markdown docs with terraform-docs" - local pth helper_dir rval - helper_dir="$(pwd)/helpers" - while read -r pth; do - if [[ -e "${pth}/README.md" ]]; then - (cd "${pth}" || return 3; "${helper_dir}"/terraform_docs .;) - rval="$?" - if [[ "${rval}" -gt 0 ]]; then - echo "Error: terraform_docs in ${pth} exit code: ${rval}" >&2 - return "${rval}" - fi - else - echo "Skipping ${pth} because README.md does not exist." - fi - done < <(find_files . -name '*.tf' -print0 \ - | compat_xargs -0 -n1 dirname \ - | sort -u) -} - -function prepare_test_variables() { - echo "Preparing terraform.tfvars files for integration tests" - #shellcheck disable=2044 - for i in $(find ./test/fixtures -type f -name terraform.tfvars.sample); do - destination=${i/%.sample/} - if [ ! -f "${destination}" ]; then - cp "${i}" "${destination}" - echo "${destination} has been created. Please edit it to reflect your GCP configuration." - fi - done -} - -function check_headers() { - echo "Checking file headers" - # Use the exclusion behavior of find_files - find_files . -type f -print0 \ - | compat_xargs -0 python test/verify_boilerplate.py -} diff --git a/test/setup/.gitignore b/test/setup/.gitignore new file mode 100644 index 0000000..0e515f8 --- /dev/null +++ b/test/setup/.gitignore @@ -0,0 +1,2 @@ +terraform.tfvars +source.sh diff --git a/test/setup/iam.tf b/test/setup/iam.tf new file mode 100644 index 0000000..8d8cb2b --- /dev/null +++ b/test/setup/iam.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + int_required_roles = [ + "roles/owner", + "roles/compute.admin" + ] +} + +resource "google_service_account" "int_test" { + project = module.project.project_id + account_id = "ci-account" + display_name = "ci-account" +} + +resource "google_project_iam_member" "int_test" { + count = length(local.int_required_roles) + + project = module.project.project_id + role = local.int_required_roles[count.index] + member = "serviceAccount:${google_service_account.int_test.email}" +} + +resource "google_service_account_key" "int_test" { + service_account_id = google_service_account.int_test.id +} diff --git a/test/setup/main.tf b/test/setup/main.tf new file mode 100644 index 0000000..7a434c4 --- /dev/null +++ b/test/setup/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_id" "project" { + byte_length = 4 +} + +locals { + project_name = "ci-event-function-${random_id.project.hex}" + region = random_shuffle.available_regions.result[0] + zone = random_shuffle.available_zones.result[0] +} + + +module "project" { + source = "terraform-google-modules/project-factory/google" + version = "~> 3.0" + + name = local.project_name + random_project_id = false + auto_create_network = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + + activate_apis = [ + "cloudresourcemanager.googleapis.com", + "storage-api.googleapis.com", + "serviceusage.googleapis.com", + "cloudfunctions.googleapis.com", + "storage-component.googleapis.com", + "sourcerepo.googleapis.com", + "compute.googleapis.com" + ] +} + +data "google_compute_regions" "available" { + project = module.project.project_id +} + +resource "random_shuffle" "available_regions" { + input = data.google_compute_regions.available.names + result_count = 1 +} + +data "google_compute_zones" "available" { + project = module.project.project_id + region = local.region +} + +resource "random_shuffle" "available_zones" { + input = data.google_compute_zones.available.names + result_count = 3 +} + diff --git a/test/boilerplate/boilerplate.sh.txt b/test/setup/make_source.sh old mode 100644 new mode 100755 similarity index 53% rename from test/boilerplate/boilerplate.sh.txt rename to test/setup/make_source.sh index ddd71c0..c91886f --- a/test/boilerplate/boilerplate.sh.txt +++ b/test/setup/make_source.sh @@ -1,4 +1,6 @@ -# Copyright 2019 Google LLC +#!/usr/bin/env bash + +# Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +13,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +project_id=$(terraform output project_id) +region=$(terraform output region) +zone=$(terraform output zone) + +{ + echo "#!/usr/bin/env bash" + echo "export TF_VAR_project_id='$project_id'" + echo "export TF_VAR_region='$region'" + echo "export TF_VAR_zone='$zone'" +} > ../source.sh + +sa_json=$(terraform output sa_key) +# shellcheck disable=SC2086 +echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh + + diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf new file mode 100644 index 0000000..c9eb8ea --- /dev/null +++ b/test/setup/outputs.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.project.project_id +} + +output "sa_key" { + value = google_service_account_key.int_test.private_key + sensitive = true +} + +output "region" { + value = local.region +} + +output "zone" { + value = local.zone +} + diff --git a/test/setup/variables.tf b/test/setup/variables.tf new file mode 100644 index 0000000..6d80b89 --- /dev/null +++ b/test/setup/variables.tf @@ -0,0 +1,26 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +variable "org_id" { + description = "The numeric organization id" +} + +variable "folder_id" { + description = "The folder to deploy in" +} + +variable "billing_account" { + description = "The billing account id associated with the project, e.g. XXXXXX-YYYYYY-ZZZZZZ" +} diff --git a/test/boilerplate/boilerplate.tf.txt b/test/setup/versions.tf similarity index 80% rename from test/boilerplate/boilerplate.tf.txt rename to test/setup/versions.tf index 83185dc..efbd8ea 100644 --- a/test/boilerplate/boilerplate.tf.txt +++ b/test/setup/versions.tf @@ -13,3 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +terraform { + required_version = ">= 0.12" +} + +provider "google" { + version = "~> 2.13.0" +} + +provider "google-beta" { + version = "~> 2.13.0" +} diff --git a/test/test_verify_boilerplate.py b/test/test_verify_boilerplate.py deleted file mode 100755 index 0d1cb44..0000000 --- a/test/test_verify_boilerplate.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Please note that this file was generated from -# [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). -# Please make sure to contribute relevant changes upstream! - -''' A simple test for the verify_boilerplate python script. -This will create a set of test files, both valid and invalid, -and confirm that the has_valid_header call returns the correct -value. - -It also checks the number of files that are found by the -get_files call. -''' -from copy import deepcopy -from tempfile import mkdtemp -from shutil import rmtree -import unittest -from verify_boilerplate import has_valid_header, get_refs, get_regexs, \ - get_args, get_files - - -class AllTestCase(unittest.TestCase): - """ - All of the setup, teardown, and tests are contained in this - class. - """ - - def write_file(self, filename, content, expected): - """ - A utility method that creates test files, and adds them to - the cases that will be tested. - - Args: - filename: (string) the file name (path) to be created. - content: (list of strings) the contents of the file. - expected: (boolean) True if the header is expected to be valid, - false if not. - """ - - file = open(filename, 'w+') - for line in content: - file.write(line + "\n") - file.close() - self.cases[filename] = expected - - def create_test_files(self, tmp_path, extension, header): - """ - Creates 2 test files for .tf, .xml, .go, etc and one for - Dockerfile, and Makefile. - - The reason for the difference is that Makefile and Dockerfile - don't have an extension. These would be substantially more - difficult to create negative test cases, unless the files - were written, deleted, and re-written. - - Args: - tmp_path: (string) the path in which to create the files - extension: (string) the file extension - header: (list of strings) the header/boilerplate content - """ - - content = "\n...blah \ncould be code or could be garbage\n" - special_cases = ["Dockerfile", "Makefile"] - header_template = deepcopy(header) - valid_filename = tmp_path + extension - valid_content = header_template.append(content) - if extension not in special_cases: - # Invalid test cases for non-*file files (.tf|.py|.sh|.yaml|.xml..) - invalid_header = [] - for line in header_template: - if "2019" in line: - invalid_header.append(line.replace('2019', 'YEAR')) - else: - invalid_header.append(line) - invalid_header.append(content) - invalid_content = invalid_header - invalid_filename = tmp_path + "invalid." + extension - self.write_file(invalid_filename, invalid_content, False) - valid_filename = tmp_path + "testfile." + extension - - valid_content = header_template - self.write_file(valid_filename, valid_content, True) - - def setUp(self): - """ - Set initial counts and values, and initializes the setup of the - test files. - """ - self.cases = {} - self.tmp_path = mkdtemp() + "/" - self.my_args = get_args() - self.my_refs = get_refs(self.my_args) - self.my_regex = get_regexs() - self.prexisting_file_count = len( - get_files(self.my_refs.keys(), self.my_args)) - for key in self.my_refs: - self.create_test_files(self.tmp_path, key, - self.my_refs.get(key)) - - def tearDown(self): - """ Delete the test directory. """ - rmtree(self.tmp_path) - - def test_files_headers(self): - """ - Confirms that the expected output of has_valid_header is correct. - """ - for case in self.cases: - if self.cases[case]: - self.assertTrue(has_valid_header(case, self.my_refs, - self.my_regex)) - else: - self.assertFalse(has_valid_header(case, self.my_refs, - self.my_regex)) - - def test_invalid_count(self): - """ - Test that the initial files found isn't zero, indicating - a problem with the code. - """ - self.assertFalse(self.prexisting_file_count == 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/verify_boilerplate.py b/test/verify_boilerplate.py deleted file mode 100644 index 74e684e..0000000 --- a/test/verify_boilerplate.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Verifies that all source files contain the necessary copyright boilerplate -# snippet. -# This is based on existing work -# https://github.com/kubernetes/test-infra/blob/master/hack -# /verify_boilerplate.py - -# Please note that this file was generated from -# [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). -# Please make sure to contribute relevant changes upstream! -from __future__ import print_function -import argparse -import glob -import os -import re -import sys - - -def get_args(): - """Parses command line arguments. - - Configures and runs argparse.ArgumentParser to extract command line - arguments. - - Returns: - An argparse.Namespace containing the arguments parsed from the - command line - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "filenames", - help="list of files to check, " - "all files if unspecified", - nargs='*') - rootdir = os.path.dirname(__file__) + "/../" - rootdir = os.path.abspath(rootdir) - parser.add_argument( - "--rootdir", default=rootdir, help="root directory to examine") - - default_boilerplate_dir = os.path.join(rootdir, "test/boilerplate") - parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) - return parser.parse_args() - - -def get_refs(ARGS): - """Converts the directory of boilerplate files into a map keyed by file - extension. - - Reads each boilerplate file's contents into an array, then adds that array - to a map keyed by the file extension. - - Returns: - A map of boilerplate lines, keyed by file extension. For example, - boilerplate.py.txt would result in the k,v pair {".py": py_lines} where - py_lines is an array containing each line of the file. - """ - refs = {} - - # Find and iterate over the absolute path for each boilerplate template - for path in glob.glob( - os.path.join(ARGS.boilerplate_dir, "boilerplate.*.txt")): - extension = os.path.basename(path).split(".")[1] - ref_file = open(path, 'r') - ref = ref_file.read().splitlines() - ref_file.close() - refs[extension] = ref - return refs - - -# pylint: disable=too-many-locals -def has_valid_header(filename, refs, regexs): - """Test whether a file has the correct boilerplate header. - - Tests each file against the boilerplate stored in refs for that file type - (based on extension), or by the entire filename (eg Dockerfile, Makefile). - Some heuristics are applied to remove build tags and shebangs, but little - variance in header formatting is tolerated. - - Args: - filename: A string containing the name of the file to test - refs: A map of boilerplate headers, keyed by file extension - regexs: a map of compiled regex objects used in verifying boilerplate - - Returns: - True if the file has the correct boilerplate header, otherwise returns - False. - """ - try: - with open(filename, 'r') as fp: # pylint: disable=invalid-name - data = fp.read() - except IOError: - return False - basename = os.path.basename(filename) - extension = get_file_extension(filename) - if extension: - ref = refs[extension] - else: - ref = refs[basename] - # remove build tags from the top of Go files - if extension == "go": - con = regexs["go_build_constraints"] - (data, found) = con.subn("", data, 1) - # remove shebang - elif extension == "sh" or extension == "py": - she = regexs["shebang"] - (data, found) = she.subn("", data, 1) - data = data.splitlines() - # if our test file is smaller than the reference it surely fails! - if len(ref) > len(data): - return False - # trim our file to the same number of lines as the reference file - data = data[:len(ref)] - year = regexs["year"] - for datum in data: - if year.search(datum): - return False - - # if we don't match the reference at this point, fail - if ref != data: - return False - return True - - -def get_file_extension(filename): - """Extracts the extension part of a filename. - - Identifies the extension as everything after the last period in filename. - - Args: - filename: string containing the filename - - Returns: - A string containing the extension in lowercase - """ - return os.path.splitext(filename)[1].split(".")[-1].lower() - - -# These directories will be omitted from header checks -SKIPPED_DIRS = [ - 'Godeps', 'third_party', '_gopath', '_output', '.git', 'vendor', - '__init__.py', 'node_modules' -] - - -def normalize_files(files): - """Extracts the files that require boilerplate checking from the files - argument. - - A new list will be built. Each path from the original files argument will - be added unless it is within one of SKIPPED_DIRS. All relative paths will - be converted to absolute paths by prepending the root_dir path parsed from - the command line, or its default value. - - Args: - files: a list of file path strings - - Returns: - A modified copy of the files list where any any path in a skipped - directory is removed, and all paths have been made absolute. - """ - newfiles = [] - for pathname in files: - if any(x in pathname for x in SKIPPED_DIRS): - continue - newfiles.append(pathname) - for idx, pathname in enumerate(newfiles): - if not os.path.isabs(pathname): - newfiles[idx] = os.path.join(ARGS.rootdir, pathname) - return newfiles - - -def get_files(extensions, ARGS): - """Generates a list of paths whose boilerplate should be verified. - - If a list of file names has been provided on the command line, it will be - treated as the initial set to search. Otherwise, all paths within rootdir - will be discovered and used as the initial set. - - Once the initial set of files is identified, it is normalized via - normalize_files() and further stripped of any file name whose extension is - not in extensions. - - Args: - extensions: a list of file extensions indicating which file types - should have their boilerplate verified - - Returns: - A list of absolute file paths - """ - files = [] - if ARGS.filenames: - files = ARGS.filenames - else: - for root, dirs, walkfiles in os.walk(ARGS.rootdir): - # don't visit certain dirs. This is just a performance improvement - # as we would prune these later in normalize_files(). But doing it - # cuts down the amount of filesystem walking we do and cuts down - # the size of the file list - for dpath in SKIPPED_DIRS: - if dpath in dirs: - dirs.remove(dpath) - for name in walkfiles: - pathname = os.path.join(root, name) - files.append(pathname) - files = normalize_files(files) - outfiles = [] - for pathname in files: - basename = os.path.basename(pathname) - extension = get_file_extension(pathname) - if extension in extensions or basename in extensions: - outfiles.append(pathname) - return outfiles - - -def get_regexs(): - """Builds a map of regular expressions used in boilerplate validation. - - There are two scenarios where these regexes are used. The first is in - validating the date referenced is the boilerplate, by ensuring it is an - acceptable year. The second is in identifying non-boilerplate elements, - like shebangs and compiler hints that should be ignored when validating - headers. - - Returns: - A map of compiled regular expression objects, keyed by mnemonic. - """ - regexs = {} - # Search for "YEAR" which exists in the boilerplate, but shouldn't in the - # real thing - regexs["year"] = re.compile('YEAR') - # dates can be 2014, 2015, 2016, 2017, 2018, or 2019, company holder names - # can be anything - regexs["date"] = re.compile('(2014|2015|2016|2017|2018|2019)') - # strip // +build \n\n build constraints - regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", - re.MULTILINE) - # strip #!.* from shell/python scripts - regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) - return regexs - - -def main(args): - """Identifies and verifies files that should have the desired boilerplate. - - Retrieves the lists of files to be validated and tests each one in turn. - If all files contain correct boilerplate, this function terminates - normally. Otherwise it prints the name of each non-conforming file and - exists with a non-zero status code. - """ - regexs = get_regexs() - refs = get_refs(args) - filenames = get_files(refs.keys(), args) - nonconforming_files = [] - for filename in filenames: - if not has_valid_header(filename, refs, regexs): - nonconforming_files.append(filename) - if nonconforming_files: - print('%d files have incorrect boilerplate headers:' % - len(nonconforming_files)) - for filename in sorted(nonconforming_files): - print(os.path.relpath(filename, args.rootdir)) - sys.exit(1) - - -if __name__ == "__main__": - ARGS = get_args() - main(ARGS) From a1114999b1612b1adc27bd0c5aa7be39471a291e Mon Sep 17 00:00:00 2001 From: Andriy Kopachevskyy Date: Wed, 11 Sep 2019 19:07:33 +0300 Subject: [PATCH 2/2] Event function: update testing to new approach Added Region and Zone variable for test/setup/variables.tf required to run example terraform files during testing. Fixes #26 --- CONTRIBUTING.md | 22 +------- README.md | 119 +--------------------------------------- test/setup/iam.tf | 9 ++- test/setup/main.tf | 29 +--------- test/setup/outputs.tf | 4 +- test/setup/variables.tf | 10 ++++ 6 files changed, 23 insertions(+), 170 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 638e685..a350db5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -CONTRIBUTING# Contributing +# Contributing This document provides guidelines for contributing to the module. @@ -6,30 +6,10 @@ This document provides guidelines for contributing to the module. The following dependencies must be installed on the development system: -- [Terraform][terraform-site] v0.12.Z - [Docker Engine][docker-engine] - [Google Cloud SDK][google-cloud-sdk] - [make] -### IAM Roles - -The Service Account which will be used to invoke this module must have -the following IAM roles: - -- Cloud Functions Developer: `roles/cloudfunctions.developer` -- Storage Admin: `roles/storage.admin` - -### APIs - -The project against which this module will be invoked must have the -following APIs enabled: - -- Cloud Functions API: `cloudfunctions.googleapis.com` -- Cloud Storage API: `storage-component.googleapis.com` - -The [Project Factory module][project-factory-module-site] can be used to -provision projects with specific APIs activated. - ## Generating Documentation for Inputs and Outputs The Inputs and Outputs tables in the READMEs of the root module, diff --git a/README.md b/README.md index bb93f4f..d183e38 100644 --- a/README.md +++ b/README.md @@ -86,121 +86,4 @@ following APIs enabled: - Cloud Storage API: `storage-component.googleapis.com` The [Project Factory module][project-factory-module-site] can be used to -provision projects with specific APIs activated. - -## Testing - -The [fixtures directory](test/fixtures) and -[integration directory](test/integration) comprise Terraform -modules and InSpec tests used to verify the behaviour of this module. - -### Testing Software Dependencies - -The following software dependencies must be installed on the system -from which the tests will be invoked: - -- [Ruby][ruby-site] v2.5 -- [Bundler][bundler-site] v1.17 - -### Integration Tests - -Integration tests are invoked using [Kitchen][kitchen-site], -[Kitchen-Terraform][kitchen-terraform-site], and [InSpec][inspec-site]. - -Kitchen instances are configured in the -[Kitchen configuration file](kitchen.yml). The instances use the modules -in [fixtures directory](test/fixtures) to invoke identically named -modules in the [examples directory](examples) and test this module. - -#### Integration Tests Configuration - -Each Kitchen instance requires a variable file named `terraform.tfvars` -to be created and populated in the associated test fixture. For -convenience, a [sample variable file][sameple-variable-file] is -available. - -A key file for a Service Account with the required -[IAM roles](#iam-roles) must be downloaded from the GCP console and -placed in the root directory of this repository. The key file must be -renamed to `credentials.json`. - -#### Integration Tests Execution - -Run `make test_integration_docker` to execute all of the Kitchen -instances in a non-interactive manner within a Docker container. - -Alternatively, the Kitchen instances can be invoked interactively: - -1. Run `make docker_run` to start the Docker container. The root - directory of this repository will be mounted in the Docker container - at `/cft/workdir/`. -1. Run `kitchen create` to initialize all Kitchen instances, or run - `kitchen create ` to initialize a specific Kitchen - instance. -1. Run `kitchen converge` to apply all Kitchen instances, or run - `kitchen converge ` to apply a specific Kitchen - instance. -1. Run `kitchen verify` to test all Kitchen instances, or run - `kitchen verify ` to test a specific Kitchen instance. -1. Run `kitchen destroy` to destroy all Kitchen instances, or run - `kitchen destroy ` to destroy a specific Kitchen - instance. - -## Linting - -Linters are available for most of the filetypes in this repository. - -### Linting Software Dependencies - -The following software dependencies must be installed on the system -from which the linting will be invoked: - -- [flake8][flake8-site]. -- [ShellCheck][shellcheck-site]. -- [terrafom validate][terraform-validate-site]. - -### Linting Execution - -Run `make check --silent` to execute all of the linters. - -Alternatively, the linters can be invoked individually. - -- Run `make check_python` to lint Python files. -- Run `make check_shell` to lint Shell files. -- Run `make check_terraform` to lint Terraform files. - -## Documentation - -The documentation of inputs and outputs for modules in this repository -is automatically generated in each module's `README.md` based on the -contents of the relevant `.tf` files. - -### Documentation Software Dependencies - -The following software dependencies must be installed on the system -from which the documentation will be generated: - -- [terraform-docs][terraform-docs-site] v0.6.0 - -### Generation - -Run `make generate_docs` to update the documentation. - -[automatic-labelling-from-localhost-example]: examples/automatic-labelling-from-localhost -[bundler-site]: https://bundler.io/ -[event-project-log-entry-submodule]: modules/event-project-log-entry -[repository-function-submodule]: modules/repository-function -[flake8-site]: https://pypi.org/project/flake8/ -[gofmt-site]: https://golang.org/cmd/gofmt/ -[hadolint-site]: https://github.com/hadolint/hadolint/ -[inspec-site]: https://inspec.io/ -[kitchen-site]: https://kitchen.ci/ -[kitchen-terraform-site]: https://github.com/newcontext-oss/kitchen-terraform/ -[project-factory-module-site]: https://github.com/terraform-google-modules/terraform-google-project-factory/ -[ruby-site]: https://ruby-lang.org/ -[sample-variable-file]: test/fixtures/shared/terraform.tfvars.sample -[shellcheck-site]: https://www.shellcheck.net/ -[terraform-docs-site]: https://github.com/segmentio/terraform-docs/releases/ -[terraform-provider-gcp-site]: https://github.com/terraform-providers/terraform-provider-google/ -[terraform-site]: https://www.terraform.io/ -[terraform-validate-site]: https://www.terraform.io/docs/commands/validate.html +provision projects with specific APIs activated. \ No newline at end of file diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 8d8cb2b..5ce270e 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -16,8 +16,13 @@ locals { int_required_roles = [ - "roles/owner", - "roles/compute.admin" + "roles/cloudfunctions.developer", + "roles/storage.admin", + "roles/compute.admin", + "roles/pubsub.admin", + "roles/logging.configWriter", + "roles/source.admin", + "roles/iam.serviceAccountUser" ] } diff --git a/test/setup/main.tf b/test/setup/main.tf index 7a434c4..af49689 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -14,14 +14,8 @@ * limitations under the License. */ -resource "random_id" "project" { - byte_length = 4 -} - locals { - project_name = "ci-event-function-${random_id.project.hex}" - region = random_shuffle.available_regions.result[0] - zone = random_shuffle.available_zones.result[0] + project_name = "ci-event-function" } @@ -30,7 +24,7 @@ module "project" { version = "~> 3.0" name = local.project_name - random_project_id = false + random_project_id = true auto_create_network = true org_id = var.org_id folder_id = var.folder_id @@ -47,22 +41,3 @@ module "project" { ] } -data "google_compute_regions" "available" { - project = module.project.project_id -} - -resource "random_shuffle" "available_regions" { - input = data.google_compute_regions.available.names - result_count = 1 -} - -data "google_compute_zones" "available" { - project = module.project.project_id - region = local.region -} - -resource "random_shuffle" "available_zones" { - input = data.google_compute_zones.available.names - result_count = 3 -} - diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index c9eb8ea..628b854 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -24,10 +24,10 @@ output "sa_key" { } output "region" { - value = local.region + value = var.region } output "zone" { - value = local.zone + value = var.zone } diff --git a/test/setup/variables.tf b/test/setup/variables.tf index 6d80b89..1a951a3 100644 --- a/test/setup/variables.tf +++ b/test/setup/variables.tf @@ -24,3 +24,13 @@ variable "folder_id" { variable "billing_account" { description = "The billing account id associated with the project, e.g. XXXXXX-YYYYYY-ZZZZZZ" } + +variable "region" { + description = "Compute region to create test resources" + default = "us-east4" +} + +variable "zone" { + description = "Compute region zone to create test resources" + default = "us-east4-a" +}