Skip to content
Permalink
Browse files
fix: use correct retry deadlines (#122)
feat: add 'from_service_account_info' factory to clients
fix: fix sphinx identifiers
  • Loading branch information
yoshi-automation committed Apr 5, 2021
1 parent 1b11642 commit 6525f86b698242d77606cffb42713d18724a2526
Showing with 5,140 additions and 3,368 deletions.
  1. +3 −19 .coveragerc
  2. +15 −0 .github/header-checker-lint.yml
  3. +3 −1 .gitignore
  4. +20 −6 .kokoro/build.sh
  5. +11 −0 .kokoro/docs/docs-presubmit.cfg
  6. +11 −0 .kokoro/samples/python3.6/periodic-head.cfg
  7. +11 −0 .kokoro/samples/python3.7/periodic-head.cfg
  8. +11 −0 .kokoro/samples/python3.8/periodic-head.cfg
  9. +28 −0 .kokoro/test-samples-against-head.sh
  10. +102 −0 .kokoro/test-samples-impl.sh
  11. +16 −80 .kokoro/test-samples.sh
  12. +1 −1 .pre-commit-config.yaml
  13. +1 −0 .trampolinerc
  14. +18 −4 CONTRIBUTING.rst
  15. +2 −2 MANIFEST.in
  16. +3 −3 UPGRADING.md
  17. +11 −0 docs/dataproc_v1/autoscaling_policy_service.rst
  18. +11 −0 docs/dataproc_v1/cluster_controller.rst
  19. +11 −0 docs/dataproc_v1/job_controller.rst
  20. +6 −12 docs/dataproc_v1/services.rst
  21. +1 −0 docs/dataproc_v1/types.rst
  22. +11 −0 docs/dataproc_v1/workflow_template_service.rst
  23. +11 −0 docs/dataproc_v1beta2/autoscaling_policy_service.rst
  24. +11 −0 docs/dataproc_v1beta2/cluster_controller.rst
  25. +11 −0 docs/dataproc_v1beta2/job_controller.rst
  26. +6 −12 docs/dataproc_v1beta2/services.rst
  27. +1 −0 docs/dataproc_v1beta2/types.rst
  28. +11 −0 docs/dataproc_v1beta2/workflow_template_service.rst
  29. +2 −2 google/cloud/dataproc_v1/__init__.py
  30. +50 −12 google/cloud/dataproc_v1/services/autoscaling_policy_service/async_client.py
  31. +47 −29 google/cloud/dataproc_v1/services/autoscaling_policy_service/client.py
  32. +18 −9 google/cloud/dataproc_v1/services/autoscaling_policy_service/pagers.py
  33. +12 −9 google/cloud/dataproc_v1/services/autoscaling_policy_service/transports/base.py
  34. +48 −64 google/cloud/dataproc_v1/services/autoscaling_policy_service/transports/grpc.py
  35. +52 −68 google/cloud/dataproc_v1/services/autoscaling_policy_service/transports/grpc_asyncio.py
  36. +81 −38 google/cloud/dataproc_v1/services/cluster_controller/async_client.py
  37. +88 −68 google/cloud/dataproc_v1/services/cluster_controller/client.py
  38. +18 −9 google/cloud/dataproc_v1/services/cluster_controller/pagers.py
  39. +15 −9 google/cloud/dataproc_v1/services/cluster_controller/transports/base.py
  40. +49 −65 google/cloud/dataproc_v1/services/cluster_controller/transports/grpc.py
  41. +53 −69 google/cloud/dataproc_v1/services/cluster_controller/transports/grpc_asyncio.py
  42. +67 −17 google/cloud/dataproc_v1/services/job_controller/async_client.py
  43. +72 −46 google/cloud/dataproc_v1/services/job_controller/client.py
  44. +18 −9 google/cloud/dataproc_v1/services/job_controller/pagers.py
  45. +16 −9 google/cloud/dataproc_v1/services/job_controller/transports/base.py
  46. +49 −65 google/cloud/dataproc_v1/services/job_controller/transports/grpc.py
  47. +53 −69 google/cloud/dataproc_v1/services/job_controller/transports/grpc_asyncio.py
  48. +84 −42 google/cloud/dataproc_v1/services/workflow_template_service/async_client.py
  49. +81 −64 google/cloud/dataproc_v1/services/workflow_template_service/client.py
  50. +18 −9 google/cloud/dataproc_v1/services/workflow_template_service/pagers.py
  51. +16 −9 google/cloud/dataproc_v1/services/workflow_template_service/transports/base.py
  52. +49 −65 google/cloud/dataproc_v1/services/workflow_template_service/transports/grpc.py
  53. +53 −69 google/cloud/dataproc_v1/services/workflow_template_service/transports/grpc_asyncio.py
  54. +103 −103 google/cloud/dataproc_v1/types/__init__.py
  55. +9 −9 google/cloud/dataproc_v1/types/autoscaling_policies.py
  56. +42 −42 google/cloud/dataproc_v1/types/clusters.py
  57. +49 −49 google/cloud/dataproc_v1/types/jobs.py
  58. +5 −5 google/cloud/dataproc_v1/types/operations.py
  59. +38 −38 google/cloud/dataproc_v1/types/workflow_templates.py
  60. +2 −2 google/cloud/dataproc_v1beta2/__init__.py
  61. +50 −12 google/cloud/dataproc_v1beta2/services/autoscaling_policy_service/async_client.py
  62. +47 −29 google/cloud/dataproc_v1beta2/services/autoscaling_policy_service/client.py
  63. +18 −9 google/cloud/dataproc_v1beta2/services/autoscaling_policy_service/pagers.py
  64. +12 −9 google/cloud/dataproc_v1beta2/services/autoscaling_policy_service/transports/base.py
  65. +48 −64 google/cloud/dataproc_v1beta2/services/autoscaling_policy_service/transports/grpc.py
  66. +52 −68 google/cloud/dataproc_v1beta2/services/autoscaling_policy_service/transports/grpc_asyncio.py
  67. +89 −48 google/cloud/dataproc_v1beta2/services/cluster_controller/async_client.py
  68. +96 −78 google/cloud/dataproc_v1beta2/services/cluster_controller/client.py
  69. +18 −9 google/cloud/dataproc_v1beta2/services/cluster_controller/pagers.py
  70. +15 −9 google/cloud/dataproc_v1beta2/services/cluster_controller/transports/base.py
  71. +49 −65 google/cloud/dataproc_v1beta2/services/cluster_controller/transports/grpc.py
  72. +53 −69 google/cloud/dataproc_v1beta2/services/cluster_controller/transports/grpc_asyncio.py
  73. +67 −17 google/cloud/dataproc_v1beta2/services/job_controller/async_client.py
  74. +72 −46 google/cloud/dataproc_v1beta2/services/job_controller/client.py
  75. +18 −9 google/cloud/dataproc_v1beta2/services/job_controller/pagers.py
  76. +16 −9 google/cloud/dataproc_v1beta2/services/job_controller/transports/base.py
  77. +49 −65 google/cloud/dataproc_v1beta2/services/job_controller/transports/grpc.py
  78. +53 −69 google/cloud/dataproc_v1beta2/services/job_controller/transports/grpc_asyncio.py
  79. +84 −42 google/cloud/dataproc_v1beta2/services/workflow_template_service/async_client.py
  80. +81 −64 google/cloud/dataproc_v1beta2/services/workflow_template_service/client.py
  81. +18 −9 google/cloud/dataproc_v1beta2/services/workflow_template_service/pagers.py
  82. +16 −9 google/cloud/dataproc_v1beta2/services/workflow_template_service/transports/base.py
  83. +49 −65 google/cloud/dataproc_v1beta2/services/workflow_template_service/transports/grpc.py
  84. +53 −69 google/cloud/dataproc_v1beta2/services/workflow_template_service/transports/grpc_asyncio.py
  85. +107 −107 google/cloud/dataproc_v1beta2/types/__init__.py
  86. +9 −9 google/cloud/dataproc_v1beta2/types/autoscaling_policies.py
  87. +44 −44 google/cloud/dataproc_v1beta2/types/clusters.py
  88. +49 −49 google/cloud/dataproc_v1beta2/types/jobs.py
  89. +5 −5 google/cloud/dataproc_v1beta2/types/operations.py
  90. +42 −42 google/cloud/dataproc_v1beta2/types/workflow_templates.py
  91. +51 −14 noxfile.py
  92. +2 −1 renovate.json
  93. +1 −1 samples/snippets/noxfile.py
  94. +2 −3 setup.py
  95. +6 −190 synth.metadata
  96. +2 −1 synth.py
  97. 0 testing/constraints-3.10.txt
  98. 0 testing/constraints-3.11.txt
  99. +10 −0 testing/constraints-3.6.txt
  100. 0 testing/constraints-3.7.txt
  101. 0 testing/constraints-3.8.txt
  102. 0 testing/constraints-3.9.txt
  103. +15 −0 tests/unit/gapic/dataproc_v1/__init__.py
  104. +228 −81 tests/unit/gapic/dataproc_v1/test_autoscaling_policy_service.py
  105. +231 −81 tests/unit/gapic/dataproc_v1/test_cluster_controller.py
  106. +249 −81 tests/unit/gapic/dataproc_v1/test_job_controller.py
  107. +263 −81 tests/unit/gapic/dataproc_v1/test_workflow_template_service.py
  108. +15 −0 tests/unit/gapic/dataproc_v1beta2/__init__.py
  109. +228 −81 tests/unit/gapic/dataproc_v1beta2/test_autoscaling_policy_service.py
  110. +231 −81 tests/unit/gapic/dataproc_v1beta2/test_cluster_controller.py
  111. +249 −81 tests/unit/gapic/dataproc_v1beta2/test_job_controller.py
  112. +263 −81 tests/unit/gapic/dataproc_v1beta2/test_workflow_template_service.py
@@ -1,27 +1,11 @@
# -*- coding: utf-8 -*-
#
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Generated by synthtool. DO NOT EDIT!
[run]
branch = True

[report]
fail_under = 100
show_missing = True
omit = google/cloud/dataproc/__init__.py
omit =
google/cloud/dataproc/__init__.py
exclude_lines =
# Re-enable the standard pragma
pragma: NO COVER
@@ -31,4 +15,4 @@ exclude_lines =
# This is added at the module level as a safeguard for if someone
# generates the code and tries to run it without pip installing. This
# makes it virtually impossible to test properly.
except pkg_resources.DistributionNotFound
except pkg_resources.DistributionNotFound
@@ -0,0 +1,15 @@
{"allowedCopyrightHolders": ["Google LLC"],
"allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"],
"ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt"],
"sourceFileExtensions": [
"ts",
"js",
"java",
"sh",
"Dockerfile",
"yaml",
"py",
"html",
"txt"
]
}
@@ -50,8 +50,10 @@ docs.metadata

# Virtual environment
env/

# Test logs
coverage.xml
sponge_log.xml
*sponge_log.xml

# System test environment variables.
system_tests/local_test_setup
@@ -15,7 +15,11 @@

set -eo pipefail

cd github/python-dataproc
if [[ -z "${PROJECT_ROOT:-}" ]]; then
PROJECT_ROOT="github/python-dataproc"
fi

cd "${PROJECT_ROOT}"

# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1
@@ -30,16 +34,26 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")

# Remove old nox
python3.6 -m pip uninstall --yes --quiet nox-automation
python3 -m pip uninstall --yes --quiet nox-automation

# Install nox
python3.6 -m pip install --upgrade --quiet nox
python3.6 -m nox --version
python3 -m pip install --upgrade --quiet nox
python3 -m nox --version

# If this is a continuous build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
cleanup() {
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
}
trap cleanup EXIT HUP
fi

# If NOX_SESSION is set, it only runs the specified session,
# otherwise run all the sessions.
if [[ -n "${NOX_SESSION:-}" ]]; then
python3.6 -m nox -s "${NOX_SESSION:-}"
python3 -m nox -s ${NOX_SESSION:-}
else
python3.6 -m nox
python3 -m nox
fi
@@ -15,3 +15,14 @@ env_vars: {
key: "TRAMPOLINE_IMAGE_UPLOAD"
value: "false"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-dataproc/.kokoro/build.sh"
}

# Only run this nox session.
env_vars: {
key: "NOX_SESSION"
value: "docs docfx"
}
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
@@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A customized test runner for samples.
#
# For periodic builds, you can specify this file for testing against head.

# `-e` enables the script to automatically fail when a command fails
# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero
set -eo pipefail
# Enables `**` to include files nested inside sub-folders
shopt -s globstar

cd github/python-dataproc

exec .kokoro/test-samples-impl.sh
@@ -0,0 +1,102 @@
#!/bin/bash
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# `-e` enables the script to automatically fail when a command fails
# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero
set -eo pipefail
# Enables `**` to include files nested inside sub-folders
shopt -s globstar

# Exit early if samples directory doesn't exist
if [ ! -d "./samples" ]; then
echo "No tests run. `./samples` not found"
exit 0
fi

# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1

# Debug: show build environment
env | grep KOKORO

# Install nox
python3.6 -m pip install --upgrade --quiet nox

# Use secrets acessor service account to get secrets
if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
gcloud auth activate-service-account \
--key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \
--project="cloud-devrel-kokoro-resources"
fi

# This script will create 3 files:
# - testing/test-env.sh
# - testing/service-account.json
# - testing/client-secrets.json
./scripts/decrypt-secrets.sh

source ./testing/test-env.sh
export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json

# For cloud-run session, we activate the service account for gcloud sdk.
gcloud auth activate-service-account \
--key-file "${GOOGLE_APPLICATION_CREDENTIALS}"

export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json

echo -e "\n******************** TESTING PROJECTS ********************"

# Switch to 'fail at end' to allow all tests to complete before exiting.
set +e
# Use RTN to return a non-zero value if the test fails.
RTN=0
ROOT=$(pwd)
# Find all requirements.txt in the samples directory (may break on whitespace).
for file in samples/**/requirements.txt; do
cd "$ROOT"
# Navigate to the project folder.
file=$(dirname "$file")
cd "$file"

echo "------------------------------------------------------------"
echo "- testing $file"
echo "------------------------------------------------------------"

# Use nox to execute the tests for the project.
python3.6 -m nox -s "$RUN_TESTS_SESSION"
EXIT=$?

# If this is a periodic build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
fi

if [[ $EXIT -ne 0 ]]; then
RTN=1
echo -e "\n Testing failed: Nox returned a non-zero exit code. \n"
else
echo -e "\n Testing completed.\n"
fi

done
cd "$ROOT"

# Workaround for Kokoro permissions issue: delete secrets
rm testing/{test-env.sh,client-secrets.json,service-account.json}

exit "$RTN"

0 comments on commit 6525f86

Please sign in to comment.