Skip to content
Permalink
Browse files
feat: add documentai v1 (#101)
  • Loading branch information
busunkim96 committed Mar 25, 2021
1 parent 9fd02a6 commit 74fabb5e260ecc27e9cf005502d79590fa7f72e4
Showing with 6,593 additions and 1,185 deletions.
  1. +11 −0 .kokoro/samples/python3.6/periodic-head.cfg
  2. +11 −0 .kokoro/samples/python3.7/periodic-head.cfg
  3. +11 −0 .kokoro/samples/python3.8/periodic-head.cfg
  4. +28 −0 .kokoro/test-samples-against-head.sh
  5. +102 −0 .kokoro/test-samples-impl.sh
  6. +16 −80 .kokoro/test-samples.sh
  7. +1 −1 .pre-commit-config.yaml
  8. +6 −0 docs/documentai_v1/document_processor_service.rst
  9. +6 −0 docs/documentai_v1/services.rst
  10. +7 −0 docs/documentai_v1/types.rst
  11. +2 −0 docs/index.rst
  12. +32 −16 google/cloud/documentai/__init__.py
  13. +63 −0 google/cloud/documentai_v1/__init__.py
  14. +2 −0 google/cloud/documentai_v1/py.typed
  15. +16 −0 google/cloud/documentai_v1/services/__init__.py
  16. +24 −0 google/cloud/documentai_v1/services/document_processor_service/__init__.py
  17. +476 −0 google/cloud/documentai_v1/services/document_processor_service/async_client.py
  18. +631 −0 google/cloud/documentai_v1/services/document_processor_service/client.py
  19. +37 −0 google/cloud/documentai_v1/services/document_processor_service/transports/__init__.py
  20. +191 −0 google/cloud/documentai_v1/services/document_processor_service/transports/base.py
  21. +333 −0 google/cloud/documentai_v1/services/document_processor_service/transports/grpc.py
  22. +341 −0 google/cloud/documentai_v1/services/document_processor_service/transports/grpc_asyncio.py
  23. +66 −0 google/cloud/documentai_v1/types/__init__.py
  24. +1,109 −0 google/cloud/documentai_v1/types/document.py
  25. +136 −0 google/cloud/documentai_v1/types/document_io.py
  26. +309 −0 google/cloud/documentai_v1/types/document_processor_service.py
  27. +78 −0 google/cloud/documentai_v1/types/geometry.py
  28. +32 −6 google/cloud/documentai_v1beta2/services/document_understanding_service/async_client.py
  29. +2 −2 google/cloud/documentai_v1beta2/services/document_understanding_service/client.py
  30. +11 −9 google/cloud/documentai_v1beta2/services/document_understanding_service/transports/base.py
  31. +38 −65 google/cloud/documentai_v1beta2/services/document_understanding_service/transports/grpc.py
  32. +42 −69 google/cloud/documentai_v1beta2/services/document_understanding_service/transports/grpc_asyncio.py
  33. +30 −30 google/cloud/documentai_v1beta2/types/__init__.py
  34. +16 −0 google/cloud/documentai_v1beta3/__init__.py
  35. +33 −2 google/cloud/documentai_v1beta3/services/document_processor_service/async_client.py
  36. +12 −9 google/cloud/documentai_v1beta3/services/document_processor_service/transports/base.py
  37. +38 −65 google/cloud/documentai_v1beta3/services/document_processor_service/transports/grpc.py
  38. +42 −69 google/cloud/documentai_v1beta3/services/document_processor_service/transports/grpc_asyncio.py
  39. +34 −16 google/cloud/documentai_v1beta3/types/__init__.py
  40. +28 −63 google/cloud/documentai_v1beta3/types/document.py
  41. +136 −0 google/cloud/documentai_v1beta3/types/document_io.py
  42. +125 −0 google/cloud/documentai_v1beta3/types/document_processor_service.py
  43. +18 −9 noxfile.py
  44. +2 −1 renovate.json
  45. +0 −180 scripts/fixup_documentai_v1beta2_keywords.py
  46. +0 −181 scripts/fixup_documentai_v1beta3_keywords.py
  47. +0 −178 scripts/fixup_keywords.py
  48. +1 −2 setup.py
  49. +15 −120 synth.metadata
  50. +12 −8 synth.py
  51. 0 testing/constraints-3.10.txt
  52. 0 testing/constraints-3.11.txt
  53. +9 −0 testing/constraints-3.6.txt
  54. 0 testing/constraints-3.7.txt
  55. 0 testing/constraints-3.8.txt
  56. 0 testing/constraints-3.9.txt
  57. +16 −0 tests/unit/gapic/documentai_v1/__init__.py
  58. +1,723 −0 tests/unit/gapic/documentai_v1/test_document_processor_service.py
  59. +15 −0 tests/unit/gapic/documentai_v1beta2/__init__.py
  60. +43 −2 tests/unit/gapic/documentai_v1beta2/test_document_understanding_service.py
  61. +15 −0 tests/unit/gapic/documentai_v1beta3/__init__.py
  62. +60 −2 tests/unit/gapic/documentai_v1beta3/test_document_processor_service.py
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
@@ -0,0 +1,11 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "INSTALL_LIBRARY_FROM_SOURCE"
value: "True"
}

env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
value: "github/python-pubsub/.kokoro/test-samples-against-head.sh"
}
@@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A customized test runner for samples.
#
# For periodic builds, you can specify this file for testing against head.

# `-e` enables the script to automatically fail when a command fails
# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero
set -eo pipefail
# Enables `**` to include files nested inside sub-folders
shopt -s globstar

cd github/python-documentai

exec .kokoro/test-samples-impl.sh
@@ -0,0 +1,102 @@
#!/bin/bash
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# `-e` enables the script to automatically fail when a command fails
# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero
set -eo pipefail
# Enables `**` to include files nested inside sub-folders
shopt -s globstar

# Exit early if samples directory doesn't exist
if [ ! -d "./samples" ]; then
echo "No tests run. `./samples` not found"
exit 0
fi

# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1

# Debug: show build environment
env | grep KOKORO

# Install nox
python3.6 -m pip install --upgrade --quiet nox

# Use secrets acessor service account to get secrets
if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
gcloud auth activate-service-account \
--key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \
--project="cloud-devrel-kokoro-resources"
fi

# This script will create 3 files:
# - testing/test-env.sh
# - testing/service-account.json
# - testing/client-secrets.json
./scripts/decrypt-secrets.sh

source ./testing/test-env.sh
export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json

# For cloud-run session, we activate the service account for gcloud sdk.
gcloud auth activate-service-account \
--key-file "${GOOGLE_APPLICATION_CREDENTIALS}"

export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json

echo -e "\n******************** TESTING PROJECTS ********************"

# Switch to 'fail at end' to allow all tests to complete before exiting.
set +e
# Use RTN to return a non-zero value if the test fails.
RTN=0
ROOT=$(pwd)
# Find all requirements.txt in the samples directory (may break on whitespace).
for file in samples/**/requirements.txt; do
cd "$ROOT"
# Navigate to the project folder.
file=$(dirname "$file")
cd "$file"

echo "------------------------------------------------------------"
echo "- testing $file"
echo "------------------------------------------------------------"

# Use nox to execute the tests for the project.
python3.6 -m nox -s "$RUN_TESTS_SESSION"
EXIT=$?

# If this is a periodic build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
fi

if [[ $EXIT -ne 0 ]]; then
RTN=1
echo -e "\n Testing failed: Nox returned a non-zero exit code. \n"
else
echo -e "\n Testing completed.\n"
fi

done
cd "$ROOT"

# Workaround for Kokoro permissions issue: delete secrets
rm testing/{test-env.sh,client-secrets.json,service-account.json}

exit "$RTN"
@@ -13,6 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# The default test runner for samples.
#
# For periodic builds, we rewinds the repo to the latest release, and
# run test-samples-impl.sh.

# `-e` enables the script to automatically fail when a command fails
# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero
@@ -24,87 +28,19 @@ cd github/python-documentai

# Run periodic samples tests at latest release
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
# preserving the test runner implementation.
cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh"
echo "--- IMPORTANT IMPORTANT IMPORTANT ---"
echo "Now we rewind the repo back to the latest release..."
LATEST_RELEASE=$(git describe --abbrev=0 --tags)
git checkout $LATEST_RELEASE
fi

# Exit early if samples directory doesn't exist
if [ ! -d "./samples" ]; then
echo "No tests run. `./samples` not found"
exit 0
fi

# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1

# Debug: show build environment
env | grep KOKORO

# Install nox
python3.6 -m pip install --upgrade --quiet nox

# Use secrets acessor service account to get secrets
if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
gcloud auth activate-service-account \
--key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \
--project="cloud-devrel-kokoro-resources"
fi

# This script will create 3 files:
# - testing/test-env.sh
# - testing/service-account.json
# - testing/client-secrets.json
./scripts/decrypt-secrets.sh

source ./testing/test-env.sh
export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json

# For cloud-run session, we activate the service account for gcloud sdk.
gcloud auth activate-service-account \
--key-file "${GOOGLE_APPLICATION_CREDENTIALS}"

export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json

echo -e "\n******************** TESTING PROJECTS ********************"

# Switch to 'fail at end' to allow all tests to complete before exiting.
set +e
# Use RTN to return a non-zero value if the test fails.
RTN=0
ROOT=$(pwd)
# Find all requirements.txt in the samples directory (may break on whitespace).
for file in samples/**/requirements.txt; do
cd "$ROOT"
# Navigate to the project folder.
file=$(dirname "$file")
cd "$file"

echo "------------------------------------------------------------"
echo "- testing $file"
echo "------------------------------------------------------------"

# Use nox to execute the tests for the project.
python3.6 -m nox -s "$RUN_TESTS_SESSION"
EXIT=$?

# If this is a periodic build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
echo "The current head is: "
echo $(git rev-parse --verify HEAD)
echo "--- IMPORTANT IMPORTANT IMPORTANT ---"
# move back the test runner implementation if there's no file.
if [ ! -f .kokoro/test-samples-impl.sh ]; then
cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh
fi
fi

if [[ $EXIT -ne 0 ]]; then
RTN=1
echo -e "\n Testing failed: Nox returned a non-zero exit code. \n"
else
echo -e "\n Testing completed.\n"
fi

done
cd "$ROOT"

# Workaround for Kokoro permissions issue: delete secrets
rm testing/{test-env.sh,client-secrets.json,service-account.json}

exit "$RTN"
exec .kokoro/test-samples-impl.sh
@@ -12,6 +12,6 @@ repos:
hooks:
- id: black
- repo: https://gitlab.com/pycqa/flake8
rev: 3.8.4
rev: 3.9.0
hooks:
- id: flake8
@@ -0,0 +1,6 @@
DocumentProcessorService
------------------------------------------

.. automodule:: google.cloud.documentai_v1.services.document_processor_service
:members:
:inherited-members:
@@ -0,0 +1,6 @@
Services for Google Cloud Documentai v1 API
===========================================
.. toctree::
:maxdepth: 2

document_processor_service
@@ -0,0 +1,7 @@
Types for Google Cloud Documentai v1 API
========================================

.. automodule:: google.cloud.documentai_v1.types
:members:
:undoc-members:
:show-inheritance:
@@ -7,6 +7,8 @@ API Reference
.. toctree::
:maxdepth: 2

documentai_v1/services
documentai_v1/types
documentai_v1beta3/services
documentai_v1beta3/types
documentai_v1beta2/services

0 comments on commit 74fabb5

Please sign in to comment.