Skip to content
Permalink
Browse files
feat: add v1beta3 (#34)
This PR was generated using Autosynth. 🌈

Synth log will be available here:
https://source.cloud.google.com/results/invocations/1b172ea4-6e2b-49f2-b1b9-cadea69c5d6e/targets

- [ ] To automatically regenerate this PR, check this box.

PiperOrigin-RevId: 324046932
Source-Link: googleapis/googleapis@26e80fe
PiperOrigin-RevId: 324035013
Source-Link: googleapis/googleapis@8cda089
PiperOrigin-RevId: 323630732
Source-Link: googleapis/googleapis@74e8cf8
PiperOrigin-RevId: 322700241
Source-Link: googleapis/googleapis@a1c9dba
PiperOrigin-RevId: 322659285
Source-Link: googleapis/googleapis@5d0eacc
PiperOrigin-RevId: 322174882
Source-Link: googleapis/googleapis@4ceb9b6
PiperOrigin-RevId: 321807443
Source-Link: googleapis/googleapis@abb30a0
PiperOrigin-RevId: 320253786
Source-Link: googleapis/googleapis@ad486bc
PiperOrigin-RevId: 319910951
Source-Link: googleapis/googleapis@8c85e06
  • Loading branch information
yoshi-automation committed Sep 30, 2020
1 parent 78959f3 commit 6145da3d5a5032f9df59ea2a499dccbf24809841
Showing with 7,376 additions and 648 deletions.
  1. +1 −1 .flake8
  2. 0 .github/snippet-bot.yml
  3. +2 −1 .gitignore
  4. +7 −1 .kokoro/build.sh
  5. +98 −0 .kokoro/docker/docs/Dockerfile
  6. +45 −0 .kokoro/docker/docs/fetch_gpg_keys.sh
  7. +19 −2 .kokoro/docs/common.cfg
  8. +17 −0 .kokoro/docs/docs-presubmit.cfg
  9. +43 −0 .kokoro/populate-secrets.sh
  10. +24 −15 .kokoro/publish-docs.sh
  11. +13 −37 .kokoro/release/common.cfg
  12. +10 −5 .kokoro/trampoline.sh
  13. +487 −0 .kokoro/trampoline_v2.sh
  14. +51 −0 .trampolinerc
  15. +0 −19 CONTRIBUTING.rst
  16. +2 −2 docs/_templates/layout.html
  17. +13 −3 docs/conf.py
  18. +6 −0 docs/documentai_v1beta3/services.rst
  19. +5 −0 docs/documentai_v1beta3/types.rst
  20. +2 −0 docs/index.rst
  21. +34 −52 google/cloud/documentai/__init__.py
  22. +46 −21 google/cloud/documentai_v1beta2/services/document_understanding_service/async_client.py
  23. +102 −60 google/cloud/documentai_v1beta2/services/document_understanding_service/client.py
  24. +80 −5 google/cloud/documentai_v1beta2/services/document_understanding_service/transports/base.py
  25. +91 −26 google/cloud/documentai_v1beta2/services/document_understanding_service/transports/grpc.py
  26. +89 −22 google/cloud/documentai_v1beta2/services/document_understanding_service/transports/grpc_asyncio.py
  27. +5 −1 google/cloud/documentai_v1beta2/types/__init__.py
  28. +132 −63 google/cloud/documentai_v1beta2/types/document.py
  29. +48 −20 google/cloud/documentai_v1beta2/types/document_understanding.py
  30. +6 −3 google/cloud/documentai_v1beta2/types/geometry.py
  31. +47 −0 google/cloud/documentai_v1beta3/__init__.py
  32. +2 −0 google/cloud/documentai_v1beta3/py.typed
  33. +16 −0 google/cloud/documentai_v1beta3/services/__init__.py
  34. +24 −0 google/cloud/documentai_v1beta3/services/document_processor_service/__init__.py
  35. +385 −0 google/cloud/documentai_v1beta3/services/document_processor_service/async_client.py
  36. +516 −0 google/cloud/documentai_v1beta3/services/document_processor_service/client.py
  37. +38 −0 google/cloud/documentai_v1beta3/services/document_processor_service/transports/__init__.py
  38. +188 −0 google/cloud/documentai_v1beta3/services/document_processor_service/transports/base.py
  39. +342 −0 google/cloud/documentai_v1beta3/services/document_processor_service/transports/grpc.py
  40. +344 −0 google/cloud/documentai_v1beta3/services/document_processor_service/transports/grpc_asyncio.py
  41. +49 −0 google/cloud/documentai_v1beta3/types/__init__.py
  42. +1,148 −0 google/cloud/documentai_v1beta3/types/document.py
  43. +278 −0 google/cloud/documentai_v1beta3/types/document_processor_service.py
  44. +78 −0 google/cloud/documentai_v1beta3/types/geometry.py
  45. +52 −6 noxfile.py
  46. +1 −0 samples/AUTHORING_GUIDE.md
  47. +1 −0 samples/CONTRIBUTING.md
  48. +14 −1 scripts/decrypt-secrets.sh
  49. +179 −0 scripts/fixup_documentai_v1beta2_keywords.py
  50. +180 −0 scripts/fixup_documentai_v1beta3_keywords.py
  51. +7 −8 synth.metadata
  52. +19 −14 synth.py
  53. +1 −0 tests/unit/gapic/documentai_v1beta2/__init__.py
  54. +534 −260 tests/unit/gapic/documentai_v1beta2/test_document_understanding_service.py
  55. +1 −0 tests/unit/gapic/documentai_v1beta3/__init__.py
  56. +1,454 −0 tests/unit/gapic/documentai_v1beta3/test_document_processor_service.py
@@ -16,7 +16,7 @@

# Generated by synthtool. DO NOT EDIT!
[flake8]
ignore = E203, E266, E501, W503, F401, F841
ignore = E203, E266, E501, W503
exclude =
# Exclude generated code.
**/proto/**
Empty file.
@@ -46,6 +46,7 @@ pip-log.txt
# Built documentation
docs/_build
bigquery/docs/generated
docs.metadata

# Virtual environment
env/
@@ -57,4 +58,4 @@ system_tests/local_test_setup

# Make sure a generated file isn't accidentally committed.
pylintrc
pylintrc.test
pylintrc.test
@@ -36,4 +36,10 @@ python3.6 -m pip uninstall --yes --quiet nox-automation
python3.6 -m pip install --upgrade --quiet nox
python3.6 -m nox --version

python3.6 -m nox
# If NOX_SESSION is set, it only runs the specified session,
# otherwise run all the sessions.
if [[ -n "${NOX_SESSION:-}" ]]; then
python3.6 -m nox -s "${NOX_SESSION:-}"
else
python3.6 -m nox
fi
@@ -0,0 +1,98 @@
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ubuntu:20.04

ENV DEBIAN_FRONTEND noninteractive

# Ensure local Python is preferred over distribution Python.
ENV PATH /usr/local/bin:$PATH

# Install dependencies.
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
apt-transport-https \
build-essential \
ca-certificates \
curl \
dirmngr \
git \
gpg-agent \
graphviz \
libbz2-dev \
libdb5.3-dev \
libexpat1-dev \
libffi-dev \
liblzma-dev \
libreadline-dev \
libsnappy-dev \
libssl-dev \
libsqlite3-dev \
portaudio19-dev \
redis-server \
software-properties-common \
ssh \
sudo \
tcl \
tcl-dev \
tk \
tk-dev \
uuid-dev \
wget \
zlib1g-dev \
&& add-apt-repository universe \
&& apt-get update \
&& apt-get -y install jq \
&& apt-get clean autoclean \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /var/cache/apt/archives/*.deb


COPY fetch_gpg_keys.sh /tmp
# Install the desired versions of Python.
RUN set -ex \
&& export GNUPGHOME="$(mktemp -d)" \
&& echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
&& /tmp/fetch_gpg_keys.sh \
&& for PYTHON_VERSION in 3.7.8 3.8.5; do \
wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
&& wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
&& gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
&& rm -r python-${PYTHON_VERSION}.tar.xz.asc \
&& mkdir -p /usr/src/python-${PYTHON_VERSION} \
&& tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
&& rm python-${PYTHON_VERSION}.tar.xz \
&& cd /usr/src/python-${PYTHON_VERSION} \
&& ./configure \
--enable-shared \
# This works only on Python 2.7 and throws a warning on every other
# version, but seems otherwise harmless.
--enable-unicode=ucs4 \
--with-system-ffi \
--without-ensurepip \
&& make -j$(nproc) \
&& make install \
&& ldconfig \
; done \
&& rm -rf "${GNUPGHOME}" \
&& rm -rf /usr/src/python* \
&& rm -rf ~/.cache/

RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
&& python3.7 /tmp/get-pip.py \
&& python3.8 /tmp/get-pip.py \
&& rm /tmp/get-pip.py

CMD ["python3.7"]
@@ -0,0 +1,45 @@
#!/bin/bash
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A script to fetch gpg keys with retry.
# Avoid jinja parsing the file.
#

function retry {
if [[ "${#}" -le 1 ]]; then
echo "Usage: ${0} retry_count commands.."
exit 1
fi
local retries=${1}
local command="${@:2}"
until [[ "${retries}" -le 0 ]]; do
$command && return 0
if [[ $? -ne 0 ]]; then
echo "command failed, retrying"
((retries--))
fi
done
return 1
}

# 3.6.9, 3.7.5 (Ned Deily)
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D

# 3.8.0 (Łukasz Langa)
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
E3FF2839C048B25C084DEBE9B26995E310250568

#
@@ -11,12 +11,12 @@ action {
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"

# Use the trampoline script to run in docker.
build_file: "python-documentai/.kokoro/trampoline.sh"
build_file: "python-documentai/.kokoro/trampoline_v2.sh"

# Configure the docker image for kokoro-trampoline.
env_vars: {
key: "TRAMPOLINE_IMAGE"
value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
}
env_vars: {
key: "TRAMPOLINE_BUILD_FILE"
@@ -28,6 +28,23 @@ env_vars: {
value: "docs-staging"
}

env_vars: {
key: "V2_STAGING_BUCKET"
value: "docs-staging-v2-staging"
}

# It will upload the docker image after successful builds.
env_vars: {
key: "TRAMPOLINE_IMAGE_UPLOAD"
value: "true"
}

# It will always build the docker image.
env_vars: {
key: "TRAMPOLINE_DOCKERFILE"
value: ".kokoro/docker/docs/Dockerfile"
}

# Fetch the token needed for reporting release status to GitHub
before_action {
fetch_keystore {
@@ -0,0 +1,17 @@
# Format: //devtools/kokoro/config/proto/build.proto

env_vars: {
key: "STAGING_BUCKET"
value: "gcloud-python-test"
}

env_vars: {
key: "V2_STAGING_BUCKET"
value: "gcloud-python-test"
}

# We only upload the image in the main `docs` build.
env_vars: {
key: "TRAMPOLINE_IMAGE_UPLOAD"
value: "false"
}
@@ -0,0 +1,43 @@
#!/bin/bash
# Copyright 2020 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eo pipefail

function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
function msg { println "$*" >&2 ;}
function println { printf '%s\n' "$(now) $*" ;}


# Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
mkdir -p ${SECRET_LOCATION}
for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
do
msg "Retrieving secret ${key}"
docker run --entrypoint=gcloud \
--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
gcr.io/google.com/cloudsdktool/cloud-sdk \
secrets versions access latest \
--project cloud-devrel-kokoro-resources \
--secret ${key} > \
"${SECRET_LOCATION}/${key}"
if [[ $? == 0 ]]; then
msg "Secret written to ${SECRET_LOCATION}/${key}"
else
msg "Error retrieving secret ${key}"
fi
done
@@ -18,26 +18,16 @@ set -eo pipefail
# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1

cd github/python-documentai

# Remove old nox
python3.6 -m pip uninstall --yes --quiet nox-automation
export PATH="${HOME}/.local/bin:${PATH}"

# Install nox
python3.6 -m pip install --upgrade --quiet nox
python3.6 -m nox --version
python3 -m pip install --user --upgrade --quiet nox
python3 -m nox --version

# build docs
nox -s docs

python3 -m pip install gcp-docuploader

# install a json parser
sudo apt-get update
sudo apt-get -y install software-properties-common
sudo add-apt-repository universe
sudo apt-get update
sudo apt-get -y install jq
python3 -m pip install --user gcp-docuploader

# create metadata
python3 -m docuploader create-metadata \
@@ -52,4 +42,23 @@ python3 -m docuploader create-metadata \
cat docs.metadata

# upload docs
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket docs-staging
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"


# docfx yaml files
nox -s docfx

# create metadata.
python3 -m docuploader create-metadata \
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
--version=$(python3 setup.py --version) \
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
--distribution-name=$(python3 setup.py --name) \
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)

cat docs.metadata

# upload docs
python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"
@@ -23,42 +23,18 @@ env_vars: {
value: "github/python-documentai/.kokoro/release.sh"
}

# Fetch the token needed for reporting release status to GitHub
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "yoshi-automation-github-key"
}
}
}

# Fetch PyPI password
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "google_cloud_pypi_password"
}
}
}

# Fetch magictoken to use with Magic Github Proxy
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "releasetool-magictoken"
}
}
# Fetch PyPI password
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "google_cloud_pypi_password"
}
}
}

# Fetch api key to use with Magic Github Proxy
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "magic-github-proxy-api-key"
}
}
}
# Tokens needed to report release status back to GitHub
env_vars: {
key: "SECRET_MANAGER_KEYS"
value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"
}

0 comments on commit 6145da3

Please sign in to comment.