feat(dev): Add Git pre-commit hooks [gapic-generator-python]

.githooks/pre-commit

@@ -0,0 +1,132 @@
+#!/bin/sh
+
+# Pre-commit Git checks.
+# Set up:
+#  ln -s .githooks/pre-commit .git/hooks/pre-commit
+
+# Constants.
+BOLD="\e[1m"
+UNSET="\e[0m"
+WHITE="\e[97m"
+RED="\e[91m"
+BACK_MAGENTA="\e[45m"
+BACK_BLUE="\e[44m"
+BACK_RED="\e[41m"
+BACK_GREEN="\e[42m"
+
+# Methods.
+function echo_error {
+  ERR_MSG=$1
+  HELP_MSG=$2
+  echo -e "$BOLD $BACK_BLUE $WHITE Precommit:\t $BACK_RED Changes NOT committed. $UNSET"
+  echo -e "$BOLD $BACK_BLUE $WHITE Precommit:\t $BACK_RED $WHITE $ERR_MSG $BACK_BLUE $HELP_MSG $UNSET"
+}
+
+function echo_status {
+  STATUS_MSG=$1
+  echo -e "$BOLD $BACK_BLUE $WHITE Precommit:\t $STATUS_MSG $UNSET"
+}
+
+function echo_success {
+  echo -e "$BOLD $BACK_BLUE $WHITE Precommit:\t $BACK_GREEN $WHITE SUCCESS. $UNSET All checks passed!"
+}
+
+function header_check_preparation {
+  echo_status "Setting up license check environment"
+  export GOPATH=$(go env GOPATH)
+  if [ $? -ne 0 ];
+  then
+      echo_status "Please install Go first, instructions can be found here: https://golang.org/doc/install."
+  else
+      export ENV_PATH=$(echo $PATH)
+      if [[ $ENV_PATH != *$GOPATH* ]];
+      then
+        echo_status "GOPATH is not in the system path, adding it now."
+        export PATH=$GOPATH/bin:$PATH
+      fi
+      which addlicense
+      if [ $? -ne 0 ];
+      then
+        echo_status "addlicense tool is not yet installed, downloading it now."
+        go get -u github.com/google/addlicense
+      fi
+  fi
+}
+
+# Disk cache.
+BAZEL_CACHE_DIR=/tmp/bazel_cache_gapic_generator_java
+if [ ! -d $BAZEL_CACHE_DIR ]
+then
+  mkdir $BAZEL_CACHE_DIR
+fi
+
+# Check only the staged files.
+NUM_TOTAL_FILES_CHANGED=$(git diff --cached --name-only | wc -l)
+NUM_PYTHON_FILES_CHANGED=$(git diff --cached --name-only "*.java" | wc -l)
+NUM_UNIT_GOLDEN_FILES_CHANGED=$(git diff --cached --name-only "src/test/*/*.golden" | wc -l)
+NUM_INTEGRATION_GOLDEN_FILES_CHANGED=$(git diff --cached --name-only "tests/integration/goldens/*/*.golden" | wc -l)
+NUM_INTEGRATION_BAZEL_FILES_CHANGED=$(git diff --cached --name-only "tests/integration/*/BUILD.bazel" | wc -l)
+NUM_BAZEL_FILES_CHANGED=$(git diff --cached --name-only "*BUILD.bazel" | wc -l)
+
+if [ $NUM_TOTAL_FILES_CHANGED -le 0 ]
+then
+  echo_error "No new files to commit." ""
+  exit 1
+fi
+
+
+if [ -x /usr/lib/git-core/google_hook ]; then
+  /usr/lib/git-core/google_hook pre-commit "$@"
+fi
+
+# Check Python format.
+if [ $NUM_PYTHON_FILES_CHANGED -gt 0 ]
+then
+  echo_status "Running Python linter..."
+  find gapic tests -name "*.py" -not -path 'tests/integration/goldens/*' | xargs autopep8 --diff --exit-code
+  FORMAT_STATUS=$?
+  if [ $FORMAT_STATUS != 0 ]
+  then
+    echo_error "Linting failed." "Please try again after running autopep8 on the gapic/ and tests/ directories."
+    exit 1
+  fi
+fi
+
+# Check unit tests.
+if [ $NUM_PYTHON_FILES_CHANGED -gt 0 ] || [ $NUM_UNIT_GOLDEN_FILES_CHANGED -gt 0 ]
+then
+  echo_status "Checking unit tests..."
+  nox -s unit-3.9
+  TEST_STATUS=$?
+  if [ $TEST_STATUS != 0 ]
+  then
+    echo_error "Tests failed." "Please fix them and try again."
+    exit 1
+  fi
+fi
+
+# Check integration tests.
+if [ $NUM_PYTHON_FILES_CHANGED -gt 0 ] \
+  || [ $NUM_INTEGRATION_GOLDEN_FILES_CHANGED -gt 0 ] \
+  || [ $NUM_INTEGRATION_BAZEL_FILES_CHANGED -gt 0 ]
+then
+  echo_status "Checking integration tests..."
+  bazel --batch test --disk_cache="$BAZEL_CACHE_DIR" //tests/integration/...
+  TEST_STATUS=$?
+  if [ $TEST_STATUS != 0 ]
+  then
+    echo_error "Tests failed." "Please fix them and try again."
+    exit 1
+  fi
+fi
+
+# Check and fix Bazel format.
+if [ $NUM_BAZEL_FILES_CHANGED -gt 0 ]
+then
+  for FILE in $(find ./ -name BUILD.bazel)
+  do
+    buildifier --lint=fix $FILE
+  done
+fi
+
+echo_success

.github/workflows/tests.yaml

@@ -313,6 +313,34 @@ jobs:
       - name: Submit coverage data to codecov.
         run: codecov
         if: always()
+  integration:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.7.0
+        with:
+          access_token: ${{ github.token }}
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install system dependencies.
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y curl pandoc unzip gcc
+      - name: Install Bazel
+        run: |
+          wget -q "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/$BAZEL_BINARY"
+          wget -q "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/$BAZEL_BINARY.sha256"
+          sha256sum -c "$BAZEL_BINARY.sha256"
+          sudo dpkg -i "$BAZEL_BINARY"
+        env:
+          BAZEL_VERSION: 3.5.0
+          BAZEL_BINARY: bazel_3.5.0-linux-x86_64.deb
+      - name: Integration Tests
+        run: bazel test tests/integration:asset tests/integration:credentials tests/integration:logging tests/integration:redis
+
   style-check:
     runs-on: ubuntu-latest
     steps:
@@ -330,4 +358,4 @@ jobs:
             python -m pip install autopep8
       - name: Check diff
         run: |
-            find gapic tests -name "*.py" | xargs autopep8 --diff --exit-code
+            find gapic tests -name "*.py" -not -path 'tests/integration/goldens/*' | xargs autopep8 --diff --exit-code

DEVELOPMENT.md

@@ -1,7 +1,47 @@
 # Development
 
-- Install dependencies with `pip install .`
-- See all nox sessions with `nox -l`
-- Execute unit tests by running one of the sessions prefixed with `unit-`
-  - Example: `nox -s unit-3.8`
-- Lint sources by running `autopep8`.
+## Setup
+
+1.  Clone this repo.
+
+2.  Copy the Git pre-commit hooks. This will automatically check the build, run
+    tests, and perform linting before each commit. (Symlinks don't seem to work,
+    but if you find a way, please add it here!)
+
+    ```sh
+    cp .githooks/pre-commit .git/hooks/pre-commit
+    ```
+
+3.  Install dependencies with `pip install .`
+
+## Unit Tests
+
+Execute unit tests by running one of the sessions prefixed with `unit-`.
+
+-   Example: `nox -s unit-3.8`
+-   See all Nox sessions with `nox -l`.
+
+## Formatting
+
+-   Lint sources by running `autopep8`. The specific command is the following.
+
+    ```
+    find gapic tests -name "*.py" -not -path 'tests/integration/goldens/*' | xargs autopep8 --diff --exit-code
+    ```
+
+## Integration Tests
+
+-   Run a single integration test for one API. This generates Python source code
+    with the microgenerator and compares them to the golden files in
+    `test/integration/goldens/asset`.
+
+    ```sh
+    bazel test //test/integration:asset
+    ```
+
+-   Update goldens files. This overwrites the golden files in
+    `test/integration/goldens/asset`.
+
+    ```sh
+    bazel run //test/integration:asset_update
+    ```

WORKSPACE

@@ -55,3 +55,11 @@ apple_rules_dependencies()
 load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies")
 
 apple_support_dependencies()
+
+load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
+
+switched_rules_by_language(
+    name = "com_google_googleapis_imports",
+    gapic = True,
+    grpc = True,
+)

repositories.bzl

@@ -61,6 +61,21 @@ def gapic_generator_python():
         urls = ["https://github.com/googleapis/gapic-generator/archive/03abac35ec0716c6f426ffc1532f9a62f1c9e6a2.zip"],
     )
 
+    _rules_gapic_version = "0.5.3"
+    _maybe(
+        http_archive,
+        name = "rules_gapic",
+        strip_prefix = "rules_gapic-%s" % _rules_gapic_version,
+        urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version],
+    )
+
+    _maybe(
+        http_archive,
+        name = "com_google_googleapis",
+        strip_prefix = "googleapis-51fe6432d4076a4c101f561967df4bf1f27818e1",
+        urls = ["https://github.com/googleapis/googleapis/archive/51fe6432d4076a4c101f561967df4bf1f27818e1.zip"],
+    )
+
 def gapic_generator_register_toolchains():
     native.register_toolchains(
         "@gapic_generator_python//:pandoc_toolchain_linux",

rules_python_gapic/py_gapic.bzl

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@com_google_api_codegen//rules_gapic:gapic.bzl", "proto_custom_library")
+load("@rules_gapic//:gapic.bzl", "proto_custom_library")
 
 def py_gapic_library(
         name,
@@ -34,7 +34,7 @@ def py_gapic_library(
 
     file_args = {}
     if grpc_service_config:
-        file_args[grpc_service_config] =  "retry-config"
+        file_args[grpc_service_config] = "retry-config"
 
     proto_custom_library(
         name = srcjar_target_name,

rules_python_gapic/py_gapic_pkg.bzl

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@com_google_api_codegen//rules_gapic:gapic_pkg.bzl", "construct_package_dir_paths")
+load("@rules_gapic//:gapic_pkg.bzl", "construct_package_dir_paths")
 
 def _py_gapic_src_pkg_impl(ctx):
     srcjar_srcs = []
@@ -66,5 +66,3 @@ def py_gapic_assembly_pkg(name, deps, assembly_name = None, **kwargs):
         package_dir = package_dir,
         **kwargs
     )
-
-