From 95b0692e9b83290b5f0bb45f7dd49f9a10bba5e8 Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Wed, 26 May 2021 04:21:08 +0000 Subject: [PATCH 1/2] changes without context autosynth cannot find the source of changes triggered by earlier changes in this repository, or by version upgrades to tools such as linters. --- .coveragerc | 4 +- .pre-commit-config.yaml | 2 +- docs/multiprocessing.rst | 4 +- noxfile.py | 88 ++++------------------------------------ setup.cfg | 8 ++-- synth.metadata | 2 +- 6 files changed, 20 insertions(+), 88 deletions(-) diff --git a/.coveragerc b/.coveragerc index d5e3f1dc..0d8e6297 100644 --- a/.coveragerc +++ b/.coveragerc @@ -17,6 +17,8 @@ # Generated by synthtool. DO NOT EDIT! [run] branch = True +omit = + google/cloud/__init__.py [report] fail_under = 100 @@ -33,4 +35,4 @@ omit = */proto/*.py */core/*.py */site-packages/*.py - pybigquery/requirements.py + google/cloud/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1bbd7878..4f00c7cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,6 @@ repos: hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.1 + rev: 3.9.2 hooks: - id: flake8 diff --git a/docs/multiprocessing.rst b/docs/multiprocessing.rst index 1cb29d4c..536d17b2 100644 --- a/docs/multiprocessing.rst +++ b/docs/multiprocessing.rst @@ -1,7 +1,7 @@ .. note:: - Because this client uses :mod:`grpcio` library, it is safe to + Because this client uses :mod:`grpc` library, it is safe to share instances across threads. In multiprocessing scenarios, the best practice is to create client instances *after* the invocation of - :func:`os.fork` by :class:`multiprocessing.Pool` or + :func:`os.fork` by :class:`multiprocessing.pool.Pool` or :class:`multiprocessing.Process`. diff --git a/noxfile.py b/noxfile.py index fd0f0101..16daec35 100644 --- a/noxfile.py +++ b/noxfile.py @@ -28,20 +28,17 @@ BLACK_PATHS = ["docs", "pybigquery", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.8" - -# We're using two Python versions to test with sqlalchemy 1.3 and 1.4. -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ - "lint", "unit", - "cover", "system", - "compliance", + "cover", + "lint", "lint_setup_py", "blacken", "docs", @@ -49,7 +46,6 @@ # Error if a python version is missing nox.options.error_on_missing_interpreters = True -nox.options.stop_on_first_error = True @nox.session(python=DEFAULT_PYTHON_VERSION) @@ -82,22 +78,6 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -def install_alembic_for_python_38(session, constraints_path): - """ - install alembic for Python 3.8 unit and system tests - - We don't require alembic and most tests should run without it, however - - - We run some unit tests (Python 3.8) to cover the alembic - registration that happens when alembic is installed. - - - We have a system test that demonstrates working with alembic and - proves that the things we think should work do work. :) - """ - if session.python == "3.8": - session.install("alembic", "-c", constraints_path) - - def default(session): # Install all test dependencies, then install this package in-place. @@ -105,7 +85,7 @@ def default(session): CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) session.install("mock", "pytest", "pytest-cov", "-c", constraints_path) - install_alembic_for_python_38(session, constraints_path) + session.install("-e", ".", "-c", constraints_path) # Run py.test against the unit tests. @@ -158,7 +138,6 @@ def system(session): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) - install_alembic_for_python_38(session, constraints_path) session.install("-e", ".", "-c", constraints_path) # Run py.test against the system tests. @@ -180,57 +159,6 @@ def system(session): ) -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -def compliance(session): - """Run the system test suite.""" - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - system_test_folder_path = os.path.join("tests", "sqlalchemy_dialect_compliance") - - # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. - if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": - session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping") - # Sanity check: Only run tests if the environment variable is set. - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip("Credentials must be set via environment variable") - # Install pyopenssl for mTLS testing. - if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": - session.install("pyopenssl") - # Sanity check: only run tests if found. - if not os.path.exists(system_test_folder_path): - session.skip("Compliance tests were not found") - - # Use pre-release gRPC for system tests. - session.install("--pre", "grpcio") - - # Install all test dependencies, then install this package into the - # virtualenv's dist-packages. - session.install( - "mock", - "pytest", - "pytest-rerunfailures", - "google-cloud-testutils", - "-c", - constraints_path, - ) - session.install("-e", ".", "-c", constraints_path) - - session.run( - "py.test", - "-vv", - f"--junitxml=compliance_{session.python}_sponge_log.xml", - "--reruns=3", - "--reruns-delay=60", - "--only-rerun=403 Exceeded rate limits", - "--only-rerun=409 Already Exists", - "--only-rerun=404 Not found", - "--only-rerun=400 Cannot execute DML over a non-existent table", - system_test_folder_path, - *session.posargs, - ) - - @nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): """Run the final coverage report. @@ -239,7 +167,7 @@ def cover(session): test runs (not system test runs), and then erases coverage data. """ session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=100") + session.run("coverage", "report", "--show-missing", "--fail-under=50") session.run("coverage", "erase") @@ -249,7 +177,7 @@ def docs(session): """Build the docs for this library.""" session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark") + session.install("sphinx==4.0.1", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( @@ -271,7 +199,9 @@ def docfx(session): """Build the docfx yaml files for this library.""" session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml") + session.install( + "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" + ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( diff --git a/setup.cfg b/setup.cfg index 897c3eff..95ac0e28 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,8 +20,8 @@ universal = 1 [sqla_testing] requirement_cls=pybigquery.requirements:Requirements -profile_file=sqlalchemy_dialect_compliance-profiles.txt +profile_file=.profiles.txt -[tool:pytest] -addopts= --tb native -v -r fxX -p no:warnings -python_files=tests/*test_*.py +[db] +default=bigquery:// +bigquery=bigquery:// diff --git a/synth.metadata b/synth.metadata index c5a588f1..3ac6f7e1 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-sqlalchemy.git", - "sha": "0a3151ba2cb5564ebc1bf2c920b4cab210cd73b8" + "sha": "af4ba179fbde3f176404ebef6443ea37622a6800" } } ], From 01bf5b02748cce007d5cf92b20caaf905d1ed35c Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 27 May 2021 10:16:00 -0600 Subject: [PATCH 2/2] Updated synth.py to recover some prevous manual changes. --- .coveragerc | 4 +- noxfile.py | 74 +++++++++++++++++++++++-- setup.cfg | 8 +-- synth.metadata | 83 ++++------------------------ synth.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 223 insertions(+), 90 deletions(-) diff --git a/.coveragerc b/.coveragerc index 0d8e6297..9b612b9b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -18,7 +18,7 @@ [run] branch = True omit = - google/cloud/__init__.py + pybigquery/requirements.py [report] fail_under = 100 @@ -35,4 +35,4 @@ omit = */proto/*.py */core/*.py */site-packages/*.py - google/cloud/__init__.py + pybigquery/requirements.py diff --git a/noxfile.py b/noxfile.py index 16daec35..d25f5877 100644 --- a/noxfile.py +++ b/noxfile.py @@ -28,23 +28,27 @@ BLACK_PATHS = ["docs", "pybigquery", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] + +# We're using two Python versions to test with sqlalchemy 1.3 and 1.4. +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9"] UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ + "lint", "unit", - "system", "cover", - "lint", + "system", + "compliance", "lint_setup_py", "blacken", "docs", ] # Error if a python version is missing +nox.options.stop_on_first_error = True nox.options.error_on_missing_interpreters = True @@ -78,6 +82,22 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") +def install_alembic_for_python_38(session, constraints_path): + """ + install alembic for Python 3.8 unit and system tests + + We do not require alembic and most tests should run without it, however + + - We run some unit tests (Python 3.8) to cover the alembic + registration that happens when alembic is installed. + + - We have a system test that demonstrates working with alembic and + proves that the things we think should work do work. :) + """ + if session.python == "3.8": + session.install("alembic", "-c", constraints_path) + + def default(session): # Install all test dependencies, then install this package in-place. @@ -86,6 +106,7 @@ def default(session): ) session.install("mock", "pytest", "pytest-cov", "-c", constraints_path) + install_alembic_for_python_38(session, constraints_path) session.install("-e", ".", "-c", constraints_path) # Run py.test against the unit tests. @@ -138,6 +159,7 @@ def system(session): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) + install_alembic_for_python_38(session, constraints_path) session.install("-e", ".", "-c", constraints_path) # Run py.test against the system tests. @@ -159,6 +181,50 @@ def system(session): ) +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def compliance(session): + """Run the SQLAlchemy dialect-compliance system tests""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_folder_path = os.path.join("tests", "sqlalchemy_dialect_compliance") + + if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": + session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping") + if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): + session.skip("Credentials must be set via environment variable") + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + if not os.path.exists(system_test_folder_path): + session.skip("Compliance tests were not found") + + session.install("--pre", "grpcio") + + session.install( + "mock", + "pytest", + "pytest-rerunfailures", + "google-cloud-testutils", + "-c", + constraints_path, + ) + session.install("-e", ".", "-c", constraints_path) + + session.run( + "py.test", + "-vv", + f"--junitxml=compliance_{session.python}_sponge_log.xml", + "--reruns=3", + "--reruns-delay=60", + "--only-rerun=403 Exceeded rate limits", + "--only-rerun=409 Already Exists", + "--only-rerun=404 Not found", + "--only-rerun=400 Cannot execute DML over a non-existent table", + system_test_folder_path, + *session.posargs, + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): """Run the final coverage report. @@ -167,7 +233,7 @@ def cover(session): test runs (not system test runs), and then erases coverage data. """ session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=50") + session.run("coverage", "report", "--show-missing", "--fail-under=100") session.run("coverage", "erase") diff --git a/setup.cfg b/setup.cfg index 95ac0e28..71e80492 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,8 +20,8 @@ universal = 1 [sqla_testing] requirement_cls=pybigquery.requirements:Requirements -profile_file=.profiles.txt +profile_file=.sqlalchemy_dialect_compliance-profiles.txt -[db] -default=bigquery:// -bigquery=bigquery:// +[tool:pytest] +addopts= --tb native -v -r fxX -p no:warnings +python_files=tests/*test_*.py diff --git a/synth.metadata b/synth.metadata index 3ac6f7e1..c5699156 100644 --- a/synth.metadata +++ b/synth.metadata @@ -3,81 +3,16 @@ { "git": { "name": ".", - "remote": "https://github.com/googleapis/python-bigquery-sqlalchemy.git", - "sha": "af4ba179fbde3f176404ebef6443ea37622a6800" + "remote": "git@github.com:googleapis/python-bigquery-sqlalchemy.git", + "sha": "95b0692e9b83290b5f0bb45f7dd49f9a10bba5e8" + } + }, + { + "git": { + "name": "synthtool", + "remote": "https://github.com/googleapis/synthtool.git", + "sha": "1778de119522ea9c9b18763c2a4577e22a60433c" } } - ], - "generatedFiles": [ - ".coveragerc", - ".flake8", - ".github/CONTRIBUTING.md", - ".github/ISSUE_TEMPLATE/bug_report.md", - ".github/ISSUE_TEMPLATE/feature_request.md", - ".github/ISSUE_TEMPLATE/support_request.md", - ".github/PULL_REQUEST_TEMPLATE.md", - ".github/header-checker-lint.yml", - ".github/release-please.yml", - ".github/snippet-bot.yml", - ".gitignore", - ".kokoro/build.sh", - ".kokoro/continuous/common.cfg", - ".kokoro/continuous/continuous.cfg", - ".kokoro/docker/docs/Dockerfile", - ".kokoro/docker/docs/fetch_gpg_keys.sh", - ".kokoro/docs/common.cfg", - ".kokoro/docs/docs-presubmit.cfg", - ".kokoro/docs/docs.cfg", - ".kokoro/populate-secrets.sh", - ".kokoro/presubmit/common.cfg", - ".kokoro/presubmit/presubmit.cfg", - ".kokoro/publish-docs.sh", - ".kokoro/release.sh", - ".kokoro/release/common.cfg", - ".kokoro/release/release.cfg", - ".kokoro/samples/lint/common.cfg", - ".kokoro/samples/lint/continuous.cfg", - ".kokoro/samples/lint/periodic.cfg", - ".kokoro/samples/lint/presubmit.cfg", - ".kokoro/samples/python3.6/common.cfg", - ".kokoro/samples/python3.6/continuous.cfg", - ".kokoro/samples/python3.6/periodic-head.cfg", - ".kokoro/samples/python3.6/periodic.cfg", - ".kokoro/samples/python3.6/presubmit.cfg", - ".kokoro/samples/python3.7/common.cfg", - ".kokoro/samples/python3.7/continuous.cfg", - ".kokoro/samples/python3.7/periodic-head.cfg", - ".kokoro/samples/python3.7/periodic.cfg", - ".kokoro/samples/python3.7/presubmit.cfg", - ".kokoro/samples/python3.8/common.cfg", - ".kokoro/samples/python3.8/continuous.cfg", - ".kokoro/samples/python3.8/periodic-head.cfg", - ".kokoro/samples/python3.8/periodic.cfg", - ".kokoro/samples/python3.8/presubmit.cfg", - ".kokoro/test-samples-against-head.sh", - ".kokoro/test-samples-impl.sh", - ".kokoro/test-samples.sh", - ".kokoro/trampoline.sh", - ".kokoro/trampoline_v2.sh", - ".pre-commit-config.yaml", - ".trampolinerc", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.rst", - "MANIFEST.in", - "docs/_static/custom.css", - "docs/_templates/layout.html", - "docs/conf.py", - "docs/multiprocessing.rst", - "noxfile.py", - "renovate.json", - "scripts/decrypt-secrets.sh", - "scripts/readme-gen/readme_gen.py", - "scripts/readme-gen/templates/README.tmpl.rst", - "scripts/readme-gen/templates/auth.tmpl.rst", - "scripts/readme-gen/templates/auth_api_key.tmpl.rst", - "scripts/readme-gen/templates/install_deps.tmpl.rst", - "scripts/readme-gen/templates/install_portaudio.tmpl.rst", - "setup.cfg", - "testing/.gitignore" ] } \ No newline at end of file diff --git a/synth.py b/synth.py index 0cd34c15..9f3ddc6f 100644 --- a/synth.py +++ b/synth.py @@ -29,8 +29,8 @@ # ---------------------------------------------------------------------------- templated_files = common.py_library( unit_test_python_versions=["3.6", "3.7", "3.8", "3.9"], - system_test_python_versions=["3.8"], - cov_level=50 + system_test_python_versions=["3.8", "3.9"], + cov_level=100 ) s.move(templated_files, excludes=[ # pybigquery was originally licensed MIT @@ -41,6 +41,12 @@ # Fixup files # ---------------------------------------------------------------------------- +s.replace( + [".coveragerc"], + "google/cloud/__init__.py", + "pybigquery/requirements.py", + ) + s.replace( ["noxfile.py"], r"[\"']google[\"']", @@ -51,6 +57,132 @@ ["noxfile.py"], "google/cloud", "pybigquery", ) +def place_before(path, text, *before_text, escape=None): + replacement = "\n".join(before_text) + "\n" + text + if escape: + for c in escape: + text = text.replace(c, '\\' + c) + s.replace([path], text, replacement) + +place_before( + "noxfile.py", + "SYSTEM_TEST_PYTHON_VERSIONS=", + "", + "# We're using two Python versions to test with sqlalchemy 1.3 and 1.4.", +) + +place_before( + "noxfile.py", + "nox.options.error_on_missing_interpreters = True", + "nox.options.stop_on_first_error = True", +) + +install_alembic_for_python_38 = ''' +def install_alembic_for_python_38(session, constraints_path): + """ + install alembic for Python 3.8 unit and system tests + + We do not require alembic and most tests should run without it, however + + - We run some unit tests (Python 3.8) to cover the alembic + registration that happens when alembic is installed. + + - We have a system test that demonstrates working with alembic and + proves that the things we think should work do work. :) + """ + if session.python == "3.8": + session.install("alembic", "-c", constraints_path) + + +''' + +place_before( + "noxfile.py", + "def default", + install_alembic_for_python_38, + ) + +place_before( + "noxfile.py", + ' session.install("-e", ".", ', + " install_alembic_for_python_38(session, constraints_path)", + escape='(') + +old_sessions = ''' + "unit", + "system", + "cover", + "lint", +''' + +new_sessions = ''' + "lint", + "unit", + "cover", + "system", + "compliance", +''' + +s.replace( ["noxfile.py"], old_sessions, new_sessions) + +compliance = ''' +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def compliance(session): + """Run the SQLAlchemy dialect-compliance system tests""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_folder_path = os.path.join("tests", "sqlalchemy_dialect_compliance") + + if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": + session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping") + if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): + session.skip("Credentials must be set via environment variable") + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + if not os.path.exists(system_test_folder_path): + session.skip("Compliance tests were not found") + + session.install("--pre", "grpcio") + + session.install( + "mock", + "pytest", + "pytest-rerunfailures", + "google-cloud-testutils", + "-c", + constraints_path, + ) + session.install("-e", ".", "-c", constraints_path) + + session.run( + "py.test", + "-vv", + f"--junitxml=compliance_{session.python}_sponge_log.xml", + "--reruns=3", + "--reruns-delay=60", + "--only-rerun=403 Exceeded rate limits", + "--only-rerun=409 Already Exists", + "--only-rerun=404 Not found", + "--only-rerun=400 Cannot execute DML over a non-existent table", + system_test_folder_path, + *session.posargs, + ) + + +''' + +place_before( + "noxfile.py", + "@nox.session(python=DEFAULT_PYTHON_VERSION)\n" + "def cover(session):", + compliance, + escape="()", + ) + + + + # Add DB config for SQLAlchemy dialect test suite. # https://github.com/sqlalchemy/sqlalchemy/blob/master/README.dialects.rst # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/89 @@ -61,11 +193,11 @@ [sqla_testing] requirement_cls=pybigquery.requirements:Requirements -profile_file=.profiles.txt +profile_file=.sqlalchemy_dialect_compliance-profiles.txt -[db] -default=bigquery:// -bigquery=bigquery:// +[tool:pytest] +addopts= --tb native -v -r fxX -p no:warnings +python_files=tests/*test_*.py """ )