diff --git a/.coveragerc b/.coveragerc index d5e3f1dc..9b612b9b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -17,6 +17,8 @@ # Generated by synthtool. DO NOT EDIT! [run] branch = True +omit = + pybigquery/requirements.py [report] fail_under = 100 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1bbd7878..4f00c7cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,6 @@ repos: hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.1 + rev: 3.9.2 hooks: - id: flake8 diff --git a/docs/multiprocessing.rst b/docs/multiprocessing.rst index 1cb29d4c..536d17b2 100644 --- a/docs/multiprocessing.rst +++ b/docs/multiprocessing.rst @@ -1,7 +1,7 @@ .. note:: - Because this client uses :mod:`grpcio` library, it is safe to + Because this client uses :mod:`grpc` library, it is safe to share instances across threads. In multiprocessing scenarios, the best practice is to create client instances *after* the invocation of - :func:`os.fork` by :class:`multiprocessing.Pool` or + :func:`os.fork` by :class:`multiprocessing.pool.Pool` or :class:`multiprocessing.Process`. diff --git a/noxfile.py b/noxfile.py index fd0f0101..d25f5877 100644 --- a/noxfile.py +++ b/noxfile.py @@ -48,8 +48,8 @@ ] # Error if a python version is missing -nox.options.error_on_missing_interpreters = True nox.options.stop_on_first_error = True +nox.options.error_on_missing_interpreters = True @nox.session(python=DEFAULT_PYTHON_VERSION) @@ -86,7 +86,7 @@ def install_alembic_for_python_38(session, constraints_path): """ install alembic for Python 3.8 unit and system tests - We don't require alembic and most tests should run without it, however + We do not require alembic and most tests should run without it, however - We run some unit tests (Python 3.8) to cover the alembic registration that happens when alembic is installed. @@ -105,6 +105,7 @@ def default(session): CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) session.install("mock", "pytest", "pytest-cov", "-c", constraints_path) + install_alembic_for_python_38(session, constraints_path) session.install("-e", ".", "-c", constraints_path) @@ -182,30 +183,23 @@ def system(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def compliance(session): - """Run the system test suite.""" + """Run the SQLAlchemy dialect-compliance system tests""" constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) system_test_folder_path = os.path.join("tests", "sqlalchemy_dialect_compliance") - # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping") - # Sanity check: Only run tests if the environment variable is set. if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): session.skip("Credentials must be set via environment variable") - # Install pyopenssl for mTLS testing. if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": session.install("pyopenssl") - # Sanity check: only run tests if found. if not os.path.exists(system_test_folder_path): session.skip("Compliance tests were not found") - # Use pre-release gRPC for system tests. session.install("--pre", "grpcio") - # Install all test dependencies, then install this package into the - # virtualenv's dist-packages. session.install( "mock", "pytest", @@ -249,7 +243,7 @@ def docs(session): """Build the docs for this library.""" session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark") + session.install("sphinx==4.0.1", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( @@ -271,7 +265,9 @@ def docfx(session): """Build the docfx yaml files for this library.""" session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml") + session.install( + "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" + ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( diff --git a/setup.cfg b/setup.cfg index 897c3eff..71e80492 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ universal = 1 [sqla_testing] requirement_cls=pybigquery.requirements:Requirements -profile_file=sqlalchemy_dialect_compliance-profiles.txt +profile_file=.sqlalchemy_dialect_compliance-profiles.txt [tool:pytest] addopts= --tb native -v -r fxX -p no:warnings diff --git a/synth.metadata b/synth.metadata index c5a588f1..c5699156 100644 --- a/synth.metadata +++ b/synth.metadata @@ -3,81 +3,16 @@ { "git": { "name": ".", - "remote": "https://github.com/googleapis/python-bigquery-sqlalchemy.git", - "sha": "0a3151ba2cb5564ebc1bf2c920b4cab210cd73b8" + "remote": "git@github.com:googleapis/python-bigquery-sqlalchemy.git", + "sha": "95b0692e9b83290b5f0bb45f7dd49f9a10bba5e8" + } + }, + { + "git": { + "name": "synthtool", + "remote": "https://github.com/googleapis/synthtool.git", + "sha": "1778de119522ea9c9b18763c2a4577e22a60433c" } } - ], - "generatedFiles": [ - ".coveragerc", - ".flake8", - ".github/CONTRIBUTING.md", - ".github/ISSUE_TEMPLATE/bug_report.md", - ".github/ISSUE_TEMPLATE/feature_request.md", - ".github/ISSUE_TEMPLATE/support_request.md", - ".github/PULL_REQUEST_TEMPLATE.md", - ".github/header-checker-lint.yml", - ".github/release-please.yml", - ".github/snippet-bot.yml", - ".gitignore", - ".kokoro/build.sh", - ".kokoro/continuous/common.cfg", - ".kokoro/continuous/continuous.cfg", - ".kokoro/docker/docs/Dockerfile", - ".kokoro/docker/docs/fetch_gpg_keys.sh", - ".kokoro/docs/common.cfg", - ".kokoro/docs/docs-presubmit.cfg", - ".kokoro/docs/docs.cfg", - ".kokoro/populate-secrets.sh", - ".kokoro/presubmit/common.cfg", - ".kokoro/presubmit/presubmit.cfg", - ".kokoro/publish-docs.sh", - ".kokoro/release.sh", - ".kokoro/release/common.cfg", - ".kokoro/release/release.cfg", - ".kokoro/samples/lint/common.cfg", - ".kokoro/samples/lint/continuous.cfg", - ".kokoro/samples/lint/periodic.cfg", - ".kokoro/samples/lint/presubmit.cfg", - ".kokoro/samples/python3.6/common.cfg", - ".kokoro/samples/python3.6/continuous.cfg", - ".kokoro/samples/python3.6/periodic-head.cfg", - ".kokoro/samples/python3.6/periodic.cfg", - ".kokoro/samples/python3.6/presubmit.cfg", - ".kokoro/samples/python3.7/common.cfg", - ".kokoro/samples/python3.7/continuous.cfg", - ".kokoro/samples/python3.7/periodic-head.cfg", - ".kokoro/samples/python3.7/periodic.cfg", - ".kokoro/samples/python3.7/presubmit.cfg", - ".kokoro/samples/python3.8/common.cfg", - ".kokoro/samples/python3.8/continuous.cfg", - ".kokoro/samples/python3.8/periodic-head.cfg", - ".kokoro/samples/python3.8/periodic.cfg", - ".kokoro/samples/python3.8/presubmit.cfg", - ".kokoro/test-samples-against-head.sh", - ".kokoro/test-samples-impl.sh", - ".kokoro/test-samples.sh", - ".kokoro/trampoline.sh", - ".kokoro/trampoline_v2.sh", - ".pre-commit-config.yaml", - ".trampolinerc", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.rst", - "MANIFEST.in", - "docs/_static/custom.css", - "docs/_templates/layout.html", - "docs/conf.py", - "docs/multiprocessing.rst", - "noxfile.py", - "renovate.json", - "scripts/decrypt-secrets.sh", - "scripts/readme-gen/readme_gen.py", - "scripts/readme-gen/templates/README.tmpl.rst", - "scripts/readme-gen/templates/auth.tmpl.rst", - "scripts/readme-gen/templates/auth_api_key.tmpl.rst", - "scripts/readme-gen/templates/install_deps.tmpl.rst", - "scripts/readme-gen/templates/install_portaudio.tmpl.rst", - "setup.cfg", - "testing/.gitignore" ] } \ No newline at end of file diff --git a/synth.py b/synth.py index 0cd34c15..9f3ddc6f 100644 --- a/synth.py +++ b/synth.py @@ -29,8 +29,8 @@ # ---------------------------------------------------------------------------- templated_files = common.py_library( unit_test_python_versions=["3.6", "3.7", "3.8", "3.9"], - system_test_python_versions=["3.8"], - cov_level=50 + system_test_python_versions=["3.8", "3.9"], + cov_level=100 ) s.move(templated_files, excludes=[ # pybigquery was originally licensed MIT @@ -41,6 +41,12 @@ # Fixup files # ---------------------------------------------------------------------------- +s.replace( + [".coveragerc"], + "google/cloud/__init__.py", + "pybigquery/requirements.py", + ) + s.replace( ["noxfile.py"], r"[\"']google[\"']", @@ -51,6 +57,132 @@ ["noxfile.py"], "google/cloud", "pybigquery", ) +def place_before(path, text, *before_text, escape=None): + replacement = "\n".join(before_text) + "\n" + text + if escape: + for c in escape: + text = text.replace(c, '\\' + c) + s.replace([path], text, replacement) + +place_before( + "noxfile.py", + "SYSTEM_TEST_PYTHON_VERSIONS=", + "", + "# We're using two Python versions to test with sqlalchemy 1.3 and 1.4.", +) + +place_before( + "noxfile.py", + "nox.options.error_on_missing_interpreters = True", + "nox.options.stop_on_first_error = True", +) + +install_alembic_for_python_38 = ''' +def install_alembic_for_python_38(session, constraints_path): + """ + install alembic for Python 3.8 unit and system tests + + We do not require alembic and most tests should run without it, however + + - We run some unit tests (Python 3.8) to cover the alembic + registration that happens when alembic is installed. + + - We have a system test that demonstrates working with alembic and + proves that the things we think should work do work. :) + """ + if session.python == "3.8": + session.install("alembic", "-c", constraints_path) + + +''' + +place_before( + "noxfile.py", + "def default", + install_alembic_for_python_38, + ) + +place_before( + "noxfile.py", + ' session.install("-e", ".", ', + " install_alembic_for_python_38(session, constraints_path)", + escape='(') + +old_sessions = ''' + "unit", + "system", + "cover", + "lint", +''' + +new_sessions = ''' + "lint", + "unit", + "cover", + "system", + "compliance", +''' + +s.replace( ["noxfile.py"], old_sessions, new_sessions) + +compliance = ''' +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def compliance(session): + """Run the SQLAlchemy dialect-compliance system tests""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_folder_path = os.path.join("tests", "sqlalchemy_dialect_compliance") + + if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false": + session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping") + if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): + session.skip("Credentials must be set via environment variable") + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + if not os.path.exists(system_test_folder_path): + session.skip("Compliance tests were not found") + + session.install("--pre", "grpcio") + + session.install( + "mock", + "pytest", + "pytest-rerunfailures", + "google-cloud-testutils", + "-c", + constraints_path, + ) + session.install("-e", ".", "-c", constraints_path) + + session.run( + "py.test", + "-vv", + f"--junitxml=compliance_{session.python}_sponge_log.xml", + "--reruns=3", + "--reruns-delay=60", + "--only-rerun=403 Exceeded rate limits", + "--only-rerun=409 Already Exists", + "--only-rerun=404 Not found", + "--only-rerun=400 Cannot execute DML over a non-existent table", + system_test_folder_path, + *session.posargs, + ) + + +''' + +place_before( + "noxfile.py", + "@nox.session(python=DEFAULT_PYTHON_VERSION)\n" + "def cover(session):", + compliance, + escape="()", + ) + + + + # Add DB config for SQLAlchemy dialect test suite. # https://github.com/sqlalchemy/sqlalchemy/blob/master/README.dialects.rst # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/89 @@ -61,11 +193,11 @@ [sqla_testing] requirement_cls=pybigquery.requirements:Requirements -profile_file=.profiles.txt +profile_file=.sqlalchemy_dialect_compliance-profiles.txt -[db] -default=bigquery:// -bigquery=bigquery:// +[tool:pytest] +addopts= --tb native -v -r fxX -p no:warnings +python_files=tests/*test_*.py """ )