From c6cc512329714b3c417b4435ee4f68e00ffffc5c Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Wed, 2 Nov 2022 17:33:19 -0700 Subject: [PATCH 1/7] Exclude pyrsistent 0.19.1 Pyrsistent 0.19.1 had a regression in 0.19.1 which was fixed in 0.19.2 The regression was on pmap where inserted elements were unreliably inserted. See https://github.com/tobgu/pyrsistent/issues/263 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 320615cf..a1f37f32 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ "numpy", "pandas", "pyarrow", - "pyrsistent", + "pyrsistent!=0.19.1", ] setup( From 2c85912c431219d45db2dfe54b7cbfbeb127c46c Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Wed, 2 Nov 2022 17:44:19 -0700 Subject: [PATCH 2/7] Update google github actions branch --- .github/workflows/bionic-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bionic-test.yml b/.github/workflows/bionic-test.yml index 10315ae5..20650eeb 100644 --- a/.github/workflows/bionic-test.yml +++ b/.github/workflows/bionic-test.yml @@ -44,7 +44,7 @@ jobs: # build failures. pip freeze - name: Set up gcloud - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@main with: service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true From 1a0c3b9734c5ce48c56d03eb2bab7333e0c742c0 Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Wed, 2 Nov 2022 18:12:13 -0700 Subject: [PATCH 3/7] Update flake8 print code Error code for print was changed from T001 to T201 https://github.com/JBKahn/flake8-print/pull/55 --- bionic/flake8/check_dnode_match.py | 2 +- setup.cfg | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bionic/flake8/check_dnode_match.py b/bionic/flake8/check_dnode_match.py index 712c522c..ff664ca6 100644 --- a/bionic/flake8/check_dnode_match.py +++ b/bionic/flake8/check_dnode_match.py @@ -216,4 +216,4 @@ def run(self): code = Path(filename).read_text() tree = ast.parse(code) for message in Checker(tree).run(): - print(message) # noqa: T001 + print(message) # noqa: T201 diff --git a/setup.cfg b/setup.cfg index 72abe899..22975006 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,8 @@ ignore = T102 # "fixme found (XXX)" per-file-ignores = # Allow print statements in example code. - example/*:T001 + build/lib/example/*:T201 + example/*:T201 # NOTE On my MacBook this plugin adds about 1 extra second to Flake8's runtime, making # it about 5s total. That's not trivial, so it might not be worth it to have this From 3fd362ef46e2e5a6c85e1985cca17f93a50d942d Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Wed, 2 Nov 2022 23:41:07 -0700 Subject: [PATCH 4/7] Pin importlib-metadata < 5 because of flake8 incompatibility See https://github.com/python/importlib_metadata/issues/406 --- bionic/deps/extras.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bionic/deps/extras.py b/bionic/deps/extras.py index ad2595e9..b04dd75d 100644 --- a/bionic/deps/extras.py +++ b/bionic/deps/extras.py @@ -54,6 +54,7 @@ def combine(*dep_lists): "flake8", "flake8-print", "flake8-fixme", + "importlib-metadata<5", # flake8 is incompatible with importlib 5.0.0 "sphinx!=3.2.0", "sphinx_rtd_theme", "sphinx-autobuild", From 117ea5762b84a3f59cbb94705bd717a85072e104 Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Wed, 2 Nov 2022 23:50:06 -0700 Subject: [PATCH 5/7] Apply black linter fixes --- bionic/__init__.py | 2 +- docs/conf.py | 16 ++++++++-------- tests/test_code_hasher.py | 2 +- tests/test_flow/conftest.py | 2 +- tests/test_flow/test_cache_api.py | 4 ++-- tests/test_flow/test_interactions.py | 2 +- tests/test_flow/test_merge.py | 4 ++-- tests/test_flow/test_persistence.py | 14 +++++++------- tests/test_flow/test_persistence_gcs.py | 6 +++--- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/bionic/__init__.py b/bionic/__init__.py index ba6a4658..ae3ff9ca 100644 --- a/bionic/__init__.py +++ b/bionic/__init__.py @@ -19,4 +19,4 @@ from . import protocol # noqa: F401 from . import util # noqa: F401 -__version__ = u"0.11.0" +__version__ = "0.11.0" diff --git a/docs/conf.py b/docs/conf.py index 006f7aaa..3e771dc6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,12 +16,12 @@ # -- Project information ----------------------------------------------------- -project = u"bionic" -copyright = u"2019, Square" -author = u"Janek Klawe" +project = "bionic" +copyright = "2019, Square" +author = "Janek Klawe" # The short X.Y version -version = u"0.11.0" +version = "0.11.0" # The full version, including alpha/beta/rc tags release = version @@ -67,7 +67,7 @@ # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [ - u"_build", + "_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints", @@ -135,7 +135,7 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, "bionic.tex", u"bionic Documentation", u"Janek Klawe", "manual"), + (master_doc, "bionic.tex", "bionic Documentation", "Janek Klawe", "manual"), ] @@ -143,7 +143,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "bionic", u"bionic Documentation", [author], 1)] +man_pages = [(master_doc, "bionic", "bionic Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -155,7 +155,7 @@ ( master_doc, "bionic", - u"bionic Documentation", + "bionic Documentation", author, "bionic", "One line description of project.", diff --git a/tests/test_code_hasher.py b/tests/test_code_hasher.py index b874d9d1..f1507b60 100644 --- a/tests/test_code_hasher.py +++ b/tests/test_code_hasher.py @@ -87,7 +87,7 @@ def dec_with_one(x): return x - one() def quadratic_eq(a, b, c): - d = b ** 2 - 4 * a * c + d = b**2 - 4 * a * c s1 = (b - cmath.sqrt(d)) / (2 * a) s2 = (-b - cmath.sqrt(d)) / (2 * a) return (s1, s2) diff --git a/tests/test_flow/conftest.py b/tests/test_flow/conftest.py index 4d777f7e..2645e366 100644 --- a/tests/test_flow/conftest.py +++ b/tests/test_flow/conftest.py @@ -200,7 +200,7 @@ def real_gcs_session_tmp_url_prefix(real_gcs_url_stem) -> Optional[str]: gcs_fs = get_gcs_fs_without_warnings() - random_hex_str = "%016x" % random.randint(0, 2 ** 64) + random_hex_str = "%016x" % random.randint(0, 2**64) path_str = f"{getpass.getuser()}/BNTESTDATA/{random_hex_str}" gs_url = real_gcs_url_stem + "/" + path_str + "/" diff --git a/tests/test_flow/test_cache_api.py b/tests/test_flow/test_cache_api.py index 12e29af5..ffcaed34 100644 --- a/tests/test_flow/test_cache_api.py +++ b/tests/test_flow/test_cache_api.py @@ -106,7 +106,7 @@ def xy(x, y): @builder def xy_squared(xy): - return xy ** 2 + return xy**2 return builder.build() @@ -134,7 +134,7 @@ def test_get_entries(preset_flow): @builder # noqa: F811 @bn.version(1) def xy(x, y): # noqa: F811 - return x ** y + return x**y tester.flow = builder.build() diff --git a/tests/test_flow/test_interactions.py b/tests/test_flow/test_interactions.py index a6732196..7a2a54ed 100644 --- a/tests/test_flow/test_interactions.py +++ b/tests/test_flow/test_interactions.py @@ -14,7 +14,7 @@ def xs(n): @builder def ys(xs): - return [x ** 2 for x in xs] + return [x**2 for x in xs] return builder diff --git a/tests/test_flow/test_merge.py b/tests/test_flow/test_merge.py index de30a373..cc2dcac4 100644 --- a/tests/test_flow/test_merge.py +++ b/tests/test_flow/test_merge.py @@ -128,7 +128,7 @@ def merge_tester(builder): @fb def x(root_x): - return root_x ** 2 + return root_x**2 tester.add("DerivedSingle", fb.build()) @@ -167,7 +167,7 @@ def x(x_y): # noqa: F811 @fb # noqa: F811 @bn.persist(False) def x(root_x): # noqa: F811 - return root_x ** 2 + return root_x**2 tester.add("DS", fb.build()) diff --git a/tests/test_flow/test_persistence.py b/tests/test_flow/test_persistence.py index 50c423b7..5329c080 100644 --- a/tests/test_flow/test_persistence.py +++ b/tests/test_flow/test_persistence.py @@ -280,7 +280,7 @@ def f(x, y): # noqa: F811 @bn.version_no_warnings(major=1, minor=1) def f(x, y): # noqa: F811 call_counter.mark() - return x ** y + return x**y assert builder.build().get("f") == 6 assert call_counter.times_called() == 0 @@ -291,7 +291,7 @@ def f(x, y): # noqa: F811 @bn.version_no_warnings(major=2) def f(x, y): # noqa: F811 call_counter.mark() - return x ** y + return x**y assert builder.build().get("f") == 8 assert call_counter.times_called() == 1 @@ -435,7 +435,7 @@ def f(x, y): # noqa: F811 @bn.version_no_warnings(major=1, minor=1) def f(x, y): # noqa: F811 call_counter.mark() - return x ** y + return x**y with raises_versioning_error_for_entity("f"): builder.build().get("f") @@ -446,7 +446,7 @@ def f(x, y): # noqa: F811 @bn.version_no_warnings(major=2) def f(x, y): # noqa: F811 call_counter.mark() - return x ** y + return x**y assert builder.build().get("f") == 8 assert call_counter.times_called() == 1 @@ -504,7 +504,7 @@ def f(x, y): # noqa: F811 assert call_counter.times_called() == 0 def op(x, y): # noqa: F811 - return x ** y + return x**y with raises_versioning_error_for_entity("f"): builder.build().get("f") @@ -733,7 +733,7 @@ def f(x, y): # noqa: F811 @bn.version_no_warnings(major=1, minor=1) def f(x, y): # noqa: F811 call_counter.mark() - return x ** y + return x**y assert builder.build().get("f") == 8 assert call_counter.times_called() == 1 @@ -744,7 +744,7 @@ def f(x, y): # noqa: F811 @bn.version_no_warnings(major=2) def f(x, y): # noqa: F811 call_counter.mark() - return x ** y + return x**y assert builder.build().get("f") == 8 assert call_counter.times_called() == 1 diff --git a/tests/test_flow/test_persistence_gcs.py b/tests/test_flow/test_persistence_gcs.py index c7448048..8057be74 100644 --- a/tests/test_flow/test_persistence_gcs.py +++ b/tests/test_flow/test_persistence_gcs.py @@ -176,7 +176,7 @@ def xy(x, y): # noqa: F811 @bn.version_no_warnings(major=1) def xy(x, y): # noqa: F811 call_counter.mark() - return x ** y + return x**y flow = builder.build() @@ -200,7 +200,7 @@ def test_indirect_versioning(preset_gcs_builder, make_counter): @bn.version_no_warnings(major=1) def xy(x, y): call_counter.mark() - return x ** y + return x**y flow = builder.build() assert flow.get("xy") == 8 @@ -240,7 +240,7 @@ def xy(x, y): # noqa: F811 @bn.version_no_warnings(major=2) def xy(x, y): # noqa: F811 call_counter.mark() - return y ** x + return y**x flow = builder.build() From ef2d5b1b66073e3f0cea5c8efc229874fe3fbec7 Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Thu, 3 Nov 2022 16:27:35 -0700 Subject: [PATCH 6/7] Pin decorator < 5 to preserve __code__ object See https://github.com/micheles/decorator/issues/129 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a1f37f32..23ac3b65 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ "pandas", "pyarrow", "pyrsistent!=0.19.1", + "decorator<5", ] setup( From 8ae96ad7315ee78ed785c6d63c5cf46836c32943 Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Fri, 4 Nov 2022 10:13:09 -0700 Subject: [PATCH 7/7] Drop support for python 3.6 --- .github/workflows/bionic-test.yml | 4 +--- bionic/code_hasher.py | 7 +++---- docs/get-started.rst | 2 +- setup.py | 4 ++-- tests/test_flow/test_persistence_compatibility.py | 11 +---------- 5 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/bionic-test.yml b/.github/workflows/bionic-test.yml index 20650eeb..5bc3bf94 100644 --- a/.github/workflows/bionic-test.yml +++ b/.github/workflows/bionic-test.yml @@ -20,10 +20,8 @@ jobs: # version.) fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.7, 3.8] include: - - python-version: 3.6 - shard-id: 0 - python-version: 3.7 shard-id: 1 - python-version: 3.8 diff --git a/bionic/code_hasher.py b/bionic/code_hasher.py index bb906f8f..4528d06e 100644 --- a/bionic/code_hasher.py +++ b/bionic/code_hasher.py @@ -37,9 +37,6 @@ PREFIX_SEPARATOR = b"$" -# This is gross but types.MethodWrapperType was introduced in Python 3.7 -# and does not exist in 3.6 which Bionic supports. -METHOD_WRAPPER_TYPE = type("".__str__) class CodeHasher: @@ -290,7 +287,9 @@ def _update_hash(self, hash_accumulator, obj, code_context): obj_bytes=self._check_and_hash(builtin_name, code_context), ) - elif inspect.ismethoddescriptor(obj) or isinstance(obj, METHOD_WRAPPER_TYPE): + elif inspect.ismethoddescriptor(obj) or isinstance( + obj, types.MethodWrapperType + ): if inspect.ismethoddescriptor(obj): type_prefix = TypePrefix.METHOD_DESCRIPTOR else: diff --git a/docs/get-started.rst b/docs/get-started.rst index 825b2a28..4e83dd5f 100644 --- a/docs/get-started.rst +++ b/docs/get-started.rst @@ -39,7 +39,7 @@ performance for some workloads. LibYAML is also available via Homebrew: brew install libyaml -Bionic supports Python 3.6 and above. +Bionic supports Python 3.7 and above. .. _extra-packages: diff --git a/setup.py b/setup.py index 23ac3b65..a49412b9 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ include_package_data=True, install_requires=requirements, extras_require=extras_require, - python_requires=">=3.6", + python_requires=">=3.7", zip_safe=False, keywords="bionic", classifiers=[ @@ -52,6 +52,6 @@ "Natural Language :: English", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", ], ) diff --git a/tests/test_flow/test_persistence_compatibility.py b/tests/test_flow/test_persistence_compatibility.py index 432321b8..8ab8bf0f 100644 --- a/tests/test_flow/test_persistence_compatibility.py +++ b/tests/test_flow/test_persistence_compatibility.py @@ -1,6 +1,5 @@ import pytest import shutil -import sys from .generate_test_compatibility_cache import Harness, CACHE_TEST_DIR @@ -26,15 +25,7 @@ def older_serialized_cache_harness(make_counter, tmp_path): # To renegerate cache, run the following command from bionic/ dir # `python -m tests.test_flow.generate_test_compatibility_cache` def test_caching_compatibility(older_serialized_cache_harness): - # The auto-versioned flow generates different bytecode hashes on Python 3.6 - # compared to 3.7 and 3.8 This is because Python 3.7 added new - # bytecode instructions like LOAD_METHOD and uses them widely. - # We skip the auto-versioned flow for Python 3.6 and only test it on - # Python 3.7+. - if sys.version_info < (3, 7): - flows = [older_serialized_cache_harness.manual_flow] - else: - flows = older_serialized_cache_harness.flows + flows = older_serialized_cache_harness.flows for flow in flows: assert (