From df59309b216ebe58bc5d4d31fe6e95b25c6cfa84 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 5 Feb 2020 09:02:48 -0600 Subject: [PATCH 1/2] Add 'Rule' to unasync for per-directory config --- src/unasync/__init__.py | 166 ++++++++++++------ .../example_custom_pkg/{src => }/__init__.py | 0 tests/data/example_custom_pkg/setup.py | 13 +- .../src/ahip/tests/__init__.py | 0 .../src/ahip/tests/test_conn.py | 8 + tests/test_unasync.py | 11 +- 6 files changed, 141 insertions(+), 57 deletions(-) rename tests/data/example_custom_pkg/{src => }/__init__.py (100%) create mode 100644 tests/data/example_custom_pkg/src/ahip/tests/__init__.py create mode 100644 tests/data/example_custom_pkg/src/ahip/tests/test_conn.py diff --git a/src/unasync/__init__.py b/src/unasync/__init__.py index 6bfd8ce..0aa604a 100644 --- a/src/unasync/__init__.py +++ b/src/unasync/__init__.py @@ -12,7 +12,17 @@ from ._version import __version__ # NOQA -ASYNC_TO_SYNC = { +__all__ = [ + "Rule", + "unasync_file", + "unasync_tokens", + "unasync_name", + "build_py", + "customize_build_py", +] + + +_ASYNC_TO_SYNC = { "__aenter__": "__enter__", "__aexit__": "__exit__", "__aiter__": "__iter__", @@ -27,6 +37,92 @@ "StopAsyncIteration": "StopIteration", } + +class Rule: + """A single set of rules for 'unasync'ing file(s)""" + + def __init__(self, fromdir, todir, additional_replacements=None): + self.fromdir = fromdir.replace("/", os.sep) + self.todir = todir.replace("/", os.sep) + + # Add any additional user-defined token replacements to our list. + self.token_replacements = _ASYNC_TO_SYNC.copy() + for key, val in (additional_replacements or {}).items(): + self.token_replacements[key] = val + + def match(self, filepath): + """Determines if a Rule matches a given filepath and if so + returns a higher comparable value if the match is more specific. + """ + file_segments = [x for x in filepath.split(os.sep) if x] + from_segments = [x for x in self.fromdir.split(os.sep) if x] + len_from_segments = len(from_segments) + + if len_from_segments > len(file_segments): + return False + + for i in range(len(file_segments) - len_from_segments + 1): + if file_segments[i : i + len_from_segments] == from_segments: + return len_from_segments, i + + return False + + def unasync_file(self, filepath): + with open(filepath, "rb") as f: + write_kwargs = {} + if sys.version_info[0] >= 3: + encoding, _ = std_tokenize.detect_encoding(f.readline) + write_kwargs["encoding"] = encoding + f.seek(0) + tokens = tokenize(f) + tokens = self.unasync_tokens(tokens) + result = untokenize(tokens) + outfilepath = filepath.replace(self.fromdir, self.todir) + makedirs_existok(os.path.dirname(outfilepath)) + with open(outfilepath, "w", **write_kwargs) as f: + print(result, file=f, end="") + + def unasync_tokens(self, tokens): + # TODO __await__, ...? + used_space = None + for space, toknum, tokval in tokens: + if tokval in ["async", "await"]: + # When removing async or await, we want to use the whitespace that + # was before async/await before the next token so that + # `print(await stuff)` becomes `print(stuff)` and not + # `print( stuff)` + used_space = space + else: + if toknum == std_tokenize.NAME: + tokval = self.unasync_name(tokval) + elif toknum == std_tokenize.STRING: + left_quote, name, right_quote = tokval[0], tokval[1:-1], tokval[-1] + tokval = left_quote + self.unasync_name(name) + right_quote + if used_space is None: + used_space = space + yield (used_space, tokval) + used_space = None + + def unasync_name(self, name): + if name in self.token_replacements: + return self.token_replacements[name] + # Convert classes prefixed with 'Async' into 'Sync' + elif len(name) > 5 and name.startswith("Async") and name[5].isupper(): + return "Sync" + name[5:] + return name + + +_DEFAULT_RULE = Rule(fromdir="/_async/", todir="/_sync/") + +unasync_tokens = _DEFAULT_RULE.unasync_tokens +unasync_name = _DEFAULT_RULE.unasync_name + + +def unasync_file(filepath, fromdir, todir): + rule = Rule(fromdir, todir) + return rule.unasync_file(filepath) + + Token = collections.namedtuple("Token", ["type", "string", "start", "end", "line"]) @@ -60,37 +156,6 @@ def tokenize(f): last_end = (tok.end[0] + 1, 0) -def unasync_name(name): - if name in ASYNC_TO_SYNC: - return ASYNC_TO_SYNC[name] - # Convert classes prefixed with 'Async' into 'Sync' - elif len(name) > 5 and name.startswith("Async") and name[5].isupper(): - return "Sync" + name[5:] - return name - - -def unasync_tokens(tokens): - # TODO __await__, ...? - used_space = None - for space, toknum, tokval in tokens: - if tokval in ["async", "await"]: - # When removing async or await, we want to use the whitespace that - # was before async/await before the next token so that - # `print(await stuff)` becomes `print(stuff)` and not - # `print( stuff)` - used_space = space - else: - if toknum == std_tokenize.NAME: - tokval = unasync_name(tokval) - elif toknum == std_tokenize.STRING: - left_quote, name, right_quote = tokval[0], tokval[1:-1], tokval[-1] - tokval = left_quote + unasync_name(name) + right_quote - if used_space is None: - used_space = space - yield (used_space, tokval) - used_space = None - - def untokenize(tokens): return "".join(space + tokval for space, tokval in tokens) @@ -103,22 +168,6 @@ def makedirs_existok(dir): raise -def unasync_file(filepath, fromdir, todir): - with open(filepath, "rb") as f: - write_kwargs = {} - if sys.version_info[0] >= 3: - encoding, _ = std_tokenize.detect_encoding(f.readline) - write_kwargs["encoding"] = encoding - f.seek(0) - tokens = tokenize(f) - tokens = unasync_tokens(tokens) - result = untokenize(tokens) - outfilepath = filepath.replace(fromdir, todir) - makedirs_existok(os.path.dirname(outfilepath)) - with open(outfilepath, "w", **write_kwargs) as f: - print(result, file=f, end="") - - class build_py(orig.build_py): """ Subclass build_py from setuptools to modify its behavior. @@ -127,10 +176,10 @@ class build_py(orig.build_py): and saves them in _sync dir. """ - RENAME_DIR_FROM_TO = ("_async", "_sync") # Controls what directory will be renamed. + UNASYNC_RULES = (_DEFAULT_RULE,) def run(self): - dir_from, dir_to = self.RENAME_DIR_FROM_TO + rules = self.UNASYNC_RULES self._updated_files = [] @@ -143,8 +192,17 @@ def run(self): # Our modification! for f in self._updated_files: - if os.sep + dir_from + os.sep in f: - unasync_file(f, dir_from, dir_to) + found_rule = None + found_weight = None + + for rule in rules: + weight = rule.match(f) + if weight and (found_weight is None or weight > found_weight): + found_rule = rule + found_weight = weight + + if found_rule: + found_rule.unasync_file(f) # Remaining base class code self.byte_compile(self.get_outputs(include_bytecode=0)) @@ -156,8 +214,8 @@ def build_module(self, module, module_file, package): return outfile, copied -def customize_build_py(rename_dir_from_to=("_async", "_sync")): +def customize_build_py(rules=(_DEFAULT_RULE,)): class _build_py(build_py): - RENAME_DIR_FROM_TO = rename_dir_from_to + UNASYNC_RULES = rules return _build_py diff --git a/tests/data/example_custom_pkg/src/__init__.py b/tests/data/example_custom_pkg/__init__.py similarity index 100% rename from tests/data/example_custom_pkg/src/__init__.py rename to tests/data/example_custom_pkg/__init__.py diff --git a/tests/data/example_custom_pkg/setup.py b/tests/data/example_custom_pkg/setup.py index c6e40ef..cf89da3 100644 --- a/tests/data/example_custom_pkg/setup.py +++ b/tests/data/example_custom_pkg/setup.py @@ -9,9 +9,18 @@ author_email="author@example.com", description="A package used to test customized unasync", url="https://github.com/pypa/sampleproject", - packages=["ahip", "ahip.some_dir"], + packages=["ahip", "ahip.some_dir", "ahip.tests"], cmdclass={ - "build_py": unasync.customize_build_py(rename_dir_from_to=("ahip", "hip")) + "build_py": unasync.customize_build_py( + rules=[ + unasync.Rule(fromdir="/ahip/", todir="/hip/"), + unasync.Rule( + fromdir="/ahip/tests/", + todir="/hip/tests/", + additional_replacements={"ahip": "hip"}, + ), + ] + ) }, package_dir={"": "src"}, ) diff --git a/tests/data/example_custom_pkg/src/ahip/tests/__init__.py b/tests/data/example_custom_pkg/src/ahip/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/example_custom_pkg/src/ahip/tests/test_conn.py b/tests/data/example_custom_pkg/src/ahip/tests/test_conn.py new file mode 100644 index 0000000..7215610 --- /dev/null +++ b/tests/data/example_custom_pkg/src/ahip/tests/test_conn.py @@ -0,0 +1,8 @@ +import ahip + + +async def test_connection(): + async def f(): + return None + + x = await f() diff --git a/tests/test_unasync.py b/tests/test_unasync.py index d5c8fcb..39ff1a8 100644 --- a/tests/test_unasync.py +++ b/tests/test_unasync.py @@ -28,6 +28,11 @@ def list_files(startpath): return output +def test_rule_on_short_path(): + rule = unasync.Rule("/ahip/tests/", "/hip/tests/") + assert rule.match("/ahip/") is False + + @pytest.mark.parametrize("source_file", TEST_FILES) def test_unasync(tmpdir, source_file): @@ -109,6 +114,10 @@ def test_project_structure_after_customized_build_py_packages(tmpdir): subprocess.check_call(["python", "setup.py", "build"], cwd=pkg_dir, env=env) _async_dir_tree = list_files(os.path.join(source_pkg_dir, "src/ahip/.")) - unasynced_dir_tree = list_files(os.path.join(pkg_dir, "build/lib/hip/.")) + unasynced_dir_path = os.path.join(pkg_dir, "build/lib/hip/.") + unasynced_dir_tree = list_files(unasynced_dir_path) assert _async_dir_tree == unasynced_dir_tree + + with open(os.path.join(unasynced_dir_path, "tests/test_conn.py")) as f: + assert "import hip\n" in f.read() From 490bbe5ab4d60a21ada0ccdee753327455733004 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 11 Feb 2020 09:03:31 -0600 Subject: [PATCH 2/2] Switch to unasync.cmdclass_build_py(), update docs --- README.rst | 24 +++++++++++++++++- docs/source/index.rst | 30 +++++++++++++++++------ src/unasync/__init__.py | 34 +++++++++----------------- tests/data/example_custom_pkg/setup.py | 4 +-- tests/data/example_mod/setup.py | 2 +- tests/data/example_pkg/setup.py | 2 +- tests/test_unasync.py | 5 ++-- 7 files changed, 64 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index 92db43c..9611fb6 100644 --- a/README.rst +++ b/README.rst @@ -44,12 +44,34 @@ And then in your :code:`setup.py` place the following code. setuptools.setup( ... - cmdclass={'build_py': unasync.build_py}, + cmdclass={'build_py': unasync.cmdclass_build_py()}, ... ) And when you will build your package you will get your synchronous code in **_sync** folder. +If you'd like to customize where certain rules are applied you can pass +customized :code:`unasync.Rule` instances to :code:`unasync.cmdclass_build_py()` + +.. code-block:: python + + import unasync + + setuptools.setup( + ... + cmdclass={'build_py': unasync.cmdclass_build_py(rules=[ + # This rule transforms files within 'ahip' -> 'hip' + # instead of the default '_async' -> '_sync'. + unasync.Rule("/ahip/", "/hip/"), + + # This rule's 'fromdir' is more specific so will take precedent + # over the above rule if the path is within /ahip/tests/... + # This rule adds an additional token replacement over the default replacements. + unasync.Rule("/ahip/tests/", "/hip/tests/", replacements={"ahip", "hip"}), + ])}, + ... + ) + Documentation ============= diff --git a/docs/source/index.rst b/docs/source/index.rst index 6f347dd..61ba167 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,9 +1,3 @@ -.. documentation master file, created by - sphinx-quickstart on Sat Jan 21 19:11:14 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - - ======= unasync ======= @@ -56,7 +50,7 @@ And then in your :code:`setup.py` place the following code. setuptools.setup( ... - cmdclass={'build_py': unasync.build_py}, + cmdclass={'build_py': unasync.cmdclass_build_py()}, ... ) @@ -70,6 +64,28 @@ Then create a file **pyproject.toml** in the root of your project and mention ** And when you will build your package you will get your synchronous code in **_sync** folder. +If you'd like to customize where certain rules are applied you can pass +customized :code:`unasync.Rule` instances to :code:`unasync.cmdclass_build_py()` + +.. code-block:: python + + import unasync + + setuptools.setup( + ... + cmdclass={'build_py': unasync.cmdclass_build_py(rules=[ + # This rule transforms files within 'ahip' -> 'hip' + # instead of the default '_async' -> '_sync'. + unasync.Rule("/ahip/", "/hip/"), + + # This rule's 'fromdir' is more specific so will take precedent + # over the above rule if the path is within /ahip/tests/... + # This rule adds an additional token replacement over the default replacements. + unasync.Rule("/ahip/tests/", "/hip/tests/", replacements={"ahip", "hip"}), + ])}, + ... + ) + .. toctree:: :maxdepth: 2 diff --git a/src/unasync/__init__.py b/src/unasync/__init__.py index 0aa604a..28e330e 100644 --- a/src/unasync/__init__.py +++ b/src/unasync/__init__.py @@ -14,11 +14,7 @@ __all__ = [ "Rule", - "unasync_file", - "unasync_tokens", - "unasync_name", - "build_py", - "customize_build_py", + "cmdclass_build_py", ] @@ -41,13 +37,13 @@ class Rule: """A single set of rules for 'unasync'ing file(s)""" - def __init__(self, fromdir, todir, additional_replacements=None): + def __init__(self, fromdir, todir, replacements=None): self.fromdir = fromdir.replace("/", os.sep) self.todir = todir.replace("/", os.sep) # Add any additional user-defined token replacements to our list. self.token_replacements = _ASYNC_TO_SYNC.copy() - for key, val in (additional_replacements or {}).items(): + for key, val in (replacements or {}).items(): self.token_replacements[key] = val def match(self, filepath): @@ -112,17 +108,6 @@ def unasync_name(self, name): return name -_DEFAULT_RULE = Rule(fromdir="/_async/", todir="/_sync/") - -unasync_tokens = _DEFAULT_RULE.unasync_tokens -unasync_name = _DEFAULT_RULE.unasync_name - - -def unasync_file(filepath, fromdir, todir): - rule = Rule(fromdir, todir) - return rule.unasync_file(filepath) - - Token = collections.namedtuple("Token", ["type", "string", "start", "end", "line"]) @@ -168,7 +153,10 @@ def makedirs_existok(dir): raise -class build_py(orig.build_py): +_DEFAULT_RULE = Rule(fromdir="/_async/", todir="/_sync/") + + +class _build_py(orig.build_py): """ Subclass build_py from setuptools to modify its behavior. @@ -214,8 +202,10 @@ def build_module(self, module, module_file, package): return outfile, copied -def customize_build_py(rules=(_DEFAULT_RULE,)): - class _build_py(build_py): +def cmdclass_build_py(rules=(_DEFAULT_RULE,)): + """Creates a 'build_py' class for use within 'cmdclass={"build_py": ...}'""" + + class _custom_build_py(_build_py): UNASYNC_RULES = rules - return _build_py + return _custom_build_py diff --git a/tests/data/example_custom_pkg/setup.py b/tests/data/example_custom_pkg/setup.py index cf89da3..bea55bf 100644 --- a/tests/data/example_custom_pkg/setup.py +++ b/tests/data/example_custom_pkg/setup.py @@ -11,13 +11,13 @@ url="https://github.com/pypa/sampleproject", packages=["ahip", "ahip.some_dir", "ahip.tests"], cmdclass={ - "build_py": unasync.customize_build_py( + "build_py": unasync.cmdclass_build_py( rules=[ unasync.Rule(fromdir="/ahip/", todir="/hip/"), unasync.Rule( fromdir="/ahip/tests/", todir="/hip/tests/", - additional_replacements={"ahip": "hip"}, + replacements={"ahip": "hip"}, ), ] ) diff --git a/tests/data/example_mod/setup.py b/tests/data/example_mod/setup.py index ac24722..d784b3d 100644 --- a/tests/data/example_mod/setup.py +++ b/tests/data/example_mod/setup.py @@ -10,5 +10,5 @@ description="A package used to test unasync", url="https://github.com/pypa/sampleproject", py_modules=["_async.some_file"], - cmdclass={"build_py": unasync.build_py}, + cmdclass={"build_py": unasync.cmdclass_build_py()}, ) diff --git a/tests/data/example_pkg/setup.py b/tests/data/example_pkg/setup.py index 98aab8e..6a159b1 100644 --- a/tests/data/example_pkg/setup.py +++ b/tests/data/example_pkg/setup.py @@ -10,6 +10,6 @@ description="A package used to test unasync", url="https://github.com/pypa/sampleproject", packages=["example_pkg", "example_pkg._async", "example_pkg._async.some_dir"], - cmdclass={"build_py": unasync.build_py}, + cmdclass={"build_py": unasync.cmdclass_build_py()}, package_dir={"": "src"}, ) diff --git a/tests/test_unasync.py b/tests/test_unasync.py index 39ff1a8..259e267 100644 --- a/tests/test_unasync.py +++ b/tests/test_unasync.py @@ -36,9 +36,8 @@ def test_rule_on_short_path(): @pytest.mark.parametrize("source_file", TEST_FILES) def test_unasync(tmpdir, source_file): - unasync.unasync_file( - os.path.join(ASYNC_DIR, source_file), fromdir=ASYNC_DIR, todir=str(tmpdir) - ) + rule = unasync.Rule(fromdir=ASYNC_DIR, todir=str(tmpdir)) + rule.unasync_file(os.path.join(ASYNC_DIR, source_file)) encoding = "latin-1" if "encoding" in source_file else "utf-8" with io.open(os.path.join(SYNC_DIR, source_file), encoding=encoding) as f: