From 2007b96795f117883925dbc60f0fbfb7e022a951 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Sun, 2 Aug 2020 16:55:14 -0400 Subject: [PATCH 01/24] simplify file write and make_directories simple file writing is simpler with pathlib.Path.write_text For make_directories, python2 is end of life. python3 doesn't require blank __init__.py files, so we can rely on pathlib.path.mkdir --- mkcodes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mkcodes.py b/mkcodes.py index adf2b76..47dbced 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -88,7 +88,6 @@ def add_inits_to_dir(path): (child / '__init__.py').touch() - @click.command() @click.argument( 'inputs', nargs=-1, required=True, type=click.Path(exists=True)) @@ -115,7 +114,10 @@ def main(inputs, output, github, safe, package_python): outputfilename.parent.mkdir(parents=True, exist_ok=True) outputfilename.write_text('\n\n'.join(codeblocks)) +<<<<<<< HEAD if package_python: add_inits_to_dir(outputbasedir) +======= +>>>>>>> simplify file write and make_directories From 33f1082f3bf8c4484eb13c9dafb5756acdcb7d77 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Tue, 4 Aug 2020 17:57:42 -0400 Subject: [PATCH 02/24] default to adding __init__.py files to directories Python unittest discover still relies on __init__.py files and doesn't support namespace packages. By default we can create these but allow for their suppression if using other test frameworks. --- mkcodes.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 47dbced..591e5b6 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -114,10 +114,8 @@ def main(inputs, output, github, safe, package_python): outputfilename.parent.mkdir(parents=True, exist_ok=True) outputfilename.write_text('\n\n'.join(codeblocks)) -<<<<<<< HEAD if package_python: add_inits_to_dir(outputbasedir) -======= ->>>>>>> simplify file write and make_directories - + if package_python: + add_inits_to_dir(outputbasedir) From f2fe0402c67ffde2e5e0d8d482697dd8f292ad2b Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Sun, 2 Aug 2020 16:54:27 -0400 Subject: [PATCH 03/24] test that non markdown extension is ignored --- tests/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test.py b/tests/test.py index 8b6ba0f..5f472d7 100644 --- a/tests/test.py +++ b/tests/test.py @@ -96,6 +96,7 @@ def test_directory_recursive(self): self.assertTrue(self._output_path_exists('nest/deep.py')) self.assertFalse(self._output_path_exists('not_markdown.py')) self.assertTrue(self._output_path_exists('nest/more/why.py')) + self.assertFalse(self._output_path_exists('not_markdown.py')) def test_multiple(self): self.call( From f727c9db83786f8d6d97e5bb565258e0b39a1b3e Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Mon, 3 Aug 2020 17:42:07 -0400 Subject: [PATCH 04/24] add per language test and data --- tests/langdata/csharp.md | 34 ++++++++++++++++++++++++++++++++++ tests/langdata/java.md | 32 ++++++++++++++++++++++++++++++++ tests/langdata/multilang.md | 28 ++++++++++++++++++++++++++++ tests/test.py | 10 ++++++++++ 4 files changed, 104 insertions(+) create mode 100644 tests/langdata/csharp.md create mode 100644 tests/langdata/java.md create mode 100644 tests/langdata/multilang.md diff --git a/tests/langdata/csharp.md b/tests/langdata/csharp.md new file mode 100644 index 0000000..ed18f0f --- /dev/null +++ b/tests/langdata/csharp.md @@ -0,0 +1,34 @@ +# dotNet is still a thing + +What if you could provide a code sample here? + +```cs +public void Sum(int a, int b) +{ + return a + b; +} +``` + +And we know that it is testable. + +```cs +[Testclass] +public class UnitTest1 +{ + [TestMethod] + public void TestMethod1() + { + //Arrange + ApplicationToTest.Calc ClassCalc = new ApplicationToTest.Calc(); + int expectedResult = 5; + + //Act + int result = ClassCalc.Sum(2,3); + + //Assert + Assert.AreEqual(expectedResult, result); + } +} +``` + +Actually checking and running these tests, that's a different matter. diff --git a/tests/langdata/java.md b/tests/langdata/java.md new file mode 100644 index 0000000..5641146 --- /dev/null +++ b/tests/langdata/java.md @@ -0,0 +1,32 @@ +# Java documentation is important + +That's a language still. Here's a java codeblock: + +```java +public class MyUnit { + public String concatenate(String one, String two){ + return one + two; + } +} +``` + +And since we have that class, let's test it + +```java +import org.junit.Test; +import static org.junit.Assert.*; + +public class MyUnitTest { + + @Test + public void testConcatenate() { + MyUnit myUnit = new MyUnit(); + + String result = myUnit.concatenate("one", "two"); + + assertEquals("onetwo", result); + + } +} + +``` diff --git a/tests/langdata/multilang.md b/tests/langdata/multilang.md new file mode 100644 index 0000000..5e8b4cb --- /dev/null +++ b/tests/langdata/multilang.md @@ -0,0 +1,28 @@ +# Comparing and contrasting + +For some ideas about an api, we might give getting started code in a simple getting started page. + +In a pinch, let's hello that world. + +```py +print("hello, world") +``` + +But maybe we want this to be enterprise grade? + +```java +class HelloWorld { + public static void main(String[] args) { + System.out.println("Hello, World!"); + } +} +``` + +New orders from the CTO: let's use Azure cloud. +```cs +class HelloWorld { + static void Main() { + System.Console.WriteLine("Hello World"); + } +} +``` diff --git a/tests/test.py b/tests/test.py index 5f472d7..aca1a2f 100644 --- a/tests/test.py +++ b/tests/test.py @@ -141,6 +141,16 @@ def test_prefixed_deep_blocks(self): self.assertIn('Ran 2 tests', proc.stderr) self.assertIn('OK', proc.stderr) + def test_other_languages(self): + subprocess.call([ + 'mkcodes', '--output', 'tests/output/test_{name}.py', '--github', + 'tests/langdata']) + self.assertTrue(self._output_path_exists('test_java.java')) + self.assertTrue(self._output_path_exists('test_csharp.cs')) + self.assertTrue(self._output_path_exists('test_multilang.cs')) + self.assertTrue(self._output_path_exists('test_multilang.java')) + self.assertTrue(self._output_path_exists('test_multilang.py')) + @unittest.skip def test_glob(self): raise NotImplementedError From 6472df4cc3592a19ea2b0d05570d362d0ec49af6 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Tue, 4 Aug 2020 08:08:15 -0400 Subject: [PATCH 05/24] remove python indicator, switch to language If mkcodes supports many language blocks, we can make the default language python. safe mode should remove that default and not output languages that aren't specified. 2 tests not passing --- mkcodes.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++++-- tests/test.py | 2 +- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 591e5b6..a1bbbc0 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -13,7 +13,7 @@ from markdown.extensions import Extension from markdown.treeprocessors import Treeprocessor - +""" def github_codeblocks(filepath, safe): codeblocks = [] codeblock_re = r'^```.*' @@ -41,6 +41,70 @@ def github_codeblocks(filepath, safe): if not re.match(codeblock_open_re, line): python = False return codeblocks +""" + +def github_codeblocks(filepath, safe): + codeblocks = [] + codeblock_re = r'^```.*' + codeblock_open_re = r'^```(`*)(py|python){0}$'.format('' if safe else '?') + + # import pudb; pu.db + with open(filepath, 'r') as f: + block = [] + language = None + in_codeblock = False + + for line in f.readlines(): + # does this line contain a codeblock begin or end? + codeblock_delimiter = re.match(codeblock_re, line) + + if in_codeblock: + if codeblock_delimiter: + # we are closing a codeblock + if language: + # finished a codeblock, append everything + codeblocks.append(''.join(block)) + + block = [] + if safe: + language = None + in_codeblock = False + else: + block.append(line) + elif codeblock_delimiter: + # beginning a codeblock + in_codeblock = True + # does it have a language? + lang_match = re.match(codeblock_open_re, line) + if not lang_match: + language = None + else: + language = lang_match.group(2) + + return codeblocks + +def github_markdown_codeblocks(filepath, safe): + import markdown + codeblocks =[] + if safe: + warnings.warn("'safe' option not available in 'github-markdown' mode.") + + class DoctestCollector(Treeprocessor): + def run(self, root): + nonlocal codeblocks + import pudb; pu.db + codeblocks = (block.text for block in root.iterfind('./pre/code')) + + class DoctestExtension(Extension): + def extendMarkdown(self, md, md_globals): + md.registerExtension(self) + md.treeprocessors.add("doctest", DoctestCollector(md), '_end') + + doctestextension = DoctestExtension() + markdowner = markdown.Markdown(extensions=['fenced_code', doctestextension]) + markdowner.convertFile(input=str(filepath), output=os.devnull) + return codeblocks + def markdown_codeblocks(filepath, safe): @@ -63,7 +127,7 @@ def extendMarkdown(self, md, md_globals): doctestextension = DoctestExtension() markdowner = markdown.Markdown(extensions=[doctestextension]) - markdowner.convertFile(str(filepath), output=os.devnull) + markdowner.convertFile(input=str(filepath), output=os.devnull) return codeblocks diff --git a/tests/test.py b/tests/test.py index aca1a2f..c9396a3 100644 --- a/tests/test.py +++ b/tests/test.py @@ -143,7 +143,7 @@ def test_prefixed_deep_blocks(self): def test_other_languages(self): subprocess.call([ - 'mkcodes', '--output', 'tests/output/test_{name}.py', '--github', + 'mkcodes', '--output', 'tests/output/test_{name}.py', 'tests/langdata']) self.assertTrue(self._output_path_exists('test_java.java')) self.assertTrue(self._output_path_exists('test_csharp.cs')) From 8ab7fbabe1af3895b61b17b1e44472bf81ad1f4b Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Tue, 4 Aug 2020 15:20:47 -0400 Subject: [PATCH 06/24] introduce default language, lang mappings rather than assume python, let's just recognize that's the default language for unknown blocsk. People can override that as needed if they want to put in some other assumptions. --- mkcodes.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index a1bbbc0..10ea584 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -42,13 +42,22 @@ def github_codeblocks(filepath, safe): python = False return codeblocks """ - -def github_codeblocks(filepath, safe): +# much easier to write the other names that an extension is known by +ext_map = {'py': ['python', 'py', 'python2', 'python3', 'py2', 'py3', 'PYTHON', 'Python']} +ext_map['cs'] = ['c#','csharp', 'c-sharp', 'cs', 'CS', 'CSHARP', 'C#'] +ext_map['java'] = ['java', 'JAVA', 'Java'] +# then invert that mapping +language_map = {} +for ext, lang_strings in ext_map.items(): + for lang_string in lang_strings: + language_map[lang_string] = ext + + +def github_codeblocks(filepath, safe, default_lang='py'): codeblocks = [] codeblock_re = r'^```.*' codeblock_open_re = r'^```(`*)(py|python){0}$'.format('' if safe else '?') - # import pudb; pu.db with open(filepath, 'r') as f: block = [] language = None @@ -76,14 +85,19 @@ def github_codeblocks(filepath, safe): in_codeblock = True # does it have a language? lang_match = re.match(codeblock_open_re, line) - if not lang_match: - language = None - else: + if lang_match: language = lang_match.group(2) - + if not safe: + # we can sub a default language if not safe + language = language or default_lang + else: + if safe: + language = None + else: + language = default_lang return codeblocks -def github_markdown_codeblocks(filepath, safe): +def github_markdown_codeblocks(filepath, safe, default_lang='py'): import markdown codeblocks =[] if safe: @@ -92,7 +106,6 @@ def github_markdown_codeblocks(filepath, safe): class DoctestCollector(Treeprocessor): def run(self, root): nonlocal codeblocks - import pudb; pu.db codeblocks = (block.text for block in root.iterfind('./pre/code')) class DoctestExtension(Extension): From 863242efa11049f6e81d951c9f08c29b65c0a129 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Tue, 4 Aug 2020 16:04:36 -0400 Subject: [PATCH 07/24] collect and write per-language code blocks as we pass through a document, get per-language codeblocks, then write those out to the correct extensionfile. By default, everything is still python - though it can be overridden --- mkcodes.py | 53 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 10ea584..94fdea5 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -54,7 +54,7 @@ def github_codeblocks(filepath, safe): def github_codeblocks(filepath, safe, default_lang='py'): - codeblocks = [] + codeblocks = {} codeblock_re = r'^```.*' codeblock_open_re = r'^```(`*)(py|python){0}$'.format('' if safe else '?') @@ -72,7 +72,11 @@ def github_codeblocks(filepath, safe, default_lang='py'): # we are closing a codeblock if language: # finished a codeblock, append everything - codeblocks.append(''.join(block)) + # codeblocks.append(''.join(block)) + # import pudb; pu.db + blocks = codeblocks.get(language, []) + blocks.append(''.join(block)) + codeblocks[language] = blocks block = [] if safe: @@ -99,14 +103,14 @@ def github_codeblocks(filepath, safe, default_lang='py'): def github_markdown_codeblocks(filepath, safe, default_lang='py'): import markdown - codeblocks =[] + codeblocks = {} if safe: warnings.warn("'safe' option not available in 'github-markdown' mode.") class DoctestCollector(Treeprocessor): def run(self, root): nonlocal codeblocks - codeblocks = (block.text for block in root.iterfind('./pre/code')) + codeblocks[default_lang] = (block.text for block in root.iterfind('./pre/code')) class DoctestExtension(Extension): def extendMarkdown(self, md, md_globals): @@ -120,10 +124,10 @@ def extendMarkdown(self, md, md_globals): -def markdown_codeblocks(filepath, safe): +def markdown_codeblocks(filepath, safe, default_lang='py'): import markdown - codeblocks = [] + codeblocks = {} if safe: warnings.warn("'safe' option not available in 'markdown' mode.") @@ -131,7 +135,7 @@ def markdown_codeblocks(filepath, safe): class DoctestCollector(Treeprocessor): def run(self, root): nonlocal codeblocks - codeblocks = (block.text for block in root.iterfind('./pre/code')) + codeblocks[default_lang] = (block.text for block in root.iterfind('./pre/code')) class DoctestExtension(Extension): def extendMarkdown(self, md, md_globals): @@ -156,43 +160,50 @@ def get_files(inputs): elif path.suffix in markdown_extensions: yield path, path.parent -def add_inits_to_dir(path): +def add_inits_along_path(from_path, to_path): """Recursively add __init__.py files to a directory This compensates for https://bugs.python.org/issue23882 and https://bugs.python.org/issue35617 """ - for child in path.rglob('*'): - if child.is_dir(): - (child / '__init__.py').touch() + to_path = to_path.expanduser().resolve() + from_path = from_path.expanduser().resolve() + if to_path.relative_to_path(from_path): + (to_path / '__init__.py').to_pathuch() + if from_path.resolve() != to_path.parent.resolve(): + add_inits_along_path(from_path, to_path.parent() @click.command() @click.argument( 'inputs', nargs=-1, required=True, type=click.Path(exists=True)) -@click.option('--output', default='{name}.py') +@click.option('--output', default='{name}.{ext}') @click.option('--github/--markdown', default=bool(not markdown_enabled), help='Github-flavored fence blocks or pure markdown.') @click.option('--safe/--unsafe', default=True, help='Allow code blocks without language hints.') @click.option('--package-python', default=True, help='Add __init__.py files to python output to aid in test discovery') -def main(inputs, output, github, safe, package_python): +@click.option('--default_lang', default='py', + help='Assumed language for code blocks without language hits.') +def main(inputs, output, github, safe, package_python, default_lang): collect_codeblocks = github_codeblocks if github else markdown_codeblocks outputbasedir = Path(output).parent outputbasename = Path(output).name for filepath, input_path in get_files(inputs): - codeblocks = collect_codeblocks(filepath, safe) + codeblocks = collect_codeblocks(filepath, safe, default_lang) if codeblocks: fp = Path(filepath) filedir = fp.parent.relative_to(input_path) filename = fp.stem - outputfilename = outputbasedir / filedir / outputbasename.format(name=filename) - outputfilename.parent.mkdir(parents=True, exist_ok=True) - outputfilename.write_text('\n\n'.join(codeblocks)) - if package_python: - add_inits_to_dir(outputbasedir) + # stitch together the OUTPUT base directory with the input directories + # add the file format at the end. + for lang, blocks in codeblocks.items(): + outputfilename = outputbasedir / filedir / outputbasename.format(name=filename, ext=lang) - if package_python: - add_inits_to_dir(outputbasedir) + # make sure path exists, don't care if it already does + outputfilename.parent.mkdir(parents=True, exist_ok=True) + outputfilename.write_text('\n\n'.join(blocks)) + if package_python and lang=='py': + add_inits_along_path(outputbasedir, outputfilename.parent) From 7d3c4867d55fe623f3630298b6d13217c1e42394 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Tue, 4 Aug 2020 16:50:07 -0400 Subject: [PATCH 08/24] make other languages work Now the default language is python codeblocks in other languages can be broken out into their own language files. There is a way to add more languages with mappings, but a language fenced codeblock will default to use that language as the output extension --- mkcodes.py | 72 +++++++------------------------------ tests/langdata/csharp.md | 2 +- tests/langdata/multilang.md | 6 ++++ tests/test.py | 5 +-- 4 files changed, 22 insertions(+), 63 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 94fdea5..b71b239 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -2,7 +2,6 @@ from pathlib import Path import re import warnings - import click try: @@ -13,38 +12,10 @@ from markdown.extensions import Extension from markdown.treeprocessors import Treeprocessor -""" -def github_codeblocks(filepath, safe): - codeblocks = [] - codeblock_re = r'^```.*' - codeblock_open_re = r'^```(`*)(py|python){0}$'.format('' if safe else '?') - - with open(filepath, 'r') as f: - block = [] - python = True - in_codeblock = False - - for line in f.readlines(): - codeblock_delimiter = re.match(codeblock_re, line) - - if in_codeblock: - if codeblock_delimiter: - if python: - codeblocks.append(''.join(block)) - block = [] - python = True - in_codeblock = False - else: - block.append(line) - elif codeblock_delimiter: - in_codeblock = True - if not re.match(codeblock_open_re, line): - python = False - return codeblocks -""" # much easier to write the other names that an extension is known by -ext_map = {'py': ['python', 'py', 'python2', 'python3', 'py2', 'py3', 'PYTHON', 'Python']} -ext_map['cs'] = ['c#','csharp', 'c-sharp', 'cs', 'CS', 'CSHARP', 'C#'] +ext_map = {'py': ['python', 'py', 'python2', 'python3', 'py2', 'py3', + 'PYTHON', 'Python']} +ext_map['cs'] = ['c#', 'csharp', 'c-sharp', 'cs', 'CS', 'CSHARP', 'C#'] ext_map['java'] = ['java', 'JAVA', 'Java'] # then invert that mapping language_map = {} @@ -56,7 +27,7 @@ def github_codeblocks(filepath, safe): def github_codeblocks(filepath, safe, default_lang='py'): codeblocks = {} codeblock_re = r'^```.*' - codeblock_open_re = r'^```(`*)(py|python){0}$'.format('' if safe else '?') + codeblock_open_re = r'^```(`*)(\w+){0}$'.format('' if safe else '?') with open(filepath, 'r') as f: block = [] @@ -74,9 +45,10 @@ def github_codeblocks(filepath, safe, default_lang='py'): # finished a codeblock, append everything # codeblocks.append(''.join(block)) # import pudb; pu.db - blocks = codeblocks.get(language, []) + ext = language_map.get(language, language) + blocks = codeblocks.get(ext, []) blocks.append(''.join(block)) - codeblocks[language] = blocks + codeblocks[ext] = blocks block = [] if safe: @@ -101,28 +73,6 @@ def github_codeblocks(filepath, safe, default_lang='py'): language = default_lang return codeblocks -def github_markdown_codeblocks(filepath, safe, default_lang='py'): - import markdown - codeblocks = {} - if safe: - warnings.warn("'safe' option not available in 'github-markdown' mode.") - - class DoctestCollector(Treeprocessor): - def run(self, root): - nonlocal codeblocks - codeblocks[default_lang] = (block.text for block in root.iterfind('./pre/code')) - - class DoctestExtension(Extension): - def extendMarkdown(self, md, md_globals): - md.registerExtension(self) - md.treeprocessors.add("doctest", DoctestCollector(md), '_end') - - doctestextension = DoctestExtension() - markdowner = markdown.Markdown(extensions=['fenced_code', doctestextension]) - markdowner.convertFile(input=str(filepath), output=os.devnull) - return codeblocks - - def markdown_codeblocks(filepath, safe, default_lang='py'): import markdown @@ -135,7 +85,8 @@ def markdown_codeblocks(filepath, safe, default_lang='py'): class DoctestCollector(Treeprocessor): def run(self, root): nonlocal codeblocks - codeblocks[default_lang] = (block.text for block in root.iterfind('./pre/code')) + codeblocks[default_lang] =\ + (block.text for block in root.iterfind('./pre/code')) class DoctestExtension(Extension): def extendMarkdown(self, md, md_globals): @@ -197,10 +148,11 @@ def main(inputs, output, github, safe, package_python, default_lang): filedir = fp.parent.relative_to(input_path) filename = fp.stem - # stitch together the OUTPUT base directory with the input directories + # stitch together the OUTPUT base directory with input directories # add the file format at the end. for lang, blocks in codeblocks.items(): - outputfilename = outputbasedir / filedir / outputbasename.format(name=filename, ext=lang) + outputfilename = outputbasedir / filedir /\ + outputbasename.format(name=filename, ext=lang) # make sure path exists, don't care if it already does outputfilename.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/langdata/csharp.md b/tests/langdata/csharp.md index ed18f0f..b75d51c 100644 --- a/tests/langdata/csharp.md +++ b/tests/langdata/csharp.md @@ -11,7 +11,7 @@ public void Sum(int a, int b) And we know that it is testable. -```cs +```csharp [Testclass] public class UnitTest1 { diff --git a/tests/langdata/multilang.md b/tests/langdata/multilang.md index 5e8b4cb..91a1652 100644 --- a/tests/langdata/multilang.md +++ b/tests/langdata/multilang.md @@ -26,3 +26,9 @@ class HelloWorld { } } ``` + +We want to have a react vue jquery frontend. Assume that the code sample below has a testable extension as the language + +```js +consol.log('Hello, world"); +``` diff --git a/tests/test.py b/tests/test.py index c9396a3..bdfb918 100644 --- a/tests/test.py +++ b/tests/test.py @@ -143,10 +143,11 @@ def test_prefixed_deep_blocks(self): def test_other_languages(self): subprocess.call([ - 'mkcodes', '--output', 'tests/output/test_{name}.py', - 'tests/langdata']) + 'mkcodes', '--output', 'tests/output/test_{name}.{ext}', + '--github', 'tests/langdata']) self.assertTrue(self._output_path_exists('test_java.java')) self.assertTrue(self._output_path_exists('test_csharp.cs')) + self.assertFalse(self._output_path_exists('test_csharp.csharp')) self.assertTrue(self._output_path_exists('test_multilang.cs')) self.assertTrue(self._output_path_exists('test_multilang.java')) self.assertTrue(self._output_path_exists('test_multilang.py')) From 4781972db67d8b8780c2cbba39c21d021d7752c8 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Tue, 4 Aug 2020 17:04:04 -0400 Subject: [PATCH 09/24] add test for unmapped extension --- tests/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test.py b/tests/test.py index bdfb918..c2be3a0 100644 --- a/tests/test.py +++ b/tests/test.py @@ -151,6 +151,7 @@ def test_other_languages(self): self.assertTrue(self._output_path_exists('test_multilang.cs')) self.assertTrue(self._output_path_exists('test_multilang.java')) self.assertTrue(self._output_path_exists('test_multilang.py')) + self.assertTrue(self._output_path_exists('test_multilang.js')) @unittest.skip def test_glob(self): From f8556dd121a166e45e50b5546f4caf8242055768 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Wed, 5 Aug 2020 06:51:43 -0400 Subject: [PATCH 10/24] clean up __init__.py addition logic Also remove some commented code, shorten comments --- mkcodes.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index b71b239..5f915ee 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -43,8 +43,6 @@ def github_codeblocks(filepath, safe, default_lang='py'): # we are closing a codeblock if language: # finished a codeblock, append everything - # codeblocks.append(''.join(block)) - # import pudb; pu.db ext = language_map.get(language, language) blocks = codeblocks.get(ext, []) blocks.append(''.join(block)) @@ -111,16 +109,20 @@ def get_files(inputs): elif path.suffix in markdown_extensions: yield path, path.parent + def add_inits_along_path(from_path, to_path): """Recursively add __init__.py files to a directory - This compensates for https://bugs.python.org/issue23882 and https://bugs.python.org/issue35617 + This compensates for https://bugs.python.org/issue23882 + and https://bugs.python.org/issue35617 """ to_path = to_path.expanduser().resolve() from_path = from_path.expanduser().resolve() - if to_path.relative_to_path(from_path): - (to_path / '__init__.py').to_pathuch() - if from_path.resolve() != to_path.parent.resolve(): - add_inits_along_path(from_path, to_path.parent() + (to_path / '__init__.py').touch() + if to_path == from_path: + return + if to_path.relative_to(from_path): + if from_path != to_path.parent: + add_inits_along_path(from_path, to_path.parent) @click.command() @@ -132,7 +134,7 @@ def add_inits_along_path(from_path, to_path): @click.option('--safe/--unsafe', default=True, help='Allow code blocks without language hints.') @click.option('--package-python', default=True, - help='Add __init__.py files to python output to aid in test discovery') + help='Add __init__.py files to python dirs for test discovery') @click.option('--default_lang', default='py', help='Assumed language for code blocks without language hits.') def main(inputs, output, github, safe, package_python, default_lang): From 833b72d33ac9ad5b577f229766afef5b573c014b Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Wed, 5 Aug 2020 06:52:35 -0400 Subject: [PATCH 11/24] test __init__.py paths For non python paths, don't declare them to be python packages. --- tests/langdata/no_py_tree/clean.md | 16 ++++++++++++++++ tests/langdata/pytree/buried.md | 12 ++++++++++++ tests/test.py | 4 ++++ 3 files changed, 32 insertions(+) create mode 100644 tests/langdata/no_py_tree/clean.md create mode 100644 tests/langdata/pytree/buried.md diff --git a/tests/langdata/no_py_tree/clean.md b/tests/langdata/no_py_tree/clean.md new file mode 100644 index 0000000..f72dbe8 --- /dev/null +++ b/tests/langdata/no_py_tree/clean.md @@ -0,0 +1,16 @@ +# Cleanliness + +If there are no python files in a directory, we don't need to add an __init__.py file to that directory. Sure, they don't hurt, but having them where they aren't needed isn't very tidy and might be confusing. + +Speaking of confusing, lets test javascript +```js +function assert(condition, message) { + if (!condition) { + message = message || "Assertion failed"; + throw new Error(message); + } +} + +assert([]+[]=="", "very sensible, adding arrays is a string") +assert({}+[]==0, "of course adding a dict to an array is 0") +``` diff --git a/tests/langdata/pytree/buried.md b/tests/langdata/pytree/buried.md new file mode 100644 index 0000000..46b780f --- /dev/null +++ b/tests/langdata/pytree/buried.md @@ -0,0 +1,12 @@ +# Test discovery + +For test discovery to work for unittest, python files generated from this document must have an `__init__.py` file added to the directory - otherwise they won't be considered testable packages. + +```python +import unittest + +class TestDiscovery(unittest.TestCase): + def test_discovery(self): + self.assertTrue(True) + +``` diff --git a/tests/test.py b/tests/test.py index c2be3a0..98b9791 100644 --- a/tests/test.py +++ b/tests/test.py @@ -152,6 +152,10 @@ def test_other_languages(self): self.assertTrue(self._output_path_exists('test_multilang.java')) self.assertTrue(self._output_path_exists('test_multilang.py')) self.assertTrue(self._output_path_exists('test_multilang.js')) + self.assertTrue(self._output_path_exists('no_py_tree/test_clean.js')) + self.assertFalse(self._output_path_exists('no_py_tree/__init__.py')) + self.assertTrue(self._output_path_exists('pytree/test_buried.py')) + self.assertTrue(self._output_path_exists('pytree/__init__.py')) @unittest.skip def test_glob(self): From fcba42de6d4c4b52f5e93db38910b9d30872ef56 Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Wed, 5 Aug 2020 07:06:34 -0400 Subject: [PATCH 12/24] correct spelling mistakes also conform new test to self.call --- tests/data/nest/more/why.md | 2 +- tests/langdata/multilang.md | 2 +- tests/test.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/data/nest/more/why.md b/tests/data/nest/more/why.md index ff50e89..b43a790 100644 --- a/tests/data/nest/more/why.md +++ b/tests/data/nest/more/why.md @@ -1,6 +1,6 @@ # why? -We want to make sure that in more complext documentation structures, which may have multiple sub directories, we are still formatting name and paths correctly. +We want to make sure that in more complex documentation structures, which may have multiple sub directories, we are still formatting name and paths correctly. ```py import unittest diff --git a/tests/langdata/multilang.md b/tests/langdata/multilang.md index 91a1652..8e15a3c 100644 --- a/tests/langdata/multilang.md +++ b/tests/langdata/multilang.md @@ -30,5 +30,5 @@ class HelloWorld { We want to have a react vue jquery frontend. Assume that the code sample below has a testable extension as the language ```js -consol.log('Hello, world"); +console.log('Hello, world"); ``` diff --git a/tests/test.py b/tests/test.py index 98b9791..e5176c0 100644 --- a/tests/test.py +++ b/tests/test.py @@ -142,9 +142,9 @@ def test_prefixed_deep_blocks(self): self.assertIn('OK', proc.stderr) def test_other_languages(self): - subprocess.call([ - 'mkcodes', '--output', 'tests/output/test_{name}.{ext}', - '--github', 'tests/langdata']) + self.call( + '--output', 'tests/output/test_{name}.{ext}', + '--github', 'tests/langdata') self.assertTrue(self._output_path_exists('test_java.java')) self.assertTrue(self._output_path_exists('test_csharp.cs')) self.assertFalse(self._output_path_exists('test_csharp.csharp')) From 6b010c2af40cf2ef31f0bd5f0300dacd626ddb0b Mon Sep 17 00:00:00 2001 From: Matt Katz Date: Wed, 5 Aug 2020 07:09:05 -0400 Subject: [PATCH 13/24] separate 3rd party imports with a blank line --- mkcodes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mkcodes.py b/mkcodes.py index 5f915ee..1fe2d8d 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -2,6 +2,7 @@ from pathlib import Path import re import warnings + import click try: From 8df94d557254d2aa27a9163a2d929b37f75a3622 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Thu, 6 Aug 2020 02:58:33 +0000 Subject: [PATCH 14/24] cleanup --- mkcodes.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 1fe2d8d..d189549 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -14,10 +14,12 @@ from markdown.treeprocessors import Treeprocessor # much easier to write the other names that an extension is known by -ext_map = {'py': ['python', 'py', 'python2', 'python3', 'py2', 'py3', - 'PYTHON', 'Python']} -ext_map['cs'] = ['c#', 'csharp', 'c-sharp', 'cs', 'CS', 'CSHARP', 'C#'] -ext_map['java'] = ['java', 'JAVA', 'Java'] +ext_map = { + 'cs': ['c#', 'csharp', 'c-sharp', 'cs', 'CS', 'CSHARP', 'C#'], + 'java': ['java', 'JAVA', 'Java'], + 'py': ['python', 'py', 'python2', 'python3', 'py2', 'py3', 'PYTHON', + 'Python'], +} # then invert that mapping language_map = {} for ext, lang_strings in ext_map.items(): @@ -45,9 +47,7 @@ def github_codeblocks(filepath, safe, default_lang='py'): if language: # finished a codeblock, append everything ext = language_map.get(language, language) - blocks = codeblocks.get(ext, []) - blocks.append(''.join(block)) - codeblocks[ext] = blocks + codeblocks.setdefault(ext, []).append(''.join(block)) block = [] if safe: @@ -84,8 +84,8 @@ def markdown_codeblocks(filepath, safe, default_lang='py'): class DoctestCollector(Treeprocessor): def run(self, root): nonlocal codeblocks - codeblocks[default_lang] =\ - (block.text for block in root.iterfind('./pre/code')) + codeblocks[default_lang] = ( + block.text for block in root.iterfind('./pre/code')) class DoctestExtension(Extension): def extendMarkdown(self, md, md_globals): From 4ff942817628a2a2b60d5c6bf05057d6796e3166 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Fri, 7 Aug 2020 19:29:31 +0000 Subject: [PATCH 15/24] Ignore capitalization of language strings. --- mkcodes.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index d189549..05814a7 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -15,10 +15,8 @@ # much easier to write the other names that an extension is known by ext_map = { - 'cs': ['c#', 'csharp', 'c-sharp', 'cs', 'CS', 'CSHARP', 'C#'], - 'java': ['java', 'JAVA', 'Java'], - 'py': ['python', 'py', 'python2', 'python3', 'py2', 'py3', 'PYTHON', - 'Python'], + 'cs': ['c#', 'csharp', 'c-sharp'], + 'py': ['python', 'python2', 'python3', 'py2', 'py3'], } # then invert that mapping language_map = {} @@ -62,6 +60,7 @@ def github_codeblocks(filepath, safe, default_lang='py'): lang_match = re.match(codeblock_open_re, line) if lang_match: language = lang_match.group(2) + language = language.lower() if language else language if not safe: # we can sub a default language if not safe language = language or default_lang @@ -160,5 +159,5 @@ def main(inputs, output, github, safe, package_python, default_lang): # make sure path exists, don't care if it already does outputfilename.parent.mkdir(parents=True, exist_ok=True) outputfilename.write_text('\n\n'.join(blocks)) - if package_python and lang=='py': + if package_python and lang == 'py': add_inits_along_path(outputbasedir, outputfilename.parent) From ef189db24c6a757dbdea9695b4cd1e5dcc5930d1 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Fri, 7 Aug 2020 19:29:47 +0000 Subject: [PATCH 16/24] Remove no-op else clause. --- mkcodes.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 05814a7..2c8a304 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -64,11 +64,6 @@ def github_codeblocks(filepath, safe, default_lang='py'): if not safe: # we can sub a default language if not safe language = language or default_lang - else: - if safe: - language = None - else: - language = default_lang return codeblocks From 5007671e912cfadec3cc773723773c18aac11403 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Fri, 7 Aug 2020 19:37:36 +0000 Subject: [PATCH 17/24] Document extension/language mappings. --- mkcodes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 2c8a304..98fca12 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -13,12 +13,16 @@ from markdown.extensions import Extension from markdown.treeprocessors import Treeprocessor -# much easier to write the other names that an extension is known by +# There does not seem to be any specification for which info strings are +# accepted, but python-markdown passes it directly to pygments, so their +# mapping can be used as a guide: +# https://github.com/pygments/pygments/blob/master/pygments/lexers/_mapping.py ext_map = { 'cs': ['c#', 'csharp', 'c-sharp'], 'py': ['python', 'python2', 'python3', 'py2', 'py3'], } -# then invert that mapping +# It's more straightforward to express the mappings by extension, but we +# actually need an inverted mapping. language_map = {} for ext, lang_strings in ext_map.items(): for lang_string in lang_strings: From acbf21337aae7b53bb1f7177e6e7a34bf57e9d56 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sun, 9 Aug 2020 14:28:30 +0000 Subject: [PATCH 18/24] Language should be reset even if --unsafe. The safety flag refers to whether a default language is used. This would have made it also accept the previous language used, which doesn't seem right. --- mkcodes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 98fca12..2a5a578 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -35,6 +35,7 @@ def github_codeblocks(filepath, safe, default_lang='py'): codeblock_open_re = r'^```(`*)(\w+){0}$'.format('' if safe else '?') with open(filepath, 'r') as f: + # Initialize State block = [] language = None in_codeblock = False @@ -51,9 +52,9 @@ def github_codeblocks(filepath, safe, default_lang='py'): ext = language_map.get(language, language) codeblocks.setdefault(ext, []).append(''.join(block)) + # Reset State block = [] - if safe: - language = None + language = None in_codeblock = False else: block.append(line) From b06d0773943bba1c361a1d590965ec386bc7511e Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sun, 9 Aug 2020 14:53:47 +0000 Subject: [PATCH 19/24] Add warning when unhinted code blocks are skipped. --- mkcodes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mkcodes.py b/mkcodes.py index 2a5a578..c94c5d3 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -51,6 +51,10 @@ def github_codeblocks(filepath, safe, default_lang='py'): # finished a codeblock, append everything ext = language_map.get(language, language) codeblocks.setdefault(ext, []).append(''.join(block)) + else: + warnings.warn('No language hint found in safe mode. ' + + 'Skipping block beginning with: ' + + block[0]) # Reset State block = [] From dda1710cc925b7985cac7e92195fe0ea71dc5ecf Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Mon, 10 Aug 2020 02:20:06 +0000 Subject: [PATCH 20/24] Use dash for cli parameters and fix typo. --- mkcodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index c94c5d3..45f2027 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -139,8 +139,8 @@ def add_inits_along_path(from_path, to_path): help='Allow code blocks without language hints.') @click.option('--package-python', default=True, help='Add __init__.py files to python dirs for test discovery') -@click.option('--default_lang', default='py', - help='Assumed language for code blocks without language hits.') +@click.option('--default-lang', default='py', + help='Assumed language for code blocks without language hints.') def main(inputs, output, github, safe, package_python, default_lang): collect_codeblocks = github_codeblocks if github else markdown_codeblocks outputbasedir = Path(output).parent From 9ef06ab83700d6d1cfea934dce999f87707ad8ef Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Wed, 12 Aug 2020 02:14:53 +0000 Subject: [PATCH 21/24] Pull sanity check out of if-statement. --- mkcodes.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index 45f2027..d886438 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -121,12 +121,15 @@ def add_inits_along_path(from_path, to_path): """ to_path = to_path.expanduser().resolve() from_path = from_path.expanduser().resolve() + + # Sanity Check: This will raise an exception if paths aren't relative. + to_path.relative_to(from_path) + (to_path / '__init__.py').touch() if to_path == from_path: return - if to_path.relative_to(from_path): - if from_path != to_path.parent: - add_inits_along_path(from_path, to_path.parent) + if from_path != to_path.parent: + add_inits_along_path(from_path, to_path.parent) @click.command() From 334e7db6cc1fae2b231e5bec9a90f0409b674078 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Wed, 12 Aug 2020 02:21:20 +0000 Subject: [PATCH 22/24] Don't create __init__.py in base output directory. --- mkcodes.py | 5 ++--- tests/test.py | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index d886438..ae46c5c 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -125,11 +125,10 @@ def add_inits_along_path(from_path, to_path): # Sanity Check: This will raise an exception if paths aren't relative. to_path.relative_to(from_path) - (to_path / '__init__.py').touch() if to_path == from_path: return - if from_path != to_path.parent: - add_inits_along_path(from_path, to_path.parent) + (to_path / '__init__.py').touch() + add_inits_along_path(from_path, to_path.parent) @click.command() diff --git a/tests/test.py b/tests/test.py index e5176c0..b83fee4 100644 --- a/tests/test.py +++ b/tests/test.py @@ -157,6 +157,9 @@ def test_other_languages(self): self.assertTrue(self._output_path_exists('pytree/test_buried.py')) self.assertTrue(self._output_path_exists('pytree/__init__.py')) + # __init__.py should not be created in the base output directory. + self.assertFalse(self._output_path_exists('__init__.py')) + @unittest.skip def test_glob(self): raise NotImplementedError From 6b4589fcb0427bc36b375d406bc4c72bd6e610a4 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Wed, 12 Aug 2020 02:26:08 +0000 Subject: [PATCH 23/24] Avoid abusing return as a mid-function "break". I would consider this anti-pattern when avoidable and I find it harder to understand. --- mkcodes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mkcodes.py b/mkcodes.py index ae46c5c..d889749 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -125,10 +125,10 @@ def add_inits_along_path(from_path, to_path): # Sanity Check: This will raise an exception if paths aren't relative. to_path.relative_to(from_path) - if to_path == from_path: - return - (to_path / '__init__.py').touch() - add_inits_along_path(from_path, to_path.parent) + # Continue recursing if we haven't reached the base output directory. + if to_path != from_path: + (to_path / '__init__.py').touch() + add_inits_along_path(from_path, to_path.parent) @click.command() From 39ea705810ac5251bf0c0988ac6a811d34a49ff7 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Wed, 12 Aug 2020 03:04:08 +0000 Subject: [PATCH 24/24] Remove redundant test assertion. --- tests/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index b83fee4..04c2ec8 100644 --- a/tests/test.py +++ b/tests/test.py @@ -96,7 +96,6 @@ def test_directory_recursive(self): self.assertTrue(self._output_path_exists('nest/deep.py')) self.assertFalse(self._output_path_exists('not_markdown.py')) self.assertTrue(self._output_path_exists('nest/more/why.py')) - self.assertFalse(self._output_path_exists('not_markdown.py')) def test_multiple(self): self.call(