diff --git a/mkcodes.py b/mkcodes.py index adf2b76..d889749 100755 --- a/mkcodes.py +++ b/mkcodes.py @@ -13,40 +13,73 @@ from markdown.extensions import Extension from markdown.treeprocessors import Treeprocessor - -def github_codeblocks(filepath, safe): - codeblocks = [] +# There does not seem to be any specification for which info strings are +# accepted, but python-markdown passes it directly to pygments, so their +# mapping can be used as a guide: +# https://github.com/pygments/pygments/blob/master/pygments/lexers/_mapping.py +ext_map = { + 'cs': ['c#', 'csharp', 'c-sharp'], + 'py': ['python', 'python2', 'python3', 'py2', 'py3'], +} +# It's more straightforward to express the mappings by extension, but we +# actually need an inverted mapping. +language_map = {} +for ext, lang_strings in ext_map.items(): + for lang_string in lang_strings: + language_map[lang_string] = ext + + +def github_codeblocks(filepath, safe, default_lang='py'): + codeblocks = {} codeblock_re = r'^```.*' - codeblock_open_re = r'^```(`*)(py|python){0}$'.format('' if safe else '?') + codeblock_open_re = r'^```(`*)(\w+){0}$'.format('' if safe else '?') with open(filepath, 'r') as f: + # Initialize State block = [] - python = True + language = None in_codeblock = False for line in f.readlines(): + # does this line contain a codeblock begin or end? codeblock_delimiter = re.match(codeblock_re, line) if in_codeblock: if codeblock_delimiter: - if python: - codeblocks.append(''.join(block)) + # we are closing a codeblock + if language: + # finished a codeblock, append everything + ext = language_map.get(language, language) + codeblocks.setdefault(ext, []).append(''.join(block)) + else: + warnings.warn('No language hint found in safe mode. ' + + 'Skipping block beginning with: ' + + block[0]) + + # Reset State block = [] - python = True + language = None in_codeblock = False else: block.append(line) elif codeblock_delimiter: + # beginning a codeblock in_codeblock = True - if not re.match(codeblock_open_re, line): - python = False + # does it have a language? + lang_match = re.match(codeblock_open_re, line) + if lang_match: + language = lang_match.group(2) + language = language.lower() if language else language + if not safe: + # we can sub a default language if not safe + language = language or default_lang return codeblocks -def markdown_codeblocks(filepath, safe): +def markdown_codeblocks(filepath, safe, default_lang='py'): import markdown - codeblocks = [] + codeblocks = {} if safe: warnings.warn("'safe' option not available in 'markdown' mode.") @@ -54,7 +87,8 @@ def markdown_codeblocks(filepath, safe): class DoctestCollector(Treeprocessor): def run(self, root): nonlocal codeblocks - codeblocks = (block.text for block in root.iterfind('./pre/code')) + codeblocks[default_lang] = ( + block.text for block in root.iterfind('./pre/code')) class DoctestExtension(Extension): def extendMarkdown(self, md, md_globals): @@ -63,7 +97,7 @@ def extendMarkdown(self, md, md_globals): doctestextension = DoctestExtension() markdowner = markdown.Markdown(extensions=[doctestextension]) - markdowner.convertFile(str(filepath), output=os.devnull) + markdowner.convertFile(input=str(filepath), output=os.devnull) return codeblocks @@ -79,43 +113,57 @@ def get_files(inputs): elif path.suffix in markdown_extensions: yield path, path.parent -def add_inits_to_dir(path): + +def add_inits_along_path(from_path, to_path): """Recursively add __init__.py files to a directory - This compensates for https://bugs.python.org/issue23882 and https://bugs.python.org/issue35617 + This compensates for https://bugs.python.org/issue23882 + and https://bugs.python.org/issue35617 """ - for child in path.rglob('*'): - if child.is_dir(): - (child / '__init__.py').touch() + to_path = to_path.expanduser().resolve() + from_path = from_path.expanduser().resolve() + + # Sanity Check: This will raise an exception if paths aren't relative. + to_path.relative_to(from_path) + # Continue recursing if we haven't reached the base output directory. + if to_path != from_path: + (to_path / '__init__.py').touch() + add_inits_along_path(from_path, to_path.parent) @click.command() @click.argument( 'inputs', nargs=-1, required=True, type=click.Path(exists=True)) -@click.option('--output', default='{name}.py') +@click.option('--output', default='{name}.{ext}') @click.option('--github/--markdown', default=bool(not markdown_enabled), help='Github-flavored fence blocks or pure markdown.') @click.option('--safe/--unsafe', default=True, help='Allow code blocks without language hints.') @click.option('--package-python', default=True, - help='Add __init__.py files to python output to aid in test discovery') -def main(inputs, output, github, safe, package_python): + help='Add __init__.py files to python dirs for test discovery') +@click.option('--default-lang', default='py', + help='Assumed language for code blocks without language hints.') +def main(inputs, output, github, safe, package_python, default_lang): collect_codeblocks = github_codeblocks if github else markdown_codeblocks outputbasedir = Path(output).parent outputbasename = Path(output).name for filepath, input_path in get_files(inputs): - codeblocks = collect_codeblocks(filepath, safe) + codeblocks = collect_codeblocks(filepath, safe, default_lang) if codeblocks: fp = Path(filepath) filedir = fp.parent.relative_to(input_path) filename = fp.stem - outputfilename = outputbasedir / filedir / outputbasename.format(name=filename) - - outputfilename.parent.mkdir(parents=True, exist_ok=True) - outputfilename.write_text('\n\n'.join(codeblocks)) - if package_python: - add_inits_to_dir(outputbasedir) - + # stitch together the OUTPUT base directory with input directories + # add the file format at the end. + for lang, blocks in codeblocks.items(): + outputfilename = outputbasedir / filedir /\ + outputbasename.format(name=filename, ext=lang) + + # make sure path exists, don't care if it already does + outputfilename.parent.mkdir(parents=True, exist_ok=True) + outputfilename.write_text('\n\n'.join(blocks)) + if package_python and lang == 'py': + add_inits_along_path(outputbasedir, outputfilename.parent) diff --git a/tests/data/nest/more/why.md b/tests/data/nest/more/why.md index ff50e89..b43a790 100644 --- a/tests/data/nest/more/why.md +++ b/tests/data/nest/more/why.md @@ -1,6 +1,6 @@ # why? -We want to make sure that in more complext documentation structures, which may have multiple sub directories, we are still formatting name and paths correctly. +We want to make sure that in more complex documentation structures, which may have multiple sub directories, we are still formatting name and paths correctly. ```py import unittest diff --git a/tests/langdata/csharp.md b/tests/langdata/csharp.md new file mode 100644 index 0000000..b75d51c --- /dev/null +++ b/tests/langdata/csharp.md @@ -0,0 +1,34 @@ +# dotNet is still a thing + +What if you could provide a code sample here? + +```cs +public void Sum(int a, int b) +{ + return a + b; +} +``` + +And we know that it is testable. + +```csharp +[Testclass] +public class UnitTest1 +{ + [TestMethod] + public void TestMethod1() + { + //Arrange + ApplicationToTest.Calc ClassCalc = new ApplicationToTest.Calc(); + int expectedResult = 5; + + //Act + int result = ClassCalc.Sum(2,3); + + //Assert + Assert.AreEqual(expectedResult, result); + } +} +``` + +Actually checking and running these tests, that's a different matter. diff --git a/tests/langdata/java.md b/tests/langdata/java.md new file mode 100644 index 0000000..5641146 --- /dev/null +++ b/tests/langdata/java.md @@ -0,0 +1,32 @@ +# Java documentation is important + +That's a language still. Here's a java codeblock: + +```java +public class MyUnit { + public String concatenate(String one, String two){ + return one + two; + } +} +``` + +And since we have that class, let's test it + +```java +import org.junit.Test; +import static org.junit.Assert.*; + +public class MyUnitTest { + + @Test + public void testConcatenate() { + MyUnit myUnit = new MyUnit(); + + String result = myUnit.concatenate("one", "two"); + + assertEquals("onetwo", result); + + } +} + +``` diff --git a/tests/langdata/multilang.md b/tests/langdata/multilang.md new file mode 100644 index 0000000..8e15a3c --- /dev/null +++ b/tests/langdata/multilang.md @@ -0,0 +1,34 @@ +# Comparing and contrasting + +For some ideas about an api, we might give getting started code in a simple getting started page. + +In a pinch, let's hello that world. + +```py +print("hello, world") +``` + +But maybe we want this to be enterprise grade? + +```java +class HelloWorld { + public static void main(String[] args) { + System.out.println("Hello, World!"); + } +} +``` + +New orders from the CTO: let's use Azure cloud. +```cs +class HelloWorld { + static void Main() { + System.Console.WriteLine("Hello World"); + } +} +``` + +We want to have a react vue jquery frontend. Assume that the code sample below has a testable extension as the language + +```js +console.log('Hello, world"); +``` diff --git a/tests/langdata/no_py_tree/clean.md b/tests/langdata/no_py_tree/clean.md new file mode 100644 index 0000000..f72dbe8 --- /dev/null +++ b/tests/langdata/no_py_tree/clean.md @@ -0,0 +1,16 @@ +# Cleanliness + +If there are no python files in a directory, we don't need to add an __init__.py file to that directory. Sure, they don't hurt, but having them where they aren't needed isn't very tidy and might be confusing. + +Speaking of confusing, lets test javascript +```js +function assert(condition, message) { + if (!condition) { + message = message || "Assertion failed"; + throw new Error(message); + } +} + +assert([]+[]=="", "very sensible, adding arrays is a string") +assert({}+[]==0, "of course adding a dict to an array is 0") +``` diff --git a/tests/langdata/pytree/buried.md b/tests/langdata/pytree/buried.md new file mode 100644 index 0000000..46b780f --- /dev/null +++ b/tests/langdata/pytree/buried.md @@ -0,0 +1,12 @@ +# Test discovery + +For test discovery to work for unittest, python files generated from this document must have an `__init__.py` file added to the directory - otherwise they won't be considered testable packages. + +```python +import unittest + +class TestDiscovery(unittest.TestCase): + def test_discovery(self): + self.assertTrue(True) + +``` diff --git a/tests/test.py b/tests/test.py index 8b6ba0f..04c2ec8 100644 --- a/tests/test.py +++ b/tests/test.py @@ -140,6 +140,25 @@ def test_prefixed_deep_blocks(self): self.assertIn('Ran 2 tests', proc.stderr) self.assertIn('OK', proc.stderr) + def test_other_languages(self): + self.call( + '--output', 'tests/output/test_{name}.{ext}', + '--github', 'tests/langdata') + self.assertTrue(self._output_path_exists('test_java.java')) + self.assertTrue(self._output_path_exists('test_csharp.cs')) + self.assertFalse(self._output_path_exists('test_csharp.csharp')) + self.assertTrue(self._output_path_exists('test_multilang.cs')) + self.assertTrue(self._output_path_exists('test_multilang.java')) + self.assertTrue(self._output_path_exists('test_multilang.py')) + self.assertTrue(self._output_path_exists('test_multilang.js')) + self.assertTrue(self._output_path_exists('no_py_tree/test_clean.js')) + self.assertFalse(self._output_path_exists('no_py_tree/__init__.py')) + self.assertTrue(self._output_path_exists('pytree/test_buried.py')) + self.assertTrue(self._output_path_exists('pytree/__init__.py')) + + # __init__.py should not be created in the base output directory. + self.assertFalse(self._output_path_exists('__init__.py')) + @unittest.skip def test_glob(self): raise NotImplementedError