Change marker token to "#@" as "#:" was showing up in the wild

Add ability to ignore a directory
Add more error catching

CHANGES.rst

@@ -1,3 +1,13 @@
+0.2.0
+=====
+
+* Add ability to ignore directories
+* Add some error handling to indicate what file has a problem for certain
+parsing errors
+* Change juli token marker from "#:" and "#::" to "#@" and "#@@" after
+discovering some code using the original in the wild
+
+
 0.1.0
 =====
 

README.rst

@@ -131,6 +131,7 @@ Additional, optional configuration values are:
 
 * ``chapter_prefix`` -- Specify what the prefix part of a chapter directory is named. If not specified, defaults to "ch"
 * ``python_globs`` -- A glob pattern that indicates which files participate in the parsing. Files that don't match will be copied without processing. If not specified it defaults to ``**/*.py``, meaning all files ending in "\*.py"
+* ``ignore_dirs`` -- A list of sub-directories that should not be processed.
 * ``[chapter_map]`` -- Chapter numbers are integers, but you may not always want that in your output structure. This map allows you to change the suffix part of a chapter directory name. Keys in the map are the chapter numbers while values are what should be used in the chapter suffix.
 * ``[subdir.XYZ]`` -- Whole directories can be marked as conditional using this TOML map. This map must specify ``range`` and ``src_dir`` attributes. The ``range`` attribute indicates what chapters this directory participates in, and the ``src_dir`` points to the conditional chapter. The ``XYZ`` portion of the nested map is ignored, it is there so you can have multiple conditional directories.
 
@@ -140,6 +141,7 @@ Here is a full example of a configuration file:
 
     output_dir = 'last_output'
     src_dir = 'code'
+    ignore_dirs = `bad_dir`
 
     chapter_prefix = "chap"
 
@@ -164,6 +166,7 @@ If your code directory contained:
     code/readme.txt
     code/between24/two_to_four.py
     code/after4/later_on.txt
+    code/bad_dir/something.py
 
 
 Then running ``juli`` with the sample configuration would result in the

clean_build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-version=`grep "__version__ = " julienne/__init__.py | cut -d "'" -f 2`
+version=`grep "__version__ = " src/julienne/__init__.py | cut -d "'" -f 2`
 
 git tag "$version"
 

load_tests.py

@@ -2,6 +2,8 @@
 import unittest, sys
 from waelstow import discover_tests
 
+sys.path.append('./src')
+
 def get_suite(labels=[]):
     return discover_tests('tests', labels)
 

pyflakes.sh

@@ -2,4 +2,4 @@
 
 echo "============================================================"
 echo "== pyflakes =="
-pyflakes julienne tests
+pyflakes src/julienne tests

src/julienne/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.0'
+__version__ = '0.2.0'

src/julienne/filemodel.py

@@ -81,7 +81,7 @@ def _parse_content(self, content):
 
         :param content: string to parse
         """
-        if content[-1] == '\n':
+        if content and content[-1] == '\n':
             content = content[:-1]
 
         self.lines = []
@@ -90,6 +90,7 @@ def _parse_content(self, content):
         boundary_set = False
         for item in content.split('\n'):
             line = Line(item, block_header)
+
             block_header = line.block_header
             self.lines.append(line)
 
@@ -138,6 +139,7 @@ def __init__(self):
         self.python_files = []
         self.conditional_dirs = []
         self.conditional_map = {}
+        self.ignore_dirs = []
 
     def set_python_filter(self, py_globs, base_dir):
         for py_glob in py_globs:
@@ -151,29 +153,39 @@ def set_dir_filter(self, subdir, base_path):
             self.conditional_dirs.append(dir_path)
             self.conditional_map[dir_path] = token
 
+    def set_ignore_dirs(self, ignore_dirs, base_path):
+        for dirname in ignore_dirs:
+            path = _convert_path(base_path, Path(dirname))
+            self.ignore_dirs.append(path)
+
 # ===========================================================================
 # Node Tree Traversal Methods
 # ===========================================================================
 
 def _process_directory(parent, dir_path, node_filter):
     for path in dir_path.iterdir():
-        if path.is_dir():
-            if path in node_filter.conditional_dirs:
-                token = node_filter.conditional_map[path]
-                node = ConditionalDirNode(path, token)
+        try:
+            if path.is_dir():
+                if path in node_filter.ignore_dirs:
+                    continue
+                elif path in node_filter.conditional_dirs:
+                    token = node_filter.conditional_map[path]
+                    node = ConditionalDirNode(path, token)
+                else:
+                    node = DirNode(path)
+
+                parent.children.append(node)
+                _process_directory(node, node.path, node_filter)
             else:
-                node = DirNode(path)
+                if path in node_filter.python_files:
+                    node = FileNode(path)
+                    node.parse_file()
+                else:
+                    node = CopyOnlyFileNode(path)
 
-            parent.children.append(node)
-            _process_directory(node, node.path, node_filter)
-        else:
-            if path in node_filter.python_files:
-                node = FileNode(path)
-                node.parse_file()
-            else:
-                node = CopyOnlyFileNode(path)
-
-            parent.children.append(node)
+                parent.children.append(node)
+        except Exception as e:
+            raise e.__class__(f"Error parsing {path}. " + str(e))
 
 
 def _traverse(chapter, node, cmd, *args):
@@ -191,14 +203,17 @@ def _traverse(chapter, node, cmd, *args):
 def _find_biggest(node, biggest=1):
     result = biggest
 
-    for child in node.children:
-        if isinstance(child, DirNode):
-            subresult = _find_biggest(child, biggest)
-            if subresult > result:
-                result = subresult
-        elif isinstance(child, FileNode):
-            if child.highest > result:
-                result = child.highest
+    try:
+        for child in node.children:
+            if isinstance(child, DirNode):
+                subresult = _find_biggest(child, biggest)
+                if subresult > result:
+                    result = subresult
+            elif isinstance(child, FileNode):
+                if hasattr(child, 'highest') and child.highest > result:
+                    result = child.highest
+    except Exception as e:
+        raise e.__class__(f"Problem occurred processing {child.path} " + str(e))
 
     return result
 
@@ -247,6 +262,9 @@ def generate_files(config_file):
     subdir = config.get('subdir', {})
     node_filter.set_dir_filter(subdir, base_path)
 
+    ignore_dirs = config.get('ignore_dirs', [])
+    node_filter.set_ignore_dirs(ignore_dirs, base_dir)
+
     # Create node structure
     root = DirNode(base_dir)
     _process_directory(root, base_dir, node_filter)

src/julienne/parser.py

@@ -51,7 +51,7 @@ def __init__(self, content, block_header=None):
         # Not a continuing block line, reset the header
         self.block_header = None
 
-        index = content.find("#:")
+        index = content.find("#@")
         if index == -1:
             # No juli comment, keep the line unconditionally
             self.conditional = False
@@ -64,7 +64,7 @@ def __init__(self, content, block_header=None):
         self.conditional = True
 
         marker = content[index:].strip()
-        if marker.startswith('#::'):
+        if marker.startswith('#@@'):
             # Line is a header for a continuing block
             self.block_header = self
             try:
@@ -74,7 +74,7 @@ def __init__(self, content, block_header=None):
                 # No space after marker, ignore this line
                 token = marker[3:]
                 self.content = None
-        else: # marker is a line marker, just "#:"
+        else: # marker is a line marker, just "#@"
             try:
                 token, rest = marker[2:].split(' ', 1)
                 self.content = f"{content[:index]}# {rest}"

tests/data/bad_code/bad_marker.py

@@ -0,0 +1 @@
+print("This should blow up")   #@ a comment without a token

tests/data/code/after4/amixed.py

@@ -1,15 +1,15 @@
 # This is a sample file
 
 a = "In all chapters"   # inline comment
-b = "In chapters 1-3"   #:1-3 comment on conditional
-c = "In chapters 1-2"   #:-2
-d = "In chapters 2 on"  #:2-
+b = "In chapters 1-3"   #@1-3 comment on conditional
+c = "In chapters 1-2"   #@-2
+d = "In chapters 2 on"  #@2-
 
-#::3-4
+#@@3-4
 #>e = "In chapters 3 to 4"  # inline comment
 #>f = "  as a block"
 
 for x in range(10):
-    #::1-2 block header with comment
+    #@@1-2 block header with comment
     #>g= "In chapters 1 and 2"
     h = "In all chapters"

tests/data/code/between24/bmixed.py

@@ -1,15 +1,15 @@
 # This is a sample file
 
 a = "In all chapters"   # inline comment
-b = "In chapters 1-3"   #:1-3 comment on conditional
-c = "In chapters 1-2"   #:-2
-d = "In chapters 2 on"  #:2-
+b = "In chapters 1-3"   #@1-3 comment on conditional
+c = "In chapters 1-2"   #@-2
+d = "In chapters 2 on"  #@2-
 
-#::3-4
+#@@3-4
 #>e = "In chapters 3 to 4"  # inline comment
 #>f = "  as a block"
 
 for x in range(10):
-    #::1-2 block header with comment
+    #@@1-2 block header with comment
     #>g= "In chapters 1 and 2"
     h = "In all chapters"

tests/data/code/between24/two_only.py

@@ -1,3 +1,3 @@
-#::2
+#@@2
 #>print("File and block only in 2")
 #>print("   this is a sub- block")

tests/data/code/condi.py

@@ -1,3 +1,3 @@
-#::2-3
+#@@2-3
 #>e = "In chapters 2 and 3 only"
 #>f = "  as a block"

tests/data/code/maxi.py

@@ -1 +1 @@
-print("only in chapter 5")    #:5
+print("only in chapter 5")    #@5

tests/data/code/mixed.py

@@ -1,15 +1,15 @@
 # This is a sample file
 
 a = "In all chapters"   # inline comment
-b = "In chapters 1-3"   #:1-3 comment on conditional
-c = "In chapters 1-2"   #:-2
-d = "In chapters 2 on"  #:2-
+b = "In chapters 1-3"   #@1-3 comment on conditional
+c = "In chapters 1-2"   #@-2
+d = "In chapters 2 on"  #@2-
 
-#::3-4
+#@@3-4
 #>e = "In chapters 3 to 4"  # inline comment
 #>f = "  as a block"
 
 for x in range(10):
-    #::1-2 block header with comment
+    #@@1-2 block header with comment
     #>g= "In chapters 1 and 2"
     h = "In all chapters"

tests/data/code/not_here/nope.py

@@ -0,0 +1 @@
+# This file should not be found as the directory is ignored

tests/data/code/under/umixed.py

@@ -1,15 +1,15 @@
 # This is a sample file
 
 a = "In all chapters"   # inline comment
-b = "In chapters 1-3"   #:1-3 comment on conditional
-c = "In chapters 1-2"   #:-2
-d = "In chapters 2 on"  #:2-
+b = "In chapters 1-3"   #@1-3 comment on conditional
+c = "In chapters 1-2"   #@-2
+d = "In chapters 2 on"  #@2-
 
-#::3-4
+#@@3-4
 #>e = "In chapters 3 to 4"  # inline comment
 #>f = "  as a block"
 
 for x in range(10):
-    #::1-2 block header with comment
+    #@@1-2 block header with comment
     #>g= "In chapters 1 and 2"
     h = "In all chapters"

tests/data/fail.toml

@@ -0,0 +1,2 @@
+output_dir = 'does_not_matter'
+src_dir = 'bad_code'

tests/data/sample.toml

@@ -1,6 +1,7 @@
 output_dir = 'last_output'
 src_dir = 'code'
 chapter_prefix = "chap"
+ignore_dirs = ['not_here', ]
 
 [chapter_map]
 4 = 'Four'

tests/test_files.py

@@ -24,3 +24,14 @@ def test_e2e(self):
         expected = here / Path('data/expected')
         result = dircmp(output, expected)
         self.assertEqual(result.diff_files, [])
+
+    def test_failures(self):
+        here = Path(__file__).parent
+        path = here / Path('data/fail.toml')
+
+        with self.assertRaises(Exception) as context:
+            generate_files(str(path))
+
+        error = context.exception
+        self.assertIn("bad_code/bad_marker.py", str(error))
+        self.assertIn("comment marker could not be parsed", str(error))

tests/test_parser.py

@@ -6,14 +6,14 @@
 # ============================================================================
 
 BLOCK1 = """\
-#::3-4
+#@@3-4
 #>e = "In chapters 3 to 4"  # inline comment
 #>f = "  as a block"\
 """
 
 BLOCK2 = """\
 for x in range(10):
-    #::1-2 block header with comment
+    #@@1-2 block header with comment
     #>g= "In chapters 1 and 2"
     h = "In all chapters"\
 """
@@ -53,25 +53,25 @@ def test_line_parsing(self):
         self.assertLine(line, False, False, text)
 
         # Test a full-range conditional line with inline comment
-        text = 'b = "In chapters 1-3"   #:1-3 comment on conditional'
+        text = 'b = "In chapters 1-3"   #@1-3 comment on conditional'
         expected = 'b = "In chapters 1-3"   # comment on conditional'
         line = Line(text)
         self.assertLine(line, True, False, expected, 1, 3)
 
         # Test lower open range conditional line
-        text = 'c = "In chapters 1-2"   #:-2'
+        text = 'c = "In chapters 1-2"   #@-2'
         expected = 'c = "In chapters 1-2"   '
         line = Line(text)
         self.assertLine(line, True, False, expected, 1, 2)
 
         # Test upper open range conditional line
-        text = 'd = "In chapters 2 on"  #:2-'
+        text = 'd = "In chapters 2 on"  #@2-'
         expected = 'd = "In chapters 2 on"  '
         line = Line(text)
         self.assertLine(line, True, False, expected, 2, -1)
 
     def test_bad_parsing(self):
-        text = 'x = 3 #:'
+        text = 'x = 3 #@'
         with self.assertRaises(ValueError):
             Line(text)