Merge branch 'feature/block-primary-attr' into dev

README.md

@@ -6,7 +6,7 @@ SALVE
 
 Authors: Stephen Rosen
 
-Version: 2.2.0
+Version: 2.3.0
 
 More information is available at [SALVE Official Site](http://salve.sirosen.net/ "SALVE")
 
@@ -39,8 +39,6 @@ Features
  - Plugin framework
  - Variable and attribute definition in manifest blocks to propogate down the block tree as defaults
  - Automatic file backup recovery using dates, generation numbers, and so forth
- - "Lightweight" block definitions based on a primary attribute, default
-     assignation of primary attribute for an ordinary block definition
 
 Fixes
 -----
@@ -50,6 +48,9 @@ Fixes
 
 Changelog
 =========
+ * 2.3.0
+    * Added support for Python 2.6 (with argparse installed)
+    * Added Primary Attribute style blocks
  * 2.2.0
     * Added Travis and Coveralls integration
     * Improved internal logging and context handling

salve/block/base.py

@@ -64,6 +64,9 @@ def __init__(self, ty, file_context):
         # this is a set of attribute identifiers which must be present
         # in order for the block to be valid
         self.min_attrs = set()
+        # the primary attribute of a block is the one handled outside of the
+        # typically block body parsing, following the block identifier
+        self.primary_attr = None
 
     def set(self, attribute_name, value):
         """

salve/block/directory_block.py

@@ -31,6 +31,7 @@ def __init__(self, file_context):
             self.path_attrs.add(attr)
         for attr in ['target']:
             self.min_attrs.add(attr)
+        self.primary_attr = 'target'
 
     def _mkdir(self, dirname):
         """

salve/block/file_block.py

@@ -31,6 +31,7 @@ def __init__(self, file_context):
             self.path_attrs.add(attr)
         for attr in ['target']:
             self.min_attrs.add(attr)
+        self.primary_attr = 'target'
 
     def compile(self):
         """

salve/block/manifest_block.py

@@ -37,6 +37,7 @@ def __init__(self, file_context, source=None):
             self.set('source', source)
         self.path_attrs.add('source')
         self.min_attrs.add('source')
+        self.primary_attr = 'source'
 
     def expand_blocks(self, root_dir, config, ancestors=None):
         """

salve/reader/parse.py

@@ -52,52 +52,75 @@ def unexpected_token(token, expected_types):
 
     # track the expected next token(s)
     expected_token_types = [Token.types.IDENTIFIER]
+    # tracks whether or not parsing is inside of a "{" "}" delimited block
+    in_block = False
+
     # the current_block and current_attr are used to build blocks
     # before they are appended to the blocks list
     current_block = None
     current_attr = None
+
     for token in tokens:
         # if the token is unexpected, throw an exception and fail
         if token.ty not in expected_token_types:
             unexpected_token(token, expected_token_types)
-        # if there is no current block, the incoming token must
-        # be an identifier, so we can use it to construct a new block
-        elif not current_block:
+
+        # if the token is an identifier found outside of a block, it is the
+        # beginning of a new block
+        if not in_block and token.ty == Token.types.IDENTIFIER:
             try:
                 b_from_id = salve.block.identifier.block_from_identifier
                 current_block = b_from_id(token)
+                blocks.append(current_block)
             except:
                 raise ParsingException('Invalid block id ' +
                     token.value, token.file_context)
-            expected_token_types = [Token.types.BLOCK_START]
-        else:
-            # if the token is a block start, do nothing
+            expected_token_types = [Token.types.BLOCK_START,
+                                    Token.types.TEMPLATE]
+            # go back to loop start (other stuff might match, and we don't want
+            # it to, since this is the block's identifier)
+            continue
+
+        # if not in a block, look for a primary attr, or {
+        if not in_block:
+            # token.ty not in (BLOCK_END, IDENTIFIER)
+            # if the token is a block start, set in_block
             if token.ty == Token.types.BLOCK_START:
+                in_block = True
                 expected_token_types = [Token.types.BLOCK_END,
-                                         Token.types.IDENTIFIER]
-            # if the token is a block end, add the current block to the
-            # list and set current_block to None
-            elif token.ty == Token.types.BLOCK_END:
-                blocks.append(current_block)
-                current_block = None
+                                        Token.types.IDENTIFIER]
+            # if the token is a template string, assign it to the
+            # primary attr
+            elif token.ty == Token.types.TEMPLATE:
+                expected_token_types = [Token.types.BLOCK_START,
+                                        Token.types.IDENTIFIER]
+                current_block.set(current_block.primary_attr, token.value)
+
+        # i.e. in_block==True
+        # look for block attribute,value pairs, or }
+        else:
+            # if the token is a block end, set current_block to None and
+            # set state to not be in block
+            if token.ty == Token.types.BLOCK_END:
+                in_block = False
                 expected_token_types = [Token.types.IDENTIFIER]
             # if the token is an identifier, it is the name of an attr
+            # (because we're in a block)
             elif token.ty == Token.types.IDENTIFIER:
                 current_attr = token.value.lower()
                 expected_token_types = [Token.types.TEMPLATE]
             # if the token is a template string, assign it to the
             # current attr
             elif token.ty == Token.types.TEMPLATE:
-                current_block.set(current_attr, token.value)
                 expected_token_types = [Token.types.BLOCK_END,
-                                         Token.types.IDENTIFIER]
-            # no meaningful else because token types must be valid, as
-            # per the earlier check for valid token type
-            else:
-                raise ValueError('SALVE Internal Error!')  # pragma: no cover
+                                        Token.types.IDENTIFIER]
+                current_block.set(current_attr, token.value)
+                current_attr = None
+
     # if the token list terminates and there is still a block in
-    # progress, it means that the block was not teminated
-    if current_block is not None:
+    # progress or a TEMPLATE could come next, it means that the block was not
+    # teminated properly
+    if in_block or Token.types.TEMPLATE in expected_token_types:
         # this PE carries no token because it is the absence of a token
         # that triggers it
         raise ParsingException('Incomplete block in token stream!',

salve/reader/tokenize.py

@@ -108,14 +108,21 @@ def unexpected_token(token_str, expected, file_context):
 
     """
     State definitions
-        FREE: Waiting for a block identifier
-        IDENTIFIER_FOUND: Got a block identifier, waiting for a {
+        FREE: Waiting for a block identifier or EOF
+
+        IDENTIFIER_FOUND: Got a block identifier, waiting for a { or primary
+        attr (template string)
+
+        PRIMARY_ATTR_FOUND: Found a primary block attr, next is either { or
+        another identifier, or EOF
+
         BLOCK: Inside of a block, waiting for an attribute identifier
             or }
+
         IDENTIFIER_FOUND_BLOCK: Inside of a block, got an attribute
             identifier, waiting for a template string value
     """
-    states = Enum('FREE', 'IDENTIFIER_FOUND', 'BLOCK',
+    states = Enum('FREE', 'IDENTIFIER_FOUND', 'PRIMARY_ATTR_FOUND', 'BLOCK',
                   'IDENTIFIER_FOUND_BLOCK')
 
     filename = get_filename(stream)
@@ -162,12 +169,34 @@ def add_token(tok, ty, file_context):
             state = states.IDENTIFIER_FOUND
 
         # if we have found a block identifier, the next token must be
-        # a block start, '{'
+        # a block start, '{', or the primary attr
         elif state is states.IDENTIFIER_FOUND:
-            if current != '{':
-                unexpected_token(current, Token.types.BLOCK_START, ctx)
-            add_token(current, Token.types.BLOCK_START, ctx)
-            state = states.BLOCK
+            # if it's a block open, cool
+            if current == '{':
+                add_token(current, Token.types.BLOCK_START, ctx)
+                state = states.BLOCK
+            # if it's a block close, uncool
+            elif current == '}':
+                unexpected_token(current, [Token.types.BLOCK_START,
+                    Token.types.TEMPLATE], ctx)
+            # anything else must be  primary attr
+            else:
+                add_token(current, Token.types.TEMPLATE, ctx)
+                state = states.PRIMARY_ATTR_FOUND
+
+        elif state is states.PRIMARY_ATTR_FOUND:
+            # if it's a block open, cool
+            if current == '{':
+                add_token(current, Token.types.BLOCK_START, ctx)
+                state = states.BLOCK
+            # if it's a block close, uncool
+            elif current == '}':
+                unexpected_token(current, [Token.types.BLOCK_START,
+                    Token.types.TEMPLATE], ctx)
+            # anything else is a new block identifier, so no block body
+            else:
+                add_token(current, Token.types.IDENTIFIER, ctx)
+                state = states.IDENTIFIER_FOUND
 
         # if we are in a block, the next token is either a block end,
         # '}', or an attribute identifier
@@ -194,7 +223,9 @@ def add_token(tok, ty, file_context):
         # get the next Maybe(Token)
         current = tokenizer.get_token()
 
-    if state is not states.FREE:
+    # we can either be FREE (i.e. last saw a '}') or PRIMARY_ATTR_FOUND (i.e.
+    # last saw a '<block_id> <attr_val>') at the end of the file
+    if state not in (states.FREE, states.PRIMARY_ATTR_FOUND):
         raise TokenizationException('Tokenizer ended in state ' +
                                      state, ctx)
 

tests/end2end/reader_test.py

@@ -123,22 +123,6 @@ def missing_open_raises_TE(self):
         assert sctx.lineno == 5
         assert sctx.filename == path
 
-    @istest
-    def double_identifier_raises_TE(self):
-        """
-        E2E: Parse File With Repeated Block ID Raises Tokenization Exception
-
-        Not only validates that a Tokenization Exception occurs, but also
-        verifies the context of the raised exception.
-        """
-        path = get_full_path('double_id.manifest')
-        e = ensure_except(salve.reader.tokenize.TokenizationException,
-                          parse_filename,
-                          path)
-        sctx = e.file_context
-        assert sctx.lineno == 5
-        assert sctx.filename == path
-
     @istest
     def missing_identifier_raises_TE(self):
         """

tests/end2end/run/failure_test.py

@@ -56,28 +56,10 @@ def missing_open_fails(self):
         argv = ['./salve.py', 'deploy', '-m', path]
         e = except_from_args(argv)
 
-        assert self.stderr.getvalue() ==\
-            "[ERROR] [PARSING] %s, line 5: " % rpath +\
-            "Unexpected token: } Expected BLOCK_START instead.\n", \
-            self.stderr.getvalue()
-        assert e.code == 1, "incorrect error code: %d" % e.code
-
-    @istest
-    def double_identifier_fails(self):
-        """
-        E2E: Run on File With Double Identifier Fails
-
-        Not only validates that a SystemExit occurs, but also
-        verifies the exit code and message of the raised exception.
-        """
-        path = get_full_path('double_id.manifest')
-        rpath = os.path.relpath(path, '.')
-        argv = ['./salve.py', 'deploy', '-m', path]
-        e = except_from_args(argv)
-
-        assert self.stderr.getvalue() ==\
-            "[ERROR] [PARSING] %s, line 5: " % rpath +\
-            "Unexpected token: file Expected BLOCK_START instead.\n", \
+        assert self.stderr.getvalue() == (
+                "[ERROR] [PARSING] %s, line 5: " % rpath +
+                "Unexpected token: } " +
+                "Expected ['BLOCK_START', 'TEMPLATE'] instead.\n"), \
             self.stderr.getvalue()
         assert e.code == 1, "incorrect error code: %d" % e.code
 

tests/unit/reader/files/invalid3.manifest

@@ -1,5 +1,3 @@
-#double-identifier, no block
+# block close following primary attr
 
-file
-
-file {}
+file primary_attr_val }

tests/unit/reader/files/invalid8.manifest

@@ -0,0 +1,3 @@
+# bare identifier fails
+
+directory

tests/unit/reader/files/valid3.manifest

@@ -1,4 +1,4 @@
-#a single empty file block
+# a single non-empty file block
 
 file {
     source /a/b/c

tests/unit/reader/files/valid4.manifest

@@ -0,0 +1,5 @@
+# a single file block with a primary attr
+
+file "/d/e/f/g" {
+    source /a/b/c
+}

tests/unit/reader/files/valid5.manifest

@@ -0,0 +1,8 @@
+# primary_attr block, followed by non-primary_attr block
+
+manifest "man man"
+
+file {
+    source "potato"
+    target "mango"
+}

tests/unit/reader/files/valid6.manifest

@@ -0,0 +1,9 @@
+# multiple Primary Attribute style blocks
+
+manifest "man man"
+
+file "bat man"
+
+directory "super man"
+
+manifest "wo man"

tests/unit/reader/files/valid7.manifest

@@ -0,0 +1,3 @@
+# primary attr block with empty body
+
+manifest "man man" {}

tests/unit/reader/files/valid8.manifest

@@ -0,0 +1,5 @@
+# primary_attr block with other attrs
+
+file "lobster" {
+    source "salad"
+}