Skip to content

Commit

Permalink
Merge branch 'feature/block-primary-attr' into dev
Browse files Browse the repository at this point in the history
  • Loading branch information
sirosen committed Jul 28, 2014
2 parents 4022b1a + aac5b10 commit 4532b8e
Show file tree
Hide file tree
Showing 19 changed files with 253 additions and 81 deletions.
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ SALVE

Authors: Stephen Rosen

Version: 2.2.0
Version: 2.3.0

More information is available at [SALVE Official Site](http://salve.sirosen.net/ "SALVE")

Expand Down Expand Up @@ -39,8 +39,6 @@ Features
- Plugin framework
- Variable and attribute definition in manifest blocks to propogate down the block tree as defaults
- Automatic file backup recovery using dates, generation numbers, and so forth
- "Lightweight" block definitions based on a primary attribute, default
assignation of primary attribute for an ordinary block definition

Fixes
-----
Expand All @@ -50,6 +48,9 @@ Fixes

Changelog
=========
* 2.3.0
* Added support for Python 2.6 (with argparse installed)
* Added Primary Attribute style blocks
* 2.2.0
* Added Travis and Coveralls integration
* Improved internal logging and context handling
Expand Down
3 changes: 3 additions & 0 deletions salve/block/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ def __init__(self, ty, file_context):
# this is a set of attribute identifiers which must be present
# in order for the block to be valid
self.min_attrs = set()
# the primary attribute of a block is the one handled outside of the
# typically block body parsing, following the block identifier
self.primary_attr = None

def set(self, attribute_name, value):
"""
Expand Down
1 change: 1 addition & 0 deletions salve/block/directory_block.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ def __init__(self, file_context):
self.path_attrs.add(attr)
for attr in ['target']:
self.min_attrs.add(attr)
self.primary_attr = 'target'

def _mkdir(self, dirname):
"""
Expand Down
1 change: 1 addition & 0 deletions salve/block/file_block.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ def __init__(self, file_context):
self.path_attrs.add(attr)
for attr in ['target']:
self.min_attrs.add(attr)
self.primary_attr = 'target'

def compile(self):
"""
Expand Down
1 change: 1 addition & 0 deletions salve/block/manifest_block.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def __init__(self, file_context, source=None):
self.set('source', source)
self.path_attrs.add('source')
self.min_attrs.add('source')
self.primary_attr = 'source'

def expand_blocks(self, root_dir, config, ancestors=None):
"""
Expand Down
63 changes: 43 additions & 20 deletions salve/reader/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,52 +52,75 @@ def unexpected_token(token, expected_types):

# track the expected next token(s)
expected_token_types = [Token.types.IDENTIFIER]
# tracks whether or not parsing is inside of a "{" "}" delimited block
in_block = False

# the current_block and current_attr are used to build blocks
# before they are appended to the blocks list
current_block = None
current_attr = None

for token in tokens:
# if the token is unexpected, throw an exception and fail
if token.ty not in expected_token_types:
unexpected_token(token, expected_token_types)
# if there is no current block, the incoming token must
# be an identifier, so we can use it to construct a new block
elif not current_block:

# if the token is an identifier found outside of a block, it is the
# beginning of a new block
if not in_block and token.ty == Token.types.IDENTIFIER:
try:
b_from_id = salve.block.identifier.block_from_identifier
current_block = b_from_id(token)
blocks.append(current_block)
except:
raise ParsingException('Invalid block id ' +
token.value, token.file_context)
expected_token_types = [Token.types.BLOCK_START]
else:
# if the token is a block start, do nothing
expected_token_types = [Token.types.BLOCK_START,
Token.types.TEMPLATE]
# go back to loop start (other stuff might match, and we don't want
# it to, since this is the block's identifier)
continue

# if not in a block, look for a primary attr, or {
if not in_block:
# token.ty not in (BLOCK_END, IDENTIFIER)
# if the token is a block start, set in_block
if token.ty == Token.types.BLOCK_START:
in_block = True
expected_token_types = [Token.types.BLOCK_END,
Token.types.IDENTIFIER]
# if the token is a block end, add the current block to the
# list and set current_block to None
elif token.ty == Token.types.BLOCK_END:
blocks.append(current_block)
current_block = None
Token.types.IDENTIFIER]
# if the token is a template string, assign it to the
# primary attr
elif token.ty == Token.types.TEMPLATE:
expected_token_types = [Token.types.BLOCK_START,
Token.types.IDENTIFIER]
current_block.set(current_block.primary_attr, token.value)

# i.e. in_block==True
# look for block attribute,value pairs, or }
else:
# if the token is a block end, set current_block to None and
# set state to not be in block
if token.ty == Token.types.BLOCK_END:
in_block = False
expected_token_types = [Token.types.IDENTIFIER]
# if the token is an identifier, it is the name of an attr
# (because we're in a block)
elif token.ty == Token.types.IDENTIFIER:
current_attr = token.value.lower()
expected_token_types = [Token.types.TEMPLATE]
# if the token is a template string, assign it to the
# current attr
elif token.ty == Token.types.TEMPLATE:
current_block.set(current_attr, token.value)
expected_token_types = [Token.types.BLOCK_END,
Token.types.IDENTIFIER]
# no meaningful else because token types must be valid, as
# per the earlier check for valid token type
else:
raise ValueError('SALVE Internal Error!') # pragma: no cover
Token.types.IDENTIFIER]
current_block.set(current_attr, token.value)
current_attr = None

# if the token list terminates and there is still a block in
# progress, it means that the block was not teminated
if current_block is not None:
# progress or a TEMPLATE could come next, it means that the block was not
# teminated properly
if in_block or Token.types.TEMPLATE in expected_token_types:
# this PE carries no token because it is the absence of a token
# that triggers it
raise ParsingException('Incomplete block in token stream!',
Expand Down
49 changes: 40 additions & 9 deletions salve/reader/tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,14 +108,21 @@ def unexpected_token(token_str, expected, file_context):

"""
State definitions
FREE: Waiting for a block identifier
IDENTIFIER_FOUND: Got a block identifier, waiting for a {
FREE: Waiting for a block identifier or EOF
IDENTIFIER_FOUND: Got a block identifier, waiting for a { or primary
attr (template string)
PRIMARY_ATTR_FOUND: Found a primary block attr, next is either { or
another identifier, or EOF
BLOCK: Inside of a block, waiting for an attribute identifier
or }
IDENTIFIER_FOUND_BLOCK: Inside of a block, got an attribute
identifier, waiting for a template string value
"""
states = Enum('FREE', 'IDENTIFIER_FOUND', 'BLOCK',
states = Enum('FREE', 'IDENTIFIER_FOUND', 'PRIMARY_ATTR_FOUND', 'BLOCK',
'IDENTIFIER_FOUND_BLOCK')

filename = get_filename(stream)
Expand Down Expand Up @@ -162,12 +169,34 @@ def add_token(tok, ty, file_context):
state = states.IDENTIFIER_FOUND

# if we have found a block identifier, the next token must be
# a block start, '{'
# a block start, '{', or the primary attr
elif state is states.IDENTIFIER_FOUND:
if current != '{':
unexpected_token(current, Token.types.BLOCK_START, ctx)
add_token(current, Token.types.BLOCK_START, ctx)
state = states.BLOCK
# if it's a block open, cool
if current == '{':
add_token(current, Token.types.BLOCK_START, ctx)
state = states.BLOCK
# if it's a block close, uncool
elif current == '}':
unexpected_token(current, [Token.types.BLOCK_START,
Token.types.TEMPLATE], ctx)
# anything else must be primary attr
else:
add_token(current, Token.types.TEMPLATE, ctx)
state = states.PRIMARY_ATTR_FOUND

elif state is states.PRIMARY_ATTR_FOUND:
# if it's a block open, cool
if current == '{':
add_token(current, Token.types.BLOCK_START, ctx)
state = states.BLOCK
# if it's a block close, uncool
elif current == '}':
unexpected_token(current, [Token.types.BLOCK_START,
Token.types.TEMPLATE], ctx)
# anything else is a new block identifier, so no block body
else:
add_token(current, Token.types.IDENTIFIER, ctx)
state = states.IDENTIFIER_FOUND

# if we are in a block, the next token is either a block end,
# '}', or an attribute identifier
Expand All @@ -194,7 +223,9 @@ def add_token(tok, ty, file_context):
# get the next Maybe(Token)
current = tokenizer.get_token()

if state is not states.FREE:
# we can either be FREE (i.e. last saw a '}') or PRIMARY_ATTR_FOUND (i.e.
# last saw a '<block_id> <attr_val>') at the end of the file
if state not in (states.FREE, states.PRIMARY_ATTR_FOUND):
raise TokenizationException('Tokenizer ended in state ' +
state, ctx)

Expand Down
16 changes: 0 additions & 16 deletions tests/end2end/reader_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,22 +123,6 @@ def missing_open_raises_TE(self):
assert sctx.lineno == 5
assert sctx.filename == path

@istest
def double_identifier_raises_TE(self):
"""
E2E: Parse File With Repeated Block ID Raises Tokenization Exception
Not only validates that a Tokenization Exception occurs, but also
verifies the context of the raised exception.
"""
path = get_full_path('double_id.manifest')
e = ensure_except(salve.reader.tokenize.TokenizationException,
parse_filename,
path)
sctx = e.file_context
assert sctx.lineno == 5
assert sctx.filename == path

@istest
def missing_identifier_raises_TE(self):
"""
Expand Down
26 changes: 4 additions & 22 deletions tests/end2end/run/failure_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,28 +56,10 @@ def missing_open_fails(self):
argv = ['./salve.py', 'deploy', '-m', path]
e = except_from_args(argv)

assert self.stderr.getvalue() ==\
"[ERROR] [PARSING] %s, line 5: " % rpath +\
"Unexpected token: } Expected BLOCK_START instead.\n", \
self.stderr.getvalue()
assert e.code == 1, "incorrect error code: %d" % e.code

@istest
def double_identifier_fails(self):
"""
E2E: Run on File With Double Identifier Fails
Not only validates that a SystemExit occurs, but also
verifies the exit code and message of the raised exception.
"""
path = get_full_path('double_id.manifest')
rpath = os.path.relpath(path, '.')
argv = ['./salve.py', 'deploy', '-m', path]
e = except_from_args(argv)

assert self.stderr.getvalue() ==\
"[ERROR] [PARSING] %s, line 5: " % rpath +\
"Unexpected token: file Expected BLOCK_START instead.\n", \
assert self.stderr.getvalue() == (
"[ERROR] [PARSING] %s, line 5: " % rpath +
"Unexpected token: } " +
"Expected ['BLOCK_START', 'TEMPLATE'] instead.\n"), \
self.stderr.getvalue()
assert e.code == 1, "incorrect error code: %d" % e.code

Expand Down
6 changes: 2 additions & 4 deletions tests/unit/reader/files/invalid3.manifest
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
#double-identifier, no block
# block close following primary attr

file

file {}
file primary_attr_val }
3 changes: 3 additions & 0 deletions tests/unit/reader/files/invalid8.manifest
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# bare identifier fails

directory
2 changes: 1 addition & 1 deletion tests/unit/reader/files/valid3.manifest
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#a single empty file block
# a single non-empty file block

file {
source /a/b/c
Expand Down
5 changes: 5 additions & 0 deletions tests/unit/reader/files/valid4.manifest
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# a single file block with a primary attr

file "/d/e/f/g" {
source /a/b/c
}
8 changes: 8 additions & 0 deletions tests/unit/reader/files/valid5.manifest
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# primary_attr block, followed by non-primary_attr block

manifest "man man"

file {
source "potato"
target "mango"
}
9 changes: 9 additions & 0 deletions tests/unit/reader/files/valid6.manifest
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# multiple Primary Attribute style blocks

manifest "man man"

file "bat man"

directory "super man"

manifest "wo man"
3 changes: 3 additions & 0 deletions tests/unit/reader/files/valid7.manifest
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# primary attr block with empty body

manifest "man man" {}
5 changes: 5 additions & 0 deletions tests/unit/reader/files/valid8.manifest
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# primary_attr block with other attrs

file "lobster" {
source "salad"
}
Loading

0 comments on commit 4532b8e

Please sign in to comment.