diff --git a/README.rst b/README.rst index 8eb76ab..ca2a3cf 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,7 @@ Reliable and fast NGINX configuration file parser. - `Command Line Tool`_ - `crossplane parse`_ + - `crossplane build`_ - `crossplane lex`_ - `crossplane format`_ - `crossplane minify`_ @@ -17,6 +18,7 @@ Reliable and fast NGINX configuration file parser. - `Python Module`_ - `crossplane.parse()`_ + - `crossplane.build()`_ - `crossplane.lex()`_ - `Contributing`_ @@ -313,6 +315,31 @@ The second, ``--tb-onerror``, will add a ``"callback"`` key to all error objects a string representation of the traceback that would have been raised by the parser if the exception had not been caught. This can be useful for logging purposes. +crossplane build +---------------- + +.. code-block:: + + usage: crossplane build [-h] [-d PATH] [-f] [-i NUM | -t] [--no-headers] + [--stdout] [-v] + filename + + builds an nginx config from a json payload + + positional arguments: + filename the file with the config payload + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + -d PATH, --dir PATH the base directory to build in + -f, --force overwrite existing files + -i NUM, --indent NUM number of spaces to indent output + -t, --tabs indent with tabs instead of spaces + --no-headers do not write header to configs + --stdout write configs to stdout instead + + crossplane lex -------------- @@ -419,18 +446,35 @@ crossplane.parse() .. code-block:: python import crossplane - crossplane.parse('/etc/nginx/nginx.conf') + payload = crossplane.parse('/etc/nginx/nginx.conf') This will return the same payload as described in the `crossplane parse`_ section, except it will be Python dicts and not one giant JSON string. +crossplane.build() +------------------ + +.. code-block:: python + + import crossplane + config = crossplane.build( + [{ + "directive": "events", + "args": [], + "block": [{ + "directive": "worker_connections", + "args": ["1024"] + }] + }] + ) + crossplane.lex() ---------------- .. code-block:: python import crossplane - crossplane.lex('/etc/nginx/nginx.conf') + tokens = crossplane.lex('/etc/nginx/nginx.conf') ``crossplane.lex`` generates 2-tuples. Inserting these pairs into a list will result in a long list similar to what you can see in the `crossplane lex`_ section when the ``--line-numbers`` flag is used, except it diff --git a/crossplane/__init__.py b/crossplane/__init__.py index 2f51a03..b3923c2 100644 --- a/crossplane/__init__.py +++ b/crossplane/__init__.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- from .parser import parse from .lexer import lex +from .builder import build -__all__ = ['parse', 'lex'] +__all__ = ['parse', 'lex', 'build'] __title__ = 'crossplane' __summary__ = 'Reliable and fast NGINX configuration file parser.' __url__ = 'https://github.com/nginxinc/crossplane' -__version__ = '0.1.3' +__version__ = '0.2.0' __author__ = 'Arie van Luttikhuizen' __email__ = 'aluttik@gmail.com' diff --git a/crossplane/__main__.py b/crossplane/__main__.py index 16fe34a..244f5e3 100644 --- a/crossplane/__main__.py +++ b/crossplane/__main__.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import os import sys from argparse import ArgumentParser, FileType, RawDescriptionHelpFormatter @@ -8,60 +9,16 @@ from . import __version__ from .lexer import lex as lex_file from .parser import parse as parse_file +from .builder import build as build_file, _enquote, DELIMITERS from .errors import NgxParserBaseException -from .compat import PY2, json +from .compat import PY2, json, input -DELIMITERS = ('{', '}', ';') - -def _escape(string): - prev, char = '', '' - for char in string: - if prev == '\\' or prev + char == '${': - prev += char - yield prev - continue - if prev == '$': - yield prev - if char not in ('\\', '$'): - yield char - prev = char - if char in ('\\', '$'): - yield char - - -def _needs_quotes(string): - if string == '': - return True - elif string in DELIMITERS: - return False - - # lexer should throw an error when variable expansion syntax - # is messed up, but just wrap it in quotes for now I guess - chars = _escape(string) - - # arguments can't start with variable expansion syntax - char = next(chars) - if char.isspace() or char in ('{', ';', '"', "'", '${'): - return True - - expanding = False - for char in chars: - if char.isspace() or char in ('{', ';', '"', "'"): - return True - elif char == ('${' if expanding else '}'): - return True - elif char == ('}' if expanding else '${'): - expanding = not expanding - - return char in ('\\', '$') or expanding - - -def _enquote(arg): - arg = str(arg.encode('utf-8') if PY2 else arg) - if _needs_quotes(arg): - arg = repr(arg.decode('string_escape') if PY2 else arg) - return arg +def _prompt_yes(): + try: + return input('overwrite? (y/n [n]) ').lower().startswith('y') + except (KeyboardInterrupt, EOFError): + sys.exit(1) def _dump_payload(obj, fp, indent): @@ -86,6 +43,66 @@ def callback(e): _dump_payload(payload, out, indent=indent) +def build(filename, dirname, force, indent, tabs, header, stdout, verbose): + with open(filename, 'r') as fp: + payload = json.load(fp) + + if dirname is None: + dirname = os.getcwd() + + existing = [] + dirs_to_make = [] + + # find which files from the json payload will overwrite existing files and + # which directories need to be created in order for the config to be built + for config in payload['config']: + path = config['file'] + if not os.path.isabs(path): + path = os.path.join(dirname, path) + dirpath = os.path.dirname(path) + if os.path.exists(path): + existing.append(path) + elif not os.path.exists(dirpath) and dirpath not in dirs_to_make: + dirs_to_make.append(dirpath) + + # ask the user if it's okay to overwrite existing files + if existing and not force and not stdout: + print('building {} would overwrite these files:'.format(filename)) + print('\n'.join(existing)) + if not _prompt_yes(): + print('not overwritten') + return + + # make directories necessary for the config to be built + for dirpath in dirs_to_make: + os.makedirs(dirpath) + + # build the nginx configuration file from the json payload + for config in payload['config']: + path = os.path.join(dirname, config['file']) + + if header: + output = ( + '# This config was built from JSON using NGINX crossplane.\n' + '# If you encounter any bugs please report them here:\n' + '# https://github.com/nginxinc/crossplane/issues\n' + '\n' + ) + else: + output = '' + + parsed = config['parsed'] + output += build_file(parsed, indent, tabs) + '\n' + + if stdout: + print('# ' + path + '\n' + output) + else: + with open(path, 'w') as fp: + fp.write(output) + if verbose: + print('wrote to ' + path) + + def lex(filename, out, indent=None, line_numbers=False): payload = list(lex_file(filename)) if not line_numbers: @@ -105,36 +122,11 @@ def minify(filename, out): def format(filename, out, indent=None, tabs=False): - padding = '\t' if tabs else ' ' * indent - - def _format(objs, depth): - margin = padding * depth - - for obj in objs: - directive = obj['directive'] - args = [_enquote(arg) for arg in obj['args']] - - if directive == 'if': - line = 'if (' + ' '.join(args) + ')' - elif args: - line = directive + ' ' + ' '.join(args) - else: - line = directive - - if obj.get('block') is None: - yield margin + line + ';' - else: - yield margin + line + ' {' - for line in _format(obj['block'], depth=depth+1): - yield line - yield margin + '}' - payload = parse_file(filename) - + parsed = payload['config'][0]['parsed'] if payload['status'] == 'ok': - config = payload['config'][0]['parsed'] - lines = _format(config, depth=0) - out.write('\n'.join(lines) + '\n') + output = build_file(parsed, indent, tabs) + '\n' + out.write(output) else: e = payload['errors'][0] raise NgxParserBaseException(e['error'], e['file'], e['line']) @@ -179,6 +171,17 @@ def create_subparser(function, help): p.add_argument('--tb-onerror', action='store_true', help='include tracebacks in config errors') p.add_argument('--single-file', action='store_true', dest='single', help='do not include other config files') + p = create_subparser(build, 'builds an nginx config from a json payload') + p.add_argument('filename', help='the file with the config payload') + p.add_argument('-v', '--verbose', action='store_true', help='verbose output') + p.add_argument('-d', '--dir', metavar='PATH', default=None, dest='dirname', help='the base directory to build in') + p.add_argument('-f', '--force', action='store_true', help='overwrite existing files') + g = p.add_mutually_exclusive_group() + g.add_argument('-i', '--indent', type=int, metavar='NUM', help='number of spaces to indent output', default=4) + g.add_argument('-t', '--tabs', action='store_true', help='indent with tabs instead of spaces') + p.add_argument('--no-headers', action='store_false', dest='header', help='do not write header to configs') + p.add_argument('--stdout', action='store_true', help='write configs to stdout instead') + p = create_subparser(lex, 'lexes tokens from an nginx config file') p.add_argument('filename', help='the nginx config file') p.add_argument('-o', '--out', type=FileType('w'), default='-', help='write output to a file') @@ -197,10 +200,10 @@ def create_subparser(function, help): g.add_argument('-t', '--tabs', action='store_true', help='indent with tabs instead of spaces') def help(command): - if command not in parser._actions[1].choices: + if command not in parser._actions[-1].choices: parser.error('unknown command %r' % command) else: - parser._actions[1].choices[command].print_help() + parser._actions[-1].choices[command].print_help() p = create_subparser(help, 'show help for commands') p.add_argument('command', help='command to show help for') diff --git a/crossplane/builder.py b/crossplane/builder.py new file mode 100644 index 0000000..3dea54a --- /dev/null +++ b/crossplane/builder.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +import codecs +import os + +from .lexer import lex +from .analyzer import analyze, enter_block_ctx +from .errors import NgxParserDirectiveError +from .compat import PY2, json + +DELIMITERS = ('{', '}', ';') + + +def _escape(string): + prev, char = '', '' + for char in string: + if prev == '\\' or prev + char == '${': + prev += char + yield prev + continue + if prev == '$': + yield prev + if char not in ('\\', '$'): + yield char + prev = char + if char in ('\\', '$'): + yield char + + +def _needs_quotes(string): + if string == '': + return True + elif string in DELIMITERS: + return False + + # lexer should throw an error when variable expansion syntax + # is messed up, but just wrap it in quotes for now I guess + chars = _escape(string) + + # arguments can't start with variable expansion syntax + char = next(chars) + if char.isspace() or char in ('{', ';', '"', "'", '${'): + return True + + expanding = False + for char in chars: + if char.isspace() or char in ('{', ';', '"', "'"): + return True + elif char == ('${' if expanding else '}'): + return True + elif char == ('}' if expanding else '${'): + expanding = not expanding + + return char in ('\\', '$') or expanding + + +def _enquote(arg): + if _needs_quotes(arg): + arg = repr(codecs.decode(arg, 'raw_unicode_escape')) + arg = arg.replace('\\\\', '\\').lstrip('u') + return arg + + +def build(payload, indent=4, tabs=False): + padding = '\t' if tabs else ' ' * indent + + def _build_lines(objs, depth): + margin = padding * depth + + for obj in objs: + directive = obj['directive'] + args = [_enquote(arg) for arg in obj['args']] + + if directive == 'if': + line = 'if (' + ' '.join(args) + ')' + elif args: + line = directive + ' ' + ' '.join(args) + else: + line = directive + + if obj.get('block') is None: + yield margin + line + ';' + else: + yield margin + line + ' {' + for line in _build_lines(obj['block'], depth+1): + yield line + yield margin + '}' + + lines = _build_lines(payload, depth=0) + return '\n'.join(lines) diff --git a/crossplane/compat.py b/crossplane/compat.py index d9b8972..2d320d6 100644 --- a/crossplane/compat.py +++ b/crossplane/compat.py @@ -8,3 +8,10 @@ PY2 = (sys.version_info[0] == 2) PY3 = (sys.version_info[0] == 3) + +if PY2: + input = raw_input + basestring = basestring +else: + input = input + basestring = str diff --git a/tests/configs/messy/nginx.conf b/tests/configs/messy/nginx.conf index 12bf364..6ea6e7b 100644 --- a/tests/configs/messy/nginx.conf +++ b/tests/configs/messy/nginx.conf @@ -14,7 +14,7 @@ user nobody; location ~ "/hello/world;"{"return" 301 /status.html;} location /foo{}location /bar{} location /\{\;\}\ #\ ab {}# hello - if ($request_method = P\{O\)\###\;"S"T ){} + if ($request_method = P\{O\)\###\;ST ){} location "/status.html" { try_files /abc/${uri} /abc/${uri}.html =404 ; } diff --git a/tests/test_build.py b/tests/test_build.py new file mode 100644 index 0000000..2cdf7d0 --- /dev/null +++ b/tests/test_build.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +import os + +import crossplane +from crossplane.compat import basestring +from crossplane.builder import _enquote + +here = os.path.dirname(__file__) + + +def assert_equal_payloads(a, b, ignore_keys=()): + assert type(a) == type(b) + if isinstance(a, list): + assert len(a) == len(b) + for args in zip(a, b): + assert_equal_payloads(*args, ignore_keys=ignore_keys) + elif isinstance(a, dict): + keys = set(a.keys()) | set(b.keys()) + keys.difference_update(ignore_keys) + for key in keys: + assert_equal_payloads(a[key], b[key], ignore_keys=ignore_keys) + elif isinstance(a, basestring): + assert _enquote(a) == _enquote(b) + else: + assert a == b + + +def compare_parsed_and_built(conf_dirname, conf_basename, tmpdir): + original_dirname = os.path.join(here, 'configs', conf_dirname) + original_path = os.path.join(original_dirname, conf_basename) + original_payload = crossplane.parse(original_path) + original_parsed = original_payload['config'][0]['parsed'] + + build1_config = crossplane.build(original_parsed) + build1_file = tmpdir.join('build1.conf') + build1_file.write(build1_config) + build1_payload = crossplane.parse(build1_file.strpath) + build1_parsed = build1_payload['config'][0]['parsed'] + + assert_equal_payloads(original_parsed, build1_parsed, ignore_keys=['line']) + + build2_config = crossplane.build(build1_parsed) + build2_file = tmpdir.join('build2.conf') + build2_file.write(build2_config) + build2_payload = crossplane.parse(build2_file.strpath) + build2_parsed = build2_payload['config'][0]['parsed'] + + assert build1_config == build2_config + assert_equal_payloads(build1_parsed, build2_parsed, ignore_keys=[]) + + +def test_build_nested_and_multiple_args(): + payload = [ + { + "directive": "events", + "args": [], + "block": [ + { + "directive": "worker_connections", + "args": ["1024"] + } + ] + }, + { + "directive": "http", + "args": [], + "block": [ + { + "directive": "server", + "args": [], + "block": [ + { + "directive": "listen", + "args": ["127.0.0.1:8080"] + }, + { + "directive": "server_name", + "args": ["default_server"] + }, + { + "directive": "location", + "args": ["/"], + "block": [ + { + "directive": "return", + "args": ["200", "foo bar baz"] + } + ] + } + ] + } + ] + } + ] + + built = crossplane.build(payload, indent=4, tabs=False) + + assert built == '\n'.join([ + 'events {', + ' worker_connections 1024;', + '}', + 'http {', + ' server {', + ' listen 127.0.0.1:8080;', + ' server_name default_server;', + ' location / {', + " return 200 'foo bar baz';", + ' }', + ' }', + '}' + ]) + + +def test_compare_parsed_and_built_simple(tmpdir): + compare_parsed_and_built('simple', 'nginx.conf', tmpdir) + + +def test_compare_parsed_and_built_messy(tmpdir): + compare_parsed_and_built('messy', 'nginx.conf', tmpdir) diff --git a/tests/test_lex.py b/tests/test_lex.py index cc9c5f5..8a5b61a 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -40,7 +40,7 @@ def test_messy_config(): ('/foo', 15), ('{', 15), ('}', 15), ('location', 15), ('/bar', 15), ('{', 15), ('}', 15), ('location', 16), ('/\\{\\;\\}\\ #\\ ab', 16), ('{', 16), ('}', 16), ('if', 17), ('($request_method', 17), ('=', 17), - ('P\\{O\\)\\###\\;', 17), ('S', 17), ('T', 17), (')', 17), ('{', 17), + ('P\\{O\\)\\###\\;ST', 17), (')', 17), ('{', 17), ('}', 17), ('location', 18), ('/status.html', 18), ('{', 18), ('try_files', 19), ('/abc/${uri} /abc/${uri}.html', 19), ('=404', 19), (';', 19), ('}', 20), ('location', 21), diff --git a/tox.ini b/tox.ini index 971be1a..ce735b4 100644 --- a/tox.ini +++ b/tox.ini @@ -19,4 +19,4 @@ usedevelop = true deps = pytest commands = - py.test {posargs} + py.test {posargs} --basetemp={envtmpdir}