From 3842233fdd039233a821a86d6f6f77da6df2367c Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 21 Mar 2018 21:49:14 -0600
Subject: [PATCH 01/89] Simple ply lexer and tests

---
 tests/test_plyparsing.py | 51 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 tests/test_plyparsing.py

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
new file mode 100644
index 000000000..a3c660f99
--- /dev/null
+++ b/tests/test_plyparsing.py
@@ -0,0 +1,51 @@
+# coding=utf-8
+"""
+Unit/functional testing for ply based parsing in cmd2
+"""
+
+import pytest
+import ply.lex as lex
+import ply.yacc as yacc
+
+class Cmd2Lexer():
+    """a ply.lex lexer for the cmd2 syntax.
+    Once initial development is completed, this code
+    should be moved into cmd2.Cmd()
+    """
+    tokens = (
+        'WORD',
+    )
+
+    t_WORD = r"[A-Za-z_]+"
+
+    def t_error(self, t):
+        print("Illegal command")
+        t.lexer.skip(1)
+
+    def build(self, **kwargs):
+        self.lexer = lex.lex(module=self, **kwargs)
+
+
+@pytest.fixture
+def cl():
+    cl = Cmd2Lexer()
+    cl.build()
+    return cl
+
+def test_lex_single_word(cl):
+    cl.lexer.input('plainword')
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'plainword'
+
+def test_lex_command_with_args(cl):
+    cl.lexer.input('command with args')
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'command'
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'with'
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'args'

From e9086e16546da0f57b414daa086d7e2bf79c8934 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 21 Mar 2018 21:54:19 -0600
Subject: [PATCH 02/89] Add requirement for ply

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4b31e5fe1..cbe2c6fd7 100755
--- a/setup.py
+++ b/setup.py
@@ -62,7 +62,7 @@
 Topic :: Software Development :: Libraries :: Python Modules
 """.splitlines())))
 
-INSTALL_REQUIRES = ['pyparsing >= 2.0.1', 'pyperclip', 'six']
+INSTALL_REQUIRES = ['pyparsing >= 2.0.1', 'ply', 'pyperclip', 'six']
 
 # Windows also requires pyreadline to ensure tab completion works
 if sys.platform.startswith('win'):

From 55329a98ef4761caa2d719c8eadf84bce538966f Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Thu, 22 Mar 2018 22:21:09 -0600
Subject: [PATCH 03/89] Add ply.yacc parser for simple command with args

---
 tests/test_plyparsing.py | 45 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
index a3c660f99..20fcc43ed 100644
--- a/tests/test_plyparsing.py
+++ b/tests/test_plyparsing.py
@@ -7,11 +7,17 @@
 import ply.lex as lex
 import ply.yacc as yacc
 
+class Cmd2Command():
+    pass
+
 class Cmd2Lexer():
     """a ply.lex lexer for the cmd2 syntax.
     Once initial development is completed, this code
     should be moved into cmd2.Cmd()
     """
+    def __init__(self):
+        self.command = Cmd2Command()
+
     tokens = (
         'WORD',
     )
@@ -22,14 +28,40 @@ def t_error(self, t):
         print("Illegal command")
         t.lexer.skip(1)
 
-    def build(self, **kwargs):
+    def build_lexer(self, **kwargs):
         self.lexer = lex.lex(module=self, **kwargs)
 
+    def p_command_and_args(self, p):
+        'command_and_args : command arglist'
+        self.command.command = p[1]
+        self.command.args = p[2]
+        p[0] = self.command
+
+    def p_command(self, p):
+        'command : WORD'
+        self.command.command = p[1]
+        p[0] = p[1]
+
+    def p_arglist(self, p):
+        'arglist : arglist WORD'
+        p[0] = '{} {}'.format(p[1], p[2])
+
+    def p_arg(self, p):
+        'arglist : WORD'
+        p[0] = p[1]
+
+    def p_error(self, p):
+        print("Syntax error in input!")
+
+    def build_parser(self, **kwargs):
+        self.parser = yacc.yacc(module=self, **kwargs)
+
 
 @pytest.fixture
 def cl():
     cl = Cmd2Lexer()
-    cl.build()
+    cl.build_lexer()
+    cl.build_parser()
     return cl
 
 def test_lex_single_word(cl):
@@ -49,3 +81,12 @@ def test_lex_command_with_args(cl):
     tok = cl.lexer.token()
     assert tok.type == 'WORD'
     assert tok.value == 'args'
+
+def test_parse_single_word(cl):
+    cl.parser.parse('plainword')
+    assert cl.command.command == 'plainword'
+
+def test_parse_command_with_args(cl):
+    cl.parser.parse('command with args')
+    assert cl.command.command == 'command'
+    assert cl.command.args == 'with args'

From c74e6ed1608cadc4434d86792552001447535d9f Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Thu, 22 Mar 2018 22:27:47 -0600
Subject: [PATCH 04/89] Rename parse results to match pyparsing variables

---
 tests/test_plyparsing.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
index 20fcc43ed..e54c54fc0 100644
--- a/tests/test_plyparsing.py
+++ b/tests/test_plyparsing.py
@@ -16,7 +16,7 @@ class Cmd2Lexer():
     should be moved into cmd2.Cmd()
     """
     def __init__(self):
-        self.command = Cmd2Command()
+        self.results = Cmd2Command()
 
     tokens = (
         'WORD',
@@ -33,13 +33,13 @@ def build_lexer(self, **kwargs):
 
     def p_command_and_args(self, p):
         'command_and_args : command arglist'
-        self.command.command = p[1]
-        self.command.args = p[2]
-        p[0] = self.command
+        self.results.command = p[1]
+        self.results.args = p[2]
+        p[0] = self.results
 
     def p_command(self, p):
         'command : WORD'
-        self.command.command = p[1]
+        self.results.command = p[1]
         p[0] = p[1]
 
     def p_arglist(self, p):
@@ -84,9 +84,9 @@ def test_lex_command_with_args(cl):
 
 def test_parse_single_word(cl):
     cl.parser.parse('plainword')
-    assert cl.command.command == 'plainword'
+    assert cl.results.command == 'plainword'
 
 def test_parse_command_with_args(cl):
     cl.parser.parse('command with args')
-    assert cl.command.command == 'command'
-    assert cl.command.args == 'with args'
+    assert cl.results.command == 'command'
+    assert cl.results.args == 'with args'

From a6f809c554d9c408f2bbe376b07cb1c596d790fc Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Fri, 30 Mar 2018 09:54:34 -0600
Subject: [PATCH 05/89] quoted arguments working

---
 tests/test_plyparsing.py | 81 +++++++++++++++++++++++++++++-----------
 1 file changed, 60 insertions(+), 21 deletions(-)

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
index e54c54fc0..f7d3598ab 100644
--- a/tests/test_plyparsing.py
+++ b/tests/test_plyparsing.py
@@ -14,15 +14,21 @@ class Cmd2Lexer():
     """a ply.lex lexer for the cmd2 syntax.
     Once initial development is completed, this code
     should be moved into cmd2.Cmd()
+
+    Unlike most python classes, the order of the methods matters here. The ply module
+    uses introspection to create an ordered list of grammer rules, so rearranging methods
+    impacts functionality.
     """
     def __init__(self):
         self.results = Cmd2Command()
 
     tokens = (
-        'WORD',
+        'WORD', 'DQWORD', 'SQWORD',
     )
 
-    t_WORD = r"[A-Za-z_]+"
+    t_WORD = r"[-A-z0-9_]+"
+    t_DQWORD = r'"(?:[^"\\]|\\.)*"'
+    t_SQWORD = r"'(?:[^'\\]|\\.)*'"
 
     def t_error(self, t):
         print("Illegal command")
@@ -31,23 +37,26 @@ def t_error(self, t):
     def build_lexer(self, **kwargs):
         self.lexer = lex.lex(module=self, **kwargs)
 
-    def p_command_and_args(self, p):
-        'command_and_args : command arglist'
-        self.results.command = p[1]
-        self.results.args = p[2]
-        p[0] = self.results
+    def p_arglist_add_argument(self, p):
+        'command : command word'
+        p[0] = '{} {}'.format(p[1], p[2])
+        self.results.command = p[0]
 
-    def p_command(self, p):
-        'command : WORD'
-        self.results.command = p[1]
+    def p_arglist_argument(self, p):
+        'command : word'
         p[0] = p[1]
+        self.results.command = p[0]
 
-    def p_arglist(self, p):
-        'arglist : arglist WORD'
-        p[0] = '{} {}'.format(p[1], p[2])
+    def p_argument_word(self, p):
+        'word : WORD'
+        p[0] = p[1]
 
-    def p_arg(self, p):
-        'arglist : WORD'
+    def p_argument_dqword(self, p):
+        'word : DQWORD'
+        p[0] = p[1]
+
+    def p_argument_sqword(self, p):
+        'word : SQWORD'
         p[0] = p[1]
 
     def p_error(self, p):
@@ -64,17 +73,32 @@ def cl():
     cl.build_parser()
     return cl
 
-def test_lex_single_word(cl):
+def test_lex_word(cl):
     cl.lexer.input('plainword')
     tok = cl.lexer.token()
     assert tok.type == 'WORD'
     assert tok.value == 'plainword'
+    assert not cl.lexer.token()
 
+def test_lex_dqword(cl):
+    cl.lexer.input('"one word"')
+    tok = cl.lexer.token()
+    assert tok.type == 'DQWORD'
+    assert tok.value == '"one word"'
+    assert not cl.lexer.token()
+
+def test_lex_sqword(cl):
+    cl.lexer.input("'one word'")
+    tok = cl.lexer.token()
+    assert tok.type == 'SQWORD'
+    assert tok.value == "'one word'"
+    assert not cl.lexer.token()
+    
 def test_lex_command_with_args(cl):
-    cl.lexer.input('command with args')
+    cl.lexer.input('123456 with args')
     tok = cl.lexer.token()
     assert tok.type == 'WORD'
-    assert tok.value == 'command'
+    assert tok.value == '123456'
     tok = cl.lexer.token()
     assert tok.type == 'WORD'
     assert tok.value == 'with'
@@ -82,11 +106,26 @@ def test_lex_command_with_args(cl):
     assert tok.type == 'WORD'
     assert tok.value == 'args'
 
-def test_parse_single_word(cl):
+def test_parse_command(cl):
     cl.parser.parse('plainword')
     assert cl.results.command == 'plainword'
 
 def test_parse_command_with_args(cl):
     cl.parser.parse('command with args')
-    assert cl.results.command == 'command'
-    assert cl.results.args == 'with args'
+    assert cl.results.command == 'command with args'
+
+def test_parse_command_with_dqarg(cl):
+    cl.parser.parse('command "with   dqarg"')
+    assert cl.results.command == 'command "with   dqarg"'
+
+def test_parse_command_with_sqarg(cl):
+    cl.parser.parse("command 'with     sqarg'")
+    assert cl.results.command == "command 'with     sqarg'"
+
+def test_parse_command_with_dqarg_and_arg(cl):
+    cl.parser.parse('command "with   dqarg" onemore   lastone')
+    assert cl.results.command == 'command "with   dqarg" onemore lastone'
+
+def test_parse_command_with_sqarg_and_arg(cl):
+    cl.parser.parse("command 'with   dqarg' onemore   lastone")
+    assert cl.results.command == "command 'with   dqarg' onemore lastone"

From 3cc8efa6e92f8cabb3e1ba57cb6f005ed2b66324 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Fri, 30 Mar 2018 11:44:37 -0600
Subject: [PATCH 06/89] hash comments working

---
 tests/test_plyparsing.py | 50 ++++++++++++++++++++++++++++++++--------
 1 file changed, 40 insertions(+), 10 deletions(-)

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
index f7d3598ab..b5b5eda3e 100644
--- a/tests/test_plyparsing.py
+++ b/tests/test_plyparsing.py
@@ -1,6 +1,13 @@
 # coding=utf-8
 """
 Unit/functional testing for ply based parsing in cmd2
+
+Notes:
+
+- Shortcuts may have to be discarded, or handled in a different way than they
+  are with pyparsing.
+- 
+
 """
 
 import pytest
@@ -23,10 +30,15 @@ def __init__(self):
         self.results = Cmd2Command()
 
     tokens = (
-        'WORD', 'DQWORD', 'SQWORD',
+        'HASHCOMMENT', 'WORD', 'DQWORD', 'SQWORD',
     )
 
-    t_WORD = r"[-A-z0-9_]+"
+    def t_HASHCOMMENT(self, t):
+        r'\#.*'
+        # no return value, token discarded
+        pass
+    
+    t_WORD = r'[-A-z0-9_$%\.:\?@!]+'
     t_DQWORD = r'"(?:[^"\\]|\\.)*"'
     t_SQWORD = r"'(?:[^'\\]|\\.)*'"
 
@@ -37,25 +49,25 @@ def t_error(self, t):
     def build_lexer(self, **kwargs):
         self.lexer = lex.lex(module=self, **kwargs)
 
-    def p_arglist_add_argument(self, p):
-        'command : command word'
+    def p_command_add_word(self, p):
+        'wordlist : wordlist word'
         p[0] = '{} {}'.format(p[1], p[2])
         self.results.command = p[0]
 
-    def p_arglist_argument(self, p):
-        'command : word'
+    def p_command_word(self, p):
+        'wordlist : word'
         p[0] = p[1]
         self.results.command = p[0]
 
-    def p_argument_word(self, p):
+    def p_word_word(self, p):
         'word : WORD'
         p[0] = p[1]
 
-    def p_argument_dqword(self, p):
+    def p_word_dqword(self, p):
         'word : DQWORD'
         p[0] = p[1]
 
-    def p_argument_sqword(self, p):
+    def p_word_sqword(self, p):
         'word : SQWORD'
         p[0] = p[1]
 
@@ -93,7 +105,14 @@ def test_lex_sqword(cl):
     assert tok.type == 'SQWORD'
     assert tok.value == "'one word'"
     assert not cl.lexer.token()
-    
+
+def test_lex_dotword(cl):
+    cl.lexer.input('dot.word')
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'dot.word'
+    assert not cl.lexer.token()
+        
 def test_lex_command_with_args(cl):
     cl.lexer.input('123456 with args')
     tok = cl.lexer.token()
@@ -106,6 +125,13 @@ def test_lex_command_with_args(cl):
     assert tok.type == 'WORD'
     assert tok.value == 'args'
 
+def test_lex_comment(cl):
+    cl.lexer.input('hi # this is all a comment')
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'hi'
+    assert not cl.lexer.token()
+
 def test_parse_command(cl):
     cl.parser.parse('plainword')
     assert cl.results.command == 'plainword'
@@ -129,3 +155,7 @@ def test_parse_command_with_dqarg_and_arg(cl):
 def test_parse_command_with_sqarg_and_arg(cl):
     cl.parser.parse("command 'with   dqarg' onemore   lastone")
     assert cl.results.command == "command 'with   dqarg' onemore lastone"
+
+def test_parse_command_with_comment(cl):
+    cl.parser.parse('command # with a comment')
+    assert cl.results.command == 'command'

From 5c00bcec2e4705793a2877025c0b9ae12f39b9d6 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Fri, 30 Mar 2018 12:14:17 -0600
Subject: [PATCH 07/89] Added support for C-style comments

---
 tests/test_plyparsing.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
index b5b5eda3e..57f2d265b 100644
--- a/tests/test_plyparsing.py
+++ b/tests/test_plyparsing.py
@@ -30,7 +30,7 @@ def __init__(self):
         self.results = Cmd2Command()
 
     tokens = (
-        'HASHCOMMENT', 'WORD', 'DQWORD', 'SQWORD',
+        'HASHCOMMENT', 'CCOMMENT', 'WORD', 'DQWORD', 'SQWORD',
     )
 
     def t_HASHCOMMENT(self, t):
@@ -38,6 +38,11 @@ def t_HASHCOMMENT(self, t):
         # no return value, token discarded
         pass
     
+    def t_CCOMMENT(self, t):
+        r'/\*.*\*/'
+        # no return value, token discarded
+        pass
+    
     t_WORD = r'[-A-z0-9_$%\.:\?@!]+'
     t_DQWORD = r'"(?:[^"\\]|\\.)*"'
     t_SQWORD = r"'(?:[^'\\]|\\.)*'"
@@ -125,13 +130,23 @@ def test_lex_command_with_args(cl):
     assert tok.type == 'WORD'
     assert tok.value == 'args'
 
-def test_lex_comment(cl):
+def test_lex_hashcomment(cl):
     cl.lexer.input('hi # this is all a comment')
     tok = cl.lexer.token()
     assert tok.type == 'WORD'
     assert tok.value == 'hi'
     assert not cl.lexer.token()
 
+def test_lex_ccomment(cl):
+    cl.lexer.input('hi /* comment */ there')
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'hi'
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'there'
+    assert not cl.lexer.token()
+    
 def test_parse_command(cl):
     cl.parser.parse('plainword')
     assert cl.results.command == 'plainword'

From 859b45d90ca5369dc59779ffa6868357099f35ec Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 2 Apr 2018 22:01:21 -0600
Subject: [PATCH 08/89] Continuing to struggle with ply

---
 tests/test_plyparsing.py | 48 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 43 insertions(+), 5 deletions(-)

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
index 57f2d265b..7928ca503 100644
--- a/tests/test_plyparsing.py
+++ b/tests/test_plyparsing.py
@@ -42,10 +42,22 @@ def t_CCOMMENT(self, t):
         r'/\*.*\*/'
         # no return value, token discarded
         pass
+
+    def t_WORD(self, t):
+        r'[-\w$%\.:\?@!]+'
+        return t
+    
+    def t_DQWORD(self, t):
+        r'"(?:[^"\\]|\\.)*"'
+        return t
     
-    t_WORD = r'[-A-z0-9_$%\.:\?@!]+'
-    t_DQWORD = r'"(?:[^"\\]|\\.)*"'
-    t_SQWORD = r"'(?:[^'\\]|\\.)*'"
+    def t_SQWORD(self, t):
+        r"'(?:[^'\\]|\\.)*'"
+        return t
+
+    # def t_PIPE(self, t):
+    #     r'[|]'
+    #     return t
 
     def t_error(self, t):
         print("Illegal command")
@@ -54,12 +66,12 @@ def t_error(self, t):
     def build_lexer(self, **kwargs):
         self.lexer = lex.lex(module=self, **kwargs)
 
-    def p_command_add_word(self, p):
+    def p_wordlist_add_word(self, p):
         'wordlist : wordlist word'
         p[0] = '{} {}'.format(p[1], p[2])
         self.results.command = p[0]
 
-    def p_command_word(self, p):
+    def p_wordlist_word(self, p):
         'wordlist : word'
         p[0] = p[1]
         self.results.command = p[0]
@@ -76,6 +88,12 @@ def p_word_sqword(self, p):
         'word : SQWORD'
         p[0] = p[1]
 
+    def p_command_and_pipe(self, p):
+        "pipeline : wordlist '|' wordlist"
+        p[0] = '{} | {}'.format(p[1], p[3])        
+        self.results.command = p[1]
+        self.results.pipeTo = p[3]
+
     def p_error(self, p):
         print("Syntax error in input!")
 
@@ -146,6 +164,16 @@ def test_lex_ccomment(cl):
     assert tok.type == 'WORD'
     assert tok.value == 'there'
     assert not cl.lexer.token()
+
+def test_lex_command_pipe(cl):
+    cl.parser.parse('command | pipeto')
+    tok = cl.lexer.token()
+    assert tok.type == 'WORD'
+    assert tok.value == 'command'
+    tok = cl.lexer.token()
+    assert tok.type == 'PIPETO'
+    assert tok.value == '| pipeto'
+    assert not cl.lexer.token()
     
 def test_parse_command(cl):
     cl.parser.parse('plainword')
@@ -174,3 +202,13 @@ def test_parse_command_with_sqarg_and_arg(cl):
 def test_parse_command_with_comment(cl):
     cl.parser.parse('command # with a comment')
     assert cl.results.command == 'command'
+
+def test_parse_command_with_simple_pipe(cl):
+    cl.parser.parse('command | pipeto')
+    assert cl.results.command == 'command'
+    assert cl.results.pipeTo == '| pipeto'
+
+# def test_parse_command_with_complex_pipe(cl):
+#     cl.parser.parse('command "with   some" args | pipeto "with  the" args')
+#     assert cl.results.command == 'command "with   some" args'
+#     assert cl.results.pipeTo == '| pipeto "with  the" args'

From 2ff92ee59737c4380f24b4ca6c94822af1131138 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 2 Apr 2018 22:01:36 -0600
Subject: [PATCH 09/89] Trying out shlex

---
 tests/test_shlexparsing.py | 90 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 tests/test_shlexparsing.py

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
new file mode 100644
index 000000000..4d1a25c5b
--- /dev/null
+++ b/tests/test_shlexparsing.py
@@ -0,0 +1,90 @@
+# coding=utf-8
+"""
+Unit/functional testing for ply based parsing in cmd2
+
+Todo List
+- multiline
+- /* */ style comments
+
+Notes:
+
+- Shortcuts may have to be discarded, or handled in a different way than they
+  are with pyparsing.
+- 
+
+"""
+
+import shlex
+import pytest
+
+class Cmd2Command():
+    pass
+
+class Cmd2Parser():
+    def parseString(self, rawinput):
+        result = Cmd2Command()
+        result.raw = rawinput
+
+        s = shlex.shlex(rawinput, posix=False, punctuation_chars=True)
+        tokens = list(s)
+        result.command = tokens[0]
+
+        # check for pipes
+        try:
+            # find the first pipe if it exists
+            pipe_pos = tokens.index('|')
+            # set everything after the first pipe to result.pipeTo
+            result.pipeTo = ' '.join(tokens[pipe_pos+1:])
+            # remove all the tokens after the pipe
+            tokens = tokens[:pipe_pos-1]
+        except ValueError:
+            result.pipeTo = None
+
+        if len(tokens) > 1:
+            result.args = ' '.join(tokens[1:])
+        else:
+            result.args = None
+
+
+
+        return result
+
+@pytest.fixture
+def parser():
+    parser = Cmd2Parser()
+    return parser
+
+@pytest.mark.parametrize('line', [
+    'plainword',
+    '"one word"',
+    "'one word'",
+])
+def test_single_word(parser, line):
+    results = parser.parseString(line)
+    assert results.command == line
+
+def test_command_with_args(parser):
+    line = 'command with args'
+    results = parser.parseString(line)
+    assert results.command == 'command'
+    assert results.args == 'with args'
+    assert not results.pipeTo
+
+def test_hashcomment(parser):
+    results = parser.parseString('hi # this is all a comment')
+    assert results.command == 'hi'
+    assert not results.args
+    assert not results.pipeTo
+
+def test_simple_piped(parser):
+    results = parser.parseString('simple | piped')
+    assert results.command == 'simple'
+    assert not results.args
+    assert results.pipeTo == 'piped'
+
+def test_double_pipe_is_not_a_pipe(parser):
+    line = 'double-pipe || is not a pipe'
+    results = parser.parseString(line)
+    assert results.command == 'double-pipe'
+    assert results.args == '|| is not a pipe'
+    assert not results.pipeTo

From be85e8c440ca1cbbeca045e1c28b810399e12d6f Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 2 Apr 2018 22:06:18 -0600
Subject: [PATCH 10/89] Comment out broken ply tests

---
 tests/test_plyparsing.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
index 7928ca503..f33527a22 100644
--- a/tests/test_plyparsing.py
+++ b/tests/test_plyparsing.py
@@ -165,15 +165,15 @@ def test_lex_ccomment(cl):
     assert tok.value == 'there'
     assert not cl.lexer.token()
 
-def test_lex_command_pipe(cl):
-    cl.parser.parse('command | pipeto')
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == 'command'
-    tok = cl.lexer.token()
-    assert tok.type == 'PIPETO'
-    assert tok.value == '| pipeto'
-    assert not cl.lexer.token()
+# def test_lex_command_pipe(cl):
+#     cl.parser.parse('command | pipeto')
+#     tok = cl.lexer.token()
+#     assert tok.type == 'WORD'
+#     assert tok.value == 'command'
+#     tok = cl.lexer.token()
+#     assert tok.type == 'PIPETO'
+#     assert tok.value == '| pipeto'
+#     assert not cl.lexer.token()
     
 def test_parse_command(cl):
     cl.parser.parse('plainword')
@@ -203,10 +203,10 @@ def test_parse_command_with_comment(cl):
     cl.parser.parse('command # with a comment')
     assert cl.results.command == 'command'
 
-def test_parse_command_with_simple_pipe(cl):
-    cl.parser.parse('command | pipeto')
-    assert cl.results.command == 'command'
-    assert cl.results.pipeTo == '| pipeto'
+# def test_parse_command_with_simple_pipe(cl):
+#     cl.parser.parse('command | pipeto')
+#     assert cl.results.command == 'command'
+#     assert cl.results.pipeTo == '| pipeto'
 
 # def test_parse_command_with_complex_pipe(cl):
 #     cl.parser.parse('command "with   some" args | pipeto "with  the" args')

From dd780edc3d080cd4ae8d62c74006af44ff540a63 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 2 Apr 2018 22:20:15 -0600
Subject: [PATCH 11/89] Add support for C-style and C++-style comments

---
 tests/test_shlexparsing.py | 44 ++++++++++++++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 4 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 4d1a25c5b..b13b9f646 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -4,17 +4,21 @@
 
 Todo List
 - multiline
-- /* */ style comments
 
 Notes:
 
 - Shortcuts may have to be discarded, or handled in a different way than they
   are with pyparsing.
-- 
+- valid comment styles:
+    - C-style -> /* comment */
+    - C++-style -> // comment
+    - Python/Shell style -> # comment
 
 """
 
+import re
 import shlex
+
 import pytest
 
 class Cmd2Command():
@@ -25,6 +29,21 @@ def parseString(self, rawinput):
         result = Cmd2Command()
         result.raw = rawinput
 
+        # strip C-style and C++-style comments
+        # shlex will handle the python/shell style comments for us
+        def replacer(match):
+                s = match.group(0)
+                if s.startswith('/'):
+                    # treat the removed comment as a space token, not an empty string
+                    return ' '
+                else:
+                    return s
+        pattern = re.compile(
+            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+            re.DOTALL | re.MULTILINE
+        )
+        rawinput = re.sub(pattern, replacer, rawinput)
+
         s = shlex.shlex(rawinput, posix=False, punctuation_chars=True)
         tokens = list(s)
         result.command = tokens[0]
@@ -38,6 +57,7 @@ def parseString(self, rawinput):
             # remove all the tokens after the pipe
             tokens = tokens[:pipe_pos-1]
         except ValueError:
+            # no pipe in the tokens
             result.pipeTo = None
 
         if len(tokens) > 1:
@@ -45,8 +65,6 @@ def parseString(self, rawinput):
         else:
             result.args = None
 
-
-
         return result
 
 @pytest.fixture
@@ -76,6 +94,24 @@ def test_hashcomment(parser):
     assert not results.args
     assert not results.pipeTo
 
+def test_c_comment(parser):
+    results = parser.parseString('hi /* this is | all a comment */')
+    assert results.command == 'hi'
+    assert not results.args
+    assert not results.pipeTo
+
+def test_parse_what_if_quoted_strings_seem_to_start_comments(parser):
+    results = parser.parseString('what if "quoted strings /* seem to " start comments?')
+    assert results.command == 'what'
+    assert results.args == 'if "quoted strings /* seem to " start comments?'
+    assert not results.pipeTo
+
+def test_cpp_comment(parser):
+    results = parser.parseString('hi // this is | all a comment */')
+    assert results.command == 'hi'
+    assert not results.args
+    assert not results.pipeTo
+
 def test_simple_piped(parser):
     results = parser.parseString('simple | piped')
     assert results.command == 'simple'

From 8ec0a4c28155e0259bc67816128cf115af262384 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 9 Apr 2018 12:55:37 -0600
Subject: [PATCH 12/89] Add support for output redirection

---
 tests/test_shlexparsing.py | 50 ++++++++++++++++++++++++++++++++++----
 1 file changed, 45 insertions(+), 5 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index b13b9f646..ab1984aee 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -28,6 +28,11 @@ class Cmd2Parser():
     def parseString(self, rawinput):
         result = Cmd2Command()
         result.raw = rawinput
+        result.command = None
+        result.args = None
+        result.pipeTo = None
+        result.output = None
+        result.outputTo = None
 
         # strip C-style and C++-style comments
         # shlex will handle the python/shell style comments for us
@@ -35,7 +40,9 @@ def replacer(match):
                 s = match.group(0)
                 if s.startswith('/'):
                     # treat the removed comment as a space token, not an empty string
-                    return ' '
+                    # return ' '
+                    # jk, always return nothing
+                    return ''
                 else:
                     return s
         pattern = re.compile(
@@ -46,7 +53,17 @@ def replacer(match):
 
         s = shlex.shlex(rawinput, posix=False, punctuation_chars=True)
         tokens = list(s)
-        result.command = tokens[0]
+
+        # check for output redirect
+        try:
+            output_pos = tokens.index('>')
+            result.output = '>'
+            result.outputTo = ' '.join(tokens[output_pos+1:])
+            # remove all the tokens after the output redirect
+            tokens = tokens[:output_pos]
+        except ValueError:
+            result.output = None
+            result.outputTo = None
 
         # check for pipes
         try:
@@ -55,15 +72,16 @@ def replacer(match):
             # set everything after the first pipe to result.pipeTo
             result.pipeTo = ' '.join(tokens[pipe_pos+1:])
             # remove all the tokens after the pipe
-            tokens = tokens[:pipe_pos-1]
+            tokens = tokens[:pipe_pos]
         except ValueError:
             # no pipe in the tokens
             result.pipeTo = None
 
+        if tokens:
+            result.command = tokens[0]
+
         if len(tokens) > 1:
             result.args = ' '.join(tokens[1:])
-        else:
-            result.args = None
 
         return result
 
@@ -100,6 +118,12 @@ def test_c_comment(parser):
     assert not results.args
     assert not results.pipeTo
 
+def test_c_comment_empty(parser):
+    results = parser.parseString('/* this is | all a comment */')
+    assert not results.command
+    assert not results.args
+    assert not results.pipeTo
+
 def test_parse_what_if_quoted_strings_seem_to_start_comments(parser):
     results = parser.parseString('what if "quoted strings /* seem to " start comments?')
     assert results.command == 'what'
@@ -124,3 +148,19 @@ def test_double_pipe_is_not_a_pipe(parser):
     assert results.command == 'double-pipe'
     assert results.args == '|| is not a pipe'
     assert not results.pipeTo
+
+def test_output_redirect(parser):
+    line = 'output into > afile.txt'
+    results = parser.parseString(line)
+    assert results.command == 'output'
+    assert results.args == 'into'
+    assert results.output == '>'
+    assert results.outputTo == 'afile.txt'
+
+def test_output_redirect_with_dash_in_path(parser):
+    line = 'output into > python-cmd2/afile.txt'
+    results = parser.parseString(line)
+    assert results.command == 'output'
+    assert results.args == 'into'
+    assert results.output == '>'
+    assert results.outputTo == 'python-cmd2/afile.txt'

From 0afd9cc7f3975d8e11915138be939acd2386bb78 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 9 Apr 2018 13:19:07 -0600
Subject: [PATCH 13/89] Add support for terminator and suffix

---
 tests/test_shlexparsing.py | 60 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 57 insertions(+), 3 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index ab1984aee..054904cf2 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -4,6 +4,8 @@
 
 Todo List
 - multiline
+- case sensitive flag
+- figure out how to let users change the terminator character
 
 Notes:
 
@@ -30,10 +32,15 @@ def parseString(self, rawinput):
         result.raw = rawinput
         result.command = None
         result.args = None
+        result.terminator = None
+        result.suffix = None
         result.pipeTo = None
         result.output = None
         result.outputTo = None
 
+        # in theory we let people change this
+        terminator = ';'
+
         # strip C-style and C++-style comments
         # shlex will handle the python/shell style comments for us
         def replacer(match):
@@ -52,6 +59,7 @@ def replacer(match):
         rawinput = re.sub(pattern, replacer, rawinput)
 
         s = shlex.shlex(rawinput, posix=False, punctuation_chars=True)
+        s.wordchars = '1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_~-.,/*?='
         tokens = list(s)
 
         # check for output redirect
@@ -62,8 +70,7 @@ def replacer(match):
             # remove all the tokens after the output redirect
             tokens = tokens[:output_pos]
         except ValueError:
-            result.output = None
-            result.outputTo = None
+            pass
 
         # check for pipes
         try:
@@ -75,8 +82,22 @@ def replacer(match):
             tokens = tokens[:pipe_pos]
         except ValueError:
             # no pipe in the tokens
-            result.pipeTo = None
+            pass
 
+        # look for the semicolon terminator
+        # we rely on shlex.shlex to split on the semicolon. If we let users
+        # change the termination character, this might break
+        try:
+            terminator_pos = tokens.index(';')
+            # everything after the first terminator gets put in suffix
+            result.terminator = tokens[terminator_pos]
+            result.suffix = ' '.join(tokens[terminator_pos+1:])
+            # remove all the tokens after and including the terminator
+            tokens = tokens[:terminator_pos]
+        except ValueError:
+            # no terminator in the tokens
+            pass
+        
         if tokens:
             result.command = tokens[0]
 
@@ -99,6 +120,19 @@ def test_single_word(parser, line):
     results = parser.parseString(line)
     assert results.command == line
 
+def test_word_plus_terminator(parser):
+    line = 'termbare;'
+    results = parser.parseString(line)
+    assert results.command == 'termbare'
+    assert results.terminator == ';'
+
+def test_suffix_after_terminator(parser):
+    line = 'termbare; suffx'
+    results = parser.parseString(line)
+    assert results.command == 'termbare'
+    assert results.terminator == ';'
+    assert results.suffix == 'suffx'
+
 def test_command_with_args(parser):
     line = 'command with args'
     results = parser.parseString(line)
@@ -149,6 +183,15 @@ def test_double_pipe_is_not_a_pipe(parser):
     assert results.args == '|| is not a pipe'
     assert not results.pipeTo
 
+def test_complex_pipe(parser):
+    line = 'command with args, terminator;sufx | piped'
+    results = parser.parseString(line)
+    assert results.command == 'command'
+    assert results.args == "with args, terminator"
+    assert results.terminator == ';'
+    assert results.suffix == 'sufx'
+    assert results.pipeTo == 'piped'
+
 def test_output_redirect(parser):
     line = 'output into > afile.txt'
     results = parser.parseString(line)
@@ -164,3 +207,14 @@ def test_output_redirect_with_dash_in_path(parser):
     assert results.args == 'into'
     assert results.output == '>'
     assert results.outputTo == 'python-cmd2/afile.txt'
+
+def test_pipe_and_redirect(parser):
+    line = 'output into;sufx | pipethrume plz > afile.txt'
+    results = parser.parseString(line)
+    assert results.command == 'output'
+    assert results.args == 'into'
+    assert results.terminator == ';'
+    assert results.suffix == 'sufx'
+    assert results.pipeTo == 'pipethrume plz'
+    assert results.output == '>'
+    assert results.outputTo == 'afile.txt'

From 4c356e5224c463a3731edaa72898a2965eb1ba58 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 9 Apr 2018 23:01:01 -0400
Subject: [PATCH 14/89] Make sure redirectors before the terminator are
 included in the args, not the redirection

---
 tests/test_shlexparsing.py | 73 +++++++++++++++++++++++++++++---------
 1 file changed, 56 insertions(+), 17 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 054904cf2..a0fe95277 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -59,9 +59,27 @@ def replacer(match):
         rawinput = re.sub(pattern, replacer, rawinput)
 
         s = shlex.shlex(rawinput, posix=False, punctuation_chars=True)
+        # these characters should be included in tokens, not used to split them
+        # we need to override the default so we can include the ','
         s.wordchars = '1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_~-.,/*?='
         tokens = list(s)
 
+        # we rely on shlex.shlex to split on the semicolon. If we let users
+        # change the termination character, this might break
+        
+        # look for the semicolon terminator
+        try:
+            terminator_pos = tokens.index(';')
+            # everything before the first terminator is the command and the args
+            (result.command, result.args) = self._command_and_args(tokens[:terminator_pos])
+            result.terminator = tokens[terminator_pos]
+            # we will set the suffix later
+            # remove all the tokens before and including the terminator
+            tokens = tokens[terminator_pos+1:]
+        except ValueError:
+            # no terminator in the tokens
+            pass
+
         # check for output redirect
         try:
             output_pos = tokens.index('>')
@@ -83,34 +101,46 @@ def replacer(match):
         except ValueError:
             # no pipe in the tokens
             pass
-
-        # look for the semicolon terminator
-        # we rely on shlex.shlex to split on the semicolon. If we let users
-        # change the termination character, this might break
-        try:
-            terminator_pos = tokens.index(';')
-            # everything after the first terminator gets put in suffix
-            result.terminator = tokens[terminator_pos]
-            result.suffix = ' '.join(tokens[terminator_pos+1:])
-            # remove all the tokens after and including the terminator
-            tokens = tokens[:terminator_pos]
-        except ValueError:
-            # no terminator in the tokens
-            pass
         
+        if result.terminator:
+            # whatever is left is the suffix
+            result.suffix = ' '.join(tokens)
+        else:
+            # no terminator, so whatever is left is the command and the args
+            (result.command, result.args) = self._command_and_args(tokens)            
+
+        return result
+    
+    def _command_and_args(self, tokens):
+        """given a list of tokens, and return a tuple of the command
+        and the args as a string.
+        """
+        command = None
+        args = None
+
         if tokens:
-            result.command = tokens[0]
+            command = tokens[0]
 
         if len(tokens) > 1:
-            result.args = ' '.join(tokens[1:])
+            args = ' '.join(tokens[1:])
 
-        return result
+        return (command, args)
 
 @pytest.fixture
 def parser():
     parser = Cmd2Parser()
     return parser
 
+@pytest.mark.parametrize('tokens,command,args', [
+    ( [], None, None),
+    ( ['command'], 'command', None ),
+    ( ['command', 'arg1', 'arg2'], 'command', 'arg1 arg2')
+])
+def test_command_and_args(parser, tokens, command, args):
+    (parsed_command, parsed_args) = parser._command_and_args(tokens)
+    assert command == parsed_command
+    assert args == parsed_args
+
 @pytest.mark.parametrize('line', [
     'plainword',
     '"one word"',
@@ -218,3 +248,12 @@ def test_pipe_and_redirect(parser):
     assert results.pipeTo == 'pipethrume plz'
     assert results.output == '>'
     assert results.outputTo == 'afile.txt'
+
+def test_has_redirect_inside_terminator(parser):
+    """The terminator designates the end of the commmand/arguments portion.  If a redirector
+    occurs before a terminator, then it will be treated as part of the arguments and not as a redirector."""
+    line = 'has > inside;'
+    results = parser.parseString(line)
+    assert results.command == 'has'
+    assert results.args == '> inside'
+    assert results.terminator == ';'
\ No newline at end of file

From 592439594c8d70f582632c9f3ae8a6c08ba9efae Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 9 Apr 2018 23:07:44 -0400
Subject: [PATCH 15/89] Add unicode to todo list

---
 tests/test_shlexparsing.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index a0fe95277..d25e5d99d 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -4,6 +4,7 @@
 
 Todo List
 - multiline
+- unicode
 - case sensitive flag
 - figure out how to let users change the terminator character
 

From f8f65b18f121dee86c41eb98aebb1db6d3055448 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Thu, 12 Apr 2018 10:55:33 -0400
Subject: [PATCH 16/89] =?UTF-8?q?mostly=20working,=20but=20no=20unicode=20?=
 =?UTF-8?q?support=E2=80=A6:(?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/test_shlexparsing.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index d25e5d99d..41ef826d7 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -21,6 +21,7 @@
 
 import re
 import shlex
+import sys
 
 import pytest
 
@@ -59,9 +60,10 @@ def replacer(match):
         )
         rawinput = re.sub(pattern, replacer, rawinput)
 
-        s = shlex.shlex(rawinput, posix=False, punctuation_chars=True)
+        s = shlex.shlex(rawinput, posix=False, punctuation_chars=';><|')
         # these characters should be included in tokens, not used to split them
         # we need to override the default so we can include the ','
+        # # s.punctuation_chars = ';><|'
         s.wordchars = '1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_~-.,/*?='
         tokens = list(s)
 
@@ -257,4 +259,11 @@ def test_has_redirect_inside_terminator(parser):
     results = parser.parseString(line)
     assert results.command == 'has'
     assert results.args == '> inside'
-    assert results.terminator == ';'
\ No newline at end of file
+    assert results.terminator == ';'
+
+@pytest.mark.skipif(sys.version_info < (3,0), reason="cmd2 unicode support requires python3")
+def test_command_with_unicode_args(parser):
+    line = 'drink café'
+    results = parser.parseString(line)
+    assert results.command == 'drink'
+    assert results.args == 'café'

From a98ff3a250b5cca122120c0b26044f5993176f1a Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 16 Apr 2018 20:23:58 -0600
Subject: [PATCH 17/89] Unicode and almost all redirection operators working

---
 tests/test_shlexparsing.py | 98 ++++++++++++++++++++++++++++++++++----
 1 file changed, 88 insertions(+), 10 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index d25e5d99d..af1d78ece 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -59,18 +59,13 @@ def replacer(match):
         )
         rawinput = re.sub(pattern, replacer, rawinput)
 
-        s = shlex.shlex(rawinput, posix=False, punctuation_chars=True)
-        # these characters should be included in tokens, not used to split them
-        # we need to override the default so we can include the ','
-        s.wordchars = '1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_~-.,/*?='
-        tokens = list(s)
-
-        # we rely on shlex.shlex to split on the semicolon. If we let users
-        # change the termination character, this might break
+        s = shlex.shlex(rawinput, posix=False)
+        s.whitespace_split = True
+        tokens = self.split_on_punctuation(list(s))
         
         # look for the semicolon terminator
         try:
-            terminator_pos = tokens.index(';')
+            terminator_pos = tokens.index(terminator)
             # everything before the first terminator is the command and the args
             (result.command, result.args) = self._command_and_args(tokens[:terminator_pos])
             result.terminator = tokens[terminator_pos]
@@ -127,6 +122,69 @@ def _command_and_args(self, tokens):
 
         return (command, args)
 
+    def split_on_punctuation(self, initial_tokens):
+        """
+        # Further splits tokens from a command line using punctuation characters
+        # as word breaks when they are in unquoted strings. Each run of punctuation
+        # characters is treated as a single token.
+
+        :param initial_tokens: the tokens as parsed by shlex
+        :return: the punctuated tokens
+        """
+        punctuation = [';'] # should be self.terminator from cmd2.py
+        if True:  # should be self.allow_redirection from cmd2.py
+            punctuation += ['|', '<', '>'] # should be REDIRECTION_CHARS from cmd2.py
+
+        punctuated_tokens = []
+
+        for cur_initial_token in initial_tokens:
+
+            # Save tokens up to 1 character in length or quoted tokens. No need to parse these.
+            if len(cur_initial_token) <= 1 or cur_initial_token[0] in ['"', "'"]: # should be QUOTES in cmd2.py
+                punctuated_tokens.append(cur_initial_token)
+                continue
+
+            # Iterate over each character in this token
+            cur_index = 0
+            cur_char = cur_initial_token[cur_index]
+
+            # Keep track of the token we are building
+            new_token = ''
+
+            while True:
+                if cur_char not in punctuation:
+
+                    # Keep appending to new_token until we hit a punctuation char
+                    while cur_char not in punctuation:
+                        new_token += cur_char
+                        cur_index += 1
+                        if cur_index < len(cur_initial_token):
+                            cur_char = cur_initial_token[cur_index]
+                        else:
+                            break
+
+                else:
+                    cur_punc = cur_char
+
+                    # Keep appending to new_token until we hit something other than cur_punc
+                    while cur_char == cur_punc:
+                        new_token += cur_char
+                        cur_index += 1
+                        if cur_index < len(cur_initial_token):
+                            cur_char = cur_initial_token[cur_index]
+                        else:
+                            break
+
+                # Save the new token
+                punctuated_tokens.append(new_token)
+                new_token = ''
+
+                # Check if we've viewed all characters
+                if cur_index >= len(cur_initial_token):
+                    break
+
+        return punctuated_tokens
+
 @pytest.fixture
 def parser():
     parser = Cmd2Parser()
@@ -257,4 +315,24 @@ def test_has_redirect_inside_terminator(parser):
     results = parser.parseString(line)
     assert results.command == 'has'
     assert results.args == '> inside'
-    assert results.terminator == ';'
\ No newline at end of file
+    assert results.terminator == ';'
+def test_parse_command_with_unicode_args(parser):
+    line = 'drink café'
+    results = parser.parseString(line)
+    assert results.command == 'drink'
+    assert results.args == 'café'
+
+def test_parse_unicode_command(parser):
+    line = 'café au lait'
+    results = parser.parseString(line)
+    assert results.command == 'café'
+    assert results.args == 'au lait'
+
+def test_parse_redirect_to_unicode_filename(parser):
+    line = 'dir home > café'
+    results = parser.parseString(line)
+    assert results.command == 'dir'
+    assert results.args == 'home'
+    assert results.output == '>'
+    assert results.outputTo == 'café'
+

From 5d314d97f0f47aa4278b47bd3db96e375c0d18f4 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 16 Apr 2018 21:07:44 -0600
Subject: [PATCH 18/89] Input from file working

---
 tests/test_shlexparsing.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index af1d78ece..10eb0b187 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -76,6 +76,15 @@ def replacer(match):
             # no terminator in the tokens
             pass
 
+        # check for input from file
+        try:
+            if tokens[0] == '<':
+                result.inputFrom = ' '.join(tokens[1:])
+                tokens = []
+        except IndexError:
+            # no input from file
+            pass
+
         # check for output redirect
         try:
             output_pos = tokens.index('>')
@@ -336,3 +345,7 @@ def test_parse_redirect_to_unicode_filename(parser):
     assert results.output == '>'
     assert results.outputTo == 'café'
 
+def test_parse_input_redirect_from_unicode_filename(parser):
+    line = '< café'
+    results = parser.parseString(line)
+    assert results.inputFrom == 'café'

From 47744016a596d3d6af539421d538ff4ba4c45f2d Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 16 Apr 2018 21:32:45 -0600
Subject: [PATCH 19/89] Extra tests and paste buffer support

---
 tests/test_shlexparsing.py | 50 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 48 insertions(+), 2 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 10eb0b187..7f4cad98e 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -4,9 +4,7 @@
 
 Todo List
 - multiline
-- unicode
 - case sensitive flag
-- figure out how to let users change the terminator character
 
 Notes:
 
@@ -24,6 +22,8 @@
 
 import pytest
 
+import cmd2
+
 class Cmd2Command():
     pass
 
@@ -95,6 +95,15 @@ def replacer(match):
         except ValueError:
             pass
 
+        # check for paste buffer
+        try:
+            output_pos = tokens.index('>>')
+            result.output = '>>'
+            # remove all tokens after the output redirect
+            tokens = tokens[:output_pos]
+        except ValueError:
+            pass
+
         # check for pipes
         try:
             # find the first pipe if it exists
@@ -199,6 +208,10 @@ def parser():
     parser = Cmd2Parser()
     return parser
 
+def test_parse_empty_string(parser):
+    results = parser.parseString('')
+    assert not results.command
+
 @pytest.mark.parametrize('tokens,command,args', [
     ( [], None, None),
     ( ['command'], 'command', None ),
@@ -238,6 +251,14 @@ def test_command_with_args(parser):
     assert results.args == 'with args'
     assert not results.pipeTo
 
+def test_parse_command_with_args_terminator_and_suffix(parser):
+    line = 'command with args and terminator; and suffix'
+    results = parser.parseString(line)
+    assert results.command == 'command'
+    assert results.args == "with args and terminator"
+    assert results.terminator == ';'
+    assert results.suffix == 'and suffix'
+
 def test_hashcomment(parser):
     results = parser.parseString('hi # this is all a comment')
     assert results.command == 'hi'
@@ -306,6 +327,16 @@ def test_output_redirect_with_dash_in_path(parser):
     assert results.output == '>'
     assert results.outputTo == 'python-cmd2/afile.txt'
 
+def test_parse_input_redirect(parser):
+    line = '< afile.txt'
+    results = parser.parseString(line)
+    assert results.inputFrom == 'afile.txt'
+
+def test_parse_input_redirect_with_dash_in_path(parser):
+    line = '< python-cmd2/afile.txt'
+    results = parser.parseString(line)
+    assert results.inputFrom == 'python-cmd2/afile.txt'
+
 def test_pipe_and_redirect(parser):
     line = 'output into;sufx | pipethrume plz > afile.txt'
     results = parser.parseString(line)
@@ -317,6 +348,13 @@ def test_pipe_and_redirect(parser):
     assert results.output == '>'
     assert results.outputTo == 'afile.txt'
 
+def test_parse_output_to_paste_buffer(parser):
+    line = 'output to paste buffer >> '
+    results = parser.parseString(line)
+    assert results.command == 'output'
+    assert results.args == 'to paste buffer'
+    assert results.output == '>>'
+
 def test_has_redirect_inside_terminator(parser):
     """The terminator designates the end of the commmand/arguments portion.  If a redirector
     occurs before a terminator, then it will be treated as part of the arguments and not as a redirector."""
@@ -349,3 +387,11 @@ def test_parse_input_redirect_from_unicode_filename(parser):
     line = '< café'
     results = parser.parseString(line)
     assert results.inputFrom == 'café'
+
+def test_empty_statement_raises_exception():
+    app = cmd2.Cmd()
+    with pytest.raises(cmd2.EmptyStatement):
+        app._complete_statement('')
+
+    with pytest.raises(cmd2.EmptyStatement):
+        app._complete_statement(' ')

From b45d49216de1468a9a3798e299ab87df1e0aae65 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 16 Apr 2018 22:40:41 -0600
Subject: [PATCH 20/89] Start multiline work

---
 tests/test_shlexparsing.py | 72 +++++++++++++++++++++++++++++++++-----
 1 file changed, 64 insertions(+), 8 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 7f4cad98e..634f1513f 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -28,6 +28,13 @@ class Cmd2Command():
     pass
 
 class Cmd2Parser():
+    # settings or variables from cmd2.py
+    terminator = ';'
+    allow_redirection = True
+    REDIRECTION_CHARS = ['|', '<', '>']
+    QUOTES = ['"', "'"]
+    multilineCommands = ['multiline']
+
     def parseString(self, rawinput):
         result = Cmd2Command()
         result.raw = rawinput
@@ -39,9 +46,6 @@ def parseString(self, rawinput):
         result.output = None
         result.outputTo = None
 
-        # in theory we let people change this
-        terminator = ';'
-
         # strip C-style and C++-style comments
         # shlex will handle the python/shell style comments for us
         def replacer(match):
@@ -65,7 +69,7 @@ def replacer(match):
         
         # look for the semicolon terminator
         try:
-            terminator_pos = tokens.index(terminator)
+            terminator_pos = tokens.index(self.terminator)
             # everything before the first terminator is the command and the args
             (result.command, result.args) = self._command_and_args(tokens[:terminator_pos])
             result.terminator = tokens[terminator_pos]
@@ -149,16 +153,16 @@ def split_on_punctuation(self, initial_tokens):
         :param initial_tokens: the tokens as parsed by shlex
         :return: the punctuated tokens
         """
-        punctuation = [';'] # should be self.terminator from cmd2.py
-        if True:  # should be self.allow_redirection from cmd2.py
-            punctuation += ['|', '<', '>'] # should be REDIRECTION_CHARS from cmd2.py
+        punctuation = [self.terminator] # should be self.terminator from cmd2.py
+        if self.allow_redirection:  # should be self.allow_redirection from cmd2.py
+            punctuation += self.REDIRECTION_CHARS # should be REDIRECTION_CHARS from cmd2.py
 
         punctuated_tokens = []
 
         for cur_initial_token in initial_tokens:
 
             # Save tokens up to 1 character in length or quoted tokens. No need to parse these.
-            if len(cur_initial_token) <= 1 or cur_initial_token[0] in ['"', "'"]: # should be QUOTES in cmd2.py
+            if len(cur_initial_token) <= 1 or cur_initial_token[0] in self.QUOTES: # should be QUOTES in cmd2.py
                 punctuated_tokens.append(cur_initial_token)
                 continue
 
@@ -203,6 +207,11 @@ def split_on_punctuation(self, initial_tokens):
 
         return punctuated_tokens
 
+######
+#
+# unit tests
+#
+######
 @pytest.fixture
 def parser():
     parser = Cmd2Parser()
@@ -363,6 +372,53 @@ def test_has_redirect_inside_terminator(parser):
     assert results.command == 'has'
     assert results.args == '> inside'
     assert results.terminator == ';'
+
+# def test_parse_unfinished_multiliine_command(parser):
+#     line = 'multiline has > inside an unfinished command'
+#     results = parser.parseString(line)
+#     assert results.multilineCommand == 'multiline'
+#     assert not 'args' in results
+
+def test_parse_multiline_command_ignores_redirectors_within_it(parser):
+    line = 'multiline has > inside;'
+    results = parser.parseString(line)
+    assert results.multilineCommand == 'multiline'
+    assert results.args == 'has > inside'
+    assert results.terminator == ';'
+
+# def test_parse_multiline_with_incomplete_comment(parser):
+#     """A terminator within a comment will be ignored and won't terminate a multiline command.
+#     Un-closed comments effectively comment out everything after the start."""
+#     line = 'multiline command /* with comment in progress;'
+#     results = parser.parseString(line)
+#     assert results.multilineCommand == 'multiline'
+#     assert not 'args' in results
+
+def test_parse_multiline_with_complete_comment(parser):
+    line = 'multiline command /* with comment complete */ is done;'
+    results = parser.parseString(line)
+    assert results.multilineCommand == 'multiline'
+    assert results.args == 'command /* with comment complete */ is done'
+    assert results.terminator == ';'
+
+# def test_parse_multiline_termninated_by_empty_line(parser):
+#     line = 'multiline command ends\n\n'
+#     results = parser.parseString(line)
+#     assert results.multilineCommand == 'multiline'
+#     assert results.args == 'command ends'
+#     assert len(results.terminator) == 2
+#     assert results.terminator[0] == '\n'
+#     assert results.terminator[1] == '\n'
+
+# def test_parse_multiline_ignores_terminators_in_comments(parser):
+#     line = 'multiline command "with term; ends" now\n\n'
+#     results = parser.parseString(line)
+#     assert results.multilineCommand == 'multiline'
+#     assert results.args == 'command "with term; ends" now'
+#     assert len(results.terminator) == 2
+#     assert results.terminator[0] == '\n'
+#     assert results.terminator[1] == '\n'
+
 def test_parse_command_with_unicode_args(parser):
     line = 'drink café'
     results = parser.parseString(line)

From b7cfb130c7c914478936366b748b04234b031119 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 18 Apr 2018 21:59:04 -0600
Subject: [PATCH 21/89] First multi-line test passes

---
 tests/test_shlexparsing.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 634f1513f..305d80d91 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -39,6 +39,7 @@ def parseString(self, rawinput):
         result = Cmd2Command()
         result.raw = rawinput
         result.command = None
+        result.multilineCommand = None
         result.args = None
         result.terminator = None
         result.suffix = None
@@ -123,6 +124,8 @@ def replacer(match):
         if result.terminator:
             # whatever is left is the suffix
             result.suffix = ' '.join(tokens)
+            if result.command in self.multilineCommands:
+                result.multilineCommand = result.command
         else:
             # no terminator, so whatever is left is the command and the args
             (result.command, result.args) = self._command_and_args(tokens)            
@@ -394,12 +397,12 @@ def test_parse_multiline_command_ignores_redirectors_within_it(parser):
 #     assert results.multilineCommand == 'multiline'
 #     assert not 'args' in results
 
-def test_parse_multiline_with_complete_comment(parser):
-    line = 'multiline command /* with comment complete */ is done;'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'command /* with comment complete */ is done'
-    assert results.terminator == ';'
+# def test_parse_multiline_with_complete_comment(parser):
+#     line = 'multiline command /* with comment complete */ is done;'
+#     results = parser.parseString(line)
+#     assert results.multilineCommand == 'multiline'
+#     assert results.args == 'command /* with comment complete */ is done'
+#     assert results.terminator == ';'
 
 # def test_parse_multiline_termninated_by_empty_line(parser):
 #     line = 'multiline command ends\n\n'
@@ -446,8 +449,8 @@ def test_parse_input_redirect_from_unicode_filename(parser):
 
 def test_empty_statement_raises_exception():
     app = cmd2.Cmd()
-    with pytest.raises(cmd2.EmptyStatement):
+    with pytest.raises(cmd2.cmd2.EmptyStatement):
         app._complete_statement('')
 
-    with pytest.raises(cmd2.EmptyStatement):
+    with pytest.raises(cmd2.cmd2.EmptyStatement):
         app._complete_statement(' ')

From 2350ec2a9b137ee3026efff1b1b3537d99cf19f2 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Fri, 20 Apr 2018 15:36:15 -0600
Subject: [PATCH 22/89] Move CommandParser class into its own file

---
 cmd2/parsing.py            | 212 ++++++++++++++++++++++++++++++++++++
 tests/test_shlexparsing.py | 217 +++----------------------------------
 2 files changed, 227 insertions(+), 202 deletions(-)
 create mode 100644 cmd2/parsing.py

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
new file mode 100644
index 000000000..41a3ed0b3
--- /dev/null
+++ b/cmd2/parsing.py
@@ -0,0 +1,212 @@
+#
+# -*- coding: utf-8 -*-
+"""Command parsing classes for cmd2"""
+
+import re
+import shlex
+
+import cmd2
+
+class Command():
+    """Store the results of a parsed command."""
+    pass
+
+class CommandParser():
+    """Parse raw text into command components."""
+    def __init__(
+            self,
+            quotes=['"', "'"],
+            allow_redirection=True,
+            redirection_chars=['|', '<', '>'],
+            terminators=[';'],
+            multilineCommands = [],
+        ):
+        self.quotes = quotes
+        self.allow_redirection = allow_redirection
+        self.redirection_chars = redirection_chars
+        self.terminators = terminators
+        self.multilineCommands = multilineCommands
+
+    def parseString(self, rawinput):
+        result = Command()
+        result.raw = rawinput
+        result.command = None
+        result.multilineCommand = None
+        result.args = None
+        result.terminator = None
+        result.suffix = None
+        result.pipeTo = None
+        result.output = None
+        result.outputTo = None
+
+        # strip C-style and C++-style comments
+        # shlex will handle the python/shell style comments for us
+        def replacer(match):
+                s = match.group(0)
+                if s.startswith('/'):
+                    # treat the removed comment as an empty string
+                    return ''
+                else:
+                    return s
+        pattern = re.compile(
+            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+            re.DOTALL | re.MULTILINE
+        )
+        rawinput = re.sub(pattern, replacer, rawinput)
+
+        s = shlex.shlex(rawinput, posix=False)
+        s.whitespace_split = True
+        tokens = self.split_on_punctuation(list(s))
+        
+        # of the valid terminators, find the first one to occur in the input
+        terminator_pos = len(tokens)+1
+        terminator = None
+        for test_terminator in self.terminators:
+            try:
+                pos = tokens.index(test_terminator)
+                if pos < terminator_pos:
+                    terminator_pos = pos
+                    terminator = test_terminator
+            except ValueError:
+                # the terminator is not in the tokens
+                pass
+
+        if terminator:
+            terminator_pos = tokens.index(terminator)
+            # everything before the first terminator is the command and the args
+            (result.command, result.args) = self._command_and_args(tokens[:terminator_pos])
+            result.terminator = tokens[terminator_pos]
+            # we will set the suffix later
+            # remove all the tokens before and including the terminator
+            tokens = tokens[terminator_pos+1:]
+
+        # check for input from file
+        try:
+            if tokens[0] == '<':
+                result.inputFrom = ' '.join(tokens[1:])
+                tokens = []
+        except IndexError:
+            # no input from file
+            pass
+
+        # check for output redirect
+        try:
+            output_pos = tokens.index('>')
+            result.output = '>'
+            result.outputTo = ' '.join(tokens[output_pos+1:])
+            # remove all the tokens after the output redirect
+            tokens = tokens[:output_pos]
+        except ValueError:
+            pass
+
+        # check for paste buffer
+        try:
+            output_pos = tokens.index('>>')
+            result.output = '>>'
+            # remove all tokens after the output redirect
+            tokens = tokens[:output_pos]
+        except ValueError:
+            pass
+
+        # check for pipes
+        try:
+            # find the first pipe if it exists
+            pipe_pos = tokens.index('|')
+            # set everything after the first pipe to result.pipeTo
+            result.pipeTo = ' '.join(tokens[pipe_pos+1:])
+            # remove all the tokens after the pipe
+            tokens = tokens[:pipe_pos]
+        except ValueError:
+            # no pipe in the tokens
+            pass
+        
+        if result.terminator:
+            # whatever is left is the suffix
+            result.suffix = ' '.join(tokens)
+        else:
+            # no terminator, so whatever is left is the command and the args
+            (result.command, result.args) = self._command_and_args(tokens)
+
+        if result.command in self.multilineCommands:
+            result.multilineCommand = result.command
+
+        return result
+    
+    def _command_and_args(self, tokens):
+        """given a list of tokens, and return a tuple of the command
+        and the args as a string.
+        """
+        command = None
+        args = None
+
+        if tokens:
+            command = tokens[0]
+
+        if len(tokens) > 1:
+            args = ' '.join(tokens[1:])
+
+        return (command, args)
+
+    def split_on_punctuation(self, initial_tokens):
+        """
+        # Further splits tokens from a command line using punctuation characters
+        # as word breaks when they are in unquoted strings. Each run of punctuation
+        # characters is treated as a single token.
+
+        :param initial_tokens: the tokens as parsed by shlex
+        :return: the punctuated tokens
+        """
+        punctuation = []
+        punctuation.extend(self.terminators)
+        if self.allow_redirection:
+            punctuation.extend(self.redirection_chars)
+
+        punctuated_tokens = []
+
+        for cur_initial_token in initial_tokens:
+
+            # Save tokens up to 1 character in length or quoted tokens. No need to parse these.
+            if len(cur_initial_token) <= 1 or cur_initial_token[0] in self.quotes:
+                punctuated_tokens.append(cur_initial_token)
+                continue
+
+            # Iterate over each character in this token
+            cur_index = 0
+            cur_char = cur_initial_token[cur_index]
+
+            # Keep track of the token we are building
+            new_token = ''
+
+            while True:
+                if cur_char not in punctuation:
+
+                    # Keep appending to new_token until we hit a punctuation char
+                    while cur_char not in punctuation:
+                        new_token += cur_char
+                        cur_index += 1
+                        if cur_index < len(cur_initial_token):
+                            cur_char = cur_initial_token[cur_index]
+                        else:
+                            break
+
+                else:
+                    cur_punc = cur_char
+
+                    # Keep appending to new_token until we hit something other than cur_punc
+                    while cur_char == cur_punc:
+                        new_token += cur_char
+                        cur_index += 1
+                        if cur_index < len(cur_initial_token):
+                            cur_char = cur_initial_token[cur_index]
+                        else:
+                            break
+
+                # Save the new token
+                punctuated_tokens.append(new_token)
+                new_token = ''
+
+                # Check if we've viewed all characters
+                if cur_index >= len(cur_initial_token):
+                    break
+
+        return punctuated_tokens
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 305d80d91..5237fd802 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -17,207 +17,20 @@
 
 """
 
-import re
-import shlex
+import cmd2
+from cmd2.parsing import CommandParser
 
 import pytest
 
-import cmd2
-
-class Cmd2Command():
-    pass
-
-class Cmd2Parser():
-    # settings or variables from cmd2.py
-    terminator = ';'
-    allow_redirection = True
-    REDIRECTION_CHARS = ['|', '<', '>']
-    QUOTES = ['"', "'"]
-    multilineCommands = ['multiline']
-
-    def parseString(self, rawinput):
-        result = Cmd2Command()
-        result.raw = rawinput
-        result.command = None
-        result.multilineCommand = None
-        result.args = None
-        result.terminator = None
-        result.suffix = None
-        result.pipeTo = None
-        result.output = None
-        result.outputTo = None
-
-        # strip C-style and C++-style comments
-        # shlex will handle the python/shell style comments for us
-        def replacer(match):
-                s = match.group(0)
-                if s.startswith('/'):
-                    # treat the removed comment as a space token, not an empty string
-                    # return ' '
-                    # jk, always return nothing
-                    return ''
-                else:
-                    return s
-        pattern = re.compile(
-            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
-            re.DOTALL | re.MULTILINE
-        )
-        rawinput = re.sub(pattern, replacer, rawinput)
-
-        s = shlex.shlex(rawinput, posix=False)
-        s.whitespace_split = True
-        tokens = self.split_on_punctuation(list(s))
-        
-        # look for the semicolon terminator
-        try:
-            terminator_pos = tokens.index(self.terminator)
-            # everything before the first terminator is the command and the args
-            (result.command, result.args) = self._command_and_args(tokens[:terminator_pos])
-            result.terminator = tokens[terminator_pos]
-            # we will set the suffix later
-            # remove all the tokens before and including the terminator
-            tokens = tokens[terminator_pos+1:]
-        except ValueError:
-            # no terminator in the tokens
-            pass
-
-        # check for input from file
-        try:
-            if tokens[0] == '<':
-                result.inputFrom = ' '.join(tokens[1:])
-                tokens = []
-        except IndexError:
-            # no input from file
-            pass
-
-        # check for output redirect
-        try:
-            output_pos = tokens.index('>')
-            result.output = '>'
-            result.outputTo = ' '.join(tokens[output_pos+1:])
-            # remove all the tokens after the output redirect
-            tokens = tokens[:output_pos]
-        except ValueError:
-            pass
-
-        # check for paste buffer
-        try:
-            output_pos = tokens.index('>>')
-            result.output = '>>'
-            # remove all tokens after the output redirect
-            tokens = tokens[:output_pos]
-        except ValueError:
-            pass
-
-        # check for pipes
-        try:
-            # find the first pipe if it exists
-            pipe_pos = tokens.index('|')
-            # set everything after the first pipe to result.pipeTo
-            result.pipeTo = ' '.join(tokens[pipe_pos+1:])
-            # remove all the tokens after the pipe
-            tokens = tokens[:pipe_pos]
-        except ValueError:
-            # no pipe in the tokens
-            pass
-        
-        if result.terminator:
-            # whatever is left is the suffix
-            result.suffix = ' '.join(tokens)
-            if result.command in self.multilineCommands:
-                result.multilineCommand = result.command
-        else:
-            # no terminator, so whatever is left is the command and the args
-            (result.command, result.args) = self._command_and_args(tokens)            
-
-        return result
-    
-    def _command_and_args(self, tokens):
-        """given a list of tokens, and return a tuple of the command
-        and the args as a string.
-        """
-        command = None
-        args = None
-
-        if tokens:
-            command = tokens[0]
-
-        if len(tokens) > 1:
-            args = ' '.join(tokens[1:])
-
-        return (command, args)
-
-    def split_on_punctuation(self, initial_tokens):
-        """
-        # Further splits tokens from a command line using punctuation characters
-        # as word breaks when they are in unquoted strings. Each run of punctuation
-        # characters is treated as a single token.
-
-        :param initial_tokens: the tokens as parsed by shlex
-        :return: the punctuated tokens
-        """
-        punctuation = [self.terminator] # should be self.terminator from cmd2.py
-        if self.allow_redirection:  # should be self.allow_redirection from cmd2.py
-            punctuation += self.REDIRECTION_CHARS # should be REDIRECTION_CHARS from cmd2.py
-
-        punctuated_tokens = []
-
-        for cur_initial_token in initial_tokens:
-
-            # Save tokens up to 1 character in length or quoted tokens. No need to parse these.
-            if len(cur_initial_token) <= 1 or cur_initial_token[0] in self.QUOTES: # should be QUOTES in cmd2.py
-                punctuated_tokens.append(cur_initial_token)
-                continue
-
-            # Iterate over each character in this token
-            cur_index = 0
-            cur_char = cur_initial_token[cur_index]
-
-            # Keep track of the token we are building
-            new_token = ''
-
-            while True:
-                if cur_char not in punctuation:
-
-                    # Keep appending to new_token until we hit a punctuation char
-                    while cur_char not in punctuation:
-                        new_token += cur_char
-                        cur_index += 1
-                        if cur_index < len(cur_initial_token):
-                            cur_char = cur_initial_token[cur_index]
-                        else:
-                            break
-
-                else:
-                    cur_punc = cur_char
-
-                    # Keep appending to new_token until we hit something other than cur_punc
-                    while cur_char == cur_punc:
-                        new_token += cur_char
-                        cur_index += 1
-                        if cur_index < len(cur_initial_token):
-                            cur_char = cur_initial_token[cur_index]
-                        else:
-                            break
-
-                # Save the new token
-                punctuated_tokens.append(new_token)
-                new_token = ''
-
-                # Check if we've viewed all characters
-                if cur_index >= len(cur_initial_token):
-                    break
-
-        return punctuated_tokens
-
-######
-#
-# unit tests
-#
-######
 @pytest.fixture
 def parser():
-    parser = Cmd2Parser()
+    parser = CommandParser(
+        quotes=['"', "'"],
+        allow_redirection=True,
+        redirection_chars=['|', '<', '>'],        
+        terminators = [';'],
+        multilineCommands = ['multiline']
+    )
     return parser
 
 def test_parse_empty_string(parser):
@@ -397,12 +210,12 @@ def test_parse_multiline_command_ignores_redirectors_within_it(parser):
 #     assert results.multilineCommand == 'multiline'
 #     assert not 'args' in results
 
-# def test_parse_multiline_with_complete_comment(parser):
-#     line = 'multiline command /* with comment complete */ is done;'
-#     results = parser.parseString(line)
-#     assert results.multilineCommand == 'multiline'
-#     assert results.args == 'command /* with comment complete */ is done'
-#     assert results.terminator == ';'
+def test_parse_multiline_with_complete_comment(parser):
+    line = 'multiline command /* with comment complete */ is done;'
+    results = parser.parseString(line)
+    assert results.multilineCommand == 'multiline'
+    assert results.args == 'command is done'
+    assert results.terminator == ';'
 
 # def test_parse_multiline_termninated_by_empty_line(parser):
 #     line = 'multiline command ends\n\n'

From 27b5ab6da0955fda521febada0070d4b53c7e255 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sat, 21 Apr 2018 16:24:45 -0600
Subject: [PATCH 23/89] A bit of renaming

---
 cmd2/parsing.py            | 22 +++++++++++-----------
 tests/test_shlexparsing.py |  3 +++
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 41a3ed0b3..41ce5743c 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -9,7 +9,16 @@
 
 class Command():
     """Store the results of a parsed command."""
-    pass
+    def __init__(self, rawinput):
+        self.raw = rawinput
+        self.command = None
+        self.multilineCommand = None
+        self.args = None
+        self.terminator = None
+        self.suffix = None
+        self.pipeTo = None
+        self.output = None
+        self.outputTo = None
 
 class CommandParser():
     """Parse raw text into command components."""
@@ -28,16 +37,7 @@ def __init__(
         self.multilineCommands = multilineCommands
 
     def parseString(self, rawinput):
-        result = Command()
-        result.raw = rawinput
-        result.command = None
-        result.multilineCommand = None
-        result.args = None
-        result.terminator = None
-        result.suffix = None
-        result.pipeTo = None
-        result.output = None
-        result.outputTo = None
+        result = Command(rawinput)
 
         # strip C-style and C++-style comments
         # shlex will handle the python/shell style comments for us
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 5237fd802..5d3c9546f 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -15,6 +15,9 @@
     - C++-style -> // comment
     - Python/Shell style -> # comment
 
+Functions in cmd2.py to be modified:
+- _complete_statement()
+
 """
 
 import cmd2

From ece08c33108a04cdeeb49af090c56e45edf46ee7 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sat, 21 Apr 2018 17:04:04 -0600
Subject: [PATCH 24/89] new shlex based parser grafted into cmd2.py

Miles to go, but the new parser is partially grafted in to cmd2, and the mumble command from example.py works.
---
 cmd2/cmd2.py | 80 +++++++++++++++++++++++++++++-----------------------
 1 file changed, 45 insertions(+), 35 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 60d1dbf8b..3dc0cb5f0 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -52,6 +52,8 @@
 # Set up readline
 from .rl_utils import rl_force_redisplay, readline, rl_type, RlType
 
+from cmd2.parsing import CommandParser
+
 if rl_type == RlType.PYREADLINE:
 
     # Save the original pyreadline display completion function since we need to override it and restore it
@@ -703,11 +705,11 @@ class Cmd(cmd.Cmd):
     """
     # Attributes used to configure the ParserManager (all are not dynamically settable at runtime)
     blankLinesAllowed = False
-    commentGrammars = pyparsing.Or([pyparsing.pythonStyleComment, pyparsing.cStyleComment])
-    commentInProgress = pyparsing.Literal('/*') + pyparsing.SkipTo(pyparsing.stringEnd ^ '*/')
-    legalChars = u'!#$%.:?@_-' + pyparsing.alphanums + pyparsing.alphas8bit
+    commentGrammars = pyparsing.Or([pyparsing.pythonStyleComment, pyparsing.cStyleComment]) # deleteme
+    commentInProgress = pyparsing.Literal('/*') + pyparsing.SkipTo(pyparsing.stringEnd ^ '*/') # deleteme
+    legalChars = u'!#$%.:?@_-' + pyparsing.alphanums + pyparsing.alphas8bit # deleteme
     multilineCommands = []
-    prefixParser = pyparsing.Empty()
+    prefixParser = pyparsing.Empty() #deleteme
     redirector = '>'        # for sending output to file
     shortcuts = {'?': 'help', '!': 'shell', '@': 'load', '@@': '_relative_load'}
     aliases = dict()
@@ -798,13 +800,20 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
         self.history = History()
         self.pystate = {}
         self.keywords = self.reserved_words + [fname[3:] for fname in dir(self) if fname.startswith('do_')]
-        self.parser_manager = ParserManager(redirector=self.redirector, terminators=self.terminators,
-                                            multilineCommands=self.multilineCommands,
-                                            legalChars=self.legalChars, commentGrammars=self.commentGrammars,
-                                            commentInProgress=self.commentInProgress,
-                                            blankLinesAllowed=self.blankLinesAllowed, prefixParser=self.prefixParser,
-                                            preparse=self.preparse, postparse=self.postparse, aliases=self.aliases,
-                                            shortcuts=self.shortcuts)
+        self.command_parser = CommandParser(
+            quotes=QUOTES,
+            allow_redirection=self.allow_redirection,
+            redirection_chars=REDIRECTION_CHARS,
+            terminators=self.terminators,
+            multilineCommands=self.multilineCommands,
+        )
+        # self.parser_manager = ParserManager(redirector=self.redirector, terminators=self.terminators,
+        #                                     multilineCommands=self.multilineCommands,
+        #                                     legalChars=self.legalChars, commentGrammars=self.commentGrammars,
+        #                                     commentInProgress=self.commentInProgress,
+        #                                     blankLinesAllowed=self.blankLinesAllowed, prefixParser=self.prefixParser,
+        #                                     preparse=self.preparse, postparse=self.postparse, aliases=self.aliases,
+        #                                     shortcuts=self.shortcuts)
         self._transcript_files = transcript_files
 
         # Used to enable the ability for a Python script to quit the application
@@ -2205,14 +2214,15 @@ def runcmds_plus_hooks(self, cmds):
 
     def _complete_statement(self, line):
         """Keep accepting lines of input until the command is complete."""
-        if not line or (not pyparsing.Or(self.commentGrammars).setParseAction(lambda x: '').transformString(line)):
-            raise EmptyStatement()
-        statement = self.parser_manager.parsed(line)
-        while statement.parsed.multilineCommand and (statement.parsed.terminator == ''):
-            statement = '%s\n%s' % (statement.parsed.raw,
-                                    self.pseudo_raw_input(self.continuation_prompt))
-            statement = self.parser_manager.parsed(statement)
-        if not statement.parsed.command:
+        #if not line or (not pyparsing.Or(self.commentGrammars).setParseAction(lambda x: '').transformString(line)):
+        #    raise EmptyStatement()
+        # statement = self.parser_manager.parsed(line) # deleteme
+        statement = self.command_parser.parseString(line)
+        #while statement.parsed.multilineCommand and (statement.parsed.terminator == ''):
+        #    statement = '%s\n%s' % (statement.parsed.raw,
+        #                            self.pseudo_raw_input(self.continuation_prompt))
+        #    statement = self.parser_manager.parsed(statement)
+        if not statement.command:
             raise EmptyStatement()
         return statement
 
@@ -2221,7 +2231,7 @@ def _redirect_output(self, statement):
 
         :param statement: ParsedString - subclass of str which also contains pyparsing ParseResults instance
         """
-        if statement.parsed.pipeTo:
+        if statement.pipeTo:
             self.kept_state = Statekeeper(self, ('stdout',))
 
             # Create a pipe with read and write sides
@@ -2236,7 +2246,7 @@ def _redirect_output(self, statement):
 
             # We want Popen to raise an exception if it fails to open the process.  Thus we don't set shell to True.
             try:
-                self.pipe_proc = subprocess.Popen(shlex.split(statement.parsed.pipeTo), stdin=subproc_stdin)
+                self.pipe_proc = subprocess.Popen(shlex.split(statement.pipeTo), stdin=subproc_stdin)
             except Exception as ex:
                 # Restore stdout to what it was and close the pipe
                 self.stdout.close()
@@ -2248,20 +2258,20 @@ def _redirect_output(self, statement):
 
                 # Re-raise the exception
                 raise ex
-        elif statement.parsed.output:
-            if (not statement.parsed.outputTo) and (not can_clip):
+        elif statement.output:
+            if (not statement.outputTo) and (not can_clip):
                 raise EnvironmentError('Cannot redirect to paste buffer; install ``xclip`` and re-run to enable')
             self.kept_state = Statekeeper(self, ('stdout',))
             self.kept_sys = Statekeeper(sys, ('stdout',))
             self.redirecting = True
-            if statement.parsed.outputTo:
+            if statement.outputTo:
                 mode = 'w'
-                if statement.parsed.output == 2 * self.redirector:
+                if statement.output == 2 * self.redirector:
                     mode = 'a'
-                sys.stdout = self.stdout = open(os.path.expanduser(statement.parsed.outputTo), mode)
+                sys.stdout = self.stdout = open(os.path.expanduser(statement.outputTo), mode)
             else:
                 sys.stdout = self.stdout = tempfile.TemporaryFile(mode="w+")
-                if statement.parsed.output == '>>':
+                if statement.output == '>>':
                     self.poutput(get_paste_buffer())
 
     def _restore_output(self, statement):
@@ -2272,7 +2282,7 @@ def _restore_output(self, statement):
         # If we have redirected output to a file or the clipboard or piped it to a shell command, then restore state
         if self.kept_state is not None:
             # If we redirected output to the clipboard
-            if statement.parsed.output and not statement.parsed.outputTo:
+            if statement.output and not statement.outputTo:
                 self.stdout.seek(0)
                 write_to_paste_buffer(self.stdout.read())
 
@@ -2310,29 +2320,29 @@ def _func_named(self, arg):
             result = target
         return result
 
-    def onecmd(self, line):
+    def onecmd(self, statement):
         """ This executes the actual do_* method for a command.
 
         If the command provided doesn't exist, then it executes _default() instead.
 
-        :param line: ParsedString - subclass of string including the pyparsing ParseResults
+        :param line: Command - a parsed command from the input stream
         :return: bool - a flag indicating whether the interpretation of commands should stop
         """
-        statement = self.parser_manager.parsed(line)
-        funcname = self._func_named(statement.parsed.command)
+        #statement = self.parser_manager.parsed(line) # deleteme
+        funcname = self._func_named(statement.command)
         if not funcname:
             return self.default(statement)
 
         # Since we have a valid command store it in the history
-        if statement.parsed.command not in self.exclude_from_history:
-            self.history.append(statement.parsed.raw)
+        if statement.command not in self.exclude_from_history:
+            self.history.append(statement.raw)
 
         try:
             func = getattr(self, funcname)
         except AttributeError:
             return self.default(statement)
 
-        stop = func(statement)
+        stop = func("{} {}".format(statement.command, statement.args))
         return stop
 
     def default(self, statement):

From 2145115a1567284807389ffc771fa0222071e09e Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 22 Apr 2018 17:07:50 -0600
Subject: [PATCH 25/89] Delete plyparsing experiment

---
 tests/test_plyparsing.py | 214 ---------------------------------------
 1 file changed, 214 deletions(-)
 delete mode 100644 tests/test_plyparsing.py

diff --git a/tests/test_plyparsing.py b/tests/test_plyparsing.py
deleted file mode 100644
index f33527a22..000000000
--- a/tests/test_plyparsing.py
+++ /dev/null
@@ -1,214 +0,0 @@
-# coding=utf-8
-"""
-Unit/functional testing for ply based parsing in cmd2
-
-Notes:
-
-- Shortcuts may have to be discarded, or handled in a different way than they
-  are with pyparsing.
-- 
-
-"""
-
-import pytest
-import ply.lex as lex
-import ply.yacc as yacc
-
-class Cmd2Command():
-    pass
-
-class Cmd2Lexer():
-    """a ply.lex lexer for the cmd2 syntax.
-    Once initial development is completed, this code
-    should be moved into cmd2.Cmd()
-
-    Unlike most python classes, the order of the methods matters here. The ply module
-    uses introspection to create an ordered list of grammer rules, so rearranging methods
-    impacts functionality.
-    """
-    def __init__(self):
-        self.results = Cmd2Command()
-
-    tokens = (
-        'HASHCOMMENT', 'CCOMMENT', 'WORD', 'DQWORD', 'SQWORD',
-    )
-
-    def t_HASHCOMMENT(self, t):
-        r'\#.*'
-        # no return value, token discarded
-        pass
-    
-    def t_CCOMMENT(self, t):
-        r'/\*.*\*/'
-        # no return value, token discarded
-        pass
-
-    def t_WORD(self, t):
-        r'[-\w$%\.:\?@!]+'
-        return t
-    
-    def t_DQWORD(self, t):
-        r'"(?:[^"\\]|\\.)*"'
-        return t
-    
-    def t_SQWORD(self, t):
-        r"'(?:[^'\\]|\\.)*'"
-        return t
-
-    # def t_PIPE(self, t):
-    #     r'[|]'
-    #     return t
-
-    def t_error(self, t):
-        print("Illegal command")
-        t.lexer.skip(1)
-
-    def build_lexer(self, **kwargs):
-        self.lexer = lex.lex(module=self, **kwargs)
-
-    def p_wordlist_add_word(self, p):
-        'wordlist : wordlist word'
-        p[0] = '{} {}'.format(p[1], p[2])
-        self.results.command = p[0]
-
-    def p_wordlist_word(self, p):
-        'wordlist : word'
-        p[0] = p[1]
-        self.results.command = p[0]
-
-    def p_word_word(self, p):
-        'word : WORD'
-        p[0] = p[1]
-
-    def p_word_dqword(self, p):
-        'word : DQWORD'
-        p[0] = p[1]
-
-    def p_word_sqword(self, p):
-        'word : SQWORD'
-        p[0] = p[1]
-
-    def p_command_and_pipe(self, p):
-        "pipeline : wordlist '|' wordlist"
-        p[0] = '{} | {}'.format(p[1], p[3])        
-        self.results.command = p[1]
-        self.results.pipeTo = p[3]
-
-    def p_error(self, p):
-        print("Syntax error in input!")
-
-    def build_parser(self, **kwargs):
-        self.parser = yacc.yacc(module=self, **kwargs)
-
-
-@pytest.fixture
-def cl():
-    cl = Cmd2Lexer()
-    cl.build_lexer()
-    cl.build_parser()
-    return cl
-
-def test_lex_word(cl):
-    cl.lexer.input('plainword')
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == 'plainword'
-    assert not cl.lexer.token()
-
-def test_lex_dqword(cl):
-    cl.lexer.input('"one word"')
-    tok = cl.lexer.token()
-    assert tok.type == 'DQWORD'
-    assert tok.value == '"one word"'
-    assert not cl.lexer.token()
-
-def test_lex_sqword(cl):
-    cl.lexer.input("'one word'")
-    tok = cl.lexer.token()
-    assert tok.type == 'SQWORD'
-    assert tok.value == "'one word'"
-    assert not cl.lexer.token()
-
-def test_lex_dotword(cl):
-    cl.lexer.input('dot.word')
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == 'dot.word'
-    assert not cl.lexer.token()
-        
-def test_lex_command_with_args(cl):
-    cl.lexer.input('123456 with args')
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == '123456'
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == 'with'
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == 'args'
-
-def test_lex_hashcomment(cl):
-    cl.lexer.input('hi # this is all a comment')
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == 'hi'
-    assert not cl.lexer.token()
-
-def test_lex_ccomment(cl):
-    cl.lexer.input('hi /* comment */ there')
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == 'hi'
-    tok = cl.lexer.token()
-    assert tok.type == 'WORD'
-    assert tok.value == 'there'
-    assert not cl.lexer.token()
-
-# def test_lex_command_pipe(cl):
-#     cl.parser.parse('command | pipeto')
-#     tok = cl.lexer.token()
-#     assert tok.type == 'WORD'
-#     assert tok.value == 'command'
-#     tok = cl.lexer.token()
-#     assert tok.type == 'PIPETO'
-#     assert tok.value == '| pipeto'
-#     assert not cl.lexer.token()
-    
-def test_parse_command(cl):
-    cl.parser.parse('plainword')
-    assert cl.results.command == 'plainword'
-
-def test_parse_command_with_args(cl):
-    cl.parser.parse('command with args')
-    assert cl.results.command == 'command with args'
-
-def test_parse_command_with_dqarg(cl):
-    cl.parser.parse('command "with   dqarg"')
-    assert cl.results.command == 'command "with   dqarg"'
-
-def test_parse_command_with_sqarg(cl):
-    cl.parser.parse("command 'with     sqarg'")
-    assert cl.results.command == "command 'with     sqarg'"
-
-def test_parse_command_with_dqarg_and_arg(cl):
-    cl.parser.parse('command "with   dqarg" onemore   lastone')
-    assert cl.results.command == 'command "with   dqarg" onemore lastone'
-
-def test_parse_command_with_sqarg_and_arg(cl):
-    cl.parser.parse("command 'with   dqarg' onemore   lastone")
-    assert cl.results.command == "command 'with   dqarg' onemore lastone"
-
-def test_parse_command_with_comment(cl):
-    cl.parser.parse('command # with a comment')
-    assert cl.results.command == 'command'
-
-# def test_parse_command_with_simple_pipe(cl):
-#     cl.parser.parse('command | pipeto')
-#     assert cl.results.command == 'command'
-#     assert cl.results.pipeTo == '| pipeto'
-
-# def test_parse_command_with_complex_pipe(cl):
-#     cl.parser.parse('command "with   some" args | pipeto "with  the" args')
-#     assert cl.results.command == 'command "with   some" args'
-#     assert cl.results.pipeTo == '| pipeto "with  the" args'

From 829c36cb22fd7f3b71548c1b742d40e7609a3aca Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 22 Apr 2018 17:11:11 -0600
Subject: [PATCH 26/89] refactor Command() to Statement(str)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Command class renamed to Statement, and is now a subclass of str.
- str’s are immutable, and the string needs to contain the arguments, so revise the parseString method
---
 cmd2/cmd2.py               | 46 +++++++++++++-------------
 cmd2/parsing.py            | 67 ++++++++++++++++++++++++++------------
 tests/test_shlexparsing.py |  2 ++
 3 files changed, 71 insertions(+), 44 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 3dc0cb5f0..4e2c053fd 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -386,20 +386,21 @@ def write_to_paste_buffer(txt: str) -> None:
     pyperclip.copy(txt)
 
 
-class ParsedString(str):
-    """Subclass of str which also stores a pyparsing.ParseResults object containing structured parse results."""
-    # pyarsing.ParseResults - structured parse results, to provide multiple means of access to the parsed data
-    parsed = None
+# deleteme
+# class ParsedString(str):
+#     """Subclass of str which also stores a pyparsing.ParseResults object containing structured parse results."""
+#     # pyarsing.ParseResults - structured parse results, to provide multiple means of access to the parsed data
+#     parsed = None
 
-    # Function which did the parsing
-    parser = None
+#     # Function which did the parsing
+#     parser = None
 
-    def full_parsed_statement(self):
-        """Used to reconstruct the full parsed statement when a command isn't recognized."""
-        new = ParsedString('%s %s' % (self.parsed.command, self.parsed.args))
-        new.parsed = self.parsed
-        new.parser = self.parser
-        return new
+#     def full_parsed_statement(self):
+#         """Used to reconstruct the full parsed statement when a command isn't recognized."""
+#         new = ParsedString('%s %s' % (self.parsed.command, self.parsed.args))
+#         new.parsed = self.parsed
+#         new.parser = self.parser
+#         return new
 
 
 def replace_with_file_contents(fname: str) -> str:
@@ -2010,8 +2011,8 @@ def preloop(self):
     def precmd(self, statement):
         """Hook method executed just before the command is processed by ``onecmd()`` and after adding it to the history.
 
-        :param statement: ParsedString - subclass of str which also contains pyparsing ParseResults instance
-        :return: ParsedString - a potentially modified version of the input ParsedString statement
+        :param statement: Statement - subclass of str which also contains the parsed input
+        :return: Statement - a potentially modified version of the input Statement object
         """
         return statement
 
@@ -2027,11 +2028,11 @@ def preparse(self, raw):
         return raw
 
     # noinspection PyMethodMayBeStatic
-    def postparse(self, parse_result):
-        """Hook that runs immediately after parsing the command-line but before ``parsed()`` returns a ParsedString.
+    def postparse(self, statement):
+        """Hook that runs immediately after parsing the user input.
 
-        :param parse_result: pyparsing.ParseResults - parsing results output by the pyparsing parser
-        :return: pyparsing.ParseResults - potentially modified ParseResults object
+        :param statement: Statement object populated by parsing
+        :return: Statement - potentially modified Statement object
         """
         return parse_result
 
@@ -2048,8 +2049,8 @@ def postparsing_precmd(self, statement):
         - raise EmptyStatement - will silently fail and do nothing
         - raise <AnyOtherException> - will fail and print an error message
 
-        :param statement: - the parsed command-line statement
-        :return: (bool, statement) - (stop, statement) containing a potentially modified version of the statement
+        :param statement: - the parsed command-line statement as a Statement object
+        :return: (bool, statement) - (stop, statement) containing a potentially modified version of the statement object
         """
         stop = False
         return stop, statement
@@ -2325,10 +2326,9 @@ def onecmd(self, statement):
 
         If the command provided doesn't exist, then it executes _default() instead.
 
-        :param line: Command - a parsed command from the input stream
+        :param statement: Command - a parsed command from the input stream
         :return: bool - a flag indicating whether the interpretation of commands should stop
         """
-        #statement = self.parser_manager.parsed(line) # deleteme
         funcname = self._func_named(statement.command)
         if not funcname:
             return self.default(statement)
@@ -2342,7 +2342,7 @@ def onecmd(self, statement):
         except AttributeError:
             return self.default(statement)
 
-        stop = func("{} {}".format(statement.command, statement.args))
+        stop = func(statement)
         return stop
 
     def default(self, statement):
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 41ce5743c..164c7735a 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -7,10 +7,19 @@
 
 import cmd2
 
-class Command():
-    """Store the results of a parsed command."""
-    def __init__(self, rawinput):
-        self.raw = rawinput
+class Statement(str):
+    """String subclass with additional attributes to store the results of parsing.
+    
+    The cmd module in the standard library passes commands around as a
+    string. To retain backwards compatibility, cmd2 does the same. However, we
+    need a place to capture the additional output of the command parsing, so we add
+    our own attributes to this subclass.
+
+    The string portion of the class contains the arguments, but not the command, nor
+    the output redirection clauses.
+    """
+    def __init__(self, object):
+        self.raw = str(object)
         self.command = None
         self.multilineCommand = None
         self.args = None
@@ -37,7 +46,7 @@ def __init__(
         self.multilineCommands = multilineCommands
 
     def parseString(self, rawinput):
-        result = Command(rawinput)
+        #result = Statement(rawinput)
 
         # strip C-style and C++-style comments
         # shlex will handle the python/shell style comments for us
@@ -67,6 +76,7 @@ def replacer(match):
                 if pos < terminator_pos:
                     terminator_pos = pos
                     terminator = test_terminator
+                    break
             except ValueError:
                 # the terminator is not in the tokens
                 pass
@@ -74,35 +84,37 @@ def replacer(match):
         if terminator:
             terminator_pos = tokens.index(terminator)
             # everything before the first terminator is the command and the args
-            (result.command, result.args) = self._command_and_args(tokens[:terminator_pos])
-            result.terminator = tokens[terminator_pos]
+            (command, args) = self._command_and_args(tokens[:terminator_pos])
+            #terminator = tokens[terminator_pos]
             # we will set the suffix later
             # remove all the tokens before and including the terminator
             tokens = tokens[terminator_pos+1:]
 
         # check for input from file
+        inputFrom = None
         try:
             if tokens[0] == '<':
-                result.inputFrom = ' '.join(tokens[1:])
+                inputFrom = ' '.join(tokens[1:])
                 tokens = []
         except IndexError:
-            # no input from file
             pass
 
+
         # check for output redirect
         try:
             output_pos = tokens.index('>')
-            result.output = '>'
-            result.outputTo = ' '.join(tokens[output_pos+1:])
+            output = '>'
+            outputTo = ' '.join(tokens[output_pos+1:])
             # remove all the tokens after the output redirect
             tokens = tokens[:output_pos]
         except ValueError:
-            pass
+            output = None
+            outputTo = None
 
         # check for paste buffer
         try:
             output_pos = tokens.index('>>')
-            result.output = '>>'
+            output = '>>'
             # remove all tokens after the output redirect
             tokens = tokens[:output_pos]
         except ValueError:
@@ -113,23 +125,36 @@ def replacer(match):
             # find the first pipe if it exists
             pipe_pos = tokens.index('|')
             # set everything after the first pipe to result.pipeTo
-            result.pipeTo = ' '.join(tokens[pipe_pos+1:])
+            pipeTo = ' '.join(tokens[pipe_pos+1:])
             # remove all the tokens after the pipe
             tokens = tokens[:pipe_pos]
         except ValueError:
             # no pipe in the tokens
-            pass
+            pipeTo = None
         
-        if result.terminator:
+        if terminator:
             # whatever is left is the suffix
-            result.suffix = ' '.join(tokens)
+            suffix = ' '.join(tokens)
         else:
             # no terminator, so whatever is left is the command and the args
-            (result.command, result.args) = self._command_and_args(tokens)
-
-        if result.command in self.multilineCommands:
-            result.multilineCommand = result.command
+            suffix = None
+            (command, args) = self._command_and_args(tokens)
 
+        if command in self.multilineCommands:
+            multilineCommand = command
+        else:
+            multilineCommand = None
+
+        result = Statement(args)
+        result.command = command
+        result.args = args
+        result.terminator = terminator
+        result.inputFrom = inputFrom
+        result.output = output
+        result.outputTo = outputTo
+        result.pipeTo = pipeTo
+        result.suffix = suffix
+        result.multilineCommand = multilineCommand
         return result
     
     def _command_and_args(self, tokens):
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 5d3c9546f..9142e1788 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -5,6 +5,8 @@
 Todo List
 - multiline
 - case sensitive flag
+- checkout Cmd2.parseline() function which parses and expands shortcuts and such
+  this code should probably be included in CommandParser
 
 Notes:
 

From f83154e2749d90bf4ec24c7c84b45dc0860e8b13 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 22 Apr 2018 19:11:29 -0600
Subject: [PATCH 27/89] =?UTF-8?q?args=20has=20to=20be=20=E2=80=98=E2=80=99?=
 =?UTF-8?q?=20not=20None?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 cmd2/parsing.py            |  9 ++++++---
 tests/test_shlexparsing.py | 11 ++---------
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 164c7735a..79d57a32a 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -22,7 +22,8 @@ def __init__(self, object):
         self.raw = str(object)
         self.command = None
         self.multilineCommand = None
-        self.args = None
+        # has to be an empty string for compatibility with standard library cmd
+        self.args = ''
         self.terminator = None
         self.suffix = None
         self.pipeTo = None
@@ -58,7 +59,8 @@ def replacer(match):
                 else:
                     return s
         pattern = re.compile(
-            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+            #r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+            r'/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
             re.DOTALL | re.MULTILINE
         )
         rawinput = re.sub(pattern, replacer, rawinput)
@@ -146,6 +148,7 @@ def replacer(match):
             multilineCommand = None
 
         result = Statement(args)
+        result.raw = rawinput
         result.command = command
         result.args = args
         result.terminator = terminator
@@ -162,7 +165,7 @@ def _command_and_args(self, tokens):
         and the args as a string.
         """
         command = None
-        args = None
+        args = ''
 
         if tokens:
             command = tokens[0]
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 9142e1788..fd0103840 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -14,7 +14,6 @@
   are with pyparsing.
 - valid comment styles:
     - C-style -> /* comment */
-    - C++-style -> // comment
     - Python/Shell style -> # comment
 
 Functions in cmd2.py to be modified:
@@ -43,8 +42,8 @@ def test_parse_empty_string(parser):
     assert not results.command
 
 @pytest.mark.parametrize('tokens,command,args', [
-    ( [], None, None),
-    ( ['command'], 'command', None ),
+    ( [], None, ''),
+    ( ['command'], 'command', '' ),
     ( ['command', 'arg1', 'arg2'], 'command', 'arg1 arg2')
 ])
 def test_command_and_args(parser, tokens, command, args):
@@ -113,12 +112,6 @@ def test_parse_what_if_quoted_strings_seem_to_start_comments(parser):
     assert results.args == 'if "quoted strings /* seem to " start comments?'
     assert not results.pipeTo
 
-def test_cpp_comment(parser):
-    results = parser.parseString('hi // this is | all a comment */')
-    assert results.command == 'hi'
-    assert not results.args
-    assert not results.pipeTo
-
 def test_simple_piped(parser):
     results = parser.parseString('simple | piped')
     assert results.command == 'simple'

From 1c98fe58f126eb3222140bfcb074f3a74346e17a Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 22 Apr 2018 20:30:45 -0600
Subject: [PATCH 28/89] Fix default() method

---
 cmd2/cmd2.py               | 4 ++--
 tests/test_shlexparsing.py | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 4e2c053fd..a124c690c 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2348,10 +2348,10 @@ def onecmd(self, statement):
     def default(self, statement):
         """Executed when the command given isn't a recognized command implemented by a do_* method.
 
-        :param statement: ParsedString - subclass of string including the pyparsing ParseResults
+        :param statement: Statement object with parsed input
         :return:
         """
-        arg = statement.full_parsed_statement()
+        arg = statement.raw
         if self.default_to_shell:
             result = os.system(arg)
             # If os.system() succeeded, then don't print warning about unknown command
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index fd0103840..02dc51d3f 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -19,6 +19,9 @@
 Functions in cmd2.py to be modified:
 - _complete_statement()
 
+Changelog Items:
+- if self.default_to_shell is true, then redirection and piping is now properly passed to the shell, previously it was truncated
+- object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything.
 """
 
 import cmd2

From 7cec2eb3689fa6e8c2a36080841f2210dfa94697 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 22 Apr 2018 20:45:13 -0600
Subject: [PATCH 29/89] Submenus now call all hooks

---
 cmd2/cmd2.py               | 13 +++++++------
 tests/test_shlexparsing.py |  4 ++++
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index a124c690c..7650d6609 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -586,7 +586,7 @@ def _copy_out_shared_attrs(self, parent_cmd, original_attributes):
 
     def __call__(self, cmd_obj):
         """Creates a subclass of Cmd wherein the given submenu can be accessed via the given command"""
-        def enter_submenu(parent_cmd, line):
+        def enter_submenu(parent_cmd, statement):
             """
             This function will be bound to do_<submenu> and will change the scope of the CLI to that of the
             submenu.
@@ -605,12 +605,13 @@ def enter_submenu(parent_cmd, line):
                 # copy over any shared attributes
                 self._copy_in_shared_attrs(parent_cmd)
 
-                if line.parsed.args:
+                if statement.args:
                     # Remove the menu argument and execute the command in the submenu
-                    line = submenu.parser_manager.parsed(line.parsed.args)
-                    submenu.precmd(line)
-                    ret = submenu.onecmd(line)
-                    submenu.postcmd(ret, line)
+                    submenu.onecmd_plus_hooks(statement.args)
+                    # statement = parent_cmd.command_parser.parseLine(statement.args)
+                    # submenu.precmd(statement)
+                    # ret = submenu.onecmd(statement)
+                    # submenu.postcmd(ret, statement)
                 else:
                     if self.reformat_prompt is not None:
                         prompt = submenu.prompt
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 02dc51d3f..b69f18980 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -18,10 +18,14 @@
 
 Functions in cmd2.py to be modified:
 - _complete_statement()
+- parsed() - expands aliases and shortcuts
 
 Changelog Items:
 - if self.default_to_shell is true, then redirection and piping is now properly passed to the shell, previously it was truncated
 - object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything.
+
+Bugs fixed:
+- submenus now all all hooks, it used to just call precmd and postcmd
 """
 
 import cmd2

From 4f7d0f18509ab4f50c88e942e7ba33a7824b58ac Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 22 Apr 2018 20:48:27 -0600
Subject: [PATCH 30/89] remove old pyparsing unit tests, they have all been
 duplicated in shlex

---
 tests/test_parsing.py | 239 ------------------------------------------
 1 file changed, 239 deletions(-)

diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 2682ec686..5825e735d 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -20,39 +20,6 @@ def hist():
     h = cmd2.cmd2.History([HistoryItem('first'), HistoryItem('second'), HistoryItem('third'), HistoryItem('fourth')])
     return h
 
-# Case-sensitive parser
-@pytest.fixture
-def parser():
-    c = cmd2.Cmd()
-    c.multilineCommands = ['multiline']
-    c.parser_manager = cmd2.cmd2.ParserManager(redirector=c.redirector, terminators=c.terminators,
-                                               multilineCommands=c.multilineCommands, legalChars=c.legalChars,
-                                               commentGrammars=c.commentGrammars, commentInProgress=c.commentInProgress,
-                                               blankLinesAllowed=c.blankLinesAllowed, prefixParser=c.prefixParser,
-                                               preparse=c.preparse, postparse=c.postparse, aliases=c.aliases,
-                                               shortcuts=c.shortcuts)
-    return c.parser_manager.main_parser
-
-# Case-sensitive ParserManager
-@pytest.fixture
-def cs_pm():
-    c = cmd2.Cmd()
-    c.multilineCommands = ['multiline']
-    c.parser_manager = cmd2.cmd2.ParserManager(redirector=c.redirector, terminators=c.terminators,
-                                               multilineCommands=c.multilineCommands, legalChars=c.legalChars,
-                                               commentGrammars=c.commentGrammars, commentInProgress=c.commentInProgress,
-                                               blankLinesAllowed=c.blankLinesAllowed, prefixParser=c.prefixParser,
-                                               preparse=c.preparse, postparse=c.postparse, aliases=c.aliases,
-                                               shortcuts=c.shortcuts)
-    return c.parser_manager
-
-
-@pytest.fixture
-def input_parser():
-    c = cmd2.Cmd()
-    return c.parser_manager.input_source_parser
-
-
 def test_history_span(hist):
     h = hist
     assert h == ['first', 'second', 'third', 'fourth']
@@ -119,212 +86,6 @@ def test_cast_problems(capsys):
     out, err = capsys.readouterr()
     assert out == expected.format(current, new)
 
-
-def test_parse_empty_string(parser):
-    assert parser.parseString('').dump() == '[]'
-
-def test_parse_only_comment(parser):
-    assert parser.parseString('/* empty command */').dump() == '[]'
-
-def test_parse_single_word(parser):
-    line = 'plainword'
-    results = parser.parseString(line)
-    assert results.command == line
-
-def test_parse_word_plus_terminator(parser):
-    line = 'termbare;'
-    results = parser.parseString(line)
-    assert results.command == 'termbare'
-    assert results.terminator == ';'
-
-def test_parse_suffix_after_terminator(parser):
-    line = 'termbare; suffx'
-    results = parser.parseString(line)
-    assert results.command == 'termbare'
-    assert results.terminator == ';'
-    assert results.suffix == 'suffx'
-
-def test_parse_command_with_args(parser):
-    line = 'command with args'
-    results = parser.parseString(line)
-    assert results.command == 'command'
-    assert results.args == 'with args'
-
-def test_parse_command_with_args_terminator_and_suffix(parser):
-    line = 'command with args and terminator; and suffix'
-    results = parser.parseString(line)
-    assert results.command == 'command'
-    assert results.args == "with args and terminator"
-    assert results.terminator == ';'
-    assert results.suffix == 'and suffix'
-
-def test_parse_simple_piped(parser):
-    line = 'simple | piped'
-    results = parser.parseString(line)
-    assert results.command == 'simple'
-    assert results.pipeTo == " piped"
-
-def test_parse_double_pipe_is_not_a_pipe(parser):
-    line = 'double-pipe || is not a pipe'
-    results = parser.parseString(line)
-    assert results.command == 'double-pipe'
-    assert results.args == '|| is not a pipe'
-    assert not 'pipeTo' in results
-
-def test_parse_complex_pipe(parser):
-    line = 'command with args, terminator;sufx | piped'
-    results = parser.parseString(line)
-    assert results.command == 'command'
-    assert results.args == "with args, terminator"
-    assert results.terminator == ';'
-    assert results.suffix == 'sufx'
-    assert results.pipeTo == ' piped'
-
-def test_parse_output_redirect(parser):
-    line = 'output into > afile.txt'
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'into'
-    assert results.output == '>'
-    assert results.outputTo == 'afile.txt'
-
-def test_parse_output_redirect_with_dash_in_path(parser):
-    line = 'output into > python-cmd2/afile.txt'
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'into'
-    assert results.output == '>'
-    assert results.outputTo == 'python-cmd2/afile.txt'
-
-
-def test_case_sensitive_parsed_single_word(cs_pm):
-    line = 'HeLp'
-    statement = cs_pm.parsed(line)
-    assert statement.parsed.command == line
-
-
-def test_parse_input_redirect(input_parser):
-    line = '< afile.txt'
-    results = input_parser.parseString(line)
-    assert results.inputFrom == line
-
-def test_parse_input_redirect_with_dash_in_path(input_parser):
-    line = "< python-cmd2/afile.txt"
-    results = input_parser.parseString(line)
-    assert results.inputFrom == line
-
-def test_parse_pipe_and_redirect(parser):
-    line = 'output into;sufx | pipethrume plz > afile.txt'
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'into'
-    assert results.terminator == ';'
-    assert results.suffix == 'sufx'
-    assert results.pipeTo == ' pipethrume plz'
-    assert results.output == '>'
-    assert results.outputTo == 'afile.txt'
-
-def test_parse_output_to_paste_buffer(parser):
-    line = 'output to paste buffer >> '
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'to paste buffer'
-    assert results.output == '>>'
-
-def test_parse_ignore_commented_redirectors(parser):
-    line = 'ignore the /* commented | > */ stuff;'
-    results = parser.parseString(line)
-    assert results.command == 'ignore'
-    assert results.args == 'the /* commented | > */ stuff'
-    assert results.terminator == ';'
-
-def test_parse_has_redirect_inside_terminator(parser):
-    """The terminator designates the end of the commmand/arguments portion.  If a redirector
-    occurs before a terminator, then it will be treated as part of the arguments and not as a redirector."""
-    line = 'has > inside;'
-    results = parser.parseString(line)
-    assert results.command == 'has'
-    assert results.args == '> inside'
-    assert results.terminator == ';'
-
-def test_parse_what_if_quoted_strings_seem_to_start_comments(parser):
-    line = 'what if "quoted strings /* seem to " start comments?'
-    results = parser.parseString(line)
-    assert results.command == 'what'
-    assert results.args == 'if "quoted strings /* seem to " start comments?'
-
-def test_parse_unfinished_multiliine_command(parser):
-    line = 'multiline has > inside an unfinished command'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert not 'args' in results
-
-def test_parse_multiline_command_ignores_redirectors_within_it(parser):
-    line = 'multiline has > inside;'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'has > inside'
-    assert results.terminator == ';'
-
-def test_parse_multiline_with_incomplete_comment(parser):
-    """A terminator within a comment will be ignored and won't terminate a multiline command.
-    Un-closed comments effectively comment out everything after the start."""
-    line = 'multiline command /* with comment in progress;'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert not 'args' in results
-
-def test_parse_multiline_with_complete_comment(parser):
-    line = 'multiline command /* with comment complete */ is done;'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'command /* with comment complete */ is done'
-    assert results.terminator == ';'
-
-def test_parse_multiline_termninated_by_empty_line(parser):
-    line = 'multiline command ends\n\n'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'command ends'
-    assert len(results.terminator) == 2
-    assert results.terminator[0] == '\n'
-    assert results.terminator[1] == '\n'
-
-def test_parse_multiline_ignores_terminators_in_comments(parser):
-    line = 'multiline command "with term; ends" now\n\n'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'command "with term; ends" now'
-    assert len(results.terminator) == 2
-    assert results.terminator[0] == '\n'
-    assert results.terminator[1] == '\n'
-
-def test_parse_command_with_unicode_args(parser):
-    line = 'drink café'
-    results = parser.parseString(line)
-    assert results.command == 'drink'
-    assert results.args == 'café'
-
-def test_parse_unicode_command(parser):
-    line = 'café au lait'
-    results = parser.parseString(line)
-    assert results.command == 'café'
-    assert results.args == 'au lait'
-
-def test_parse_redirect_to_unicode_filename(parser):
-    line = 'dir home > café'
-    results = parser.parseString(line)
-    assert results.command == 'dir'
-    assert results.args == 'home'
-    assert results.output == '>'
-    assert results.outputTo == 'café'
-
-def test_parse_input_redirect_from_unicode_filename(input_parser):
-    line = '< café'
-    results = input_parser.parseString(line)
-    assert results.inputFrom == line
-
-
 def test_empty_statement_raises_exception():
     app = cmd2.Cmd()
     with pytest.raises(cmd2.cmd2.EmptyStatement):

From a735180192ff900234e9c2c3ab4a7786ad7ec8d3 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 22 Apr 2018 21:30:49 -0600
Subject: [PATCH 31/89] Fix default_to_shell unit tests

---
 cmd2/cmd2.py       | 6 +-----
 tests/test_cmd2.py | 4 ++--
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 7650d6609..c11816a5b 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -608,10 +608,6 @@ def enter_submenu(parent_cmd, statement):
                 if statement.args:
                     # Remove the menu argument and execute the command in the submenu
                     submenu.onecmd_plus_hooks(statement.args)
-                    # statement = parent_cmd.command_parser.parseLine(statement.args)
-                    # submenu.precmd(statement)
-                    # ret = submenu.onecmd(statement)
-                    # submenu.postcmd(ret, statement)
                 else:
                     if self.reformat_prompt is not None:
                         prompt = submenu.prompt
@@ -2035,7 +2031,7 @@ def postparse(self, statement):
         :param statement: Statement object populated by parsing
         :return: Statement - potentially modified Statement object
         """
-        return parse_result
+        return statement
 
     # noinspection PyMethodMayBeStatic
     def postparsing_precmd(self, statement):
diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py
index 35ef4c0f1..6075ca7bd 100644
--- a/tests/test_cmd2.py
+++ b/tests/test_cmd2.py
@@ -961,7 +961,7 @@ def test_default_to_shell_good(capsys):
         line = 'dir'
     else:
         line = 'ls'
-    statement = app.parser_manager.parsed(line)
+    statement = app.command_parser.parseString(line)
     retval = app.default(statement)
     assert not retval
     out, err = capsys.readouterr()
@@ -971,7 +971,7 @@ def test_default_to_shell_failure(capsys):
     app = cmd2.Cmd()
     app.default_to_shell = True
     line = 'ls does_not_exist.xyz'
-    statement = app.parser_manager.parsed(line)
+    statement = app.command_parser.parseString(line)
     retval = app.default(statement)
     assert not retval
     out, err = capsys.readouterr()

From 65bf06a6e9712c87802bf8c319442a8b4cb00e6f Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 22 Apr 2018 21:55:33 -0600
Subject: [PATCH 32/89] Updates to comments and todo list

---
 cmd2/parsing.py            | 4 +---
 tests/test_shlexparsing.py | 2 ++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 79d57a32a..5bb8d6547 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -47,9 +47,7 @@ def __init__(
         self.multilineCommands = multilineCommands
 
     def parseString(self, rawinput):
-        #result = Statement(rawinput)
-
-        # strip C-style and C++-style comments
+        # strip C-style comments
         # shlex will handle the python/shell style comments for us
         def replacer(match):
                 s = match.group(0)
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index b69f18980..0029ca071 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -7,6 +7,8 @@
 - case sensitive flag
 - checkout Cmd2.parseline() function which parses and expands shortcuts and such
   this code should probably be included in CommandParser
+- get rid of legalChars
+
 
 Notes:
 

From 7ac59187ffbc14e2eb14a00866231c8b18e1a087 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 18:34:08 -0600
Subject: [PATCH 33/89] Shortcut and alias processing added to CommandParser()

---
 cmd2/parsing.py            | 44 +++++++++++++++++++++++++++++++++++---
 tests/test_shlexparsing.py | 20 ++++++++++++++---
 2 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 5bb8d6547..dece2b5e5 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -31,7 +31,10 @@ def __init__(self, object):
         self.outputTo = None
 
 class CommandParser():
-    """Parse raw text into command components."""
+    """Parse raw text into command components.
+    
+    Shortcuts is a list of tuples with each tuple containing the shortcut and the expansion.
+    """
     def __init__(
             self,
             quotes=['"', "'"],
@@ -39,14 +42,18 @@ def __init__(
             redirection_chars=['|', '<', '>'],
             terminators=[';'],
             multilineCommands = [],
+            aliases = {},
+            shortcuts = [],
         ):
         self.quotes = quotes
         self.allow_redirection = allow_redirection
         self.redirection_chars = redirection_chars
         self.terminators = terminators
         self.multilineCommands = multilineCommands
+        self.aliases = aliases
+        self.shortcuts = shortcuts
 
-    def parseString(self, rawinput):
+    def parseString(self, line):
         # strip C-style comments
         # shlex will handle the python/shell style comments for us
         def replacer(match):
@@ -61,7 +68,22 @@ def replacer(match):
             r'/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
             re.DOTALL | re.MULTILINE
         )
-        rawinput = re.sub(pattern, replacer, rawinput)
+        line = re.sub(pattern, replacer, line)
+        rawinput = line
+
+        # expand shortcuts, have to do this first because
+        # a shortcut can expand into multiple tokens, ie '!ls' becomes
+        # 'shell ls'
+        for (shortcut, expansion) in self.shortcuts:
+            if  rawinput.startswith(shortcut):
+                # If the next character after the shortcut isn't a space, then insert one
+                shortcut_len = len(shortcut)
+                if len(rawinput) == shortcut_len or rawinput[shortcut_len] != ' ':
+                    expansion += ' '
+
+                # Expand the shortcut
+                rawinput = rawinput.replace(shortcut, expansion, 1)
+                break
 
         s = shlex.shlex(rawinput, posix=False)
         s.whitespace_split = True
@@ -140,11 +162,27 @@ def replacer(match):
             suffix = None
             (command, args) = self._command_and_args(tokens)
 
+        # expand aliases
+        # make a copy of aliases so we can edit it
+        tmp_aliases = list(self.aliases.keys())
+        keep_expanding = len(tmp_aliases) > 0
+
+        while keep_expanding:
+            for cur_alias in tmp_aliases:
+                keep_expanding = False
+                if command == cur_alias:
+                    command = self.aliases[cur_alias]
+                    tmp_aliases.remove(cur_alias)
+                    keep_expanding = len(tmp_aliases) > 0
+                    break
+
+        # set multiline
         if command in self.multilineCommands:
             multilineCommand = command
         else:
             multilineCommand = None
 
+        # build Statement object
         result = Statement(args)
         result.raw = rawinput
         result.command = command
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 0029ca071..b8d4b208b 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -12,11 +12,10 @@
 
 Notes:
 
-- Shortcuts may have to be discarded, or handled in a different way than they
-  are with pyparsing.
 - valid comment styles:
     - C-style -> /* comment */
     - Python/Shell style -> # comment
+- we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
 
 Functions in cmd2.py to be modified:
 - _complete_statement()
@@ -42,7 +41,9 @@ def parser():
         allow_redirection=True,
         redirection_chars=['|', '<', '>'],        
         terminators = [';'],
-        multilineCommands = ['multiline']
+        multilineCommands = ['multiline'],
+        aliases = {'helpalias': 'help', '42': 'theanswer'},
+        shortcuts = [('?', 'help'), ('!', 'shell')]
     )
     return parser
 
@@ -274,3 +275,16 @@ def test_empty_statement_raises_exception():
 
     with pytest.raises(cmd2.cmd2.EmptyStatement):
         app._complete_statement(' ')
+
+@pytest.mark.parametrize('line,command,args', [
+    ('helpalias', 'help', ''),
+    ('helpalias mycommand', 'help', 'mycommand'),
+    ('42', 'theanswer', ''),
+    ('42 arg1 arg2', 'theanswer', 'arg1 arg2'),
+    ('!ls', 'shell', 'ls'),
+    ('!ls -al /tmp', 'shell', 'ls -al /tmp'),
+])
+def test_alias_and_shortcut_expansion(parser, line, command, args):
+    statement = parser.parseString(line)
+    assert statement.command == command
+    assert statement.args == args

From f47568f8dfdf0a9c909c266b8de3233d1ae8a4fa Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 18:40:14 -0600
Subject: [PATCH 34/89] Shortcuts and aliases fully implemented.

---
 cmd2/cmd2.py    |  2 ++
 cmd2/parsing.py | 14 +++++++-------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index c11816a5b..e34db7d52 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -804,6 +804,8 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
             redirection_chars=REDIRECTION_CHARS,
             terminators=self.terminators,
             multilineCommands=self.multilineCommands,
+            aliases=self.aliases,
+            shortcuts=self.shortcuts,
         )
         # self.parser_manager = ParserManager(redirector=self.redirector, terminators=self.terminators,
         #                                     multilineCommands=self.multilineCommands,
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index dece2b5e5..c81106677 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -53,7 +53,7 @@ def __init__(
         self.aliases = aliases
         self.shortcuts = shortcuts
 
-    def parseString(self, line):
+    def parseString(self, rawinput):
         # strip C-style comments
         # shlex will handle the python/shell style comments for us
         def replacer(match):
@@ -68,24 +68,24 @@ def replacer(match):
             r'/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
             re.DOTALL | re.MULTILINE
         )
-        line = re.sub(pattern, replacer, line)
-        rawinput = line
+        rawinput = re.sub(pattern, replacer, rawinput)
+        line = rawinput
 
         # expand shortcuts, have to do this first because
         # a shortcut can expand into multiple tokens, ie '!ls' becomes
         # 'shell ls'
         for (shortcut, expansion) in self.shortcuts:
-            if  rawinput.startswith(shortcut):
+            if  line.startswith(shortcut):
                 # If the next character after the shortcut isn't a space, then insert one
                 shortcut_len = len(shortcut)
-                if len(rawinput) == shortcut_len or rawinput[shortcut_len] != ' ':
+                if len(line) == shortcut_len or line[shortcut_len] != ' ':
                     expansion += ' '
 
                 # Expand the shortcut
-                rawinput = rawinput.replace(shortcut, expansion, 1)
+                line = line.replace(shortcut, expansion, 1)
                 break
 
-        s = shlex.shlex(rawinput, posix=False)
+        s = shlex.shlex(line, posix=False)
         s.whitespace_split = True
         tokens = self.split_on_punctuation(list(s))
         

From 2534df2987a5aec226ce5bfd5b21f1d8f22ed3f2 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 18:45:32 -0600
Subject: [PATCH 35/89] No more importing pyparsing

---
 cmd2/cmd2.py | 37 -------------------------------------
 1 file changed, 37 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index e34db7d52..8aed075bf 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -46,7 +46,6 @@
 import unittest
 from code import InteractiveConsole
 
-import pyparsing
 import pyperclip
 
 # Set up readline
@@ -118,12 +117,6 @@ def __subclasshook__(cls, C):
 
 __version__ = '0.9.0'
 
-# Pyparsing enablePackrat() can greatly speed up parsing, but problems have been seen in Python 3 in the past
-pyparsing.ParserElement.enablePackrat()
-
-# Override the default whitespace chars in Pyparsing so that newlines are not treated as whitespace
-pyparsing.ParserElement.setDefaultWhitespaceChars(' \t')
-
 
 # The next 2 variables and associated setter functions effect how arguments are parsed for decorated commands
 # which use one of the decorators: @with_argument_list, @with_argparser, or @with_argparser_and_unknown_args
@@ -385,24 +378,6 @@ def write_to_paste_buffer(txt: str) -> None:
     """
     pyperclip.copy(txt)
 
-
-# deleteme
-# class ParsedString(str):
-#     """Subclass of str which also stores a pyparsing.ParseResults object containing structured parse results."""
-#     # pyarsing.ParseResults - structured parse results, to provide multiple means of access to the parsed data
-#     parsed = None
-
-#     # Function which did the parsing
-#     parser = None
-
-#     def full_parsed_statement(self):
-#         """Used to reconstruct the full parsed statement when a command isn't recognized."""
-#         new = ParsedString('%s %s' % (self.parsed.command, self.parsed.args))
-#         new.parsed = self.parsed
-#         new.parser = self.parser
-#         return new
-
-
 def replace_with_file_contents(fname: str) -> str:
     """Action to perform when successfully matching parse element definition for inputFrom parser.
 
@@ -703,11 +678,7 @@ class Cmd(cmd.Cmd):
     """
     # Attributes used to configure the ParserManager (all are not dynamically settable at runtime)
     blankLinesAllowed = False
-    commentGrammars = pyparsing.Or([pyparsing.pythonStyleComment, pyparsing.cStyleComment]) # deleteme
-    commentInProgress = pyparsing.Literal('/*') + pyparsing.SkipTo(pyparsing.stringEnd ^ '*/') # deleteme
-    legalChars = u'!#$%.:?@_-' + pyparsing.alphanums + pyparsing.alphas8bit # deleteme
     multilineCommands = []
-    prefixParser = pyparsing.Empty() #deleteme
     redirector = '>'        # for sending output to file
     shortcuts = {'?': 'help', '!': 'shell', '@': 'load', '@@': '_relative_load'}
     aliases = dict()
@@ -807,13 +778,6 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
             aliases=self.aliases,
             shortcuts=self.shortcuts,
         )
-        # self.parser_manager = ParserManager(redirector=self.redirector, terminators=self.terminators,
-        #                                     multilineCommands=self.multilineCommands,
-        #                                     legalChars=self.legalChars, commentGrammars=self.commentGrammars,
-        #                                     commentInProgress=self.commentInProgress,
-        #                                     blankLinesAllowed=self.blankLinesAllowed, prefixParser=self.prefixParser,
-        #                                     preparse=self.preparse, postparse=self.postparse, aliases=self.aliases,
-        #                                     shortcuts=self.shortcuts)
         self._transcript_files = transcript_files
 
         # Used to enable the ability for a Python script to quit the application
@@ -897,7 +861,6 @@ def visible_prompt(self):
         return strip_ansi(self.prompt)
 
     def _finalize_app_parameters(self):
-        self.commentGrammars.ignore(pyparsing.quotedString).setParseAction(lambda x: '')
         # noinspection PyUnresolvedReferences
         self.shortcuts = sorted(self.shortcuts.items(), reverse=True)
 

From b3d71457e951d9d382787cb82fdf77f32951337c Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 20:16:55 -0600
Subject: [PATCH 36/89] Fix parsing of input redirection and appending output

---
 cmd2/parsing.py            | 15 ++++++++-------
 tests/test_cmd2.py         |  2 +-
 tests/test_shlexparsing.py | 18 +++++++++++++++++-
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index c81106677..ec8e2e846 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -115,14 +115,16 @@ def replacer(match):
         # check for input from file
         inputFrom = None
         try:
-            if tokens[0] == '<':
-                inputFrom = ' '.join(tokens[1:])
-                tokens = []
-        except IndexError:
+            input_pos = tokens.index('<')
+            inputFrom = ' '.join(tokens[input_pos+1:])
+            tokens = tokens[:input_pos]
+        except ValueError:
             pass
 
 
         # check for output redirect
+        output = None
+        outputTo = None
         try:
             output_pos = tokens.index('>')
             output = '>'
@@ -130,13 +132,12 @@ def replacer(match):
             # remove all the tokens after the output redirect
             tokens = tokens[:output_pos]
         except ValueError:
-            output = None
-            outputTo = None
+            pass
 
-        # check for paste buffer
         try:
             output_pos = tokens.index('>>')
             output = '>>'
+            outputTo = ' '.join(tokens[output_pos+1:])
             # remove all tokens after the output redirect
             tokens = tokens[:output_pos]
         except ValueError:
diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py
index 6075ca7bd..f2ee16af7 100644
--- a/tests/test_cmd2.py
+++ b/tests/test_cmd2.py
@@ -607,7 +607,7 @@ def test_input_redirection(base_app, request):
 
     # NOTE: File 'redirect.txt" contains 1 word "history"
 
-    # Verify that redirecting input ffom a file works
+    # Verify that redirecting input from a file works
     out = run_cmd(base_app, 'help < {}'.format(filename))
     assert out == normalize(HELP_HISTORY)
 
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index b8d4b208b..b9caed7c2 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -8,7 +8,8 @@
 - checkout Cmd2.parseline() function which parses and expands shortcuts and such
   this code should probably be included in CommandParser
 - get rid of legalChars
-
+- move remaining tests in test_parsing.py to test_cmd2.py
+- rename test_shlexparsing.py to test_parsing.py
 
 Notes:
 
@@ -160,11 +161,26 @@ def test_output_redirect_with_dash_in_path(parser):
     assert results.output == '>'
     assert results.outputTo == 'python-cmd2/afile.txt'
 
+def test_output_redirect_append(parser):
+    line = 'output appended to >> /tmp/afile.txt'
+    results = parser.parseString(line)
+    assert results.command == 'output'
+    assert results.args == 'appended to'
+    assert results.output == '>>'
+    assert results.outputTo == '/tmp/afile.txt'
+
 def test_parse_input_redirect(parser):
     line = '< afile.txt'
     results = parser.parseString(line)
     assert results.inputFrom == 'afile.txt'
 
+def test_parse_input_redirect_after_command(parser):
+    line = 'help < afile.txt'
+    results = parser.parseString(line)
+    assert results.command == 'help'
+    assert results.args == ''
+    assert results.inputFrom == 'afile.txt'
+
 def test_parse_input_redirect_with_dash_in_path(parser):
     line = '< python-cmd2/afile.txt'
     results = parser.parseString(line)

From 7f7adaf2fa211e877987aef075affe2a7082dbc5 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 20:41:04 -0600
Subject: [PATCH 37/89] More work on multiline

---
 cmd2/cmd2.py               |  8 ++++----
 cmd2/parsing.py            |  4 ++++
 tests/test_shlexparsing.py | 12 +++++++-----
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 8aed075bf..d9cbd2b02 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2181,10 +2181,10 @@ def _complete_statement(self, line):
         #    raise EmptyStatement()
         # statement = self.parser_manager.parsed(line) # deleteme
         statement = self.command_parser.parseString(line)
-        #while statement.parsed.multilineCommand and (statement.parsed.terminator == ''):
-        #    statement = '%s\n%s' % (statement.parsed.raw,
-        #                            self.pseudo_raw_input(self.continuation_prompt))
-        #    statement = self.parser_manager.parsed(statement)
+        while statement.multilineCommand and (statement.terminator == ''):
+            line = '%s\n%s' % (statement.raw,
+                                    self.pseudo_raw_input(self.continuation_prompt))
+            statement = self.command_parser.parseString(line)
         if not statement.command:
             raise EmptyStatement()
         return statement
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index ec8e2e846..f4f9a6a38 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -180,6 +180,10 @@ def replacer(match):
         # set multiline
         if command in self.multilineCommands:
             multilineCommand = command
+            # return no arguments if this is a "partial" command,
+            # i.e. we have a multiline command but no terminator yet
+            if not terminator:
+                args = ''
         else:
             multilineCommand = None
 
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index b9caed7c2..d09078205 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -213,11 +213,13 @@ def test_has_redirect_inside_terminator(parser):
     assert results.args == '> inside'
     assert results.terminator == ';'
 
-# def test_parse_unfinished_multiliine_command(parser):
-#     line = 'multiline has > inside an unfinished command'
-#     results = parser.parseString(line)
-#     assert results.multilineCommand == 'multiline'
-#     assert not 'args' in results
+def test_parse_unfinished_multiliine_command(parser):
+    line = 'multiline has > inside an unfinished command'
+    statement = parser.parseString(line)
+    assert statement.multilineCommand == 'multiline'
+    assert not statement.args
+    assert not statement.terminator
+    
 
 def test_parse_multiline_command_ignores_redirectors_within_it(parser):
     line = 'multiline has > inside;'

From 4411d8d68c57e8cfca323b80369a8d3c5f11c9d4 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 21:13:33 -0600
Subject: [PATCH 38/89] Multiline support mostly done

---
 cmd2/cmd2.py               |  2 +-
 cmd2/parsing.py            | 15 ++++++++++++--
 tests/test_cmd2.py         |  2 +-
 tests/test_shlexparsing.py | 41 +++++++++++++++++++-------------------
 4 files changed, 36 insertions(+), 24 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index d9cbd2b02..8165f4113 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2181,7 +2181,7 @@ def _complete_statement(self, line):
         #    raise EmptyStatement()
         # statement = self.parser_manager.parsed(line) # deleteme
         statement = self.command_parser.parseString(line)
-        while statement.multilineCommand and (statement.terminator == ''):
+        while statement.multilineCommand and not statement.terminator:
             line = '%s\n%s' % (statement.raw,
                                     self.pseudo_raw_input(self.continuation_prompt))
             statement = self.command_parser.parseString(line)
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index f4f9a6a38..2c01fb70b 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -7,6 +7,8 @@
 
 import cmd2
 
+BLANK_LINE = '\n\n'
+
 class Statement(str):
     """String subclass with additional attributes to store the results of parsing.
     
@@ -85,13 +87,19 @@ def replacer(match):
                 line = line.replace(shortcut, expansion, 1)
                 break
 
+        # handle the special case/hardcoded terminator of a blank line
+        # we have to do this before we shlex on whitespace because it
+        # destroys all unquoted whitespace in the input
+        terminator = None
+        if line[-2:] == BLANK_LINE:
+            terminator = BLANK_LINE
+
         s = shlex.shlex(line, posix=False)
         s.whitespace_split = True
         tokens = self.split_on_punctuation(list(s))
         
         # of the valid terminators, find the first one to occur in the input
         terminator_pos = len(tokens)+1
-        terminator = None
         for test_terminator in self.terminators:
             try:
                 pos = tokens.index(test_terminator)
@@ -104,7 +112,10 @@ def replacer(match):
                 pass
 
         if terminator:
-            terminator_pos = tokens.index(terminator)
+            if terminator == BLANK_LINE:
+                terminator_pos = len(tokens)+1
+            else:
+                terminator_pos = tokens.index(terminator)
             # everything before the first terminator is the command and the args
             (command, args) = self._command_and_args(tokens[:terminator_pos])
             #terminator = tokens[terminator_pos]
diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py
index f2ee16af7..068ea08fe 100644
--- a/tests/test_cmd2.py
+++ b/tests/test_cmd2.py
@@ -1358,7 +1358,7 @@ def test_multiline_complete_statement_without_terminator(multiline_app):
     line = '{} {}'.format(command, args)
     statement = multiline_app._complete_statement(line)
     assert statement == args
-    assert statement.parsed.command == command
+    assert statement.command == command
 
 
 def test_clipboard_failure(capsys):
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index d09078205..269a5753a 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -232,9 +232,10 @@ def test_parse_multiline_command_ignores_redirectors_within_it(parser):
 #     """A terminator within a comment will be ignored and won't terminate a multiline command.
 #     Un-closed comments effectively comment out everything after the start."""
 #     line = 'multiline command /* with comment in progress;'
-#     results = parser.parseString(line)
-#     assert results.multilineCommand == 'multiline'
-#     assert not 'args' in results
+#     statement = parser.parseString(line)
+#     assert statement.multilineCommand == 'multiline'
+#     assert statement.args == 'command'
+#     assert not statement.terminator
 
 def test_parse_multiline_with_complete_comment(parser):
     line = 'multiline command /* with comment complete */ is done;'
@@ -243,23 +244,23 @@ def test_parse_multiline_with_complete_comment(parser):
     assert results.args == 'command is done'
     assert results.terminator == ';'
 
-# def test_parse_multiline_termninated_by_empty_line(parser):
-#     line = 'multiline command ends\n\n'
-#     results = parser.parseString(line)
-#     assert results.multilineCommand == 'multiline'
-#     assert results.args == 'command ends'
-#     assert len(results.terminator) == 2
-#     assert results.terminator[0] == '\n'
-#     assert results.terminator[1] == '\n'
-
-# def test_parse_multiline_ignores_terminators_in_comments(parser):
-#     line = 'multiline command "with term; ends" now\n\n'
-#     results = parser.parseString(line)
-#     assert results.multilineCommand == 'multiline'
-#     assert results.args == 'command "with term; ends" now'
-#     assert len(results.terminator) == 2
-#     assert results.terminator[0] == '\n'
-#     assert results.terminator[1] == '\n'
+def test_parse_multiline_termninated_by_empty_line(parser):
+    line = 'multiline command ends\n\n'
+    results = parser.parseString(line)
+    assert results.multilineCommand == 'multiline'
+    assert results.args == 'command ends'
+    assert len(results.terminator) == 2
+    assert results.terminator[0] == '\n'
+    assert results.terminator[1] == '\n'
+
+def test_parse_multiline_ignores_terminators_in_comments(parser):
+    line = 'multiline command "with term; ends" now\n\n'
+    results = parser.parseString(line)
+    assert results.multilineCommand == 'multiline'
+    assert results.args == 'command "with term; ends" now'
+    assert len(results.terminator) == 2
+    assert results.terminator[0] == '\n'
+    assert results.terminator[1] == '\n'
 
 def test_parse_command_with_unicode_args(parser):
     line = 'drink café'

From 4cee87baf6df2896c60d2d3a8811e9e65ed24c84 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 22:02:58 -0600
Subject: [PATCH 39/89] Fix some old bugs and bad behavior in multiline input

---
 cmd2/cmd2.py | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 8165f4113..4528b14e7 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2182,9 +2182,33 @@ def _complete_statement(self, line):
         # statement = self.parser_manager.parsed(line) # deleteme
         statement = self.command_parser.parseString(line)
         while statement.multilineCommand and not statement.terminator:
-            line = '%s\n%s' % (statement.raw,
-                                    self.pseudo_raw_input(self.continuation_prompt))
+            if not self.quit_on_sigint:
+                try:
+                    newline = self.pseudo_raw_input(self.continuation_prompt)
+                    if newline == 'eof':
+                        # they entered either a blank line, or we hit an EOF
+                        # for some other reason. Turn the literal 'eof'
+                        # into a blank line, which serves as a command
+                        # terminator
+                        newline = '\n'
+                        self.poutput(newline)
+                    line = '%s\n%s' % (statement.raw, newline)
+                except KeyboardInterrupt:
+                    self.poutput('^C')
+                    statement = self.command_parser.parseString('')
+                    break
+            else:
+                newline = self.pseudo_raw_input(self.continuation_prompt)
+                if newline == 'eof':
+                    # they entered either a blank line, or we hit an EOF
+                    # for some other reason. Turn the literal 'eof'
+                    # into a blank line, which serves as a command
+                    # terminator
+                    newline = '\n'
+                    self.poutput(newline)
+                line = '%s\n%s' % (statement.raw, newline)
             statement = self.command_parser.parseString(line)
+
         if not statement.command:
             raise EmptyStatement()
         return statement

From c378b1da3c1533f538ae616e550dd4ffd270d85f Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 22:43:41 -0600
Subject: [PATCH 40/89] Another multiline fix

---
 cmd2/cmd2.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 4528b14e7..b4c1dbf65 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2176,7 +2176,13 @@ def runcmds_plus_hooks(self, cmds):
             return stop
 
     def _complete_statement(self, line):
-        """Keep accepting lines of input until the command is complete."""
+        """Keep accepting lines of input until the command is complete.
+        
+        There is some pretty hacky code here to handle some quirks of
+        self.pseudo_raw_input(). It returns a literal 'eof' if the input
+        pipe runs out. We can't refactor it because we need to retain
+        backwards compatibility with the standard library version of cmd.
+        """
         #if not line or (not pyparsing.Or(self.commentGrammars).setParseAction(lambda x: '').transformString(line)):
         #    raise EmptyStatement()
         # statement = self.parser_manager.parsed(line) # deleteme
@@ -2192,7 +2198,7 @@ def _complete_statement(self, line):
                         # terminator
                         newline = '\n'
                         self.poutput(newline)
-                    line = '%s\n%s' % (statement.raw, newline)
+                    line = '{}\n{}\n'.format(statement.raw, newline)
                 except KeyboardInterrupt:
                     self.poutput('^C')
                     statement = self.command_parser.parseString('')
@@ -2206,7 +2212,7 @@ def _complete_statement(self, line):
                     # terminator
                     newline = '\n'
                     self.poutput(newline)
-                line = '%s\n%s' % (statement.raw, newline)
+                line = '{}\n{}\n'.format(statement.raw, newline)
             statement = self.command_parser.parseString(line)
 
         if not statement.command:

From 3441d1e5abcd95ff9100905a9184400d590c600d Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 22:55:24 -0600
Subject: [PATCH 41/89] Multiline now working

---
 cmd2/cmd2.py               | 4 ++--
 cmd2/parsing.py            | 4 ++--
 tests/test_shlexparsing.py | 8 ++------
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index b4c1dbf65..7b01a6534 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2198,7 +2198,7 @@ def _complete_statement(self, line):
                         # terminator
                         newline = '\n'
                         self.poutput(newline)
-                    line = '{}\n{}\n'.format(statement.raw, newline)
+                    line = '{}\n{}'.format(statement.raw, newline)
                 except KeyboardInterrupt:
                     self.poutput('^C')
                     statement = self.command_parser.parseString('')
@@ -2212,7 +2212,7 @@ def _complete_statement(self, line):
                     # terminator
                     newline = '\n'
                     self.poutput(newline)
-                line = '{}\n{}\n'.format(statement.raw, newline)
+                line = '{}\n{}'.format(statement.raw, newline)
             statement = self.command_parser.parseString(line)
 
         if not statement.command:
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 2c01fb70b..ffeb8bbe1 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -7,7 +7,7 @@
 
 import cmd2
 
-BLANK_LINE = '\n\n'
+BLANK_LINE = '\n'
 
 class Statement(str):
     """String subclass with additional attributes to store the results of parsing.
@@ -91,7 +91,7 @@ def replacer(match):
         # we have to do this before we shlex on whitespace because it
         # destroys all unquoted whitespace in the input
         terminator = None
-        if line[-2:] == BLANK_LINE:
+        if line[-1:] == BLANK_LINE:
             terminator = BLANK_LINE
 
         s = shlex.shlex(line, posix=False)
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 269a5753a..936e60920 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -249,18 +249,14 @@ def test_parse_multiline_termninated_by_empty_line(parser):
     results = parser.parseString(line)
     assert results.multilineCommand == 'multiline'
     assert results.args == 'command ends'
-    assert len(results.terminator) == 2
-    assert results.terminator[0] == '\n'
-    assert results.terminator[1] == '\n'
+    assert results.terminator == '\n'
 
 def test_parse_multiline_ignores_terminators_in_comments(parser):
     line = 'multiline command "with term; ends" now\n\n'
     results = parser.parseString(line)
     assert results.multilineCommand == 'multiline'
     assert results.args == 'command "with term; ends" now'
-    assert len(results.terminator) == 2
-    assert results.terminator[0] == '\n'
-    assert results.terminator[1] == '\n'
+    assert results.terminator == '\n'
 
 def test_parse_command_with_unicode_args(parser):
     line = 'drink café'

From 4d2143d3537a5a7b2a964ffa425ef5702aa8c15c Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 23 Apr 2018 22:59:12 -0600
Subject: [PATCH 42/89] Add todo for implementing input redirection

---
 tests/test_shlexparsing.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 936e60920..9c2c8f9ba 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -3,7 +3,7 @@
 Unit/functional testing for ply based parsing in cmd2
 
 Todo List
-- multiline
+- implement input redirection
 - case sensitive flag
 - checkout Cmd2.parseline() function which parses and expands shortcuts and such
   this code should probably be included in CommandParser
@@ -19,7 +19,6 @@
 - we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
 
 Functions in cmd2.py to be modified:
-- _complete_statement()
 - parsed() - expands aliases and shortcuts
 
 Changelog Items:

From 8297d4d1c0a4f56c6c952059fb7fc2b43b1050ed Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Tue, 24 Apr 2018 21:15:54 -0600
Subject: [PATCH 43/89] Refactoring and code cleanup

- rename CommandParser to StatementParser
- move tests from test_shlexparsing.py to test_parsing.py
- standardize the output of the parse() method into a variable called statement.
---
 cmd2/cmd2.py               | 197 +--------------------
 cmd2/parsing.py            |   6 +-
 tests/test_cmd2.py         |  87 ++++++++--
 tests/test_parsing.py      | 346 +++++++++++++++++++++++++++++--------
 tests/test_shlexparsing.py | 284 +-----------------------------
 5 files changed, 358 insertions(+), 562 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 43973b64a..41f46d5c9 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -52,7 +52,7 @@
 from .rl_utils import rl_force_redisplay, readline, rl_type, RlType
 from .argparse_completer import AutoCompleter, ACArgumentParser
 
-from cmd2.parsing import CommandParser
+from cmd2.parsing import StatementParser
 
 if rl_type == RlType.PYREADLINE:
 
@@ -348,23 +348,6 @@ def write_to_paste_buffer(txt: str) -> None:
     """
     pyperclip.copy(txt)
 
-def replace_with_file_contents(fname: str) -> str:
-    """Action to perform when successfully matching parse element definition for inputFrom parser.
-
-    :param fname: filename
-    :return: contents of file "fname"
-    """
-    try:
-        # Any outer quotes are not part of the filename
-        unquoted_file = strip_quotes(fname[0])
-        with open(os.path.expanduser(unquoted_file)) as source_file:
-            result = source_file.read()
-    except IOError:
-        result = '< %s' % fname[0]  # wasn't a file after all
-
-    # TODO: IF pyparsing input parser logic gets fixed to support empty file, add support to get from paste buffer
-    return result
-
 
 class EmbeddedConsoleExit(SystemExit):
     """Custom exception class for use with the py command."""
@@ -646,7 +629,7 @@ class Cmd(cmd.Cmd):
 
     Line-oriented command interpreters are often useful for test harnesses, internal tools, and rapid prototypes.
     """
-    # Attributes used to configure the ParserManager (all are not dynamically settable at runtime)
+    # Attributes used to configure the StatementParser, best not to change these at runtime
     blankLinesAllowed = False
     multilineCommands = []
     redirector = '>'        # for sending output to file
@@ -739,7 +722,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
         self.history = History()
         self.pystate = {}
         self.keywords = self.reserved_words + [fname[3:] for fname in dir(self) if fname.startswith('do_')]
-        self.command_parser = CommandParser(
+        self.statement_parser = StatementParser(
             quotes=QUOTES,
             allow_redirection=self.allow_redirection,
             redirection_chars=REDIRECTION_CHARS,
@@ -2121,10 +2104,7 @@ def _complete_statement(self, line):
         pipe runs out. We can't refactor it because we need to retain
         backwards compatibility with the standard library version of cmd.
         """
-        #if not line or (not pyparsing.Or(self.commentGrammars).setParseAction(lambda x: '').transformString(line)):
-        #    raise EmptyStatement()
-        # statement = self.parser_manager.parsed(line) # deleteme
-        statement = self.command_parser.parseString(line)
+        statement = self.statement_parser.parse(line)
         while statement.multilineCommand and not statement.terminator:
             if not self.quit_on_sigint:
                 try:
@@ -2139,7 +2119,7 @@ def _complete_statement(self, line):
                     line = '{}\n{}'.format(statement.raw, newline)
                 except KeyboardInterrupt:
                     self.poutput('^C')
-                    statement = self.command_parser.parseString('')
+                    statement = self.statement_parser.parse('')
                     break
             else:
                 newline = self.pseudo_raw_input(self.continuation_prompt)
@@ -2151,7 +2131,7 @@ def _complete_statement(self, line):
                     newline = '\n'
                     self.poutput(newline)
                 line = '{}\n{}'.format(statement.raw, newline)
-            statement = self.command_parser.parseString(line)
+            statement = self.statement_parser.parse(line)
 
         if not statement.command:
             raise EmptyStatement()
@@ -2160,7 +2140,7 @@ def _complete_statement(self, line):
     def _redirect_output(self, statement):
         """Handles output redirection for >, >>, and |.
 
-        :param statement: ParsedString - subclass of str which also contains pyparsing ParseResults instance
+        :param statement: Statement - a parsed statement from the user
         """
         if statement.pipeTo:
             self.kept_state = Statekeeper(self, ('stdout',))
@@ -2208,7 +2188,7 @@ def _redirect_output(self, statement):
     def _restore_output(self, statement):
         """Handles restoring state after output redirection as well as the actual pipe operation if present.
 
-        :param statement: ParsedString - subclass of str which also contains pyparsing ParseResults instance
+        :param statement: Statement object which contains the parsed input from the user
         """
         # If we have redirected output to a file or the clipboard or piped it to a shell command, then restore state
         if self.kept_state is not None:
@@ -3312,167 +3292,6 @@ def cmdloop(self, intro=None):
         self.postloop()
 
 
-# noinspection PyPep8Naming
-class ParserManager:
-    """
-    Class which encapsulates all of the pyparsing parser functionality for cmd2 in a single location.
-    """
-    def __init__(self, redirector, terminators, multilineCommands, legalChars, commentGrammars, commentInProgress,
-                 blankLinesAllowed, prefixParser, preparse, postparse, aliases, shortcuts):
-        """Creates and uses parsers for user input according to app's parameters."""
-
-        self.commentGrammars = commentGrammars
-        self.preparse = preparse
-        self.postparse = postparse
-        self.aliases = aliases
-        self.shortcuts = shortcuts
-
-        self.main_parser = self._build_main_parser(redirector=redirector, terminators=terminators,
-                                                   multilineCommands=multilineCommands, legalChars=legalChars,
-                                                   commentInProgress=commentInProgress,
-                                                   blankLinesAllowed=blankLinesAllowed, prefixParser=prefixParser)
-        self.input_source_parser = self._build_input_source_parser(legalChars=legalChars,
-                                                                   commentInProgress=commentInProgress)
-
-    def _build_main_parser(self, redirector, terminators, multilineCommands, legalChars, commentInProgress,
-                           blankLinesAllowed, prefixParser):
-        """Builds a PyParsing parser for interpreting user commands."""
-
-        # Build several parsing components that are eventually compiled into overall parser
-        output_destination_parser = (pyparsing.Literal(redirector * 2) |
-                                     (pyparsing.WordStart() + redirector) |
-                                     pyparsing.Regex('[^=]' + redirector))('output')
-
-        terminator_parser = pyparsing.Or(
-            [(hasattr(t, 'parseString') and t) or pyparsing.Literal(t) for t in terminators])('terminator')
-        string_end = pyparsing.stringEnd ^ '\nEOF'
-        multilineCommand = pyparsing.Or(
-            [pyparsing.Keyword(c, caseless=False) for c in multilineCommands])('multilineCommand')
-        oneline_command = (~multilineCommand + pyparsing.Word(legalChars))('command')
-        pipe = pyparsing.Keyword('|', identChars='|')
-        do_not_parse = self.commentGrammars | commentInProgress | pyparsing.quotedString
-        after_elements = \
-            pyparsing.Optional(pipe + pyparsing.SkipTo(output_destination_parser ^ string_end,
-                                                       ignore=do_not_parse)('pipeTo')) + \
-            pyparsing.Optional(output_destination_parser +
-                               pyparsing.SkipTo(string_end, ignore=do_not_parse).
-                               setParseAction(lambda x: strip_quotes(x[0].strip()))('outputTo'))
-
-        multilineCommand.setParseAction(lambda x: x[0])
-        oneline_command.setParseAction(lambda x: x[0])
-
-        if blankLinesAllowed:
-            blankLineTerminationParser = pyparsing.NoMatch
-        else:
-            blankLineTerminator = (pyparsing.lineEnd + pyparsing.lineEnd)('terminator')
-            blankLineTerminator.setResultsName('terminator')
-            blankLineTerminationParser = ((multilineCommand ^ oneline_command) +
-                                          pyparsing.SkipTo(blankLineTerminator, ignore=do_not_parse).setParseAction(
-                                                   lambda x: x[0].strip())('args') + blankLineTerminator)('statement')
-
-        multilineParser = (((multilineCommand ^ oneline_command) +
-                            pyparsing.SkipTo(terminator_parser,
-                                             ignore=do_not_parse).setParseAction(lambda x: x[0].strip())('args') +
-                            terminator_parser)('statement') +
-                           pyparsing.SkipTo(output_destination_parser ^ pipe ^ string_end,
-                                            ignore=do_not_parse).setParseAction(lambda x: x[0].strip())('suffix') +
-                           after_elements)
-        multilineParser.ignore(commentInProgress)
-
-        singleLineParser = ((oneline_command +
-                             pyparsing.SkipTo(terminator_parser ^ string_end ^ pipe ^ output_destination_parser,
-                                              ignore=do_not_parse).setParseAction(
-                                 lambda x: x[0].strip())('args'))('statement') +
-                            pyparsing.Optional(terminator_parser) + after_elements)
-
-        blankLineTerminationParser = blankLineTerminationParser.setResultsName('statement')
-
-        parser = prefixParser + (
-            string_end |
-            multilineParser |
-            singleLineParser |
-            blankLineTerminationParser |
-            multilineCommand + pyparsing.SkipTo(string_end, ignore=do_not_parse)
-        )
-        parser.ignore(self.commentGrammars)
-        return parser
-
-    @staticmethod
-    def _build_input_source_parser(legalChars, commentInProgress):
-        """Builds a PyParsing parser for alternate user input sources (from file, pipe, etc.)"""
-
-        input_mark = pyparsing.Literal('<')
-        input_mark.setParseAction(lambda x: '')
-
-        # Also allow spaces, slashes, and quotes
-        file_name = pyparsing.Word(legalChars + ' /\\"\'')
-
-        input_from = file_name('inputFrom')
-        input_from.setParseAction(replace_with_file_contents)
-        # a not-entirely-satisfactory way of distinguishing < as in "import from" from <
-        # as in "lesser than"
-        inputParser = input_mark + pyparsing.Optional(input_from) + pyparsing.Optional('>') + \
-            pyparsing.Optional(file_name) + (pyparsing.stringEnd | '|')
-        inputParser.ignore(commentInProgress)
-        return inputParser
-
-    def parsed(self, raw):
-        """ This function is where the actual parsing of each line occurs.
-
-        :param raw: str - the line of text as it was entered
-        :return: ParsedString - custom subclass of str with extra attributes
-        """
-        if isinstance(raw, ParsedString):
-            p = raw
-        else:
-            # preparse is an overridable hook; default makes no changes
-            s = self.preparse(raw)
-            s = self.input_source_parser.transformString(s.lstrip())
-            s = self.commentGrammars.transformString(s)
-
-            # Make a copy of aliases so we can edit it
-            tmp_aliases = list(self.aliases.keys())
-            keep_expanding = len(tmp_aliases) > 0
-
-            # Expand aliases
-            while keep_expanding:
-                for cur_alias in tmp_aliases:
-                    keep_expanding = False
-
-                    if s == cur_alias or s.startswith(cur_alias + ' '):
-                        s = s.replace(cur_alias, self.aliases[cur_alias], 1)
-
-                        # Do not expand the same alias more than once
-                        tmp_aliases.remove(cur_alias)
-                        keep_expanding = len(tmp_aliases) > 0
-                        break
-
-            # Expand command shortcut to its full command name
-            for (shortcut, expansion) in self.shortcuts:
-                if s.startswith(shortcut):
-                    # If the next character after the shortcut isn't a space, then insert one
-                    shortcut_len = len(shortcut)
-                    if len(s) == shortcut_len or s[shortcut_len] != ' ':
-                        expansion += ' '
-
-                    # Expand the shortcut
-                    s = s.replace(shortcut, expansion, 1)
-                    break
-
-            try:
-                result = self.main_parser.parseString(s)
-            except pyparsing.ParseException:
-                # If we have a parsing failure, treat it is an empty command and move to next prompt
-                result = self.main_parser.parseString('')
-            result['raw'] = raw
-            result['command'] = result.multilineCommand or result.command
-            result = self.postparse(result)
-            p = ParsedString(result.args)
-            p.parsed = result
-            p.parser = self.parsed
-        return p
-
-
 class HistoryItem(str):
     """Class used to represent an item in the History list.
 
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index ffeb8bbe1..9204305ba 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -1,6 +1,6 @@
 #
 # -*- coding: utf-8 -*-
-"""Command parsing classes for cmd2"""
+"""Statement parsing classes for cmd2"""
 
 import re
 import shlex
@@ -32,7 +32,7 @@ def __init__(self, object):
         self.output = None
         self.outputTo = None
 
-class CommandParser():
+class StatementParser():
     """Parse raw text into command components.
     
     Shortcuts is a list of tuples with each tuple containing the shortcut and the expansion.
@@ -55,7 +55,7 @@ def __init__(
         self.aliases = aliases
         self.shortcuts = shortcuts
 
-    def parseString(self, rawinput):
+    def parse(self, rawinput):
         # strip C-style comments
         # shlex will handle the python/shell style comments for us
         def replacer(match):
diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py
index 27168af17..0a4dfbc64 100644
--- a/tests/test_cmd2.py
+++ b/tests/test_cmd2.py
@@ -114,6 +114,49 @@ def test_base_show_readonly(base_app):
     assert out == expected
 
 
+def test_cast():
+    cast = cmd2.cmd2.cast
+
+    # Boolean
+    assert cast(True, True) == True
+    assert cast(True, False) == False
+    assert cast(True, 0) == False
+    assert cast(True, 1) == True
+    assert cast(True, 'on') == True
+    assert cast(True, 'off') == False
+    assert cast(True, 'ON') == True
+    assert cast(True, 'OFF') == False
+    assert cast(True, 'y') == True
+    assert cast(True, 'n') == False
+    assert cast(True, 't') == True
+    assert cast(True, 'f') == False
+
+    # Non-boolean same type
+    assert cast(1, 5) == 5
+    assert cast(3.4, 2.7) == 2.7
+    assert cast('foo', 'bar') == 'bar'
+    assert cast([1,2], [3,4]) == [3,4]
+
+def test_cast_problems(capsys):
+    cast = cmd2.cmd2.cast
+
+    expected = 'Problem setting parameter (now {}) to {}; incorrect type?\n'
+
+    # Boolean current, with new value not convertible to bool
+    current = True
+    new = [True, True]
+    assert cast(current, new) == current
+    out, err = capsys.readouterr()
+    assert out == expected.format(current, new)
+
+    # Non-boolean current, with new value not convertible to current type
+    current = 1
+    new = 'octopus'
+    assert cast(current, new) == current
+    out, err = capsys.readouterr()
+    assert out == expected.format(current, new)
+
+
 def test_base_set(base_app):
     out = run_cmd(base_app, 'set quiet True')
     expected = normalize("""
@@ -217,6 +260,34 @@ def test_base_error(base_app):
     assert out == ["*** Unknown syntax: meow"]
 
 
+@pytest.fixture
+def hist():
+    from cmd2.cmd2 import HistoryItem
+    h = cmd2.cmd2.History([HistoryItem('first'), HistoryItem('second'), HistoryItem('third'), HistoryItem('fourth')])
+    return h
+
+def test_history_span(hist):
+    h = hist
+    assert h == ['first', 'second', 'third', 'fourth']
+    assert h.span('-2..') == ['third', 'fourth']
+    assert h.span('2..3') == ['second', 'third']    # Inclusive of end
+    assert h.span('3') == ['third']
+    assert h.span(':') == h
+    assert h.span('2..') == ['second', 'third', 'fourth']
+    assert h.span('-1') == ['fourth']
+    assert h.span('-2..-3') == ['third', 'second']
+    assert h.span('*') == h
+
+def test_history_get(hist):
+    h = hist
+    assert h == ['first', 'second', 'third', 'fourth']
+    assert h.get('') == h
+    assert h.get('-2') == h[:-2]
+    assert h.get('5') == []
+    assert h.get('2-3') == ['second']           # Exclusive of end
+    assert h.get('ir') == ['first', 'third']    # Normal string search for all elements containing "ir"
+    assert h.get('/i.*d/') == ['third']         # Regex string search "i", then anything, then "d"
+
 def test_base_history(base_app):
     run_cmd(base_app, 'help')
     run_cmd(base_app, 'shortcuts')
@@ -604,18 +675,6 @@ def test_allow_redirection(base_app):
     # Verify that no file got created
     assert not os.path.exists(filename)
 
-
-def test_input_redirection(base_app, request):
-    test_dir = os.path.dirname(request.module.__file__)
-    filename = os.path.join(test_dir, 'redirect.txt')
-
-    # NOTE: File 'redirect.txt" contains 1 word "history"
-
-    # Verify that redirecting input from a file works
-    out = run_cmd(base_app, 'help < {}'.format(filename))
-    assert out == normalize(HELP_HISTORY)
-
-
 def test_pipe_to_shell(base_app, capsys):
     if sys.platform == "win32":
         # Windows
@@ -965,7 +1024,7 @@ def test_default_to_shell_good(capsys):
         line = 'dir'
     else:
         line = 'ls'
-    statement = app.command_parser.parseString(line)
+    statement = app.statement_parser.parse(line)
     retval = app.default(statement)
     assert not retval
     out, err = capsys.readouterr()
@@ -975,7 +1034,7 @@ def test_default_to_shell_failure(capsys):
     app = cmd2.Cmd()
     app.default_to_shell = True
     line = 'ls does_not_exist.xyz'
-    statement = app.command_parser.parseString(line)
+    statement = app.statement_parser.parse(line)
     retval = app.default(statement)
     assert not retval
     out, err = capsys.readouterr()
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 5825e735d..6fa7d6292 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -1,90 +1,284 @@
 # coding=utf-8
 """
-Unit/functional testing for helper functions/classes in the cmd2.py module.
-
-These are primarily tests related to parsing.  Moreover, they are mostly a port of the old doctest tests which were
-problematic because they worked properly for some versions of pyparsing but not for others.
+Test the parsing logic in parsing.py
 
 Copyright 2017 Todd Leonhardt <todd.leonhardt@gmail.com>
 Released under MIT license, see LICENSE file
 """
-import sys
-
 import cmd2
+from cmd2.parsing import StatementParser
+
 import pytest
 
 
 @pytest.fixture
-def hist():
-    from cmd2.cmd2 import HistoryItem
-    h = cmd2.cmd2.History([HistoryItem('first'), HistoryItem('second'), HistoryItem('third'), HistoryItem('fourth')])
-    return h
-
-def test_history_span(hist):
-    h = hist
-    assert h == ['first', 'second', 'third', 'fourth']
-    assert h.span('-2..') == ['third', 'fourth']
-    assert h.span('2..3') == ['second', 'third']    # Inclusive of end
-    assert h.span('3') == ['third']
-    assert h.span(':') == h
-    assert h.span('2..') == ['second', 'third', 'fourth']
-    assert h.span('-1') == ['fourth']
-    assert h.span('-2..-3') == ['third', 'second']
-    assert h.span('*') == h
-
-def test_history_get(hist):
-    h = hist
-    assert h == ['first', 'second', 'third', 'fourth']
-    assert h.get('') == h
-    assert h.get('-2') == h[:-2]
-    assert h.get('5') == []
-    assert h.get('2-3') == ['second']           # Exclusive of end
-    assert h.get('ir') == ['first', 'third']    # Normal string search for all elements containing "ir"
-    assert h.get('/i.*d/') == ['third']         # Regex string search "i", then anything, then "d"
-
-
-def test_cast():
-    cast = cmd2.cmd2.cast
-
-    # Boolean
-    assert cast(True, True) == True
-    assert cast(True, False) == False
-    assert cast(True, 0) == False
-    assert cast(True, 1) == True
-    assert cast(True, 'on') == True
-    assert cast(True, 'off') == False
-    assert cast(True, 'ON') == True
-    assert cast(True, 'OFF') == False
-    assert cast(True, 'y') == True
-    assert cast(True, 'n') == False
-    assert cast(True, 't') == True
-    assert cast(True, 'f') == False
-
-    # Non-boolean same type
-    assert cast(1, 5) == 5
-    assert cast(3.4, 2.7) == 2.7
-    assert cast('foo', 'bar') == 'bar'
-    assert cast([1,2], [3,4]) == [3,4]
-
-
-def test_cast_problems(capsys):
-    cast = cmd2.cmd2.cast
-
-    expected = 'Problem setting parameter (now {}) to {}; incorrect type?\n'
-
-    # Boolean current, with new value not convertible to bool
-    current = True
-    new = [True, True]
-    assert cast(current, new) == current
-    out, err = capsys.readouterr()
-    assert out == expected.format(current, new)
-
-    # Non-boolean current, with new value not convertible to current type
-    current = 1
-    new = 'octopus'
-    assert cast(current, new) == current
-    out, err = capsys.readouterr()
-    assert out == expected.format(current, new)
+def parser():
+    parser = StatementParser(
+        quotes=['"', "'"],
+        allow_redirection=True,
+        redirection_chars=['|', '<', '>'],        
+        terminators = [';'],
+        multilineCommands = ['multiline'],
+        aliases = {'helpalias': 'help', '42': 'theanswer'},
+        shortcuts = [('?', 'help'), ('!', 'shell')]
+    )
+    return parser
+
+def test_parse_empty_string(parser):
+    statement = parser.parse('')
+    assert not statement.command
+
+@pytest.mark.parametrize('tokens,command,args', [
+    ( [], None, ''),
+    ( ['command'], 'command', '' ),
+    ( ['command', 'arg1', 'arg2'], 'command', 'arg1 arg2')
+])
+def test_command_and_args(parser, tokens, command, args):
+    (parsed_command, parsed_args) = parser._command_and_args(tokens)
+    assert command == parsed_command
+    assert args == parsed_args
+
+@pytest.mark.parametrize('line', [
+    'plainword',
+    '"one word"',
+    "'one word'",
+])
+def test_single_word(parser, line):
+    statement = parser.parse(line)
+    assert statement.command == line
+
+def test_word_plus_terminator(parser):
+    line = 'termbare;'
+    statement = parser.parse(line)
+    assert statement.command == 'termbare'
+    assert statement.terminator == ';'
+
+def test_suffix_after_terminator(parser):
+    line = 'termbare; suffx'
+    statement = parser.parse(line)
+    assert statement.command == 'termbare'
+    assert statement.terminator == ';'
+    assert statement.suffix == 'suffx'
+
+def test_command_with_args(parser):
+    line = 'command with args'
+    statement = parser.parse(line)
+    assert statement.command == 'command'
+    assert statement.args == 'with args'
+    assert not statement.pipeTo
+
+def test_parse_command_with_args_terminator_and_suffix(parser):
+    line = 'command with args and terminator; and suffix'
+    statement = parser.parse(line)
+    assert statement.command == 'command'
+    assert statement.args == "with args and terminator"
+    assert statement.terminator == ';'
+    assert statement.suffix == 'and suffix'
+
+def test_hashcomment(parser):
+    statement = parser.parse('hi # this is all a comment')
+    assert statement.command == 'hi'
+    assert not statement.args
+    assert not statement.pipeTo
+
+def test_c_comment(parser):
+    statement = parser.parse('hi /* this is | all a comment */')
+    assert statement.command == 'hi'
+    assert not statement.args
+    assert not statement.pipeTo
+
+def test_c_comment_empty(parser):
+    statement = parser.parse('/* this is | all a comment */')
+    assert not statement.command
+    assert not statement.args
+    assert not statement.pipeTo
+
+def test_parse_what_if_quoted_strings_seem_to_start_comments(parser):
+    statement = parser.parse('what if "quoted strings /* seem to " start comments?')
+    assert statement.command == 'what'
+    assert statement.args == 'if "quoted strings /* seem to " start comments?'
+    assert not statement.pipeTo
+
+def test_simple_piped(parser):
+    statement = parser.parse('simple | piped')
+    assert statement.command == 'simple'
+    assert not statement.args
+    assert statement.pipeTo == 'piped'
+
+def test_double_pipe_is_not_a_pipe(parser):
+    line = 'double-pipe || is not a pipe'
+    statement = parser.parse(line)
+    assert statement.command == 'double-pipe'
+    assert statement.args == '|| is not a pipe'
+    assert not statement.pipeTo
+
+def test_complex_pipe(parser):
+    line = 'command with args, terminator;sufx | piped'
+    statement = parser.parse(line)
+    assert statement.command == 'command'
+    assert statement.args == "with args, terminator"
+    assert statement.terminator == ';'
+    assert statement.suffix == 'sufx'
+    assert statement.pipeTo == 'piped'
+
+def test_output_redirect(parser):
+    line = 'output into > afile.txt'
+    statement = parser.parse(line)
+    assert statement.command == 'output'
+    assert statement.args == 'into'
+    assert statement.output == '>'
+    assert statement.outputTo == 'afile.txt'
+
+def test_output_redirect_with_dash_in_path(parser):
+    line = 'output into > python-cmd2/afile.txt'
+    statement = parser.parse(line)
+    assert statement.command == 'output'
+    assert statement.args == 'into'
+    assert statement.output == '>'
+    assert statement.outputTo == 'python-cmd2/afile.txt'
+
+def test_output_redirect_append(parser):
+    line = 'output appended to >> /tmp/afile.txt'
+    statement = parser.parse(line)
+    assert statement.command == 'output'
+    assert statement.args == 'appended to'
+    assert statement.output == '>>'
+    assert statement.outputTo == '/tmp/afile.txt'
+
+def test_parse_input_redirect(parser):
+    line = '< afile.txt'
+    statement = parser.parse(line)
+    assert statement.inputFrom == 'afile.txt'
+
+def test_parse_input_redirect_after_command(parser):
+    line = 'help < afile.txt'
+    statement = parser.parse(line)
+    assert statement.command == 'help'
+    assert statement.args == ''
+    assert statement.inputFrom == 'afile.txt'
+
+def test_parse_input_redirect_with_dash_in_path(parser):
+    line = '< python-cmd2/afile.txt'
+    statement = parser.parse(line)
+    assert statement.inputFrom == 'python-cmd2/afile.txt'
+
+def test_pipe_and_redirect(parser):
+    line = 'output into;sufx | pipethrume plz > afile.txt'
+    statement = parser.parse(line)
+    assert statement.command == 'output'
+    assert statement.args == 'into'
+    assert statement.terminator == ';'
+    assert statement.suffix == 'sufx'
+    assert statement.pipeTo == 'pipethrume plz'
+    assert statement.output == '>'
+    assert statement.outputTo == 'afile.txt'
+
+def test_parse_output_to_paste_buffer(parser):
+    line = 'output to paste buffer >> '
+    statement = parser.parse(line)
+    assert statement.command == 'output'
+    assert statement.args == 'to paste buffer'
+    assert statement.output == '>>'
+
+def test_has_redirect_inside_terminator(parser):
+    """The terminator designates the end of the commmand/arguments portion.  If a redirector
+    occurs before a terminator, then it will be treated as part of the arguments and not as a redirector."""
+    line = 'has > inside;'
+    statement = parser.parse(line)
+    assert statement.command == 'has'
+    assert statement.args == '> inside'
+    assert statement.terminator == ';'
+
+def test_parse_unfinished_multiliine_command(parser):
+    line = 'multiline has > inside an unfinished command'
+    statement = parser.parse(line)
+    assert statement.multilineCommand == 'multiline'
+    assert not statement.args
+    assert not statement.terminator
+    
+
+def test_parse_multiline_command_ignores_redirectors_within_it(parser):
+    line = 'multiline has > inside;'
+    statement = parser.parse(line)
+    assert statement.multilineCommand == 'multiline'
+    assert statement.args == 'has > inside'
+    assert statement.terminator == ';'
+
+# def test_parse_multiline_with_incomplete_comment(parser):
+#     """A terminator within a comment will be ignored and won't terminate a multiline command.
+#     Un-closed comments effectively comment out everything after the start."""
+#     line = 'multiline command /* with comment in progress;'
+#     statement = parser.parse(line)
+#     assert statement.multilineCommand == 'multiline'
+#     assert statement.args == 'command'
+#     assert not statement.terminator
+
+def test_parse_multiline_with_complete_comment(parser):
+    line = 'multiline command /* with comment complete */ is done;'
+    statement = parser.parse(line)
+    assert statement.multilineCommand == 'multiline'
+    assert statement.args == 'command is done'
+    assert statement.terminator == ';'
+
+def test_parse_multiline_termninated_by_empty_line(parser):
+    line = 'multiline command ends\n\n'
+    statement = parser.parse(line)
+    assert statement.multilineCommand == 'multiline'
+    assert statement.args == 'command ends'
+    assert statement.terminator == '\n'
+
+def test_parse_multiline_ignores_terminators_in_comments(parser):
+    line = 'multiline command "with term; ends" now\n\n'
+    statement = parser.parse(line)
+    assert statement.multilineCommand == 'multiline'
+    assert statement.args == 'command "with term; ends" now'
+    assert statement.terminator == '\n'
+
+def test_parse_command_with_unicode_args(parser):
+    line = 'drink café'
+    statement = parser.parse(line)
+    assert statement.command == 'drink'
+    assert statement.args == 'café'
+
+def test_parse_unicode_command(parser):
+    line = 'café au lait'
+    statement = parser.parse(line)
+    assert statement.command == 'café'
+    assert statement.args == 'au lait'
+
+def test_parse_redirect_to_unicode_filename(parser):
+    line = 'dir home > café'
+    statement = parser.parse(line)
+    assert statement.command == 'dir'
+    assert statement.args == 'home'
+    assert statement.output == '>'
+    assert statement.outputTo == 'café'
+
+def test_parse_input_redirect_from_unicode_filename(parser):
+    line = '< café'
+    statement = parser.parse(line)
+    assert statement.inputFrom == 'café'
+
+def test_empty_statement_raises_exception():
+    app = cmd2.Cmd()
+    with pytest.raises(cmd2.cmd2.EmptyStatement):
+        app._complete_statement('')
+
+    with pytest.raises(cmd2.cmd2.EmptyStatement):
+        app._complete_statement(' ')
+
+@pytest.mark.parametrize('line,command,args', [
+    ('helpalias', 'help', ''),
+    ('helpalias mycommand', 'help', 'mycommand'),
+    ('42', 'theanswer', ''),
+    ('42 arg1 arg2', 'theanswer', 'arg1 arg2'),
+    ('!ls', 'shell', 'ls'),
+    ('!ls -al /tmp', 'shell', 'ls -al /tmp'),
+])
+def test_alias_and_shortcut_expansion(parser, line, command, args):
+    statement = parser.parse(line)
+    assert statement.command == command
+    assert statement.args == args
 
 def test_empty_statement_raises_exception():
     app = cmd2.Cmd()
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 9c2c8f9ba..2ff796413 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -3,13 +3,13 @@
 Unit/functional testing for ply based parsing in cmd2
 
 Todo List
-- implement input redirection
 - case sensitive flag
 - checkout Cmd2.parseline() function which parses and expands shortcuts and such
   this code should probably be included in CommandParser
 - get rid of legalChars
 - move remaining tests in test_parsing.py to test_cmd2.py
 - rename test_shlexparsing.py to test_parsing.py
+- look at parsed() - expands aliases and shortcuts, see if it can be refactored
 
 Notes:
 
@@ -18,287 +18,11 @@
     - Python/Shell style -> # comment
 - we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
 
-Functions in cmd2.py to be modified:
-- parsed() - expands aliases and shortcuts
 
 Changelog Items:
 - if self.default_to_shell is true, then redirection and piping is now properly passed to the shell, previously it was truncated
-- object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything.
+- object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything. The parser is now self.statement_parser instead of self.command_parser.
+- input redirection no longer supported. Use the load command instead.
+- submenus now call all hooks, it used to just call precmd and postcmd
 
-Bugs fixed:
-- submenus now all all hooks, it used to just call precmd and postcmd
 """
-
-import cmd2
-from cmd2.parsing import CommandParser
-
-import pytest
-
-@pytest.fixture
-def parser():
-    parser = CommandParser(
-        quotes=['"', "'"],
-        allow_redirection=True,
-        redirection_chars=['|', '<', '>'],        
-        terminators = [';'],
-        multilineCommands = ['multiline'],
-        aliases = {'helpalias': 'help', '42': 'theanswer'},
-        shortcuts = [('?', 'help'), ('!', 'shell')]
-    )
-    return parser
-
-def test_parse_empty_string(parser):
-    results = parser.parseString('')
-    assert not results.command
-
-@pytest.mark.parametrize('tokens,command,args', [
-    ( [], None, ''),
-    ( ['command'], 'command', '' ),
-    ( ['command', 'arg1', 'arg2'], 'command', 'arg1 arg2')
-])
-def test_command_and_args(parser, tokens, command, args):
-    (parsed_command, parsed_args) = parser._command_and_args(tokens)
-    assert command == parsed_command
-    assert args == parsed_args
-
-@pytest.mark.parametrize('line', [
-    'plainword',
-    '"one word"',
-    "'one word'",
-])
-def test_single_word(parser, line):
-    results = parser.parseString(line)
-    assert results.command == line
-
-def test_word_plus_terminator(parser):
-    line = 'termbare;'
-    results = parser.parseString(line)
-    assert results.command == 'termbare'
-    assert results.terminator == ';'
-
-def test_suffix_after_terminator(parser):
-    line = 'termbare; suffx'
-    results = parser.parseString(line)
-    assert results.command == 'termbare'
-    assert results.terminator == ';'
-    assert results.suffix == 'suffx'
-
-def test_command_with_args(parser):
-    line = 'command with args'
-    results = parser.parseString(line)
-    assert results.command == 'command'
-    assert results.args == 'with args'
-    assert not results.pipeTo
-
-def test_parse_command_with_args_terminator_and_suffix(parser):
-    line = 'command with args and terminator; and suffix'
-    results = parser.parseString(line)
-    assert results.command == 'command'
-    assert results.args == "with args and terminator"
-    assert results.terminator == ';'
-    assert results.suffix == 'and suffix'
-
-def test_hashcomment(parser):
-    results = parser.parseString('hi # this is all a comment')
-    assert results.command == 'hi'
-    assert not results.args
-    assert not results.pipeTo
-
-def test_c_comment(parser):
-    results = parser.parseString('hi /* this is | all a comment */')
-    assert results.command == 'hi'
-    assert not results.args
-    assert not results.pipeTo
-
-def test_c_comment_empty(parser):
-    results = parser.parseString('/* this is | all a comment */')
-    assert not results.command
-    assert not results.args
-    assert not results.pipeTo
-
-def test_parse_what_if_quoted_strings_seem_to_start_comments(parser):
-    results = parser.parseString('what if "quoted strings /* seem to " start comments?')
-    assert results.command == 'what'
-    assert results.args == 'if "quoted strings /* seem to " start comments?'
-    assert not results.pipeTo
-
-def test_simple_piped(parser):
-    results = parser.parseString('simple | piped')
-    assert results.command == 'simple'
-    assert not results.args
-    assert results.pipeTo == 'piped'
-
-def test_double_pipe_is_not_a_pipe(parser):
-    line = 'double-pipe || is not a pipe'
-    results = parser.parseString(line)
-    assert results.command == 'double-pipe'
-    assert results.args == '|| is not a pipe'
-    assert not results.pipeTo
-
-def test_complex_pipe(parser):
-    line = 'command with args, terminator;sufx | piped'
-    results = parser.parseString(line)
-    assert results.command == 'command'
-    assert results.args == "with args, terminator"
-    assert results.terminator == ';'
-    assert results.suffix == 'sufx'
-    assert results.pipeTo == 'piped'
-
-def test_output_redirect(parser):
-    line = 'output into > afile.txt'
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'into'
-    assert results.output == '>'
-    assert results.outputTo == 'afile.txt'
-
-def test_output_redirect_with_dash_in_path(parser):
-    line = 'output into > python-cmd2/afile.txt'
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'into'
-    assert results.output == '>'
-    assert results.outputTo == 'python-cmd2/afile.txt'
-
-def test_output_redirect_append(parser):
-    line = 'output appended to >> /tmp/afile.txt'
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'appended to'
-    assert results.output == '>>'
-    assert results.outputTo == '/tmp/afile.txt'
-
-def test_parse_input_redirect(parser):
-    line = '< afile.txt'
-    results = parser.parseString(line)
-    assert results.inputFrom == 'afile.txt'
-
-def test_parse_input_redirect_after_command(parser):
-    line = 'help < afile.txt'
-    results = parser.parseString(line)
-    assert results.command == 'help'
-    assert results.args == ''
-    assert results.inputFrom == 'afile.txt'
-
-def test_parse_input_redirect_with_dash_in_path(parser):
-    line = '< python-cmd2/afile.txt'
-    results = parser.parseString(line)
-    assert results.inputFrom == 'python-cmd2/afile.txt'
-
-def test_pipe_and_redirect(parser):
-    line = 'output into;sufx | pipethrume plz > afile.txt'
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'into'
-    assert results.terminator == ';'
-    assert results.suffix == 'sufx'
-    assert results.pipeTo == 'pipethrume plz'
-    assert results.output == '>'
-    assert results.outputTo == 'afile.txt'
-
-def test_parse_output_to_paste_buffer(parser):
-    line = 'output to paste buffer >> '
-    results = parser.parseString(line)
-    assert results.command == 'output'
-    assert results.args == 'to paste buffer'
-    assert results.output == '>>'
-
-def test_has_redirect_inside_terminator(parser):
-    """The terminator designates the end of the commmand/arguments portion.  If a redirector
-    occurs before a terminator, then it will be treated as part of the arguments and not as a redirector."""
-    line = 'has > inside;'
-    results = parser.parseString(line)
-    assert results.command == 'has'
-    assert results.args == '> inside'
-    assert results.terminator == ';'
-
-def test_parse_unfinished_multiliine_command(parser):
-    line = 'multiline has > inside an unfinished command'
-    statement = parser.parseString(line)
-    assert statement.multilineCommand == 'multiline'
-    assert not statement.args
-    assert not statement.terminator
-    
-
-def test_parse_multiline_command_ignores_redirectors_within_it(parser):
-    line = 'multiline has > inside;'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'has > inside'
-    assert results.terminator == ';'
-
-# def test_parse_multiline_with_incomplete_comment(parser):
-#     """A terminator within a comment will be ignored and won't terminate a multiline command.
-#     Un-closed comments effectively comment out everything after the start."""
-#     line = 'multiline command /* with comment in progress;'
-#     statement = parser.parseString(line)
-#     assert statement.multilineCommand == 'multiline'
-#     assert statement.args == 'command'
-#     assert not statement.terminator
-
-def test_parse_multiline_with_complete_comment(parser):
-    line = 'multiline command /* with comment complete */ is done;'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'command is done'
-    assert results.terminator == ';'
-
-def test_parse_multiline_termninated_by_empty_line(parser):
-    line = 'multiline command ends\n\n'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'command ends'
-    assert results.terminator == '\n'
-
-def test_parse_multiline_ignores_terminators_in_comments(parser):
-    line = 'multiline command "with term; ends" now\n\n'
-    results = parser.parseString(line)
-    assert results.multilineCommand == 'multiline'
-    assert results.args == 'command "with term; ends" now'
-    assert results.terminator == '\n'
-
-def test_parse_command_with_unicode_args(parser):
-    line = 'drink café'
-    results = parser.parseString(line)
-    assert results.command == 'drink'
-    assert results.args == 'café'
-
-def test_parse_unicode_command(parser):
-    line = 'café au lait'
-    results = parser.parseString(line)
-    assert results.command == 'café'
-    assert results.args == 'au lait'
-
-def test_parse_redirect_to_unicode_filename(parser):
-    line = 'dir home > café'
-    results = parser.parseString(line)
-    assert results.command == 'dir'
-    assert results.args == 'home'
-    assert results.output == '>'
-    assert results.outputTo == 'café'
-
-def test_parse_input_redirect_from_unicode_filename(parser):
-    line = '< café'
-    results = parser.parseString(line)
-    assert results.inputFrom == 'café'
-
-def test_empty_statement_raises_exception():
-    app = cmd2.Cmd()
-    with pytest.raises(cmd2.cmd2.EmptyStatement):
-        app._complete_statement('')
-
-    with pytest.raises(cmd2.cmd2.EmptyStatement):
-        app._complete_statement(' ')
-
-@pytest.mark.parametrize('line,command,args', [
-    ('helpalias', 'help', ''),
-    ('helpalias mycommand', 'help', 'mycommand'),
-    ('42', 'theanswer', ''),
-    ('42 arg1 arg2', 'theanswer', 'arg1 arg2'),
-    ('!ls', 'shell', 'ls'),
-    ('!ls -al /tmp', 'shell', 'ls -al /tmp'),
-])
-def test_alias_and_shortcut_expansion(parser, line, command, args):
-    statement = parser.parseString(line)
-    assert statement.command == command
-    assert statement.args == args

From 9170550b2a32a763dafc1d7b14b43d97c542a0db Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Tue, 24 Apr 2018 22:40:29 -0600
Subject: [PATCH 44/89] Add test for alias expansion on incomplete multiline
 commands

---
 tests/test_parsing.py      | 36 ++++++++++++++++++------------------
 tests/test_shlexparsing.py |  7 +++++++
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 6fa7d6292..1913938eb 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -19,7 +19,7 @@ def parser():
         redirection_chars=['|', '<', '>'],        
         terminators = [';'],
         multilineCommands = ['multiline'],
-        aliases = {'helpalias': 'help', '42': 'theanswer'},
+        aliases = {'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline'},
         shortcuts = [('?', 'help'), ('!', 'shell')]
     )
     return parser
@@ -193,9 +193,9 @@ def test_parse_unfinished_multiliine_command(parser):
     line = 'multiline has > inside an unfinished command'
     statement = parser.parse(line)
     assert statement.multilineCommand == 'multiline'
-    assert not statement.args
+    assert statement.command == 'multiline'
+    assert statement.args == 'has > inside an unfinished command'
     assert not statement.terminator
-    
 
 def test_parse_multiline_command_ignores_redirectors_within_it(parser):
     line = 'multiline has > inside;'
@@ -204,14 +204,14 @@ def test_parse_multiline_command_ignores_redirectors_within_it(parser):
     assert statement.args == 'has > inside'
     assert statement.terminator == ';'
 
-# def test_parse_multiline_with_incomplete_comment(parser):
-#     """A terminator within a comment will be ignored and won't terminate a multiline command.
-#     Un-closed comments effectively comment out everything after the start."""
-#     line = 'multiline command /* with comment in progress;'
-#     statement = parser.parse(line)
-#     assert statement.multilineCommand == 'multiline'
-#     assert statement.args == 'command'
-#     assert not statement.terminator
+def test_parse_multiline_with_incomplete_comment(parser):
+    """A terminator within a comment will be ignored and won't terminate a multiline command.
+    Un-closed comments effectively comment out everything after the start."""
+    line = 'multiline command /* with comment in progress;'
+    statement = parser.parse(line)
+    assert statement.multilineCommand == 'multiline'
+    assert statement.args == 'command'
+    assert not statement.terminator
 
 def test_parse_multiline_with_complete_comment(parser):
     line = 'multiline command /* with comment complete */ is done;'
@@ -280,10 +280,10 @@ def test_alias_and_shortcut_expansion(parser, line, command, args):
     assert statement.command == command
     assert statement.args == args
 
-def test_empty_statement_raises_exception():
-    app = cmd2.Cmd()
-    with pytest.raises(cmd2.cmd2.EmptyStatement):
-        app._complete_statement('')
-
-    with pytest.raises(cmd2.cmd2.EmptyStatement):
-        app._complete_statement(' ')
+def test_alias_on_multiline_command(parser):
+    line = 'anothermultiline has > inside an unfinished command'
+    statement = parser.parse(line)
+    assert statement.multilineCommand == 'multiline'
+    assert statement.command == 'multiline'
+    assert statement.args == 'has > inside an unfinished command'
+    assert not statement.terminator
diff --git a/tests/test_shlexparsing.py b/tests/test_shlexparsing.py
index 2ff796413..c9f2e85a6 100644
--- a/tests/test_shlexparsing.py
+++ b/tests/test_shlexparsing.py
@@ -11,6 +11,13 @@
 - rename test_shlexparsing.py to test_parsing.py
 - look at parsed() - expands aliases and shortcuts, see if it can be refactored
 
+Questions:
+- say I have a command called 'fred' which is a multiline command. If I make an alias
+  for fred called 'george' is george a multiline command? I think the answer is yes.
+  If you want a multi-line synonym for a command that isn't multiline, do it like
+  example.py does. If the answer is no, then I need to rework StatementParser.parse()
+- 
+
 Notes:
 
 - valid comment styles:

From 1862ac84672755e1ae0c7adf4bc2933c29021af8 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Tue, 24 Apr 2018 22:40:41 -0600
Subject: [PATCH 45/89] Add type hinting

---
 cmd2/parsing.py | 71 ++++++++++++++++++++++++++++++-------------------
 1 file changed, 43 insertions(+), 28 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 9204305ba..9030a5f88 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -4,6 +4,7 @@
 
 import re
 import shlex
+from typing import List, Tuple
 
 import cmd2
 
@@ -55,7 +56,7 @@ def __init__(
         self.aliases = aliases
         self.shortcuts = shortcuts
 
-    def parse(self, rawinput):
+    def parse(self, rawinput: str) -> Statement:
         # strip C-style comments
         # shlex will handle the python/shell style comments for us
         def replacer(match):
@@ -67,12 +68,15 @@ def replacer(match):
                     return s
         pattern = re.compile(
             #r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
-            r'/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+            r'/\*.*?(\*/|$)|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
             re.DOTALL | re.MULTILINE
         )
         rawinput = re.sub(pattern, replacer, rawinput)
         line = rawinput
 
+        command = None
+        args = ''
+
         # expand shortcuts, have to do this first because
         # a shortcut can expand into multiple tokens, ie '!ls' becomes
         # 'shell ls'
@@ -97,7 +101,11 @@ def replacer(match):
         s = shlex.shlex(line, posix=False)
         s.whitespace_split = True
         tokens = self.split_on_punctuation(list(s))
-        
+
+        if tokens:
+            command_to_expand = tokens[0]
+            tokens[0] = self.expand_aliases(command_to_expand)
+
         # of the valid terminators, find the first one to occur in the input
         terminator_pos = len(tokens)+1
         for test_terminator in self.terminators:
@@ -118,10 +126,18 @@ def replacer(match):
                 terminator_pos = tokens.index(terminator)
             # everything before the first terminator is the command and the args
             (command, args) = self._command_and_args(tokens[:terminator_pos])
-            #terminator = tokens[terminator_pos]
             # we will set the suffix later
             # remove all the tokens before and including the terminator
             tokens = tokens[terminator_pos+1:]
+        else:
+            (testcommand, testargs) = self._command_and_args(tokens)
+            if testcommand in self.multilineCommands:
+                # no terminator on this line but we have a multiline command
+                # everything else on the line is part of the args
+                # because redirectors can only be after a terminator
+                command = testcommand
+                args = testargs
+                tokens = []
 
         # check for input from file
         inputFrom = None
@@ -132,7 +148,6 @@ def replacer(match):
         except ValueError:
             pass
 
-
         # check for output redirect
         output = None
         outputTo = None
@@ -172,29 +187,13 @@ def replacer(match):
         else:
             # no terminator, so whatever is left is the command and the args
             suffix = None
-            (command, args) = self._command_and_args(tokens)
-
-        # expand aliases
-        # make a copy of aliases so we can edit it
-        tmp_aliases = list(self.aliases.keys())
-        keep_expanding = len(tmp_aliases) > 0
-
-        while keep_expanding:
-            for cur_alias in tmp_aliases:
-                keep_expanding = False
-                if command == cur_alias:
-                    command = self.aliases[cur_alias]
-                    tmp_aliases.remove(cur_alias)
-                    keep_expanding = len(tmp_aliases) > 0
-                    break
+            if not command:
+                # command could already have been set, if so, don't set it again
+                (command, args) = self._command_and_args(tokens)
 
         # set multiline
         if command in self.multilineCommands:
             multilineCommand = command
-            # return no arguments if this is a "partial" command,
-            # i.e. we have a multiline command but no terminator yet
-            if not terminator:
-                args = ''
         else:
             multilineCommand = None
 
@@ -211,8 +210,24 @@ def replacer(match):
         result.suffix = suffix
         result.multilineCommand = multilineCommand
         return result
-    
-    def _command_and_args(self, tokens):
+
+    def expand_aliases(self, command: str) -> str:
+        """Given a command, expand any aliases for the command"""
+        # make a copy of aliases so we can edit it
+        tmp_aliases = list(self.aliases.keys())
+        keep_expanding = len(tmp_aliases) > 0
+
+        while keep_expanding:
+            for cur_alias in tmp_aliases:
+                keep_expanding = False
+                if command == cur_alias:
+                    command = self.aliases[cur_alias]
+                    tmp_aliases.remove(cur_alias)
+                    keep_expanding = len(tmp_aliases) > 0
+                    break
+        return command
+
+    def _command_and_args(self, tokens: List[str]) -> Tuple[str, str]:
         """given a list of tokens, and return a tuple of the command
         and the args as a string.
         """
@@ -227,7 +242,7 @@ def _command_and_args(self, tokens):
 
         return (command, args)
 
-    def split_on_punctuation(self, initial_tokens):
+    def split_on_punctuation(self, tokens: List[str]) -> List[str]:
         """
         # Further splits tokens from a command line using punctuation characters
         # as word breaks when they are in unquoted strings. Each run of punctuation
@@ -243,7 +258,7 @@ def split_on_punctuation(self, initial_tokens):
 
         punctuated_tokens = []
 
-        for cur_initial_token in initial_tokens:
+        for cur_initial_token in tokens:
 
             # Save tokens up to 1 character in length or quoted tokens. No need to parse these.
             if len(cur_initial_token) <= 1 or cur_initial_token[0] in self.quotes:

From be9a01013b78d15ac9dfd69b88f2f9d8047ad252 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Tue, 24 Apr 2018 22:49:22 -0600
Subject: [PATCH 46/89] Cleanup todo list

---
 tests/test_shlexparsing.py => SHLEX_TODO.txt | 25 +++++---------------
 1 file changed, 6 insertions(+), 19 deletions(-)
 rename tests/test_shlexparsing.py => SHLEX_TODO.txt (69%)

diff --git a/tests/test_shlexparsing.py b/SHLEX_TODO.txt
similarity index 69%
rename from tests/test_shlexparsing.py
rename to SHLEX_TODO.txt
index c9f2e85a6..371a6a714 100644
--- a/tests/test_shlexparsing.py
+++ b/SHLEX_TODO.txt
@@ -1,30 +1,18 @@
-# coding=utf-8
-"""
-Unit/functional testing for ply based parsing in cmd2
 
-Todo List
+Notes on conversion from pyparsing to shlex taking place in the ply branch
+
+Todo List:
 - case sensitive flag
-- checkout Cmd2.parseline() function which parses and expands shortcuts and such
-  this code should probably be included in CommandParser
+- refactor Cmd2.parseline() to use StatementParser.parse()
 - get rid of legalChars
-- move remaining tests in test_parsing.py to test_cmd2.py
-- rename test_shlexparsing.py to test_parsing.py
-- look at parsed() - expands aliases and shortcuts, see if it can be refactored
+- delete test_shlexparsing.py once I have all this data captured elsewhere
+- we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
 
 Questions:
 - say I have a command called 'fred' which is a multiline command. If I make an alias
   for fred called 'george' is george a multiline command? I think the answer is yes.
   If you want a multi-line synonym for a command that isn't multiline, do it like
   example.py does. If the answer is no, then I need to rework StatementParser.parse()
-- 
-
-Notes:
-
-- valid comment styles:
-    - C-style -> /* comment */
-    - Python/Shell style -> # comment
-- we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
-
 
 Changelog Items:
 - if self.default_to_shell is true, then redirection and piping is now properly passed to the shell, previously it was truncated
@@ -32,4 +20,3 @@
 - input redirection no longer supported. Use the load command instead.
 - submenus now call all hooks, it used to just call precmd and postcmd
 
-"""

From 08c5121cb314b45859c1e857e9cde2485a9de4fa Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Tue, 24 Apr 2018 22:53:17 -0600
Subject: [PATCH 47/89] Add todo for #327

---
 SHLEX_TODO.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index 371a6a714..774cbc9d7 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -5,8 +5,9 @@ Todo List:
 - case sensitive flag
 - refactor Cmd2.parseline() to use StatementParser.parse()
 - get rid of legalChars
-- delete test_shlexparsing.py once I have all this data captured elsewhere
+- get rid of POSIX_SHLEX and STRIP_QUOTES_FOR_NON_POSIX - those options should be ignored
 - we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
+- delete SHLEX_TODO.txt once everything is done
 
 Questions:
 - say I have a command called 'fred' which is a multiline command. If I make an alias

From b20a5166d0fe931e61bfec6beec4bb4aa7c131a4 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 25 Apr 2018 14:41:06 -0600
Subject: [PATCH 48/89] Remove references to legalChars

---
 SHLEX_TODO.txt | 1 -
 cmd2/cmd2.py   | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index 371a6a714..9b75328f8 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -4,7 +4,6 @@ Notes on conversion from pyparsing to shlex taking place in the ply branch
 Todo List:
 - case sensitive flag
 - refactor Cmd2.parseline() to use StatementParser.parse()
-- get rid of legalChars
 - delete test_shlexparsing.py once I have all this data captured elsewhere
 - we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
 
diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 41f46d5c9..17a6917cf 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -635,7 +635,7 @@ class Cmd(cmd.Cmd):
     redirector = '>'        # for sending output to file
     shortcuts = {'?': 'help', '!': 'shell', '@': 'load', '@@': '_relative_load'}
     aliases = dict()
-    terminators = [';']     # make sure your terminators are not in legalChars!
+    terminators = [';']
 
     # Attributes which are NOT dynamically settable at runtime
     allow_cli_args = True       # Should arguments passed on the command-line be processed as commands?

From a3ec4f1c51d091464ae0940368f2bdb906b287b7 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 25 Apr 2018 17:20:28 -0600
Subject: [PATCH 49/89] Update todo

---
 SHLEX_TODO.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index 9b75328f8..e57c03a94 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -4,7 +4,8 @@ Notes on conversion from pyparsing to shlex taking place in the ply branch
 Todo List:
 - case sensitive flag
 - refactor Cmd2.parseline() to use StatementParser.parse()
-- delete test_shlexparsing.py once I have all this data captured elsewhere
+- refactor tab completion to use StatementParser instead of parseline()
+- delete SHLEX_TODO.txt once I have all this data captured elsewhere
 - we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
 
 Questions:
@@ -18,4 +19,6 @@ Changelog Items:
 - object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything. The parser is now self.statement_parser instead of self.command_parser.
 - input redirection no longer supported. Use the load command instead.
 - submenus now call all hooks, it used to just call precmd and postcmd
+- cmd2 ignores identchars. The standard library cmd uses those characters to split the first "word" of the input, but cmd2 hasn't used those for a while, and the new parsing logic parses on whitespace, which has the added benefit of full unicode support, unlike cmd or prior versions of cmd2.
+
 

From a343ed182e07b89165db7b6a059ab79f9d6a0b15 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 25 Apr 2018 21:39:46 -0600
Subject: [PATCH 50/89] Todo updates

---
 SHLEX_TODO.txt | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index e57c03a94..7e79946cb 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -2,17 +2,12 @@
 Notes on conversion from pyparsing to shlex taking place in the ply branch
 
 Todo List:
-- case sensitive flag
 - refactor Cmd2.parseline() to use StatementParser.parse()
 - refactor tab completion to use StatementParser instead of parseline()
 - delete SHLEX_TODO.txt once I have all this data captured elsewhere
-- we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
 
 Questions:
-- say I have a command called 'fred' which is a multiline command. If I make an alias
-  for fred called 'george' is george a multiline command? I think the answer is yes.
-  If you want a multi-line synonym for a command that isn't multiline, do it like
-  example.py does. If the answer is no, then I need to rework StatementParser.parse()
+
 
 Changelog Items:
 - if self.default_to_shell is true, then redirection and piping is now properly passed to the shell, previously it was truncated

From 05ee395f0d487fc67979ce3d0824bdaadff5c811 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 25 Apr 2018 22:17:56 -0600
Subject: [PATCH 51/89] Remove POSIX_SHLEX and STRIP_QUOTES_FOR_NON_POSIX

---
 SHLEX_TODO.txt           | 15 +++++--------
 cmd2/__init__.py         |  2 +-
 cmd2/cmd2.py             | 48 ++++++----------------------------------
 docs/unfreefeatures.rst  | 10 ---------
 tests/test_argparse.py   | 14 ------------
 tests/test_cmd2.py       |  7 +-----
 tests/test_transcript.py |  5 -----
 7 files changed, 14 insertions(+), 87 deletions(-)

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index 774cbc9d7..b9ffe70d4 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -2,22 +2,17 @@
 Notes on conversion from pyparsing to shlex taking place in the ply branch
 
 Todo List:
-- case sensitive flag
 - refactor Cmd2.parseline() to use StatementParser.parse()
-- get rid of legalChars
-- get rid of POSIX_SHLEX and STRIP_QUOTES_FOR_NON_POSIX - those options should be ignored
-- we now ignore self.identchars, which breaks backwards compatibility with the cmd in the standard library
+- refactor tab completion to use StatementParser instead of parseline()
 - delete SHLEX_TODO.txt once everything is done
 
 Questions:
-- say I have a command called 'fred' which is a multiline command. If I make an alias
-  for fred called 'george' is george a multiline command? I think the answer is yes.
-  If you want a multi-line synonym for a command that isn't multiline, do it like
-  example.py does. If the answer is no, then I need to rework StatementParser.parse()
 
 Changelog Items:
-- if self.default_to_shell is true, then redirection and piping is now properly passed to the shell, previously it was truncated
+- if self.default_to_shell is true, then redirection and piping are now properly passed to the shell, previously it was truncated
 - object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything. The parser is now self.statement_parser instead of self.command_parser.
 - input redirection no longer supported. Use the load command instead.
 - submenus now call all hooks, it used to just call precmd and postcmd
-
+- cmd2 ignores identchars. The standardlibrary cmd uses those characters to split the first "word" of the input, but cmd2 hasn't used those for a while, and the new parsing logic parses on whitespace, which has the added benefit of full unicode support, unlike cmd or prior versions of cmd2.
+- set_posix_shlex function and POSIX_SHLEX variable have been removed. Parsing behavior is now always posix=false.
+- set_strip_quotes function and STRIP_QUOTES_FOR_NON_POSIX have been removed. Quotes are always stripped from arguments.
diff --git a/cmd2/__init__.py b/cmd2/__init__.py
index 8e744e03d..2dd089776 100644
--- a/cmd2/__init__.py
+++ b/cmd2/__init__.py
@@ -1,5 +1,5 @@
 #
 # -*- coding: utf-8 -*-
 #
-from .cmd2 import __version__, Cmd, set_posix_shlex, set_strip_quotes, AddSubmenu, CmdResult, categorize
+from .cmd2 import __version__, Cmd, AddSubmenu, CmdResult, categorize
 from .cmd2 import with_argument_list, with_argparser, with_argparser_and_unknown_args, with_category
diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 17a6917cf..cd80970b0 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -119,16 +119,6 @@ def __subclasshook__(cls, C):
 __version__ = '0.9.0'
 
 
-# The next 2 variables and associated setter functions effect how arguments are parsed for decorated commands
-# which use one of the decorators: @with_argument_list, @with_argparser, or @with_argparser_and_unknown_args
-# The defaults are sane and maximize ease of use for new applications based on cmd2.
-
-# Use POSIX or Non-POSIX (Windows) rules for splitting a command-line string into a list of arguments via shlex.split()
-POSIX_SHLEX = False
-
-# Strip outer quotes for convenience if POSIX_SHLEX = False
-STRIP_QUOTES_FOR_NON_POSIX = True
-
 # Used for tab completion and word breaks. Do not change.
 QUOTES = ['"', "'"]
 REDIRECTION_CHARS = ['|', '<', '>']
@@ -153,24 +143,6 @@ def categorize(func: Union[Callable, Iterable], category: str) -> None:
         setattr(func, HELP_CATEGORY, category)
 
 
-def set_posix_shlex(val: bool) -> None:
-    """ Allows user of cmd2 to choose between POSIX and non-POSIX splitting of args for decorated commands.
-
-    :param val: True => POSIX,  False => Non-POSIX
-    """
-    global POSIX_SHLEX
-    POSIX_SHLEX = val
-
-
-def set_strip_quotes(val: bool) -> None:
-    """ Allows user of cmd2 to choose whether to automatically strip outer-quotes when POSIX_SHLEX is False.
-
-    :param val: True => strip quotes on args for decorated commands if POSIX_SHLEX is False.
-    """
-    global STRIP_QUOTES_FOR_NON_POSIX
-    STRIP_QUOTES_FOR_NON_POSIX = val
-
-
 def _which(editor: str) -> Optional[str]:
     try:
         editor_path = subprocess.check_output(['which', editor], stderr=subprocess.STDOUT).strip()
@@ -202,13 +174,12 @@ def parse_quoted_string(cmdline: str) -> List[str]:
         lexed_arglist = cmdline
     else:
         # Use shlex to split the command line into a list of arguments based on shell rules
-        lexed_arglist = shlex.split(cmdline, posix=POSIX_SHLEX)
-        # If not using POSIX shlex, make sure to strip off outer quotes for convenience
-        if not POSIX_SHLEX and STRIP_QUOTES_FOR_NON_POSIX:
-            temp_arglist = []
-            for arg in lexed_arglist:
-                temp_arglist.append(strip_quotes(arg))
-            lexed_arglist = temp_arglist
+        lexed_arglist = shlex.split(cmdline, posix=False)
+        # strip off outer quotes for convenience
+        temp_arglist = []
+        for arg in lexed_arglist:
+            temp_arglist.append(strip_quotes(arg))
+        lexed_arglist = temp_arglist
     return lexed_arglist
 
 
@@ -2732,12 +2703,7 @@ def cmdenvironment(self):
         Commands may be terminated with: {}
         Arguments at invocation allowed: {}
         Output redirection and pipes allowed: {}
-        Parsing of command arguments:
-            Shell lexer mode for command argument splitting: {}
-            Strip Quotes after splitting arguments: {}
-        """.format(str(self.terminators), self.allow_cli_args, self.allow_redirection,
-                   "POSIX" if POSIX_SHLEX else "non-POSIX",
-                   "True" if STRIP_QUOTES_FOR_NON_POSIX and not POSIX_SHLEX else "False")
+        """.format(str(self.terminators), self.allow_cli_args, self.allow_redirection)
         return read_only_settings
 
     def show(self, args, parameter):
diff --git a/docs/unfreefeatures.rst b/docs/unfreefeatures.rst
index d420797d8..bcf0fa41a 100644
--- a/docs/unfreefeatures.rst
+++ b/docs/unfreefeatures.rst
@@ -129,16 +129,6 @@ that module.
 ``cmd2`` defines a few decorators which change the behavior of
 how arguments get parsed for and passed to a ``do_`` method.  See the section :ref:`decorators` for more information.
 
-Controlling how arguments are parsed for commands with flags
-------------------------------------------------------------
-There are a couple functions which can globally effect how arguments are parsed for commands with flags:
-
-.. autofunction:: cmd2.set_posix_shlex
-
-.. autofunction:: cmd2.set_strip_quotes
-
-.. _argparse: https://docs.python.org/3/library/argparse.html
-
 
 poutput, pfeedback, perror, ppaged
 ==================================
diff --git a/tests/test_argparse.py b/tests/test_argparse.py
index 6a9a93a74..f7c6eaba2 100644
--- a/tests/test_argparse.py
+++ b/tests/test_argparse.py
@@ -124,8 +124,6 @@ def test_argparse_basic_command(argparse_app):
     assert out == ['hello']
 
 def test_argparse_quoted_arguments(argparse_app):
-    argparse_app.POSIX = False
-    argparse_app.STRIP_QUOTES_FOR_NON_POSIX = True
     out = run_cmd(argparse_app, 'say "hello there"')
     assert out == ['hello there']
 
@@ -138,21 +136,9 @@ def test_argparse_with_list_and_empty_doc(argparse_app):
     assert out == ['HELLO WORLD!']
 
 def test_argparse_quoted_arguments_multiple(argparse_app):
-    argparse_app.POSIX = False
-    argparse_app.STRIP_QUOTES_FOR_NON_POSIX = True
     out = run_cmd(argparse_app, 'say "hello  there" "rick & morty"')
     assert out == ['hello  there rick & morty']
 
-def test_argparse_quoted_arguments_posix(argparse_app):
-    argparse_app.POSIX = True
-    out = run_cmd(argparse_app, 'tag strong this should be loud')
-    assert out == ['<strong>this should be loud</strong>']
-
-def test_argparse_quoted_arguments_posix_multiple(argparse_app):
-    argparse_app.POSIX = True
-    out = run_cmd(argparse_app, 'tag strong this "should  be" loud')
-    assert out == ['<strong>this should  be loud</strong>']
-
 def test_argparse_help_docstring(argparse_app):
     out = run_cmd(argparse_app, 'help say')
     assert out[0].startswith('usage: say')
diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py
index 0a4dfbc64..2c9b03df3 100644
--- a/tests/test_cmd2.py
+++ b/tests/test_cmd2.py
@@ -104,13 +104,8 @@ def test_base_show_readonly(base_app):
         Commands may be terminated with: {}
         Arguments at invocation allowed: {}
         Output redirection and pipes allowed: {}
-        Parsing of command arguments:
-            Shell lexer mode for command argument splitting: {}
-            Strip Quotes after splitting arguments: {}
             
-""".format(base_app.terminators, base_app.allow_cli_args, base_app.allow_redirection,
-           "POSIX" if cmd2.cmd2.POSIX_SHLEX else "non-POSIX",
-           "True" if cmd2.cmd2.STRIP_QUOTES_FOR_NON_POSIX and not cmd2.cmd2.POSIX_SHLEX else "False"))
+""".format(base_app.terminators, base_app.allow_cli_args, base_app.allow_redirection))
     assert out == expected
 
 
diff --git a/tests/test_transcript.py b/tests/test_transcript.py
index 8ee5f3f6f..bba56cab4 100644
--- a/tests/test_transcript.py
+++ b/tests/test_transcript.py
@@ -15,7 +15,6 @@
 import pytest
 
 import cmd2
-from cmd2 import set_posix_shlex, set_strip_quotes
 from .conftest import run_cmd, StdOut, normalize
 
 class CmdLineApp(cmd2.Cmd):
@@ -35,10 +34,6 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.intro = 'This is an intro banner ...'
 
-        # Configure how arguments are parsed for commands using decorators
-        set_posix_shlex(False)
-        set_strip_quotes(True)
-
     speak_parser = argparse.ArgumentParser()
     speak_parser.add_argument('-p', '--piglatin', action="store_true", help="atinLay")
     speak_parser.add_argument('-s', '--shout', action="store_true", help="N00B EMULATION MODE")

From 739d3f42715e59b61432cd7fbedacae4a4f80a16 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Thu, 26 Apr 2018 20:21:52 -0600
Subject: [PATCH 52/89] First stage of refactoring cmd2.parseline() for tab
 completion

---
 cmd2/cmd2.py             |   5 ++
 cmd2/parsing.py          | 104 ++++++++++++++++++++++++++++++---------
 tests/test_completion.py |  38 --------------
 tests/test_parsing.py    |  38 +++++++++++++-
 4 files changed, 122 insertions(+), 63 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index cd80970b0..89e078027 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -1588,6 +1588,11 @@ def complete(self, text, state):
 
                 # Parse the command line
                 command, args, expanded_line = self.parseline(line)
+                
+                # use these lines instead of the one above
+                # statement = self.command_parser.parse_command_only(line)
+                # command = statement.command
+                # expanded_line = statement.command_and_args
 
                 # We overwrote line with a properly formatted but fully stripped version
                 # Restore the end spaces since line is only supposed to be lstripped when
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 9030a5f88..45715b32a 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -10,6 +10,14 @@
 
 BLANK_LINE = '\n'
 
+def _comment_replacer(match):
+    s = match.group(0)
+    if s.startswith('/'):
+        # treat the removed comment as an empty string
+        return ''
+    else:
+        return s
+
 class Statement(str):
     """String subclass with additional attributes to store the results of parsing.
     
@@ -32,6 +40,11 @@ def __init__(self, object):
         self.pipeTo = None
         self.output = None
         self.outputTo = None
+    
+    @property
+    def command_and_args(self):
+        """Combine command and args with a space separating them"""
+        return '{} {}'.format('' if self.command is None else self.command, self.args).strip()
 
 class StatementParser():
     """Parse raw text into command components.
@@ -56,40 +69,28 @@ def __init__(
         self.aliases = aliases
         self.shortcuts = shortcuts
 
-    def parse(self, rawinput: str) -> Statement:
-        # strip C-style comments
-        # shlex will handle the python/shell style comments for us
-        def replacer(match):
-                s = match.group(0)
-                if s.startswith('/'):
-                    # treat the removed comment as an empty string
-                    return ''
-                else:
-                    return s
-        pattern = re.compile(
-            #r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+        self.comment_pattern = re.compile(
             r'/\*.*?(\*/|$)|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
             re.DOTALL | re.MULTILINE
         )
-        rawinput = re.sub(pattern, replacer, rawinput)
-        line = rawinput
 
+    def parse(self, rawinput: str) -> Statement:
+        """Parse input into a Statement object, stripping comments, expanding
+        aliases and shortcuts, and extracting output redirection directives.
+        """
+        # strip C-style comments
+        # shlex will handle the python/shell style comments for us
+        # save rawinput for later
+        rawinput = re.sub(self.comment_pattern, _comment_replacer, rawinput)
+        # we are going to modify line, so create a copy of the raw input
+        line = rawinput
         command = None
         args = ''
 
         # expand shortcuts, have to do this first because
         # a shortcut can expand into multiple tokens, ie '!ls' becomes
         # 'shell ls'
-        for (shortcut, expansion) in self.shortcuts:
-            if  line.startswith(shortcut):
-                # If the next character after the shortcut isn't a space, then insert one
-                shortcut_len = len(shortcut)
-                if len(line) == shortcut_len or line[shortcut_len] != ' ':
-                    expansion += ' '
-
-                # Expand the shortcut
-                line = line.replace(shortcut, expansion, 1)
-                break
+        line = self.expand_shortcuts(line)
 
         # handle the special case/hardcoded terminator of a blank line
         # we have to do this before we shlex on whitespace because it
@@ -98,10 +99,12 @@ def replacer(match):
         if line[-1:] == BLANK_LINE:
             terminator = BLANK_LINE
 
+        # split the input on whitespace
         s = shlex.shlex(line, posix=False)
         s.whitespace_split = True
         tokens = self.split_on_punctuation(list(s))
 
+        # expand aliases
         if tokens:
             command_to_expand = tokens[0]
             tokens[0] = self.expand_aliases(command_to_expand)
@@ -211,6 +214,59 @@ def replacer(match):
         result.multilineCommand = multilineCommand
         return result
 
+    def parse_command_only(self, rawinput: str) -> Statement:
+        """Partially parse input into a Statement object. The command is
+        identified, and shortcuts and aliases are expanded.
+        Terminators, multiline commands, and output redirection are not
+        parsed.
+        """
+        # strip C-style comments
+        # shlex will handle the python/shell style comments for us
+        # save rawinput for later
+        rawinput = re.sub(self.comment_pattern, _comment_replacer, rawinput)
+        # we are going to modify line, so create a copy of the raw input
+        line = rawinput
+        command = None
+        args = ''
+
+        # expand shortcuts, have to do this first because
+        # a shortcut can expand into multiple tokens, ie '!ls' becomes
+        # 'shell ls'
+        line = self.expand_shortcuts(line)
+
+        # split the input on whitespace
+        s = shlex.shlex(line, posix=False)
+        s.whitespace_split = True
+        tokens = self.split_on_punctuation(list(s))
+
+        # expand aliases
+        if tokens:
+            command_to_expand = tokens[0]
+            tokens[0] = self.expand_aliases(command_to_expand)
+
+        (command, args) = self._command_and_args(tokens)
+
+        # build Statement object
+        result = Statement(args)
+        result.raw = rawinput
+        result.command = command
+        result.args = args
+        return result
+
+    def expand_shortcuts(self, line: str) -> str:
+        """Expand shortcuts at the beginning of input."""
+        for (shortcut, expansion) in self.shortcuts:
+            if  line.startswith(shortcut):
+                # If the next character after the shortcut isn't a space, then insert one
+                shortcut_len = len(shortcut)
+                if len(line) == shortcut_len or line[shortcut_len] != ' ':
+                    expansion += ' '
+
+                # Expand the shortcut
+                line = line.replace(shortcut, expansion, 1)
+                break
+        return line
+
     def expand_aliases(self, command: str) -> str:
         """Given a command, expand any aliases for the command"""
         # make a copy of aliases so we can edit it
diff --git a/tests/test_completion.py b/tests/test_completion.py
index cf45f2810..f916f2e85 100644
--- a/tests/test_completion.py
+++ b/tests/test_completion.py
@@ -581,44 +581,6 @@ def test_tokens_for_completion_redirect_off(cmd2_app):
     assert expected_tokens == tokens
     assert expected_raw_tokens == raw_tokens
 
-def test_parseline_command_and_args(cmd2_app):
-    line = 'help history'
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command == 'help'
-    assert args == 'history'
-    assert line == out_line
-
-def test_parseline_emptyline(cmd2_app):
-    line = ''
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command is None
-    assert args is None
-    assert line is out_line
-
-def test_parseline_strips_line(cmd2_app):
-    line = '  help history  '
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command == 'help'
-    assert args == 'history'
-    assert line.strip() == out_line
-
-def test_parseline_expands_alias(cmd2_app):
-    # Create the alias
-    cmd2_app.do_alias(['fake', 'pyscript'])
-
-    line = 'fake foobar.py'
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command == 'pyscript'
-    assert args == 'foobar.py'
-    assert line.replace('fake', 'pyscript') == out_line
-
-def test_parseline_expands_shortcuts(cmd2_app):
-    line = '!cat foobar.txt'
-    command, args, out_line = cmd2_app.parseline(line)
-    assert command == 'shell'
-    assert args == 'cat foobar.txt'
-    assert line.replace('!', 'shell ') == out_line
-
 def test_add_opening_quote_basic_no_text(cmd2_app):
     text = ''
     line = 'test_basic {}'.format(text)
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 1913938eb..d7a872b3d 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -19,7 +19,7 @@ def parser():
         redirection_chars=['|', '<', '>'],        
         terminators = [';'],
         multilineCommands = ['multiline'],
-        aliases = {'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline'},
+        aliases = {'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline', 'fake': 'pyscript'},
         shortcuts = [('?', 'help'), ('!', 'shell')]
     )
     return parser
@@ -27,6 +27,8 @@ def parser():
 def test_parse_empty_string(parser):
     statement = parser.parse('')
     assert not statement.command
+    assert not statement.args
+    assert statement.raw == ''
 
 @pytest.mark.parametrize('tokens,command,args', [
     ( [], None, ''),
@@ -287,3 +289,37 @@ def test_alias_on_multiline_command(parser):
     assert statement.command == 'multiline'
     assert statement.args == 'has > inside an unfinished command'
     assert not statement.terminator
+
+def test_parse_command_only_command_and_args(parser):
+    line = 'help history'
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'help'
+    assert statement.args == 'history'
+    assert statement.command_and_args == line
+
+def test_parse_command_only_emptyline(parser):
+    line = ''
+    statement = parser.parse_command_only(line)
+    assert statement.command is None
+    assert statement.args is ''
+    assert statement.command_and_args is line
+
+def test_parse_command_only_strips_line(parser):
+    line = '  help history  '
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'help'
+    assert statement.args == 'history'
+    assert statement.command_and_args == line.strip()
+
+def test_parse_command_only_expands_alias(parser):
+    line = 'fake foobar.py'
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'pyscript'
+    assert statement.args == 'foobar.py'
+
+def test_parse_command_only_expands_shortcuts(parser):
+    line = '!cat foobar.txt'
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'shell'
+    assert statement.args == 'cat foobar.txt'
+    assert statement.command_and_args == line.replace('!', 'shell ')

From a57eaefe7fe5a980a22568056df3cc75be146f5c Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Thu, 26 Apr 2018 21:53:20 -0600
Subject: [PATCH 53/89] No more xdist or forking

---
 tox.ini | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/tox.ini b/tox.ini
index 6749418b8..0d511db59 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,11 +16,9 @@ deps =
   pyperclip
   pytest
   pytest-cov
-  pytest-forked
-  pytest-xdist
   wcwidth
 commands =
-  py.test {posargs: -n 2} --cov --cov-report=term-missing --forked
+  py.test {posargs} --cov --cov-report=term-missing
   codecov
 
 [testenv:py35]
@@ -29,10 +27,8 @@ deps =
   pyparsing
   pyperclip
   pytest
-  pytest-forked
-  pytest-xdist
   wcwidth
-commands = py.test -v -n2 --forked
+commands = py.test -v
 
 [testenv:py35-win]
 deps =
@@ -41,8 +37,7 @@ deps =
   pyperclip
   pyreadline
   pytest
-  pytest-xdist
-commands = py.test -v -n2
+commands = py.test -v
 
 [testenv:py36]
 deps =
@@ -51,11 +46,9 @@ deps =
   pyperclip
   pytest
   pytest-cov
-  pytest-forked
-  pytest-xdist
   wcwidth
 commands =
-  py.test {posargs: -n 2} --cov --cov-report=term-missing --forked
+  py.test {posargs:} --cov --cov-report=term-missing
   codecov
 
 [testenv:py36-win]
@@ -66,9 +59,8 @@ deps =
   pyreadline
   pytest
   pytest-cov
-  pytest-xdist
 commands =
-  py.test {posargs: -n 2} --cov --cov-report=term-missing
+  py.test {posargs} --cov --cov-report=term-missing
   codecov
 
 [testenv:py37]
@@ -76,8 +68,6 @@ deps =
   pyparsing
   pyperclip
   pytest
-  pytest-forked
-  pytest-xdist
   wcwidth
-commands = py.test -v -n2 --forked
+commands = py.test -v
 

From a5501592da787f85e358ace415cb6a4f93b674ed Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Thu, 26 Apr 2018 21:57:45 -0600
Subject: [PATCH 54/89] Remove references to pytest-xdist and pytest-forked

---
 CONTRIBUTING.md | 29 ++++++-----------------------
 1 file changed, 6 insertions(+), 23 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index e0cbd57b2..f4ebcbefc 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -45,7 +45,6 @@ The tables below list all prerequisites along with the minimum required version
 | Prerequisite                                        | Minimum Version |
 | --------------------------------------------------- | --------------- |
 | [Python](https://www.python.org/downloads/)         | `3.4`           |
-| [pyparsing](http://pyparsing.wikispaces.com)        | `2.1`           |
 | [pyperclip](https://github.com/asweigart/pyperclip) | `1.6`           |
 
 #### Additional prerequisites to run cmd2 unit tests
@@ -63,15 +62,13 @@ The tables below list all prerequisites along with the minimum required version
 ### Optional prerequisites for enhanced unit test features
 | Prerequisite                                | Minimum Version |
 | ------------------------------------------- | --------------- |
-| [pytest-forked](https://pypi.python.org/pypi/pytest-forked)| `0.2` |
-| [pytest-xdist](https://pypi.python.org/pypi/pytest-xdist)| `1.15` |
 | [pytest-cov](https://pypi.python.org/pypi/pytest-cov) | `1.8` |
 
 If Python is already installed in your machine, run the following commands to validate the versions:
 
 ```shell
 python -V
-pip freeze | grep pyparsing
+pip freeze | grep pyperclip
 ```
 
 If your versions are lower than the prerequisite versions, you should update.
@@ -190,7 +187,7 @@ Once you have cmd2 cloned, before you start any cmd2 application, you first need
 
 ```bash
 # Install cmd2 prerequisites
-pip install -U pyparsing pyperclip
+pip install -U pyperclip
 
 # Install prerequisites for running cmd2 unit tests
 pip install -U pytest
@@ -198,8 +195,8 @@ pip install -U pytest
 # Install prerequisites for building cmd2 documentation
 pip install -U sphinx sphinx-rtd-theme
 
-# Install optional prerequisites for running unit tests in parallel and doing code coverage analysis
-pip install -U pytest-xdist pytest-cov pytest-forked
+# Install optional prerequisites for doing code coverage analysis
+pip install -U pytest-cov
 ```
 
 For doing cmd2 development, you actually do NOT want to have cmd2 installed as a Python package.  
@@ -259,27 +256,13 @@ py.test
 
 and ensure all tests pass.
 
-If you have the `pytest-xdist` pytest distributed testing plugin installed, then you can use it to
-dramatically speed up test execution by running tests in parallel on multiple cores like so:
-
-```shell
-py.test -n4
-```
-where `4` should be replaced by the number of parallel threads you wish to run for testing.
-
-If you have the `pytest-forked` pytest plugin (not avilable on Windows) for running tests in isolated formed processes, 
-you can speed things up even further:
-
-```shell
-py.test -nauto --forked
-```
 
 #### Measuring code coverage
 
 Code coverage can be measured as follows:
 
 ```shell
-py.test -nauto --cov=cmd2 --cov-report=term-missing --cov-report=html --forked
+py.test --cov=cmd2 --cov-report=term-missing --cov-report=html
 ```
 
 Then use your web browser of choice to look at the results which are in `<cmd2>/htmlcov/index.html`.
@@ -287,7 +270,7 @@ Then use your web browser of choice to look at the results which are in `<cmd2>/
 ### Squash Your Commits
 When you make a pull request, it is preferable for all of your changes to be in one commit.
 
-If you have made more then one commit, then you will can _squash_ your commits.
+If you have made more then one commit, then you can _squash_ your commits.
 
 To do this, see [Squashing Your Commits](http://forum.freecodecamp.com/t/how-to-squash-multiple-commits-into-one-with-git/13231).
 

From 070e46b0c8d521cd851cd5092ad48d7890f803eb Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Thu, 26 Apr 2018 22:11:55 -0600
Subject: [PATCH 55/89] Fix broken tox builds with codecov

---
 tox.ini | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tox.ini b/tox.ini
index 0d511db59..6d7d063ec 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,10 +15,9 @@ deps =
   pyparsing
   pyperclip
   pytest
-  pytest-cov
   wcwidth
 commands =
-  py.test {posargs} --cov --cov-report=term-missing
+  py.test {posargs}
   codecov
 
 [testenv:py35]
@@ -48,7 +47,7 @@ deps =
   pytest-cov
   wcwidth
 commands =
-  py.test {posargs:} --cov --cov-report=term-missing
+  py.test
   codecov
 
 [testenv:py36-win]
@@ -58,9 +57,8 @@ deps =
   pyperclip
   pyreadline
   pytest
-  pytest-cov
 commands =
-  py.test {posargs} --cov --cov-report=term-missing
+  py.test {posargs}
   codecov
 
 [testenv:py37]

From 618b89104e4d0bf28e4ec3471e6c1d95b46b95c0 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Thu, 26 Apr 2018 22:27:36 -0600
Subject: [PATCH 56/89] =?UTF-8?q?pytest=20with=20=E2=80=94cov=20fails=20tw?=
 =?UTF-8?q?o=20unit=20tests.=20without=20=E2=80=94cov=20they=20pass=20:(?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tox.ini | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/tox.ini b/tox.ini
index 6d7d063ec..e74ce16f1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,18 +12,17 @@ setenv =
 [testenv:py34]
 deps =
   codecov
-  pyparsing
   pyperclip
   pytest
+  pytest-cov
   wcwidth
 commands =
-  py.test {posargs}
+  py.test {posargs} --cov
   codecov
 
 [testenv:py35]
 deps =
   mock
-  pyparsing
   pyperclip
   pytest
   wcwidth
@@ -32,7 +31,6 @@ commands = py.test -v
 [testenv:py35-win]
 deps =
   mock
-  pyparsing
   pyperclip
   pyreadline
   pytest
@@ -41,29 +39,27 @@ commands = py.test -v
 [testenv:py36]
 deps =
   codecov
-  pyparsing
   pyperclip
   pytest
   pytest-cov
   wcwidth
 commands =
-  py.test
+  py.test {posargs} --cov
   codecov
 
 [testenv:py36-win]
 deps =
   codecov
-  pyparsing
   pyperclip
   pyreadline
   pytest
+  pytest-cov
 commands =
-  py.test {posargs}
+  py.test {posargs} --cov
   codecov
 
 [testenv:py37]
 deps =
-  pyparsing
   pyperclip
   pytest
   wcwidth

From 16bf37bf0141446f46dfce2d1ba2b9ed0de7ec44 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Fri, 27 Apr 2018 07:52:14 -0600
Subject: [PATCH 57/89] Add description of comment-matching regex

---
 cmd2/parsing.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 45715b32a..b6c58db71 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -69,6 +69,30 @@ def __init__(
         self.aliases = aliases
         self.shortcuts = shortcuts
 
+        # this regular expression matches C-style comments and quoted
+        # strings, i.e. stuff between single or double quote marks
+        # it's used with _comment_replacer() to strip out the C-style
+        # comments, while leaving C-style comments that are inside either
+        # double or single quotes.
+        #
+        # this big regular expression can be broken down into 3 regular
+        # expressions that are OR'ed together.
+        #
+        # /\*.*?(\*/|$)          matches C-style comments, with an optional
+        #                        closing '*/'. The optional closing '*/' is
+        #                        there to retain backward compatibility with
+        #                        the pyparsing implementation of cmd2 < 0.9.0
+        # \'(?:\\.|[^\\\'])*\'   matches a single quoted string, allowing
+        #                        for embedded backslash escaped single quote
+        #                        marks
+        # "(?:\\.|[^\\"])*"      matches a double quoted string, allowing
+        #                        for embedded backslash escaped double quote
+        #                        marks
+        #
+        # by way of reminder the (?:...) regular expression syntax is just
+        # a non-capturing version of regular parenthesis. We need the non-
+        # capturing syntax because _comment_replacer() looks at match
+        # groups
         self.comment_pattern = re.compile(
             r'/\*.*?(\*/|$)|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
             re.DOTALL | re.MULTILINE

From 0d4cf470486ca20f9c7d6580161a3ff689172523 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Fri, 27 Apr 2018 11:14:25 -0600
Subject: [PATCH 58/89] Fix broken interrupt unit tests

---
 tests/test_cmd2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py
index 2c9b03df3..9ec7f642f 100644
--- a/tests/test_cmd2.py
+++ b/tests/test_cmd2.py
@@ -964,6 +964,7 @@ def do_say(self, arg):
 @pytest.fixture
 def say_app():
     app = SayApp()
+    app.allow_cli_args = False
     app.stdout = StdOut()
     return app
 

From 62b2d2624bc530b0a9197806d34e639706038448 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Fri, 27 Apr 2018 11:24:58 -0600
Subject: [PATCH 59/89] =?UTF-8?q?Remove=20unused=20=E2=80=98<=E2=80=98=20f?=
 =?UTF-8?q?rom=20REDIRECTION=5FCHARS?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 cmd2/cmd2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 89e078027..226852029 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -121,7 +121,7 @@ def __subclasshook__(cls, C):
 
 # Used for tab completion and word breaks. Do not change.
 QUOTES = ['"', "'"]
-REDIRECTION_CHARS = ['|', '<', '>']
+REDIRECTION_CHARS = ['|', '>']
 
 # optional attribute, when tagged on a function, allows cmd2 to categorize commands
 HELP_CATEGORY = 'help_category'

From c40322236f9ca82e5e7ac0ab87fd55f95c4bd57f Mon Sep 17 00:00:00 2001
From: Todd Leonhardt <todd.leonhardt@gmail.com>
Date: Fri, 27 Apr 2018 14:24:45 -0400
Subject: [PATCH 60/89] Fixes for constants being in separate file

---
 cmd2/cmd2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 630e79b4a..2c004c85f 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -680,9 +680,9 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
         self.pystate = {}
         self.keywords = self.reserved_words + [fname[3:] for fname in dir(self) if fname.startswith('do_')]
         self.statement_parser = StatementParser(
-            quotes=QUOTES,
+            quotes=constants.QUOTES,
             allow_redirection=self.allow_redirection,
-            redirection_chars=REDIRECTION_CHARS,
+            redirection_chars=constants.REDIRECTION_CHARS,
             terminators=self.terminators,
             multilineCommands=self.multilineCommands,
             aliases=self.aliases,

From 8c69e0e75454a5665ed2d428aba3c052b97b68b5 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Fri, 27 Apr 2018 15:11:31 -0600
Subject: [PATCH 61/89] =?UTF-8?q?Remove=20=E2=80=98<=E2=80=98=20from=20RED?=
 =?UTF-8?q?IRECTION=5FCHARS,=20it=E2=80=99s=20no=20longer=20used.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 cmd2/constants.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd2/constants.py b/cmd2/constants.py
index 6784264fd..838650e5b 100644
--- a/cmd2/constants.py
+++ b/cmd2/constants.py
@@ -6,7 +6,7 @@
 
 # Used for command parsing, tab completion and word breaks. Do not change.
 QUOTES = ['"', "'"]
-REDIRECTION_CHARS = ['|', '<', '>']
+REDIRECTION_CHARS = ['|', '>']
 
 # Regular expression to match ANSI escape codes
 ANSI_ESCAPE_RE = re.compile(r'\x1b[^m]*m')

From 85a3cc1320bafc2a44c75b630835bcc0635b61de Mon Sep 17 00:00:00 2001
From: Todd Leonhardt <todd.leonhardt@gmail.com>
Date: Sat, 28 Apr 2018 12:24:11 -0400
Subject: [PATCH 62/89] Removed pyparsing from setup.py as a dependency

Also:
- updated README.md to correctly state 3rd-party dependencies
- Updated docs/requirements.txt to no longer include pyparsing for ReadTheDocs build
---
 README.md             | 2 +-
 docs/requirements.txt | 3 +--
 setup.py              | 4 ++--
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index bf381ba59..f456694c7 100755
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@ pip install -U cmd2
 ```
 
 cmd2 works with Python 3.4+ on Windows, macOS, and Linux. It is pure Python code with
-the only 3rd-party dependencies being on [pyparsing](http://pyparsing.wikispaces.com), and [pyperclip](https://github.com/asweigart/pyperclip).
+the only 3rd-party dependencies being on [colorama](https://github.com/tartley/colorama), and [pyperclip](https://github.com/asweigart/pyperclip).
 Windows has an additional dependency on [pyreadline](https://pypi.python.org/pypi/pyreadline). Non-Windows platforms
 have an additional dependency on [wcwidth](https://pypi.python.org/pypi/wcwidth). Finally, Python
 3.4 has an additional dependency on [contextlib2](https://pypi.python.org/pypi/contextlib2).
diff --git a/docs/requirements.txt b/docs/requirements.txt
index d22f93414..011603fb3 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,3 @@
-pyparsing
+colorama
 pyperclip
 wcwidth
-colorama
diff --git a/setup.py b/setup.py
index 3020cf24d..d5043ce6d 100755
--- a/setup.py
+++ b/setup.py
@@ -61,7 +61,7 @@
 Topic :: Software Development :: Libraries :: Python Modules
 """.splitlines())))
 
-INSTALL_REQUIRES = ['pyparsing >= 2.1.0', 'pyperclip >= 1.5.27', 'colorama']
+INSTALL_REQUIRES = ['pyperclip >= 1.5.27', 'colorama']
 
 EXTRAS_REQUIRE = {
     # Windows also requires pyreadline to ensure tab completion works
@@ -83,7 +83,7 @@
         INSTALL_REQUIRES.append('typing')
 
 TESTS_REQUIRE = ['pytest', 'pytest-xdist']
-DOCS_REQUIRE = ['sphinx', 'sphinx_rtd_theme', 'pyparsing', 'pyperclip', 'wcwidth']
+DOCS_REQUIRE = ['sphinx', 'sphinx_rtd_theme', 'pyperclip', 'wcwidth']
 
 setup(
     name="cmd2",

From 5c14b3845f6a872e3e5b236f8caab6b4f3472f8f Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 09:38:15 -0600
Subject: [PATCH 63/89] Cleanup requested changes in pull request

---
 cmd2/cmd2.py    | 18 +++++++++---------
 cmd2/parsing.py | 15 +++++++--------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 40c024d08..630ff0340 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -42,7 +42,7 @@
 import sys
 import tempfile
 import traceback
-from typing import Callable, List, Optional, Union
+from typing import Callable, List, Optional, Union, Tuple
 import unittest
 from code import InteractiveConsole
 
@@ -55,7 +55,7 @@
 from .rl_utils import rl_force_redisplay, readline, rl_type, RlType
 from .argparse_completer import AutoCompleter, ACArgumentParser
 
-from cmd2.parsing import StatementParser
+from cmd2.parsing import StatementParser, Statement
 
 if rl_type == RlType.PYREADLINE:
 
@@ -1586,7 +1586,7 @@ def complete(self, text, state):
 
                 # Parse the command line
                 command, args, expanded_line = self.parseline(line)
-                
+
                 # use these lines instead of the one above
                 # statement = self.command_parser.parse_command_only(line)
                 # command = statement.command
@@ -1865,7 +1865,7 @@ def preloop(self):
         # Register a default SIGINT signal handler for Ctrl+C
         signal.signal(signal.SIGINT, self.sigint_handler)
 
-    def precmd(self, statement):
+    def precmd(self, statement: Statement) -> Statement:
         """Hook method executed just before the command is processed by ``onecmd()`` and after adding it to the history.
 
         :param statement: Statement - subclass of str which also contains the parsed input
@@ -1876,7 +1876,7 @@ def precmd(self, statement):
     # -----  Methods which are cmd2-specific lifecycle hooks which are not present in cmd -----
 
     # noinspection PyMethodMayBeStatic
-    def preparse(self, raw):
+    def preparse(self, raw: str) -> str:
         """Hook method executed just before the command line is interpreted, but after the input prompt is generated.
 
         :param raw: str - raw command line input
@@ -1885,7 +1885,7 @@ def preparse(self, raw):
         return raw
 
     # noinspection PyMethodMayBeStatic
-    def postparse(self, statement):
+    def postparse(self, statement: Statement) -> Statement:
         """Hook that runs immediately after parsing the user input.
 
         :param statement: Statement object populated by parsing
@@ -1894,7 +1894,7 @@ def postparse(self, statement):
         return statement
 
     # noinspection PyMethodMayBeStatic
-    def postparsing_precmd(self, statement):
+    def postparsing_precmd(self, statement: Statement) -> Tuple[bool, Statement]:
         """This runs after parsing the command-line, but before anything else; even before adding cmd to history.
 
         NOTE: This runs before precmd() and prior to any potential output redirection or piping.
@@ -1913,7 +1913,7 @@ def postparsing_precmd(self, statement):
         return stop, statement
 
     # noinspection PyMethodMayBeStatic
-    def postparsing_postcmd(self, stop):
+    def postparsing_postcmd(self, stop: bool) -> bool:
         """This runs after everything else, including after postcmd().
 
         It even runs when an empty line is entered.  Thus, if you need to do something like update the prompt due
@@ -2072,7 +2072,7 @@ def runcmds_plus_hooks(self, cmds):
 
     def _complete_statement(self, line):
         """Keep accepting lines of input until the command is complete.
-        
+
         There is some pretty hacky code here to handle some quirks of
         self.pseudo_raw_input(). It returns a literal 'eof' if the input
         pipe runs out. We can't refactor it because we need to retain
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index b6c58db71..22b558b3b 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -6,8 +6,6 @@
 import shlex
 from typing import List, Tuple
 
-import cmd2
-
 BLANK_LINE = '\n'
 
 def _comment_replacer(match):
@@ -20,7 +18,7 @@ def _comment_replacer(match):
 
 class Statement(str):
     """String subclass with additional attributes to store the results of parsing.
-    
+
     The cmd module in the standard library passes commands around as a
     string. To retain backwards compatibility, cmd2 does the same. However, we
     need a place to capture the additional output of the command parsing, so we add
@@ -29,8 +27,9 @@ class Statement(str):
     The string portion of the class contains the arguments, but not the command, nor
     the output redirection clauses.
     """
-    def __init__(self, object):
-        self.raw = str(object)
+    def __init__(self, obj):
+        super().__init__()
+        self.raw = str(obj)
         self.command = None
         self.multilineCommand = None
         # has to be an empty string for compatibility with standard library cmd
@@ -40,7 +39,7 @@ def __init__(self, object):
         self.pipeTo = None
         self.output = None
         self.outputTo = None
-    
+
     @property
     def command_and_args(self):
         """Combine command and args with a space separating them"""
@@ -48,7 +47,7 @@ def command_and_args(self):
 
 class StatementParser():
     """Parse raw text into command components.
-    
+
     Shortcuts is a list of tuples with each tuple containing the shortcut and the expansion.
     """
     def __init__(
@@ -207,7 +206,7 @@ def parse(self, rawinput: str) -> Statement:
         except ValueError:
             # no pipe in the tokens
             pipeTo = None
-        
+
         if terminator:
             # whatever is left is the suffix
             suffix = ' '.join(tokens)

From fd3512a80600a3262030d55ed03f4930757395f6 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 09:52:10 -0600
Subject: [PATCH 64/89] Move quotes and redirection_chars from arguments to
 constants

Since the tab completion code relies on these same constants, if we allow them to be passed to the statement parser, we could have a situation where the statement parser and tab completion return different results.
---
 cmd2/cmd2.py          |  2 --
 cmd2/parsing.py       | 46 ++++++++++++++++++++++++++-----------------
 tests/test_parsing.py |  2 --
 3 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 630ff0340..626769fe5 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -687,9 +687,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
         self.pystate = {}
         self.keywords = self.reserved_words + [fname[3:] for fname in dir(self) if fname.startswith('do_')]
         self.statement_parser = StatementParser(
-            quotes=constants.QUOTES,
             allow_redirection=self.allow_redirection,
-            redirection_chars=constants.REDIRECTION_CHARS,
             terminators=self.terminators,
             multilineCommands=self.multilineCommands,
             aliases=self.aliases,
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 22b558b3b..b4f0e9c17 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -6,15 +6,17 @@
 import shlex
 from typing import List, Tuple
 
+from . import constants
+
 BLANK_LINE = '\n'
 
 def _comment_replacer(match):
-    s = match.group(0)
-    if s.startswith('/'):
-        # treat the removed comment as an empty string
+    matched_string = match.group(0)
+    if matched_string.startswith('/'):
+        # the matched string was a comment, so remove it
         return ''
-    else:
-        return s
+    # the matched string was a quoted string, return the match
+    return matched_string
 
 class Statement(str):
     """String subclass with additional attributes to store the results of parsing.
@@ -52,21 +54,29 @@ class StatementParser():
     """
     def __init__(
             self,
-            quotes=['"', "'"],
             allow_redirection=True,
-            redirection_chars=['|', '<', '>'],
-            terminators=[';'],
-            multilineCommands = [],
-            aliases = {},
+            terminators=None,
+            multilineCommands = None,
+            aliases = None,
             shortcuts = [],
         ):
-        self.quotes = quotes
         self.allow_redirection = allow_redirection
-        self.redirection_chars = redirection_chars
-        self.terminators = terminators
-        self.multilineCommands = multilineCommands
-        self.aliases = aliases
-        self.shortcuts = shortcuts
+        if terminators is None:
+            self.terminators = [';']
+        else:
+            self.terminators = terminators
+        if multilineCommands is None:
+            self.multilineCommands = []
+        else:
+            self.multilineCommands = multilineCommands
+        if aliases is None:
+            self.aliases = {}
+        else:
+            self.aliases = aliases
+        if shortcuts is None:
+            self.shortcuts = []
+        else:
+            self.shortcuts = shortcuts
 
         # this regular expression matches C-style comments and quoted
         # strings, i.e. stuff between single or double quote marks
@@ -333,14 +343,14 @@ def split_on_punctuation(self, tokens: List[str]) -> List[str]:
         punctuation = []
         punctuation.extend(self.terminators)
         if self.allow_redirection:
-            punctuation.extend(self.redirection_chars)
+            punctuation.extend(constants.REDIRECTION_CHARS)
 
         punctuated_tokens = []
 
         for cur_initial_token in tokens:
 
             # Save tokens up to 1 character in length or quoted tokens. No need to parse these.
-            if len(cur_initial_token) <= 1 or cur_initial_token[0] in self.quotes:
+            if len(cur_initial_token) <= 1 or cur_initial_token[0] in constants.QUOTES:
                 punctuated_tokens.append(cur_initial_token)
                 continue
 
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index d7a872b3d..b920b440b 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -14,9 +14,7 @@
 @pytest.fixture
 def parser():
     parser = StatementParser(
-        quotes=['"', "'"],
         allow_redirection=True,
-        redirection_chars=['|', '<', '>'],        
         terminators = [';'],
         multilineCommands = ['multiline'],
         aliases = {'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline', 'fake': 'pyscript'},

From 85c63d41286e62460b3b80c465349a5a0476266c Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 09:59:13 -0600
Subject: [PATCH 65/89] multilineCommand -> multiline_command

---
 SHLEX_TODO.txt        |  1 +
 cmd2/cmd2.py          |  6 +++---
 cmd2/parsing.py       | 18 +++++++++---------
 tests/test_parsing.py | 16 ++++++++--------
 4 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index b9ffe70d4..b8526afa7 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -12,6 +12,7 @@ Changelog Items:
 - if self.default_to_shell is true, then redirection and piping are now properly passed to the shell, previously it was truncated
 - object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything. The parser is now self.statement_parser instead of self.command_parser.
 - input redirection no longer supported. Use the load command instead.
+- multilineCommand attribute is no multiline_command
 - submenus now call all hooks, it used to just call precmd and postcmd
 - cmd2 ignores identchars. The standardlibrary cmd uses those characters to split the first "word" of the input, but cmd2 hasn't used those for a while, and the new parsing logic parses on whitespace, which has the added benefit of full unicode support, unlike cmd or prior versions of cmd2.
 - set_posix_shlex function and POSIX_SHLEX variable have been removed. Parsing behavior is now always posix=false.
diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 626769fe5..136d86b2b 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -595,7 +595,7 @@ class Cmd(cmd.Cmd):
     """
     # Attributes used to configure the StatementParser, best not to change these at runtime
     blankLinesAllowed = False
-    multilineCommands = []
+    multiline_commands = []
     redirector = '>'        # for sending output to file
     shortcuts = {'?': 'help', '!': 'shell', '@': 'load', '@@': '_relative_load'}
     aliases = dict()
@@ -689,7 +689,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, persistent_histor
         self.statement_parser = StatementParser(
             allow_redirection=self.allow_redirection,
             terminators=self.terminators,
-            multilineCommands=self.multilineCommands,
+            multiline_commands=self.multiline_commands,
             aliases=self.aliases,
             shortcuts=self.shortcuts,
         )
@@ -2077,7 +2077,7 @@ def _complete_statement(self, line):
         backwards compatibility with the standard library version of cmd.
         """
         statement = self.statement_parser.parse(line)
-        while statement.multilineCommand and not statement.terminator:
+        while statement.multiline_command and not statement.terminator:
             if not self.quit_on_sigint:
                 try:
                     newline = self.pseudo_raw_input(self.continuation_prompt)
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index b4f0e9c17..039378565 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -33,7 +33,7 @@ def __init__(self, obj):
         super().__init__()
         self.raw = str(obj)
         self.command = None
-        self.multilineCommand = None
+        self.multiline_command = None
         # has to be an empty string for compatibility with standard library cmd
         self.args = ''
         self.terminator = None
@@ -56,7 +56,7 @@ def __init__(
             self,
             allow_redirection=True,
             terminators=None,
-            multilineCommands = None,
+            multiline_commands = None,
             aliases = None,
             shortcuts = [],
         ):
@@ -65,10 +65,10 @@ def __init__(
             self.terminators = [';']
         else:
             self.terminators = terminators
-        if multilineCommands is None:
+        if multiline_commands is None:
             self.multilineCommands = []
         else:
-            self.multilineCommands = multilineCommands
+            self.multiline_commands = multiline_commands
         if aliases is None:
             self.aliases = {}
         else:
@@ -167,7 +167,7 @@ def parse(self, rawinput: str) -> Statement:
             tokens = tokens[terminator_pos+1:]
         else:
             (testcommand, testargs) = self._command_and_args(tokens)
-            if testcommand in self.multilineCommands:
+            if testcommand in self.multiline_commands:
                 # no terminator on this line but we have a multiline command
                 # everything else on the line is part of the args
                 # because redirectors can only be after a terminator
@@ -228,10 +228,10 @@ def parse(self, rawinput: str) -> Statement:
                 (command, args) = self._command_and_args(tokens)
 
         # set multiline
-        if command in self.multilineCommands:
-            multilineCommand = command
+        if command in self.multiline_commands:
+            multiline_command = command
         else:
-            multilineCommand = None
+            multiline_command = None
 
         # build Statement object
         result = Statement(args)
@@ -244,7 +244,7 @@ def parse(self, rawinput: str) -> Statement:
         result.outputTo = outputTo
         result.pipeTo = pipeTo
         result.suffix = suffix
-        result.multilineCommand = multilineCommand
+        result.multiline_command = multiline_command
         return result
 
     def parse_command_only(self, rawinput: str) -> Statement:
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index b920b440b..3fc7c1714 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -16,7 +16,7 @@ def parser():
     parser = StatementParser(
         allow_redirection=True,
         terminators = [';'],
-        multilineCommands = ['multiline'],
+        multiline_commands = ['multiline'],
         aliases = {'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline', 'fake': 'pyscript'},
         shortcuts = [('?', 'help'), ('!', 'shell')]
     )
@@ -192,7 +192,7 @@ def test_has_redirect_inside_terminator(parser):
 def test_parse_unfinished_multiliine_command(parser):
     line = 'multiline has > inside an unfinished command'
     statement = parser.parse(line)
-    assert statement.multilineCommand == 'multiline'
+    assert statement.multiline_command == 'multiline'
     assert statement.command == 'multiline'
     assert statement.args == 'has > inside an unfinished command'
     assert not statement.terminator
@@ -200,7 +200,7 @@ def test_parse_unfinished_multiliine_command(parser):
 def test_parse_multiline_command_ignores_redirectors_within_it(parser):
     line = 'multiline has > inside;'
     statement = parser.parse(line)
-    assert statement.multilineCommand == 'multiline'
+    assert statement.multiline_command == 'multiline'
     assert statement.args == 'has > inside'
     assert statement.terminator == ';'
 
@@ -209,28 +209,28 @@ def test_parse_multiline_with_incomplete_comment(parser):
     Un-closed comments effectively comment out everything after the start."""
     line = 'multiline command /* with comment in progress;'
     statement = parser.parse(line)
-    assert statement.multilineCommand == 'multiline'
+    assert statement.multiline_command == 'multiline'
     assert statement.args == 'command'
     assert not statement.terminator
 
 def test_parse_multiline_with_complete_comment(parser):
     line = 'multiline command /* with comment complete */ is done;'
     statement = parser.parse(line)
-    assert statement.multilineCommand == 'multiline'
+    assert statement.multiline_command == 'multiline'
     assert statement.args == 'command is done'
     assert statement.terminator == ';'
 
 def test_parse_multiline_termninated_by_empty_line(parser):
     line = 'multiline command ends\n\n'
     statement = parser.parse(line)
-    assert statement.multilineCommand == 'multiline'
+    assert statement.multiline_command == 'multiline'
     assert statement.args == 'command ends'
     assert statement.terminator == '\n'
 
 def test_parse_multiline_ignores_terminators_in_comments(parser):
     line = 'multiline command "with term; ends" now\n\n'
     statement = parser.parse(line)
-    assert statement.multilineCommand == 'multiline'
+    assert statement.multiline_command == 'multiline'
     assert statement.args == 'command "with term; ends" now'
     assert statement.terminator == '\n'
 
@@ -283,7 +283,7 @@ def test_alias_and_shortcut_expansion(parser, line, command, args):
 def test_alias_on_multiline_command(parser):
     line = 'anothermultiline has > inside an unfinished command'
     statement = parser.parse(line)
-    assert statement.multilineCommand == 'multiline'
+    assert statement.multiline_command == 'multiline'
     assert statement.command == 'multiline'
     assert statement.args == 'has > inside an unfinished command'
     assert not statement.terminator

From 975818feb4f24c25e84b3586cfe68230f1ac84f5 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 10:00:54 -0600
Subject: [PATCH 66/89] _command_and_args switched to static method

---
 cmd2/parsing.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 039378565..63766a8c7 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -316,7 +316,8 @@ def expand_aliases(self, command: str) -> str:
                     break
         return command
 
-    def _command_and_args(self, tokens: List[str]) -> Tuple[str, str]:
+    @staticmethod
+    def _command_and_args(tokens: List[str]) -> Tuple[str, str]:
         """given a list of tokens, and return a tuple of the command
         and the args as a string.
         """

From fbc6d0b39fa1e84ea3de3b38b700c45189146429 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 12:43:11 -0600
Subject: [PATCH 67/89] pipeTo -> pipe_to

---
 cmd2/cmd2.py          |  4 ++--
 cmd2/parsing.py       | 16 ++++++++--------
 tests/test_parsing.py | 18 +++++++++---------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 136d86b2b..985e2b66d 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2114,7 +2114,7 @@ def _redirect_output(self, statement):
 
         :param statement: Statement - a parsed statement from the user
         """
-        if statement.pipeTo:
+        if statement.pipe_to:
             self.kept_state = Statekeeper(self, ('stdout',))
 
             # Create a pipe with read and write sides
@@ -2129,7 +2129,7 @@ def _redirect_output(self, statement):
 
             # We want Popen to raise an exception if it fails to open the process.  Thus we don't set shell to True.
             try:
-                self.pipe_proc = subprocess.Popen(shlex.split(statement.pipeTo), stdin=subproc_stdin)
+                self.pipe_proc = subprocess.Popen(shlex.split(statement.pipe_to), stdin=subproc_stdin)
             except Exception as ex:
                 # Restore stdout to what it was and close the pipe
                 self.stdout.close()
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 63766a8c7..9b548716d 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -38,7 +38,7 @@ def __init__(self, obj):
         self.args = ''
         self.terminator = None
         self.suffix = None
-        self.pipeTo = None
+        self.pipe_to = None
         self.output = None
         self.outputTo = None
 
@@ -56,9 +56,9 @@ def __init__(
             self,
             allow_redirection=True,
             terminators=None,
-            multiline_commands = None,
-            aliases = None,
-            shortcuts = [],
+            multiline_commands=None,
+            aliases=None,
+            shortcuts=[],
         ):
         self.allow_redirection = allow_redirection
         if terminators is None:
@@ -209,13 +209,13 @@ def parse(self, rawinput: str) -> Statement:
         try:
             # find the first pipe if it exists
             pipe_pos = tokens.index('|')
-            # set everything after the first pipe to result.pipeTo
-            pipeTo = ' '.join(tokens[pipe_pos+1:])
+            # set everything after the first pipe to result.pipe_to
+            pipe_to = ' '.join(tokens[pipe_pos+1:])
             # remove all the tokens after the pipe
             tokens = tokens[:pipe_pos]
         except ValueError:
             # no pipe in the tokens
-            pipeTo = None
+            pipe_to = None
 
         if terminator:
             # whatever is left is the suffix
@@ -242,7 +242,7 @@ def parse(self, rawinput: str) -> Statement:
         result.inputFrom = inputFrom
         result.output = output
         result.outputTo = outputTo
-        result.pipeTo = pipeTo
+        result.pipe_to = pipe_to
         result.suffix = suffix
         result.multiline_command = multiline_command
         return result
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 3fc7c1714..01674136b 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -65,7 +65,7 @@ def test_command_with_args(parser):
     statement = parser.parse(line)
     assert statement.command == 'command'
     assert statement.args == 'with args'
-    assert not statement.pipeTo
+    assert not statement.pipe_to
 
 def test_parse_command_with_args_terminator_and_suffix(parser):
     line = 'command with args and terminator; and suffix'
@@ -79,38 +79,38 @@ def test_hashcomment(parser):
     statement = parser.parse('hi # this is all a comment')
     assert statement.command == 'hi'
     assert not statement.args
-    assert not statement.pipeTo
+    assert not statement.pipe_to
 
 def test_c_comment(parser):
     statement = parser.parse('hi /* this is | all a comment */')
     assert statement.command == 'hi'
     assert not statement.args
-    assert not statement.pipeTo
+    assert not statement.pipe_to
 
 def test_c_comment_empty(parser):
     statement = parser.parse('/* this is | all a comment */')
     assert not statement.command
     assert not statement.args
-    assert not statement.pipeTo
+    assert not statement.pipe_to
 
 def test_parse_what_if_quoted_strings_seem_to_start_comments(parser):
     statement = parser.parse('what if "quoted strings /* seem to " start comments?')
     assert statement.command == 'what'
     assert statement.args == 'if "quoted strings /* seem to " start comments?'
-    assert not statement.pipeTo
+    assert not statement.pipe_to
 
 def test_simple_piped(parser):
     statement = parser.parse('simple | piped')
     assert statement.command == 'simple'
     assert not statement.args
-    assert statement.pipeTo == 'piped'
+    assert statement.pipe_to == 'piped'
 
 def test_double_pipe_is_not_a_pipe(parser):
     line = 'double-pipe || is not a pipe'
     statement = parser.parse(line)
     assert statement.command == 'double-pipe'
     assert statement.args == '|| is not a pipe'
-    assert not statement.pipeTo
+    assert not statement.pipe_to
 
 def test_complex_pipe(parser):
     line = 'command with args, terminator;sufx | piped'
@@ -119,7 +119,7 @@ def test_complex_pipe(parser):
     assert statement.args == "with args, terminator"
     assert statement.terminator == ';'
     assert statement.suffix == 'sufx'
-    assert statement.pipeTo == 'piped'
+    assert statement.pipe_to == 'piped'
 
 def test_output_redirect(parser):
     line = 'output into > afile.txt'
@@ -169,7 +169,7 @@ def test_pipe_and_redirect(parser):
     assert statement.args == 'into'
     assert statement.terminator == ';'
     assert statement.suffix == 'sufx'
-    assert statement.pipeTo == 'pipethrume plz'
+    assert statement.pipe_to == 'pipethrume plz'
     assert statement.output == '>'
     assert statement.outputTo == 'afile.txt'
 

From 83fd707dcf9916c9f483e4417a2c3b2b083c8da2 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 12:44:38 -0600
Subject: [PATCH 68/89] outputTo -> output_to

---
 cmd2/cmd2.py          |  8 ++++----
 cmd2/parsing.py       | 10 +++++-----
 tests/test_parsing.py | 10 +++++-----
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 985e2b66d..0c7389914 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2142,16 +2142,16 @@ def _redirect_output(self, statement):
                 # Re-raise the exception
                 raise ex
         elif statement.output:
-            if (not statement.outputTo) and (not can_clip):
+            if (not statement.output_to) and (not can_clip):
                 raise EnvironmentError('Cannot redirect to paste buffer; install ``xclip`` and re-run to enable')
             self.kept_state = Statekeeper(self, ('stdout',))
             self.kept_sys = Statekeeper(sys, ('stdout',))
             self.redirecting = True
-            if statement.outputTo:
+            if statement.output_to:
                 mode = 'w'
                 if statement.output == 2 * self.redirector:
                     mode = 'a'
-                sys.stdout = self.stdout = open(os.path.expanduser(statement.outputTo), mode)
+                sys.stdout = self.stdout = open(os.path.expanduser(statement.output_to), mode)
             else:
                 sys.stdout = self.stdout = tempfile.TemporaryFile(mode="w+")
                 if statement.output == '>>':
@@ -2165,7 +2165,7 @@ def _restore_output(self, statement):
         # If we have redirected output to a file or the clipboard or piped it to a shell command, then restore state
         if self.kept_state is not None:
             # If we redirected output to the clipboard
-            if statement.output and not statement.outputTo:
+            if statement.output and not statement.output_to:
                 self.stdout.seek(0)
                 write_to_paste_buffer(self.stdout.read())
 
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 9b548716d..0ba0736db 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -40,7 +40,7 @@ def __init__(self, obj):
         self.suffix = None
         self.pipe_to = None
         self.output = None
-        self.outputTo = None
+        self.output_to = None
 
     @property
     def command_and_args(self):
@@ -186,11 +186,11 @@ def parse(self, rawinput: str) -> Statement:
 
         # check for output redirect
         output = None
-        outputTo = None
+        output_to = None
         try:
             output_pos = tokens.index('>')
             output = '>'
-            outputTo = ' '.join(tokens[output_pos+1:])
+            output_to = ' '.join(tokens[output_pos+1:])
             # remove all the tokens after the output redirect
             tokens = tokens[:output_pos]
         except ValueError:
@@ -199,7 +199,7 @@ def parse(self, rawinput: str) -> Statement:
         try:
             output_pos = tokens.index('>>')
             output = '>>'
-            outputTo = ' '.join(tokens[output_pos+1:])
+            output_to = ' '.join(tokens[output_pos+1:])
             # remove all tokens after the output redirect
             tokens = tokens[:output_pos]
         except ValueError:
@@ -241,7 +241,7 @@ def parse(self, rawinput: str) -> Statement:
         result.terminator = terminator
         result.inputFrom = inputFrom
         result.output = output
-        result.outputTo = outputTo
+        result.output_to = output_to
         result.pipe_to = pipe_to
         result.suffix = suffix
         result.multiline_command = multiline_command
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 01674136b..cfc5aa762 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -127,7 +127,7 @@ def test_output_redirect(parser):
     assert statement.command == 'output'
     assert statement.args == 'into'
     assert statement.output == '>'
-    assert statement.outputTo == 'afile.txt'
+    assert statement.output_to == 'afile.txt'
 
 def test_output_redirect_with_dash_in_path(parser):
     line = 'output into > python-cmd2/afile.txt'
@@ -135,7 +135,7 @@ def test_output_redirect_with_dash_in_path(parser):
     assert statement.command == 'output'
     assert statement.args == 'into'
     assert statement.output == '>'
-    assert statement.outputTo == 'python-cmd2/afile.txt'
+    assert statement.output_to == 'python-cmd2/afile.txt'
 
 def test_output_redirect_append(parser):
     line = 'output appended to >> /tmp/afile.txt'
@@ -143,7 +143,7 @@ def test_output_redirect_append(parser):
     assert statement.command == 'output'
     assert statement.args == 'appended to'
     assert statement.output == '>>'
-    assert statement.outputTo == '/tmp/afile.txt'
+    assert statement.output_to == '/tmp/afile.txt'
 
 def test_parse_input_redirect(parser):
     line = '< afile.txt'
@@ -171,7 +171,7 @@ def test_pipe_and_redirect(parser):
     assert statement.suffix == 'sufx'
     assert statement.pipe_to == 'pipethrume plz'
     assert statement.output == '>'
-    assert statement.outputTo == 'afile.txt'
+    assert statement.output_to == 'afile.txt'
 
 def test_parse_output_to_paste_buffer(parser):
     line = 'output to paste buffer >> '
@@ -252,7 +252,7 @@ def test_parse_redirect_to_unicode_filename(parser):
     assert statement.command == 'dir'
     assert statement.args == 'home'
     assert statement.output == '>'
-    assert statement.outputTo == 'café'
+    assert statement.output_to == 'café'
 
 def test_parse_input_redirect_from_unicode_filename(parser):
     line = '< café'

From c7fa6969ffde1247122a714b597d088ca6af9166 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 13:03:05 -0600
Subject: [PATCH 69/89] Fix incorrect error message

---
 cmd2/cmd2.py       | 2 +-
 tests/test_cmd2.py | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 0c7389914..360743ede 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2719,7 +2719,7 @@ def show(self, args, parameter):
             if args.all:
                 self.poutput('\nRead only settings:{}'.format(self.cmdenvironment()))
         else:
-            raise LookupError("Parameter '%s' not supported (type 'show' for list of parameters)." % param)
+            raise LookupError("Parameter '%s' not supported (type 'set' for list of parameters)." % param)
 
     set_parser = ACArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
     set_parser.add_argument('-a', '--all', action='store_true', help='display read-only settings as well')
diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py
index 9ec7f642f..17760d4d2 100644
--- a/tests/test_cmd2.py
+++ b/tests/test_cmd2.py
@@ -104,7 +104,6 @@ def test_base_show_readonly(base_app):
         Commands may be terminated with: {}
         Arguments at invocation allowed: {}
         Output redirection and pipes allowed: {}
-            
 """.format(base_app.terminators, base_app.allow_cli_args, base_app.allow_redirection))
     assert out == expected
 
@@ -167,7 +166,7 @@ def test_set_not_supported(base_app, capsys):
     run_cmd(base_app, 'set qqq True')
     out, err = capsys.readouterr()
     expected = normalize("""
-EXCEPTION of type 'LookupError' occurred with message: 'Parameter 'qqq' not supported (type 'show' for list of parameters).'
+EXCEPTION of type 'LookupError' occurred with message: 'Parameter 'qqq' not supported (type 'set' for list of parameters).'
 To enable full traceback, run the following command:  'set debug true'
 """)
     assert normalize(str(err)) == expected
@@ -1384,7 +1383,7 @@ def test_which_editor_bad():
 
 class MultilineApp(cmd2.Cmd):
     def __init__(self, *args, **kwargs):
-        self.multilineCommands = ['orate']
+        self.multiline_commands = ['orate']
         super().__init__(*args, **kwargs)
 
     orate_parser = argparse.ArgumentParser()

From 00093dd2cff5b0532c9c6154b396acf49043df66 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 15:26:51 -0600
Subject: [PATCH 70/89] Remove trailing newline from cmdenvironment()

---
 cmd2/cmd2.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 360743ede..5419a6fa5 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -2694,9 +2694,8 @@ def cmdenvironment(self):
         read_only_settings = """
         Commands may be terminated with: {}
         Arguments at invocation allowed: {}
-        Output redirection and pipes allowed: {}
-        """.format(str(self.terminators), self.allow_cli_args, self.allow_redirection)
-        return read_only_settings
+        Output redirection and pipes allowed: {}"""
+        return read_only_settings.format(str(self.terminators), self.allow_cli_args, self.allow_redirection)
 
     def show(self, args, parameter):
         param = ''

From 7b2d8a23b978f408cc1fe949e23c0aae97ed54a3 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 15:29:54 -0600
Subject: [PATCH 71/89] multilineCommands -> multiline_commands

---
 README.md                    |  4 ++--
 SHLEX_TODO.txt               |  2 +-
 cmd2/parsing.py              | 18 +++++++++---------
 docs/settingchanges.rst      |  2 +-
 docs/unfreefeatures.rst      |  2 +-
 examples/arg_print.py        |  2 +-
 examples/argparse_example.py |  2 +-
 examples/example.py          |  2 +-
 examples/pirate.py           |  2 +-
 tests/test_transcript.py     |  4 ++--
 10 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index f456694c7..beca75817 100755
--- a/README.md
+++ b/README.md
@@ -91,7 +91,7 @@ Instructions for implementing each feature follow.
 
 - Multi-line commands
 
-    Any command accepts multi-line input when its name is listed in `Cmd.multilineCommands`.
+    Any command accepts multi-line input when its name is listed in `Cmd.multiline_commands`.
     The program will keep expecting input until a line ends with any of the characters
     in `Cmd.terminators` .  The default terminators are `;` and `/n` (empty newline).
 
@@ -165,7 +165,7 @@ class CmdLineApp(cmd2.Cmd):
     MUMBLE_LAST = ['right?']
 
     def __init__(self):
-        self.multilineCommands = ['orate']
+        self.multiline_commands = ['orate']
         self.maxrepeats = 3
 
         # Add stuff to settable and shortcuts before calling base class initializer
diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index b8526afa7..761debee3 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -12,7 +12,7 @@ Changelog Items:
 - if self.default_to_shell is true, then redirection and piping are now properly passed to the shell, previously it was truncated
 - object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything. The parser is now self.statement_parser instead of self.command_parser.
 - input redirection no longer supported. Use the load command instead.
-- multilineCommand attribute is no multiline_command
+- multilineCommand attribute is now multiline_command
 - submenus now call all hooks, it used to just call precmd and postcmd
 - cmd2 ignores identchars. The standardlibrary cmd uses those characters to split the first "word" of the input, but cmd2 hasn't used those for a while, and the new parsing logic parses on whitespace, which has the added benefit of full unicode support, unlike cmd or prior versions of cmd2.
 - set_posix_shlex function and POSIX_SHLEX variable have been removed. Parsing behavior is now always posix=false.
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 0ba0736db..75bbd1c44 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -66,7 +66,7 @@ def __init__(
         else:
             self.terminators = terminators
         if multiline_commands is None:
-            self.multilineCommands = []
+            self.multiline_commands = []
         else:
             self.multiline_commands = multiline_commands
         if aliases is None:
@@ -133,9 +133,9 @@ def parse(self, rawinput: str) -> Statement:
             terminator = BLANK_LINE
 
         # split the input on whitespace
-        s = shlex.shlex(line, posix=False)
-        s.whitespace_split = True
-        tokens = self.split_on_punctuation(list(s))
+        lexer = shlex.shlex(line, posix=False)
+        lexer.whitespace_split = True
+        tokens = self.split_on_punctuation(list(lexer))
 
         # expand aliases
         if tokens:
@@ -268,9 +268,9 @@ def parse_command_only(self, rawinput: str) -> Statement:
         line = self.expand_shortcuts(line)
 
         # split the input on whitespace
-        s = shlex.shlex(line, posix=False)
-        s.whitespace_split = True
-        tokens = self.split_on_punctuation(list(s))
+        lexer = shlex.shlex(line, posix=False)
+        lexer.whitespace_split = True
+        tokens = self.split_on_punctuation(list(lexer))
 
         # expand aliases
         if tokens:
@@ -304,7 +304,7 @@ def expand_aliases(self, command: str) -> str:
         """Given a command, expand any aliases for the command"""
         # make a copy of aliases so we can edit it
         tmp_aliases = list(self.aliases.keys())
-        keep_expanding = len(tmp_aliases) > 0
+        keep_expanding = bool(tmp_aliases)
 
         while keep_expanding:
             for cur_alias in tmp_aliases:
@@ -312,7 +312,7 @@ def expand_aliases(self, command: str) -> str:
                 if command == cur_alias:
                     command = self.aliases[cur_alias]
                     tmp_aliases.remove(cur_alias)
-                    keep_expanding = len(tmp_aliases) > 0
+                    keep_expanding = bool(tmp_aliases)
                     break
         return command
 
diff --git a/docs/settingchanges.rst b/docs/settingchanges.rst
index 539bbc9a7..f005fb492 100644
--- a/docs/settingchanges.rst
+++ b/docs/settingchanges.rst
@@ -44,7 +44,7 @@ To define more shortcuts, update the dict ``App.shortcuts`` with the
   Shortcuts need to be created by updating the ``shortcuts`` dictionary attribute prior to calling the
   ``cmd2.Cmd`` super class ``__init__()`` method.  Moreover, that super class init method needs to be called after
   updating the ``shortcuts`` attribute  This warning applies in general to many other attributes which are not
-  settable at runtime such as ``commentGrammars``, ``multilineCommands``, etc.
+  settable at runtime such as ``multiline_commands``, etc.
 
 
 Aliases
diff --git a/docs/unfreefeatures.rst b/docs/unfreefeatures.rst
index bcf0fa41a..16d0eb088 100644
--- a/docs/unfreefeatures.rst
+++ b/docs/unfreefeatures.rst
@@ -7,7 +7,7 @@ Multiline commands
 
 Command input may span multiple lines for the
 commands whose names are listed in the
-parameter ``app.multilineCommands``.  These
+parameter ``app.multiline_commands``.  These
 commands will be executed only
 after the user has entered a *terminator*.
 By default, the command terminators is
diff --git a/examples/arg_print.py b/examples/arg_print.py
index 18fa483f1..bd8ff6bea 100755
--- a/examples/arg_print.py
+++ b/examples/arg_print.py
@@ -30,7 +30,7 @@ def __init__(self):
         # Make sure to call this super class __init__ *after* setting commentGrammars and/or updating shortcuts
         super().__init__()
         # NOTE: It is critical that the super class __init__ method be called AFTER updating certain parameters which
-        # are not settable at runtime.  This includes the commentGrammars, shortcuts, multilineCommands, etc.
+        # are not settable at runtime.  This includes the shortcuts, multiline_commands, etc.
 
     def do_aprint(self, arg):
         """Print the argument string this basic command is called with."""
diff --git a/examples/argparse_example.py b/examples/argparse_example.py
index e9b377baa..3d436323f 100755
--- a/examples/argparse_example.py
+++ b/examples/argparse_example.py
@@ -20,7 +20,7 @@
 class CmdLineApp(Cmd):
     """ Example cmd2 application. """
     def __init__(self, ip_addr=None, port=None, transcript_files=None):
-        self.multilineCommands = ['orate']
+        self.multiline_commands = ['orate']
         self.shortcuts.update({'&': 'speak'})
         self.maxrepeats = 3
 
diff --git a/examples/example.py b/examples/example.py
index 612d81e55..35e2c49f4 100755
--- a/examples/example.py
+++ b/examples/example.py
@@ -27,7 +27,7 @@ class CmdLineApp(Cmd):
     MUMBLE_LAST = ['right?']
 
     def __init__(self):
-        self.multilineCommands = ['orate']
+        self.multiline_commands = ['orate']
         self.maxrepeats = 3
 
         # Add stuff to settable and shortcuts before calling base class initializer
diff --git a/examples/pirate.py b/examples/pirate.py
index 7fe3884b0..bcf9a3687 100755
--- a/examples/pirate.py
+++ b/examples/pirate.py
@@ -14,7 +14,7 @@ class Pirate(Cmd):
     """A piratical example cmd2 application involving looting and drinking."""
     def __init__(self):
         self.default_to_shell = True
-        self.multilineCommands = ['sing']
+        self.multiline_commands = ['sing']
         self.terminators = Cmd.terminators + ['...']
         self.songcolor = 'blue'
 
diff --git a/tests/test_transcript.py b/tests/test_transcript.py
index bba56cab4..6330ab099 100644
--- a/tests/test_transcript.py
+++ b/tests/test_transcript.py
@@ -24,7 +24,7 @@ class CmdLineApp(cmd2.Cmd):
     MUMBLE_LAST = ['right?']
 
     def __init__(self, *args, **kwargs):
-        self.multilineCommands = ['orate']
+        self.multiline_commands = ['orate']
         self.maxrepeats = 3
         self.redirector = '->'
 
@@ -125,7 +125,7 @@ def test_base_with_transcript(_cmdline_app):
 
 Documented commands (type help <topic>):
 ========================================
-alias  help     load    orate  pyscript  say  shell      speak  
+alias  help     load    orate  pyscript  say  shell      speak
 edit   history  mumble  py     quit      set  shortcuts  unalias
 
 (Cmd) help say

From 39e74f6b60b8701b14da29693f909ab407242f73 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 15:33:18 -0600
Subject: [PATCH 72/89] pylint cleanups

---
 tests/test_parsing.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index cfc5aa762..eb26c47dc 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -5,20 +5,20 @@
 Copyright 2017 Todd Leonhardt <todd.leonhardt@gmail.com>
 Released under MIT license, see LICENSE file
 """
+import pytest
+
 import cmd2
 from cmd2.parsing import StatementParser
 
-import pytest
-
 
 @pytest.fixture
 def parser():
     parser = StatementParser(
         allow_redirection=True,
-        terminators = [';'],
-        multiline_commands = ['multiline'],
-        aliases = {'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline', 'fake': 'pyscript'},
-        shortcuts = [('?', 'help'), ('!', 'shell')]
+        terminators=[';'],
+        multiline_commands=['multiline'],
+        aliases={'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline', 'fake': 'pyscript'},
+        shortcuts=[('?', 'help'), ('!', 'shell')]
     )
     return parser
 
@@ -29,9 +29,9 @@ def test_parse_empty_string(parser):
     assert statement.raw == ''
 
 @pytest.mark.parametrize('tokens,command,args', [
-    ( [], None, ''),
-    ( ['command'], 'command', '' ),
-    ( ['command', 'arg1', 'arg2'], 'command', 'arg1 arg2')
+    ([], None, ''),
+    (['command'], 'command', ''),
+    (['command', 'arg1', 'arg2'], 'command', 'arg1 arg2')
 ])
 def test_command_and_args(parser, tokens, command, args):
     (parsed_command, parsed_args) = parser._command_and_args(tokens)

From 1297be25d68b0c8e511b819c906f0d5e0a942606 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 16:09:54 -0600
Subject: [PATCH 73/89] Really get rid of the inputFrom stuff, including
 documentation

---
 README.md             |  2 +-
 cmd2/parsing.py       | 10 ----------
 setup.py              |  8 ++++----
 tests/test_parsing.py | 22 ----------------------
 4 files changed, 5 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index beca75817..e58d912c7 100755
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@ Main Features
 - Python scripting of your application with ``pyscript``
 - Run shell commands with ``!``
 - Pipe command output to shell commands with `|`
-- Redirect command output to file with `>`, `>>`; input from file with `<`
+- Redirect command output to file with `>`, `>>`
 - Bare `>`, `>>` with no filename send output to paste buffer (clipboard)
 - `py` enters interactive Python console (opt-in `ipy` for IPython console)
 - Option to display long output using a pager with ``cmd2.Cmd.ppaged()``
diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 75bbd1c44..a6e67096a 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -175,15 +175,6 @@ def parse(self, rawinput: str) -> Statement:
                 args = testargs
                 tokens = []
 
-        # check for input from file
-        inputFrom = None
-        try:
-            input_pos = tokens.index('<')
-            inputFrom = ' '.join(tokens[input_pos+1:])
-            tokens = tokens[:input_pos]
-        except ValueError:
-            pass
-
         # check for output redirect
         output = None
         output_to = None
@@ -239,7 +230,6 @@ def parse(self, rawinput: str) -> Statement:
         result.command = command
         result.args = args
         result.terminator = terminator
-        result.inputFrom = inputFrom
         result.output = output
         result.output_to = output_to
         result.pipe_to = pipe_to
diff --git a/setup.py b/setup.py
index d5043ce6d..b776e99f0 100755
--- a/setup.py
+++ b/setup.py
@@ -10,9 +10,9 @@
 
 VERSION = '0.9.0'
 DESCRIPTION = "cmd2 - a tool for building interactive command line applications in Python"
-LONG_DESCRIPTION = """cmd2 is a tool for building interactive command line applications in Python. Its goal is to make 
-it quick and easy for developers to build feature-rich and user-friendly interactive command line applications.  It 
-provides a simple API which is an extension of Python's built-in cmd module.  cmd2 provides a wealth of features on top 
+LONG_DESCRIPTION = """cmd2 is a tool for building interactive command line applications in Python. Its goal is to make
+it quick and easy for developers to build feature-rich and user-friendly interactive command line applications.  It
+provides a simple API which is an extension of Python's built-in cmd module.  cmd2 provides a wealth of features on top
 of cmd to make your life easier and eliminates much of the boilerplate code which would be necessary when using cmd.
 
 The latest documentation for cmd2 can be read online here:
@@ -25,7 +25,7 @@
     - Python scripting of your application with ``pyscript``
     - Run shell commands with ``!``
     - Pipe command output to shell commands with `|`
-    - Redirect command output to file with `>`, `>>`; input from file with `<`
+    - Redirect command output to file with `>`, `>>`
     - Bare `>`, `>>` with no filename send output to paste buffer (clipboard)
     - `py` enters interactive Python console (opt-in `ipy` for IPython console)
     - Multi-line commands
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index eb26c47dc..4ca6838e0 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -145,23 +145,6 @@ def test_output_redirect_append(parser):
     assert statement.output == '>>'
     assert statement.output_to == '/tmp/afile.txt'
 
-def test_parse_input_redirect(parser):
-    line = '< afile.txt'
-    statement = parser.parse(line)
-    assert statement.inputFrom == 'afile.txt'
-
-def test_parse_input_redirect_after_command(parser):
-    line = 'help < afile.txt'
-    statement = parser.parse(line)
-    assert statement.command == 'help'
-    assert statement.args == ''
-    assert statement.inputFrom == 'afile.txt'
-
-def test_parse_input_redirect_with_dash_in_path(parser):
-    line = '< python-cmd2/afile.txt'
-    statement = parser.parse(line)
-    assert statement.inputFrom == 'python-cmd2/afile.txt'
-
 def test_pipe_and_redirect(parser):
     line = 'output into;sufx | pipethrume plz > afile.txt'
     statement = parser.parse(line)
@@ -254,11 +237,6 @@ def test_parse_redirect_to_unicode_filename(parser):
     assert statement.output == '>'
     assert statement.output_to == 'café'
 
-def test_parse_input_redirect_from_unicode_filename(parser):
-    line = '< café'
-    statement = parser.parse(line)
-    assert statement.inputFrom == 'café'
-
 def test_empty_statement_raises_exception():
     app = cmd2.Cmd()
     with pytest.raises(cmd2.cmd2.EmptyStatement):

From 1097de13a26cab91a3e35bcdba801c19de07b06b Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 16:12:31 -0600
Subject: [PATCH 74/89] Clean up documentation references to pyparsing

---
 docs/freefeatures.rst   |  8 ++-----
 docs/unfreefeatures.rst | 48 +++++++++--------------------------------
 examples/arg_print.py   |  5 +----
 3 files changed, 13 insertions(+), 48 deletions(-)

diff --git a/docs/freefeatures.rst b/docs/freefeatures.rst
index a7a112fcc..b3f038b01 100644
--- a/docs/freefeatures.rst
+++ b/docs/freefeatures.rst
@@ -32,11 +32,7 @@ Comments
 Comments are omitted from the argument list
 before it is passed to a ``do_`` method.  By
 default, both Python-style and C-style comments
-are recognized; you may change this by overriding
-``app.commentGrammars`` with a different pyparsing_
-grammar (see the arg_print_ example for specifically how to to this).
-
-Comments can be useful in :ref:`scripts`, but would
+are recognized. Comments can be useful in :ref:`scripts`, but would
 be pointless within an interactive session.
 
 ::
@@ -49,7 +45,6 @@ be pointless within an interactive session.
   (Cmd) speak it was /* not */ delicious! # Yuck!
   it was  delicious!
 
-.. _pyparsing: http://pyparsing.wikispaces.com/
 .. _arg_print: https://github.com/python-cmd2/cmd2/blob/master/examples/arg_print.py
 
 Startup Initialization Script
@@ -102,6 +97,7 @@ Output redirection
 As in a Unix shell, output of a command can be redirected:
 
   - sent to a file with ``>``, as in ``mycommand args > filename.txt``
+  - appended to a file with ``>>``, as in ``mycommand args >> filename.txt``
   - piped (``|``) as input to operating-system commands, as in
     ``mycommand args | wc``
   - sent to the paste buffer, ready for the next Copy operation, by
diff --git a/docs/unfreefeatures.rst b/docs/unfreefeatures.rst
index 16d0eb088..6f22c1e06 100644
--- a/docs/unfreefeatures.rst
+++ b/docs/unfreefeatures.rst
@@ -22,11 +22,9 @@ Parsed statements
 =================
 
 ``cmd2`` passes ``arg`` to a ``do_`` method (or
-``default``) as a ParsedString, a subclass of
-string that includes an attribute ``parsed``.
-``parsed`` is a ``pyparsing.ParseResults``
-object produced by applying a pyparsing_
-grammar applied to ``arg``.  It may include:
+``default``) as a Statement, a subclass of
+string that includes many attributes of the parsed
+input:
 
 command
     Name of the command called
@@ -37,47 +35,21 @@ raw
 terminator
     Character used to end a multiline command
 
-suffix
-    Remnant of input after terminator
+command_and_args
+    A string of just the command and the arguments, with
+    output redirection or piping to shell commands removed
 
-::
-
-    def do_parsereport(self, arg):
-        self.stdout.write(arg.parsed.dump() + '\n')
+If ``Statement`` does not contain an attribute,
+querying for it will return ``None``.
 
-::
-
-    (Cmd) parsereport A B /* C */ D; E
-    ['parsereport', 'A B  D', ';', 'E']
-    - args: A B  D
-    - command: parsereport
-    - raw: parsereport A B /* C */ D; E
-    - statement: ['parsereport', 'A B  D', ';']
-        - args: A B  D
-        - command: parsereport
-        - terminator: ;
-    - suffix: E
-    - terminator: ;
-
-If ``parsed`` does not contain an attribute,
-querying for it will return ``None``.  (This
-is a characteristic of ``pyparsing.ParseResults``.)
-
-The parsing grammar and process currently employed
-by cmd2 is stable, but is likely significantly more
-complex than it needs to be.  Future ``cmd2`` releases may
-change it somewhat (hopefully reducing complexity).
-
-(Getting ``arg`` as a ``ParsedString`` is
+(Getting ``arg`` as a ``Statement`` is
 technically "free", in that it requires no application
 changes from the cmd_ standard, but there will
 be no result unless you change your application
-to *use* ``arg.parsed``.)
+to *use* any of the additional attributes.)
 
 .. _cmd: https://docs.python.org/3/library/cmd.html
 
-.. _pyparsing: http://pyparsing.wikispaces.com/
-
 
 Environment parameters
 ======================
diff --git a/examples/arg_print.py b/examples/arg_print.py
index bd8ff6bea..95e8ff015 100755
--- a/examples/arg_print.py
+++ b/examples/arg_print.py
@@ -21,13 +21,10 @@ class ArgumentAndOptionPrinter(cmd2.Cmd):
     """ Example cmd2 application where we create commands that just print the arguments they are called with."""
 
     def __init__(self):
-        # Uncomment this line to disable Python-style comments but still allow C-style comments
-        # self.commentGrammars = pyparsing.Or([pyparsing.cStyleComment])
-
         # Create command aliases which are shorter
         self.shortcuts.update({'$': 'aprint', '%': 'oprint'})
 
-        # Make sure to call this super class __init__ *after* setting commentGrammars and/or updating shortcuts
+        # Make sure to call this super class __init__ *after* setting and/or updating shortcuts
         super().__init__()
         # NOTE: It is critical that the super class __init__ method be called AFTER updating certain parameters which
         # are not settable at runtime.  This includes the shortcuts, multiline_commands, etc.

From ce6fab8e2341359a0f0f2ac97b4d39797907bb22 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 16:12:42 -0600
Subject: [PATCH 75/89] More todos

---
 SHLEX_TODO.txt | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index 761debee3..2aabef323 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -4,6 +4,10 @@ Notes on conversion from pyparsing to shlex taking place in the ply branch
 Todo List:
 - refactor Cmd2.parseline() to use StatementParser.parse()
 - refactor tab completion to use StatementParser instead of parseline()
+- self.redirector doesn't really work any more, either make it work or expire it
+- clarify Statement.args whether it should be None or '', and whether it should be a string or a list of arguments,
+  include verifying documentation in unfreefeatures.txt
+- make sure Statement.command_and_args can handle quoted string arguments well. It might have to return a list of args, not a string
 - delete SHLEX_TODO.txt once everything is done
 
 Questions:
@@ -11,6 +15,7 @@ Questions:
 Changelog Items:
 - if self.default_to_shell is true, then redirection and piping are now properly passed to the shell, previously it was truncated
 - object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything. The parser is now self.statement_parser instead of self.command_parser.
+- self.commentGrammers is no longer supported or available. Comments are C-style or python style.
 - input redirection no longer supported. Use the load command instead.
 - multilineCommand attribute is now multiline_command
 - submenus now call all hooks, it used to just call precmd and postcmd

From 3a7e713ad191bf9763d04c907d4fb27a0d52208b Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 16:34:52 -0600
Subject: [PATCH 76/89] Add new unit test

---
 tests/test_parsing.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 4ca6838e0..8c026631b 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -299,3 +299,10 @@ def test_parse_command_only_expands_shortcuts(parser):
     assert statement.command == 'shell'
     assert statement.args == 'cat foobar.txt'
     assert statement.command_and_args == line.replace('!', 'shell ')
+
+def test_parse_command_only_quoted_args(parser):
+    line = 'shell "/tmp/directory with spaces/doit.sh"'
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'shell'
+    assert statement.args == '"/tmp/directory with spaces/doit.sh"'
+    assert statement.command_and_args == line

From efce15990f2485666f38178ca444c786ac05901d Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 16:54:36 -0600
Subject: [PATCH 77/89] Add test_command_with_quoted_args()

---
 tests/test_parsing.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 8c026631b..53c941d2b 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -67,6 +67,12 @@ def test_command_with_args(parser):
     assert statement.args == 'with args'
     assert not statement.pipe_to
 
+def test_command_with_quoted_args(parser):
+    line = 'command with "quoted args" and "some not"'
+    statement = parser.parse(line)
+    assert statement.command == 'command'
+    assert statement.args == 'with "quoted args" and "some not"'
+
 def test_parse_command_with_args_terminator_and_suffix(parser):
     line = 'command with args and terminator; and suffix'
     statement = parser.parse(line)

From eecdc5cd3e155e21b65c7e87c4b82832d5232430 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 19:07:05 -0600
Subject: [PATCH 78/89] Missed a mutable argument

---
 cmd2/parsing.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index a6e67096a..c1795f337 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -58,7 +58,7 @@ def __init__(
             terminators=None,
             multiline_commands=None,
             aliases=None,
-            shortcuts=[],
+            shortcuts=None,
         ):
         self.allow_redirection = allow_redirection
         if terminators is None:

From 4b903e0cc868a5410691f7c655efbad9d427124f Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 19:09:20 -0600
Subject: [PATCH 79/89] Add some documentation

---
 cmd2/parsing.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index c1795f337..3c56bcc60 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -28,6 +28,31 @@ class Statement(str):
 
     The string portion of the class contains the arguments, but not the command, nor
     the output redirection clauses.
+
+    :var raw:               string containing exactly what we input by the user
+    :type raw:              str
+    :var command:           the command, i.e. the first whitespace delimited word
+    :type command:          str or None
+    :var multiline_command: if the command is a multiline command, the name of the
+                            command, otherwise None
+    :type command:          str or None
+    :var args:              the arguments to the command, not including any output
+                            redirection or terminators. quoted arguments remain
+                            quoted.
+    :type args:             str
+    :var terminator:        the charater which terminated the multiline command, if
+                            there was one
+    :type terminator:       str or None
+    :var suffix:            characters appearing after the terminator but before output
+                            redirection, if any
+    :type suffix:           str or None
+    :var pipe_to:           if output was piped to a shell command, the shell command
+    :type pipe_to:          str or None
+    :var output:            if output was redirected, the redirection token, i.e. '>>'
+    :type output:           str or None
+    :var output_to:         if output was redirected, the destination, usually a filename
+    :type output_to:        str or None
+
     """
     def __init__(self, obj):
         super().__init__()
@@ -44,7 +69,10 @@ def __init__(self, obj):
 
     @property
     def command_and_args(self):
-        """Combine command and args with a space separating them"""
+        """Combine command and args with a space separating them.
+
+        Quoted arguments remain quoted.
+        """
         return '{} {}'.format('' if self.command is None else self.command, self.args).strip()
 
 class StatementParser():

From 0140cf9425d2512935e2c53913efc83d383ba79d Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Sun, 29 Apr 2018 20:05:16 -0600
Subject: [PATCH 80/89] Internal refactoring of parsing and tokenizing code

---
 cmd2/parsing.py | 50 ++++++++++++++++++++++++++++++-------------------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 3c56bcc60..4da8d8145 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -10,13 +10,6 @@
 
 BLANK_LINE = '\n'
 
-def _comment_replacer(match):
-    matched_string = match.group(0)
-    if matched_string.startswith('/'):
-        # the matched string was a comment, so remove it
-        return ''
-    # the matched string was a quoted string, return the match
-    return matched_string
 
 class Statement(str):
     """String subclass with additional attributes to store the results of parsing.
@@ -75,6 +68,7 @@ def command_and_args(self):
         """
         return '{} {}'.format('' if self.command is None else self.command, self.args).strip()
 
+
 class StatementParser():
     """Parse raw text into command components.
 
@@ -135,16 +129,25 @@ def __init__(
             re.DOTALL | re.MULTILINE
         )
 
+    def tokenize(self, line: str) -> List[str]:
+        """Tokenize a string into a list"""
+        lexer = shlex.shlex(line, posix=False)
+        lexer.whitespace_split = True
+        tokens = self._split_on_punctuation(list(lexer))
+        return tokens
+
     def parse(self, rawinput: str) -> Statement:
-        """Parse input into a Statement object, stripping comments, expanding
-        aliases and shortcuts, and extracting output redirection directives.
+        """Tokenize the input and parse it into a Statement object, stripping
+        comments, expanding aliases and shortcuts, and extracting output
+        redirection directives.
         """
         # strip C-style comments
         # shlex will handle the python/shell style comments for us
         # save rawinput for later
-        rawinput = re.sub(self.comment_pattern, _comment_replacer, rawinput)
+        rawinput = re.sub(self.comment_pattern, self._comment_replacer, rawinput)
         # we are going to modify line, so create a copy of the raw input
         line = rawinput
+
         command = None
         args = ''
 
@@ -160,10 +163,7 @@ def parse(self, rawinput: str) -> Statement:
         if line[-1:] == BLANK_LINE:
             terminator = BLANK_LINE
 
-        # split the input on whitespace
-        lexer = shlex.shlex(line, posix=False)
-        lexer.whitespace_split = True
-        tokens = self.split_on_punctuation(list(lexer))
+        tokens = self.tokenize(line)
 
         # expand aliases
         if tokens:
@@ -274,7 +274,7 @@ def parse_command_only(self, rawinput: str) -> Statement:
         # strip C-style comments
         # shlex will handle the python/shell style comments for us
         # save rawinput for later
-        rawinput = re.sub(self.comment_pattern, _comment_replacer, rawinput)
+        rawinput = re.sub(self.comment_pattern, self._comment_replacer, rawinput)
         # we are going to modify line, so create a copy of the raw input
         line = rawinput
         command = None
@@ -288,7 +288,7 @@ def parse_command_only(self, rawinput: str) -> Statement:
         # split the input on whitespace
         lexer = shlex.shlex(line, posix=False)
         lexer.whitespace_split = True
-        tokens = self.split_on_punctuation(list(lexer))
+        tokens = self._split_on_punctuation(list(lexer))
 
         # expand aliases
         if tokens:
@@ -319,7 +319,10 @@ def expand_shortcuts(self, line: str) -> str:
         return line
 
     def expand_aliases(self, command: str) -> str:
-        """Given a command, expand any aliases for the command"""
+        """Given a command, expand any aliases for the command.
+
+        If an alias contains shortcuts, the shortcuts will be expanded too.
+        """
         # make a copy of aliases so we can edit it
         tmp_aliases = list(self.aliases.keys())
         keep_expanding = bool(tmp_aliases)
@@ -332,7 +335,7 @@ def expand_aliases(self, command: str) -> str:
                     tmp_aliases.remove(cur_alias)
                     keep_expanding = bool(tmp_aliases)
                     break
-        return command
+        return self.expand_shortcuts(command)
 
     @staticmethod
     def _command_and_args(tokens: List[str]) -> Tuple[str, str]:
@@ -350,7 +353,16 @@ def _command_and_args(tokens: List[str]) -> Tuple[str, str]:
 
         return (command, args)
 
-    def split_on_punctuation(self, tokens: List[str]) -> List[str]:
+    @staticmethod
+    def _comment_replacer(match):
+        matched_string = match.group(0)
+        if matched_string.startswith('/'):
+            # the matched string was a comment, so remove it
+            return ''
+        # the matched string was a quoted string, return the match
+        return matched_string
+
+    def _split_on_punctuation(self, tokens: List[str]) -> List[str]:
         """
         # Further splits tokens from a command line using punctuation characters
         # as word breaks when they are in unquoted strings. Each run of punctuation

From 119a6e49f1c37ea43059c74f7a2441a275731554 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 30 Apr 2018 00:01:47 -0600
Subject: [PATCH 81/89] Fix nested alias and shortcut expansion

---
 cmd2/parsing.py       | 168 ++++++++++++++++++++----------------------
 tests/test_parsing.py |  24 +++++-
 2 files changed, 100 insertions(+), 92 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 4da8d8145..7046b674a 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -8,7 +8,7 @@
 
 from . import constants
 
-BLANK_LINE = '\n'
+LINE_FEED = '\n'
 
 
 class Statement(str):
@@ -129,10 +129,35 @@ def __init__(
             re.DOTALL | re.MULTILINE
         )
 
+        # aliases have to be a word, so make a regular expression
+        # that matches the first word in the line. This regex has two
+        # parts, the first parenthesis enclosed group matches one
+        # or more non-whitespace characters, and the second group
+        # matches either a whitespace character or the end of the
+        # string. We use \A and \Z to ensure we always match the
+        # beginning and end of a string that may have multiple
+        # lines
+        self.command_pattern = re.compile(r'\A(\S+)(\s|\Z)')
+
+
     def tokenize(self, line: str) -> List[str]:
-        """Tokenize a string into a list"""
+        """Lex a string into a list of tokens.
+
+        Comments are removed, and shortcuts and aliases are expanded.
+        """
+
+        # strip C-style comments
+        # shlex will handle the python/shell style comments for us
+        line = re.sub(self.comment_pattern, self._comment_replacer, line)
+
+        # expand shortcuts and aliases
+        line = self._expand(line)
+
+        # split on whitespace
         lexer = shlex.shlex(line, posix=False)
         lexer.whitespace_split = True
+
+        # custom lexing
         tokens = self._split_on_punctuation(list(lexer))
         return tokens
 
@@ -141,34 +166,19 @@ def parse(self, rawinput: str) -> Statement:
         comments, expanding aliases and shortcuts, and extracting output
         redirection directives.
         """
-        # strip C-style comments
-        # shlex will handle the python/shell style comments for us
-        # save rawinput for later
-        rawinput = re.sub(self.comment_pattern, self._comment_replacer, rawinput)
-        # we are going to modify line, so create a copy of the raw input
-        line = rawinput
-
-        command = None
-        args = ''
-
-        # expand shortcuts, have to do this first because
-        # a shortcut can expand into multiple tokens, ie '!ls' becomes
-        # 'shell ls'
-        line = self.expand_shortcuts(line)
 
         # handle the special case/hardcoded terminator of a blank line
-        # we have to do this before we shlex on whitespace because it
+        # we have to do this before we tokenize because tokenizing
         # destroys all unquoted whitespace in the input
         terminator = None
-        if line[-1:] == BLANK_LINE:
-            terminator = BLANK_LINE
+        if rawinput[-1:] == LINE_FEED:
+            terminator = LINE_FEED
 
-        tokens = self.tokenize(line)
+        command = None
+        args = ''
 
-        # expand aliases
-        if tokens:
-            command_to_expand = tokens[0]
-            tokens[0] = self.expand_aliases(command_to_expand)
+        # lex the input into a list of tokens
+        tokens = self.tokenize(rawinput)
 
         # of the valid terminators, find the first one to occur in the input
         terminator_pos = len(tokens)+1
@@ -184,7 +194,7 @@ def parse(self, rawinput: str) -> Statement:
                 pass
 
         if terminator:
-            if terminator == BLANK_LINE:
+            if terminator == LINE_FEED:
                 terminator_pos = len(tokens)+1
             else:
                 terminator_pos = tokens.index(terminator)
@@ -228,7 +238,7 @@ def parse(self, rawinput: str) -> Statement:
         try:
             # find the first pipe if it exists
             pipe_pos = tokens.index('|')
-            # set everything after the first pipe to result.pipe_to
+            # save everything after the first pipe
             pipe_to = ' '.join(tokens[pipe_pos+1:])
             # remove all the tokens after the pipe
             tokens = tokens[:pipe_pos]
@@ -252,18 +262,18 @@ def parse(self, rawinput: str) -> Statement:
         else:
             multiline_command = None
 
-        # build Statement object
-        result = Statement(args)
-        result.raw = rawinput
-        result.command = command
-        result.args = args
-        result.terminator = terminator
-        result.output = output
-        result.output_to = output_to
-        result.pipe_to = pipe_to
-        result.suffix = suffix
-        result.multiline_command = multiline_command
-        return result
+        # build the statement
+        statement = Statement(args)
+        statement.raw = rawinput
+        statement.command = command
+        statement.args = args
+        statement.terminator = terminator
+        statement.output = output
+        statement.output_to = output_to
+        statement.pipe_to = pipe_to
+        statement.suffix = suffix
+        statement.multiline_command = multiline_command
+        return statement
 
     def parse_command_only(self, rawinput: str) -> Statement:
         """Partially parse input into a Statement object. The command is
@@ -271,41 +281,42 @@ def parse_command_only(self, rawinput: str) -> Statement:
         Terminators, multiline commands, and output redirection are not
         parsed.
         """
-        # strip C-style comments
-        # shlex will handle the python/shell style comments for us
-        # save rawinput for later
-        rawinput = re.sub(self.comment_pattern, self._comment_replacer, rawinput)
-        # we are going to modify line, so create a copy of the raw input
-        line = rawinput
-        command = None
-        args = ''
-
-        # expand shortcuts, have to do this first because
-        # a shortcut can expand into multiple tokens, ie '!ls' becomes
-        # 'shell ls'
-        line = self.expand_shortcuts(line)
-
-        # split the input on whitespace
-        lexer = shlex.shlex(line, posix=False)
-        lexer.whitespace_split = True
-        tokens = self._split_on_punctuation(list(lexer))
-
-        # expand aliases
-        if tokens:
-            command_to_expand = tokens[0]
-            tokens[0] = self.expand_aliases(command_to_expand)
+        # lex the input into a list of tokens
+        tokens = self.tokenize(rawinput)
 
+        # parse out the command and everything else
         (command, args) = self._command_and_args(tokens)
 
-        # build Statement object
-        result = Statement(args)
-        result.raw = rawinput
-        result.command = command
-        result.args = args
-        return result
+        # build the statement
+        statement = Statement(args)
+        statement.raw = rawinput
+        statement.command = command
+        statement.args = args
+        return statement
+
+    def _expand(self, line: str) -> str:
+        """Expand shortcuts and aliases"""
 
-    def expand_shortcuts(self, line: str) -> str:
-        """Expand shortcuts at the beginning of input."""
+        # expand aliases
+        # make a copy of aliases so we can edit it
+        tmp_aliases = list(self.aliases.keys())
+        keep_expanding = bool(tmp_aliases)
+        while keep_expanding:
+            for cur_alias in tmp_aliases:
+                keep_expanding = False
+                # apply our regex to line
+                match = self.command_pattern.search(line)
+                if match:
+                    # we got a match, extract the command
+                    command = match.group(1)
+                    if command == cur_alias:
+                        # rebuild line with the expanded alias
+                        line = self.aliases[cur_alias] + match.group(2) + line[match.end(2):]
+                        tmp_aliases.remove(cur_alias)
+                        keep_expanding = bool(tmp_aliases)
+                        break
+
+        # expand shortcuts
         for (shortcut, expansion) in self.shortcuts:
             if  line.startswith(shortcut):
                 # If the next character after the shortcut isn't a space, then insert one
@@ -318,25 +329,6 @@ def expand_shortcuts(self, line: str) -> str:
                 break
         return line
 
-    def expand_aliases(self, command: str) -> str:
-        """Given a command, expand any aliases for the command.
-
-        If an alias contains shortcuts, the shortcuts will be expanded too.
-        """
-        # make a copy of aliases so we can edit it
-        tmp_aliases = list(self.aliases.keys())
-        keep_expanding = bool(tmp_aliases)
-
-        while keep_expanding:
-            for cur_alias in tmp_aliases:
-                keep_expanding = False
-                if command == cur_alias:
-                    command = self.aliases[cur_alias]
-                    tmp_aliases.remove(cur_alias)
-                    keep_expanding = bool(tmp_aliases)
-                    break
-        return self.expand_shortcuts(command)
-
     @staticmethod
     def _command_and_args(tokens: List[str]) -> Tuple[str, str]:
         """given a list of tokens, and return a tuple of the command
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 53c941d2b..1e745bda9 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -17,7 +17,11 @@ def parser():
         allow_redirection=True,
         terminators=[';'],
         multiline_commands=['multiline'],
-        aliases={'helpalias': 'help', '42': 'theanswer', 'anothermultiline': 'multiline', 'fake': 'pyscript'},
+        aliases={'helpalias': 'help',
+                 '42': 'theanswer',
+                 'l': '!ls -al',
+                 'anothermultiline': 'multiline',
+                 'fake': 'pyscript'},
         shortcuts=[('?', 'help'), ('!', 'shell')]
     )
     return parser
@@ -28,6 +32,17 @@ def test_parse_empty_string(parser):
     assert not statement.args
     assert statement.raw == ''
 
+@pytest.mark.parametrize('line,tokens', [
+    ('command', ['command']),
+    ('command /* with some comment */ arg', ['command', 'arg']),
+    ('command arg1 arg2 # comment at the end', ['command', 'arg1', 'arg2']),
+    ('42 arg1 arg2', ['theanswer', 'arg1', 'arg2']),
+    ('l', ['shell', 'ls', '-al'])
+])
+def test_tokenize(parser, line, tokens):
+    tokens_to_test = parser.tokenize(line)
+    assert tokens_to_test == tokens
+
 @pytest.mark.parametrize('tokens,command,args', [
     ([], None, ''),
     (['command'], 'command', ''),
@@ -258,6 +273,7 @@ def test_empty_statement_raises_exception():
     ('42 arg1 arg2', 'theanswer', 'arg1 arg2'),
     ('!ls', 'shell', 'ls'),
     ('!ls -al /tmp', 'shell', 'ls -al /tmp'),
+    ('l', 'shell', 'ls -al')
 ])
 def test_alias_and_shortcut_expansion(parser, line, command, args):
     statement = parser.parse(line)
@@ -307,8 +323,8 @@ def test_parse_command_only_expands_shortcuts(parser):
     assert statement.command_and_args == line.replace('!', 'shell ')
 
 def test_parse_command_only_quoted_args(parser):
-    line = 'shell "/tmp/directory with spaces/doit.sh"'
+    line = 'l "/tmp/directory with spaces/doit.sh"'
     statement = parser.parse_command_only(line)
     assert statement.command == 'shell'
-    assert statement.args == '"/tmp/directory with spaces/doit.sh"'
-    assert statement.command_and_args == line
+    assert statement.args == 'ls -al "/tmp/directory with spaces/doit.sh"'
+    assert statement.command_and_args == line.replace('l', 'shell ls -al')

From b6b4ba212abae5a1394448d0ea5aab34158ef64c Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 30 Apr 2018 08:33:25 -0600
Subject: [PATCH 82/89] remove unused file

---
 tests/redirect.txt | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 tests/redirect.txt

diff --git a/tests/redirect.txt b/tests/redirect.txt
deleted file mode 100644
index c375d5ba0..000000000
--- a/tests/redirect.txt
+++ /dev/null
@@ -1 +0,0 @@
-history

From 32363e20947cfa70239234d6ccc8e39cac643aa7 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 30 Apr 2018 16:43:30 -0600
Subject: [PATCH 83/89] extract strip_quotes() to utils module

---
 cmd2/cmd2.py  | 21 +++------------------
 cmd2/utils.py | 12 ++++++++++++
 2 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py
index 5419a6fa5..e99e46594 100755
--- a/cmd2/cmd2.py
+++ b/cmd2/cmd2.py
@@ -147,21 +147,6 @@ def _which(editor: str) -> Optional[str]:
     return editor_path
 
 
-def strip_quotes(arg: str) -> str:
-    """ Strip outer quotes from a string.
-
-     Applies to both single and double quotes.
-
-    :param arg:  string to strip outer quotes from
-    :return: same string with potentially outer quotes stripped
-    """
-    quote_chars = '"' + "'"
-
-    if len(arg) > 1 and arg[0] == arg[-1] and arg[0] in quote_chars:
-        arg = arg[1:-1]
-    return arg
-
-
 def parse_quoted_string(cmdline: str) -> List[str]:
     """Parse a quoted string into a list of arguments."""
     if isinstance(cmdline, list):
@@ -173,7 +158,7 @@ def parse_quoted_string(cmdline: str) -> List[str]:
         # strip off outer quotes for convenience
         temp_arglist = []
         for arg in lexed_arglist:
-            temp_arglist.append(strip_quotes(arg))
+            temp_arglist.append(utils.strip_quotes(arg))
         lexed_arglist = temp_arglist
     return lexed_arglist
 
@@ -1032,7 +1017,7 @@ def tokens_for_completion(self, line, begidx, endidx):
             raw_tokens = initial_tokens
 
         # Save the unquoted tokens
-        tokens = [strip_quotes(cur_token) for cur_token in raw_tokens]
+        tokens = [utils.strip_quotes(cur_token) for cur_token in raw_tokens]
 
         # If the token being completed had an unclosed quote, we need
         # to remove the closing quote that was added in order for it
@@ -2780,7 +2765,7 @@ def do_shell(self, command):
                 # an unclosed quote, so we only need to check the first character.
                 first_char = tokens[index][0]
                 if first_char in constants.QUOTES:
-                    tokens[index] = strip_quotes(tokens[index])
+                    tokens[index] = utils.strip_quotes(tokens[index])
 
                 tokens[index] = os.path.expanduser(tokens[index])
 
diff --git a/cmd2/utils.py b/cmd2/utils.py
index 33215dc02..6abab94c0 100644
--- a/cmd2/utils.py
+++ b/cmd2/utils.py
@@ -11,3 +11,15 @@ def strip_ansi(text: str) -> str:
     :return: the same string with any ANSI escape codes removed
     """
     return constants.ANSI_ESCAPE_RE.sub('', text)
+
+def strip_quotes(arg: str) -> str:
+    """ Strip outer quotes from a string.
+
+     Applies to both single and double quotes.
+
+    :param arg:  string to strip outer quotes from
+    :return: same string with potentially outer quotes stripped
+    """
+    if len(arg) > 1 and arg[0] == arg[-1] and arg[0] in constants.QUOTES:
+        arg = arg[1:-1]
+    return arg

From eab09a82213ce09bd24ffe637a6187decf8fcf01 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 30 Apr 2018 16:48:52 -0600
Subject: [PATCH 84/89] Updated changelog

---
 CHANGELOG.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bb5779940..5823c8df5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,23 @@
 ## 0.9.0 (TBD, 2018)
+* Bug Fixes
+    * If self.default_to_shell is true, then redirection and piping are now properly passed to the shell. Previously it was truncated.
+    * Submenus now call all hooks, it used to just call precmd and postcmd.
 * Enhancements
     * Automatic completion of ``argparse`` arguments via ``cmd2.argparse_completer.AutoCompleter``
         * See the [tab_autocompletion.py](https://github.com/python-cmd2/cmd2/blob/master/examples/tab_autocompletion.py) example for a demonstration of how to use this feature
     * ``cmd2`` no longer depends on the ``six`` module
     * ``cmd2`` is now a multi-file Python package instead of a single-file module
+    * Switch command parsing from pyparsing to custom code which utilizes shlex.
+        * The object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object, which is a subclass of ``str``. The statement object has many attributes which give you access to various components of the parsed input. If you were using anything but the string in your do_* methods, this change will require you to update your code.
+        * ``commentGrammers`` is no longer supported or available. Comments are C-style or python style.
+        * Input redirection no longer supported. Use the load command instead.
+        * ``multilineCommand`` attribute is ``now multiline_command``
+        * ``identchars`` is now ignored. The standardlibrary cmd uses those characters to split the first "word" of the input, but cmd2 hasn't used those for a while, and the new parsing logic parses on whitespace, which has the added benefit of full unicode support, unlike cmd or prior versions of cmd2.
+        * ``set_posix_shlex`` function and ``POSIX_SHLEX`` variable have been removed. Parsing behavior is now always the more forgiving ``posix=false``.
+        * ``set_strip_quotes`` function and ``STRIP_QUOTES_FOR_NON_POSIX`` have been removed. Quotes are stripped from arguments when presented as a list (a la ``sys.argv``), and present when arguments are presented as a string (like the string passed to do_*).
+* Changes
+    * ``strip_ansi()`` and ``strip_quotes()`` functions have moved to new utils module
+    * Several constants moved to new constants module
 * Deletions (potentially breaking changes)
     * Deleted all ``optparse`` code which had previously been deprecated in release 0.8.0
         * The ``options`` decorator no longer exists

From 2bd1c8fcd2e86546cb8a33c38e86dc8119cee671 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Mon, 30 Apr 2018 16:50:55 -0600
Subject: [PATCH 85/89] Remove changelog items from todo list

---
 SHLEX_TODO.txt | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
index 2aabef323..70e439ce8 100644
--- a/SHLEX_TODO.txt
+++ b/SHLEX_TODO.txt
@@ -11,14 +11,3 @@ Todo List:
 - delete SHLEX_TODO.txt once everything is done
 
 Questions:
-
-Changelog Items:
-- if self.default_to_shell is true, then redirection and piping are now properly passed to the shell, previously it was truncated
-- object passed to do_* methods has changed. It no longer is the pyparsing object, it's a new Statement object. A side effect of this is that we now have a clean interface between the parsing logic and the rest of cmd2. If we need to change the parser in the future, we can do it without breaking anything. The parser is now self.statement_parser instead of self.command_parser.
-- self.commentGrammers is no longer supported or available. Comments are C-style or python style.
-- input redirection no longer supported. Use the load command instead.
-- multilineCommand attribute is now multiline_command
-- submenus now call all hooks, it used to just call precmd and postcmd
-- cmd2 ignores identchars. The standardlibrary cmd uses those characters to split the first "word" of the input, but cmd2 hasn't used those for a while, and the new parsing logic parses on whitespace, which has the added benefit of full unicode support, unlike cmd or prior versions of cmd2.
-- set_posix_shlex function and POSIX_SHLEX variable have been removed. Parsing behavior is now always posix=false.
-- set_strip_quotes function and STRIP_QUOTES_FOR_NON_POSIX have been removed. Quotes are always stripped from arguments.

From dbf4846e8bc0e6ca38c928d8fe4752f9b6173803 Mon Sep 17 00:00:00 2001
From: Todd Leonhardt <todd.leonhardt@gmail.com>
Date: Tue, 1 May 2018 20:40:24 -0700
Subject: [PATCH 86/89] Updated setup.py

Changes include:
- Removed support for versions of setuptools prior to 18.0 (dating to early 2015)
    - This removed some extra logic related to conditional dependencies and simplified the imports
- Added a python_requires statement to require Python 3.4 or newer
    - I believe this requires setuptools >= 34.4
---
 setup.py | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/setup.py b/setup.py
index b776e99f0..639afe1ce 100755
--- a/setup.py
+++ b/setup.py
@@ -3,9 +3,6 @@
 """
 Setuptools setup file, used to install or test 'cmd2'
 """
-import sys
-
-import setuptools
 from setuptools import setup
 
 VERSION = '0.9.0'
@@ -72,18 +69,7 @@
     ":python_version<'3.5'": ['contextlib2', 'typing'],
 }
 
-if int(setuptools.__version__.split('.')[0]) < 18:
-    EXTRAS_REQUIRE = {}
-    if sys.platform.startswith('win'):
-        INSTALL_REQUIRES.append('pyreadline')
-    else:
-        INSTALL_REQUIRES.append('wcwidth')
-    if sys.version_info < (3, 5):
-        INSTALL_REQUIRES.append('contextlib2')
-        INSTALL_REQUIRES.append('typing')
-
 TESTS_REQUIRE = ['pytest', 'pytest-xdist']
-DOCS_REQUIRE = ['sphinx', 'sphinx_rtd_theme', 'pyperclip', 'wcwidth']
 
 setup(
     name="cmd2",
@@ -98,6 +84,7 @@
     platforms=['any'],
     packages=['cmd2'],
     keywords='command prompt console cmd',
+    python_requires='>=3.4',
     install_requires=INSTALL_REQUIRES,
     extras_require=EXTRAS_REQUIRE,
     tests_require=TESTS_REQUIRE,

From 802000bc56ba41955cdd6ffa9043cdc715e023d6 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 2 May 2018 09:14:49 -0600
Subject: [PATCH 87/89] =?UTF-8?q?Ensure=20args=20is=20=E2=80=98=E2=80=99?=
 =?UTF-8?q?=20for=20backwards=20compatibility=20with=20cmd?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 cmd2/parsing.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 7046b674a..2af8ff01f 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -32,7 +32,7 @@ class Statement(str):
     :var args:              the arguments to the command, not including any output
                             redirection or terminators. quoted arguments remain
                             quoted.
-    :type args:             str
+    :type args:             str or None
     :var terminator:        the charater which terminated the multiline command, if
                             there was one
     :type terminator:       str or None
@@ -52,8 +52,7 @@ def __init__(self, obj):
         self.raw = str(obj)
         self.command = None
         self.multiline_command = None
-        # has to be an empty string for compatibility with standard library cmd
-        self.args = ''
+        self.args = None
         self.terminator = None
         self.suffix = None
         self.pipe_to = None
@@ -175,7 +174,7 @@ def parse(self, rawinput: str) -> Statement:
             terminator = LINE_FEED
 
         command = None
-        args = ''
+        args = None
 
         # lex the input into a list of tokens
         tokens = self.tokenize(rawinput)
@@ -263,9 +262,13 @@ def parse(self, rawinput: str) -> Statement:
             multiline_command = None
 
         # build the statement
-        statement = Statement(args)
+        # string representation of args must be an empty string instead of
+        # None for compatibility with standard library cmd
+        statement = Statement('' if args is None else args)
         statement.raw = rawinput
         statement.command = command
+        # if there are no args we will use None since we don't have to worry
+        # about compatibility wiht standard library cmd
         statement.args = args
         statement.terminator = terminator
         statement.output = output
@@ -331,8 +334,11 @@ def _expand(self, line: str) -> str:
 
     @staticmethod
     def _command_and_args(tokens: List[str]) -> Tuple[str, str]:
-        """given a list of tokens, and return a tuple of the command
+        """Given a list of tokens, return a tuple of the command
         and the args as a string.
+
+        The args string will be '' instead of None to retain backwards compatibility
+        with cmd in the standard library.
         """
         command = None
         args = ''

From ad634b2e7f68392727246f796647b92d67172011 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 2 May 2018 19:09:30 -0700
Subject: [PATCH 88/89] Add argv to Statement object

---
 cmd2/parsing.py         | 33 ++++++++++++++++++-----
 docs/unfreefeatures.rst | 19 ++++++++++---
 tests/test_parsing.py   | 59 ++++++++++++++++++++++++++++++++++-------
 3 files changed, 93 insertions(+), 18 deletions(-)

diff --git a/cmd2/parsing.py b/cmd2/parsing.py
index 2af8ff01f..908e9272f 100644
--- a/cmd2/parsing.py
+++ b/cmd2/parsing.py
@@ -7,6 +7,7 @@
 from typing import List, Tuple
 
 from . import constants
+from . import utils
 
 LINE_FEED = '\n'
 
@@ -33,6 +34,10 @@ class Statement(str):
                             redirection or terminators. quoted arguments remain
                             quoted.
     :type args:             str or None
+    :var: argv:             a list of arguments a la sys.argv. Quotes, if any, are removed
+                            from the elements of the list, and aliases and shortcuts
+                            are expanded
+    :type argv:             list
     :var terminator:        the charater which terminated the multiline command, if
                             there was one
     :type terminator:       str or None
@@ -53,6 +58,7 @@ def __init__(self, obj):
         self.command = None
         self.multiline_command = None
         self.args = None
+        self.argv = None
         self.terminator = None
         self.suffix = None
         self.pipe_to = None
@@ -65,7 +71,14 @@ def command_and_args(self):
 
         Quoted arguments remain quoted.
         """
-        return '{} {}'.format('' if self.command is None else self.command, self.args).strip()
+        if self.command and self.args:
+            rtn = '{} {}'.format(self.command, self.args)
+        elif self.command:
+            # we are trusting that if we get here that self.args is None
+            rtn = self.command
+        else:
+            rtn = None
+        return rtn
 
 
 class StatementParser():
@@ -175,6 +188,7 @@ def parse(self, rawinput: str) -> Statement:
 
         command = None
         args = None
+        argv = None
 
         # lex the input into a list of tokens
         tokens = self.tokenize(rawinput)
@@ -198,7 +212,8 @@ def parse(self, rawinput: str) -> Statement:
             else:
                 terminator_pos = tokens.index(terminator)
             # everything before the first terminator is the command and the args
-            (command, args) = self._command_and_args(tokens[:terminator_pos])
+            argv = tokens[:terminator_pos]
+            (command, args) = self._command_and_args(argv)
             # we will set the suffix later
             # remove all the tokens before and including the terminator
             tokens = tokens[terminator_pos+1:]
@@ -210,6 +225,7 @@ def parse(self, rawinput: str) -> Statement:
                 # because redirectors can only be after a terminator
                 command = testcommand
                 args = testargs
+                argv = tokens
                 tokens = []
 
         # check for output redirect
@@ -253,7 +269,8 @@ def parse(self, rawinput: str) -> Statement:
             suffix = None
             if not command:
                 # command could already have been set, if so, don't set it again
-                (command, args) = self._command_and_args(tokens)
+                argv = tokens
+                (command, args) = self._command_and_args(argv)
 
         # set multiline
         if command in self.multiline_commands:
@@ -268,8 +285,9 @@ def parse(self, rawinput: str) -> Statement:
         statement.raw = rawinput
         statement.command = command
         # if there are no args we will use None since we don't have to worry
-        # about compatibility wiht standard library cmd
+        # about compatibility with standard library cmd
         statement.args = args
+        statement.argv = list(map(lambda x: utils.strip_quotes(x), argv))
         statement.terminator = terminator
         statement.output = output
         statement.output_to = output_to
@@ -291,10 +309,13 @@ def parse_command_only(self, rawinput: str) -> Statement:
         (command, args) = self._command_and_args(tokens)
 
         # build the statement
-        statement = Statement(args)
+        # string representation of args must be an empty string instead of
+        # None for compatibility with standard library cmd
+        statement = Statement('' if args is None else args)
         statement.raw = rawinput
         statement.command = command
         statement.args = args
+        statement.argv = tokens
         return statement
 
     def _expand(self, line: str) -> str:
@@ -341,7 +362,7 @@ def _command_and_args(tokens: List[str]) -> Tuple[str, str]:
         with cmd in the standard library.
         """
         command = None
-        args = ''
+        args = None
 
         if tokens:
             command = tokens[0]
diff --git a/docs/unfreefeatures.rst b/docs/unfreefeatures.rst
index 6f22c1e06..e69f8a7e5 100644
--- a/docs/unfreefeatures.rst
+++ b/docs/unfreefeatures.rst
@@ -29,15 +29,28 @@ input:
 command
     Name of the command called
 
+args
+    The arguments to the command with output redirection
+    or piping to shell commands removed
+
+command_and_args
+    A string of just the command and the arguments, with
+    output redirection or piping to shell commands removed
+
+argv
+    A list of arguments a-la ``sys.argv``, including
+    the command as ``argv[0]`` and the subsequent
+    arguments as additional items in the list.
+    Quotes around arguments will be stripped as will
+    any output redirection or piping portions of the command
+
 raw
     Full input exactly as typed.
 
 terminator
     Character used to end a multiline command
 
-command_and_args
-    A string of just the command and the arguments, with
-    output redirection or piping to shell commands removed
+
 
 If ``Statement`` does not contain an attribute,
 querying for it will return ``None``.
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 1e745bda9..3f8fc07ca 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -10,6 +10,7 @@
 import cmd2
 from cmd2.parsing import StatementParser
 
+from cmd2 import utils
 
 @pytest.fixture
 def parser():
@@ -44,8 +45,8 @@ def test_tokenize(parser, line, tokens):
     assert tokens_to_test == tokens
 
 @pytest.mark.parametrize('tokens,command,args', [
-    ([], None, ''),
-    (['command'], 'command', ''),
+    ([], None, None),
+    (['command'], 'command', None),
     (['command', 'arg1', 'arg2'], 'command', 'arg1 arg2')
 ])
 def test_command_and_args(parser, tokens, command, args):
@@ -61,12 +62,15 @@ def test_command_and_args(parser, tokens, command, args):
 def test_single_word(parser, line):
     statement = parser.parse(line)
     assert statement.command == line
+    assert not statement.args
+    assert statement.argv == [utils.strip_quotes(line)]
 
 def test_word_plus_terminator(parser):
     line = 'termbare;'
     statement = parser.parse(line)
     assert statement.command == 'termbare'
     assert statement.terminator == ';'
+    assert statement.argv == ['termbare']
 
 def test_suffix_after_terminator(parser):
     line = 'termbare; suffx'
@@ -74,19 +78,21 @@ def test_suffix_after_terminator(parser):
     assert statement.command == 'termbare'
     assert statement.terminator == ';'
     assert statement.suffix == 'suffx'
+    assert statement.argv == ['termbare']
 
 def test_command_with_args(parser):
     line = 'command with args'
     statement = parser.parse(line)
     assert statement.command == 'command'
     assert statement.args == 'with args'
-    assert not statement.pipe_to
+    assert statement.argv == ['command', 'with', 'args']
 
 def test_command_with_quoted_args(parser):
     line = 'command with "quoted args" and "some not"'
     statement = parser.parse(line)
     assert statement.command == 'command'
     assert statement.args == 'with "quoted args" and "some not"'
+    assert statement.argv == ['command', 'with', 'quoted args', 'and', 'some not']
 
 def test_parse_command_with_args_terminator_and_suffix(parser):
     line = 'command with args and terminator; and suffix'
@@ -95,35 +101,40 @@ def test_parse_command_with_args_terminator_and_suffix(parser):
     assert statement.args == "with args and terminator"
     assert statement.terminator == ';'
     assert statement.suffix == 'and suffix'
+    assert statement.argv == ['command', 'with', 'args', 'and', 'terminator']
 
 def test_hashcomment(parser):
     statement = parser.parse('hi # this is all a comment')
     assert statement.command == 'hi'
     assert not statement.args
-    assert not statement.pipe_to
+    assert statement.argv == ['hi']
 
 def test_c_comment(parser):
     statement = parser.parse('hi /* this is | all a comment */')
     assert statement.command == 'hi'
     assert not statement.args
     assert not statement.pipe_to
+    assert statement.argv == ['hi']
 
 def test_c_comment_empty(parser):
     statement = parser.parse('/* this is | all a comment */')
     assert not statement.command
     assert not statement.args
     assert not statement.pipe_to
+    assert not statement.argv
 
 def test_parse_what_if_quoted_strings_seem_to_start_comments(parser):
     statement = parser.parse('what if "quoted strings /* seem to " start comments?')
     assert statement.command == 'what'
     assert statement.args == 'if "quoted strings /* seem to " start comments?'
     assert not statement.pipe_to
+    assert statement.argv == ['what', 'if', 'quoted strings /* seem to ', 'start', 'comments?']
 
 def test_simple_piped(parser):
     statement = parser.parse('simple | piped')
     assert statement.command == 'simple'
     assert not statement.args
+    assert statement.argv == ['simple']
     assert statement.pipe_to == 'piped'
 
 def test_double_pipe_is_not_a_pipe(parser):
@@ -131,6 +142,7 @@ def test_double_pipe_is_not_a_pipe(parser):
     statement = parser.parse(line)
     assert statement.command == 'double-pipe'
     assert statement.args == '|| is not a pipe'
+    assert statement.argv == ['double-pipe', '||', 'is', 'not', 'a', 'pipe']
     assert not statement.pipe_to
 
 def test_complex_pipe(parser):
@@ -138,6 +150,7 @@ def test_complex_pipe(parser):
     statement = parser.parse(line)
     assert statement.command == 'command'
     assert statement.args == "with args, terminator"
+    assert statement.argv == ['command', 'with', 'args,', 'terminator']
     assert statement.terminator == ';'
     assert statement.suffix == 'sufx'
     assert statement.pipe_to == 'piped'
@@ -147,6 +160,7 @@ def test_output_redirect(parser):
     statement = parser.parse(line)
     assert statement.command == 'output'
     assert statement.args == 'into'
+    assert statement.argv == ['output', 'into']
     assert statement.output == '>'
     assert statement.output_to == 'afile.txt'
 
@@ -155,6 +169,7 @@ def test_output_redirect_with_dash_in_path(parser):
     statement = parser.parse(line)
     assert statement.command == 'output'
     assert statement.args == 'into'
+    assert statement.argv == ['output', 'into']
     assert statement.output == '>'
     assert statement.output_to == 'python-cmd2/afile.txt'
 
@@ -163,6 +178,7 @@ def test_output_redirect_append(parser):
     statement = parser.parse(line)
     assert statement.command == 'output'
     assert statement.args == 'appended to'
+    assert statement.argv == ['output', 'appended', 'to']
     assert statement.output == '>>'
     assert statement.output_to == '/tmp/afile.txt'
 
@@ -171,6 +187,7 @@ def test_pipe_and_redirect(parser):
     statement = parser.parse(line)
     assert statement.command == 'output'
     assert statement.args == 'into'
+    assert statement.argv == ['output', 'into']
     assert statement.terminator == ';'
     assert statement.suffix == 'sufx'
     assert statement.pipe_to == 'pipethrume plz'
@@ -182,6 +199,7 @@ def test_parse_output_to_paste_buffer(parser):
     statement = parser.parse(line)
     assert statement.command == 'output'
     assert statement.args == 'to paste buffer'
+    assert statement.argv == ['output', 'to', 'paste', 'buffer']
     assert statement.output == '>>'
 
 def test_has_redirect_inside_terminator(parser):
@@ -191,6 +209,7 @@ def test_has_redirect_inside_terminator(parser):
     statement = parser.parse(line)
     assert statement.command == 'has'
     assert statement.args == '> inside'
+    assert statement.argv == ['has', '>', 'inside']
     assert statement.terminator == ';'
 
 def test_parse_unfinished_multiliine_command(parser):
@@ -199,6 +218,7 @@ def test_parse_unfinished_multiliine_command(parser):
     assert statement.multiline_command == 'multiline'
     assert statement.command == 'multiline'
     assert statement.args == 'has > inside an unfinished command'
+    assert statement.argv == ['multiline', 'has', '>', 'inside', 'an', 'unfinished', 'command']
     assert not statement.terminator
 
 def test_parse_multiline_command_ignores_redirectors_within_it(parser):
@@ -206,6 +226,7 @@ def test_parse_multiline_command_ignores_redirectors_within_it(parser):
     statement = parser.parse(line)
     assert statement.multiline_command == 'multiline'
     assert statement.args == 'has > inside'
+    assert statement.argv == ['multiline', 'has', '>', 'inside']
     assert statement.terminator == ';'
 
 def test_parse_multiline_with_incomplete_comment(parser):
@@ -214,28 +235,36 @@ def test_parse_multiline_with_incomplete_comment(parser):
     line = 'multiline command /* with comment in progress;'
     statement = parser.parse(line)
     assert statement.multiline_command == 'multiline'
+    assert statement.command == 'multiline'
     assert statement.args == 'command'
+    assert statement.argv == ['multiline', 'command']
     assert not statement.terminator
 
 def test_parse_multiline_with_complete_comment(parser):
     line = 'multiline command /* with comment complete */ is done;'
     statement = parser.parse(line)
     assert statement.multiline_command == 'multiline'
+    assert statement.command == 'multiline'
     assert statement.args == 'command is done'
+    assert statement.argv == ['multiline', 'command', 'is', 'done']
     assert statement.terminator == ';'
 
 def test_parse_multiline_termninated_by_empty_line(parser):
     line = 'multiline command ends\n\n'
     statement = parser.parse(line)
     assert statement.multiline_command == 'multiline'
+    assert statement.command == 'multiline'
     assert statement.args == 'command ends'
+    assert statement.argv == ['multiline', 'command', 'ends']
     assert statement.terminator == '\n'
 
 def test_parse_multiline_ignores_terminators_in_comments(parser):
     line = 'multiline command "with term; ends" now\n\n'
     statement = parser.parse(line)
     assert statement.multiline_command == 'multiline'
+    assert statement.command == 'multiline'
     assert statement.args == 'command "with term; ends" now'
+    assert statement.argv == ['multiline', 'command', 'with term; ends', 'now']
     assert statement.terminator == '\n'
 
 def test_parse_command_with_unicode_args(parser):
@@ -243,18 +272,21 @@ def test_parse_command_with_unicode_args(parser):
     statement = parser.parse(line)
     assert statement.command == 'drink'
     assert statement.args == 'café'
+    assert statement.argv == ['drink', 'café']
 
 def test_parse_unicode_command(parser):
     line = 'café au lait'
     statement = parser.parse(line)
     assert statement.command == 'café'
     assert statement.args == 'au lait'
+    assert statement.argv == ['café', 'au', 'lait']
 
 def test_parse_redirect_to_unicode_filename(parser):
     line = 'dir home > café'
     statement = parser.parse(line)
     assert statement.command == 'dir'
     assert statement.args == 'home'
+    assert statement.argv == ['dir', 'home']
     assert statement.output == '>'
     assert statement.output_to == 'café'
 
@@ -267,9 +299,9 @@ def test_empty_statement_raises_exception():
         app._complete_statement(' ')
 
 @pytest.mark.parametrize('line,command,args', [
-    ('helpalias', 'help', ''),
+    ('helpalias', 'help', None),
     ('helpalias mycommand', 'help', 'mycommand'),
-    ('42', 'theanswer', ''),
+    ('42', 'theanswer', None),
     ('42 arg1 arg2', 'theanswer', 'arg1 arg2'),
     ('!ls', 'shell', 'ls'),
     ('!ls -al /tmp', 'shell', 'ls -al /tmp'),
@@ -293,20 +325,27 @@ def test_parse_command_only_command_and_args(parser):
     statement = parser.parse_command_only(line)
     assert statement.command == 'help'
     assert statement.args == 'history'
+    assert statement.argv == ['help', 'history']
     assert statement.command_and_args == line
 
 def test_parse_command_only_emptyline(parser):
     line = ''
     statement = parser.parse_command_only(line)
+    # statement is a subclass of str(), the value of the str
+    # should be '', to retain backwards compatibility with
+    # the cmd in the standard library
+    assert statement == ''
     assert statement.command is None
-    assert statement.args is ''
-    assert statement.command_and_args is line
+    assert statement.args is None
+    assert not statement.argv
+    assert statement.command_and_args == None
 
 def test_parse_command_only_strips_line(parser):
     line = '  help history  '
     statement = parser.parse_command_only(line)
     assert statement.command == 'help'
     assert statement.args == 'history'
+    assert statement.argv == ['help', 'history']
     assert statement.command_and_args == line.strip()
 
 def test_parse_command_only_expands_alias(parser):
@@ -314,13 +353,15 @@ def test_parse_command_only_expands_alias(parser):
     statement = parser.parse_command_only(line)
     assert statement.command == 'pyscript'
     assert statement.args == 'foobar.py'
+    assert statement.argv == ['pyscript', 'foobar.py']
 
 def test_parse_command_only_expands_shortcuts(parser):
     line = '!cat foobar.txt'
     statement = parser.parse_command_only(line)
     assert statement.command == 'shell'
     assert statement.args == 'cat foobar.txt'
-    assert statement.command_and_args == line.replace('!', 'shell ')
+    assert statement.argv == ['shell', 'cat', 'foobar.txt']
+    assert statement.command_and_args == 'shell cat foobar.txt'
 
 def test_parse_command_only_quoted_args(parser):
     line = 'l "/tmp/directory with spaces/doit.sh"'

From fa94eed90cf81b24e5b83c2b4c7e16025d849996 Mon Sep 17 00:00:00 2001
From: kotfu <kotfu@kotfu.net>
Date: Wed, 2 May 2018 19:13:23 -0700
Subject: [PATCH 89/89] Remove SHLEX_TODO.txt

---
 SHLEX_TODO.txt | 13 -------------
 1 file changed, 13 deletions(-)
 delete mode 100644 SHLEX_TODO.txt

diff --git a/SHLEX_TODO.txt b/SHLEX_TODO.txt
deleted file mode 100644
index 70e439ce8..000000000
--- a/SHLEX_TODO.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-
-Notes on conversion from pyparsing to shlex taking place in the ply branch
-
-Todo List:
-- refactor Cmd2.parseline() to use StatementParser.parse()
-- refactor tab completion to use StatementParser instead of parseline()
-- self.redirector doesn't really work any more, either make it work or expire it
-- clarify Statement.args whether it should be None or '', and whether it should be a string or a list of arguments,
-  include verifying documentation in unfreefeatures.txt
-- make sure Statement.command_and_args can handle quoted string arguments well. It might have to return a list of args, not a string
-- delete SHLEX_TODO.txt once everything is done
-
-Questions: