[spec/command-sub] Pin down the behavior of backticks and \.

backticks are parsed very differently than $() by all shells, although
POSIX doesn't talk about this.  An extra level of dequoting \ is
necessary.

We will probably have to write a simple BacktickLexer and then put the
"real" Lexer on top of that, feeding it to the CommandParser and
WordParser.

frontend/lex.py

@@ -145,6 +145,7 @@
 ]
 
 # Kind.Left that are valid in double-quoted modes.
+
 _LEFT_SUBS = [
   C('`', Id.Left_Backtick),
   C('$(', Id.Left_CommandSub),

osh/word_parse.py

@@ -657,6 +657,11 @@ def _ReadCommandSubPart(self, left_id):
       self.lexer.PushHint(Id.Op_RParen, right_id)
 
     elif left_id == Id.Left_Backtick:
+
+      # TODO: Backticks are parsed VERY differently than $().  They need their
+      # own lexer to get rid of backslashes!  virtualenv depends on this.  See
+      # the 7 failing cases in spec/command-sub.test.sh.
+
       right_id = Id.Eof_Backtick
       self.lexer.PushHint(Id.Left_Backtick, right_id)
 

spec/command-sub.test.sh

@@ -26,7 +26,7 @@ echo `echo -n l; echo -n s`
 ## stdout: ls
 
 #### Nested backticks
-# Inner `` are escaped!  # Not sure how to do triple..  Seems like an unlikely
+# Inner `` are escaped!  Not sure how to do triple..  Seems like an unlikely
 # use case.  Not sure if I even want to support this!
 echo X > $TMP/000000-first
 echo `\`echo -n l; echo -n s\` $TMP | grep 000000-first`
@@ -104,3 +104,114 @@ x
 0
 0
 ## END
+
+#### Double Quotes in Command Sub in Double Quotes
+# virtualenv's bin/activate uses this.
+# This is weird!  Double quotes within `` is different than double quotes
+# within $()!  All shells agree.
+# I think this is related to the nested backticks case!
+echo "x $(echo hi)"
+echo "x $(echo "hi")"
+echo "x $(echo \"hi\")"
+echo "x `echo hi`"
+echo "x `echo "hi"`"
+echo "x `echo \"hi\"`"
+## STDOUT:
+x hi
+x hi
+x "hi"
+x hi
+x hi
+x hi
+## END
+
+#### Escaped quote in [[ ]]
+file=$TMP/command-sub-dbracket
+#rm -f $file
+echo "123 `[[ $(echo \\" > $file) ]]` 456";
+cat $file
+## STDOUT:
+123  456
+"
+## END
+
+#### Quoting $ within ``
+echo 1 `echo $`
+echo 2 `echo \$`
+echo 3 `echo \\$`
+echo 4 `echo \\\$`
+echo 5 `echo \\\\$`
+
+echo "1 `echo $`"
+echo "2 `echo \$`"
+echo "3 `echo \\$`"
+echo "4 `echo \\\$`"
+echo "5 `echo \\\\$`"
+## STDOUT:
+1 $
+2 $
+3 $
+4 $
+5 \$
+1 $
+2 $
+3 $
+4 $
+5 \$
+## END
+
+#### Quoting \ within ``
+# You need FOUR backslashes to make a literal \.
+echo [1 `echo \ `]
+echo [2 `echo \\ `]
+echo [3 `echo \\\\ `]
+echo "[1 `echo \ `]"
+echo "[2 `echo \\ `]"
+echo "[3 `echo \\\\ `]"
+
+## STDOUT:
+[1 ]
+[2 ]
+[3 \]
+[1  ]
+[2  ]
+[3 \]
+## END
+
+#### Quoting ( within ``
+echo 1 `echo \(`
+echo 2 `echo \\(`
+echo 3 `echo \\ \\(`
+
+echo "1 `echo \(`"
+echo "2 `echo \\(`"
+echo "3 `echo \\ \\(`"
+## STDOUT:
+1 (
+2 (
+3 (
+1 (
+2 (
+3  (
+## END
+
+#### \ with non-operator characters within ``
+echo [1 `echo \%]`
+echo [2 `echo \\%]`
+echo [3 `echo \\\%]`
+echo [4 `echo \\\\%]`
+
+echo "[1 `echo \%`]"
+echo "[2 `echo \\%`]"
+echo "[3 `echo \\\%`]"
+echo "[4 `echo \\\\%`]"
+## STDOUT:
+[1 %]
+[2 %]
+[3 \%]
+[4 \%]
+[1 %]
+[2 %]
+[3 \%]
+[4 \%]
+## END

test/spec.sh

@@ -343,7 +343,7 @@ builtin-getopts() {
 }
 
 builtin-bracket() {
-  sh-spec spec/builtin-bracket.test.sh --osh-failures-allowed 1 \
+  sh-spec spec/builtin-bracket.test.sh \
     ${REF_SHELLS[@]} $OSH_LIST "$@"
 }
 
@@ -393,7 +393,7 @@ arith() {
 }
 
 command-sub() {
-  sh-spec spec/command-sub.test.sh --osh-failures-allowed 1 \
+  sh-spec spec/command-sub.test.sh --osh-failures-allowed 7 \
     ${REF_SHELLS[@]} $OSH_LIST "$@"
 }
 
@@ -443,7 +443,7 @@ special-vars() {
 
 # dash/mksh don't implement this.
 introspect() {
-  sh-spec spec/introspect.test.sh --osh-failures-allowed 1 \
+  sh-spec spec/introspect.test.sh \
     $BASH $OSH_LIST "$@"
 }