Brainstorming about a rewrite of TildeSubPart.

We want to maintain the arena invariant: the spans "sum up" to the
original file.

- osh2oil: expose the fact the "synthetic" LiteralPart after the tilde
  has no span_id
- Adjust spec tests for tilde.
- Add comments.

core/word.py

@@ -250,9 +250,8 @@ def LeftMostSpanForWord(w):
     return const.NO_INTEGER
 
 
-# This is needed for DoWord I guess?  IT makes it easier to write the fixer.
 def RightMostSpanForWord(w):
-  # TODO: Really we should use par
+  """Needed for here doc delimiters."""
   if w.tag == word_e.CompoundWord:
     if len(w.parts) == 0:
       # TODO: Use EmptyWord instead
@@ -265,6 +264,20 @@ def RightMostSpanForWord(w):
   return w.token.span_id
 
 
+# From bash, general.c, unquoted_tilde_word():
+# POSIX.2, 3.6.1:  A tilde-prefix consists of an unquoted tilde character at
+# the beginning of the word, followed by all of the characters preceding the
+# first unquoted slash in the word, or all the characters in the word if there
+# is no slash...If none of the characters in the tilde-prefix are quoted, the
+# characters in the tilde-prefix following the tilde shell be treated as a
+# possible login name. 
+#define TILDE_END(c)    ((c) == '\0' || (c) == '/' || (c) == ':')
+#
+# So an unquoted tilde can ALWAYS start a new lex mode?  You respect quotes and
+# substitutions.
+#
+# We only detect ~Lit_Chars and split.  So we might as well just write a regex.
+
 def TildeDetect(word):
   """Detect tilde expansion.
 
@@ -397,8 +410,8 @@ def LooksLikeAssignment(w):
   """Tests whether a word looks like FOO=bar.
 
   Returns:
-    (string, CompoundWord) if it looks like FOO=bar
-    False                  if it doesn't
+    (string, op, CompoundWord) if it looks like FOO=bar
+    False                      if it doesn't
 
   s=1
   s+=1
@@ -431,9 +444,9 @@ def LooksLikeAssignment(w):
 
   rhs = ast.CompoundWord()
   if len(w.parts) == 1:
-    # This fake SingleQuotedPart is necesssary so that EmptyUnquoted elision
-    # isn't applied.  EMPTY= is like EMPTY=''.
-    # TODO: This part doesn't have spids, so it might break some invariants.
+    # This is necessary so that EmptyUnquoted elision isn't applied.  EMPTY= is
+    # like EMPTY=''.
+    # TODO: Change to EmptyWord
     rhs.parts.append(ast.EmptyPart())
   else:
     for p in w.parts[1:]:

osh/lex.py

@@ -208,6 +208,7 @@
   C('}', Id.Lit_RBrace),  # Also for var sub ${a}
   C(',', Id.Lit_Comma),
   C('~', Id.Lit_Tilde),  # For tilde expansion
+                         # TODO: Add the rest of Lit_Chars minus / here.
 
   R(r'[ \t\r]+', Id.WS_Space),
 

osh/osh.asdl

@@ -76,6 +76,7 @@ module osh
                 id? prefix_op,  -- prefix # or ! operators
                 bracket_op? bracket_op
                 suffix_op? suffix_op)
+    -- This should be token tilde, token rest
   | TildeSubPart(string prefix)
     -- For command sub and process sub: $(...)  <(...)  >(...)
   | CommandSubPart(command command_list, token left_token)

osh/word_parse.py

@@ -1111,11 +1111,7 @@ def _ReadWord(self, lex_mode):
         Kind.VSub, Kind.Lit, Kind.Left, Kind.KW, Kind.Assign, Kind.ControlFlow,
         Kind.BoolUnary, Kind.BoolBinary, Kind.ExtGlob):
       # We're beginning a word.  If we see Id.Lit_Pound, change to
-      # lex_mode_e.COMMENT and read until end of line.  (TODO: How to add
-      # comments to AST?)
-
-      # TODO: Can we do the same thing for Tilde here?  Enter a state where we
-      # look for / too.
+      # lex_mode_e.COMMENT and read until end of line.
       if self.token_type == Id.Lit_Pound:
         self._Next(lex_mode_e.COMMENT)
         self._Peek()

spec/brace-expansion.test.sh

@@ -149,20 +149,25 @@ HOME=/home/foo
 echo ~
 HOME=/home/bar
 echo ~
-## stdout-json: "/home/foo\n/home/bar\n"
+## STDOUT:
+/home/foo
+/home/bar
+## END
 
 #### Tilde expansion with brace expansion
+# NOTE: osh matches mksh.  Is that OK?
 # The brace expansion happens FIRST.  After that, the second token has tilde
 # FIRST, so it gets expanded.  The first token has an unexpanded tilde, because
 # it's not in the leading position.
 # NOTE: mksh gives different behavior!  So it probably doesn't matter that
-# much...
+# much
 HOME=/home/bob
 echo {foo~,~}/bar
 ## stdout: foo~/bar /home/bob/bar
 ## OK mksh stdout: foo~/bar ~/bar
 
 #### Two kinds of tilde expansion
+# NOTE: osh matches mksh.  Is that OK?
 # ~/foo and ~bar
 HOME=/home/bob
 echo ~{/src,root}
@@ -177,7 +182,10 @@ foo='~'
 echo $foo
 # In the second instance, we expand into a literal ~, and since var expansion
 # comes after tilde expansion, it is NOT tried again.
-## stdout-json: "/home/bob\n~\n"
+## STDOUT:
+/home/bob
+~
+## END
 
 #### Number range expansion
 echo -{1..8..3}-

test/arena.sh

@@ -23,8 +23,13 @@ here-doc() {
   compare test/arena/here-multiple.sh
 }
 
+tilde() {
+  compare test/arena/tilde.sh
+}
+
 readonly -a PASSING=(
   here-doc
+  tilde
 )
 
 all-passing() {

test/arena/tilde.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo ~/src
+echo ~andy/git
+bin=~andy/bin
+echo not~an~expansion~
+echo 
+echo ~{/src,root}
+echo {foo~,~}/bar

tools/osh2oil.py

@@ -1053,10 +1053,10 @@ def DoWordPart(self, node, local_symbols, quoted=False):
       # TODO: We might want to do it all on the word level though.  For
       # example, foo"bar" becomes "foobar" in oil.
       spid = node.token.span_id
-      if spid is None:
+      if spid == const.NO_INTEGER:
         #raise RuntimeError('%s has no span_id' % node.token)
         # TODO: Fix word.TildeDetect to construct proper tokens.
-        print('WARNING:%s has no span_id' % node.token, file=sys.stderr)
+        log('WARNING: %s has no span_id' % node.token)
       else:
         self.cursor.PrintUntil(spid + 1)