parse.y: indented hereoc

* parse.y: add heredoc <<~ syntax.  [Feature #9098]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@52916 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,7 @@
+Mon Dec  7 23:39:49 2015  Ben Miller  <bjmllr@gmail.com>
+
+	* parse.y: add heredoc <<~ syntax.  [Feature #9098]
+
 Mon Dec  7 23:06:16 2015  Kazuhiro NISHIYAMA  <zn@mbf.nifty.com>
 
 	* prelude.rb (IO#read_nonblock): [DOC] add missing options to

doc/syntax/literals.rdoc

@@ -196,6 +196,20 @@ Note that the while the closing identifier may be indented, the content is
 always treated as if it is flush left.  If you indent the content those spaces
 will appear in the output.
 
+To have indented content as well as an indented closing identifier, you can use
+a "squiggly" heredoc, which uses a "~" instead of a "-" after <tt><<</tt>:
+
+    expected_result = <<~SQUIGGLY_HEREDOC
+      This would contain specially formatted text.
+
+      That might span many lines
+    SQUIGGLY_HEREDOC
+
+The indentation of the least-indented line will be removed from each line of
+the content.  Note that empty lines and lines consisting solely of literal tabs
+and spaces will be ignored for the purposes of determining indentation, but
+escaped tabs and spaces are considered non-indentation characters.
+
 A heredoc allows interpolation and escaped characters.  You may disable
 interpolation and escaping by surrounding the opening identifier with single
 quotes:

ext/ripper/lib/ripper/lexer.rb

@@ -44,28 +44,56 @@ def Ripper.lex(src, filename = '-', lineno = 1)
   end
 
   class Lexer < ::Ripper   #:nodoc: internal use only
+    Elem = Struct.new(:pos, :event, :tok)
+
     def tokenize
-      lex().map {|pos, event, tok| tok }
+      parse().sort_by(&:pos).map(&:tok)
     end
 
     def lex
-      parse().sort_by {|pos, event, tok| pos }
+      parse().sort_by(&:pos).map(&:to_a)
     end
 
     def parse
       @buf = []
+      @stack = []
       super
+      @buf.flatten!
       @buf
     end
 
     private
 
+    def on_heredoc_dedent(v, w)
+      @buf.each do |e|
+        if e.event == :on_tstring_content
+          if (n = dedent_string(e.tok, w)) > 0
+            e.pos[1] += n
+          end
+        end
+      end
+      v
+    end
+
+    def on_heredoc_beg(tok)
+      @stack.push @buf
+      buf = []
+      @buf << buf
+      @buf = buf
+      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+    end
+
+    def on_heredoc_end(tok)
+      @buf.push Elem.new([lineno(), column()], __callee__, tok)
+      @buf = @stack.pop
+    end
+
     def _push_token(tok)
-      @buf.push [[lineno(), column()], __callee__, tok]
+      @buf.push Elem.new([lineno(), column()], __callee__, tok)
     end
 
-    SCANNER_EVENTS.each do |event|
-      alias_method "on_#{event}", :_push_token
+    (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
+      alias_method event, :_push_token
     end
   end
 

ext/ripper/lib/ripper/sexp.rb

@@ -62,7 +62,35 @@ def Ripper.sexp_raw(src, filename = '-', lineno = 1)
   class SexpBuilder < ::Ripper   #:nodoc:
     private
 
-    PARSER_EVENTS.each do |event|
+    def dedent_element(e, width)
+      if (n = dedent_string(e[1], width)) > 0
+        e[2][1] += n
+      end
+      e
+    end
+
+    def on_heredoc_dedent(val, width)
+      sub = proc do |cont|
+        cont.map! do |e|
+          if Array === e
+            case e[0]
+            when :@tstring_content
+              e = dedent_element(e, width)
+            when /_add\z/
+              e[1] = sub[e[1]]
+            end
+          elsif String === e
+            dedent_string(e, width)
+          end
+          e
+        end
+      end
+      sub[val]
+      val
+    end
+
+    events = private_instance_methods(false).grep(/\Aon_/) {$'.to_sym}
+    (PARSER_EVENTS - events).each do |event|
       module_eval(<<-End, __FILE__, __LINE__ + 1)
         def on_#{event}(*args)
           args.unshift :#{event}
@@ -83,6 +111,19 @@ def on_#{event}(tok)
   class SexpBuilderPP < SexpBuilder #:nodoc:
     private
 
+    def on_heredoc_dedent(val, width)
+      val.map! do |e|
+        next e if Symbol === e and /_content\z/ =~ e
+        if Array === e and e[0] == :@tstring_content
+          e = dedent_element(e, width)
+        elsif String === e
+          dedent_string(e, width)
+        end
+        e
+      end
+      val
+    end
+
     def _dispatch_event_new
       []
     end