Support interpolation and here docs in ruby strings.

TODO

@@ -25,7 +25,6 @@ add other languages
 
   x xml
   x ruby
-  _ support here docs in ruby
 
   _ erb
   _ yaml

Text/Highlighting/Illuminate/Ruby.x

@@ -23,8 +23,11 @@ $hexdigit = [$digit A-F a-f]
            "self"|"true"|"__FILE__"|"__LINE__"|"and"|"not"|"or"|"def"|"class"|
            "module"|"catch"|"fail"|"load"|"throw")
 @alert = (TODO|FIXME|BUG)[\:]?
+@singlestring = \' ([^ \' \\] | \\ .)* \'
 @string = ( \" ([^ \" \\] | \\ .)* \" 
-          | \' ([^ \' \\] | \\ .)* \')
+          | @singlestring
+          | \` [^ \`]* \`)
+@identifier = [$alpha \_] $wordchar*
 
 tokens :-
 
@@ -48,7 +51,26 @@ tokens :-
 <def> {
   [$alpha \_] $wordchar*   { tok Function ==> popContext }
 }
-<0> {
+<backtickstring> {
+ \# \{                   { tok CBracket ==> pushContext (interp, Plain) }
+ \\ .                    { tok String } 
+ \`                      { tok String ==> popContext }
+}
+<doublestring> {
+ \# \{                   { tok CBracket ==> pushContext (interp, Plain) }
+ \\ .                    { tok String } 
+ \"                      { tok String ==> popContext }
+}
+<heredoc> {
+  @string                { tok String ==> hereDoc ==> popContext }
+  @identifier            { tok VarId ==> hereDoc ==> popContext } 
+  \# / [^\{]  { tok Comment ==> pushContext (linecomment, Comment) }
+  [ \t]+                 { tok Whitespace }
+  -- if we hit end of line w/o an identifier, exit context
+  \n                     { tok Whitespace ==> popContext }
+}
+<interp>   \}            { tok CBracket ==> popContext }
+<0,interp> {
   ^ "=begin" / [ \t\r\n] { tok Comment ==> pushContext (blockcomment, Comment) }
   "require" / $white     { tok Preproc }
   \/ ([^\n\/]+|\\\/)* \/ { tok Regex }
@@ -58,19 +80,17 @@ tokens :-
   $white ^ "def" / $white  { tok Keyword ==> pushContext (def, Plain) }
   @keyword / ~$wordchar  { tok Keyword }
   -- ? and ! as symbols as part of a method name
-  $alphanum+ [\?\!]?     { tok VarId }
+  @identifier  [\?\!]?   { tok VarId }
   @number                { tok Number }
-  @string                { tok String }
-  \` [^ \`] \`           { tok String }
-  -- note: #{var} is not a comment:
-  \# / [^\{]  { tok Comment ==> pushContext (linecomment, Comment) }
-
-  -- TODO: here docs ... need some way to store and access the end string
-  --  "<<" \-? (@string | $wordchar+)  { tok String ==> pushContext 
+  @singlestring          { tok String }
+  \`                     { tok String ==> pushContext (backtickstring, String) }
+  \"                     { tok String ==> pushContext (doublestring, String) }
+  \#                     { tok Comment ==> pushContext (linecomment, Comment) }
 
+  "<<" \-?               { tok Symbol ==> pushContext (heredoc, Plain) }
   $symbol                { tok Symbol }
   [\{ \}]                { tok CBracket }
-  [$alpha \_]$wordchar*  { tok VarId }
+  @identifier            { tok VarId }
   $white+                { tok Whitespace }
 }
 

alex/AlexWrapper-illuminate

@@ -2,6 +2,7 @@ import Text.Highlighting.Illuminate.Types
 import Data.Sequence (Seq, (><), (<|), (|>), singleton, viewl, ViewL(..))
 import qualified Data.Sequence as Seq (empty)
 import qualified Data.Foldable as F
+import Data.List (span, break, splitAt, isPrefixOf)
 {-# LINE 1 "templates/wrappers.hs" #-}
 {-# LINE 1 "templates/wrappers.hs" #-}
 {-# LINE 1 "<built-in>" #-}
@@ -228,6 +229,36 @@ popContext toks = do
            alexSetStartCode $ fst y
            return toks
 
+scanForStop :: String -> String -> Int
+scanForStop stop ('\n':xs) = 1 +
+  let (sps, xs') = span (`elem` " \t") xs
+      offset = length sps + length stop
+  in  if stop `isPrefixOf` xs'
+         then case (takeWhile (`elem` " \t") $ drop (length stop) xs') of
+                   ('\n':_)  -> offset
+                   []        -> offset
+                   _         -> offset + scanForStop stop (drop offset xs')
+         else length sps + scanForStop stop xs'
+scanForStop stop (_:xs) = 1 + scanForStop stop xs
+scanForStop stop [] = 0
+
+hereDoc :: Tokens -> Alex Tokens
+hereDoc toks = Alex $ \s ->
+  let inp = alex_inp s
+      unescape ('\\':x:xs) = x : unescape xs
+      unescape (x:xs) = x : unescape xs
+      unescape [] = []
+      unstring = tail . init . unescape
+  in  case viewl toks of
+        ((tt,str) :< empty) | tt `elem` [String, VarId] ->
+            let stop = if tt == String then unstring str else str
+                off = scanForStop stop inp
+                (res, newinp) = splitAt off inp
+                newpos = foldl alexMove (alex_pos s) res
+                s' = s{alex_pos = newpos, alex_inp = newinp, alex_chr = '\n'}
+            in  Right (s', toks |> (String, res))
+        _                   -> Left "hereDoc expecting String or VarId"
+
 (==>) :: (AlexInput -> Int -> Alex a) -> (a -> Alex b) -> (AlexInput -> Int -> Alex b)
 (act1 ==> act2) inp len = do
   res <- act1 inp len

illuminate.cabal

@@ -35,7 +35,8 @@ Flag executable
   Default:           True
 
 Library
-  Build-Depends:     base >= 3 && < 5, containers, xhtml, html, utf8-string, hscolour, FileManip, filepath
+  Build-Depends:     base >= 3 && < 5, containers, xhtml, html, utf8-string,
+                     hscolour, FileManip, filepath
   Build-Tools:       alex >= 2 && < 3
   Exposed-Modules:   Text.Highlighting.Illuminate
                      Text.Highlighting.Illuminate.Types