Skip to content
Browse files

Add support for efficient lexing of strict bytestrings

  • Loading branch information...
1 parent c69fca3 commit 197317b03cf9ab82def73b1c3b1fa4925738e812 @donsbot donsbot committed
Showing with 39 additions and 5 deletions.
  1. +1 −0 Setup.lhs
  2. +38 −5 templates/wrappers.hs
View
1 Setup.lhs
@@ -65,6 +65,7 @@ wrappers :: [(FilePath,[String])]
wrappers = [
("AlexWrapper-basic", ["-DALEX_BASIC"]),
("AlexWrapper-basic-bytestring", ["-DALEX_BASIC_BYTESTRING"]),
+ ("AlexWrapper-strict-bytestring", ["-DALEX_STRICT_BYTESTRING"]),
("AlexWrapper-posn", ["-DALEX_POSN"]),
("AlexWrapper-posn-bytestring", ["-DALEX_POSN_BYTESTRING"]),
("AlexWrapper-monad", ["-DALEX_MONAD"]),
View
43 templates/wrappers.hs
@@ -8,6 +8,12 @@
import qualified Data.ByteString.Lazy.Char8 as ByteString
+#elif defined(ALEX_STRICT_BYTESTRING)
+
+import qualified Data.ByteString.Char8 as ByteString
+import qualified Data.ByteString.Internal as ByteString
+import qualified Data.ByteString.Unsafe as ByteString
+
#endif
-- -----------------------------------------------------------------------------
@@ -268,11 +274,38 @@ alexInputPrevChar (c,_) = c
-- alexScanTokens :: String -> [token]
alexScanTokens str = go ('\n',str)
where go inp@(_,str) =
- case alexScan inp 0 of
- AlexEOF -> []
- AlexError _ -> error "lexical error"
- AlexSkip inp' len -> go inp'
- AlexToken inp' len act -> act (ByteString.take (fromIntegral len) str) : go inp'
+ case alexScan inp 0 of
+ AlexEOF -> []
+ AlexError _ -> error "lexical error"
+ AlexSkip inp' len -> go inp'
+ AlexToken inp' len act -> act (ByteString.take (fromIntegral len) str) : go inp'
+
+
+#endif
+
+#ifdef ALEX_STRICT_BYTESTRING
+
+data AlexInput = AlexInput { alexChar :: {-# UNPACK #-}!Char
+ , alexStr :: {-# UNPACK #-}!ByteString.ByteString }
+
+alexGetChar (AlexInput _ cs)
+ | ByteString.null cs = Nothing
+ | otherwise = Just $! (ByteString.head cs, AlexInput c cs')
+ where
+ (c,cs') = (ByteString.w2c (ByteString.unsafeHead cs)
+ , ByteString.unsafeTail cs)
+
+alexInputPrevChar = alexChar
+
+-- alexScanTokens :: String -> [token]
+alexScanTokens str = go (AlexInput '\n' str)
+ where go inp@(AlexInput _ str) =
+ case alexScan inp 0 of
+ AlexEOF -> []
+ AlexError _ -> error "lexical error"
+ AlexSkip inp' len -> go inp'
+ AlexToken inp' len act -> act (ByteString.unsafeTake len str) : go inp'
+
#endif

0 comments on commit 197317b

Please sign in to comment.
Something went wrong with that request. Please try again.