Permalink
Browse files

Initial import

  • Loading branch information...
0 parents commit bdb29e0d2ff5f772b5521d7a657698622b945e0a @yav committed Sep 27, 2008
@@ -0,0 +1 @@
+dist
@@ -0,0 +1,8 @@
+Copyright (c) 2008 Thomas Hallgren
+Copyright (c) 2008 Iavor S. Diatchki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,64 @@
+module Language.Haskell.Lexer
+ ( PosToken
+ , Token(..)
+ , lexerPass0
+ , lexerPass0'
+ , lexerPass1
+ , rmSpace
+ , layoutPre
+ , module Language.Haskell.Lexer.Position
+ ) where
+
+import Language.Haskell.Lexer.Lex(haskellLex)
+import Language.Haskell.Lexer.Utils
+import Language.Haskell.Lexer.Layout(layoutPre,PosToken)
+import Language.Haskell.Lexer.Position
+import Data.List(mapAccumL)
+
+default(Int)
+
+-- | The function 'lexerPass1' handles the part of lexical analysis that
+-- can be done independently of the parser---the tokenization and the
+-- addition of the extra layout tokens \<n\> and {n}, as specified in
+-- section 9.3 of the revised Haskell 98 Report.
+lexerPass1 :: String -> [PosToken]
+lexerPass1 = lexerPass1Only . lexerPass0
+
+lexerPass1Only :: [PosToken] -> [PosToken]
+lexerPass1Only = layoutPre . rmSpace
+
+-- | Remove token that are not meaningful (e.g., white space and comments).
+rmSpace :: [PosToken] -> [PosToken]
+rmSpace = filter (notWhite.fst)
+
+notWhite :: Token -> Bool
+notWhite t = t/=Whitespace &&
+ t/=Commentstart && t/=Comment &&
+ t/=NestedComment
+
+-- | Tokenize and add position information. Preserves white space,
+-- and does not insert extra tokens due to layout.
+lexerPass0 :: String -> [PosToken]
+lexerPass0 = lexerPass0' startPos
+
+-- | Same as 'lexerPass0', except that it uses the given start position.
+lexerPass0' :: Pos -> String -> [PosToken]
+lexerPass0' pos0 = addPos . haskellLex . rmcr
+ where
+ addPos = snd . mapAccumL pos pos0
+
+ pos p (t,s) = {-seq p'-} (p',(t,(p,s)))
+ where p' = nextPos p s
+-- where s = reverse r
+
+
+-- | Since #nextPos# examines one character at a time, it will increase the line
+-- number by 2 if it sees \CR\LF, which can happen when reading DOS files on
+-- a Unix like system.
+-- Since the extra \CR characters can cause trouble later as well, we choose
+-- to simply remove them here.
+rmcr :: String -> String
+rmcr ('\CR':'\LF':s) = '\LF':rmcr s
+rmcr (c:s) = c:rmcr s
+rmcr "" = ""
+
@@ -0,0 +1,61 @@
+module Language.Haskell.Lexer.Layout (layoutPre,PosToken) where
+
+import Language.Haskell.Lexer.Tokens
+import Language.Haskell.Lexer.Position
+
+type PosToken = (Token,(Pos,String))
+
+-- | This is an implementation of Haskell layout, as specified in
+-- section 9.3 of the revised Haskell 98 report.
+-- This preprocessor inserts the extra \<n\> and {n} tokens.
+layoutPre :: [PosToken] -> [PosToken]
+layoutPre = indent . open
+
+open :: [PosToken] -> [PosToken]
+open = open1
+
+{-+
+If the first lexeme of a module is not { or module, then it is preceded
+by {n} where n is the indentation of the lexeme.
+-}
+open1 :: [PosToken] -> [PosToken]
+open1 (t1@(Reservedid,(_,"module")):ts) = t1:open2 ts
+open1 (t1@(Special,(_,"{")):ts) = t1:open2 ts
+open1 ts@((_,(p,_)):_) = (Open (column p),(p,"")):open2 ts
+open1 [] = []
+
+{-+
+If a let, where, do, or of keyword is not followed by the lexeme {,
+the token {n} is inserted after the keyword, where n is the indentation of
+the next lexeme if there is one, or 0 if the end of file has been reached.
+-}
+open2 :: [PosToken] -> [PosToken]
+open2 (t1:ts1) | isLtoken t1 =
+ case ts1 of
+ t2@(_,(p,_)):ts2 ->
+ if notLBrace t2
+ then t1:(Open (column p),(p,"")):open2 ts1
+ else t1:t2:open2 ts2
+ [] -> t1:(Open 0,(fst (snd t1),"")):[]
+ where
+ isLtoken (Reservedid,(_,s)) = s `elem` ["let","where","do","of"]
+ isLtoken _ = False
+
+ notLBrace (Special,(_,"{")) = False
+ notLBrace _ = True
+open2 (t:ts) = t:open2 ts
+open2 [] = []
+
+{-+
+(This is from the original Haskell 98 report.)
+The first token on each line (not including tokens already annotated) is
+preceeded by &lt;n&gt;, where n is the indentation of the token.
+-}
+indent :: [PosToken] -> [PosToken]
+indent (t1@(Open _,(p,_)):ts) = t1:indent2 (line p) ts
+indent (t1@(_,(p,_)):ts) = (Indent (column p),(p,"")):t1:indent2 (line p) ts
+indent [] = []
+
+indent2 :: Int -> [PosToken] -> [PosToken]
+indent2 r (t1@(_,(p,_)):ts) | line p==r = t1:indent2 r ts
+indent2 _ ts = indent ts
Oops, something went wrong.

0 comments on commit bdb29e0

Please sign in to comment.