Move lexer to core for all to import.

plutus-core/plutus-core.cabal

@@ -202,6 +202,7 @@ library
         PlutusCore.Generators.Internal.TypeEvalCheck
         PlutusCore.Generators.Internal.TypedBuiltinGen
         PlutusCore.Generators.Internal.Utils
+        PlutusCore.Lexer.Lexer
         PlutusCore.Lexer.Type
         PlutusCore.Parsable
         PlutusCore.Parser.Internal

plutus-core/untyped-plutus-core/src/UntypedPlutusCore/Parser/Lexer.hs → plutus-core/plutus-core/src/PlutusCore/Lexer/Lexer.hs

@@ -1,15 +1,17 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE RecordWildCards   #-}
 {-# LANGUAGE TypeFamilies      #-}
-module UntypedPlutusCore.Lexer where
+module PlutusCore.Lexer.Lexer where
 
-import           PlutusCore.Lexer
+import           PlutusCore.Lexer      (AlexPosn)
 import           PlutusCore.Lexer.Type as L (Token)
 import           PlutusPrelude         (NonEmpty ((:|)), Pretty (pretty), Render (render))
 
 import qualified Data.List             as DL
 import qualified Data.List.NonEmpty    as NE
 import           Data.Proxy            (Proxy (..))
+import qualified Data.Set              as Set
+import           Data.Void             (Void)
 import           Text.Megaparsec
 
 data WithPos a = WithPos
@@ -19,54 +21,60 @@ data WithPos a = WithPos
   , tokenVal    :: a
   } deriving (Eq, Ord, Show)
 
-data MyStream = MyStream
-  { myStreamInput :: String -- for showing offending lines
-  , unMyStream    :: [WithPos (L.Token AlexPosn)]
+data TkStream = TkStream
+  { tkStreamInput :: String -- for showing offending lines
+  , unTkStream    :: [WithPos (L.Token AlexPosn)]
   }
-instance Stream MyStream where
-  type Token  MyStream = WithPos (L.Token AlexPosn)
-  type Tokens MyStream = [WithPos (L.Token AlexPosn)]
+
+-- data KwStream = KwStream
+--   { kwStreamInput :: String
+--   , unKwStream :: [WithPos Keyword ]
+--   }
+
+instance Stream TkStream where
+  type Token  TkStream = WithPos (L.Token AlexPosn)
+  type Tokens TkStream = [WithPos (L.Token AlexPosn)]
 
   tokenToChunk Proxy x = [x]
   tokensToChunk Proxy xs = xs
   chunkToTokens Proxy = id
   chunkLength Proxy = length
   chunkEmpty Proxy = null
-  take1_ (MyStream _ []) = Nothing
-  take1_ (MyStream str (t:ts)) = Just
+  take1_ (TkStream _ []) = Nothing
+  take1_ (TkStream str (t:ts)) = Just
     ( t
-    , MyStream (drop (tokensLength pxy (t:|[])) str) ts
+    , TkStream (drop (tokensLength pxy (t:|[])) str) ts
     )
-  takeN_ n (MyStream str s)
-    | n <= 0    = Just ([], MyStream str s)
+  takeN_ n (TkStream str s)
+    | n <= 0    = Just ([], TkStream str s)
     | null s    = Nothing
     | otherwise =
         let (x, s') = splitAt n s
         in case NE.nonEmpty x of
-          Nothing  -> Just (x, MyStream str s')
-          Just nex -> Just (x, MyStream (drop (tokensLength pxy nex) str) s')
-  takeWhile_ f (MyStream str s) =
+          Nothing  -> Just (x, TkStream str s')
+          Just nex -> Just (x, TkStream (drop (tokensLength pxy nex) str) s')
+  takeWhile_ f (TkStream str s) =
     let (x, s') = DL.span f s
     in case NE.nonEmpty x of
-      Nothing  -> (x, MyStream str s')
-      Just nex -> (x, MyStream (drop (tokensLength pxy nex) str) s')
+      Nothing  -> (x, TkStream str s')
+      Just nex -> (x, TkStream (drop (tokensLength pxy nex) str) s')
 
 showToken :: L.Token AlexPosn -> String
 showToken = render . pretty
 
-instance VisualStream MyStream where
+instance VisualStream TkStream where
   showTokens Proxy = unwords
     . NE.toList
     . fmap (showToken . tokenVal)
   tokensLength Proxy xs = sum (tokenLength <$> xs)
 
-instance TraversableStream MyStream where
+instance TraversableStream TkStream where
   reachOffset o PosState {..} =
     ( Just (prefix ++ restOfLine)
     , PosState
-        { pstateInput = MyStream
-            { myStreamInput = postStr
-            , unMyStream = post
+        { pstateInput = TkStream
+            { tkStreamInput = postStr
+            , unTkStream = post
             }
         , pstateOffset = max pstateOffset o
         , pstateSourcePos = newSourcePos
@@ -84,13 +92,29 @@ instance TraversableStream MyStream where
         case post of
           []    -> pstateSourcePos
           (x:_) -> startPos x
-      (pre, post) = splitAt (o - pstateOffset) (unMyStream pstateInput)
-      (preStr, postStr) = splitAt tokensConsumed (myStreamInput pstateInput)
+      (pre, post) = splitAt (o - pstateOffset) (unTkStream pstateInput)
+      (preStr, postStr) = splitAt tokensConsumed (tkStreamInput pstateInput)
       tokensConsumed =
         case NE.nonEmpty pre of
           Nothing    -> 0
           Just nePre -> tokensLength pxy nePre
       restOfLine = takeWhile (/= '\n') postStr
 
-pxy :: Proxy MyStream
+pxy :: Proxy TkStream
 pxy = Proxy
+
+type Parser = Parsec Void TkStream
+
+liftToken :: L.Token AlexPosn -> WithPos (L.Token AlexPosn)
+liftToken = WithPos pos pos 0
+  where
+    pos = initialPos ""
+
+pToken :: L.Token AlexPosn -> Parser (L.Token AlexPosn)
+pToken c = token test (Set.singleton . Tokens . nes . liftToken $ c)
+  where
+    test (WithPos _ _ _ x) =
+      if x == c
+        then Just x
+        else Nothing
+    nes x = x :| []