Tokenise helper

This commit is contained in:
Igor Ranieri 2025-09-24 10:41:09 +02:00
parent 7ae868932d
commit 00a6e11f67

View file

@ -62,6 +62,15 @@ data Token
| EOF
deriving (Eq, Show)
located :: Parser a -> Parser (SourcePos, a)
located p = (,) <$> getPosition <*> p
startPosition :: Parser a -> Parser SourcePos
startPosition = fmap fst . located
tokenise :: [Parser a] -> Parser [(SourcePos, a)]
tokenise = sequence . map located
lexer :: String -> Either ParseError [LocatedToken]
lexer = Parsec.runParser lexText initialParserState "input" . Text.pack
@ -175,12 +184,6 @@ anchors = do
where
anchor' = (string "#" <|> string "\\#")
located :: Parser a -> Parser (SourcePos, a)
located p = (,) <$> getPosition <*> p
startPosition :: Parser a -> Parser SourcePos
startPosition = fmap fst . located
-- "Module.Name"
-- "Module.Name#anchor"
-- "Module.Name\#anchor" -- this has been deprecated for 9 years, thanks Ben
@ -208,27 +211,14 @@ modules = do
conChar = satisfy (\c -> isAlphaNum c || c == '_')
linkRaw :: Lexer
linkRaw = do
pos1 <- getPosition
void $ string "["
pos2 <- getPosition
text <- anyUntil $ Text.pack <$> string "]"
pos3 <- getPosition
void $ "]"
pos4 <- getPosition
void $ "("
pos5 <- getPosition
link' <- anyUntil $ Text.pack <$> string ")"
pos6 <- getPosition
void $ ")"
pure $
[ (pos1, BracketOpen)
, (pos2, Token text)
, (pos3, BracketClose)
, (pos4, ParenOpen)
, (pos5, Token link')
, (pos6, ParenClose)
linkRaw =
tokenise
[ BracketOpen <$ "["
, Token <$> anyUntil "]"
, BracketClose <$ "]"
, ParenOpen <$ "("
, Token <$> anyUntil ")"
, ParenClose <$ ")"
]
link :: Lexer