ref(lexer): attempt to not try on every token for a better error message

src/Lexer.hs

@@ -83,24 +83,32 @@ lexText = go
       Just _ -> pure []
       Nothing -> do
         toks <-
-          choice $
+          -- backtracking here so we always have a chance to try "other", the "catch-all-leave-to-parser-to-deal-with" choice
-            Parsec.try
+          -- TODO: is this desirable? do we throw lexer error at all?
-              <$> [ mathMultiline
+          try
-                  , mathInline
+            ( choice
-                  , escape -- maths go before escape to avoid mismatch
+                -- Sorted in
-                  , headers
+                -- - longest to shortest parse path
-                  , newlineToken
+                -- - highest frequency to lowest frequency (for performance?)
-                  , spaceToken
+                -- - more exact to more freeform (the latter can be the former but not vice versa)
-                  , link
+                [ spaceToken
-                  , labeledLink
+                , newlineToken
-                  , module_
+                , -- starts with "\"
-                  , anchor
+                  try mathMultiline
-                  , numericEntity
+                , try mathInline
-                  , textElement
+                , escape
-                  , quotes
+                , headers
-                  , birdTrack
+                , labeledLink
-                  , other
+                , link
-                  ]
+                , anchor
+                , numericEntity
+                , textElement
+                , try module_
+                , quotes
+                , birdTrack
+                ]
+            )
+            <|> other
         rest <- go
         pure (toks <> rest)
 
@@ -239,6 +247,7 @@ mathMultiline = delimited "\\[" "\\]" MathMultilineOpen MathMultilineClose
 mathInline :: Lexer
 mathInline = delimited "\\(" "\\)" MathInlineOpen MathInlineClose
 
+-- TODO: make sure this starts at column 0?
 birdTrack :: Lexer
 birdTrack = delimitedNoTrailing ">> " eol BirdTrack