Revert "chore!: tokenize file in one step (#14)"

This reverts commit dd3abfe.

src/cli.rs

@@ -1,4 +1,4 @@
-use lemmy_help::LemmyHelp;
+use lemmy_help::{frontend::Lua, LemmyHelp};
 use lexopt::{
     Arg::{Long, Short, Value},
     Parser,
@@ -58,6 +58,7 @@ impl Cli {
 
         for f in &self.files {
             let source = read_to_string(f).unwrap();
+            let source = Lua::parse(&source).unwrap();
             lemmy.for_help(&source).unwrap();
         }
 

src/frontend/lua.rs

@@ -0,0 +1,83 @@
+use chumsky::{
+    prelude::{choice, end, filter, just, take_until, Simple},
+    text::{ident, keyword, newline, TextParser},
+    Parser,
+};
+
+#[derive(Debug)]
+pub struct Lua;
+
+impl Lua {
+    pub fn parse(src: &str) -> Result<String, Vec<Simple<char>>> {
+        Ok(Self::lex(src)?.join("\n"))
+    }
+
+    pub fn lex(src: &str) -> Result<Vec<String>, Vec<Simple<char>>> {
+        let dotted = ident()
+            .then(just('.').or(just(':')))
+            .then(ident())
+            .map(|((m, k), f)| format!("{m}{k}{f}"));
+
+        let expr = dotted.padded().then_ignore(just('='));
+
+        let func = keyword("function").padded();
+
+        let local = keyword("local").padded();
+
+        let triple = just("---");
+
+        let node = choice((
+            triple
+                .then(filter(|c| *c != '\n').repeated().collect::<String>())
+                .map(|(s, x)| format!("{s}{x}")),
+            local.ignore_then(choice((
+                func.clone()
+                    .ignore_then(ident())
+                    .map(|x| format!("---@func {x} private")),
+                ident()
+                    .padded()
+                    .then_ignore(just('='))
+                    .map(|x| format!("---@expr {x} private")),
+            ))),
+            func.clone()
+                .ignore_then(dotted)
+                .map(|x| format!("---@func {x} public")),
+            choice((
+                expr.then_ignore(func)
+                    .map(|x| format!("---@func {x} public")),
+                expr.map(|x| format!("---@expr {x} public")),
+            )),
+            keyword("return")
+                .ignore_then(ident().padded())
+                .then_ignore(end())
+                .map(|x| format!("---@export {x}")),
+        ))
+        .map(Some);
+
+        let private = triple
+            .then_ignore(just("@private"))
+            .then_ignore(
+                choice((
+                    // eat up all the emmylua, if any, then one valid token
+                    triple
+                        .then(take_until(newline().or(end())))
+                        .padded()
+                        .repeated()
+                        .ignore_then(ident()),
+                    // if there is no emmylua, just eat the next token
+                    // so the next parser won't recognize the code
+                    ident(),
+                ))
+                .padded(),
+            )
+            .to(None);
+
+        let misc = take_until(newline()).to(None);
+
+        choice((private, node, misc))
+            .padded()
+            .repeated()
+            .flatten()
+            .parse(src)
+    }
+}

src/frontend/mod.rs

@@ -0,0 +1,2 @@
+mod lua;
+pub use lua::*;

src/lib.rs

@@ -1,4 +1,5 @@
 mod parser;
 pub use parser::*;
 
+pub mod frontend;
 pub mod view;

src/parser/emmy.rs

@@ -2,7 +2,7 @@ use std::ops::Range;
 
 use chumsky::{
     prelude::{any, choice, end, filter, just, take_until, Simple},
-    text::{ident, keyword, newline, TextParser},
+    text::{ident, newline, TextParser},
     Parser,
 };
 
@@ -55,7 +55,6 @@ pub enum TagType {
     Usage(String),
     Comment(String),
     Empty,
-    Skip,
 }
 
 type Spanned = (TagType, Range<usize>);
@@ -65,11 +64,20 @@ pub struct Emmy;
 
 impl Emmy {
     pub fn parse(src: &str) -> Result<Vec<Spanned>, Vec<Simple<char>>> {
-        let triple = just("---");
+        let comment = take_until(newline().or(end())).map(|(x, _)| x.iter().collect());
 
-        let local = keyword("local").padded();
+        let ty = filter(|x: &char| !x.is_whitespace())
+            .repeated()
+            .padded()
+            .collect();
 
-        let func = keyword("function").padded();
+        let name = ident().padded();
+
+        let desc = choice((
+            end().to(None),
+            just("---").rewind().to(None),
+            comment.clone().map(Some),
+        ));
 
         let dotted = choice((
             ident()
@@ -80,47 +88,37 @@ impl Emmy {
         ))
         .padded();
 
-        let expr = dotted.clone().padded().then_ignore(just('='));
-
-        let comment = take_until(newline().or(end())).map(|(x, _)| x.iter().collect());
-
-        let ty = filter(|x: &char| !x.is_whitespace()).repeated().collect();
-
-        let name = ident().padded();
-
-        let desc = choice((
-            end().to(None),
-            triple.rewind().to(None),
-            comment.clone().map(Some),
-        ));
-
-        let private = just("private").then_ignore(
-            choice((
-                // eat up all the emmylua, if any, then one valid token
-                triple
-                    .then(take_until(newline().or(end())))
-                    .padded()
-                    .repeated()
-                    .ignore_then(ident()),
-                // if there is no emmylua, just eat the next token
-                // so the next parser won't recognize the code
-                ident(),
-            ))
-            .padded(),
-        );
-
-        let misc = take_until(newline());
-
         let tags = just('@')
             .ignore_then(choice((
-                private.to(TagType::Skip),
                 just("mod")
-                    .ignore_then(ty.padded())
+                    .ignore_then(ty)
                     .then(desc.clone())
                     .map(|(name, desc)| TagType::Module { name, desc }),
                 just("divider")
                     .ignore_then(any().padded())
                     .map(TagType::Divider),
+                just("func")
+                    .ignore_then(dotted.clone())
+                    .then(comment.clone())
+                    .padded()
+                    .map(|((prefix, scope, name), _)| TagType::Func {
+                        prefix,
+                        name,
+                        scope,
+                    }),
+                just("expr")
+                    .ignore_then(dotted)
+                    .then(comment.clone())
+                    .padded()
+                    .map(|((prefix, scope, name), _)| TagType::Expr {
+                        prefix,
+                        name,
+                        scope,
+                    }),
+                just("export")
+                    .ignore_then(ident().padded())
+                    .then_ignore(end())
+                    .map(TagType::Export),
                 just("brief").ignore_then(
                     choice((
                         just("[[").to(TagType::BriefStart),
@@ -129,41 +127,33 @@ impl Emmy {
                     .padded(),
                 ),
                 just("param")
-                    .ignore_then(ty.padded()) // I am using `ty` here because param can have `?`
-                    .then(ty.padded())
+                    .ignore_then(ty) // I am using `ty` here because param can have `?`
+                    .then(ty)
                     .then(desc.clone())
                     .map(|((name, ty), desc)| TagType::Param { name, ty, desc }),
                 just("return")
-                    .ignore_then(
-                        ty.then(choice((
-                            newline().to((None, None)),
-                            ident()
-                                .then(newline().to(None).or(desc.clone().padded()))
-                                .padded()
-                                .map(|(name, desc)| (Some(name), desc)),
-                        )))
-                        .padded(),
-                    )
-                    .map(|(ty, (name, desc))| TagType::Return { ty, name, desc }),
+                    .ignore_then(ty)
+                    .then(name.or_not())
+                    .then(desc.clone())
+                    .map(|((ty, name), desc)| TagType::Return { ty, name, desc }),
                 just("class")
                     .ignore_then(name)
                     .then(desc.clone())
                     .map(|(name, desc)| TagType::Class(name, desc)),
                 just("field")
                     .ignore_then(name)
-                    .then(ty.padded())
+                    .then(ty)
                     .then(desc.clone())
                     .map(|((name, ty), desc)| TagType::Field { name, ty, desc }),
                 just("alias")
                     .ignore_then(name)
-                    .then(ty.padded())
+                    .then(ty)
                     .then(desc.clone())
                     .map(|((name, ty), desc)| TagType::Alias { ty, name, desc }),
-                just("type").ignore_then(
-                    ty.then(choice((newline().to(None), desc.padded())))
-                        .padded()
-                        .map(|(name, desc)| TagType::Type(name, desc)),
-                ),
+                just("type")
+                    .ignore_then(name)
+                    .then(desc)
+                    .map(|(name, desc)| TagType::Type(name, desc)),
                 just("tag")
                     .ignore_then(comment.clone().padded())
                     .map(TagType::Tag),
@@ -183,53 +173,10 @@ impl Emmy {
             .or(newline().to(TagType::Empty))
             .or(comment.map(TagType::Comment));
 
-        choice((
-            triple.ignore_then(tags),
-            local.ignore_then(choice((
-                func.clone().ignore_then(ident()).map(|name| TagType::Func {
-                    name,
-                    prefix: None,
-                    scope: Scope::Local,
-                }),
-                ident()
-                    .padded()
-                    .then_ignore(just('='))
-                    .map(|name| TagType::Expr {
-                        name,
-                        prefix: None,
-                        scope: Scope::Local,
-                    }),
-            ))),
-            func.clone()
-                .ignore_then(dotted)
-                .map(|(prefix, scope, name)| TagType::Func {
-                    prefix,
-                    name,
-                    scope,
-                }),
-            choice((
-                expr.clone()
-                    .then_ignore(func)
-                    .map(|(prefix, scope, name)| TagType::Func {
-                        prefix,
-                        name,
-                        scope,
-                    }),
-                expr.map(|(prefix, scope, name)| TagType::Expr {
-                    prefix,
-                    name,
-                    scope,
-                }),
-            )),
-            keyword("return")
-                .ignore_then(ident().padded())
-                .then_ignore(end())
-                .map(TagType::Export),
-            misc.to(TagType::Skip),
-        ))
-        .padded()
-        .map_with_span(|t, r| (t, r))
-        .repeated()
-        .parse(src)
+        just("---")
+            .ignore_then(tags)
+            .map_with_span(|t, r| (t, r))
+            .repeated()
+            .parse(src)
     }
 }

src/parser/mod.rs

@@ -87,7 +87,8 @@ impl_parse!(Node, Option<Self>, {
         select! { TagType::Export(x) => Self::Export(x) },
     ))
     .map(Some)
-    // This will skip useless nodes
+    // This will skip extra nodes which were probably injected by the fronted parsers
+    // i.e. ---@func | ---@expr
     .or(any().to(None))
 });