chore(book): document extras

book/src/extras.md

@@ -1 +1,48 @@
 # Using `Extras`
+
+When deriving the `Logos` traits, you may want to convey some internal state
+between your tokens. That is where `Logos::Extras` comes to the rescue.
+
+Each `Lexer` has a public field called `extras` that can be accessed and
+mutated to keep track and modify some internal state. By default,
+this field is set to `()`, but its type can by modified using the derive
+attribute `#[logos(extras = <some type>)]` on your `enum` declaration.
+
+For example, one may want to know the location, both line and column indices,
+of each token. This is especially useful when one needs to report an erroneous
+token to the user, in an user-friendly manner.
+
+```rust,no_run,noplayground
+{{#include ../../logos/examples/extras.rs:tokens}}
+```
+
+The above token definition will hold two tokens: `Newline` and `Word`.
+The former is only used to keep track of the line numbering and will be skipped
+using `Skip` as a return value from its callback function. The latter will be
+a word with `(line, column)` indices.
+
+To make it easy, the lexer will contain the following two extras:
+
++ `extras.0`: the line number;
++ `extras.1`: the char index of the current line.
+
+We now have to define the two callback functions:
+
+```rust,no_run,noplayground
+{{#include ../../logos/examples/extras.rs:callbacks}}
+```
+
+Extras can of course be used for more complicate logic, and there is no limit
+to what you can store within the public `extras` field.
+
+Finally, we provide you the full code that you should be able to run with[^1]:
+```bash
+cd logos/logos
+cargo run --example extras Cargo.toml
+```
+
+[^1] You first need to clone [this repository](https://github.com/maciejhirsz/logos).
+
+```rust,no_run,noplayground
+{{#include ../../logos/examples/extras.rs:all}}
+```

logos/Cargo.toml

@@ -37,6 +37,10 @@ export_derive = ["logos-derive"]
 name = "brainfuck"
 path = "examples/brainfuck.rs"
 
+[[example]]
+name = "extras"
+path = "examples/extras.rs"
+
 [[example]]
 name = "json"
 path = "examples/json.rs"

logos/examples/extras.rs

@@ -0,0 +1,76 @@
+//! Print line and column positions for each word in a file.
+//!
+//! Usage:
+//!     cargo run --example extras <path/to/file>
+//!
+//! Example:
+//!     cargo run --example extras Cargo.toml
+//!
+//! This is a small example on how to use
+//! [`Extras`](https://docs.rs/logos/latest/logos/trait.Logos.html#associatedtype.Extras)
+//! to convey some (mutable) internal state from token to token.
+//!
+//! Here, the extras will be a tuple with the following fields:
+//!
+//! + 0. the line number;
+//! + 1. the char index of the current line.
+//!
+//! From then, one can easily compute the column number of some token by computing:
+//!
+//! ```rust,no_run,no_playgroud
+//! fn get_column(lex: &Lexer<Token>) -> usize {
+//!     lex.span().start - lex.extras.1
+//! }
+//! ```
+
+/* ANCHOR: all */
+use logos::{Lexer, Logos, Skip};
+use std::env;
+use std::fs;
+
+/* ANCHOR: callbacks */
+/// Update the line count and the char index.
+fn newline_callback(lex: &mut Lexer<Token>) -> Skip {
+    lex.extras.0 += 1;
+    lex.extras.1 = lex.span().end;
+    Skip
+}
+
+/// Compute the line and column position for the current word.
+fn word_callback(lex: &mut Lexer<Token>) -> (usize, usize) {
+    let line = lex.extras.0;
+    let column = lex.span().start - lex.extras.1;
+
+    (line, column)
+}
+/* ANCHOR_END: callbacks */
+
+/* ANCHOR: tokens */
+/// Simple tokens to retrieve words and their location.
+#[derive(Debug, Logos)]
+#[logos(extras = (usize, usize))]
+enum Token {
+    #[regex(r"\n", newline_callback)]
+    Newline,
+
+    #[regex(r"\w+", word_callback)]
+    Word((usize, usize)),
+}
+/* ANCHOR_END: tokens */
+
+fn main() {
+    let src = fs::read_to_string(env::args().nth(1).expect("Expected file argument"))
+        .expect("Failed to read file");
+
+    let mut lex = Token::lexer(src.as_str());
+
+    while let Some(token) = lex.next() {
+        match token {
+            Ok(Token::Word((line, column))) => {
+                println!("Word '{}' found at ({}, {})", lex.slice(), line, column);
+            }
+            _ => (),
+        }
+    }
+}
+/* ANCHOR_END: all */