1 files changed, 27 insertions, 10 deletions

diff --git a/crates/rebel-parse/examples/parse-string.rs b/crates/rebel-parse/examples/parse-string.rs
index ba6a378..47e30a5 100644
--- a/crates/rebel-parse/examples/parse-string.rs
+++ b/crates/rebel-parse/examples/parse-string.rs
@@ -1,11 +1,12 @@
-use std::{fmt::Debug, time::Instant};
+use std::{fmt::Debug, process, time::Instant};
 
 use clap::{Parser, ValueEnum};
 
-use rebel_parse::recipe;
+use rebel_parse::{recipe, tokenize};
 
-#[derive(Clone, Debug, ValueEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, ValueEnum)]
 enum Rule {
+	Tokenize,
 	Recipe,
 	RecipeStmt,
 	Body,
@@ -28,14 +29,31 @@ fn main() {
 	}
 
 	let start = Instant::now();
+	let result = tokenize::token_stream(input);
+	let dur = Instant::now().duration_since(start);
+	println!("Tokenization took {} us", dur.as_micros());
+
+	let tokens = match result {
+		Ok(value) => value,
+		Err(err) => {
+			println!("{err}");
+			process::exit(1);
+		}
+	};
+
+	let start = Instant::now();
 	let result = match opts.rule {
-		Rule::Recipe => recipe::recipe(input).map(as_debug),
-		Rule::RecipeStmt => recipe::recipe_stmt(input).map(as_debug),
-		Rule::Body => recipe::body(input).map(as_debug),
-		Rule::BodyStmt => recipe::body_stmt(input).map(as_debug),
-		Rule::Expr => recipe::expr(input).map(as_debug),
+		Rule::Tokenize => Ok(as_debug(tokens)),
+		Rule::Recipe => recipe::recipe(&tokens).map(as_debug),
+		Rule::RecipeStmt => recipe::recipe_stmt(&tokens).map(as_debug),
+		Rule::Body => recipe::body(&tokens).map(as_debug),
+		Rule::BodyStmt => recipe::body_stmt(&tokens).map(as_debug),
+		Rule::Expr => recipe::expr(&tokens).map(as_debug),
 	};
-	let dur = Instant::now().duration_since(start);
+	if opts.rule != Rule::Tokenize {
+		let dur = Instant::now().duration_since(start);
+		println!("Parsing took {} us", dur.as_micros());
+	}
 
 	match result {
 		Ok(value) => {
@@ -45,5 +63,4 @@ fn main() {
 			println!("{err}");
 		}
 	};
-	println!("Took {} us", dur.as_micros());
 }