From fe853d0662cf4845cf6505711d964a5c3fb0cf70 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Sun, 28 Apr 2024 09:31:37 +0200 Subject: rebel-parse: tokenize: use positive list of allowed characters for punctuation Speed up tokenization, and fixes treating incomplete strings' quotes as punctiation. --- crates/rebel-parse/src/grammar/tokenize.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'crates') diff --git a/crates/rebel-parse/src/grammar/tokenize.rs b/crates/rebel-parse/src/grammar/tokenize.rs index a68d7c6..4758e9e 100644 --- a/crates/rebel-parse/src/grammar/tokenize.rs +++ b/crates/rebel-parse/src/grammar/tokenize.rs @@ -23,7 +23,12 @@ peg::parser! { = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } rule punct_char() -> char - = !number() !string() !ident() !__ ch:[_] { ch } + = !comment_start() ch:[ + | '~' | '!' | '@' | '#' | '$' | '%' | '^' | '&' + | '*' | '-' | '=' | '+' | '|' | ';' | ':' | ',' + | '<' | '.' | '>' | '/' | '\'' | '?' | '(' | ')' + | '[' | ']' | '{' | '}' + ] { ch } rule spacing() -> Spacing = &punct_char() { Spacing::Joint } @@ -95,6 +100,10 @@ peg::parser! { rule _ = quiet!{__?} + rule comment_start() + = "//" + / "/*" + rule comment() = "//" (!newline() [_])* (newline() / ![_]) / "/*" (!"*/" [_])* "*/" -- cgit v1.2.3