diff options
| author | Matthias Schiffer <mschiffer@universe-factory.net> | 2024-04-28 09:31:37 +0200 |
|---|---|---|
| committer | Matthias Schiffer <mschiffer@universe-factory.net> | 2024-04-28 09:31:37 +0200 |
| commit | fe853d0662cf4845cf6505711d964a5c3fb0cf70 (patch) | |
| tree | 4fb560fd053c25db1149b7e51cf095add8333afb /crates | |
| parent | 0565ef1692f2fc4712cc55ebe25dc88f99d57c38 (diff) | |
| download | rebel-fe853d0662cf4845cf6505711d964a5c3fb0cf70.tar rebel-fe853d0662cf4845cf6505711d964a5c3fb0cf70.zip | |
rebel-parse: tokenize: use positive list of allowed characters for punctuation
Speed up tokenization, and fixes treating incomplete strings' quotes as
punctiation.
Diffstat (limited to 'crates')
| -rw-r--r-- | crates/rebel-parse/src/grammar/tokenize.rs | 11 |
1 files changed, 10 insertions, 1 deletions
diff --git a/crates/rebel-parse/src/grammar/tokenize.rs b/crates/rebel-parse/src/grammar/tokenize.rs index a68d7c6..4758e9e 100644 --- a/crates/rebel-parse/src/grammar/tokenize.rs +++ b/crates/rebel-parse/src/grammar/tokenize.rs @@ -23,7 +23,12 @@ peg::parser! { = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } rule punct_char() -> char - = !number() !string() !ident() !__ ch:[_] { ch } + = !comment_start() ch:[ + | '~' | '!' | '@' | '#' | '$' | '%' | '^' | '&' + | '*' | '-' | '=' | '+' | '|' | ';' | ':' | ',' + | '<' | '.' | '>' | '/' | '\'' | '?' | '(' | ')' + | '[' | ']' | '{' | '}' + ] { ch } rule spacing() -> Spacing = &punct_char() { Spacing::Joint } @@ -95,6 +100,10 @@ peg::parser! { rule _ = quiet!{__?} + rule comment_start() + = "//" + / "/*" + rule comment() = "//" (!newline() [_])* (newline() / ![_]) / "/*" (!"*/" [_])* "*/" |