diff options
Diffstat (limited to 'crates/rebel-parse/src/grammar/tokenize.rs')
| -rw-r--r-- | crates/rebel-parse/src/grammar/tokenize.rs | 11 |
1 files changed, 10 insertions, 1 deletions
diff --git a/crates/rebel-parse/src/grammar/tokenize.rs b/crates/rebel-parse/src/grammar/tokenize.rs index a68d7c6..4758e9e 100644 --- a/crates/rebel-parse/src/grammar/tokenize.rs +++ b/crates/rebel-parse/src/grammar/tokenize.rs @@ -23,7 +23,12 @@ peg::parser! { = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } rule punct_char() -> char - = !number() !string() !ident() !__ ch:[_] { ch } + = !comment_start() ch:[ + | '~' | '!' | '@' | '#' | '$' | '%' | '^' | '&' + | '*' | '-' | '=' | '+' | '|' | ';' | ':' | ',' + | '<' | '.' | '>' | '/' | '\'' | '?' | '(' | ')' + | '[' | ']' | '{' | '}' + ] { ch } rule spacing() -> Spacing = &punct_char() { Spacing::Joint } @@ -95,6 +100,10 @@ peg::parser! { rule _ = quiet!{__?} + rule comment_start() + = "//" + / "/*" + rule comment() = "//" (!newline() [_])* (newline() / ![_]) / "/*" (!"*/" [_])* "*/" |