diff options
Diffstat (limited to 'crates/rebel-parse/src/grammar/tokenize.rs')
| -rw-r--r-- | crates/rebel-parse/src/grammar/tokenize.rs | 21 |
1 files changed, 11 insertions, 10 deletions
diff --git a/crates/rebel-parse/src/grammar/tokenize.rs b/crates/rebel-parse/src/grammar/tokenize.rs index b497e23..826f4ce 100644 --- a/crates/rebel-parse/src/grammar/tokenize.rs +++ b/crates/rebel-parse/src/grammar/tokenize.rs @@ -8,7 +8,8 @@ peg::parser! { = _ tokens:(token() ** _) _ { tokens } pub rule token() -> Token<'input> - = literal:literal() { Token::Literal(literal) } + = number:number() { Token::Number(number) } + / string:string() { Token::String(string) } / ident:ident() { Token::Ident(ident) } / punct:punct() { Token::Punct(punct) } @@ -22,24 +23,24 @@ peg::parser! { = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } rule punct_char() -> char - = !literal() !ident() !__ ch:[_] { ch } + = !number() !string() !ident() !__ ch:[_] { ch } rule spacing() -> Spacing = &punct_char() { Spacing::Joint } / { Spacing::Alone } - rule literal() -> Literal<'input> - = content:$(['0'..='9'] ['0'..='9' | 'a'..='z' | 'A'..='Z' | '_']*) { - Literal { content, kind: LiteralKind::Number } - } - / "\"" content:$(string_char()*) "\"" { - Literal { content, kind: LiteralKind::String } + rule number() -> &'input str + = $(['0'..='9'] ['0'..='9' | 'a'..='z' | 'A'..='Z' | '_']*) + + rule string() -> String<'input> + = "\"" content:$(string_char()*) "\"" { + String { content, kind: StringKind::String } } / "r\"" content:$([^'"']*) "\"" { - Literal { content, kind: LiteralKind::RawString } + String { content, kind: StringKind::RawString } } / "```" newline() content:$((!"```" [_])+) "```" { - Literal { content, kind: LiteralKind::ScriptString } + String { content, kind: StringKind::ScriptString } } rule string_char() |