diff options
| author | Matthias Schiffer <mschiffer@universe-factory.net> | 2024-04-25 19:42:48 +0200 |
|---|---|---|
| committer | Matthias Schiffer <mschiffer@universe-factory.net> | 2024-04-25 21:14:22 +0200 |
| commit | 14be157edccaace010edcf41555174babb984260 (patch) | |
| tree | cd25fdf446b507991b3d5bb7d3aa7c28b51b9df3 | |
| parent | bf44f9a24df6713f89ed95eaa69eab28f102d0cc (diff) | |
| download | rebel-14be157edccaace010edcf41555174babb984260.tar rebel-14be157edccaace010edcf41555174babb984260.zip | |
rebel-parse: tokenize: handle escapes in regular strings
| -rw-r--r-- | crates/rebel-parse/src/grammar/tokenize.rs | 27 |
1 files changed, 22 insertions, 5 deletions
diff --git a/crates/rebel-parse/src/grammar/tokenize.rs b/crates/rebel-parse/src/grammar/tokenize.rs index a64c5e1..811da0d 100644 --- a/crates/rebel-parse/src/grammar/tokenize.rs +++ b/crates/rebel-parse/src/grammar/tokenize.rs @@ -33,9 +33,9 @@ peg::parser! { = $(['0'..='9'] ['0'..='9' | 'a'..='z' | 'A'..='Z' | '_']*) rule string() -> String<'input> - = "\"" chars:$(string_char()*) "\"" { + = "\"" pieces:string_piece()* "\"" { String { - pieces: vec![StringPiece::Chars(chars)], + pieces, kind: StringKind::String, } } @@ -52,9 +52,26 @@ peg::parser! { } } - rule string_char() - = [^'"' | '\\'] - / "\\" [_] + rule string_piece() -> StringPiece<'input> + = chars:$([^'"' | '\\']+) { StringPiece::Chars(chars) } + / "\\" escape:string_escape() { StringPiece::Escape(escape) } + + rule string_escape() -> char + = "n" { '\n' } + / "r" { '\r' } + / "t" { '\t' } + / "\\" { '\\' } + / "\"" { '"' } + / "0" { '\0' } + / "x" digits:$(['0'..='7'] hex_digit()) { + u8::from_str_radix(digits, 16).unwrap().into() + } + / "u{" digits:$(hex_digit()*<1,6>) "}" { ? + u32::from_str_radix(digits, 16).unwrap().try_into().or(Err("Invalid unicode escape")) + } + + rule hex_digit() + = ['0'..='9' | 'a'..='f' | 'A'..='F'] /// Mandatory whitespace rule __ |