diff options
Diffstat (limited to 'crates/rebel-parse/src/grammar/tokenize.rs')
| -rw-r--r-- | crates/rebel-parse/src/grammar/tokenize.rs | 24 |
1 files changed, 20 insertions, 4 deletions
diff --git a/crates/rebel-parse/src/grammar/tokenize.rs b/crates/rebel-parse/src/grammar/tokenize.rs index 4758e9e..a696852 100644 --- a/crates/rebel-parse/src/grammar/tokenize.rs +++ b/crates/rebel-parse/src/grammar/tokenize.rs @@ -2,6 +2,16 @@ use crate::token::*; pub use rules::*; +static KEYWORDS: phf::Map<&'static str, Keyword> = phf::phf_map! { + "true" => Keyword::True, + "false" => Keyword::False, + "fetch" => Keyword::Fetch, + "task" => Keyword::Task, + "struct" => Keyword::Struct, + "set" => Keyword::Set, + "map" => Keyword::Map, +}; + peg::parser! { pub grammar rules() for str { pub rule token_stream() -> TokenStream<'input> @@ -10,14 +20,20 @@ peg::parser! { pub rule token() -> Token<'input> = number:number() { Token::Number(number) } / string:string() { Token::Str(string) } - / ident:ident() { Token::Ident(ident) } + / token:ident_or_keyword() { token } / punct:punct() { Token::Punct(punct) } - rule ident() -> &'input str - = $( + rule ident_or_keyword() -> Token<'input> + = s:$( ['a'..='z' | 'A' ..='Z' | '_' ] ['a'..='z' | 'A' ..='Z' | '_' | '0'..='9']* - ) + ) { + if let Some(kw) = KEYWORDS.get(s) { + Token::Keyword(*kw) + } else { + Token::Ident(s) + } + } rule punct() -> Punct = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } |