diff options
author | Matthias Schiffer <mschiffer@universe-factory.net> | 2024-04-28 22:45:17 +0200 |
---|---|---|
committer | Matthias Schiffer <mschiffer@universe-factory.net> | 2024-04-28 22:45:17 +0200 |
commit | 74ed467e20f75f9301c1008015fc8c8c8e8571f3 (patch) | |
tree | ef98ebdf76bbb477e0a0c845ce9d7fcc76e7082b /crates/rebel-parse/src/grammar/tokenize.rs | |
parent | 611f6aaa7f6fc1b116bc681dec8cb05c4a6a51cd (diff) | |
download | rebel-74ed467e20f75f9301c1008015fc8c8c8e8571f3.tar rebel-74ed467e20f75f9301c1008015fc8c8c8e8571f3.zip |
rebel-parse: use phf for faster keyword parsing
Diffstat (limited to 'crates/rebel-parse/src/grammar/tokenize.rs')
-rw-r--r-- | crates/rebel-parse/src/grammar/tokenize.rs | 24 |
1 files changed, 20 insertions, 4 deletions
diff --git a/crates/rebel-parse/src/grammar/tokenize.rs b/crates/rebel-parse/src/grammar/tokenize.rs index 4758e9e..a696852 100644 --- a/crates/rebel-parse/src/grammar/tokenize.rs +++ b/crates/rebel-parse/src/grammar/tokenize.rs @@ -2,6 +2,16 @@ use crate::token::*; pub use rules::*; +static KEYWORDS: phf::Map<&'static str, Keyword> = phf::phf_map! { + "true" => Keyword::True, + "false" => Keyword::False, + "fetch" => Keyword::Fetch, + "task" => Keyword::Task, + "struct" => Keyword::Struct, + "set" => Keyword::Set, + "map" => Keyword::Map, +}; + peg::parser! { pub grammar rules() for str { pub rule token_stream() -> TokenStream<'input> @@ -10,14 +20,20 @@ peg::parser! { pub rule token() -> Token<'input> = number:number() { Token::Number(number) } / string:string() { Token::Str(string) } - / ident:ident() { Token::Ident(ident) } + / token:ident_or_keyword() { token } / punct:punct() { Token::Punct(punct) } - rule ident() -> &'input str - = $( + rule ident_or_keyword() -> Token<'input> + = s:$( ['a'..='z' | 'A' ..='Z' | '_' ] ['a'..='z' | 'A' ..='Z' | '_' | '0'..='9']* - ) + ) { + if let Some(kw) = KEYWORDS.get(s) { + Token::Keyword(*kw) + } else { + Token::Ident(s) + } + } rule punct() -> Punct = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } |