use crate::token::*; pub use rules::*; peg::parser! { pub grammar rules() for str { pub rule token_stream() -> Vec> = _ tokens:(token() ** _) _ { tokens } pub rule token() -> Token<'input> = number:number() { Token::Number(number) } / string:string() { Token::String(string) } / ident:ident() { Token::Ident(ident) } / punct:punct() { Token::Punct(punct) } rule ident() -> &'input str = $( ['a'..='z' | 'A' ..='Z' | '_' ] ['a'..='z' | 'A' ..='Z' | '_' | '0'..='9']* ) rule punct() -> Punct = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } rule punct_char() -> char = !number() !string() !ident() !__ ch:[_] { ch } rule spacing() -> Spacing = &punct_char() { Spacing::Joint } / { Spacing::Alone } rule number() -> &'input str = $(['0'..='9'] ['0'..='9' | 'a'..='z' | 'A'..='Z' | '_']*) rule string() -> String<'input> = "\"" content:$(string_char()*) "\"" { String { content, kind: StringKind::String } } / "r\"" content:$([^'"']*) "\"" { String { content, kind: StringKind::RawString } } / "```" newline() content:$((!"```" [_])+) "```" { String { content, kind: StringKind::ScriptString } } rule string_char() = [^'"' | '\\'] / "\\" [_] /// Mandatory whitespace rule __ = ([' ' | '\t'] / quiet!{newline()} / quiet!{comment()})+ /// Optional whitespace rule _ = quiet!{__?} rule comment() = "//" (!newline() [_])* (newline() / ![_]) / "/*" (!"*/" [_])* "*/" rule newline() = ['\n' | '\r'] } }