use crate::token::*; pub use rules::*; peg::parser! { pub grammar rules() for str { pub rule token_stream() -> Vec> = _ tokens:(token() ** _) _ { tokens } pub rule token() -> Token<'input> = literal:literal() { Token::Literal(literal) } / ident:ident() { Token::Ident(ident) } / punct:punct() { Token::Punct(punct) } rule ident() -> Ident<'input> = name:$( ['a'..='z' | 'A' ..='Z' | '_' ] ['a'..='z' | 'A' ..='Z' | '_' | '0'..='9']* ) { Ident(name) } rule punct() -> Punct = ch:punct_char() spacing:spacing() { Punct(ch, spacing) } rule punct_char() -> char = !literal() !ident() !__ ch:[_] { ch } rule spacing() -> Spacing = &punct_char() { Spacing::Joint } / { Spacing::Alone } rule literal() -> Literal<'input> = content:$(['0'..='9'] ['0'..='9' | 'a'..='z' | 'A'..='Z' | '_']*) { Literal { content, kind: LiteralKind::Number } } / "\"" content:$(string_char()*) "\"" { Literal { content, kind: LiteralKind::String } } / "r\"" content:$([^'"']*) "\"" { Literal { content, kind: LiteralKind::RawString } } / "```" newline() content:$((!"```" [_])+) "```" { Literal { content, kind: LiteralKind::ScriptString } } rule string_char() = [^'"' | '\\'] / "\\" [_] /// Mandatory whitespace rule __ = ([' ' | '\t'] / quiet!{newline()} / quiet!{comment()})+ /// Optional whitespace rule _ = quiet!{__?} rule comment() = "//" (!newline() [_])* (newline() / ![_]) / "/*" (!"*/" [_])* "*/" rule newline() = ['\n' | '\r'] } }