trinitrix
/
trixy
Archived
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(parser): Use predictable names for tokens

This commit is contained in:
Benedikt Peetz 2024-03-24 19:27:25 +01:00
parent add0d170eb
commit a4513e8c51
Signed by: bpeetz
GPG Key ID: A5E94010C3A642AD
4 changed files with 67 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
trixy-parser/src/lexing/mod.rs
View File

@ -169,16 +169,19 @@ pub enum TokenKind {
Semicolon, Semicolon,
Comma, Comma,
Arrow, Arrow,
BraceOpen,
BraceClose,
ParenOpen,
ParenClose,
SquareOpen,
SquareClose,
PoundSign, PoundSign,
EqualsSign, EqualsSign,
StringLiteral(String), StringLiteral(String),
CurlyBracketOpen,
CurlyBracketClose,
CurvedBracketOpen,
CurvedBracketClose,
AngledBracketOpen,
AngledBracketClose,
SquareBracketOpen,
SquareBracketClose,
Comment(String), Comment(String),
/// This is not a real TokenKind, but only used for error handling /// This is not a real TokenKind, but only used for error handling
@ -230,14 +233,17 @@ impl Display for TokenKind {
TokenKind::Semicolon => f.write_str("SEMICOLON"), TokenKind::Semicolon => f.write_str("SEMICOLON"),
TokenKind::Comma => f.write_str("COMMA"), TokenKind::Comma => f.write_str("COMMA"),
TokenKind::Arrow => f.write_str("ARROW"), TokenKind::Arrow => f.write_str("ARROW"),
TokenKind::BraceOpen => f.write_str("BRACEOPEN"),
TokenKind::BraceClose => f.write_str("BRACECLOSE"),
TokenKind::ParenOpen => f.write_str("PARENOPEN"),
TokenKind::ParenClose => f.write_str("PARENCLOSE"),
TokenKind::Dummy => f.write_str("DUMMY"), TokenKind::Dummy => f.write_str("DUMMY"),
TokenKind::SquareOpen => f.write_str("SQUAREOPEN"),
TokenKind::SquareClose => f.write_str("SQUARECLOSE"),
TokenKind::StringLiteral(text) => write!(f, r#"STRING_LITERAL("{}")"#, text), TokenKind::StringLiteral(text) => write!(f, r#"STRING_LITERAL("{}")"#, text),
TokenKind::CurlyBracketOpen => f.write_str("CURLY_BRACKET_OPEN"),
TokenKind::CurlyBracketClose => f.write_str("CURLY_BRACKET_CLOSE"),
TokenKind::CurvedBracketOpen => f.write_str("CURVED_BRACKET_OPEN"),
TokenKind::CurvedBracketClose => f.write_str("CURVED_BRACKET_CLOSE"),
TokenKind::AngledBracketOpen => f.write_str("ANGLED_BRACKET_OPEN"),
TokenKind::AngledBracketClose => f.write_str("ANGLED_BRACKET_CLOSE"),
TokenKind::SquareBracketOpen => f.write_str("SQUARE_BRACKET_OPEN"),
TokenKind::SquareBracketClose => f.write_str("SQUARE_BRACKET_CLOSE"),
TokenKind::Comment(text) => write!(f, "COMMENT({})", text), TokenKind::Comment(text) => write!(f, "COMMENT({})", text),
} }
} }
@ -325,22 +331,31 @@ macro_rules! token {
[,] => { $crate::lexing::TokenKind::Comma }; [,] => { $crate::lexing::TokenKind::Comma };
[Arrow] => { $crate::lexing::TokenKind::Arrow }; [Arrow] => { $crate::lexing::TokenKind::Arrow };
[->] => { $crate::lexing::TokenKind::Arrow }; [->] => { $crate::lexing::TokenKind::Arrow };
[SquareOpen] => { $crate::lexing::TokenKind::SquareOpen };
[<] => { $crate::lexing::TokenKind::SquareOpen };
[SquareClose] => { $crate::lexing::TokenKind::SquareClose };
[>] => { $crate::lexing::TokenKind::SquareClose};
[BraceOpen] => { $crate::lexing::TokenKind::BraceOpen };
// [{] => { $crate::lexing::TokenKind::BraceOpen };
[BraceClose] => { $crate::lexing::TokenKind::BraceClose };
// [}] => { $crate::lexing::TokenKind::BraceClose };
[ParenOpen] => { $crate::lexing::TokenKind::ParenOpen };
[PoundSign] => { $crate::lexing::TokenKind::PoundSign }; [PoundSign] => { $crate::lexing::TokenKind::PoundSign };
[#] => { $crate::lexing::TokenKind::PoundSign }; [#] => { $crate::lexing::TokenKind::PoundSign };
[EqualsSign] => { $crate::lexing::TokenKind::EqualsSign }; [EqualsSign] => { $crate::lexing::TokenKind::EqualsSign };
[=] => { $crate::lexing::TokenKind::EqualsSign }; [=] => { $crate::lexing::TokenKind::EqualsSign };
[AngledBracketOpen] => { $crate::lexing::TokenKind::AngledBracketOpen };
[<] => { $crate::lexing::TokenKind::AngledBracketOpen };
[AngledBracketClose] => { $crate::lexing::TokenKind::AngledBracketClose };
[>] => { $crate::lexing::TokenKind::AngledBracketClose };
[CurlyBracketOpen] => { $crate::lexing::TokenKind::CurlyBracketOpen};
// [{] => { $crate::lexing::TokenKind::CurlyBracketOpen };
[CurlyBracketClose] => { $crate::lexing::TokenKind::CurlyBracketClose};
// [}] => { $crate::lexing::TokenKind::CurlyBracketClose };
[CurvedBracketOpen] => { $crate::lexing::TokenKind::CurvedBracketOpen};
// [(] => { $crate::lexing::TokenKind::ParenthesisOpen }; // [(] => { $crate::lexing::TokenKind::ParenthesisOpen };
[ParenClose] => { $crate::lexing::TokenKind::ParenClose }; [CurvedBracketClose] => { $crate::lexing::TokenKind::CurvedBracketClose};
// [)] => { $crate::lexing::TokenKind::ParenthesisClose }; // [)] => { $crate::lexing::TokenKind::ParenthesisClose };
[SquareBracketOpen] => { $crate::lexing::TokenKind::SquareBracketOpen};
// [[] => { $crate::lexing::TokenKind::ParenthesisOpen };
[SquareBracketClose] => { $crate::lexing::TokenKind::SquareBracketClose};
// []] => { $crate::lexing::TokenKind::ParenthesisClose };
[mod] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#mod) }; [mod] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#mod) };
[fn] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#fn) }; [fn] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#fn) };

24
trixy-parser/src/lexing/test.rs
View File

@ -45,7 +45,7 @@ mod commands {
}, },
Token { Token {
span: TokenSpan { start: 14, end: 15 }, span: TokenSpan { start: 14, end: 15 },
kind: TokenKind::BraceOpen, kind: TokenKind::CurlyBracketOpen,
}, },
Token { Token {
span: TokenSpan { start: 20, end: 22 }, span: TokenSpan { start: 20, end: 22 },
@ -57,7 +57,7 @@ mod commands {
}, },
Token { Token {
span: TokenSpan { start: 29, end: 30 }, span: TokenSpan { start: 29, end: 30 },
kind: TokenKind::ParenOpen, kind: TokenKind::CurvedBracketOpen,
}, },
Token { Token {
span: TokenSpan { start: 30, end: 35 }, span: TokenSpan { start: 30, end: 35 },
@ -73,7 +73,7 @@ mod commands {
}, },
Token { Token {
span: TokenSpan { start: 43, end: 44 }, span: TokenSpan { start: 43, end: 44 },
kind: TokenKind::ParenClose, kind: TokenKind::CurvedBracketClose,
}, },
Token { Token {
span: TokenSpan { start: 45, end: 47 }, span: TokenSpan { start: 45, end: 47 },
@ -89,7 +89,7 @@ mod commands {
}, },
Token { Token {
span: TokenSpan { start: 56, end: 57 }, span: TokenSpan { start: 56, end: 57 },
kind: TokenKind::BraceClose, kind: TokenKind::CurlyBracketClose,
}, },
]; ];
TokenStream { TokenStream {
@ -136,19 +136,19 @@ mod mod {{
}, },
Token { Token {
span: TokenSpan { start: 9, end: 10 }, span: TokenSpan { start: 9, end: 10 },
kind: TokenKind::BraceOpen, kind: TokenKind::CurlyBracketOpen,
}, },
Token { Token {
span: TokenSpan { start: 10, end: 11 }, span: TokenSpan { start: 10, end: 11 },
kind: TokenKind::BraceOpen, kind: TokenKind::CurlyBracketOpen,
}, },
Token { Token {
span: TokenSpan { start: 12, end: 13 }, span: TokenSpan { start: 12, end: 13 },
kind: TokenKind::BraceClose, kind: TokenKind::CurlyBracketClose,
}, },
Token { Token {
span: TokenSpan { start: 13, end: 14 }, span: TokenSpan { start: 13, end: 14 },
kind: TokenKind::BraceClose, kind: TokenKind::CurlyBracketClose,
}, },
]; ];
TokenStream { TokenStream {
@ -191,19 +191,19 @@ fn test_comments() {
}, },
Token { Token {
span: TokenSpan { start: 41, end: 42 }, span: TokenSpan { start: 41, end: 42 },
kind: TokenKind::BraceOpen, kind: TokenKind::CurlyBracketOpen,
}, },
Token { Token {
span: TokenSpan { start: 42, end: 43 }, span: TokenSpan { start: 42, end: 43 },
kind: TokenKind::BraceOpen, kind: TokenKind::CurlyBracketOpen,
}, },
Token { Token {
span: TokenSpan { start: 53, end: 54 }, span: TokenSpan { start: 53, end: 54 },
kind: TokenKind::BraceClose, kind: TokenKind::CurlyBracketClose,
}, },
Token { Token {
span: TokenSpan { start: 54, end: 55 }, span: TokenSpan { start: 54, end: 55 },
kind: TokenKind::BraceClose, kind: TokenKind::CurlyBracketClose,
}, },
]; ];
TokenStream { TokenStream {

15
trixy-parser/src/lexing/tokenizer.rs
View File

@ -88,15 +88,18 @@ impl<'a> Tokenizer<'a> {
}; };
let (tok, length) = match next { let (tok, length) = match next {
'(' => (TokenKind::ParenOpen, 1), '(' => (TokenKind::CurvedBracketOpen, 1),
')' => (TokenKind::ParenClose, 1), ')' => (TokenKind::CurvedBracketClose, 1),
'{' => (TokenKind::BraceOpen, 1), '{' => (TokenKind::CurlyBracketOpen, 1),
'}' => (TokenKind::BraceClose, 1), '}' => (TokenKind::CurlyBracketClose, 1),
'<' => (TokenKind::AngledBracketOpen, 1),
'>' => (TokenKind::AngledBracketClose, 1),
'[' => (TokenKind::SquareBracketOpen, 1),
']' => (TokenKind::SquareBracketClose, 1),
':' => (TokenKind::Colon, 1), ':' => (TokenKind::Colon, 1),
';' => (TokenKind::Semicolon, 1), ';' => (TokenKind::Semicolon, 1),
',' => (TokenKind::Comma, 1), ',' => (TokenKind::Comma, 1),
'<' => (TokenKind::SquareOpen, 1),
'>' => (TokenKind::SquareClose, 1),
'#' => (TokenKind::PoundSign, 1), '#' => (TokenKind::PoundSign, 1),
'=' => (TokenKind::EqualsSign, 1), '=' => (TokenKind::EqualsSign, 1),
'"' => tokenize_literal_string(self.remaining_text, "\"")?, '"' => tokenize_literal_string(self.remaining_text, "\"")?,

18
trixy-parser/src/parsing/unchecked/mod.rs
View File

@ -222,9 +222,9 @@ impl Parser {
..Default::default() ..Default::default()
}; };
self.expect(token![BraceOpen])?; self.expect(token![CurlyBracketOpen])?;
while !self.expect_peek(token![BraceClose]) { while !self.expect_peek(token![CurlyBracketClose]) {
let next = self.parse_next()?; let next = self.parse_next()?;
match next { match next {
Declaration::Function(function) => namespace.functions.push(function), Declaration::Function(function) => namespace.functions.push(function),
@ -236,7 +236,7 @@ impl Parser {
} }
} }
self.expect(token![BraceClose])?; self.expect(token![CurlyBracketClose])?;
Ok(namespace) Ok(namespace)
} }
@ -244,7 +244,7 @@ impl Parser {
let attributes = self.parse_attributes()?; let attributes = self.parse_attributes()?;
self.expect(token![enum])?; self.expect(token![enum])?;
let identifier = self.expect(token![Ident])?; let identifier = self.expect(token![Ident])?;
self.expect(token![BraceOpen])?; self.expect(token![CurlyBracketOpen])?;
let mut states = vec![]; let mut states = vec![];
if self.expect_peek(token![Ident]) || self.expect_peek(token![#]) { if self.expect_peek(token![Ident]) || self.expect_peek(token![#]) {
@ -267,7 +267,7 @@ impl Parser {
} }
} }
} }
self.expect(token![BraceClose])?; self.expect(token![CurlyBracketClose])?;
Ok(Enumeration { Ok(Enumeration {
identifier, identifier,
states, states,
@ -279,7 +279,7 @@ impl Parser {
let attributes = self.parse_attributes()?; let attributes = self.parse_attributes()?;
self.expect(token![struct])?; self.expect(token![struct])?;
let name = self.expect(token![Ident])?; let name = self.expect(token![Ident])?;
self.expect(token![BraceOpen])?; self.expect(token![CurlyBracketOpen])?;
let mut contents = vec![]; let mut contents = vec![];
if self.expect_peek(token![Ident]) { if self.expect_peek(token![Ident]) {
@ -293,7 +293,7 @@ impl Parser {
break; break;
} }
} }
self.expect(token![BraceClose])?; self.expect(token![CurlyBracketClose])?;
Ok(Structure { Ok(Structure {
identifier: name, identifier: name,
@ -325,7 +325,7 @@ impl Parser {
let attributes = self.parse_attributes()?; let attributes = self.parse_attributes()?;
self.expect(token![fn])?; self.expect(token![fn])?;
let name = self.expect(token![Ident])?; let name = self.expect(token![Ident])?;
self.expect(token![ParenOpen])?; self.expect(token![CurvedBracketOpen])?;
let mut inputs = vec![]; let mut inputs = vec![];
if self.expect_peek(token![Ident]) { if self.expect_peek(token![Ident]) {
@ -336,7 +336,7 @@ impl Parser {
inputs.push(self.parse_named_type()?); inputs.push(self.parse_named_type()?);
} }
self.expect(token![ParenClose])?; self.expect(token![CurvedBracketClose])?;
let mut output_type = None; let mut output_type = None;
if self.expect_peek(token![->]) { if self.expect_peek(token![->]) {
self.expect(token![->])?; self.expect(token![->])?;