From dad99d1a1007aaa221801ceaac0d1abb44d181f4 Mon Sep 17 00:00:00 2001 From: Soispha Date: Sun, 24 Mar 2024 19:27:25 +0100 Subject: [PATCH] refactor(parser): Use predictable names for tokens --- trixy-parser/src/lexing/mod.rs | 59 ++++++++++++++--------- trixy-parser/src/lexing/test.rs | 24 ++++----- trixy-parser/src/lexing/tokenizer.rs | 15 +++--- trixy-parser/src/parsing/unchecked/mod.rs | 18 +++---- 4 files changed, 67 insertions(+), 49 deletions(-) diff --git a/trixy-parser/src/lexing/mod.rs b/trixy-parser/src/lexing/mod.rs index 152e760..febd2bc 100644 --- a/trixy-parser/src/lexing/mod.rs +++ b/trixy-parser/src/lexing/mod.rs @@ -169,16 +169,19 @@ pub enum TokenKind { Semicolon, Comma, Arrow, - BraceOpen, - BraceClose, - ParenOpen, - ParenClose, - SquareOpen, - SquareClose, PoundSign, EqualsSign, StringLiteral(String), + CurlyBracketOpen, + CurlyBracketClose, + CurvedBracketOpen, + CurvedBracketClose, + AngledBracketOpen, + AngledBracketClose, + SquareBracketOpen, + SquareBracketClose, + Comment(String), /// This is not a real TokenKind, but only used for error handling @@ -230,14 +233,17 @@ impl Display for TokenKind { TokenKind::Semicolon => f.write_str("SEMICOLON"), TokenKind::Comma => f.write_str("COMMA"), TokenKind::Arrow => f.write_str("ARROW"), - TokenKind::BraceOpen => f.write_str("BRACEOPEN"), - TokenKind::BraceClose => f.write_str("BRACECLOSE"), - TokenKind::ParenOpen => f.write_str("PARENOPEN"), - TokenKind::ParenClose => f.write_str("PARENCLOSE"), TokenKind::Dummy => f.write_str("DUMMY"), - TokenKind::SquareOpen => f.write_str("SQUAREOPEN"), - TokenKind::SquareClose => f.write_str("SQUARECLOSE"), TokenKind::StringLiteral(text) => write!(f, r#"STRING_LITERAL("{}")"#, text), + + TokenKind::CurlyBracketOpen => f.write_str("CURLY_BRACKET_OPEN"), + TokenKind::CurlyBracketClose => f.write_str("CURLY_BRACKET_CLOSE"), + TokenKind::CurvedBracketOpen => f.write_str("CURVED_BRACKET_OPEN"), + TokenKind::CurvedBracketClose => f.write_str("CURVED_BRACKET_CLOSE"), + TokenKind::AngledBracketOpen => f.write_str("ANGLED_BRACKET_OPEN"), + TokenKind::AngledBracketClose => f.write_str("ANGLED_BRACKET_CLOSE"), + TokenKind::SquareBracketOpen => f.write_str("SQUARE_BRACKET_OPEN"), + TokenKind::SquareBracketClose => f.write_str("SQUARE_BRACKET_CLOSE"), TokenKind::Comment(text) => write!(f, "COMMENT({})", text), } } @@ -325,22 +331,31 @@ macro_rules! token { [,] => { $crate::lexing::TokenKind::Comma }; [Arrow] => { $crate::lexing::TokenKind::Arrow }; [->] => { $crate::lexing::TokenKind::Arrow }; - [SquareOpen] => { $crate::lexing::TokenKind::SquareOpen }; - [<] => { $crate::lexing::TokenKind::SquareOpen }; - [SquareClose] => { $crate::lexing::TokenKind::SquareClose }; - [>] => { $crate::lexing::TokenKind::SquareClose}; - [BraceOpen] => { $crate::lexing::TokenKind::BraceOpen }; - // [{] => { $crate::lexing::TokenKind::BraceOpen }; - [BraceClose] => { $crate::lexing::TokenKind::BraceClose }; - // [}] => { $crate::lexing::TokenKind::BraceClose }; - [ParenOpen] => { $crate::lexing::TokenKind::ParenOpen }; [PoundSign] => { $crate::lexing::TokenKind::PoundSign }; [#] => { $crate::lexing::TokenKind::PoundSign }; [EqualsSign] => { $crate::lexing::TokenKind::EqualsSign }; [=] => { $crate::lexing::TokenKind::EqualsSign }; + + + [AngledBracketOpen] => { $crate::lexing::TokenKind::AngledBracketOpen }; + [<] => { $crate::lexing::TokenKind::AngledBracketOpen }; + + [AngledBracketClose] => { $crate::lexing::TokenKind::AngledBracketClose }; + [>] => { $crate::lexing::TokenKind::AngledBracketClose }; + + [CurlyBracketOpen] => { $crate::lexing::TokenKind::CurlyBracketOpen}; + // [{] => { $crate::lexing::TokenKind::CurlyBracketOpen }; + [CurlyBracketClose] => { $crate::lexing::TokenKind::CurlyBracketClose}; + // [}] => { $crate::lexing::TokenKind::CurlyBracketClose }; + [CurvedBracketOpen] => { $crate::lexing::TokenKind::CurvedBracketOpen}; // [(] => { $crate::lexing::TokenKind::ParenthesisOpen }; - [ParenClose] => { $crate::lexing::TokenKind::ParenClose }; + [CurvedBracketClose] => { $crate::lexing::TokenKind::CurvedBracketClose}; // [)] => { $crate::lexing::TokenKind::ParenthesisClose }; + [SquareBracketOpen] => { $crate::lexing::TokenKind::SquareBracketOpen}; + // [[] => { $crate::lexing::TokenKind::ParenthesisOpen }; + [SquareBracketClose] => { $crate::lexing::TokenKind::SquareBracketClose}; + // []] => { $crate::lexing::TokenKind::ParenthesisClose }; + [mod] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#mod) }; [fn] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#fn) }; diff --git a/trixy-parser/src/lexing/test.rs b/trixy-parser/src/lexing/test.rs index 9112712..b867f71 100644 --- a/trixy-parser/src/lexing/test.rs +++ b/trixy-parser/src/lexing/test.rs @@ -45,7 +45,7 @@ mod commands { }, Token { span: TokenSpan { start: 14, end: 15 }, - kind: TokenKind::BraceOpen, + kind: TokenKind::CurlyBracketOpen, }, Token { span: TokenSpan { start: 20, end: 22 }, @@ -57,7 +57,7 @@ mod commands { }, Token { span: TokenSpan { start: 29, end: 30 }, - kind: TokenKind::ParenOpen, + kind: TokenKind::CurvedBracketOpen, }, Token { span: TokenSpan { start: 30, end: 35 }, @@ -73,7 +73,7 @@ mod commands { }, Token { span: TokenSpan { start: 43, end: 44 }, - kind: TokenKind::ParenClose, + kind: TokenKind::CurvedBracketClose, }, Token { span: TokenSpan { start: 45, end: 47 }, @@ -89,7 +89,7 @@ mod commands { }, Token { span: TokenSpan { start: 56, end: 57 }, - kind: TokenKind::BraceClose, + kind: TokenKind::CurlyBracketClose, }, ]; TokenStream { @@ -136,19 +136,19 @@ mod mod {{ }, Token { span: TokenSpan { start: 9, end: 10 }, - kind: TokenKind::BraceOpen, + kind: TokenKind::CurlyBracketOpen, }, Token { span: TokenSpan { start: 10, end: 11 }, - kind: TokenKind::BraceOpen, + kind: TokenKind::CurlyBracketOpen, }, Token { span: TokenSpan { start: 12, end: 13 }, - kind: TokenKind::BraceClose, + kind: TokenKind::CurlyBracketClose, }, Token { span: TokenSpan { start: 13, end: 14 }, - kind: TokenKind::BraceClose, + kind: TokenKind::CurlyBracketClose, }, ]; TokenStream { @@ -191,19 +191,19 @@ fn test_comments() { }, Token { span: TokenSpan { start: 41, end: 42 }, - kind: TokenKind::BraceOpen, + kind: TokenKind::CurlyBracketOpen, }, Token { span: TokenSpan { start: 42, end: 43 }, - kind: TokenKind::BraceOpen, + kind: TokenKind::CurlyBracketOpen, }, Token { span: TokenSpan { start: 53, end: 54 }, - kind: TokenKind::BraceClose, + kind: TokenKind::CurlyBracketClose, }, Token { span: TokenSpan { start: 54, end: 55 }, - kind: TokenKind::BraceClose, + kind: TokenKind::CurlyBracketClose, }, ]; TokenStream { diff --git a/trixy-parser/src/lexing/tokenizer.rs b/trixy-parser/src/lexing/tokenizer.rs index 9ef8c47..3904911 100644 --- a/trixy-parser/src/lexing/tokenizer.rs +++ b/trixy-parser/src/lexing/tokenizer.rs @@ -88,15 +88,18 @@ impl<'a> Tokenizer<'a> { }; let (tok, length) = match next { - '(' => (TokenKind::ParenOpen, 1), - ')' => (TokenKind::ParenClose, 1), - '{' => (TokenKind::BraceOpen, 1), - '}' => (TokenKind::BraceClose, 1), + '(' => (TokenKind::CurvedBracketOpen, 1), + ')' => (TokenKind::CurvedBracketClose, 1), + '{' => (TokenKind::CurlyBracketOpen, 1), + '}' => (TokenKind::CurlyBracketClose, 1), + '<' => (TokenKind::AngledBracketOpen, 1), + '>' => (TokenKind::AngledBracketClose, 1), + '[' => (TokenKind::SquareBracketOpen, 1), + ']' => (TokenKind::SquareBracketClose, 1), + ':' => (TokenKind::Colon, 1), ';' => (TokenKind::Semicolon, 1), ',' => (TokenKind::Comma, 1), - '<' => (TokenKind::SquareOpen, 1), - '>' => (TokenKind::SquareClose, 1), '#' => (TokenKind::PoundSign, 1), '=' => (TokenKind::EqualsSign, 1), '"' => tokenize_literal_string(self.remaining_text, "\"")?, diff --git a/trixy-parser/src/parsing/unchecked/mod.rs b/trixy-parser/src/parsing/unchecked/mod.rs index c50456f..a244fae 100644 --- a/trixy-parser/src/parsing/unchecked/mod.rs +++ b/trixy-parser/src/parsing/unchecked/mod.rs @@ -222,9 +222,9 @@ impl Parser { ..Default::default() }; - self.expect(token![BraceOpen])?; + self.expect(token![CurlyBracketOpen])?; - while !self.expect_peek(token![BraceClose]) { + while !self.expect_peek(token![CurlyBracketClose]) { let next = self.parse_next()?; match next { Declaration::Function(function) => namespace.functions.push(function), @@ -236,7 +236,7 @@ impl Parser { } } - self.expect(token![BraceClose])?; + self.expect(token![CurlyBracketClose])?; Ok(namespace) } @@ -244,7 +244,7 @@ impl Parser { let attributes = self.parse_attributes()?; self.expect(token![enum])?; let identifier = self.expect(token![Ident])?; - self.expect(token![BraceOpen])?; + self.expect(token![CurlyBracketOpen])?; let mut states = vec![]; if self.expect_peek(token![Ident]) || self.expect_peek(token![#]) { @@ -267,7 +267,7 @@ impl Parser { } } } - self.expect(token![BraceClose])?; + self.expect(token![CurlyBracketClose])?; Ok(Enumeration { identifier, states, @@ -279,7 +279,7 @@ impl Parser { let attributes = self.parse_attributes()?; self.expect(token![struct])?; let name = self.expect(token![Ident])?; - self.expect(token![BraceOpen])?; + self.expect(token![CurlyBracketOpen])?; let mut contents = vec![]; if self.expect_peek(token![Ident]) { @@ -293,7 +293,7 @@ impl Parser { break; } } - self.expect(token![BraceClose])?; + self.expect(token![CurlyBracketClose])?; Ok(Structure { identifier: name, @@ -325,7 +325,7 @@ impl Parser { let attributes = self.parse_attributes()?; self.expect(token![fn])?; let name = self.expect(token![Ident])?; - self.expect(token![ParenOpen])?; + self.expect(token![CurvedBracketOpen])?; let mut inputs = vec![]; if self.expect_peek(token![Ident]) { @@ -336,7 +336,7 @@ impl Parser { inputs.push(self.parse_named_type()?); } - self.expect(token![ParenClose])?; + self.expect(token![CurvedBracketClose])?; let mut output_type = None; if self.expect_peek(token![->]) { self.expect(token![->])?;