trinitrix
/
trixy
Archived
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(parser): Add support for parsing attributes

This commit is contained in:
Benedikt Peetz 2024-03-24 19:16:52 +01:00
parent bf3eb61110
commit 98c3b009df
Signed by: bpeetz
GPG Key ID: A5E94010C3A642AD
9 changed files with 444 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
trixy-parser/docs/grammar.ebnf
View File

@ -20,7 +20,6 @@
#*) #*)
# (* # (*
# Trixy is fully whitespace independent, this means that you can # Trixy is fully whitespace independent, this means that you can
# interleave whitespace in the definitions. # interleave whitespace in the definitions.
@ -31,22 +30,31 @@
CommandSpec = {Function | Namespace | Enumeration | Structure } ; CommandSpec = {Function | Namespace | Enumeration | Structure } ;
Function = {DocComment} "fn" Identifier "(" [NamedType {"," NamedType }] ")" [ "->" Type ] ";" ; Function = {DocComment} {Attribute} "fn" Identifier "(" [NamedType {"," NamedType }] ")" [ "->" Type ] ";" ;
Namespace = {DocComment} "mod" Identifier "{" {Function | Namespace | Enumeration | Structure} "}" ; Namespace = {DocComment} {Attribute} "mod" Identifier "{" {Function | Namespace | Enumeration | Structure} "}" ;
Structure = {DocComment} "struct" Identifier "{" [DocNamedType {"," DocNamedType } [","]] "}"; Structure = {DocComment} {Attribute} "struct" Identifier "{" [DocNamedType {"," DocNamedType } [","]] "}";
Enumeration = {DocComment} "enum" Identifier "{" [DocIdentifier {"," DocIdentifier} [","]] "}"; Enumeration = {DocComment} {Attribute} "enum" Identifier "{" [DocIdentifier {"," DocIdentifier} [","]] "}";
Type = Identifier ["<" Type {"," Type} ">"]; Type = Identifier ["<" Type {"," Type} ">"];
StringLiteral = ["r"] "\"" {ANYTHING} "\"" | "r" "#" {"#"} "\"" {ANYTHING} "#" {"#"} "\"";
Identifier = (CHARACTER | "_") { NUMBER | CHARACTER | "_" } ; Identifier = (CHARACTER | "_") { NUMBER | CHARACTER | "_" } ;
DocIdentifier = {DocComment} (CHARACTER | "_") { NUMBER | CHARACTER | "_" } ; DocIdentifier = {DocComment} {Attribute} (CHARACTER | "_") { NUMBER | CHARACTER | "_" } ;
NamedType = Identifier ":" Type; NamedType = Identifier ":" Type;
DocNamedType = {DocComment} Identifier ":" Type; DocNamedType = {DocComment} {Attribute} Identifier ":" Type;
# (* This is syntax sugar for a `DocAttribute` *)
DocComment = "///" {ANYTHING} LineEnding; DocComment = "///" {ANYTHING} LineEnding;
Attribute = "#" "[" AttributeValue "]" LineEnding;
AttributeValue = DeriveAttribute | DocAttribute | ErrorAttribute | MsgAttribute;
ErrorAttribute = "error";
MsgAttribute = "msg" "(" StringLiteral ")";
DeriveAttribute = "derive" "(" "Error" ")";
DocAttribute = "doc" "=" StringLiteral;
Comment = "//" [ NOT ("/" {ANYTHING} LineEnding) | "//"] {ANYTHING} LineEnding; Comment = "//" [ NOT ("/" {ANYTHING} LineEnding) | "//"] {ANYTHING} LineEnding;
LineEnding = "\\n" | "\\r" | "\\r\\n"; LineEnding = "\\n" | "\\r" | "\\r\\n";

11
trixy-parser/src/command_spec/checked.rs
View File

@ -176,11 +176,20 @@ impl TokenKind {
pub enum Attribute { pub enum Attribute {
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
doc(String), doc(String),
#[allow(non_camel_case_types)]
derive(DeriveValue),
#[allow(non_camel_case_types)]
error,
#[allow(non_camel_case_types)]
msg(String),
} }
impl From<unchecked::Attribute> for Attribute { impl From<unchecked::Attribute> for Attribute {
fn from(value: unchecked::Attribute) -> Self { fn from(value: unchecked::Attribute) -> Self {
match value { match value {
unchecked::Attribute::doc { content: name, .. } => Self::doc(name), unchecked::Attribute::doc { content: name, .. } => Self::doc(name.content),
unchecked::Attribute::derive { value, .. } => Self::derive(value),
unchecked::Attribute::error { .. } => Self::error,
unchecked::Attribute::msg { content, .. } => Self::msg(content.content),
} }
} }
} }

59
trixy-parser/src/command_spec/unchecked.rs
View File

@ -71,7 +71,64 @@ pub enum Declaration {
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
pub enum Attribute { pub enum Attribute {
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
doc { content: String, span: TokenSpan }, doc {
content: StringLiteral,
span: TokenSpan,
},
#[allow(non_camel_case_types)]
derive { value: DeriveValue, span: TokenSpan },
#[allow(non_camel_case_types)]
error { span: TokenSpan },
#[allow(non_camel_case_types)]
msg {
content: StringLiteral,
span: TokenSpan,
},
}
impl Display for Attribute {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Attribute::doc { .. } => f.write_str("doc"),
Attribute::derive { .. } => f.write_str("derive"),
Attribute::error { .. } => f.write_str("error"),
Attribute::msg { .. } => f.write_str("msg"),
}
}
}
impl Attribute {
pub fn span(&self) -> TokenSpan {
match self {
Attribute::doc { span, .. } => *span,
Attribute::derive { span, .. } => *span,
Attribute::error { span, .. } => *span,
Attribute::msg { span, .. } => *span,
}
}
}
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
pub struct StringLiteral {
pub(crate) content: String,
pub(crate) span: TokenSpan,
}
impl From<Token> for StringLiteral {
fn from(value: Token) -> Self {
let span = *value.span();
let content = match value.kind {
TokenKind::StringLiteral(content) => content,
_ => unreachable!("A string literal was expected"),
};
Self { content, span }
}
}
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
pub enum DeriveValue {
Error,
} }
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]

69
trixy-parser/src/lexing/mod.rs
View File

@ -163,6 +163,7 @@ impl Token {
#[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord, Clone)] #[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord, Clone)]
pub enum TokenKind { pub enum TokenKind {
Keyword(Keyword), Keyword(Keyword),
AttributeKeyword(AttributeKeyword),
Identifier(String), Identifier(String),
Colon, Colon,
Semicolon, Semicolon,
@ -174,8 +175,10 @@ pub enum TokenKind {
ParenClose, ParenClose,
SquareOpen, SquareOpen,
SquareClose, SquareClose,
PoundSign,
EqualsSign,
StringLiteral(String),
DocComment(String),
Comment(String), Comment(String),
/// This is not a real TokenKind, but only used for error handling /// This is not a real TokenKind, but only used for error handling
@ -190,13 +193,18 @@ impl TokenKind {
return true; return true;
} }
} }
if let TokenKind::Comment(_) = self { if let TokenKind::AttributeKeyword(_) = self {
if let TokenKind::Comment(_) = other { if let TokenKind::AttributeKeyword(_) = other {
return true; return true;
} }
} }
if let TokenKind::DocComment(_) = self { if let TokenKind::StringLiteral(_) = self {
if let TokenKind::DocComment(_) = other { if let TokenKind::StringLiteral(_) = other {
return true;
}
}
if let TokenKind::Comment(_) = self {
if let TokenKind::Comment(_) = other {
return true; return true;
} }
} }
@ -208,6 +216,7 @@ impl Display for TokenKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
TokenKind::Keyword(word) => write!(f, "KEYWORD({})", word), TokenKind::Keyword(word) => write!(f, "KEYWORD({})", word),
TokenKind::AttributeKeyword(word) => write!(f, "ATTRIBUTE_KEYWORD({})", word),
TokenKind::Identifier(ident) => { TokenKind::Identifier(ident) => {
if ident.is_empty() { if ident.is_empty() {
write!(f, "IDENTIFIER") write!(f, "IDENTIFIER")
@ -215,6 +224,8 @@ impl Display for TokenKind {
write!(f, "IDENTIFIER({})", ident) write!(f, "IDENTIFIER({})", ident)
} }
} }
TokenKind::EqualsSign => f.write_str("EQUALS_SIGN"),
TokenKind::PoundSign => f.write_str("POUND_SIGN"),
TokenKind::Colon => f.write_str("COLON"), TokenKind::Colon => f.write_str("COLON"),
TokenKind::Semicolon => f.write_str("SEMICOLON"), TokenKind::Semicolon => f.write_str("SEMICOLON"),
TokenKind::Comma => f.write_str("COMMA"), TokenKind::Comma => f.write_str("COMMA"),
@ -226,7 +237,7 @@ impl Display for TokenKind {
TokenKind::Dummy => f.write_str("DUMMY"), TokenKind::Dummy => f.write_str("DUMMY"),
TokenKind::SquareOpen => f.write_str("SQUAREOPEN"), TokenKind::SquareOpen => f.write_str("SQUAREOPEN"),
TokenKind::SquareClose => f.write_str("SQUARECLOSE"), TokenKind::SquareClose => f.write_str("SQUARECLOSE"),
TokenKind::DocComment(text) => write!(f, "DOCCOMMENT({})", text), TokenKind::StringLiteral(text) => write!(f, r#"STRING_LITERAL("{}")"#, text),
TokenKind::Comment(text) => write!(f, "COMMENT({})", text), TokenKind::Comment(text) => write!(f, "COMMENT({})", text),
} }
} }
@ -249,6 +260,26 @@ pub enum Keyword {
r#enum, r#enum,
} }
/// Keywords used in attributes: (#[<keyword>(<value>)])
#[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)]
pub enum AttributeKeyword {
/// Derive a trait
#[allow(non_camel_case_types)]
derive,
/// Document the attached item
#[allow(non_camel_case_types)]
doc,
/// Mark the beginning of an error
#[allow(non_camel_case_types)]
error,
/// Encompass an error message
#[allow(non_camel_case_types)]
msg,
}
impl Display for Keyword { impl Display for Keyword {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self { match self {
@ -260,6 +291,17 @@ impl Display for Keyword {
} }
} }
impl Display for AttributeKeyword {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
AttributeKeyword::derive => f.write_str("derive"),
AttributeKeyword::doc => f.write_str("doc"),
AttributeKeyword::error => f.write_str("error"),
AttributeKeyword::msg => f.write_str("msg"),
}
}
}
/// Shorthand macro for generating a token from *anything* which can be /// Shorthand macro for generating a token from *anything* which can be
/// converted into a `TokenKind`, or any of the `TokenKind` variants. /// converted into a `TokenKind`, or any of the `TokenKind` variants.
/// ///
@ -267,11 +309,11 @@ impl Display for Keyword {
/// ///
/// ``` /// ```
/// use trixy_parser::token; /// use trixy_parser::token;
/// # fn main() { ///# fn main() {
/// token![mod]; /// token![mod];
/// token![;]; /// token![;];
/// token![Arrow]; /// token![Arrow];
/// # } ///# }
/// ``` /// ```
#[macro_export] #[macro_export]
macro_rules! token { macro_rules! token {
@ -292,6 +334,10 @@ macro_rules! token {
[BraceClose] => { $crate::lexing::TokenKind::BraceClose }; [BraceClose] => { $crate::lexing::TokenKind::BraceClose };
// [}] => { $crate::lexing::TokenKind::BraceClose }; // [}] => { $crate::lexing::TokenKind::BraceClose };
[ParenOpen] => { $crate::lexing::TokenKind::ParenOpen }; [ParenOpen] => { $crate::lexing::TokenKind::ParenOpen };
[PoundSign] => { $crate::lexing::TokenKind::PoundSign };
[#] => { $crate::lexing::TokenKind::PoundSign };
[EqualsSign] => { $crate::lexing::TokenKind::EqualsSign };
[=] => { $crate::lexing::TokenKind::EqualsSign };
// [(] => { $crate::lexing::TokenKind::ParenthesisOpen }; // [(] => { $crate::lexing::TokenKind::ParenthesisOpen };
[ParenClose] => { $crate::lexing::TokenKind::ParenClose }; [ParenClose] => { $crate::lexing::TokenKind::ParenClose };
// [)] => { $crate::lexing::TokenKind::ParenthesisClose }; // [)] => { $crate::lexing::TokenKind::ParenthesisClose };
@ -301,13 +347,16 @@ macro_rules! token {
[struct] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#struct) }; [struct] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#struct) };
[enum] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#enum) }; [enum] => { $crate::lexing::TokenKind::Keyword($crate::lexing::Keyword::r#enum) };
// The `derive` here is completely arbitrary. It is only for comparisons (see `same_kind`)
[AttributeKeyword] => { $crate::lexing::TokenKind::AttributeKeyword($crate::lexing::AttributeKeyword::derive) };
// This is only works for checking for a identifier or comment // This is only works for checking for a identifier or comment
// see the `same_kind` method on TokenKind // see the `same_kind` method on TokenKind
[Ident] => { $crate::lexing::TokenKind::Identifier("".to_owned()) }; [Ident] => { $crate::lexing::TokenKind::Identifier("".to_owned()) };
[Identifier] => { $crate::lexing::TokenKind::Identifier("".to_owned()) }; [Identifier] => { $crate::lexing::TokenKind::Identifier("".to_owned()) };
[DocComment] => { $crate::lexing::TokenKind::DocComment("".to_owned()) }; [StringLiteral] => { $crate::lexing::TokenKind::StringLiteral("".to_owned()) };
[DocCommentMatch] => { $crate::lexing::TokenKind::DocComment(_doc_comment) };
[Comment] => { $crate::lexing::TokenKind::Comment("".to_owned()) }; [Comment] => { $crate::lexing::TokenKind::Comment("".to_owned()) };
} }

156
trixy-parser/src/lexing/tokenizer.rs
View File

@ -28,7 +28,7 @@ use crate::{
use super::{ use super::{
error::{LexingError, SpannedLexingError}, error::{LexingError, SpannedLexingError},
Token, TokenKind, AttributeKeyword, Token, TokenKind,
}; };
pub(super) struct Tokenizer<'a> { pub(super) struct Tokenizer<'a> {
@ -57,6 +57,20 @@ impl<'a> Tokenizer<'a> {
SpannedLexingError { source: e, context } SpannedLexingError { source: e, context }
})?; })?;
// if let TokenKind::StringLiteral(string) = &token_kind {
// if string == "" {
// eprintln!(
// "Got an empty StringLiteral '{}', with span: {}..{}",
// string,
// start,
// start + index
// );
// eprintln!(
// "Removing following text: '{}'\n",
// &self.remaining_text[..index],
// );
// }
// }
self.chomp(index); // end - start self.chomp(index); // end - start
let end = self.current_index; let end = self.current_index;
@ -83,6 +97,10 @@ impl<'a> Tokenizer<'a> {
',' => (TokenKind::Comma, 1), ',' => (TokenKind::Comma, 1),
'<' => (TokenKind::SquareOpen, 1), '<' => (TokenKind::SquareOpen, 1),
'>' => (TokenKind::SquareClose, 1), '>' => (TokenKind::SquareClose, 1),
'#' => (TokenKind::PoundSign, 1),
'=' => (TokenKind::EqualsSign, 1),
'"' => tokenize_literal_string(self.remaining_text, "\"")?,
'r' => try_to_tokenize_raw_literal_string(self.remaining_text)?,
'-' => tokenize_arrow(self.remaining_text)?, '-' => tokenize_arrow(self.remaining_text)?,
'/' => tokenize_comment(self.remaining_text)?, '/' => tokenize_comment(self.remaining_text)?,
@ -167,38 +185,76 @@ fn tokenize_comment(text: &str) -> Result<(TokenKind, usize), LexingError> {
Err(LexingError::ExpectedComment) Err(LexingError::ExpectedComment)
} else { } else {
let text: &str = &text[2..]; let text: &str = &text[2..];
if let Some('/') = text.chars().next() { if end_of_line(&text) {
let text = &text[1..]; Ok((TokenKind::Comment("".to_owned()), 1 + 2))
if end_of_line(&text) {
Ok((TokenKind::DocComment("".to_owned()), 1 + 3))
} else {
let (doc_comment, chars_read) = take_while(text, |ch| ch != '\n' && ch != '\r')?;
// trim whitespace
let doc_comment = doc_comment.trim_start();
let doc_comment = doc_comment.trim_end();
Ok((
TokenKind::DocComment(doc_comment.to_owned()),
chars_read + 3,
))
}
} else { } else {
if end_of_line(&text) { let (comment, chars_read) = take_while(text, |ch| ch != '\n' && ch != '\r')?;
Ok((TokenKind::Comment("".to_owned()), 1 + 2))
} else {
let (comment, chars_read) = take_while(text, |ch| ch != '\n' && ch != '\r')?;
// trim whitespace // trim trailing whitespace (only at the end to avoid removing wanted whitespace)
let comment = comment.trim_start(); let comment = comment.trim_end();
let comment = comment.trim_end();
Ok((TokenKind::Comment(comment.to_owned()), chars_read + 2)) Ok((TokenKind::Comment(comment.to_owned()), chars_read + 2))
}
} }
} }
} }
/// We check if the r is the beginning of a literal string, otherwise, we tokenize a identifier
fn try_to_tokenize_raw_literal_string(text: &str) -> Result<(TokenKind, usize), LexingError> {
// remove the 'r' at the begining
let text_without_r = &text[1..];
let next_char = &text_without_r[0..1];
if next_char == "#" {
// The string is also escaped, count the hashtags
let (delimeter, chars_read) = take_while(text_without_r, |ch| ch == '#')?;
let delimeter = format!("\"{}", delimeter);
let (token, length) = tokenize_literal_string(&text_without_r[chars_read..], &delimeter)?;
// The 1 is the size of the extra 'r'
Ok((token, length + 1))
} else if next_char == "\"" {
// regular raw string literal
let (token, length) = tokenize_literal_string(text_without_r, "\"")?;
// The 1 is the size of the extra 'r'
Ok((token, length + 1))
} else {
// if the 'r' is not followed by either an '#' or a '"', it must be part of an identifier
tokenize_ident(text)
}
}
fn tokenize_literal_string(text: &str, delimeter: &str) -> Result<(TokenKind, usize), LexingError> {
// The first char is always a quote (")
assert_eq!(&text[..1], "\"");
let text_without_quote = &text[1..];
if &text_without_quote[0..delimeter.len()] == delimeter {
// eprintln!(
// "Got a direct delimeter, removing: '{}'",
// &text[..1 + delimeter.len()]
// );
// eprintln!("Next up to parse: '{}'\n", &text[1 + delimeter.len()..20]);
// The literal string does not contain anything
Ok((TokenKind::StringLiteral("".to_owned()), 1 + delimeter.len()))
} else {
let mut predicates: Vec<_> = delimeter
.chars()
.map(|ch| {
// eprintln!("Condition, which needs to match: |ch| ch == '{}'", ch);
move |ch2| ch2 == ch
})
.collect();
let (literal, chars_read) =
take_until_succesive_match(text_without_quote, &mut predicates)?;
// The second number read here is the last quote
Ok((
TokenKind::StringLiteral(literal.to_owned()),
chars_read + 1 + delimeter.len(),
))
}
}
fn tokenize_ident(text: &str) -> Result<(TokenKind, usize), LexingError> { fn tokenize_ident(text: &str) -> Result<(TokenKind, usize), LexingError> {
let (got, chars_read) = take_while(text, |ch| ch == '_' || ch.is_alphanumeric())?; let (got, chars_read) = take_while(text, |ch| ch == '_' || ch.is_alphanumeric())?;
@ -208,6 +264,12 @@ fn tokenize_ident(text: &str) -> Result<(TokenKind, usize), LexingError> {
"fn" => TokenKind::Keyword(Keyword::r#fn), "fn" => TokenKind::Keyword(Keyword::r#fn),
"struct" => TokenKind::Keyword(Keyword::r#struct), "struct" => TokenKind::Keyword(Keyword::r#struct),
"enum" => TokenKind::Keyword(Keyword::r#enum), "enum" => TokenKind::Keyword(Keyword::r#enum),
"derive" => TokenKind::AttributeKeyword(AttributeKeyword::derive),
"doc" => TokenKind::AttributeKeyword(AttributeKeyword::doc),
"error" => TokenKind::AttributeKeyword(AttributeKeyword::error),
"msg" => TokenKind::AttributeKeyword(AttributeKeyword::msg),
other => TokenKind::Identifier(other.to_string()), other => TokenKind::Identifier(other.to_string()),
}; };
@ -252,6 +314,48 @@ where
Ok((&data[..current_index], current_index)) Ok((&data[..current_index], current_index))
} }
} }
/// Consume bytes until all the predicates match in successive ways
fn take_until_succesive_match<'a, F>(
data: &'a str,
preds: &mut [F],
) -> Result<(&'a str, usize), LexingError>
where
F: FnMut(char) -> bool,
{
assert!(!preds.is_empty(), "Predicates need to be provided");
let mut current_index = 0;
'outer: for ch in data.chars() {
let should_stop = preds[0](ch);
current_index += ch.len_utf8();
if should_stop {
// eprintln!("First predicate did match char: {:#?}", ch);
if preds.len() == 1 {
// eprintln!("Only one predicate provided, which matched: {:#?}\n", ch);
break 'outer;
}
'inner: for pred in &mut preds[1..] {
let ch = &data.chars().nth(current_index).expect("This should exists");
// eprintln!("Checking pred with char: {:#?}", ch);
if pred(*ch) {
// eprintln!("Predicate did match char: {:#?}\n", ch);
break 'outer;
}
// eprintln!("Predicate did not match char: {:#?}\n", ch);
current_index += ch.len_utf8();
break 'inner;
}
}
}
if current_index == 0 {
Err(LexingError::NoMatchesTaken)
} else {
Ok((&data[..current_index], current_index))
}
}
/// Skips input until the remaining string pattern starts with the pattern /// Skips input until the remaining string pattern starts with the pattern
fn skip_until<'a>(mut src: &'a str, pattern: &str) -> &'a str { fn skip_until<'a>(mut src: &'a str, pattern: &str) -> &'a str {

9
trixy-parser/src/parsing/checked/error.rs
View File

@ -24,7 +24,7 @@ use thiserror::Error;
use std::{error::Error, fmt::Display}; use std::{error::Error, fmt::Display};
use crate::{ use crate::{
command_spec::checked::Identifier, command_spec::{checked::Identifier, unchecked::Attribute},
error::{AdditionalHelp, ErrorContext, ErrorContextDisplay}, error::{AdditionalHelp, ErrorContext, ErrorContextDisplay},
lexing::TokenSpan, lexing::TokenSpan,
parsing::unchecked::error::SpannedParsingError as OldSpannedParsingError, parsing::unchecked::error::SpannedParsingError as OldSpannedParsingError,
@ -63,6 +63,11 @@ pub enum ParsingError {
r#type: Identifier, r#type: Identifier,
span: TokenSpan, span: TokenSpan,
}, },
#[error("The {specified} attribute can't be used here!")]
WrongAttributeInPosition {
specified: Attribute,
span: TokenSpan,
},
} }
impl ParsingError { impl ParsingError {
@ -74,6 +79,7 @@ impl ParsingError {
ParsingError::EnumWithNamespaceNamePascal { enum_span, .. } => enum_span, ParsingError::EnumWithNamespaceNamePascal { enum_span, .. } => enum_span,
ParsingError::NotEnoughGenericArgs { span, .. } => span, ParsingError::NotEnoughGenericArgs { span, .. } => span,
ParsingError::TooManyGenericArgs { span, .. } => span, ParsingError::TooManyGenericArgs { span, .. } => span,
ParsingError::WrongAttributeInPosition { span, .. } => span,
} }
} }
} }
@ -87,6 +93,7 @@ impl AdditionalHelp for ParsingError {
| ParsingError::EnumWithNamespaceName {..} => "Change the name of this Enumeration as the generation process in trixy-macros needs to use this name".to_owned(), | ParsingError::EnumWithNamespaceName {..} => "Change the name of this Enumeration as the generation process in trixy-macros needs to use this name".to_owned(),
ParsingError::NotEnoughGenericArgs { got, expected_min, .. } => format!("Add generic args until you have gone from {} to {}", got, expected_min), ParsingError::NotEnoughGenericArgs { got, expected_min, .. } => format!("Add generic args until you have gone from {} to {}", got, expected_min),
ParsingError::TooManyGenericArgs { got, expected_max, .. } => format!("Remove generic args until you have gone from {} to {}", got, expected_max), ParsingError::TooManyGenericArgs { got, expected_max, .. } => format!("Remove generic args until you have gone from {} to {}", got, expected_max),
ParsingError::WrongAttributeInPosition { .. } => format!("Remove this attribute"),
} }
} }
} }

60
trixy-parser/src/parsing/checked/mod.rs
View File

@ -19,7 +19,7 @@
* If not, see <https://www.gnu.org/licenses/>. * If not, see <https://www.gnu.org/licenses/>.
*/ */
use std::mem; use std::{iter, mem};
use convert_case::{Case, Casing}; use convert_case::{Case, Casing};
use trixy_types::BASE_TYPES; use trixy_types::BASE_TYPES;
@ -27,8 +27,8 @@ use trixy_types::BASE_TYPES;
use crate::{ use crate::{
command_spec::{ command_spec::{
checked::{ checked::{
CommandSpec, DocIdentifier, DocNamedType, Enumeration, Function, Identifier, NamedType, self, CommandSpec, DocIdentifier, DocNamedType, Enumeration, Function, Identifier,
Namespace, Structure, Type, NamedType, Namespace, Structure, Type,
}, },
unchecked::{ unchecked::{
CommandSpec as UncheckedCommandSpec, DocNamedType as UncheckedDocNamedType, CommandSpec as UncheckedCommandSpec, DocNamedType as UncheckedDocNamedType,
@ -39,7 +39,7 @@ use crate::{
Variant, Variant,
}, },
error::ErrorContext, error::ErrorContext,
lexing::{TokenKind, TokenSpan}, lexing::{Token, TokenKind, TokenSpan, TokenStream},
}; };
use self::error::{ParsingError, SpannedParsingError}; use self::error::{ParsingError, SpannedParsingError};
@ -48,6 +48,39 @@ pub mod error;
#[cfg(test)] #[cfg(test)]
mod test; mod test;
macro_rules! take_attrs {
($name:expr, $($types:ident),*) => {
$name
.attributes
.into_iter()
.map(|val| {
take_attrs!{@process_val val, $($types),*}
})
.collect::<Result<Vec<_>, _>>()?
};
(@process_val_last $iden:ident) => {
{
let span = $iden.span();
return Err(ParsingError::WrongAttributeInPosition {
specified: $iden,
span,
});
}
};
(@process_val $iden:ident, $val:ident) => {
if let $crate::command_spec::unchecked::Attribute::$val{..} = $iden {
return Ok($iden.into());
};
take_attrs!{@process_val_last $iden}
};
(@process_val $iden:ident, $val:ident, $($other:tt),+ $(,)*) => {
if let $crate::command_spec::unchecked::Attribute::$val{..} = $iden {
return Ok($iden.into());
};
take_attrs!{@process_val $iden, $($other),*}
};
}
struct Parser { struct Parser {
command_spec: UncheckedCommandSpec, command_spec: UncheckedCommandSpec,
structures: Vec<UncheckedStructure>, structures: Vec<UncheckedStructure>,
@ -57,6 +90,7 @@ struct Parser {
impl UncheckedCommandSpec { impl UncheckedCommandSpec {
pub fn process(self, original_file: String) -> Result<CommandSpec, SpannedParsingError> { pub fn process(self, original_file: String) -> Result<CommandSpec, SpannedParsingError> {
let original_file = TokenStream::replace(&original_file).to_string();
let checked = Parser { let checked = Parser {
command_spec: self, command_spec: self,
structures: vec![], structures: vec![],
@ -68,12 +102,6 @@ impl UncheckedCommandSpec {
} }
} }
macro_rules! pass_attrs_along {
($name:ident) => {
$name.attributes.into_iter().map(|a| a.into()).collect()
};
}
impl Parser { impl Parser {
fn parse(mut self) -> Result<CommandSpec, SpannedParsingError> { fn parse(mut self) -> Result<CommandSpec, SpannedParsingError> {
let namespace: UncheckedNamespace = let namespace: UncheckedNamespace =
@ -142,7 +170,7 @@ impl Parser {
structures, structures,
enumerations, enumerations,
namespaces, namespaces,
attributes: pass_attrs_along!(namespace), attributes: take_attrs! {namespace, doc},
}) })
} }
@ -165,7 +193,7 @@ impl Parser {
identifier, identifier,
inputs, inputs,
output, output,
attributes: pass_attrs_along!(function), attributes: take_attrs! {function, doc},
}) })
} }
@ -202,7 +230,7 @@ impl Parser {
mem::take(&mut state.token.kind).to_identifier(Variant::DocNamedType); mem::take(&mut state.token.kind).to_identifier(Variant::DocNamedType);
DocIdentifier { DocIdentifier {
name: ident.name, name: ident.name,
attributes: pass_attrs_along!(state), attributes: take_attrs! {state, doc, msg},
variant: Variant::DocNamedType, variant: Variant::DocNamedType,
} }
}) })
@ -211,7 +239,7 @@ impl Parser {
Ok(Enumeration { Ok(Enumeration {
identifier, identifier,
states, states,
attributes: pass_attrs_along!(enumeration), attributes: take_attrs! {enumeration, doc, derive, error},
}) })
} }
@ -231,7 +259,7 @@ impl Parser {
Ok(Structure { Ok(Structure {
identifier, identifier,
contents, contents,
attributes: pass_attrs_along!(structure), attributes: take_attrs! {structure, doc, derive},
}) })
} }
@ -254,7 +282,7 @@ impl Parser {
Ok(DocNamedType { Ok(DocNamedType {
name, name,
r#type, r#type,
attributes: pass_attrs_along!(doc_named_type), attributes: take_attrs! {doc_named_type, doc},
}) })
} }

15
trixy-parser/src/parsing/unchecked/error.rs
View File

@ -23,7 +23,7 @@ use std::{error::Error, fmt::Display};
use thiserror::Error; use thiserror::Error;
use crate::{ use crate::{
command_spec::unchecked::Attribute, command_spec::unchecked::{Attribute, StringLiteral},
error::{AdditionalHelp, ErrorContext, ErrorContextDisplay}, error::{AdditionalHelp, ErrorContext, ErrorContextDisplay},
lexing::{TokenKind, TokenSpan}, lexing::{TokenKind, TokenSpan},
}; };
@ -46,19 +46,23 @@ pub enum ParsingError {
#[error("Expected a Keyword to start a new declaration, but found: '{actual}'")] #[error("Expected a Keyword to start a new declaration, but found: '{actual}'")]
ExpectedKeyword { actual: TokenKind, span: TokenSpan }, ExpectedKeyword { actual: TokenKind, span: TokenSpan },
#[error("DocComment does not have target")] #[error("Attribute does not have target")]
TrailingDocComment { TrailingAttribute {
comments: Vec<Attribute>, comments: Vec<Attribute>,
span: TokenSpan, span: TokenSpan,
}, },
#[error("Derive value is not known")]
WrongDeriveValue { specified: StringLiteral },
} }
impl ParsingError { impl ParsingError {
pub fn span(&self) -> &TokenSpan { pub fn span(&self) -> &TokenSpan {
match self { match self {
ParsingError::ExpectedDifferentToken { span, .. } => span, ParsingError::ExpectedDifferentToken { span, .. } => span,
ParsingError::ExpectedKeyword { span, .. } => span, ParsingError::ExpectedKeyword { span, .. } => span,
ParsingError::TrailingDocComment { span, .. } => span, ParsingError::TrailingAttribute { span, .. } => span,
ParsingError::UnexpectedEOF { span, .. } => span, ParsingError::UnexpectedEOF { span, .. } => span,
ParsingError::WrongDeriveValue { specified } => &specified.span,
} }
} }
@ -81,8 +85,9 @@ impl AdditionalHelp for ParsingError {
ParsingError::ExpectedKeyword { actual, .. } => format!( ParsingError::ExpectedKeyword { actual, .. } => format!(
"I expected a keyword (that is something like 'fn' or 'mod') but you put a '{}' there!", "I expected a keyword (that is something like 'fn' or 'mod') but you put a '{}' there!",
actual), actual),
ParsingError::TrailingDocComment { .. } => "I expected some target (a function, namespace, enum, or something like this) which this doc comment annotates, but you put nothing there".to_owned(), ParsingError::TrailingAttribute { .. } => "I expected some target (a function, namespace, enum, or something like this) which this attribute annotates, but you put nothing there".to_owned(),
ParsingError::UnexpectedEOF { expected, .. } => format!("Put the expected token ('{expected}') here."), ParsingError::UnexpectedEOF { expected, .. } => format!("Put the expected token ('{expected}') here."),
ParsingError::WrongDeriveValue { specified } => format!("'{}' is not a valid derive value! Take a look a the grammar file", specified.content),
} }
} }
} }

187
trixy-parser/src/parsing/unchecked/mod.rs
View File

@ -19,15 +19,15 @@
* If not, see <https://www.gnu.org/licenses/>. * If not, see <https://www.gnu.org/licenses/>.
*/ */
use std::mem; use std::{iter::once, mem};
use crate::{ use crate::{
command_spec::unchecked::{ command_spec::unchecked::{
Attribute, CommandSpec, Declaration, DocNamedType, DocToken, Enumeration, Function, Attribute, CommandSpec, Declaration, DeriveValue, DocNamedType, DocToken, Enumeration,
NamedType, Namespace, Structure, Type, Function, NamedType, Namespace, StringLiteral, Structure, Type,
}, },
error::ErrorContext, error::ErrorContext,
lexing::{Token, TokenKind, TokenSpan, TokenStream}, lexing::{AttributeKeyword, Token, TokenKind, TokenSpan, TokenStream},
token, token,
}; };
@ -46,7 +46,7 @@ impl TokenStream {
pub(super) struct Parser { pub(super) struct Parser {
token_stream: TokenStream, token_stream: TokenStream,
active_doc_comments: Vec<Attribute>, current_attributes: Vec<Attribute>,
last_span: TokenSpan, last_span: TokenSpan,
} }
@ -55,7 +55,7 @@ impl Parser {
token_stream.reverse(); token_stream.reverse();
Self { Self {
token_stream, token_stream,
active_doc_comments: vec![], current_attributes: vec![],
last_span: TokenSpan::default(), last_span: TokenSpan::default(),
} }
} }
@ -89,38 +89,21 @@ impl Parser {
token![fn] => Ok(Declaration::Function(self.parse_function()?)), token![fn] => Ok(Declaration::Function(self.parse_function()?)),
token![struct] => Ok(Declaration::Structure(self.parse_structure()?)), token![struct] => Ok(Declaration::Structure(self.parse_structure()?)),
token![enum] => Ok(Declaration::Enumeration(self.parse_enumeration()?)), token![enum] => Ok(Declaration::Enumeration(self.parse_enumeration()?)),
token![DocCommentMatch] => { token![#] => {
while self.expect_peek(token![DocComment]) { let attributes = self.parse_attributes()?;
let comment_to_push = { self.current_attributes.extend(attributes);
let doc_comment = self.expect(token![DocComment])?;
let span = *doc_comment.span();
let name = if let TokenKind::DocComment(content) = doc_comment.kind {
content
} else {
unreachable!("The expect should have accounted for that case");
};
Attribute::doc {
content: name,
span,
}
};
self.active_doc_comments.push(comment_to_push);
}
if self.token_stream.is_empty() { if self.token_stream.is_empty() {
fn get_span(attr: Option<&Attribute>) -> TokenSpan { fn get_span(attr: Option<&Attribute>) -> TokenSpan {
match attr.expect("Something should be here") { attr.expect("Something should be here").span()
Attribute::doc { span, .. } => *span,
}
} }
let span = TokenSpan::from_range( let span = TokenSpan::from_range(
get_span(self.active_doc_comments.first()), get_span(self.current_attributes.first()),
get_span(self.active_doc_comments.last()), get_span(self.current_attributes.last()),
); );
Err(ParsingError::TrailingDocComment { Err(ParsingError::TrailingAttribute {
comments: mem::take(&mut self.active_doc_comments), comments: mem::take(&mut self.current_attributes),
span, span,
}) })
} else { } else {
@ -159,29 +142,78 @@ impl Parser {
}) })
} }
fn parse_doc_comments(&mut self) -> Result<Vec<Attribute>, ParsingError> { fn parse_bracket_string_literal(&mut self) -> Result<StringLiteral, ParsingError> {
let mut attrs = mem::take(&mut self.active_doc_comments); self.expect(token![CurvedBracketOpen])?;
let string_literal = self.expect(token![StringLiteral])?;
self.expect(token![CurvedBracketClose])?;
let string_literal = Into::<StringLiteral>::into(string_literal);
Ok(string_literal)
}
while self.expect_peek(token![DocComment]) { fn parse_attribute_value(&mut self) -> Result<Attribute, ParsingError> {
attrs.push({ let ident = self.expect(token![AttributeKeyword])?;
let doc_comment = self.expect(token![DocComment])?; let span = *ident.span();
let span = *doc_comment.span(); let TokenKind::AttributeKeyword(keyword) = ident.kind() else {
let name = if let TokenKind::DocComment(content) = doc_comment.kind { unreachable!("This is checked in the `expect` above")
content };
} else {
unreachable!("The expect should have accounted for that case"); let attribute = match keyword {
}; AttributeKeyword::derive => {
Attribute::doc { let string_literal = self.parse_bracket_string_literal()?;
content: name, match string_literal.content.as_str() {
span, "Error" => Ok(Attribute::derive {
value: DeriveValue::Error,
span,
}),
_ => Err(error::ParsingError::WrongDeriveValue {
specified: string_literal,
}),
} }
}
AttributeKeyword::doc => {
self.expect(token![=])?;
let string_literal = self.expect(token![StringLiteral])?;
let string_literal = Into::<StringLiteral>::into(string_literal);
if self.expect_peek(token![PoundSign]) {
dbg!(&self.token_stream);
}
Ok(Attribute::doc {
content: string_literal,
span,
})
}
AttributeKeyword::error => Ok(Attribute::error { span }),
AttributeKeyword::msg => {
let string_literal = self.parse_bracket_string_literal()?;
Ok(Attribute::msg {
content: string_literal,
span,
})
}
}?;
Ok(attribute)
}
fn parse_attributes(&mut self) -> Result<Vec<Attribute>, ParsingError> {
let mut attrs = mem::take(&mut self.current_attributes);
while self.expect_peek(token![#]) {
attrs.push({
self.expect(token![#])?;
self.expect(token![SquareBracketOpen])?;
let attribue = self.parse_attribute_value()?;
self.expect(token![SquareBracketClose])?;
attribue
}); });
} }
Ok(attrs) Ok(attrs)
} }
fn parse_namespace(&mut self) -> Result<Namespace, ParsingError> { fn parse_namespace(&mut self) -> Result<Namespace, ParsingError> {
let attributes = self.parse_doc_comments()?; let attributes = self.parse_attributes()?;
self.expect(token![mod])?; self.expect(token![mod])?;
let mut namespace = Namespace { let mut namespace = Namespace {
@ -209,29 +241,30 @@ impl Parser {
} }
fn parse_enumeration(&mut self) -> Result<Enumeration, ParsingError> { fn parse_enumeration(&mut self) -> Result<Enumeration, ParsingError> {
let attributes = self.parse_doc_comments()?; let attributes = self.parse_attributes()?;
self.expect(token![enum])?; self.expect(token![enum])?;
let identifier = self.expect(token![Ident])?; let identifier = self.expect(token![Ident])?;
self.expect(token![BraceOpen])?; self.expect(token![BraceOpen])?;
let mut states = vec![]; let mut states = vec![];
if self.expect_peek(token![Ident]) { if self.expect_peek(token![Ident]) || self.expect_peek(token![#]) {
let attributes = self.parse_doc_comments()?; let attributes = self.parse_attributes()?;
states.push(DocToken { states.push(DocToken {
token: self.expect(token![Ident])?, token: self.expect(token![Ident])?,
attributes, attributes,
}); });
}
while self.expect_peek(token![Comma]) { while self.expect_peek(token![Comma]) {
self.expect(token![Comma])?; self.expect(token![Comma])?;
if self.expect_peek(token![Ident]) { if self.expect_peek(token![Ident]) || self.expect_peek(token![#]) {
let attributes = self.parse_doc_comments()?; let attributes = self.parse_attributes()?;
states.push(DocToken { states.push(DocToken {
token: self.expect(token![Ident])?, token: self.expect(token![Ident])?,
attributes, attributes,
}); });
} else { } else {
break; break;
}
} }
} }
self.expect(token![BraceClose])?; self.expect(token![BraceClose])?;
@ -243,7 +276,7 @@ impl Parser {
} }
fn parse_structure(&mut self) -> Result<Structure, ParsingError> { fn parse_structure(&mut self) -> Result<Structure, ParsingError> {
let attributes = self.parse_doc_comments()?; let attributes = self.parse_attributes()?;
self.expect(token![struct])?; self.expect(token![struct])?;
let name = self.expect(token![Ident])?; let name = self.expect(token![Ident])?;
self.expect(token![BraceOpen])?; self.expect(token![BraceOpen])?;
@ -254,7 +287,7 @@ impl Parser {
} }
while self.expect_peek(token![Comma]) { while self.expect_peek(token![Comma]) {
self.expect(token![Comma])?; self.expect(token![Comma])?;
if self.expect_peek(token![Ident]) || self.expect_peek(token![DocComment]) { if self.expect_peek(token![Ident]) || self.expect_peek(token![PoundSign]) {
contents.push(self.parse_doc_named_type()?); contents.push(self.parse_doc_named_type()?);
} else { } else {
break; break;
@ -277,7 +310,7 @@ impl Parser {
} }
fn parse_doc_named_type(&mut self) -> Result<DocNamedType, ParsingError> { fn parse_doc_named_type(&mut self) -> Result<DocNamedType, ParsingError> {
let attributes = self.parse_doc_comments()?; let attributes = self.parse_attributes()?;
let name = self.expect(token![Ident])?; let name = self.expect(token![Ident])?;
self.expect(token![Colon])?; self.expect(token![Colon])?;
let r#type = self.parse_type()?; let r#type = self.parse_type()?;
@ -289,7 +322,7 @@ impl Parser {
} }
fn parse_function(&mut self) -> Result<Function, ParsingError> { fn parse_function(&mut self) -> Result<Function, ParsingError> {
let attributes = self.parse_doc_comments()?; let attributes = self.parse_attributes()?;
self.expect(token![fn])?; self.expect(token![fn])?;
let name = self.expect(token![Ident])?; let name = self.expect(token![Ident])?;
self.expect(token![ParenOpen])?; self.expect(token![ParenOpen])?;
@ -322,20 +355,20 @@ impl Parser {
/// For example: /// For example:
/// ///
/// ```dont_run /// ```dont_run
/// use trixy_lang_parser::{ ///#use trixy_lang_parser::{
/// lexing::{Keyword, TokenKind, TokenStream}, ///# lexing::{Keyword, TokenKind, TokenStream},
/// parsing::unchecked::Parser, ///# parsing::unchecked::Parser,
/// token, ///# token,
/// }; ///#};
/// ///
/// # fn main() { ///# fn main() {
/// let token_stream = TokenStream::lex("mod {}").unwrap(); /// let token_stream = TokenStream::lex("mod {}").unwrap();
/// let parser = Parser::new(token_stream); /// let parser = Parser::new(token_stream);
/// assert_eq!(parser.expect(token![mod]).unwrap(), TokenKind::Keyword(Keyword::mod)); /// assert_eq!(parser.expect(token![mod]).unwrap(), TokenKind::Keyword(Keyword::mod));
/// assert_eq!(parser.expect(token![BraceOpen]).unwrap(), TokenKind::BraceOpen); /// assert_eq!(parser.expect(token![CurlyBracketOpen]).unwrap(), TokenKind::BraceOpen);
/// assert_eq!(parser.expect(token![BraceClose]).unwrap(), TokenKind::BraceClose); /// assert_eq!(parser.expect(token![CurlyBracketClose]).unwrap(), TokenKind::BraceClose);
/// assert!(parser.expect(token![BraceClose]).is_err()); /// assert!(parser.expect(token![CurlyBracketClose]).is_err());
/// # } ///# }
/// ``` /// ```
/// ///
pub(super) fn expect(&mut self, token: TokenKind) -> Result<Token, ParsingError> { pub(super) fn expect(&mut self, token: TokenKind) -> Result<Token, ParsingError> {