Parcel/core-parser/inc/tokenizer.h

69 lines
1.8 KiB
C

#ifndef PARCEL_TOKENIZER_H
#define PARCEL_TOKENIZER_H
#include <parcel/utility/utility.h>
typedef struct pac_token pac_token_s;
typedef struct pac_tlist pac_tlist_s; // Token List
typedef enum
{
PAC_TOKEN_STRAY = 0,
PAC_TOKEN_WORD,
PAC_TOKEN_KEYWORD_TRUE,
PAC_TOKEN_KEYWORD_FALSE,
PAC_TOKEN_KEYWORD_ALPHA,
PAC_TOKEN_KEYWORD_WORD,
PAC_TOKEN_KEYWORD_INTEGER,
PAC_TOKEN_LIT_STRING,
PAC_TOKEN_LIT_RUNE, // TODO
PAC_TOKEN_LIT_INTEGER, // TODO
PAC_TOKEN_SIGN_OPENING_TAG,
PAC_TOKEN_SIGN_CLOSING_TAG,
PAC_TOKEN_SIGN_OPENING_BRACKET,
PAC_TOKEN_SIGN_CLOSING_BRACKET,
PAC_TOKEN_SIGN_EQUALS,
PAC_TOKEN_SIGN_SEMICOLON,
PAC_TOKEN_SIGN_COLON,
PAC_TOKEN_SIGN_COMMA,
PAC_TOKEN_SIGN_UNDERSCORE,
PAC_TOKEN_SIGN_VERTICAL_BAR,
PAC_TOKEN_SIGN_AMPERSAND,
PAC_TOKEN_SIGN_DOLLAR,
PAC_TOKEN_SIGN_SLASH,
PAC_TOKEN_SIGN_HASH,
PAC_TOKEN_SIGN_AT,
PAC_TOKEN_SIGN_PLUS,
PAC_TOKEN_SIGN_MINUS,
} pac_token_e;
struct pac_token
{
pac_token_e type;
usz_t offset;
usz_t length;
usz_t line;
usz_t column;
};
struct pac_tlist
{
char *source;
usz_t num_tokens;
pac_token_s *tokens;
};
pac_tlist_s pac_tokenize_grammar (char *source, usz_t len_source);
void pac_delete_token_list (pac_tlist_s list);
void pac_display_tlist (pac_tlist_s list);
pac_token_e pac_word_to_token_type (char *word, usz_t length);
char * pac_stringify_token_type (pac_token_e type);
#endif