Parcel/code/inc/tokenizer.h

62 lines
1.6 KiB
C
Raw Normal View History

2023-11-26 18:49:54 +00:00
#ifndef PARCEL_TOKENIZER_H
#define PARCEL_TOKENIZER_H
#include <utility.h>
typedef struct pac_token pac_token_s;
typedef struct pac_tlist pac_tlist_s; // Token List
typedef enum
{
PAC_TOKEN_STRAY = 0,
PAC_TOKEN_WORD,
PAC_TOKEN_KEYWORD_TRUE,
PAC_TOKEN_KEYWORD_FALSE,
PAC_TOKEN_KEYWORD_ALPHA,
PAC_TOKEN_KEYWORD_WORD,
PAC_TOKEN_KEYWORD_INTEGER,
PAC_TOKEN_LIT_STRING,
PAC_TOKEN_LIT_RUNE, // TODO
PAC_TOKEN_LIT_INTEGER, // TODO
PAC_TOKEN_SIGN_OPEN_TAG,
PAC_TOKEN_SIGN_CLOSE_TAG,
PAC_TOKEN_SIGN_EQUALS,
PAC_TOKEN_SIGN_COLON,
PAC_TOKEN_SIGN_COMMA,
PAC_TOKEN_SIGN_HYPHEN,
PAC_TOKEN_SIGN_UNDERSCORE,
PAC_TOKEN_SIGN_VERTICAL_BAR,
PAC_TOKEN_SIGN_SEMICOLON
} pac_token_e;
struct pac_token
{
pac_token_e type;
usz_t offset;
usz_t length;
usz_t line;
usz_t column;
2023-11-26 18:49:54 +00:00
};
struct pac_tlist
{
char *source;
usz_t num_tokens;
pac_token_s *tokens;
// cursor: An index into the 'tokens'-array; used in later stages.
usz_t cursor;
};
pac_token_e pac_word_to_token_type (char *word, usz_t length);
pac_tlist_s pac_tokenize_grammar (char *source, usz_t len_source);
char * pac_stringify_token_type (pac_token_e type);
void pac_display_tlist (pac_tlist_s list);
#endif