69 lines
1.8 KiB
C
69 lines
1.8 KiB
C
|
|
#ifndef PARCEL_TOKENIZER_H
|
|
#define PARCEL_TOKENIZER_H
|
|
|
|
#include <parcel/utility/utility.h>
|
|
|
|
typedef struct pac_token pac_token_s;
|
|
typedef struct pac_tlist pac_tlist_s; // Token List
|
|
|
|
typedef enum
|
|
{
|
|
PAC_TOKEN_STRAY = 0,
|
|
PAC_TOKEN_WORD,
|
|
PAC_TOKEN_KEYWORD_TRUE,
|
|
PAC_TOKEN_KEYWORD_FALSE,
|
|
PAC_TOKEN_KEYWORD_ALPHA,
|
|
PAC_TOKEN_KEYWORD_WORD,
|
|
PAC_TOKEN_KEYWORD_INTEGER,
|
|
|
|
PAC_TOKEN_LIT_STRING,
|
|
PAC_TOKEN_LIT_RUNE, // TODO
|
|
PAC_TOKEN_LIT_INTEGER, // TODO
|
|
|
|
PAC_TOKEN_SIGN_OPENING_TAG,
|
|
PAC_TOKEN_SIGN_CLOSING_TAG,
|
|
PAC_TOKEN_SIGN_OPENING_BRACKET,
|
|
PAC_TOKEN_SIGN_CLOSING_BRACKET,
|
|
PAC_TOKEN_SIGN_EQUALS,
|
|
PAC_TOKEN_SIGN_SEMICOLON,
|
|
PAC_TOKEN_SIGN_COLON,
|
|
PAC_TOKEN_SIGN_COMMA,
|
|
PAC_TOKEN_SIGN_UNDERSCORE,
|
|
PAC_TOKEN_SIGN_VERTICAL_BAR,
|
|
PAC_TOKEN_SIGN_AMPERSAND,
|
|
PAC_TOKEN_SIGN_DOLLAR,
|
|
PAC_TOKEN_SIGN_SLASH,
|
|
PAC_TOKEN_SIGN_HASH,
|
|
PAC_TOKEN_SIGN_AT,
|
|
PAC_TOKEN_SIGN_PLUS,
|
|
PAC_TOKEN_SIGN_MINUS,
|
|
|
|
} pac_token_e;
|
|
|
|
struct pac_token
|
|
{
|
|
pac_token_e type;
|
|
usz_t offset;
|
|
usz_t length;
|
|
|
|
usz_t line;
|
|
usz_t column;
|
|
};
|
|
|
|
struct pac_tlist
|
|
{
|
|
char *source;
|
|
usz_t num_tokens;
|
|
pac_token_s *tokens;
|
|
};
|
|
|
|
pac_tlist_s pac_tokenize_grammar (char *source, usz_t len_source);
|
|
void pac_delete_token_list (pac_tlist_s list);
|
|
void pac_display_tlist (pac_tlist_s list);
|
|
|
|
pac_token_e pac_word_to_token_type (char *word, usz_t length);
|
|
char * pac_stringify_token_type (pac_token_e type);
|
|
|
|
#endif
|