59 lines
1.5 KiB
C
59 lines
1.5 KiB
C
|
|
||
|
|
#ifndef PARCEL_TOKENIZER_H
|
||
|
|
#define PARCEL_TOKENIZER_H
|
||
|
|
|
||
|
|
#include <utility.h>
|
||
|
|
|
||
|
|
typedef struct pac_token pac_token_s;
|
||
|
|
typedef struct pac_tlist pac_tlist_s; // Token List
|
||
|
|
|
||
|
|
typedef enum
|
||
|
|
{
|
||
|
|
PAC_TOKEN_STRAY = 0,
|
||
|
|
PAC_TOKEN_WORD,
|
||
|
|
PAC_TOKEN_KEYWORD_TRUE,
|
||
|
|
PAC_TOKEN_KEYWORD_FALSE,
|
||
|
|
PAC_TOKEN_KEYWORD_ALPHA,
|
||
|
|
PAC_TOKEN_KEYWORD_WORD,
|
||
|
|
PAC_TOKEN_KEYWORD_INTEGER,
|
||
|
|
|
||
|
|
PAC_TOKEN_LIT_STRING,
|
||
|
|
PAC_TOKEN_LIT_RUNE, // TODO
|
||
|
|
PAC_TOKEN_LIT_INTEGER, // TODO
|
||
|
|
|
||
|
|
PAC_TOKEN_SIGN_OPEN_TAG,
|
||
|
|
PAC_TOKEN_SIGN_CLOSE_TAG,
|
||
|
|
PAC_TOKEN_SIGN_EQUALS,
|
||
|
|
PAC_TOKEN_SIGN_COLON,
|
||
|
|
PAC_TOKEN_SIGN_COMMA,
|
||
|
|
PAC_TOKEN_SIGN_HYPHEN,
|
||
|
|
PAC_TOKEN_SIGN_UNDERSCORE,
|
||
|
|
PAC_TOKEN_SIGN_VERTICAL_BAR,
|
||
|
|
PAC_TOKEN_SIGN_SEMICOLON
|
||
|
|
|
||
|
|
} pac_token_e;
|
||
|
|
|
||
|
|
struct pac_token
|
||
|
|
{
|
||
|
|
pac_token_e type;
|
||
|
|
usz_t offset;
|
||
|
|
usz_t length;
|
||
|
|
};
|
||
|
|
|
||
|
|
struct pac_tlist
|
||
|
|
{
|
||
|
|
char *source;
|
||
|
|
usz_t num_tokens;
|
||
|
|
pac_token_s *tokens;
|
||
|
|
|
||
|
|
// cursor: An index into the 'tokens'-array; used in later stages.
|
||
|
|
usz_t cursor;
|
||
|
|
};
|
||
|
|
|
||
|
|
pac_token_e pac_word_to_token_type (char *word, usz_t length);
|
||
|
|
pac_tlist_s pac_tokenize_grammar (char *source, usz_t len_source);
|
||
|
|
char * pac_stringify_token_type (pac_token_e type);
|
||
|
|
void pac_display_tlist (pac_tlist_s list);
|
||
|
|
|
||
|
|
#endif
|