Parcel/core-parser/inc/tokenizer.h

#ifndef PARCEL_TOKENIZER_H
#define PARCEL_TOKENIZER_H

#include <parcel/utility/utility.h>

typedef struct pac_token                    pac_token_s;
typedef struct pac_tlist                    pac_tlist_s;            // Token List

typedef enum
{
    PAC_TOKEN_STRAY             = 0,
    PAC_TOKEN_WORD,
    PAC_TOKEN_KEYWORD_TRUE,
    PAC_TOKEN_KEYWORD_FALSE,
    PAC_TOKEN_KEYWORD_ALPHA,
    PAC_TOKEN_KEYWORD_WORD,
    PAC_TOKEN_KEYWORD_INTEGER,

    PAC_TOKEN_LIT_STRING,
    PAC_TOKEN_LIT_RUNE,                 // TODO
    PAC_TOKEN_LIT_INTEGER,              // TODO

    PAC_TOKEN_SIGN_OPENING_TAG,
    PAC_TOKEN_SIGN_CLOSING_TAG,
    PAC_TOKEN_SIGN_OPENING_BRACKET,
    PAC_TOKEN_SIGN_CLOSING_BRACKET,
    PAC_TOKEN_SIGN_EQUALS,
    PAC_TOKEN_SIGN_SEMICOLON,
    PAC_TOKEN_SIGN_COLON,
    PAC_TOKEN_SIGN_COMMA,
    PAC_TOKEN_SIGN_UNDERSCORE,
    PAC_TOKEN_SIGN_VERTICAL_BAR,
    PAC_TOKEN_SIGN_AMPERSAND,
    PAC_TOKEN_SIGN_DOLLAR,
    PAC_TOKEN_SIGN_SLASH,
    PAC_TOKEN_SIGN_HASH,
    PAC_TOKEN_SIGN_AT,
    PAC_TOKEN_SIGN_PLUS,
    PAC_TOKEN_SIGN_MINUS,

} pac_token_e;

struct pac_token
{
    pac_token_e                 type;
    usz_t                       offset;
    usz_t                       length;
    
    usz_t                       line;
    usz_t                       column;
};

struct pac_tlist
{
    char                       *source;
    usz_t                       num_tokens;
    pac_token_s                *tokens;
};

pac_tlist_s             pac_tokenize_grammar            (char *source, usz_t len_source);
void                    pac_delete_token_list           (pac_tlist_s list);
void                    pac_display_tlist               (pac_tlist_s list);

pac_token_e             pac_word_to_token_type          (char *word, usz_t length);
char *                  pac_stringify_token_type        (pac_token_e type);

#endif