63 lines
1.6 KiB
C
63 lines
1.6 KiB
C
|
|
||
|
|
#ifndef CARROT_TOKENIZER_H
|
||
|
|
#define CARROT_TOKENIZER_H
|
||
|
|
|
||
|
|
#include <stdbool.h>
|
||
|
|
#include <stdint.h>
|
||
|
|
#include <stdio.h>
|
||
|
|
#include <stdlib.h>
|
||
|
|
#include <string.h>
|
||
|
|
|
||
|
|
typedef enum
|
||
|
|
{
|
||
|
|
CARROT_TOKEN_WORD,
|
||
|
|
CARROT_TOKEN_SPECIAL,
|
||
|
|
CARROT_TOKEN_SPECIAL_POINT,
|
||
|
|
CARROT_TOKEN_SPECIAL_COMMA,
|
||
|
|
CARROT_TOKEN_SPECIAL_COLON,
|
||
|
|
CARROT_TOKEN_SPECIAL_SEMICOLON,
|
||
|
|
CARROT_TOKEN_SPECIAL_EQUALS_SIGN,
|
||
|
|
CARROT_TOKEN_SPECIAL_AMPERSAND,
|
||
|
|
CARROT_TOKEN_SPECIAL_VERTICAL_BAR,
|
||
|
|
CARROT_TOKEN_SPECIAL_OPENING_ARROW,
|
||
|
|
CARROT_TOKEN_SPECIAL_CLOSING_ARROW,
|
||
|
|
CARROT_TOKEN_SPECIAL_OPENING_CURLY_BRACE,
|
||
|
|
CARROT_TOKEN_SPECIAL_CLOSING_CURLY_BRACE,
|
||
|
|
CARROT_TOKEN_SPECIAL_OPENING_PARENTHESIS,
|
||
|
|
CARROT_TOKEN_SPECIAL_CLOSING_PARENTHESIS,
|
||
|
|
CARROT_TOKEN_SPECIAL_OPENING_SQUARE_BRACKET,
|
||
|
|
CARROT_TOKEN_SPECIAL_CLOSING_SQUARE_BRACKET,
|
||
|
|
CARROT_TOKEN_LITERAL_NUMERIC,
|
||
|
|
CARROT_TOKEN_LITERAL_STRING,
|
||
|
|
CARROT_TOKEN_LITERAL_CHARACTER,
|
||
|
|
CARROT_TOKEN_STREAM_END
|
||
|
|
|
||
|
|
} carrot_token_e;
|
||
|
|
|
||
|
|
typedef struct
|
||
|
|
{
|
||
|
|
carrot_token_e type;
|
||
|
|
uint32_t length;
|
||
|
|
uint32_t char_index;
|
||
|
|
|
||
|
|
uint32_t line_index;
|
||
|
|
uint32_t column_index;
|
||
|
|
|
||
|
|
} carrot_token_s;
|
||
|
|
|
||
|
|
typedef struct
|
||
|
|
{
|
||
|
|
uint32_t num_tokens;
|
||
|
|
carrot_token_s *tokens;
|
||
|
|
|
||
|
|
uint32_t len_source;
|
||
|
|
char *source;
|
||
|
|
|
||
|
|
} carrot_token_stream_s;
|
||
|
|
|
||
|
|
carrot_token_stream_s carrot_tokenize (char *source, uint32_t len_source);
|
||
|
|
void carrot_print_tokens (carrot_token_stream_s *stream);
|
||
|
|
|
||
|
|
#endif
|
||
|
|
|