#ifndef CARROT_TOKENIZER_H
#define CARROT_TOKENIZER_H

#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

typedef enum
{
    CARROT_TOKEN_WORD,
    CARROT_TOKEN_SPECIAL,
    CARROT_TOKEN_SPECIAL_POINT,
    CARROT_TOKEN_SPECIAL_COMMA,
    CARROT_TOKEN_SPECIAL_COLON,
    CARROT_TOKEN_SPECIAL_SEMICOLON,
    CARROT_TOKEN_SPECIAL_EQUALS_SIGN,
    CARROT_TOKEN_SPECIAL_AMPERSAND,
    CARROT_TOKEN_SPECIAL_VERTICAL_BAR,
    CARROT_TOKEN_SPECIAL_OPENING_ARROW,
    CARROT_TOKEN_SPECIAL_CLOSING_ARROW,
    CARROT_TOKEN_SPECIAL_OPENING_CURLY_BRACE,
    CARROT_TOKEN_SPECIAL_CLOSING_CURLY_BRACE,
    CARROT_TOKEN_SPECIAL_OPENING_PARENTHESIS,
    CARROT_TOKEN_SPECIAL_CLOSING_PARENTHESIS,
    CARROT_TOKEN_SPECIAL_OPENING_SQUARE_BRACKET,
    CARROT_TOKEN_SPECIAL_CLOSING_SQUARE_BRACKET,
    CARROT_TOKEN_LITERAL_NUMERIC,
    CARROT_TOKEN_LITERAL_STRING,
    CARROT_TOKEN_LITERAL_CHARACTER,
    CARROT_TOKEN_STREAM_END
    
} carrot_token_e;

typedef struct
{
    carrot_token_e                  type;
    uint32_t                        length;
    uint32_t                        char_index;
    
    uint32_t                        line_index;
    uint32_t                        column_index;
    
} carrot_token_s;

typedef struct
{
    uint32_t                        num_tokens;
    carrot_token_s                 *tokens;
    
    uint32_t                        len_source;
    char                           *source;
    
} carrot_token_stream_s;

carrot_token_stream_s   carrot_tokenize             (char *source, uint32_t len_source);
void                    carrot_print_tokens         (carrot_token_stream_s *stream);

#endif