epickh
/
Parcel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: epickh:3c59e4fb9321a99982fe6adaccb056672667ab0d
Branches Tags
epickh:main
..
compare: epickh:cf124ffcdc18da3e30d5641c8272fc20e9172197
Branches Tags
epickh:main

No commits in common. "3c59e4fb9321a99982fe6adaccb056672667ab0d" and "cf124ffcdc18da3e30d5641c8272fc20e9172197" have entirely different histories.
3c59e4fb93 ... cf124ffcdc

11 changed files with 76 additions and 524 deletions
Show Stats Expand all files Collapse all files

15
code/inc/ast.h
View File

@ -3,7 +3,6 @@
#define PARCEL_AST_H
#include <utility.h>
#include <logger.h>
#include <tokenizer.h>
typedef struct pac_ast pac_ast_s;
@ -80,18 +79,6 @@ struct pac_ast_item
} data;
};
typedef struct pac_ast_grower
{
pac_tlist_s *token_list;
pac_logger_s logger;
bool_t failed;
} pac_ast_grower_s;
pac_ast_s pac_grow_ast (pac_tlist_s tokens);
bool_t pac_ast_handle_invalid_reference_name_token (pac_ast_grower_s *grower);
bool_t pac_ast_handle_missing_reference_close_tag (pac_ast_grower_s *grower);
bool_t pac_ast_handle_reference_with_equals_sign (pac_ast_grower_s *grower);
pac_ast_s pac_grow_ast (pac_tlist_s tokens);
#endif // PARCEL_AST_H

105
code/inc/logger.h
View File

@ -1,105 +0,0 @@
#ifndef PARCEL_LOGGER_H
#define PARCEL_LOGGER_H
#include <utility.h>
typedef enum
{
PAC_SYNTAX_ERROR,
PAC_NAMING_ERROR,
PAC_INTERNAL_ERROR
} pac_error_e;
typedef enum
{
PAC_SYNTAX_ERROR_UNSPECIFIED,
PAC_SYNTAX_ERROR_STRAY,
PAC_SYNTAX_ERROR_MISSING_TOKEN,
PAC_SYNTAX_ERROR_ODD_TOKEN
} pac_syntax_error_e;
typedef enum
{
PAC_NAMING_ERROR_INVALID_RULE_NAME
} pac_naming_error_e;
typedef struct pac_syntax_error
{
usz_t line;
usz_t column;
pac_syntax_error_e type;
union pac_syntax_error_specifics
{
struct pac_syntax_error_stray
{
rune_t sign;
} stray;
struct pac_syntax_error_missing_token
{
char *wanted_token;
char *hint;
} missing_token;
struct pac_syntax_error_odd_token
{
usz_t num_valid_options;
char *valid_options[16];
char *found_token;
char *hint;
} odd_token;
} specifics;
} pac_syntax_error_s;
typedef struct pac_naming_error
{
usz_t line;
usz_t column;
pac_naming_error_e type;
union pac_naming_error_specifics
{
struct pac_naming_error_invalid_rule_name
{
char *given_rule_name;
} invalid_rule_name;
} specifics;
} pac_naming_error_s;
typedef struct pac_error
{
pac_error_e type;
union pac_error_data
{
pac_syntax_error_s syntax_error;
pac_naming_error_s naming_error;
} specifics;
} pac_error_s;
typedef struct pac_logger
{
usz_t allocated_errors;
usz_t num_errors;
pac_error_s *errors;
pac_arena_s string_arena;
} pac_logger_s;
pac_logger_s pac_create_logger ();
void pac_log_syntax_error (pac_logger_s *logger, pac_syntax_error_s error);
void pac_log_naming_error (pac_logger_s *logger, pac_naming_error_s error);
void * pac_log_alloc (pac_logger_s *logger, usz_t num_bytes);
#endif // PARCEL_LOGGER_H

3
code/inc/tokenizer.h
View File

@ -38,9 +38,6 @@ struct pac_token
pac_token_e type;
usz_t offset;
usz_t length;
usz_t line;
usz_t column;
};
struct pac_tlist

39
code/inc/utility.h
View File

@ -34,38 +34,11 @@ typedef i64_t isz_t;
bool_t pac_rune_is_lower_letter (rune_t rune);
bool_t pac_rune_is_upper_letter (rune_t rune);
bool_t pac_rune_is_letter (rune_t rune);
bool_t pac_rune_is_digit (rune_t rune);
bool_t pac_rune_is_blank (rune_t rune);
bool_t pac_rune_is_sign (rune_t rune);
void pac_memory_copy (void *destination, void *source, usz_t length);
void pac_memory_fill (void *region, usz_t len_region, u8_t byte);
void pac_memory_zero (void *region, usz_t len_region);
typedef struct pac_arena pac_arena_s;
struct pac_arena
{
usz_t len_allocation;
usz_t offset;
void *allocation;
pac_arena_s *continuation;
};
pac_arena_s pac_create_arena (usz_t size);
void pac_delete_arena (pac_arena_s arena);
void pac_free_arena (pac_arena_s *arena);
pac_arena_s * pac_new_arena (usz_t size);
void * pac_arena_alloc (pac_arena_s *arena, usz_t length);
bool_t pac_rune_is_lower_letter (rune_t rune);
bool_t pac_rune_is_upper_letter (rune_t rune);
bool_t pac_rune_is_letter (rune_t rune);
bool_t pac_rune_is_digit (rune_t rune);
bool_t pac_rune_is_blank (rune_t rune);
bool_t pac_rune_is_sign (rune_t rune);
#endif // Include Guard (TN_UTIL_TYPES_H)

59
code/src/allocator.c
View File

@ -1,59 +0,0 @@
#include <utility.h>
#include <stdlib.h>
pac_arena_s pac_create_arena(usz_t size)
{
pac_arena_s arena;
arena.len_allocation = size;
arena.offset = 0;
arena.allocation = malloc(size);
arena.continuation = NULL;
return arena;
}
void pac_delete_arena(pac_arena_s arena)
{
if(arena.continuation != NULL)
{
pac_free_arena(arena.continuation);
}
free(arena.allocation);
}
pac_arena_s * pac_new_arena(usz_t size)
{
pac_arena_s *arena = malloc(sizeof(pac_arena_s) + size);
arena->len_allocation = size;
arena->offset = 0;
arena->allocation = arena + sizeof(pac_arena_s);
arena->continuation = NULL;
return arena;
}
void pac_free_arena(pac_arena_s *arena)
{
if(arena->continuation != NULL)
{
pac_free_arena(arena->continuation);
}
free(arena);
}
void * pac_arena_alloc(pac_arena_s *arena, usz_t length)
{
if((arena->offset + length) > arena->len_allocation)
{
if(arena->continuation == NULL)
arena->continuation = pac_new_arena(length*2);
return pac_arena_alloc(arena->continuation, length);
}
void *block = arena->allocation + arena->offset;
arena->offset += length;
return block;
}

127
code/src/ast.c
View File

@ -1,86 +1,78 @@
#include <parcel.h>
#include <ast.h>
#include <logger.h>
#include <tokenizer.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#define CURSTR &grower->token_list->source[grower->token_list->tokens[grower->token_list->cursor].offset]
#define CURTOK (grower->token_list->tokens[grower->token_list->cursor])
#define CURSOR (grower->token_list->cursor)
#define SKIP_TOKEN ++grower->token_list->cursor
#define REWIND_TOKEN --grower->token_list->cursor
#define TOKEN_AT(index) (grower->token_list->tokens[index])
#define END_REACHED (grower->token_list->cursor >= grower->token_list->num_tokens)
#define CURRENT_STRING &tlist->source[tlist->tokens[tlist->cursor].offset]
#define CURRENT (tlist->tokens[tlist->cursor])
#define SKIP_TOKEN ++tlist->cursor
#define TOKEN_AT(index) (tlist->tokens[index])
#define END_REACHED (tlist->cursor >= tlist->num_tokens)
i32_t pac_grow_reference(pac_ast_grower_s *grower, pac_ast_reference_s *reference)
i32_t pac_grow_reference(pac_tlist_s *tlist, pac_ast_reference_s *reference)
{
if(CURTOK.type != PAC_TOKEN_SIGN_OPEN_TAG)
if(CURRENT.type != PAC_TOKEN_SIGN_OPEN_TAG)
{
// This is only possible with malformed state.
return -1;
}
SKIP_TOKEN;
if(CURTOK.type != PAC_TOKEN_WORD)
if(CURRENT.type != PAC_TOKEN_WORD)
{
return pac_ast_handle_invalid_reference_name_token(grower);
return -1;
}
usz_t len_name = CURTOK.length;
char *name = CURSTR;
usz_t len_name = CURRENT.length;
char *name = CURRENT_STRING;
SKIP_TOKEN;
if(CURTOK.type != PAC_TOKEN_SIGN_CLOSE_TAG)
if(CURRENT.type != PAC_TOKEN_SIGN_CLOSE_TAG)
{
return pac_ast_handle_missing_reference_close_tag(grower);
return -1;
}
SKIP_TOKEN;
if(CURTOK.type == PAC_TOKEN_SIGN_EQUALS)
{
return pac_ast_handle_reference_with_equals_sign(grower);
}
reference->len_name = len_name;
reference->name = malloc(len_name+1);
pac_memory_copy(reference->name, name, len_name);
memcpy(reference->name, name, len_name);
reference->name[len_name] = 0x00;
return 3;
}
i32_t pac_grow_item(pac_ast_grower_s *grower, pac_ast_item_s *item)
i32_t pac_grow_item(pac_tlist_s *tlist, pac_ast_item_s *item)
{
item->type = PAC_AST_ITEM_INVALID;
pac_memory_zero(item, sizeof(pac_ast_item_s));
if(CURTOK.type == PAC_TOKEN_LIT_STRING)
memset(item, 0x00, sizeof(pac_ast_item_s));
if(CURRENT.type == PAC_TOKEN_LIT_STRING)
{
item->type = PAC_AST_ITEM_LITERAL;
item->data.literal.length = CURTOK.length;
item->data.literal.length = CURRENT.length;
item->data.literal.string = malloc(item->data.literal.length + 1);
pac_memory_copy(item->data.literal.string, CURSTR, CURTOK.length);
memcpy(item->data.literal.string, CURRENT_STRING, CURRENT.length);
item->data.literal.string[item->data.literal.length] = 0x00;
SKIP_TOKEN;
return 1;
}
if(CURTOK.type == PAC_TOKEN_SIGN_OPEN_TAG)
if(CURRENT.type == PAC_TOKEN_SIGN_OPEN_TAG)
{
item->type = PAC_AST_ITEM_REFERENCE;
return pac_grow_reference(grower, &item->data.reference);
return pac_grow_reference(tlist, &item->data.reference);
}
if(CURTOK.type == PAC_TOKEN_KEYWORD_WORD)
if(CURRENT.type == PAC_TOKEN_KEYWORD_WORD)
{
item->type = PAC_AST_ITEM_SET;
item->data.set = PAC_AST_SET_WORD;
SKIP_TOKEN;
return 1;
SKIP_TOKEN;
}
if(CURTOK.type == PAC_TOKEN_KEYWORD_INTEGER)
if(CURRENT.type == PAC_TOKEN_KEYWORD_INTEGER)
{
item->type = PAC_AST_ITEM_SET;
item->data.set = PAC_AST_SET_INTEGER;
@ -90,11 +82,11 @@ i32_t pac_grow_item(pac_ast_grower_s *grower, pac_ast_item_s *item)
return -1;
}
i32_t pac_grow_variant(pac_ast_grower_s *grower, pac_ast_variant_s *variant, char *rule_name, usz_t variant_index)
i32_t pac_grow_variant(pac_tlist_s *tlist, pac_ast_variant_s *variant)
{
usz_t start_index = CURSOR;
usz_t start_index = tlist->cursor;
pac_memory_zero(variant, sizeof(pac_ast_variant_s));
memset(variant, 0x00, sizeof(pac_ast_variant_s));
usz_t items_capacity = 8;
variant->items = calloc(sizeof(pac_ast_item_s), items_capacity);
@ -105,60 +97,57 @@ i32_t pac_grow_variant(pac_ast_grower_s *grower, pac_ast_variant_s *variant, cha
items_capacity *= 2;
variant->items = calloc(sizeof(pac_ast_item_s), items_capacity);
}
i32_t success = pac_grow_item(grower, &variant->items[variant->num_items]);
i32_t success = pac_grow_item(tlist, &variant->items[variant->num_items]);
++variant->num_items;
if(success < 0)
{
return success - 1;
}
if(
(CURTOK.type == PAC_TOKEN_SIGN_VERTICAL_BAR)
|| (CURTOK.type == PAC_TOKEN_SIGN_SEMICOLON)
) {
return CURSOR - start_index;
}
if(CURTOK.type != PAC_TOKEN_SIGN_COMMA)
{
if(CURRENT.type == PAC_TOKEN_SIGN_VERTICAL_BAR)
return tlist->cursor - start_index;
if(CURRENT.type == PAC_TOKEN_SIGN_SEMICOLON)
return tlist->cursor - start_index;
if(CURRENT.type != PAC_TOKEN_SIGN_COMMA)
return -1;
}
SKIP_TOKEN;
}
return -1;
}
i32_t pac_grow_rule(pac_ast_grower_s *grower, pac_ast_rule_s *rule)
i32_t pac_grow_rule(pac_tlist_s *tlist, pac_ast_rule_s *rule)
{
pac_memory_zero(rule, sizeof(pac_ast_rule_s));
memset(rule, 0x00, sizeof(pac_ast_rule_s));
// Parse the header
usz_t start_index = CURSOR;
usz_t start_index = tlist->cursor;
if(CURTOK.type != PAC_TOKEN_SIGN_OPEN_TAG)
if(CURRENT.type != PAC_TOKEN_SIGN_OPEN_TAG)
{
return -1;
}
SKIP_TOKEN;
if(CURTOK.type != PAC_TOKEN_WORD)
if(CURRENT.type != PAC_TOKEN_WORD)
{
puts("A rule name must be a single word!");
return -1;
}
usz_t len_name = CURTOK.length;
usz_t name_in_source = CURSTR;
usz_t len_name = CURRENT.length;
usz_t name_start = CURRENT.offset;
SKIP_TOKEN;
if(CURTOK.type != PAC_TOKEN_SIGN_CLOSE_TAG)
if(CURRENT.type != PAC_TOKEN_SIGN_CLOSE_TAG)
{
puts("Missing Tag closing sign!");
return -1;
}
SKIP_TOKEN;
if(CURTOK.type != PAC_TOKEN_SIGN_EQUALS)
if(CURRENT.type != PAC_TOKEN_SIGN_EQUALS)
return -1;
SKIP_TOKEN;
@ -166,7 +155,7 @@ i32_t pac_grow_rule(pac_ast_grower_s *grower, pac_ast_rule_s *rule)
// Parse all variants
rule->name = malloc(len_name + 1);
pac_memory_copy(rule->name, name_in_source, len_name);
memcpy(rule->name, &tlist->source[name_start], len_name);
rule->name[len_name] = 0;
usz_t variants_capacity = 4;
@ -178,7 +167,7 @@ i32_t pac_grow_rule(pac_ast_grower_s *grower, pac_ast_rule_s *rule)
variants_capacity *= 2;
rule->variants = realloc(rule->variants, sizeof(pac_ast_variant_s) * variants_capacity);
}
i32_t success = pac_grow_variant(grower, &rule->variants[rule->num_variants], rule->name, rule->num_variants);
i32_t success = pac_grow_variant(tlist, &rule->variants[rule->num_variants]);
++rule->num_variants;
if(success < 0)
@ -186,15 +175,14 @@ i32_t pac_grow_rule(pac_ast_grower_s *grower, pac_ast_rule_s *rule)
printf("Failed parsing a rule's variant!");
while(!END_REACHED)
{
if(CURTOK.type == PAC_TOKEN_SIGN_VERTICAL_BAR)
if(CURRENT.type == PAC_TOKEN_SIGN_VERTICAL_BAR)
{
printf("Continuing with next variant.\n");
break;
}
if(CURTOK.type == PAC_TOKEN_SIGN_SEMICOLON)
if(CURRENT.type == PAC_TOKEN_SIGN_SEMICOLON)
{
printf("Continuing with next rule.\n");
SKIP_TOKEN;
return 2;
}
SKIP_TOKEN;
@ -202,13 +190,13 @@ i32_t pac_grow_rule(pac_ast_grower_s *grower, pac_ast_rule_s *rule)
continue;
}
if(CURTOK.type == PAC_TOKEN_SIGN_SEMICOLON)
if(CURRENT.type == PAC_TOKEN_SIGN_SEMICOLON)
{
SKIP_TOKEN;
return CURSOR - start_index;
return tlist->cursor - start_index;
}
if(CURTOK.type != PAC_TOKEN_SIGN_VERTICAL_BAR)
if(CURRENT.type != PAC_TOKEN_SIGN_VERTICAL_BAR)
{
return -1;
}
@ -224,11 +212,6 @@ pac_ast_s pac_grow_ast(pac_tlist_s tokens)
ast.num_rules = 0;
ast.rules = malloc(sizeof(pac_ast_rule_s) * rules_capacity);
pac_ast_grower_s grower;
grower.logger = pac_create_logger();
grower.token_list = &tokens;
grower.failed = FALSE;
while(tokens.cursor < tokens.num_tokens)
{
if(ast.num_rules >= rules_capacity)
@ -236,7 +219,7 @@ pac_ast_s pac_grow_ast(pac_tlist_s tokens)
rules_capacity *= 2;
ast.rules = realloc(ast.rules, sizeof(pac_ast_rule_s) * rules_capacity);
}
int success = pac_grow_rule(&grower, &ast.rules[ast.num_rules]);
int success = pac_grow_rule(&tokens, &ast.rules[ast.num_rules]);
if(success < 0)
{
printf("Failed parsing a rule at index: %lu!\n", tokens.cursor);

115
code/src/ast_errors.c
View File

@ -1,115 +0,0 @@
#include <ast.h>
#include <logger.h>
#include <tokenizer.h>
#define CURSTR &grower->token_list->source[grower->token_list->tokens[grower->token_list->cursor].offset]
#define CURTOK (grower->token_list->tokens[grower->token_list->cursor])
#define CURSOR (grower->token_list->cursor)
#define SKIP_TOKEN ++grower->token_list->cursor
#define REWIND_TOKEN --grower->token_list->cursor
#define TOKEN_AT(index) (grower->token_list->tokens[(index)])
#define END_REACHED (grower->token_list->cursor >= grower->token_list->num_tokens)
// pac_ast_builder_is_at_item_start: An utility function which returns wheter
// the parser is at the start of an item at in the current state.
bool_t pac_ast_builder_is_at_item_start(pac_ast_grower_s *grower)
{
switch(CURTOK.type)
{
case PAC_TOKEN_LIT_STRING:
case PAC_TOKEN_SIGN_OPEN_TAG:
case PAC_TOKEN_KEYWORD_WORD:
case PAC_TOKEN_KEYWORD_INTEGER:
return TRUE;
default: break;
}
return FALSE;
}
// Returns whether the parser can continue to catch errors
bool_t pac_ast_handle_invalid_reference_name_token(pac_ast_grower_s *grower)
{
usz_t open_tag_src_offset = TOKEN_AT(CURSOR-1).offset;
usz_t len_reference_name = 0;
// Find closing tag for getting the name of the word for the error message.
usz_t tried_tokens = 0;
while(tried_tokens < 3)
{
if(TOKEN_AT(CURSOR + tried_tokens).type == PAC_TOKEN_SIGN_CLOSE_TAG)
{
len_reference_name = TOKEN_AT(CURSOR + tried_tokens).offset - open_tag_src_offset;
++len_reference_name; // Take the closing tag into the name
break;
}
++tried_tokens;
}
// If no closing tag could be found, use the token after the opening tag.
if(len_reference_name == 0)
len_reference_name = ((CURTOK.offset + CURTOK.length) - open_tag_src_offset);
pac_naming_error_s error;
error.type = PAC_NAMING_ERROR_INVALID_RULE_NAME;
error.line = CURTOK.line;
error.column = CURTOK.column;
error.specifics.invalid_rule_name.given_rule_name = pac_log_alloc(&grower->logger, len_reference_name + 1);
pac_memory_copy(error.specifics.invalid_rule_name.given_rule_name, &grower->token_list->source[open_tag_src_offset], len_reference_name);
pac_log_naming_error(&grower->logger, error);
grower->failed = TRUE;
return TRUE;
}
// Returns whether the parser can continue to catch errors
bool_t pac_ast_handle_missing_reference_close_tag(pac_ast_grower_s *grower)
{
pac_syntax_error_s error;
error.type = PAC_SYNTAX_ERROR_MISSING_TOKEN;
error.line = CURTOK.line;
error.column = CURTOK.column;
error.specifics.missing_token.hint = NULL;
error.specifics.missing_token.wanted_token = "Tag Closer (>)";
pac_log_syntax_error(&grower->logger, error);
return TRUE;
}
// Returns whether the parser can continue to catch errors
bool_t pac_ast_handle_reference_with_equals_sign(pac_ast_grower_s *grower)
{
pac_syntax_error_s error;
error.type = PAC_SYNTAX_ERROR_ODD_TOKEN;
error.line = CURTOK.line;
error.column = CURTOK.column;
error.specifics.odd_token.hint = "There might be a semicolon missing 4 tokens ago.";
error.specifics.odd_token.num_valid_options = 1;
error.specifics.odd_token.valid_options[0] = "Any Rule Item";
pac_log_syntax_error(&grower->logger, error);
// TODO: Handle the tokens following this as a new rule.
return FALSE;
}
// Returns whether the parser can continue to catch errors
bool_t pac_ast_handle_unknown_item_type(pac_ast_grower_s *grower)
{
pac_syntax_error_s error;
error.type = PAC_SYNTAX_ERROR_ODD_TOKEN;
error.line = CURTOK.line;
error.column = CURTOK.column;
error.specifics.odd_token.num_valid_options = 1;
error.specifics.odd_token.valid_options[0] = "Any Rule Item";
pac_log_syntax_error(&grower->logger, error);
while(!pac_ast_builder_is_at_item_start(grower))
SKIP_TOKEN;
return TRUE;
}

2
code/src/linker.c
View File

@ -15,7 +15,7 @@ void pac_copy_empty_rules_from_ast_to_grammar(pac_grammar_s *grammar, pac_ast_s
pac_rule_s *rule = &grammar->rules[rule_index];
usz_t len_rule_name = strlen(ast_rule.name);
rule->name = malloc(len_rule_name + 1);
pac_memory_copy(rule->name, ast_rule.name, len_rule_name);
memcpy(rule->name, ast_rule.name, len_rule_name);
rule->name[len_rule_name] = 0x00;
}
}

47
code/src/logger.c
View File

@ -1,47 +0,0 @@
#include <logger.h>
#include <stdlib.h>
pac_logger_s pac_create_logger()
{
pac_logger_s logger;
logger.allocated_errors = 1024;
logger.num_errors = 0;
logger.errors = calloc(sizeof(pac_error_s), logger.allocated_errors);
logger.string_arena = pac_create_arena(32768);
return logger;
}
void pac_resize_log_if_needed(pac_logger_s *logger)
{
if(logger->num_errors >= logger->allocated_errors)
{
logger->allocated_errors *= 2;
logger->errors =
realloc(logger->errors, sizeof(pac_error_s) * logger->allocated_errors);
}
}
void pac_log_syntax_error(pac_logger_s *logger, pac_syntax_error_s error)
{
pac_resize_log_if_needed(logger);
logger->errors[logger->num_errors].type = PAC_SYNTAX_ERROR;
logger->errors[logger->num_errors].specifics.syntax_error = error;
++logger->num_errors;
}
void pac_log_naming_error(pac_logger_s *logger, pac_naming_error_s error)
{
pac_resize_log_if_needed(logger);
logger->errors[logger->num_errors].type = PAC_NAMING_ERROR;
logger->errors[logger->num_errors].specifics.naming_error = error;
++logger->num_errors;
}
void * pac_log_alloc(pac_logger_s *logger, usz_t num_bytes)
{
return pac_arena_alloc(&logger->string_arena, num_bytes);
}

55
code/src/tokenizer.c
View File

@ -52,10 +52,7 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
list.source = source;
list.num_tokens = 0;
list.tokens = calloc(sizeof(pac_token_s), tokens_capacity);
usz_t line = 1;
usz_t column = 1;
for(usz_t offset = 0; offset < len_source; ++offset, ++column)
for(usz_t offset = 0; offset < len_source; ++offset)
{
// The subject of the current iteration of this loop;
// the rune which is being looked at
@ -67,31 +64,18 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
list.tokens = realloc(list.tokens, sizeof(pac_token_s) * tokens_capacity);
}
if(subject == '\n')
{
++line;
column = 0; // The for()-header will make it 1.
continue;
}
if(pac_rune_is_blank(subject))
continue;
if(subject == '"')
{
usz_t start_offset = offset;
usz_t start_line = line;
usz_t start_column = column;
while(offset < len_source)
{
if(subject == '\\')
{
++offset;
++column;
}
++offset;
++column;
++offset;
subject = source[offset];
if(subject == '"')
@ -101,8 +85,6 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
token.type = PAC_TOKEN_LIT_STRING;
token.offset = start_offset + 1; // +1 for skipping the starting quotation mark
token.length = (offset - start_offset) - 1; // -1 for leaving the ending quotation mark out
token.line = start_line;
token.column = start_column;
list.tokens[list.num_tokens] = token;
++list.num_tokens;
@ -112,12 +94,9 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
if(pac_rune_is_letter(subject) || (subject == '_'))
{
usz_t start_offset = offset;
usz_t start_line = line;
usz_t start_column = column;
while(offset < len_source)
{
++offset;
++column;
subject = source[offset];
if(!pac_rune_is_letter(subject) && !pac_rune_is_digit(subject) && (subject != '_'))
@ -129,10 +108,7 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
token.offset = start_offset;
token.length = offset - start_offset;
token.type = pac_convert_word_to_token_type(&source[start_offset], token.length);
token.line = start_line;
token.column = start_column;
--offset; // The for() - header will skip to the character after the word.
--column;
list.tokens[list.num_tokens] = token;
++list.num_tokens;
@ -141,7 +117,8 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
if(subject == '#')
{
usz_t offset_copy = offset + 1;
usz_t offset_copy = offset;
++offset_copy;
if(offset_copy < len_source)
{
@ -159,23 +136,17 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
}
else if(second_sign == '[')
{
usz_t line_copy = line;
usz_t column_copy = column + 1; // +1, because there also is one at
while(offset_copy < len_source) // the creation of 'offset_copy'.
while(offset_copy < len_source)
{
if(source[offset_copy] == ']')
{
// TODO: There should be a check for wheter there is one square bracket following this one.
break;
}
if(source[offset_copy] == '\n')
++line_copy;
// If there's an escaped character here, do one jump more
if(source[offset_copy] == '\\')
++offset_copy;
++offset_copy;
++column_copy;
}
line = line_copy;
column = column_copy;
}
offset = offset_copy;
}
@ -188,8 +159,6 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
token.type = pac_convert_sign_to_token_type(subject);
token.offset = offset;
token.length = 1;
token.line = line;
token.column = column;
list.tokens[list.num_tokens] = token;
++list.num_tokens;
@ -232,7 +201,7 @@ char * pac_stringify_token_type(pac_token_e type)
char pac_spaces[256];
char * pac_create_spaces_for_indent(u8_t count)
{
pac_memory_fill(pac_spaces, 256, ' ');
memset(pac_spaces, ' ', 256);
pac_spaces[count] = 0x00;
return &pac_spaces[0];
}
@ -245,14 +214,12 @@ void pac_display_tlist(pac_tlist_s list)
pac_token_s token = list.tokens[index];
char content[token.length+1];
pac_memory_copy(&content[0], &list.source[token.offset], token.length);
memcpy(&content[0], &list.source[token.offset], token.length);
content[token.length] = 0;
char *token_type_string = pac_stringify_token_type(token.type);
char *token_type_indent = pac_create_spaces_for_indent(24 - strlen(token_type_string));
printf("Column %-3lu @ Line %lu: [%s]%s %s\n",
token.column,
token.line,
printf("[%s]:%s %s\n",
token_type_string,
token_type_indent,
content

29
code/src/utility.c
View File

@ -1,29 +0,0 @@
#include <utility.h>
void pac_memory_copy(void *destination, void *source, usz_t length)
{
u8_t *source_8 = source;
u8_t *destination_8 = destination;
usz_t index = 0;
while(index < length)
{
destination_8[index] = source_8[index];
++index;
}
}
void pac_memory_fill(void *region, usz_t len_region, u8_t byte)
{
u8_t *region_8 = region;
usz_t index = 0;
while(index < len_region)
{
region_8[index] = byte;
++index;
}
}
void pac_memory_zero(void *region, usz_t len_region)
{
pac_memory_fill(region, len_region, 0);
}