Added a sample for the AST

code/inc/ast.h

@@ -3,6 +3,7 @@
 #define PARCEL_AST_H
 
 #include <utility.h>
+#include <tokenizer.h>
 
 typedef struct pac_ast                  pac_ast_s;
 typedef struct pac_ast_rule             pac_ast_rule_s;
@@ -58,14 +59,26 @@ typedef enum
     
 } pac_outline_e;
 
+typedef enum
+{
+    PAC_AST_ITEM_INVALID            = 0x00,
+    PAC_AST_ITEM_REFERENCE,
+    PAC_AST_ITEM_LITERAL,
+    PAC_AST_ITEM_OUTLINE
+
+} pac_ast_item_e;
+
 struct pac_ast_item
 {
-    bool_t                      is_literal;
+    pac_ast_item_e              type;
     union pac_item_data
     {
         pac_ast_literal_s           literal;
         pac_outline_e               outline;
+        pac_ast_reference_s         reference;
     } data;
 };
 
+pac_ast_s               pac_grow_ast                    (pac_tlist_s tokens);
+
 #endif // PARCEL_AST_H

code/src/ast.c

@@ -6,7 +6,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#define CURRENT_STRING      (&tlist->source[tlist->tokens[tlist->cursor].offset])
+#define CURRENT_STRING       &tlist->source[tlist->tokens[tlist->cursor].offset]
 #define CURRENT             (tlist->tokens[tlist->cursor])
 #define SKIP_TOKEN           ++tlist->cursor
 #define TOKEN_AT(index)     (tlist->tokens[index])
@@ -14,18 +14,44 @@
 
 i32_t pac_grow_reference(pac_tlist_s *tlist, pac_ast_reference_s *reference)
 {
+    if(CURRENT.type != PAC_TOKEN_SIGN_OPEN_TAG)
+    {
+        return -1;
+    }
+    SKIP_TOKEN;
 
+    
+    if(CURRENT.type != PAC_TOKEN_WORD)
+    {
+        return -1;
+    }
+    usz_t                   len_name                = CURRENT.length;
+    char                   *name                    = CURRENT_STRING;
+    SKIP_TOKEN;
+
+    if(CURRENT.type != PAC_TOKEN_SIGN_CLOSE_TAG)
+    {
+        return -1;
+    }
+    SKIP_TOKEN;
+
+    reference->len_name         = len_name;
+    reference->name             = malloc(len_name+1);
+    memcpy(reference->name, name, len_name);
+    reference->name[len_name]   = 0x00;
+
+    return 3;
 }
 
 i32_t pac_grow_item(pac_tlist_s *tlist, pac_ast_item_s *item)
 {
-    item->is_literal            = FALSE;
+    item->type                  = PAC_AST_ITEM_INVALID;
     memset(item, 0x00, sizeof(pac_ast_item_s));
     switch(CURRENT.type)
     {
         case PAC_TOKEN_LIT_STRING:
         {
-            item->is_literal            = TRUE;
+            item->type                  = PAC_AST_ITEM_LITERAL;
             item->data.literal.length   = CURRENT.length;
             item->data.literal.string   = malloc(item->data.literal.length + 1);
             memcpy(item->data.literal.string, CURRENT_STRING, CURRENT.length);
@@ -33,14 +59,22 @@ i32_t pac_grow_item(pac_tlist_s *tlist, pac_ast_item_s *item)
             SKIP_TOKEN;
         } return 1;
         
+        case PAC_TOKEN_SIGN_OPEN_TAG:
+        {
+            item->type                  = PAC_AST_ITEM_REFERENCE;
+            return pac_grow_reference(tlist, &item->data.reference);
+        }
+
         case PAC_TOKEN_KEYWORD_WORD:
         {
+            item->type                  = PAC_AST_ITEM_OUTLINE;
             item->data.outline          = PAC_OUTLINE_WORD;
             SKIP_TOKEN;
         } return 1;
         
         case PAC_TOKEN_KEYWORD_INTEGER:
         {
+            item->type                  = PAC_AST_ITEM_OUTLINE;
             item->data.outline          = PAC_OUTLINE_INTEGER;
             SKIP_TOKEN;
         } return 1;
@@ -136,7 +170,7 @@ i32_t pac_grow_rule(pac_tlist_s *tlist, pac_ast_rule_s *rule)
 
         if(success < 0)
         {
-            printf("Failed parsing variant %u of rule '%s'. ");
+            printf("Failed parsing variant %u of rule '%d'. ", rule->num_variants);
             while(!END_REACHED)
             {
                 if(CURRENT.type == PAC_TOKEN_SIGN_VERTICAL_BAR)
@@ -147,11 +181,11 @@ i32_t pac_grow_rule(pac_tlist_s *tlist, pac_ast_rule_s *rule)
                 if(CURRENT.type == PAC_TOKEN_SIGN_SEMICOLON)
                 {
                     printf("Continuing with next rule.\n");
-                    SKIP_TOKEN;
                     return 2;
                 }
                 SKIP_TOKEN;
             }
+            continue;
         }
         
         if(CURRENT.type == PAC_TOKEN_SIGN_SEMICOLON)
@@ -192,15 +226,3 @@ pac_ast_s pac_grow_ast(pac_tlist_s tokens)
     }
     return ast;
 }
-
-pac_grammar_s pac_convert_grammar(char *source)
-{
-    usz_t                   len_source              = strlen(source);
-    pac_tlist_s             tokens                  = pac_tokenize_grammar(source, len_source);
-    pac_display_tlist(tokens);
-
-    pac_ast_s               ast                     = pac_grow_ast(tokens);
-
-    pac_grammar_s           grammar;
-    return grammar;
-}

code/src/tokenizer.c

@@ -99,7 +99,7 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source)
                 ++offset;
                 subject                         = source[offset];
                 
-                if(!pac_rune_is_letter(subject) && (subject != '_'))
+                if(!pac_rune_is_letter(subject) && !pac_rune_is_digit(subject) && (subject != '_'))
                 {
                     break;
                 }

samples/sample_ast_01.parcel

@@ -0,0 +1,3 @@
+
+<root> = <ref1>, "+", <ref2> | <ref1>, "-", <ref2>;
+