From 138e361c547d4f16cd5579b4c9f41fff67dbf243 Mon Sep 17 00:00:00 2001 From: Eric-Paul Ickhorn Date: Tue, 28 Nov 2023 16:00:10 +0100 Subject: [PATCH] Improved tokenizer to make it possible to have digits in a word --- code/src/tokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/src/tokenizer.c b/code/src/tokenizer.c index 1f59cfc..6ddcd31 100644 --- a/code/src/tokenizer.c +++ b/code/src/tokenizer.c @@ -99,7 +99,7 @@ pac_tlist_s pac_tokenize_grammar(char *source, usz_t len_source) ++offset; subject = source[offset]; - if(!pac_rune_is_letter(subject) && (subject != '_')) + if(!pac_rune_is_letter(subject) && !pac_rune_is_digit(subject) && (subject != '_')) { break; }