Fixing up lexer build and preprocessor parsing logic

custom/4coder_base_types.h

@@ -172,6 +172,9 @@ typedef void Void_Func(void);
 #define stringify_(a) #a
 #define stringify(a) stringify_(a)
 
+#define function static
+#define api(x)
+
 #define internal static
 #define local_persist static
 #define global static
@@ -297,8 +300,6 @@ global_const f32 epsilon_f32 = 5.96046448e-8f;
 
 #define require(c) Stmnt( if (!(c)){ return(0); } )
 
-#define API_EXPORT
-
 ////////////////////////////////
 
 global_const u32 bit_1  = 0x00000001;

custom/4coder_file.h

@@ -16,25 +16,16 @@
 #if OS_WINDOWS
 
 //// WINDOWS BEGIN ////
+#undef function
 #define UNICODE
 #include <Windows.h>
 typedef TCHAR Filename_Character;
 #define SLASH '\\'
+#define function static
 //// WINDOWS END ////
 
-#elif OS_LINUX || OS_MAC
-
-//// UNIX BEGIN ////
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <dirent.h>
-typedef char Filename_Character;
-#define SLASH '/'
-//// UNIX END ////
-
 #else
-# error metdata generator not supported on this platform
+# error 4coder file not defined
 #endif
 
 struct Cross_Platform_File_Info{

custom/bin/build_one_time.bat

@@ -1,32 +1,41 @@
 @echo off
 
-REM usage: build_generator <src-file> <binary-output-path> [release]
+REM usage: <script> <target> [binary-output-path [mode]]
-REM src-file: a relative path to the target of a unity-build for a generator
+REM  src-file : a relative path to the target of a unity-build for a one time executable
-REM binary-output-path: a relative path where the generator.exe will be written, assumed to be "." if unset
+REM  binary-output-path : a relative path where the generator.exe will be written, 
+REM                       assumed to be "." if unset
+REM  mode : if set to "release" builds with optimizations
 
-set code_home=%~dp0
+set location=%cd%
-if %code_home:~-1%==\ (set code_home=%code_home:~0,-1%)
+set me="%~dp0"
+cd %me%
+cd ..
+set custom_root=%cd%
+set custom_bin=%custom_root%\bin
+cd %location%
 
 if NOT "%Platform%" == "X64" IF NOT "%Platform%" == "x64" (call "%code_home%\windows_scripts\setup_cl_x64.bat")
 
-set src=%1
+set target=%1
-if "%src%" == "" (echo error: no input file & exit)
+if "%target%" == "" (echo error: no input file & exit)
+set full_target=%target%
+if NOT "%target:~1,1%" == ":" (set full_target="%cd%\%target%")
 
 set dst=%2
 if "%dst%" == "" (set dst=".")
 
-set opts=/W4 /wd4310 /wd4100 /wd4201 /wd4505 /wd4996 /wd4127 /wd4510 /wd4512 /wd4610 /wd4457 /wd4146 /WX
-set opts=%opts% /GR- /nologo /FC
 set debug=/Zi
 set release=/O2 /Zi
-
 set mode=%debug%
 if "%3" == "release" (set mode=%release%)
 
-set full_src="%cd%\%src%"
+set opts=/W4 /wd4310 /wd4100 /wd4201 /wd4505 /wd4996 /wd4127 /wd4510 /wd4512 /wd4610 /wd4457 /wd4146 /WX
+set opts=%opts% /GR- /nologo /FC
+set opts=%opts% /I%custom_root%
+set opts=%opts% %mode%
 
 pushd %dst%
-call cl /I"%code_home%" %opts% %mode% %full_src% /Fegenerator
+call cl /I"%code_home%" %opts% %full_target% /Fegenerator
 popd
 
 

custom/languages/4coder_lex_gen_cpp.cpp

@@ -235,21 +235,21 @@ build_language_model(void){
     Keyword_Set *pp_directive_set = sm_begin_key_set("pp_directives");
     
     sm_select_base_kind(TokenBaseKind_Preprocessor);
-    sm_key("PPInclude", "#include");
+    sm_key("PPInclude", "include");
-    sm_key("PPVersion", "#version");
+    sm_key("PPVersion", "version");
-    sm_key("PPDefine", "#define");
+    sm_key("PPDefine", "define");
-    sm_key("PPUndef", "#undef");
+    sm_key("PPUndef", "undef");
-    sm_key("PPIf", "#if");
+    sm_key("PPIf", "if");
-    sm_key("PPIfDef", "#ifdef");
+    sm_key("PPIfDef", "ifdef");
-    sm_key("PPIfNDef", "#ifndef");
+    sm_key("PPIfNDef", "ifndef");
-    sm_key("PPElse", "#else");
+    sm_key("PPElse", "else");
-    sm_key("PPElIf", "#elif");
+    sm_key("PPElIf", "elif");
-    sm_key("PPEndIf", "#endif");
+    sm_key("PPEndIf", "endif");
-    sm_key("PPError", "#error");
+    sm_key("PPError", "error");
-    sm_key("PPImport", "#import");
+    sm_key("PPImport", "import");
-    sm_key("PPUsing", "#using");
+    sm_key("PPUsing", "using");
-    sm_key("PPLine", "#line");
+    sm_key("PPLine", "line");
-    sm_key("PPPragma", "#pragma");
+    sm_key("PPPragma", "pragma");
     
     sm_select_base_kind(TokenBaseKind_LexError);
     sm_key_fallback("PPUnknown");
@@ -310,7 +310,9 @@ build_language_model(void){
     AddState(LL_number);
     AddState(ULL_number);
     
+    AddState(pp_directive_whitespace);
     AddState(pp_directive);
+    AddState(pp_directive_emit);
     
     AddState(include_pointy);
     AddState(include_quotes);
@@ -401,7 +403,7 @@ build_language_model(void){
     
     sm_case_flagged(is_include_body, false, "\"", string);
     sm_case("\'", character);
-    sm_case_flagged(is_pp_body, false, "#", pp_directive);
+    sm_case_flagged(is_pp_body, false, "#", pp_directive_whitespace);
     {
         State *operator_state = smo_op_set_lexer_root(pp_ops, root, "LexError");
         sm_case_peek_flagged(is_pp_body, true, "#", operator_state);
@@ -771,18 +773,40 @@ build_language_model(void){
     
     ////
     
-    sm_select_state(pp_directive);
+    sm_select_state(pp_directive_whitespace);
-    sm_set_flag(is_pp_body, true);
+    sm_delim_mark_first();
+    sm_case(" \t\f\v", pp_directive_whitespace);
     sm_case("abcdefghijklmnopqrstuvwxyz"
             "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
             "_"
             "0123456789",
             pp_directive);
+    {
+        Emit_Rule *emit = sm_emit_rule();
+        sm_emit_handler_direct("LexError");
+        sm_fallback_peek(emit);
+    }
+    
+    ////
+    
+    sm_select_state(pp_directive);
+    sm_set_flag(is_pp_body, true);
+    sm_case("abcdefghijklmnopqrstuvwxyz"
+            "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+            "_"
+            "0123456789",
+            pp_directive);
+    sm_fallback_peek(pp_directive_emit);
+    
+    ////
+    
+    sm_select_state(pp_directive_emit);
+    sm_delim_mark_one_past_last();
     {
         Emit_Rule *emit = sm_emit_rule();
         sm_emit_check_set_flag("PPInclude", is_include_body, true);
         sm_emit_check_set_flag("PPError", is_error_body, true);
-        sm_emit_handler_keys(pp_directive_set);
+        sm_emit_handler_keys_delim(pp_directive_set);
         sm_fallback_peek(emit);
     }
     

custom/lexer_generator/4coder_lex_gen_main.cpp

@@ -106,6 +106,7 @@ typedef i32 Emit_Handler_Kind;
 enum{
     EmitHandlerKind_Direct,
     EmitHandlerKind_Keywords,
+    EmitHandlerKind_KeywordsDelim,
 };
 
 struct Emit_Handler{
@@ -494,11 +495,10 @@ smi_emit_rule(Arena *arena){
 }
 
 internal Emit_Handler*
-smi_emit_handler(Arena *arena, Emit_Rule *rule, String_Const_u8 name, Flag *flag_check){
+smi_emit_handler__inner(Arena *arena, Emit_Rule *rule, Emit_Handler_Kind kind, Flag *flag_check){
     Emit_Handler *handler = push_array_zero(arena, Emit_Handler, 1);
-    handler->kind = EmitHandlerKind_Direct;
+    handler->kind = kind;
     handler->flag_check = flag_check;
-    handler->token_name = name;
     if (rule != 0){
         sll_queue_push(rule->first, rule->last, handler);
         rule->count += 1;
@@ -506,16 +506,24 @@ smi_emit_handler(Arena *arena, Emit_Rule *rule, String_Const_u8 name, Flag *flag
     return(handler);
 }
 
+internal Emit_Handler*
+smi_emit_handler(Arena *arena, Emit_Rule *rule, String_Const_u8 name, Flag *flag_check){
+    Emit_Handler *handler = smi_emit_handler__inner(arena, rule, EmitHandlerKind_Direct, flag_check);
+    handler->token_name = name;
+    return(handler);
+}
+
 internal Emit_Handler*
 smi_emit_handler(Arena *arena, Emit_Rule *rule, Keyword_Set *set, Flag *flag_check){
-    Emit_Handler *handler = push_array_zero(arena, Emit_Handler, 1);
+    Emit_Handler *handler = smi_emit_handler__inner(arena, rule, EmitHandlerKind_Keywords, flag_check);
-    handler->kind = EmitHandlerKind_Keywords;
+    handler->keywords = set;
-    handler->flag_check = flag_check;
+    return(handler);
+}
+
+internal Emit_Handler*
+smi_emit_handler_delim(Arena *arena, Emit_Rule *rule, Keyword_Set *set, Flag *flag_check){
+    Emit_Handler *handler = smi_emit_handler__inner(arena, rule, EmitHandlerKind_KeywordsDelim, flag_check);
     handler->keywords = set;
-    if (rule != 0){
-        sll_queue_push(rule->first, rule->last, handler);
-        rule->count += 1;
-    }
     return(handler);
 }
 
@@ -1388,6 +1396,17 @@ sm_emit_handler_keys(Keyword_Set *set){
     sm_emit_handler_keys(0, set);
 }
 
+internal void
+sm_emit_handler_keys_delim(Flag *flag_check, Keyword_Set *set){
+    Emit_Rule *rule = helper_ctx.selected_emit_rule;
+    smi_emit_handler(helper_ctx.arena, rule, set, flag_check);
+}
+
+internal void
+sm_emit_handler_keys_delim(Keyword_Set *set){
+    sm_emit_handler_keys_delim(0, set);
+}
+
 internal Transition*
 sm_case(String_Const_u8 str, Flag *flag_check, b32 flag_check_value, State *dst, Transition_Consume_Rule consume_rule, Emit_Rule *emit){
     Transition *transition = smi_case(&helper_ctx.primary_ctx, helper_ctx.selected_state, TransitionCaseKind_CharaterArray, str,

project.4coder

@@ -39,7 +39,7 @@ command_list = {
  
  { .name = "build C++ lexer generator",
    .out = "*compilation*", .footer_panel = true, .save_dirty_files = true,
-   .cmd = { { "custom\\bin\\build_one_time 4coder_lex_gen_cpp.cpp ..\\build", .os = "win" }, }, },
+   .cmd = { { "custom\\bin\\build_one_time custom\\languages\\4coder_lex_gen_cpp.cpp ..\\build", .os = "win" }, }, },
  
  { .name = "build token tester",
    .out = "*compilation*", .footer_panel = true, .save_dirty_files = true,