Fixed lexer for preprocssor directives completely; fixed up build paths some more

4ed_coroutine.cpp

@@ -46,9 +46,9 @@ coroutine_main(void *ptr){
         }
         Assert(me->type != CoroutineType_Root);
         Assert(me->yield_ctx != 0);
-        Assert(me->function != 0);
+        Assert(me->func != 0);
         
-        me->function(me);
+        me->func(me);
         
         // NOTE(allen): Wake up the caller and set this coroutine back to being dead.
         Coroutine *other = me->yield_ctx;
@@ -56,7 +56,7 @@ coroutine_main(void *ptr){
         Assert(other->state == CoroutineState_Waiting);
         
         coroutine__pass_control(me, other, CoroutineState_Dead, CoroutinePassControl_ExitMe);
-        me->function = 0;
+        me->func = 0;
     }
 }
 
@@ -124,7 +124,7 @@ internal Coroutine*
 coroutine_create(Coroutine_Group *coroutines, Coroutine_Function *func){
     Coroutine *result = coroutine_system_alloc(coroutines);
     Assert(result->state == CoroutineState_Dead);
-    result->function = func;
+    result->func = func;
     return(result);
 }
 

4ed_coroutine.h

@@ -37,7 +37,7 @@ struct Coroutine{
     System_Thread thread;
     System_Condition_Variable cv;
     struct Coroutine_Group *sys;
-    Coroutine_Function *function;
+    Coroutine_Function *func;
     Coroutine *yield_ctx;
     Coroutine_State state;
     Coroutine_Type type;

custom/bin/build_one_time.bat

@@ -35,7 +35,7 @@ set opts=%opts% /I%custom_root%
 set opts=%opts% %mode%
 
 pushd %dst%
-call cl /I"%code_home%" %opts% %full_target% /Fegenerator
+call cl /I"%code_home%" %opts% %full_target% /Feone_time
 popd
 
 

custom/languages/4coder_lex_gen_cpp.cpp

@@ -311,6 +311,7 @@ build_language_model(void){
     AddState(ULL_number);
     
     AddState(pp_directive_whitespace);
+    AddState(pp_directive_first);
     AddState(pp_directive);
     AddState(pp_directive_emit);
     
@@ -774,13 +775,12 @@ build_language_model(void){
     ////
     
     sm_select_state(pp_directive_whitespace);
-    sm_delim_mark_first();
     sm_case(" \t\f\v", pp_directive_whitespace);
-    sm_case("abcdefghijklmnopqrstuvwxyz"
+    sm_case_peek("abcdefghijklmnopqrstuvwxyz"
-            "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-            "_"
+                 "_"
-            "0123456789",
+                 "0123456789",
-            pp_directive);
+                 pp_directive_first);
     {
         Emit_Rule *emit = sm_emit_rule();
         sm_emit_handler_direct("LexError");
@@ -789,8 +789,14 @@ build_language_model(void){
     
     ////
     
-    sm_select_state(pp_directive);
+    sm_select_state(pp_directive_first);
+    sm_delim_mark_first();
     sm_set_flag(is_pp_body, true);
+    sm_fallback_peek(pp_directive);
+    
+    ////
+    
+    sm_select_state(pp_directive);
     sm_case("abcdefghijklmnopqrstuvwxyz"
             "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
             "_"

custom/languages/4coder_lexer_cpp_test.cpp

@@ -408,6 +408,12 @@ R"bar(foo
 )bar"
 
 #error ``` Foo bar is bad news bears! ```
+#   define foo bar
+#    define bar baz
+#define baz bish
+
+# error wishy washy
+#define swish(x,y,...) switch(x){ y(__VA_ARGS__) }
 
 #endif
 
@@ -438,7 +444,7 @@ print_token_list(Token_List *list, String_Const_u8 text){
                    token_base_kind_names[token->kind],
                    token_cpp_kind_names[token->sub_kind],
                    token->flags, token->sub_flags);
-            printf("\t:%.*s:\n", token->size, text.str + token->pos);
+            printf("\t:%.*s:\n", (i32)token->size, text.str + token->pos);
         }
     }
 }

custom/lexer_generator/4coder_lex_gen_main.cpp

@@ -1399,7 +1399,7 @@ sm_emit_handler_keys(Keyword_Set *set){
 internal void
 sm_emit_handler_keys_delim(Flag *flag_check, Keyword_Set *set){
     Emit_Rule *rule = helper_ctx.selected_emit_rule;
-    smi_emit_handler(helper_ctx.arena, rule, set, flag_check);
+    smi_emit_handler_delim(helper_ctx.arena, rule, set, flag_check);
 }
 
 internal void
@@ -2419,6 +2419,7 @@ opt_emit_rule_match(Emit_Rule *rule_a, Emit_Rule *rule_b){
                 }
             }break;
             case EmitHandlerKind_Keywords:
+            case EmitHandlerKind_KeywordsDelim:
             {
                 if (handler_a->keywords != handler_b->keywords){
                     result = false;
@@ -3470,6 +3471,30 @@ gen_SLOW_action_list__cont_flow(Arena *scratch, Token_Kind_Set tokens, Flag_Set
                                 keep_looping = false;
                             }
                         }break;
+                        
+                        case EmitHandlerKind_KeywordsDelim:
+                        {
+                            Keyword_Set *keywords = handler->keywords;
+                            fprintf(out, "Lexeme_Table_Lookup lookup = "
+                                    "lexeme_table_lookup(%.*s_hash_array, %.*s_key_array, "
+                                    "%.*s_value_array, %.*s_slot_count, %.*s_seed, "
+                                    "delim_first, (delim_one_past_last - delim_first));\n",
+                                    string_expand(keywords->pretty_name),
+                                    string_expand(keywords->pretty_name),
+                                    string_expand(keywords->pretty_name),
+                                    string_expand(keywords->pretty_name),
+                                    string_expand(keywords->pretty_name));
+                            fprintf(out, "if (lookup.found_match){\n");
+                            fprintf(out, "token.kind = lookup.base_kind;\n");
+                            fprintf(out, "token.sub_kind = lookup.sub_kind;\n");
+                            fprintf(out, "break;\n");
+                            fprintf(out, "}\n");
+                            if (handler->keywords->has_fallback_token_kind){
+                                gen_emit__direct(scratch, tokens,
+                                                 keywords->fallback_name, out);
+                                keep_looping = false;
+                            }
+                        }break;
                     }
                     
                     if (handler->flag_check != 0){

platform_win32/win32_4ed.cpp

@@ -46,7 +46,10 @@
 #include "4ed_render_target.h"
 #include "4ed.h"
 
+#undef function
 #include <Windows.h>
+#define function static
+
 #include "win32_gl.h"
 
 //////////////////////////////

project.4coder

@@ -22,16 +22,16 @@ load_paths = {
  { load_paths_only, .os = "mac"  },
 };
 
-build_x86_win32 = "echo build: x86 & build.bat /DDEV_BUILD_X86";
+build_x64_win32 = "echo build: x64 & bin\\build.bat";
-build_x86_linux = "echo build: x86 & ./build.sh -DDEV_BUILD_X86";
+build_x64_linux = "echo build: x64 & bin/build.sh";
-build_x86_mac   = "echo build: x86 & ./build.sh -DDEV_BUILD_X86";
+build_x64_mac   = "echo build: x64 & bin/build.sh";
 
 command_list = {
  { .name = "build x64",
    .out = "*compilation*", .footer_panel = true, .save_dirty_files = true,
-   .cmd = { {"echo build: x64 & bin\\build.bat" , .os = "win"  },
+   .cmd = { {build_x64_win32, .os = "win"  },
-            {"echo build: x64 & bin/build.sh", .os = "linux"},
+            {build_x64_linux, .os = "linux"},
-            {"echo build: x64 & bin/build.sh", .os = "mac"  }, }, },
+            {build_x64_max  , .os = "mac"  }, }, },
  
  { .name = "run one time",
    .out = "*run*", .footer_panel = false, .save_dirty_files = false,
@@ -43,7 +43,7 @@ command_list = {
  
  { .name = "build token tester",
    .out = "*compilation*", .footer_panel = true, .save_dirty_files = true,
-   .cmd = { { "custom\\bin\\build_one_time languages\\4coder_lexer_cpp_test.cpp ..\\build", .os = "win" }, }, }
+   .cmd = { { "custom\\bin\\build_one_time custom\\languages\\4coder_lexer_cpp_test.cpp ..\\build", .os = "win" }, }, }
 };
 
 fkey_command[1] = "build x64";