lexer table generator started

4coder_custom.cpp

@@ -1,12 +1,11 @@
-// Set to Custom_None if you're going to drop in your own include/get_bindings call.
+// Delete CustomCurrent to define your own customizations or
-// or choose one of the preexisting customizations
+// set it to one of the preexisting options
 #define Custom_Current Custom_Default
 
-#define Custom_None -1
+#define Custom_Default 1
-#define Custom_Default 0 
 
 // The following customization schemes are power users only:
-#define Custom_HandmadeHero 1
+#define Custom_HandmadeHero 2
 
 
 // TOP
@@ -14,7 +13,7 @@
 #if Custom_Current == Custom_Default
 # include "4coder_default_bindings.cpp"
 #elif Custom_Current == Custom_HandmadeHero
-# include "power/4coder_handmade_hero.cpp"
+# include "power/4coder_casey.cpp"
 #endif
 
 extern "C" GET_BINDING_DATA(get_bindings){
@@ -22,7 +21,7 @@ extern "C" GET_BINDING_DATA(get_bindings){
     Bind_Helper *context = &context_actual;
     
 #if Custom_Current == Custom_Default
-    default_get_bindings(context);
+    default_get_bindings(context, true);
 #elif Custom_Current == Custom_HandmadeHero
     casey_get_bindings(context);
 #endif

4coder_custom.h

@@ -1,4 +1,7 @@
 
+#ifndef FCODER_CUSTOM_H
+#define FCODER_CUSTOM_H
+
 #include "4coder_version.h"
 #include "4coder_keycodes.h"
 #include "4coder_style.h"
@@ -183,7 +186,6 @@ enum Command_ID{
     cmdid_interactive_open,
     cmdid_reopen,
     cmdid_save,
-    cmdid_interactive_save_as,
     cmdid_change_active_panel,
     cmdid_interactive_switch_buffer,
     cmdid_interactive_kill_buffer,
@@ -233,6 +235,7 @@ enum Param_ID{
     par_cli_command,
     par_clear_blank_lines,
     par_use_tabs,
+    par_save_update_name,
     
     // never below this
     par_type_count
@@ -599,3 +602,5 @@ struct Binding_Unit{
         } hook;
     };
 };
+
+#endif

4coder_default.cpp

@@ -212,6 +212,11 @@ CUSTOM_COMMAND_SIG(open_file_in_quotes){
     }
 }
 
+CUSTOM_COMMAND_SIG(save_as){
+    push_parameter(app, par_save_update_name, 1);
+    exec_command(app, cmdid_save);
+}
+
 CUSTOM_COMMAND_SIG(goto_line){
     int line_number;
     Query_Bar bar;

4coder_default_bindings.cpp

@@ -10,7 +10,11 @@ unsigned char blink_t = 0;
 // 2^24 of them so don't be wasteful!
 enum My_Maps{
     my_code_map,
-    my_html_map
+    my_html_map,
+    // for testing
+    my_empty_map1,
+    my_empty_map2,
+    my_maps_count
 };
 
 HOOK_SIG(my_start){
@@ -40,13 +44,14 @@ HOOK_SIG(my_start){
 
 HOOK_SIG(my_file_settings){
     // NOTE(allen|a4): In hooks that want parameters, such as this file
-    // created hook.  The file created hook is guaranteed to have only
+    // opened hook.  The file created hook is guaranteed to have only
     // and exactly one buffer parameter.  In normal command callbacks
     // there are no parameter buffers.
     Buffer_Summary buffer = app->get_parameter_buffer(app, 0);
     assert(buffer.exists);
 
     int treat_as_code = 0;
+    int wrap_lines = 1;
 
     if (buffer.file_name && buffer.size < (16 << 20)){
         String ext = file_extension(make_string(buffer.file_name, buffer.file_name_len));
@@ -56,6 +61,13 @@ HOOK_SIG(my_file_settings){
         else if (match(ext, make_lit_string("hpp"))) treat_as_code = 1;
     }
 
+    if (treat_as_code){
+        wrap_lines = 0;
+    }
+    if (buffer.file_name[0] == '*'){
+        wrap_lines = 0;
+    }
+
     push_parameter(app, par_lex_as_cpp_file, treat_as_code);
     push_parameter(app, par_wrap_lines, !treat_as_code);
     push_parameter(app, par_key_mapid, (treat_as_code)?((int)my_code_map):((int)mapid_file));
@@ -66,8 +78,9 @@ HOOK_SIG(my_file_settings){
 }
 
 HOOK_SIG(my_frame){
-    // NOTE(allen|a4): Please use me sparingly! This get's called roughly once every *33 ms* if everything is going well.
+    // NOTE(allen|a4): Please use me sparingly! This get's called roughly once every *33 ms*
-    // But if you start doing a lot in here, there's nothing 4codes does to stop you from making it a lot slower.
+    // if everything is going well. But if you start doing a lot in here, there's nothing 4codes does
+    // to stop you from making it a lot slower.
 
     int result = 0;
     Theme_Color theme_color_1[] = {
@@ -120,7 +133,7 @@ CUSTOM_COMMAND_SIG(write_div){
 }
 
 CUSTOM_COMMAND_SIG(begin_html_mode){
-    push_parameter(app, par_key_mapid, my_html_map);
+    push_parameter(app, par_key_mapid, my_empty_map1);
     exec_command(app, cmdid_set_settings);
 }
 
@@ -291,12 +304,14 @@ CUSTOM_COMMAND_SIG(ruin_theme){
     app->set_theme_colors(app, colors, count);
 }
 
-void default_get_bindings(Bind_Helper *context){
+void default_get_bindings(Bind_Helper *context, int set_hooks){
     // NOTE(allen|a3.1): Hooks have no loyalties to maps. All hooks are global
     // and once set they always apply, regardless of what map is active.
-    set_hook(context, hook_start, my_start);
+    if (set_hooks){
-    set_hook(context, hook_open_file, my_file_settings);
+        set_hook(context, hook_start, my_start);
-    //set_hook(context, hook_frame, my_frame); // Example of a frame hook, but disabled by default.
+        set_hook(context, hook_open_file, my_file_settings);
+        //set_hook(context, hook_frame, my_frame); // Example of a frame hook, but disabled by default.
+    }
 
     set_scroll_rule(context, smooth_scroll_rule);
 
@@ -312,6 +327,7 @@ void default_get_bindings(Bind_Helper *context){
     bind(context, 'i', MDFR_CTRL, cmdid_interactive_switch_buffer);
     bind(context, 'c', MDFR_ALT, cmdid_open_color_tweaker);
     bind(context, 'o', MDFR_ALT, open_in_other);
+    bind(context, 'w', MDFR_CTRL, save_as);
 
     bind(context, 'm', MDFR_ALT, build_search);
     bind(context, 'x', MDFR_ALT, execute_arbitrary_command);
@@ -326,14 +342,21 @@ void default_get_bindings(Bind_Helper *context){
     bind(context, '~', MDFR_ALT, ruin_theme);
 
     end_map(context);
-    
+
-    
+
     begin_map(context, my_html_map);
     inherit_map(context, mapid_file);
     bind(context, 'h', MDFR_ALT, write_h);
     bind(context, 'd', MDFR_ALT, write_div);
     end_map(context);
 
+    begin_map(context, my_empty_map1);
+    inherit_map(context, mapid_nomap);
+    end_map(context);
+
+    begin_map(context, my_empty_map2);
+    inherit_map(context, mapid_nomap);
+    end_map(context);
 
     begin_map(context, my_code_map);
 
@@ -438,16 +461,15 @@ void default_get_bindings(Bind_Helper *context){
 
     bind(context, 'K', MDFR_CTRL, cmdid_kill_buffer);
     bind(context, 'O', MDFR_CTRL, cmdid_reopen);
-    bind(context, 'w', MDFR_CTRL, cmdid_interactive_save_as);
     bind(context, 's', MDFR_CTRL, cmdid_save);
-    
+
     bind(context, '\n', MDFR_SHIFT, write_and_auto_tab);
     bind(context, ' ', MDFR_SHIFT, cmdid_write_character);
-    
+
     bind(context, 'e', MDFR_CTRL, cmdid_center_view);
-    
+
     bind(context, 'T', MDFR_CTRL | MDFR_ALT, begin_html_mode);
-    
+
     end_map(context);
 }
 

4ed.cpp

@@ -94,7 +94,7 @@ app_get_or_add_map_index(Models *models, i32 mapid){
     i32 *map_id_table = models->map_id_table;
     for (result = 0; result < user_map_count; ++result){
         if (map_id_table[result] == mapid) break;
-        if (map_id_table[result] == 0){
+        if (map_id_table[result] == -1){
             map_id_table[result] = mapid;
             break;
         }
@@ -936,10 +936,11 @@ COMMAND_DECL(save){
     USE_FILE(file, view);
 
     Delay *delay = &models->delay1;
-
+    
     char *filename = 0;
     int filename_len = 0;
     int buffer_id = -1;
+    int update_names = 0;
 
     Command_Parameter *end = param_stack_end(&command->part);
     Command_Parameter *param = param_stack_first(&command->part, end);
@@ -952,27 +953,50 @@ COMMAND_DECL(save){
         else if (v == par_buffer_id && param->param.value.type == dynamic_type_int){
             buffer_id = dynamic_to_int(&param->param.value);
         }
+        else if (v == par_save_update_name){
+			update_names = dynamic_to_bool(&param->param.value);
+		}
     }
 
-    String name = {};
+#if 0
-    if (filename){
+#endif
-        name = make_string(filename, filename_len);
-    }
-    else if (file){
-        name = file->name.source_path;
-    }
 
-    if (name.size != 0){
+    if (buffer_id != -1){
-        if (buffer_id == -1){
+        file = working_set_get_active_file(&models->working_set, buffer_id);
-            if (file){
+	}
-                delayed_save(delay, name, file);
+    
+    if (update_names){
+        String name = {};
+        if (filename){
+            name = make_string(filename, filename_len);
+        }
+
+        if (file){
+            if (name.str){
+                if (!file->state.is_dummy && file_is_ready(file)){
+                    delayed_save_as(delay, name, file);
+                }
+            }
+            else{
+                view_show_interactive(system, view, &models->map_ui,
+                    IAct_Save_As, IInt_Sys_File_List, make_lit_string("Save As: "));
             }
         }
-        else{
+    }
-            file = working_set_get_active_file(&models->working_set, buffer_id);
+    else{
-            
+        String name = {};
-            if (!file->state.is_dummy && file_is_ready(file) && buffer_needs_save(file)){
+        if (filename){
-                delayed_save(delay, name, file);
+            name = make_string(filename, filename_len);
+        }
+        else if (file){
+            name = file->name.source_path;
+        }
+        
+        if (name.size != 0){
+            if (file){
+                if (!file->state.is_dummy && file_is_ready(file)){
+                    delayed_save(delay, name, file);
+                }
             }
             else{
                 delayed_save(delay, name);
@@ -981,15 +1005,6 @@ COMMAND_DECL(save){
     }
 }
 
-COMMAND_DECL(interactive_save_as){
-    ProfileMomentFunction();
-    USE_VIEW(view);
-    USE_MODELS(models);
-
-    view_show_interactive(system, view, &models->map_ui,
-        IAct_Save_As, IInt_Sys_File_List, make_lit_string("Save As: "));
-}
-
 COMMAND_DECL(change_active_panel){
     ProfileMomentFunction();
     USE_MODELS(models);
@@ -2301,9 +2316,7 @@ extern "C"{
     GET_ACTIVE_VIEW_SIG(external_get_active_view){
         Command_Data *cmd = (Command_Data*)app->cmd_context;
         View_Summary view = {};
-
         fill_view_summary(&view, cmd->view, &cmd->vars->live_set, &cmd->models->working_set);
-
         return(view);
     }
 
@@ -2318,6 +2331,7 @@ extern "C"{
         Command_Data *cmd = (Command_Data*)app->cmd_context;
         Live_Views *live_set;
         View *vptr;
+        Editing_File *file;
         int result = 0;
         int view_id;
 
@@ -2326,15 +2340,19 @@ extern "C"{
             view_id = view->view_id - 1;
             if (view_id >= 0 && view_id < live_set->max){
                 vptr = live_set->views + view_id;
-                result = 1;
+                file = vptr->file;
-                if (seek.type == buffer_seek_line_char && seek.character <= 0){
+                if (file && !file->state.is_loading){
-                    seek.character = 1;
+                    result = 1;
+                    if (seek.type == buffer_seek_line_char && seek.character <= 0){
+                        seek.character = 1;
+                    }
+                    vptr->cursor = view_compute_cursor(vptr, seek);
+                    if (set_preferred_x){
+                        vptr->preferred_x = view_get_cursor_x(vptr);
+                    }
+                    fill_view_summary(view, vptr, live_set, &cmd->models->working_set);
+                    file->state.cursor_pos = vptr->cursor.pos;
                 }
-                vptr->cursor = view_compute_cursor(vptr, seek);
-                if (set_preferred_x){
-                    vptr->preferred_x = view_get_cursor_x(vptr);
-                }
-                fill_view_summary(view, vptr, live_set, &cmd->models->working_set);
             }
         }
 
@@ -2658,7 +2676,7 @@ setup_command_table(){
     SET(interactive_open);
     SET(reopen);
     SET(save);
-    SET(interactive_save_as);
+    //SET(interactive_save_as);
     SET(change_active_panel);
     SET(interactive_switch_buffer);
     SET(interactive_kill_buffer);
@@ -3083,14 +3101,15 @@ App_Read_Command_Line_Sig(app_read_command_line){
     }
     else{
         vars = app_setup_memory(memory);
+
+        settings = &vars->models.settings;
+        *settings = {};
+        settings->font_size = 16;
+        
         if (clparams.argc > 1){
             init_command_line_settings(&vars->models.settings, plat_settings, clparams);
         }
-        else{
+        
-            settings = &vars->models.settings;
-            *settings = {};
-            settings->font_size = 16;
-        }
         *files = vars->models.settings.init_files;
         *file_count = &vars->models.settings.init_files_count;
     }
@@ -3209,6 +3228,7 @@ App_Init_Sig(app_init){
 
                 models->map_id_table = push_array(
                     &models->mem.part, i32, user_map_count);
+                memset(models->map_id_table, -1, user_map_count*sizeof(i32));
 
                 models->user_maps = push_array(
                     &models->mem.part, Command_Map, user_map_count);
@@ -4350,29 +4370,9 @@ App_Step_Sig(app_step){
                         }
                     }
                 }break;
-
+                
-                case DACT_SAVE_AS:
-                {
-                    // TODO(allen): deduplicate
-                    Editing_File *file = 0;
-                    if (panel){
-                        file = panel->view->file;
-                    }
-                    else if (string.str && string.size > 0){
-                        file = working_set_lookup_file(working_set, string);
-                    }
-                    if (file){
-                        i32 sys_id = file_save_and_set_names(system, exchange, mem, working_set, file, string.str);
-                        if (sys_id){
-                            app_push_file_binding(vars, sys_id, file->id.id);
-                        }
-                        else{
-                            delayed_action_repush(&models->delay2, act);
-                        }
-                    }
-                }break;
-
                 case DACT_SAVE:
+                case DACT_SAVE_AS:
                 {
                     if (!file){
                         if (panel){
@@ -4386,9 +4386,12 @@ App_Step_Sig(app_step){
                     }
                     // TODO(allen): We could handle the case where someone tries to save the same thing
                     // twice... that would be nice to have under control.
-                    if (file){
+                    if (file && buffer_needs_save(file)){
                         i32 sys_id = file_save(system, exchange, mem, file, file->name.source_path.str);
                         if (sys_id){
+                            if (act->type == DACT_SAVE_AS){
+                                file_set_name(working_set, file, string.str);
+                            }
                             // TODO(allen): This is fishy! Shouldn't we bind it to a file name instead? This file
                             // might be killed before we get notified that the saving is done!
                             app_push_file_binding(vars, sys_id, file->id.id);

build.bat

@@ -1,4 +1,5 @@
 @echo off
 
-"w:\4ed\misc\build_all.bat" /DFRED_SUPER /DFRED_NOT_PACKAGE /Zi
+"w:\4ed\misc\build_exp.bat" /Zi
+REM "w:\4ed\misc\build_all.bat" /DFRED_SUPER /DFRED_NOT_PACKAGE /Zi
 REM "w:\4ed\misc\build_all.bat" /O2 /Zi

power/4coder_casey.cpp

@@ -11,48 +11,96 @@
    merchantability, fitness for a particular purpose, or non-infringement.
 */
 
-/* TODO(casey): Here are our current issues
+/* NOTE(allen): Should be fixed now
+     - Need file limit to be substantially higher than 128 (65536?)
+     ~ file limit is now over 8 million
+
+     - Font size is too big
+     ~ -f N option on command line, default N = 16
+
+     - Asking for a buffer to be saved if you have not modified the buffer should not save the
+       buffer, or perhaps more "safely", it should diff the buffer against the existing on-disk
+       contents and only save if there is a detected change between them.
+
+     - Search:
+       - Needs to be case-insensitive, or at least have the option to be
+       - Needs to replace using the case of the thing being replaced, or at least have the option to do so
+
+     - Bug with opening too many files where it simply no longer can switch to a buffer at all?
+     ~ I assume this refers to a file limit issue, if not then maybe it's not actually fixed.
+
+     - Bug where opening the same buffer with open-file leads to a confusing situation
+       where you don't know what you're editing or something??
+
+     - Bug where replacing v4 with rectangle2 only replaces some instances???
+     ~ For the interested programmer: the range recomputation wasn't working right so it was always
+          using the original range from cursor to mark.  So if the string gets too long later occurances of
+          v4 get pushed outside of the range.
+
+     - Bug in search where extra backspaces after there are no characters yet "remembers"
+       how many you hit and then eats that many real characters you type?
 
-   - Display:
-     - Bug in scroll callback that seems to always pass the same view id instead of 
-       the correct id for each view?
-     - Jumping to subsequent errors in a file seems to jump to an unrelated position
-       then scroll back to the actual position, which results in lots of extra scrolling
-     - Need a way of highlighting the current line like Emacs does for the benefit
-       of people on The Stream(TM)
      - Some way to recenter the view so that the line containing the cursor becomes the
        center line vertically.
+     ~ cmdid_center_view
+
+     - Have buffers normalize slashes to always be forward-slash - right now I'm doing this manually
+*/
+
+/* TODO(casey): Here are our current issues
+
+   - High priority:
+     - Would like the option to indent to hanging parentheses, equals signs, etc. instead of
+       always just "one tab in from the previous line".
+       - Actually, maybe just expose the dirty state, so that the user can decide whether to
+         save or not?  Not sure...
+     - Replace:
+       - Needs to be case-insensitive, or at least have the option to be
+       - Needs to replace using the case of the thing being replaced, or at least have the option to do so
+     - Auto-complete doesn't pick nearby words first, it seems, which makes it much slower to use?
+     - Bug with not being able to switch-to-corresponding-file in another buffer
+       without accidentally bringing up the file open dialog?
+     
+   - Display:
+     - Need a word-wrap mode that wraps at word boundaries instead of characters
+     - Need to be able to set a word wrap length at something other than the window
+?FIXED First go-to-line for a file seems to still just go to the beginning of the buffer?
+       Not sure Allen's right about the slash problem, but either way, we need some
+       way to fix it.
+     - Need a way of highlighting the current line like Emacs does for the benefit
+       of people on The Stream(TM)
      - NOTE / IMPORTANT / TODO highlighting?  Ability to customize?  Whatever.
      - Some kind of parentheses highlighting?  I can write this myself, but I
        would need some way of adding highlight information to the buffer.
 
    - Indentation:
      - Multiple // lines don't seem to indent properly.  The first one will go to the correct place, but the subsequent ones will go to the first column regardless?
-     - Would like the option to indent to hanging parentheses, equals signs, etc. instead of
-       always just "one tab in from the previous line".
-     - Crash bug with paste-and-indent that sometimes leaves things unindented then crashes
      - Need to have better indentation / wrapping control for typing in comments. 
        Right now it's a bit worse than Emacs, which does automatically put you at
        the same margin as the prev. line (4coder just goes back to column 1).  It'd
        be nice if it go _better_ than Emacs, with no need to manually flow comments,
        etc.
+     - Up/down arrows and mouse clicks on wrapped lines don't seem to work properly after the second wrap 
+       (eg., a line wrapped to more than 2 physical lines on the screen.)
 
-   - Buffer management:
+   - Buffer management: 
-     - Bug in view iteration such that buffer_id is sometimes set to 0, so you can't find the view
-       for a buffer?
-     - Have buffers normalize slashes to always be forward-slash - right now I'm doing this manually   
      - Switch-to-buffer with no typing, just return, should switch to the most recently
        used buffer that is not currently displayed in a view.
-       - Kill-buffer should perform this switch automatically, or it should be easy
+  ?FIXED Kill-buffer should perform this switch automatically, or it should be easy
-         to build a custom kill buffer that does
+       to build a custom kill buffer that does
      - Seems like there's no way to switch to buffers whose names are substrings of other
        buffers' names without using the mouse?
+       - Also, mouse-clicking on buffers doesn't seem to work reliably?  Often it just goes to a 
+         blank window?
 
    - Need auto-complete for things like "arbitrary command", with options listed, etc.,
      so this should either be built into 4ed, or the custom DLL should have the ability
      to display possible completions and iterate over internal cmdid's, etc.  Possibly
      the latter, for maximal ability of customizers to add their own commands?
 
+   - Default directory for file open / build search should be that of the current
+     buffer, not tracked separately?  Probably I should code this on my own.
+
    - Macro recording/playback
 */
 
@@ -61,19 +109,33 @@
 #include <math.h>
 #include <stdio.h>
 
+#include "..\4coder_default.cpp"
+
+enum maps{
+	my_code_map
+};
+
 #ifndef Assert
 #define internal static
 #define Assert assert 
 #endif
 
+struct Parsed_Error
+{
+    int exists;
+
+    String target_file_name;
+    int target_line_number;
+
+    int source_buffer_id;
+    int source_position;
+};
+
 static bool GlobalEditMode;
 static char *GlobalCompilationBufferName = "*compilation*";
 
 // TODO(casey): If 4coder gets variables at some point, this would go in a variable.
 static char BuildDirectory[4096] = "./";
-static int ErrorParsingPosition;
-static int ErrorParsingLastJumpLine;
-static int ErrorParsingLastBufferID;
 
 enum token_type
 {
@@ -358,6 +420,8 @@ CUSTOM_COMMAND_SIG(casey_kill_to_end_of_line)
 
 CUSTOM_COMMAND_SIG(casey_paste_and_tab)
 {
+    // NOTE(allen): Paste puts the mark at the beginning and the cursor at
+    // the end of the pasted chunk, so it is all set for cmdid_auto_tab_range
     exec_command(app, cmdid_paste);
     exec_command(app, cmdid_auto_tab_range);
 }
@@ -417,7 +481,7 @@ SwitchToOrLoadFile(struct Application_Links *app, String FileName, bool CreateIf
             // to interactive open to tell it to fail if the file isn't there?
             exec_command(app, cmdid_interactive_open);
 
-            Result.buffer = app->get_buffer_by_name(app, FileName.str, FileName.size);
+            Result.buffer = app->get_buffer_by_name(app, FileName.str, FileName.size);            
 
             Result.Loaded = true;
             Result.Switched = true;
@@ -433,8 +497,6 @@ CUSTOM_COMMAND_SIG(casey_load_todo)
     SwitchToOrLoadFile(app, ToDoFileName, true);
 }
 
-inline String Empty() {String Result = {}; return(Result);}
-
 CUSTOM_COMMAND_SIG(casey_build_search)
 {
     int keep_going = 1;
@@ -524,6 +586,18 @@ CUSTOM_COMMAND_SIG(casey_find_corresponding_file)
     }
 }
 
+CUSTOM_COMMAND_SIG(casey_find_corresponding_file_other_window)
+{
+    View_Summary old_view = app->get_active_view(app);
+    Buffer_Summary buffer = app->get_buffer(app, old_view.buffer_id);
+
+    exec_command(app, cmdid_change_active_panel);
+    View_Summary new_view = app->get_active_view(app);
+    app->view_set_buffer(app, &new_view, buffer.buffer_id);
+
+//    exec_command(app, casey_find_corresponding_file);
+}
+
 CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking)
 {
     exec_command(app, cmdid_change_active_panel);
@@ -541,19 +615,25 @@ CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking)
 
     String dir = make_string(app->memory, 0, app->memory_size);
     append(&dir, BuildDirectory);
+    for(int At = 0;
+        At < dir.size;
+        ++At)
+    {
+        if(dir.str[At] == '/')
+        {
+            dir.str[At] = '\\';
+        }
+    }
+
 
     push_parameter(app, par_flags, CLI_OverlapWithConflict);
     push_parameter(app, par_name, GlobalCompilationBufferName, (int)strlen(GlobalCompilationBufferName));
     push_parameter(app, par_cli_path, dir.str, dir.size);
 
-    if(append(&dir, "build"))
+    if(append(&dir, "build.bat"))
     {
         push_parameter(app, par_cli_command, dir.str, dir.size);
         exec_command(app, cmdid_command_line);
-        ErrorParsingPosition = 0;
-        ErrorParsingLastJumpLine = 0;
-        ErrorParsingLastBufferID = 0;
-
         exec_command(app, cmdid_change_active_panel);
     }
     else{
@@ -561,21 +641,10 @@ CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking)
     }
 }
 
-struct Parsed_Error
-{
-    int exists;
-
-    String target_file_name;
-    int target_line_number;
-
-    int source_buffer_id;
-    int source_position;
-};
-
 internal bool
 casey_errors_are_the_same(Parsed_Error a, Parsed_Error b)
 {
-    bool result = ((a.exists == b.exists) && match(a.target_file_name, b.target_file_name) && (a.target_line_number == b.target_line_number));
+    bool result = ((a.exists == b.exists) && compare(a.target_file_name, b.target_file_name) && (a.target_line_number == b.target_line_number));
 
     return(result);
 }
@@ -621,6 +690,7 @@ casey_parse_error(Application_Links *app, Buffer_Summary buffer, View_Summary vi
     int size = end - start;
 
     char *ParsingRegion = (char *)malloc(size + 1);
+//    char *ParsingRegion = (char *)app->push_memory(app, size + 1);
     app->buffer_read_range(app, &buffer, start, end, ParsingRegion);
     ParsingRegion[size] = 0;
     tokenizer Tokenizer = {ParsingRegion};
@@ -661,14 +731,7 @@ casey_parse_error(Application_Links *app, Buffer_Summary buffer, View_Summary vi
                         result.target_line_number = line_number;
                         result.source_buffer_id = buffer.buffer_id;
                         result.source_position = start + (int)(ColonToken.Text - ParsingRegion);
-                        
+
-                        int start_pos;
-                        for (start_pos = 0;
-                            start_pos < result.target_file_name.size && result.target_file_name.str[start_pos] == ' ';
-                            ++start_pos);
-                        
-                        result.target_file_name = substr(result.target_file_name, start_pos);
-                        
                         break;
                     }
                 }
@@ -938,6 +1001,8 @@ CUSTOM_COMMAND_SIG(casey_quick_calc)
 internal void
 OpenProject(Application_Links *app, char *ProjectFileName)
 {
+    int TotalOpenAttempts = 0;
+
     FILE *ProjectFile = fopen(ProjectFileName, "r");
     if(ProjectFile)
     {
@@ -954,22 +1019,22 @@ OpenProject(Application_Links *app, char *ProjectFileName)
             BuildDirectory[BuildDirSize] = 0;
         }
 
+        char SourceFileDirectoryName[4096];
         char FileDirectoryName[4096];
-        while(fgets(FileDirectoryName, sizeof(FileDirectoryName) - 1, ProjectFile))
+        while(fgets(SourceFileDirectoryName, sizeof(SourceFileDirectoryName) - 1, ProjectFile))
         {
             // NOTE(allen|a3.4.4): Here we get the list of files in this directory.
             // Notice that we free_file_list at the end.
-            String dir = make_string(app->memory, 0, app->memory_size);
+            String dir = make_string(FileDirectoryName, 0, sizeof(FileDirectoryName));
-            append(&dir, FileDirectoryName);
+            append(&dir, SourceFileDirectoryName);
-            if(dir.size && dir.str[dir.size] == '\n')
+            if(dir.size && dir.str[dir.size-1] == '\n')
             {
                 --dir.size;
             }
 
-            if(dir.size && dir.str[dir.size] != '/')
+            if(dir.size && dir.str[dir.size-1] != '/')
             {
-                dir.str[dir.size] = '/';
+                dir.str[dir.size++] = '/';
-                ++dir.size;
             }
 
             File_List list = app->get_file_list(app, dir.str, dir.size);
@@ -992,6 +1057,7 @@ OpenProject(Application_Links *app, char *ProjectFileName)
                         push_parameter(app, par_name, dir.str, dir.size);
                         push_parameter(app, par_do_in_background, 1);
                         exec_command(app, cmdid_interactive_open);
+                        ++TotalOpenAttempts;
                     }
                 }
             }
@@ -1130,7 +1196,7 @@ DEFINE_MODAL_KEY(modal_t, casey_load_todo);
 DEFINE_MODAL_KEY(modal_u, cmdid_undo);
 DEFINE_MODAL_KEY(modal_v, casey_switch_buffer_other_window);
 DEFINE_MODAL_KEY(modal_w, cmdid_cut);
-DEFINE_MODAL_KEY(modal_x, casey_open_file_other_window);
+DEFINE_MODAL_KEY(modal_x, casey_find_corresponding_file_other_window);
 DEFINE_MODAL_KEY(modal_y, auto_tab_line_at_cursor);
 DEFINE_MODAL_KEY(modal_z, cmdid_interactive_open);
 
@@ -1185,7 +1251,9 @@ HOOK_SIG(casey_file_settings)
     if(treat_as_project)
     {
         OpenProject(app, buffer.file_name);
-        exec_command(app, cmdid_kill_buffer);
+        // NOTE(casey): Don't actually want to kill this, or you can never edit the project.
+//        exec_command(app, cmdid_kill_buffer);
+
     }
 
     return(0);
@@ -1238,10 +1306,10 @@ struct Casey_Scroll_Velocity
 };
 
 Casey_Scroll_Velocity casey_scroll_velocity_[16] = {0};
-Casey_Scroll_Velocity *casey_scroll_velocity = casey_scroll_velocity_ - 1;
+Casey_Scroll_Velocity *casey_scroll_velocity = casey_scroll_velocity_;
 
 SCROLL_RULE_SIG(casey_smooth_scroll_rule){
-    float dt = 1.0f/30.0f; // TODO(casey): Why do I not get the timestep here?
+    float dt = 1.0f/60.0f; // TODO(casey): Why do I not get the timestep here?
     Casey_Scroll_Velocity *velocity = casey_scroll_velocity + view_id;
     int result = 0;
     if(is_new_target)
@@ -1265,7 +1333,6 @@ SCROLL_RULE_SIG(casey_smooth_scroll_rule){
         velocity->t = 0;
         *scroll_x = target_x;
         *scroll_y = target_y;
-        result = 1;
     }
 
     return(result);
@@ -1294,6 +1361,7 @@ internal BOOL CALLBACK win32_find_4coder_window(HWND Window, LPARAM LParam)
 internal void
 win32_toggle_fullscreen(void)
 {
+#if 0
     // NOTE(casey): This follows Raymond Chen's prescription
     // for fullscreen toggling, see:
     // http://blogs.msdn.com/b/oldnewthing/archive/2010/04/12/9994016.aspx
@@ -1322,6 +1390,9 @@ win32_toggle_fullscreen(void)
                         SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER |
                         SWP_NOOWNERZORDER | SWP_FRAMECHANGED);
     }
+#else
+    ShowWindow(GlobalWindowHandle, SW_MAXIMIZE);
+#endif
 }
 
 HOOK_SIG(casey_start)
@@ -1330,7 +1401,7 @@ HOOK_SIG(casey_start)
     app->change_theme(app, literal("Handmade Hero"));
     app->change_font(app, literal("liberation mono"));
 
-    //win32_toggle_fullscreen();
+    win32_toggle_fullscreen();
 
     return(0);
 }

power/4coder_chronal.cpp

@@ -0,0 +1,304 @@
+
+// TOP
+#include "4coder_default.cpp"
+
+//#include "chr_winutils.h"
+
+#ifndef literal
+#define literal(s) (s), (sizeof(s)-1)
+#endif
+
+#define rgb_color(r, g, b) (r << 16 + g << 8 + b << 0)
+#define hex_color(hex) hex
+
+const int color_margin_normal = 0x341313;
+const int color_margin_insert = 0x5a3619;
+
+enum Vim_Maps {
+    mapid_normal = mapid_global,
+    mapid_insert = 0,
+    mapid_replace,
+    mapid_visual,
+
+    // There are a bunch of different chord "starters" that result in keys having
+    // different behaviors. There's no better way to handle this right now than
+    // just explicitly creating maps for each one.
+
+    //TODO(chronister): Chords can be built up, so this can have potentially huge
+    //combinatorics... what I *want* is a way to build up an actual stack of commands
+    //...
+
+    mapid_chord_delete,
+    mapid_chord_yank,
+    mapid_chord_g,
+};
+
+HOOK_SIG(chronal_init){
+    exec_command(app, cmdid_open_panel_vsplit);
+    exec_command(app, cmdid_change_active_panel);
+
+    app->change_theme(app, literal("4coder"));
+    app->change_font(app, literal("hack"));
+
+    const int color_bg = 0x15100f;
+    const int color_bar = 0x1c1212;
+    const int color_bar_hover = 0x261414;
+    const int color_bar_active = 0x341313;
+    const int color_text = 0x916550;
+    const int color_comment = 0x9d5b25;
+    const int color_string_literal = 0x9c2d21;
+    const int color_num_literals = 0xc56211;
+    const int color_keyword = 0xf74402;
+    Theme_Color colors[] = {
+        { Stag_Back, color_bg },
+        { Stag_Margin, color_bar },
+        { Stag_Margin_Hover, color_bar_hover },
+        { Stag_Margin_Active, color_margin_normal },
+        { Stag_Bar, color_bar },
+        { Stag_Bar_Active, color_bar_active },
+        { Stag_Base, color_text },
+        { Stag_Default, color_text },
+        { Stag_Comment, color_comment },
+        { Stag_Int_Constant, color_num_literals },
+        { Stag_Float_Constant, color_num_literals },
+        { Stag_Str_Constant, color_string_literal },
+        { Stag_Char_Constant, color_string_literal },
+        { Stag_Bool_Constant, color_keyword },
+        { Stag_Keyword, color_keyword },
+        { Stag_Special_Character, color_keyword },
+        { Stag_Preproc, color_keyword },
+    };
+
+    app->set_theme_colors(app, colors, ArrayCount(colors));
+
+    push_parameter(app, par_key_mapid, mapid_normal);
+    exec_command(app, cmdid_set_settings);
+
+    // no meaning for return
+    return(0);
+}
+
+HOOK_SIG(chronal_file_settings){
+    // NOTE(allen|a4): In hooks that want parameters, such as this file
+    // created hook.  The file created hook is guaranteed to have only
+    // and exactly one buffer parameter.  In normal command callbacks
+    // there are no parameter buffers.
+    Buffer_Summary buffer = app->get_parameter_buffer(app, 0);
+    assert(buffer.exists);
+
+    int treat_as_code = 0;
+
+    if (buffer.file_name && buffer.size < (16 << 20)){
+        String ext = file_extension(make_string(buffer.file_name, buffer.file_name_len));
+        if (match(ext, make_lit_string("cpp"))) treat_as_code = 1;
+        else if (match(ext, make_lit_string("h"))) treat_as_code = 1;
+        else if (match(ext, make_lit_string("c"))) treat_as_code = 1;
+        else if (match(ext, make_lit_string("hpp"))) treat_as_code = 1;
+    }
+
+    push_parameter(app, par_lex_as_cpp_file, treat_as_code);
+    push_parameter(app, par_wrap_lines, !treat_as_code);
+    push_parameter(app, par_key_mapid, mapid_normal);
+    exec_command(app, cmdid_set_settings);
+
+    // no meaning for return
+    return(0);
+}
+
+/*                 *
+ * Custom commands *
+ *                 */
+
+
+
+CUSTOM_COMMAND_SIG(do_nothing){
+}
+
+CUSTOM_COMMAND_SIG(enter_insert_mode){
+    push_parameter(app, par_key_mapid, mapid_insert);
+    exec_command(app, cmdid_set_settings);
+
+    Theme_Color colors[] = {
+        { Stag_Bar_Active, color_margin_insert },
+        { Stag_Margin_Active, color_margin_insert },
+    };
+    app->set_theme_colors(app, colors, ArrayCount(colors));
+}
+
+CUSTOM_COMMAND_SIG(enter_normal_mode){
+    push_parameter(app, par_key_mapid, mapid_normal);
+    exec_command(app, cmdid_set_settings);
+
+    Theme_Color colors[] = {
+        { Stag_Bar_Active, color_margin_normal },
+        { Stag_Margin_Active, color_margin_normal },
+    };
+    app->set_theme_colors(app, colors, ArrayCount(colors));
+}
+
+CUSTOM_COMMAND_SIG(seek_forward_word_start){
+    View_Summary view;
+    view = app->get_active_view(app);
+    push_parameter(app, par_flags, BoundryToken);
+    exec_command(app, cmdid_seek_right);
+    app->refresh_view(app, &view);
+}
+
+CUSTOM_COMMAND_SIG(seek_backward_word_start){
+    View_Summary view;
+    view = app->get_active_view(app);
+    push_parameter(app, par_flags, BoundryToken | BoundryWhitespace);
+    exec_command(app, cmdid_seek_left);
+    app->refresh_view(app, &view);
+}
+
+CUSTOM_COMMAND_SIG(seek_forward_word_end){
+    View_Summary view;
+    view = app->get_active_view(app);
+    push_parameter(app, par_flags, BoundryToken | BoundryWhitespace);
+    exec_command(app, cmdid_seek_right);
+    app->refresh_view(app, &view);
+}
+
+CUSTOM_COMMAND_SIG(newline_then_insert_before){
+    exec_command(app, cmdid_seek_beginning_of_line);
+    write_string(app, make_lit_string("\n"));
+    exec_command(app, cmdid_move_left);
+    exec_command(app, enter_insert_mode);
+}
+
+CUSTOM_COMMAND_SIG(newline_then_insert_after){
+    exec_command(app, cmdid_seek_end_of_line);
+    write_string(app, make_lit_string("\n"));
+    exec_command(app, enter_insert_mode);
+}
+
+CUSTOM_COMMAND_SIG(begin_chord_delete){
+    push_parameter(app, par_key_mapid, mapid_chord_delete);
+    exec_command(app, cmdid_set_settings);
+}
+
+CUSTOM_COMMAND_SIG(delete_line){
+    View_Summary view;
+    Buffer_Summary buffer;
+    int pos1, pos2;
+    view = app->get_active_view(app);
+
+    exec_command(app, cmdid_seek_beginning_of_line);
+    app->refresh_view(app, &view);
+    pos1 = view.cursor.pos;
+
+    exec_command(app, cmdid_seek_end_of_line);
+    app->refresh_view(app, &view);
+    pos2 = view.cursor.pos;
+    
+    Range range = make_range(pos1, pos2);
+    buffer = app->get_buffer(app, view.buffer_id);
+    app->buffer_replace_range(app, &buffer, range.start, range.end, 0, 0);
+}
+
+CUSTOM_COMMAND_SIG(delete_word){
+    View_Summary view;
+    Buffer_Summary buffer;
+    int pos1, pos2;
+    view = app->get_active_view(app);
+
+    exec_command(app, seek_backward_word_start);
+    app->refresh_view(app, &view);
+    pos1 = view.cursor.pos;
+
+    exec_command(app, seek_forward_word_end);
+    app->refresh_view(app, &view);
+    pos2 = view.cursor.pos;
+    
+    Range range = make_range(pos1, pos2);
+    buffer = app->get_buffer(app, view.buffer_id);
+    app->buffer_replace_range(app, &buffer, range.start, range.end, 0, 0);
+    
+}
+
+void
+chronal_get_bindings(Bind_Helper *context){
+    set_hook(context, hook_start, chronal_init);
+    set_hook(context, hook_open_file, chronal_file_settings);
+
+    set_scroll_rule(context, smooth_scroll_rule);
+
+    /*                          *
+     * SECTION: Vim keybindings *
+     *                          */
+
+    /* Normal mode.
+     * aka "It's eating all my input, help!" mode.
+     * Shortcuts for navigation, entering various modes,
+     * dealing with the editor.
+     */
+    begin_map(context, mapid_normal);
+
+    bind_vanilla_keys(context, do_nothing);
+
+    bind(context, 'w', MDFR_NONE, seek_forward_word_start);
+    bind(context, 'e', MDFR_NONE, seek_forward_word_end);
+    bind(context, 'b', MDFR_NONE, seek_backward_word_start);
+    bind(context, '$', MDFR_NONE, cmdid_seek_end_of_line);
+    bind(context, '0', MDFR_NONE, cmdid_seek_beginning_of_line);
+
+    bind(context, 'h', MDFR_NONE, cmdid_move_left);
+    bind(context, 'j', MDFR_NONE, cmdid_move_down);
+    bind(context, 'k', MDFR_NONE, cmdid_move_up);
+    bind(context, 'l', MDFR_NONE, cmdid_move_right);
+
+    bind(context, 'u', MDFR_CTRL, cmdid_page_up);
+    bind(context, 'd', MDFR_CTRL, cmdid_page_down);
+
+    bind(context, 'x', MDFR_NONE, cmdid_delete);
+
+    bind(context, 'u', MDFR_NONE, cmdid_undo);
+    bind(context, 'r', MDFR_CTRL, cmdid_redo);
+
+    bind(context, '/', MDFR_NONE, search);
+
+    bind(context, 'i', MDFR_NONE, enter_insert_mode);
+    bind(context, 'o', MDFR_NONE, newline_then_insert_after);
+    bind(context, 'O', MDFR_NONE, newline_then_insert_before);
+
+    bind(context, 'n', MDFR_CTRL, cmdid_word_complete);
+
+    // TEMP (will be replaced later by :statusbar commands)
+    bind(context, 'o', MDFR_CTRL, cmdid_interactive_open);
+    bind(context, 'c', MDFR_CTRL, cmdid_open_color_tweaker);
+    end_map(context);
+
+    /* Insert mode
+     * You type and it goes into the buffer. Nice and simple.
+     * Escape to exit.
+     */
+    begin_map(context, mapid_insert);
+    inherit_map(context, mapid_nomap);
+
+    bind_vanilla_keys(context, cmdid_write_character);
+    bind(context, key_back, MDFR_NONE, cmdid_backspace);
+
+    bind(context, key_esc, MDFR_NONE, enter_normal_mode);
+
+    end_map(context);
+
+#if 1
+
+    /* Chord "modes".
+     * They're not really an explicit mode per-say, but the meaning of key presses
+     * does change once a chord starts, and is context-dependent.
+     * TODO(chronister): I want these to properly build on each other.
+     */
+    begin_map(context, mapid_chord_delete);
+    inherit_map(context, mapid_nomap);
+
+    bind(context, 'd', MDFR_NONE, delete_line);
+    bind(context, 'w', MDFR_NONE, delete_word);
+
+    end_map(context);
+
+#endif
+}
+
+

power/4coder_experiments.cpp

@@ -0,0 +1,54 @@
+
+#include "4coder_default_bindings.cpp"
+
+enum Experiment_Maps{
+	my_experiment_map = my_maps_count
+};
+
+HOOK_SIG(my_file_settings){
+    Buffer_Summary buffer = app->get_parameter_buffer(app, 0);
+    assert(buffer.exists);
+
+    int treat_as_code = 0;
+    int wrap_lines = 1;
+
+    if (buffer.file_name && buffer.size < (16 << 20)){
+        String ext = file_extension(make_string(buffer.file_name, buffer.file_name_len));
+        if (match(ext, make_lit_string("cpp"))) treat_as_code = 1;
+        else if (match(ext, make_lit_string("h"))) treat_as_code = 1;
+        else if (match(ext, make_lit_string("c"))) treat_as_code = 1;
+        else if (match(ext, make_lit_string("hpp"))) treat_as_code = 1;
+    }
+
+    if (treat_as_code){
+        wrap_lines = 0;
+    }
+    if (buffer.file_name[0] == '*'){
+        wrap_lines = 0;
+    }
+
+    push_parameter(app, par_lex_as_cpp_file, treat_as_code);
+    push_parameter(app, par_wrap_lines, !treat_as_code);
+    push_parameter(app, par_key_mapid, (treat_as_code)?((int)my_experiment_map):((int)mapid_file));
+    exec_command(app, cmdid_set_settings);
+
+    // no meaning for return
+    return(0);
+}
+
+CUSTOM_COMMAND_SIG(kill_rect){
+	// TODO
+}
+
+void experiments_get_bindings(Bind_Helper *context){
+    default_get_bindings(context, 0);
+    
+    set_hook(context, hook_start, my_start);
+    set_hook(context, hook_open_file, experiment_file_settings);
+    
+    begin_map(context, my_experiment_map);
+    inherit_map(my_code_map);
+    
+    
+    end_map(context);
+}

test/4cpp_lexer_fsms.h

@@ -0,0 +1,97 @@
+/*
+ * FSMs for 4c++ lexer
+ * 
+ * 23.03.2016 (dd.mm.yyyy)
+ */
+
+// TOP
+
+enum Lex_State{
+    LS_default,
+    LS_identifier,
+    LS_pound,
+    LS_pp,
+    LS_char,
+    LS_char_slashed,
+    LS_string,
+    LS_string_slashed,
+    LS_number,
+    LS_number0,
+    LS_float,
+    LS_crazy_float0,
+    LS_crazy_float1,
+    LS_hex,
+    LS_comment_pre,
+    LS_comment,
+    LS_comment_slashed,
+    LS_comment_block,
+    LS_comment_block_ending,
+    LS_dot,
+    LS_ellipsis,
+    LS_less,
+    LS_less_less,
+    LS_more,
+    LS_more_more,
+    LS_minus,
+    LS_arrow,
+    LS_and,
+    LS_or,
+    LS_plus,
+    LS_colon,
+    LS_star,
+    LS_modulo,
+    LS_caret,
+    LS_eq,
+    LS_bang,
+    LS_error_message,
+    //
+    LS_count
+};
+
+enum Lex_Int_State{
+	LSINT_default,
+    LSINT_u,
+    LSINT_l,
+    LSINT_L,
+    LSINT_ul,
+    LSINT_uL,
+    LSINT_ll,
+    LSINT_extra
+};
+
+enum Lex_INC_State{
+    LSINC_default,
+    LSINC_quotes,
+    LSINC_pointy,
+    LSINC_junk,
+};
+
+enum Lex_PP_State{
+    LSPP_default,
+    LSPP_include,
+    LSPP_macro_identifier,
+    LSPP_identifier,
+    LSPP_body_if,
+    LSPP_body,
+    LSPP_number,
+    LSPP_error,
+    LSPP_junk,
+    //
+    LSPP_count
+};
+
+struct Whitespace_FSM{
+    unsigned char pp_state;
+    unsigned char white_done;
+};
+
+struct Lex_FSM{
+    unsigned char state;
+    unsigned char int_state;
+    unsigned char emit_token;
+    unsigned char multi_line;
+};
+
+// BOTTOM
+
+

test/4cpp_lexer_tables.c

@@ -0,0 +1,86 @@
+unsigned char main_fsm_eqclasses[] = {
+0,1,1,1,1,1,1,1,1,2,3,4,4,4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,5,6,7,8,9,10,11,8,8,12,13,8,14,15,16,17,18,18,18,18,18,18,18,18,18,19,8,20,21,22,8,8,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,8,25,8,26,24,1,23,23,23,23,27,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,28,24,24,8,29,8,8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+};
+
+const int num_eq_classes = 30;
+
+unsigned char main_fsm_table[] = {
+37,0,0,0,0,35,6,2,37,32,27,4,31,29,25,19,14,9,8,30,21,34,23,1,1,37,33,1,1,28,
+38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,1,1,38,38,38,38,1,1,38,38,1,1,38,
+39,3,2,39,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
+40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,3,3,40,40,40,40,3,3,40,40,3,3,40,
+41,4,4,4,4,4,4,4,4,4,4,41,4,4,4,4,4,4,4,4,4,4,4,4,4,5,4,4,4,4,
+42,4,4,6,5,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
+43,6,6,6,6,6,43,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,6,6,6,6,
+44,6,6,6,7,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
+45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,10,45,8,8,45,45,45,45,45,45,45,45,45,45,45,
+46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,10,46,8,8,46,46,46,46,46,46,46,46,46,13,46,
+47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,10,10,47,47,47,47,47,47,47,47,11,47,47,
+48,48,48,48,48,48,48,48,48,48,48,48,48,48,12,48,48,12,12,48,48,48,48,48,48,48,48,48,48,48,
+49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,12,12,49,49,49,49,49,49,49,49,49,49,49,
+50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,13,13,50,50,50,50,13,50,50,50,13,50,50,
+51,51,51,51,51,51,51,51,51,51,51,51,17,51,51,51,15,51,51,51,51,51,51,51,51,51,51,51,51,51,
+52,15,15,52,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,15,15,15,15,
+53,15,15,15,16,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,
+54,17,17,17,17,17,17,17,17,17,17,17,18,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
+55,17,17,17,17,17,17,17,17,17,17,17,18,17,17,17,55,17,17,17,17,17,17,17,17,17,17,17,17,17,
+56,56,56,56,56,56,56,56,56,56,56,56,56,56,56,20,56,10,10,56,56,56,56,56,56,56,56,56,56,56,
+57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,
+58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,22,58,58,58,58,58,58,58,58,58,
+59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,
+60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,24,60,60,60,60,60,60,60,
+61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,
+62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,26,62,62,62,62,62,62,62,
+63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,
+64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,
+65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,
+66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,
+67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,
+68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,
+69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,
+70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,
+71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,
+72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,
+73,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,
+};
+
+unsigned char main_fsm_multiline_table[] = {
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+};
+

test/4cpp_new_lexer.h

@@ -5,6 +5,8 @@
 #define FCPP_NEW_LEXER_INC
 
 #include "../4cpp_lexer_types.h"
+#include "4cpp_lexer_fsms.h"
+#include "4cpp_lexer_tables.c"
 
 namespace new_lex{
 //
@@ -332,80 +334,6 @@ cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token_i, int shift_amoun
     }
 }
 
-enum Lex_State{
-    LS_default,
-    LS_identifier,
-    LS_pound,
-    LS_pp,
-    LS_char,
-    LS_char_slashed,
-    LS_string,
-    LS_string_slashed,
-    LS_number,
-    LS_number0,
-    LS_float,
-    LS_crazy_float0,
-    LS_crazy_float1,
-    LS_hex,
-    LS_comment_pre,
-    LS_comment,
-    LS_comment_slashed,
-    LS_comment_block,
-    LS_comment_block_ending,
-    LS_dot,
-    LS_ellipsis,
-    LS_less,
-    LS_less_less,
-    LS_more,
-    LS_more_more,
-    LS_minus,
-    LS_arrow,
-    LS_and,
-    LS_or,
-    LS_plus,
-    LS_colon,
-    LS_star,
-    LS_modulo,
-    LS_caret,
-    LS_eq,
-    LS_bang,
-    LS_error_message,
-    //
-    LS_count
-};
-
-enum Lex_Int_State{
-	LSINT_default,
-    LSINT_u,
-    LSINT_l,
-    LSINT_L,
-    LSINT_ul,
-    LSINT_uL,
-    LSINT_ll,
-    LSINT_extra
-};
-
-enum Lex_INC_State{
-    LSINC_default,
-    LSINC_quotes,
-    LSINC_pointy,
-    LSINC_junk,
-};
-
-enum Lex_PP_State{
-    LSPP_default,
-    LSPP_include,
-    LSPP_macro_identifier,
-    LSPP_identifier,
-    LSPP_body_if,
-    LSPP_body,
-    LSPP_number,
-    LSPP_error,
-    LSPP_junk,
-    //
-    LSPP_count
-};
-
 enum Pos_Update_Rule{
     PUR_none,
     PUR_unget_whitespace
@@ -493,19 +421,6 @@ cpp_push_token_nonalloc(Cpp_Token *out_tokens, int *token_i, Cpp_Token token){
     }
 }
 
-struct Whitespace_FSM{
-    unsigned char pp_state;
-    unsigned char white_done;
-};
-
-struct Lex_FSM{
-    unsigned char state;
-    unsigned char int_state;
-    unsigned char emit_token;
-    unsigned char multi_line;
-    unsigned char completed;
-};
-
 struct Lex_Data{
     char *tb;
     int tb_pos;
@@ -600,7 +515,7 @@ int_fsm(Lex_FSM fsm, char c){
 }
 
 Lex_FSM
-main_fsm(Lex_FSM fsm, unsigned char pp_state, char c){
+main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
     if (c == 0) fsm.emit_token = 1;
     else
         switch (pp_state){
@@ -1030,14 +945,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
             for (; S.wfsm.white_done == 0 && S.pos < end_pos;){
                 c = chunk[S.pos++];
                 wfsm = S.wfsm;
-                {
+                wfsm = whitespace_skip_fsm(wfsm, c);
-                    if (wfsm.pp_state != LSPP_default){
-                        if (c == '\n') wfsm.pp_state = LSPP_default;
-                    }
-                    if (!(c == ' ' || c == '\n' || c == '\t' || c == '\r' || c == '\f' || c == '\v')){
-                        wfsm.white_done = 1;
-                    }
-                }
                 S.wfsm = wfsm;
             }
             if (S.wfsm.white_done == 0){
@@ -1052,401 +960,26 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
         S.tb_pos = 0;
         S.fsm = {0};
         for(;;){
-            for (; S.fsm.emit_token == 0 && S.pos < end_pos;){
+            if (S.pp_state == LSPP_default){
-                c = chunk[S.pos++];
+				for (; S.fsm.state < LS_count && S.pos < end_pos;){
-                S.tb[S.tb_pos++] = c;
+                    c = chunk[S.pos++];
-                
+                    S.tb[S.tb_pos++] = c;
-                fsm = S.fsm;
+                    
-                
+                    int i = S.fsm.state*num_eq_classes + main_fsm_eqclasses[c];
-                {
+                    S.fsm.multi_line |= main_fsm_multiline_table[i];
-                    if (c == 0) fsm.emit_token = 1;
+                    S.fsm.state = main_fsm_table[i];
-                    else
+                }
-                        switch (S.pp_state){
+                S.fsm.emit_token = (S.fsm.state >= LS_count);
-                        case LSPP_error:
+			}
-                        fsm.state = LS_error_message;
+            else{
-                        if (c == '\n') fsm.emit_token = 1;
+                for (; S.fsm.emit_token == 0 && S.pos < end_pos;){
-                        break;
+                    c = chunk[S.pos++];
-
+                    S.tb[S.tb_pos++] = c;
-                        case LSPP_include:
+
-                        switch (fsm.state){
+                    fsm = S.fsm;
-                            case LSINC_default:
+                    fsm = main_fsm(fsm, S.pp_state, c);
-                            switch (c){
+                    S.fsm = fsm;
-                                case '"': fsm.state = LSINC_quotes; break;
-                                case '<': fsm.state = LSINC_pointy; break;
-                                default: fsm.state = LSINC_junk; break;
-                            }
-                            break;
-
-                            case LSINC_quotes:
-                            if (c == '"') fsm.emit_token = 1;
-                            break;
-
-                            case LSINC_pointy:
-                            if (c == '>') fsm.emit_token = 1;
-                            break;
-
-                            case LSINC_junk:
-                            if (c == '\n') fsm.emit_token = 1;
-                            break;
-                        }
-                        break;
-
-                        default:
-                        switch (fsm.state){
-                            case LS_default:
-                            if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'){
-                                fsm.state = LS_identifier;
-                            }
-                            else if (c >= '1' && c <= '9'){
-                                fsm.state = LS_number;
-                            }
-                            else if (c == '0'){
-                                fsm.state = LS_number0;
-                            }
-                            else switch (c){
-                                case '\'': fsm.state = LS_char; break;
-                                case '"': fsm.state = LS_string; break;
-
-                                case '/': fsm.state = LS_comment_pre; break;
-
-                                case '.': fsm.state = LS_dot; break;
-
-                                case '<': fsm.state = LS_less; break;
-                                case '>': fsm.state = LS_more; break;
-
-                                case '-': fsm.state = LS_minus; break;
-
-                                case '&': fsm.state = LS_and; break;
-                                case '|': fsm.state = LS_or; break;
-
-                                case '+': fsm.state = LS_plus; break;
-
-                                case ':': fsm.state = LS_colon; break;
-
-                                case '*': fsm.state = LS_star; break;
-
-                                case '%': fsm.state = LS_modulo; break;
-                                case '^': fsm.state = LS_caret; break;
-
-                                case '=': fsm.state = LS_eq; break;
-                                case '!': fsm.state = LS_bang; break;
-
-                                case '#': fsm.state = LS_pound; break;
-
-#define OperCase(op,type) case op: fsm.emit_token = 1; break;
-                                OperCase('{', CPP_TOKEN_BRACE_OPEN);
-                                OperCase('}', CPP_TOKEN_BRACE_CLOSE);
-
-                                OperCase('[', CPP_TOKEN_BRACKET_OPEN);
-                                OperCase(']', CPP_TOKEN_BRACKET_CLOSE);
-
-                                OperCase('(', CPP_TOKEN_PARENTHESE_OPEN);
-                                OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE);
-
-                                OperCase('~', CPP_TOKEN_TILDE);
-                                OperCase(',', CPP_TOKEN_COMMA);
-                                OperCase(';', CPP_TOKEN_SEMICOLON);
-                                OperCase('?', CPP_TOKEN_TERNARY_QMARK);
-
-                                OperCase('@', CPP_TOKEN_JUNK);
-                                OperCase('$', CPP_TOKEN_JUNK);
-                                OperCase('\\', CPP_TOKEN_JUNK);
-#undef OperCase
-                            }
-                            break;
-
-                            case LS_identifier:
-                            if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
-                                fsm.emit_token = 1;
-                            }
-                            break;
-
-                            case LS_pound:
-                            if (S.pp_state == LSPP_default){
-                                if (c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'){
-                                    fsm.state = LS_pound;
-                                }
-                                else if (c == '\n'){
-                                    fsm.emit_token = 1;
-                                }
-                                else{
-                                    fsm.state = LS_pp;
-                                }
-                            }
-                            else{
-                                switch (c){
-                                    case '#': fsm.emit_token = 1; break;
-                                    default: fsm.emit_token = 1; break;
-                                }
-                            }
-                            break;
-
-                            case LS_pp:
-                            if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
-                                fsm.emit_token = 1;
-                            }
-                            break;
-
-                            case LS_char:
-                            switch(c){
-                                case '\'': fsm.emit_token = 1; break;
-                                case '\\': fsm.state = LS_char_slashed; break;
-                            }
-                            break;
-
-                            case LS_char_slashed:
-                            switch (c){
-                                case '\r': case '\f': case '\v': break;
-                                case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
-                                default: fsm.state = LS_char; break;
-                            }
-                            break;
-
-                            case LS_string:
-                            switch(c){
-                                case '\"': fsm.emit_token = 1; break;
-                                case '\\': fsm.state = LS_string_slashed; break;
-                            }
-                            break;
-
-                            case LS_string_slashed:
-                            switch (c){
-                                case '\r': case '\f': case '\v': break;
-                                case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
-                                default: fsm.state = LS_string; break;
-                            }
-                            break;
-
-                            case LS_number:
-                            if (c >= '0' && c <= '9'){
-                                fsm.state = LS_number;
-                            }
-                            else{
-                                switch (c){
-                                    case '.': fsm.state = LS_float; break;
-                                    default: fsm.emit_token = 1; break;
-                                }
-                            }
-                            break;
-
-                            case LS_number0:
-                            if (c >= '0' && c <= '9'){
-                                fsm.state = LS_number;
-                            }
-                            else if (c == 'x'){
-                                fsm.state = LS_hex;
-                            }
-                            else if (c == '.'){
-                                fsm.state = LS_float;
-                            }
-                            else{
-                                fsm.emit_token = 1;
-                            }
-                            break;
-
-                            case LS_float:
-                            if (!(c >= '0' && c <= '9')){
-                                switch (c){
-                                    case 'e': fsm.state = LS_crazy_float0; break;
-                                    default: fsm.emit_token = 1; break;
-                                }
-                            }
-                            break;
-
-                            case LS_crazy_float0:
-                            {
-                                if ((c >= '0' && c <= '9') || c == '-'){
-                                    fsm.state = LS_crazy_float1;
-                                }
-                                else{
-                                    fsm.emit_token = 1;
-                                }
-                            }
-                            break;
-
-                            case LS_crazy_float1:
-                            {
-                                if (!(c >= '0' && c <= '9')){
-                                    fsm.emit_token = 1;
-                                }
-                            }
-                            break;
-
-                            case LS_hex:
-                            if (!(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F')){
-                                fsm.emit_token = 1;
-                            }
-                            break;
-
-                            case LS_dot:
-                            if (c >= '0' && c <= '9'){
-                                fsm.state = LS_float;
-                            }
-                            else
-                                switch (c){
-                                case '.': fsm.state = LS_ellipsis; break;
-                                case '*': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_ellipsis: fsm.emit_token = 1; break;
-
-                            case LS_less:
-                            switch (c){
-                                case '<': fsm.state = LS_less_less; break;
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_less_less:
-                            switch (c){
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_more:
-                            switch (c){
-                                case '>': fsm.state = LS_more_more; break;
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_more_more:
-                            switch (c){
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_comment_pre:
-                            switch (c){
-                                case '/': fsm.state = LS_comment; break;
-                                case '*': fsm.state = LS_comment_block; break;
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_comment:
-                            switch (c){
-                                case '\\': fsm.state = LS_comment_slashed; break;
-                                case '\n': fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_comment_slashed:
-                            switch (c){
-                                case '\r': case '\f': case '\v': break;
-                                default: fsm.state = LS_comment; break;
-                            }
-                            break;
-
-                            case LS_comment_block:
-                            switch (c){
-                                case '*': fsm.state = LS_comment_block_ending; break;
-                            }
-                            break;
-
-                            case LS_comment_block_ending:
-                            switch (c){
-                                case '*': fsm.state = LS_comment_block_ending; break;
-                                case '/': fsm.emit_token = 1; break;
-                                default: fsm.state = LS_comment_block; break;
-                            }
-                            break;
-
-                            case LS_minus:
-                            switch (c){
-                                case '>': fsm.state = LS_arrow; break;
-                                case '-': fsm.emit_token = 1; break;
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_arrow:
-                            switch (c){
-                                case '*': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_and:
-                            switch (c){
-                                case '&': fsm.emit_token = 1; break;
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_or:
-                            switch (c){
-                                case '|': fsm.emit_token = 1; break;
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_plus:
-                            switch (c){
-                                case '+': fsm.emit_token = 1; break;
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_colon:
-                            switch (c){
-                                case ':': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_star:
-                            switch (c){
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_modulo:
-                            switch (c){
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_caret:
-                            switch (c){
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_eq:
-                            switch (c){
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-
-                            case LS_bang:
-                            switch (c){
-                                case '=': fsm.emit_token = 1; break;
-                                default: fsm.emit_token = 1; break;
-                            }
-                            break;
-                        }
-                        break;
-                    }
                 }
-                
-                //fsm = main_fsm(fsm, S.pp_state, c);
-                S.fsm = fsm;
             }
             if (S.fsm.emit_token == 0){
                 DrYield(3, 1);
@@ -1457,6 +990,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
         Assert(S.fsm.emit_token == 1);
         
         if (c != 0){
+            if (S.fsm.state >= LS_count) S.fsm.state -= LS_count;
             pos_update_rule = PUR_none;
             if (S.pp_state == LSPP_include){
                 switch (S.fsm.state){

test/experiment.cpp

@@ -30,25 +30,25 @@ dump_file(char *filename){
     Data data = {};
     HANDLE file;
     DWORD hi, lo;
-    
+
     file = CreateFile(filename, GENERIC_READ, 0, 0, 
         OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
 
     if (file != INVALID_HANDLE_VALUE){
         lo = GetFileSize(file, &hi);
         assert(hi == 0);
-        
+
         data.size = (int)lo;
         data.data = (byte*)malloc(data.size + 1);
-        
+
         ReadFile(file, data.data, lo, &lo, 0);
         data.data[data.size] = 0;
-        
+
         assert((int)lo == data.size);
-        
+
         CloseHandle(file);
     }
-    
+
     return(data);
 }
 
@@ -60,11 +60,11 @@ typedef struct File_Info{
 typedef struct File_List{
     // Ignore this, it's for internal stuff.
     void *block;
-    
+
     // The list of files and folders.
     File_Info *infos;
     int count;
-    
+
     // Ignore this, it's for internal stuff.
     int block_size;
 } File_List;
@@ -87,13 +87,13 @@ system_set_file_list(File_List *file_list, String directory){
         append(&dir, directory);
         char trail_str[] = "\\*";
         append(&dir, trail_str);
-        
+
         char *c_str_dir = make_c_str(dir);
-        
+
         WIN32_FIND_DATA find_data;
         HANDLE search;
         search = FindFirstFileA(c_str_dir, &find_data);
-        
+
         if (search != INVALID_HANDLE_VALUE){            
             i32 count = 0;
             i32 file_count = 0;
@@ -116,12 +116,12 @@ system_set_file_list(File_List *file_list, String directory){
                 file_list->block = Win32GetMemory(required_size);
                 file_list->block_size = required_size;
             }
-            
+
             file_list->infos = (File_Info*)file_list->block;
             char *name = (char*)(file_list->infos + file_count);
             if (file_list->block){
                 search = FindFirstFileA(c_str_dir, &find_data);
-                
+
                 if (search != INVALID_HANDLE_VALUE){
                     File_Info *info = file_list->infos;
                     more_files = 1;
@@ -130,7 +130,7 @@ system_set_file_list(File_List *file_list, String directory){
                             !match(find_data.cFileName, "..")){
                             info->folder = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
                             info->filename.str = name;
-                            
+
                             i32 i = 0;
                             for(;find_data.cFileName[i];++i) *name++ = find_data.cFileName[i];
                             info->filename.size = i;
@@ -142,9 +142,9 @@ system_set_file_list(File_List *file_list, String directory){
                         more_files = FindNextFile(search, &find_data);
                     }while(more_files);
                     FindClose(search);
-                    
+
                     file_list->count = file_count;
-                    
+
                 }else{
                     Win32FreeMemory(file_list->block);
                     file_list->block = 0;
@@ -210,14 +210,14 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks){
     int k, chunk_size, is_last;
 
     extension = file_extension(make_string_slowly(filename));
-    
+
     if (match(extension, "cpp") || match(extension, "h")){
         file_data = dump_file(filename);
         if (file_data.size < (100 << 10)){
             pass = 1;
             if (verbose >= 0) printf("testing on file: %s\n", filename);
             exp->test_total++;
-            
+
             exp->correct_stack.count = 0;
             exp->testing_stack.count = 0;
 
@@ -226,16 +226,16 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks){
 
             file_cpp.data = (char*)file_data.data;
             file_cpp.size = file_data.size;
-            
+
             ld.tb = (char*)malloc(file_data.size + 1);
-            
+
             {
                 i64 start;
-                
+
                 start = __rdtsc();
                 cpp_lex_file_nonalloc(file_cpp, &exp->correct_stack, lex_data);
                 time.handcoded += (__rdtsc() - start);
-                
+
                 start = __rdtsc();
                 if (chunks){
                     int relevant_size = file_data.size + 1;
@@ -246,19 +246,19 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks){
                             chunk_size = relevant_size - k;
                             is_last = 1;
                         }
-                        
+
                         int result = new_lex::cpp_lex_nonalloc(&ld, (char*)file_data.data + k, chunk_size, &exp->testing_stack);
                         if (result == 0 || result == 2) break;
-					}
+                    }
                 }
                 else{
                     new_lex::cpp_lex_nonalloc(&ld, (char*)file_data.data, file_data.size, &exp->testing_stack);
                 }
                 time.fsm += (__rdtsc() - start);
             }
-            
+
             free(ld.tb);
-            
+
             if (exp->correct_stack.count != exp->testing_stack.count){
                 pass = 0;
                 if (verbose >= 0){
@@ -307,7 +307,7 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks){
                 if (verbose >= 0) printf("test failed, you failed, fix it now!\n\n");
             }
         }
-        
+
         free(file_data.data);
     }
 }
@@ -334,37 +334,37 @@ show_time(Times t, int repeats, char *type){
 #define BASE_DIR "w:/4ed/data/test/"
 
 int main(){
-    
+
     int repeats = 100;
     int verbose_level = -1;
-    int chunk_start = 1;
+    int chunk_start = 0;
-    int chunk_end = 16;
+    int chunk_end = 0;
-#define TEST_FILE "lexer_test.cpp"    
+#define TEST_FILE "autotab.cpp"    
 #define SINGLE_ITEM 0
-    
+
     int chunks = (chunk_start > 0 && chunk_start <= chunk_end);
     int c = 0;
-    
+
     char test_directory[] = BASE_DIR;
     File_List all_files = {};
     Experiment exp = {};
     Experiment chunk_exp = {};
     Times exp_t = {};
     Times chunk_exp_t = {};
-    
+
     init_test_stack(&exp.correct_stack);
     init_test_stack(&exp.testing_stack);
-    
+
     init_test_stack(&chunk_exp.correct_stack);
     init_test_stack(&chunk_exp.testing_stack);
-    
+
     AllowLocal(test_directory);
     AllowLocal(all_files);
-    
+
 #if SINGLE_ITEM
     (void)(repeats);
     (void)(verbose_level);
-    
+
     if (chunks){
         begin_t(&chunk_exp_t);
         printf("With chunks of %d\n", chunks);
@@ -373,14 +373,14 @@ int main(){
         }
         end_t(&chunk_exp_t);
     }
-    
+
     begin_t(&exp_t);
     printf("Unchunked\n");
     run_experiment(&exp, BASE_DIR TEST_FILE, 1, 0);
     end_t(&exp_t);
-    
+
 #else
-    
+
     system_set_file_list(&all_files, make_lit_string(test_directory));
 
     for (int j = 0; j < repeats; ++j){
@@ -393,7 +393,7 @@ int main(){
                     }
                     end_t(&chunk_exp_t);
                 }
-                
+
                 begin_t(&exp_t);
                 if (verbose_level == -1 && chunks){
                     for (c = chunk_start; c <= chunk_end; ++c){
@@ -413,7 +413,7 @@ int main(){
         printf("chunks of sizes %d through %d tested\n", chunk_start, chunk_end);
         printf("chunked passed %d / %d tests\n", chunk_exp.passed_total, chunk_exp.test_total);
     }
-    
+
     printf("unchunk passed %d / %d tests\n", exp.passed_total, exp.test_total);
 
     if (passed(exp) && (chunks == 0 || passed(chunk_exp))){
@@ -421,8 +421,8 @@ int main(){
             show_time(chunk_exp_t, repeats, "Chunked");
         }
         show_time(exp_t, repeats, "Unchunked");
-	}
+    }
-    
+
     return(0);
 }
 

test/fsm_table_generator.cpp

@@ -0,0 +1,592 @@
+/*
+ * FSM table generator:
+ *  Generate FSM tables as ".c" files from FSM functions.
+ *
+ * 23.03.2016 (dd.mm.yyyy)
+ */
+
+// TOP
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "4cpp_lexer_fsms.h"
+
+Whitespace_FSM
+whitespace_skip_fsm(Whitespace_FSM wfsm, char c){
+    if (wfsm.pp_state != LSPP_default){
+        if (c == '\n') wfsm.pp_state = LSPP_default;
+    }
+    if (!(c == ' ' || c == '\n' || c == '\t' || c == '\r' || c == '\f' || c == '\v')){
+        wfsm.white_done = 1;
+    }
+    return(wfsm);
+}
+
+Lex_FSM
+int_fsm(Lex_FSM fsm, char c){
+    switch (fsm.int_state){
+        case LSINT_default:
+        switch (c){
+            case 'u': case 'U': fsm.int_state = LSINT_u; break;
+            case 'l': fsm.int_state = LSINT_l; break;
+            case 'L': fsm.int_state = LSINT_L; break;
+            default: fsm.emit_token = 1; break;
+        }
+        break;
+
+        case LSINT_u:
+        switch (c){
+            case 'l': fsm.int_state = LSINT_ul; break;
+            case 'L': fsm.int_state = LSINT_uL; break;
+            default: fsm.emit_token = 1; break;
+        }
+        break;
+
+        case LSINT_l:
+        switch (c){
+            case 'l': fsm.int_state = LSINT_ll; break;
+            case 'U': case 'u': fsm.int_state = LSINT_extra; break;
+            default: fsm.emit_token = 1; break;
+        }
+        break;
+
+        case LSINT_L:
+        switch (c){
+            case 'L': fsm.int_state = LSINT_ll; break;
+            case 'U': case 'u': fsm.int_state = LSINT_extra; break;
+            default: fsm.emit_token = 1; break;
+        }
+        break;
+
+        case LSINT_ul:
+        switch (c){
+            case 'l': fsm.int_state = LSINT_extra; break;
+            default: fsm.emit_token = 1; break;
+        }
+        break;
+
+        case LSINT_uL:
+        switch (c){
+            case 'L': fsm.int_state = LSINT_extra; break;
+            default: fsm.emit_token = 1; break;
+        }
+        break;
+
+        case LSINT_ll:
+        switch (c){
+            case 'u': case 'U': fsm.int_state = LSINT_extra; break;
+            default: fsm.emit_token = 1; break;
+        }
+        break;
+
+        case LSINT_extra:
+        fsm.emit_token = 1;
+        break;
+    }
+    return(fsm);
+}
+
+Lex_FSM
+main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
+    if (c == 0) fsm.emit_token = 1;
+    else
+        switch (pp_state){
+        case LSPP_error:
+        fsm.state = LS_error_message;
+        if (c == '\n') fsm.emit_token = 1;
+        break;
+
+        case LSPP_include:
+        switch (fsm.state){
+            case LSINC_default:
+            switch (c){
+                case '"': fsm.state = LSINC_quotes; break;
+                case '<': fsm.state = LSINC_pointy; break;
+                default: fsm.state = LSINC_junk; break;
+            }
+            break;
+
+            case LSINC_quotes:
+            if (c == '"') fsm.emit_token = 1;
+            break;
+
+            case LSINC_pointy:
+            if (c == '>') fsm.emit_token = 1;
+            break;
+
+            case LSINC_junk:
+            if (c == '\n') fsm.emit_token = 1;
+            break;
+        }
+        break;
+
+        default:
+        switch (fsm.state){
+            case LS_default:
+            if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'){
+                fsm.state = LS_identifier;
+            }
+            else if (c >= '1' && c <= '9'){
+                fsm.state = LS_number;
+            }
+            else if (c == '0'){
+                fsm.state = LS_number0;
+            }
+            else switch (c){
+                case '\'': fsm.state = LS_char; break;
+                case '"': fsm.state = LS_string; break;
+
+                case '/': fsm.state = LS_comment_pre; break;
+
+                case '.': fsm.state = LS_dot; break;
+
+                case '<': fsm.state = LS_less; break;
+                case '>': fsm.state = LS_more; break;
+
+                case '-': fsm.state = LS_minus; break;
+
+                case '&': fsm.state = LS_and; break;
+                case '|': fsm.state = LS_or; break;
+
+                case '+': fsm.state = LS_plus; break;
+
+                case ':': fsm.state = LS_colon; break;
+
+                case '*': fsm.state = LS_star; break;
+
+                case '%': fsm.state = LS_modulo; break;
+                case '^': fsm.state = LS_caret; break;
+
+                case '=': fsm.state = LS_eq; break;
+                case '!': fsm.state = LS_bang; break;
+
+                case '#': fsm.state = LS_pound; break;
+
+#define OperCase(op,type) case op: fsm.emit_token = 1; break;
+                OperCase('{', CPP_TOKEN_BRACE_OPEN);
+                OperCase('}', CPP_TOKEN_BRACE_CLOSE);
+
+                OperCase('[', CPP_TOKEN_BRACKET_OPEN);
+                OperCase(']', CPP_TOKEN_BRACKET_CLOSE);
+
+                OperCase('(', CPP_TOKEN_PARENTHESE_OPEN);
+                OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE);
+
+                OperCase('~', CPP_TOKEN_TILDE);
+                OperCase(',', CPP_TOKEN_COMMA);
+                OperCase(';', CPP_TOKEN_SEMICOLON);
+                OperCase('?', CPP_TOKEN_TERNARY_QMARK);
+
+                OperCase('@', CPP_TOKEN_JUNK);
+                OperCase('$', CPP_TOKEN_JUNK);
+                OperCase('\\', CPP_TOKEN_JUNK);
+#undef OperCase
+            }
+            break;
+
+            case LS_identifier:
+            if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
+                fsm.emit_token = 1;
+            }
+            break;
+
+            case LS_pound:
+            if (pp_state == LSPP_default){
+                if (c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'){
+                    fsm.state = LS_pound;
+                }
+                else if (c == '\n'){
+                    fsm.emit_token = 1;
+                }
+                else{
+                    fsm.state = LS_pp;
+                }
+            }
+            else{
+                switch (c){
+                    case '#': fsm.emit_token = 1; break;
+                    default: fsm.emit_token = 1; break;
+                }
+            }
+            break;
+
+            case LS_pp:
+            if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
+                fsm.emit_token = 1;
+            }
+            break;
+
+            case LS_char:
+            switch(c){
+                case '\'': fsm.emit_token = 1; break;
+                case '\\': fsm.state = LS_char_slashed; break;
+            }
+            break;
+
+            case LS_char_slashed:
+            switch (c){
+                case '\r': case '\f': case '\v': break;
+                case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
+                default: fsm.state = LS_char; break;
+            }
+            break;
+
+            case LS_string:
+            switch(c){
+                case '\"': fsm.emit_token = 1; break;
+                case '\\': fsm.state = LS_string_slashed; break;
+            }
+            break;
+
+            case LS_string_slashed:
+            switch (c){
+                case '\r': case '\f': case '\v': break;
+                case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
+                default: fsm.state = LS_string; break;
+            }
+            break;
+
+            case LS_number:
+            if (c >= '0' && c <= '9'){
+                fsm.state = LS_number;
+            }
+            else{
+                switch (c){
+                    case '.': fsm.state = LS_float; break;
+                    default: fsm.emit_token = 1; break;
+                }
+            }
+            break;
+
+            case LS_number0:
+            if (c >= '0' && c <= '9'){
+                fsm.state = LS_number;
+            }
+            else if (c == 'x'){
+                fsm.state = LS_hex;
+            }
+            else if (c == '.'){
+                fsm.state = LS_float;
+            }
+            else{
+                fsm.emit_token = 1;
+            }
+            break;
+
+            case LS_float:
+            if (!(c >= '0' && c <= '9')){
+                switch (c){
+                    case 'e': fsm.state = LS_crazy_float0; break;
+                    default: fsm.emit_token = 1; break;
+                }
+            }
+            break;
+
+            case LS_crazy_float0:
+            {
+                if ((c >= '0' && c <= '9') || c == '-'){
+                    fsm.state = LS_crazy_float1;
+                }
+                else{
+                    fsm.emit_token = 1;
+                }
+            }
+            break;
+
+            case LS_crazy_float1:
+            {
+                if (!(c >= '0' && c <= '9')){
+                    fsm.emit_token = 1;
+                }
+            }
+            break;
+
+            case LS_hex:
+            if (!(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F')){
+                fsm.emit_token = 1;
+            }
+            break;
+
+            case LS_dot:
+            if (c >= '0' && c <= '9'){
+                fsm.state = LS_float;
+            }
+            else
+                switch (c){
+                case '.': fsm.state = LS_ellipsis; break;
+                case '*': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_ellipsis: fsm.emit_token = 1; break;
+
+            case LS_less:
+            switch (c){
+                case '<': fsm.state = LS_less_less; break;
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_less_less:
+            switch (c){
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_more:
+            switch (c){
+                case '>': fsm.state = LS_more_more; break;
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_more_more:
+            switch (c){
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_comment_pre:
+            switch (c){
+                case '/': fsm.state = LS_comment; break;
+                case '*': fsm.state = LS_comment_block; break;
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_comment:
+            switch (c){
+                case '\\': fsm.state = LS_comment_slashed; break;
+                case '\n': fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_comment_slashed:
+            switch (c){
+                case '\r': case '\f': case '\v': break;
+                default: fsm.state = LS_comment; break;
+            }
+            break;
+
+            case LS_comment_block:
+            switch (c){
+                case '*': fsm.state = LS_comment_block_ending; break;
+            }
+            break;
+
+            case LS_comment_block_ending:
+            switch (c){
+                case '*': fsm.state = LS_comment_block_ending; break;
+                case '/': fsm.emit_token = 1; break;
+                default: fsm.state = LS_comment_block; break;
+            }
+            break;
+
+            case LS_minus:
+            switch (c){
+                case '>': fsm.state = LS_arrow; break;
+                case '-': fsm.emit_token = 1; break;
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_arrow:
+            switch (c){
+                case '*': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_and:
+            switch (c){
+                case '&': fsm.emit_token = 1; break;
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_or:
+            switch (c){
+                case '|': fsm.emit_token = 1; break;
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_plus:
+            switch (c){
+                case '+': fsm.emit_token = 1; break;
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_colon:
+            switch (c){
+                case ':': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_star:
+            switch (c){
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_modulo:
+            switch (c){
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_caret:
+            switch (c){
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_eq:
+            switch (c){
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+
+            case LS_bang:
+            switch (c){
+                case '=': fsm.emit_token = 1; break;
+                default: fsm.emit_token = 1; break;
+            }
+            break;
+        }
+        break;
+    }
+    return(fsm);
+}
+
+void
+begin_table(FILE *file, char *type, char *table_name){
+    fprintf(file, "unsigned %s %s[] = {\n", type, table_name);
+}
+
+void
+do_table_item(FILE *file, unsigned short item){
+    fprintf(file, "%d,", (int)item);
+}
+
+void
+end_row(FILE *file){
+    fprintf(file, "\n");
+}
+
+void
+end_table(FILE *file){
+    fprintf(file, "};\n\n");
+}
+
+int main(){
+    FILE *file;
+    file = fopen("4cpp_lexer_tables.c", "wb");
+    
+    unsigned char *full_transition_table = (unsigned char*)malloc(LS_count * 256);
+    unsigned char *marks = (unsigned char*)malloc(LS_count * 256);
+    unsigned char *eq_class = (unsigned char*)malloc(LS_count * 256);
+    unsigned char *eq_class_rep = (unsigned char*)malloc(LS_count * 256);
+    memset(marks, 0, 256);
+    
+    int i = 0;
+    Lex_FSM fsm = {0};
+    Lex_FSM new_fsm;
+    for (unsigned short c = 0; c < 256; ++c){
+        for (unsigned char state = 0; state < LS_count; ++state){
+            fsm.state = state;
+            fsm.emit_token = 0;
+            new_fsm = main_fsm(fsm, LSPP_default, (unsigned char)c);
+            full_transition_table[i++] = new_fsm.state + LS_count*new_fsm.emit_token;
+        }
+    }
+    
+    unsigned char eq_class_counter = 0;
+    unsigned char *c_line = full_transition_table;
+    for (unsigned short c = 0; c < 256; ++c){
+        if (marks[c] == 0){
+            eq_class[c] = eq_class_counter;
+            eq_class_rep[eq_class_counter] = (unsigned char)c;
+            unsigned char *c2_line = c_line + LS_count;
+            for (unsigned short c2 = c + 1; c2 < 256; ++c2){
+                if (memcmp(c_line, c2_line, LS_count) == 0){
+                    marks[c2] = 1;
+                    eq_class[c2] = eq_class_counter;
+                }
+                c2_line += LS_count;
+            }
+            ++eq_class_counter;
+        }
+        c_line += LS_count;
+	}
+    
+    unsigned char *reduced_transition_table = (unsigned char*)malloc(LS_count * eq_class_counter);
+    unsigned char *reduced_multiline_table = (unsigned char*)malloc(LS_count * eq_class_counter);
+    i = 0;
+    for (unsigned char state = 0; state < LS_count; ++state){
+        fsm.state = state;
+        for (unsigned short eq = 0; eq < eq_class_counter; ++eq){
+            fsm.emit_token = 0;
+            fsm.multi_line = 0;
+            new_fsm = main_fsm(fsm, LSPP_default, eq_class_rep[eq]);
+            reduced_transition_table[i] = new_fsm.state + LS_count*new_fsm.emit_token;
+            reduced_multiline_table[i] = new_fsm.multi_line;
+            ++i;
+        }
+    }
+
+    begin_table(file, "char", "main_fsm_eqclasses");
+    for (unsigned short c = 0; c < 256; ++c){
+        do_table_item(file, eq_class[c]);
+    }
+    end_row(file);
+    end_table(file);
+    
+    fprintf(file, "const int num_eq_classes = %d;\n\n", eq_class_counter);
+    
+    i = 0;
+    begin_table(file, "char", "main_fsm_table");
+    for (unsigned char state = 0; state < LS_count; ++state){
+        for (unsigned short c = 0; c < eq_class_counter; ++c){
+            do_table_item(file, reduced_transition_table[i++]);
+		}
+        end_row(file);
+	}
+    end_table(file);
+    
+    i = 0;
+    begin_table(file, "char", "main_fsm_multiline_table");
+    for (unsigned char state = 0; state < LS_count; ++state){
+        for (unsigned short c = 0; c < eq_class_counter; ++c){
+            do_table_item(file, reduced_multiline_table[i++]);
+        }
+        end_row(file);
+    }
+    end_table(file);
+    
+    fclose(file);
+    
+    return(0);
+}
+
+// BOTTOM
+
+

win32_4ed.cpp

@@ -463,26 +463,33 @@ Sys_Set_File_List_Sig(system_set_file_list){
     }
 }
 
-// TODO(allen): proper "is terminated" check
 internal
 Sys_File_Unique_Hash_Sig(system_file_unique_hash){
     Unique_Hash hash = {0};
     BY_HANDLE_FILE_INFORMATION info;
     HANDLE handle;
-    
+    char space[1024];
-    handle = CreateFile(filename.str, GENERIC_READ, 0, 0,
+    String str;
-        OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
+
-    
+    if (filename.size < sizeof(space)){
-    *success = 0;
+        str = make_fixed_width_string(space);
-    if (handle && handle != INVALID_HANDLE_VALUE){
+        copy(&str, filename);
-        if (GetFileInformationByHandle(handle, &info)){
+        terminate_with_null(&str);
-            hash.d[2] = info.dwVolumeSerialNumber;
-            hash.d[1] = info.nFileIndexHigh;
-            hash.d[0] = info.nFileIndexLow;
-            *success = 1;
-        }
         
-        CloseHandle(handle);
+        handle = CreateFile(str.str, GENERIC_READ, 0, 0,
+            OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
+
+        *success = 0;
+        if (handle && handle != INVALID_HANDLE_VALUE){
+            if (GetFileInformationByHandle(handle, &info)){
+                hash.d[2] = info.dwVolumeSerialNumber;
+                hash.d[1] = info.nFileIndexHigh;
+                hash.d[0] = info.nFileIndexLow;
+                *success = 1;
+            }
+
+            CloseHandle(handle);
+        }
     }
     
     return(hash);