Virtual whitespace flashing fixed by edit chain in tick callback

4ed.cpp

@@ -11,7 +11,7 @@
 
 internal void
 output_file_append(Thread_Context *tctx, Models *models, Editing_File *file, String_Const_u8 value){
-         i64 end = buffer_size(&file->state.buffer);
+    i64 end = buffer_size(&file->state.buffer);
     Edit_Behaviors behaviors = {};
     edit_single(tctx, models, file, Ii64(end), value, behaviors);
 }
@@ -24,7 +24,7 @@ file_cursor_to_end(Thread_Context *tctx, Models *models, Editing_File *file){
     for (Panel *panel = layout_get_first_open_panel(layout);
          panel != 0;
          panel = layout_get_next_open_panel(layout, panel)){
-        View *view = panel->view;
+          View *view = panel->view;
         if (view->file != file){
             continue;
         }
@@ -82,7 +82,7 @@ fill_hardcode_default_style(Color_Table color_table){
     color_table.vals[Stag_Text_Cycle_2] = 0xFF00A000;
     color_table.vals[Stag_Text_Cycle_3] = 0xFF0030B0;
     color_table.vals[Stag_Text_Cycle_4] = 0xFFA0A000;
-    
+     
     color_table.vals[Stag_Line_Numbers_Back] = 0xFF101010;
     color_table.vals[Stag_Line_Numbers_Text] = 0xFF404040;
 }
@@ -798,6 +798,14 @@ App_Step_Sig(app_step){
             models->color_table = color_table;
         }
         
+        Application_Links app = {};
+        app.tctx = tctx;
+        app.cmd_context = models;
+        
+        if (models->tick != 0){
+            models->tick(&app, frame);
+        }
+        
         begin_render_section(target, models->frame_counter, literal_dt, animation_dt);
         models->in_render_mode = true;
         
@@ -811,9 +819,6 @@ App_Step_Sig(app_step){
             if (ctx != 0){
                 Render_Caller_Function *render_caller = ctx->ctx.render_caller;
                 if (render_caller != 0){
-                    Application_Links app = {};
-                    app.tctx = tctx;
-                    app.cmd_context = models;
                     render_caller(&app, frame, view_get_id(live_views, view));
                 }
             }

4ed_api_implementation.cpp

@@ -2168,6 +2168,10 @@ set_custom_hook(Application_Links *app, Hook_ID hook_id, Void_Func *func_ptr){
         {
             models->view_event_handler = (Custom_Command_Function*)func_ptr;
         }break;
+        case HookID_Tick:
+        {
+            models->tick = (Tick_Function*)func_ptr;
+        }break;
         case HookID_RenderCaller:
         {
             models->render_caller = (Render_Caller_Function*)func_ptr;

4ed_app_models.h

@@ -49,6 +49,7 @@ struct Models{
     Child_Process_Container child_processes;
     Custom_API config_api;
     
+    Tick_Function *tick;
     Render_Caller_Function *render_caller;
     Delta_Rule_Function *delta_rule;
     umem delta_rule_memory_size;

custom/4coder_default_bindings.cpp

@@ -20,6 +20,7 @@ custom_layer_init(Application_Links *app){
     setup_default_mapping(&framework_mapping);
     async_task_handler_init(app, &global_async_system);
     code_index_init();
+    buffer_modified_set_init();
     
     Profile_Global_List *list = get_core_profile_list(app);
     ProfileThreadName(tctx, list, string_u8_litexpr("main"));

custom/4coder_default_framework.cpp

@@ -460,7 +460,6 @@ default_4coder_initialize(Application_Links *app, String_Const_u8_Array file_nam
     buffer_map_id      = managed_id_declare(app, SCu8("DEFAULT.buffer_map_id"     ));
 	buffer_eol_setting = managed_id_declare(app, SCu8("DEFAULT.buffer_eol_setting"));
     buffer_lex_task = managed_id_declare(app, SCu8("DEFAULT.buffer_lex_task"));
-    buffer_parse_task = managed_id_declare(app, SCu8("DEFAULT.buffer_parse_task"));
     
     sticky_jump_marker_handle = managed_id_declare(app, SCu8("DEFAULT.sticky_jump_marker_handle"));
     attachment_tokens = managed_id_declare(app, SCu8("DEFAULT.tokens"));
@@ -572,5 +571,70 @@ default_4coder_one_panel(Application_Links *app){
     default_4coder_one_panel(app, file_names);
 }
 
+////////////////////////////////
+
+function void
+buffer_modified_set_init(void){
+    Buffer_Modified_Set *set = &global_buffer_modified_set;
+    
+    block_zero_struct(set);
+    Base_Allocator *allocator = get_base_allocator_system();
+    set->arena = make_arena(allocator);
+    set->id_to_node = make_table_u64_u64(allocator, 100);
+}
+
+function Buffer_Modified_Node*
+buffer_modified_set__alloc_node(Buffer_Modified_Set *set){
+    Buffer_Modified_Node *result = set->free;
+    if (result == 0){
+        result = push_array(&set->arena, Buffer_Modified_Node, 1);
+    }
+    else{
+        sll_stack_pop(set->free);
+    }
+    return(result);
+}
+
+function void
+buffer_mark_as_modified(Buffer_ID buffer){
+    Buffer_Modified_Set *set = &global_buffer_modified_set;
+    
+    Table_Lookup lookup = table_lookup(&set->id_to_node, (u64)buffer);
+    if (!lookup.found_match){
+        Buffer_Modified_Node *node = buffer_modified_set__alloc_node(set);
+        zdll_push_back(set->first, set->last, node);
+        node->buffer = buffer;
+        table_insert(&set->id_to_node, (u64)buffer, (u64)PtrAsInt(node));
+    }
+}
+
+function void
+buffer_unmark_as_modified(Buffer_ID buffer){
+    Buffer_Modified_Set *set = &global_buffer_modified_set;
+    
+    Table_Lookup lookup = table_lookup(&set->id_to_node, (u64)buffer);
+    if (lookup.found_match){
+        u64 val = 0;
+        table_read(&set->id_to_node, (u64)buffer, &val);
+        Buffer_Modified_Node *node = (Buffer_Modified_Node*)IntAsPtr(val);
+        zdll_remove(set->first, set->last, node);
+        table_erase(&set->id_to_node, lookup);
+        sll_stack_push(set->free, node);
+    }
+}
+
+function void
+buffer_modified_set_clear(void){
+    Buffer_Modified_Set *set = &global_buffer_modified_set;
+    
+    table_clear(&set->id_to_node);
+    if (set->last != 0){
+        set->last->next = set->free;
+        set->free = set->first;
+        set->first = 0;
+        set->last = 0;
+    }
+}
+
 // BOTTOM
 

custom/4coder_default_framework.h

@@ -78,6 +78,22 @@ do_buffer_kill_user_check(Application_Links *app, Buffer_ID buffer, View_ID view
 function b32
 do_4coder_close_user_check(Application_Links *app, View_ID view);
 
+////////////////////////////////
+
+struct Buffer_Modified_Node{
+    Buffer_Modified_Node *next;
+    Buffer_Modified_Node *prev;
+    Buffer_ID buffer;
+};
+
+struct Buffer_Modified_Set{
+    Arena arena;
+    Buffer_Modified_Node *free;
+    Buffer_Modified_Node *first;
+    Buffer_Modified_Node *last;
+    Table_u64_u64 id_to_node;
+};
+
 #endif
 
 // BOTTOM

custom/4coder_default_framework_variables.cpp

@@ -46,7 +46,6 @@ global Managed_ID view_word_complete_menu = 0;
 global Managed_ID buffer_map_id = 0;
 global Managed_ID buffer_eol_setting = 0;
 global Managed_ID buffer_lex_task = 0;
-global Managed_ID buffer_parse_task = 0;
 
 global Managed_ID sticky_jump_marker_handle = 0;
 
@@ -86,5 +85,7 @@ enum{
     default_maps_count,
 };
 
+global Buffer_Modified_Set global_buffer_modified_set = {};
+
 // BOTTOM
 

custom/4coder_default_hooks.cpp

@@ -235,6 +235,45 @@ MODIFY_COLOR_TABLE_SIG(default_modify_color_table){
 }
 #endif
 
+function void
+default_tick(Application_Links *app, Frame_Info frame_info){
+    Scratch_Block scratch(app);
+    
+    for (Buffer_Modified_Node *node = global_buffer_modified_set.first;
+         node != 0;
+         node = node->next){
+        Temp_Memory_Block temp(scratch);
+        Buffer_ID buffer_id = node->buffer;
+        
+        Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
+        
+        String_Const_u8 contents = push_whole_buffer(app, scratch, buffer_id);
+        Token_Array *tokens_ptr = scope_attachment(app, scope, attachment_tokens, Token_Array);
+        if (tokens_ptr == 0){
+            continue;
+        }
+        if (tokens_ptr->count == 0){
+            continue;
+        }
+        Token_Array tokens = *tokens_ptr;
+        
+        Arena arena = make_arena_system(KB(16));
+        Code_Index_File *index = push_array_zero(&arena, Code_Index_File, 1);
+        index->buffer = buffer_id;
+        
+        Generic_Parse_State state = {};
+        generic_parse_init(app, &arena, contents, &tokens, &state);
+        generic_parse_full_input_breaks(index, &state, max_i32);
+        
+        code_index_lock();
+        code_index_set_file(buffer_id, arena, index);
+        code_index_unlock();
+        buffer_clear_layout_cache(app, buffer_id);
+    }
+    
+    buffer_modified_set_clear();
+}
+
 function Rect_f32
 default_buffer_region(Application_Links *app, View_ID view_id, Rect_f32 region){
     Buffer_ID buffer = view_get_buffer(app, view_id, Access_Always);
@@ -661,6 +700,71 @@ parse_async__inner(Async_Context *actx, Buffer_ID buffer_id,
     }
 }
 
+function void
+do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){
+    Application_Links *app = actx->app;
+    ProfileScope(app, "async lex");
+    Thread_Context *tctx = get_thread_context(app);
+    Scratch_Block scratch(tctx);
+    
+    String_Const_u8 contents = {};
+    {
+        ProfileBlock(app, "async lex contents (before mutex)");
+        system_acquire_global_frame_mutex(tctx);
+        ProfileBlock(app, "async lex contents (after mutex)");
+        contents = push_whole_buffer(app, scratch, buffer_id);
+        system_release_global_frame_mutex(tctx);
+    }
+    
+    i32 limit_factor = 10000;
+    
+    Token_List list = {};
+    b32 canceled = false;
+    
+        Lex_State_Cpp state = {};
+        lex_full_input_cpp_init(&state, contents);
+        for (;;){
+            ProfileBlock(app, "async lex block");
+            if (lex_full_input_cpp_breaks(scratch, &list, &state, limit_factor)){
+                break;
+            }
+            if (async_check_canceled(actx)){
+                canceled = true;
+                break;
+            }
+        }
+    
+    if (!canceled){
+        ProfileBlock(app, "async lex save results (before mutex)");
+        system_acquire_global_frame_mutex(tctx);
+        ProfileBlock(app, "async lex save results (after mutex)");
+        Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
+        if (scope != 0){
+            Base_Allocator *allocator = managed_scope_allocator(app, scope);
+            Token_Array *tokens_ptr = scope_attachment(app, scope, attachment_tokens,
+                                                       Token_Array);
+            base_free(allocator, tokens_ptr->tokens);
+            Token_Array tokens = {};
+            tokens.tokens = base_array(allocator, Token, list.total_count);
+            tokens.count = list.total_count;
+            tokens.max = list.total_count;
+            token_fill_memory_from_list(tokens.tokens, &list);
+            block_copy_struct(tokens_ptr, &tokens);
+        }
+        buffer_mark_as_modified(buffer_id);
+        system_release_global_frame_mutex(tctx);
+    }
+}
+
+function void
+do_full_lex_async(Async_Context *actx, Data data){
+    if (data.size == sizeof(Buffer_ID)){
+        Buffer_ID buffer = *(Buffer_ID*)data.data;
+        do_full_lex_async__inner(actx, buffer);
+    }
+}
+
+#if 0
 function void
 do_full_lex_and_parse_async__inner(Async_Context *actx, Buffer_ID buffer_id){
     Application_Links *app = actx->app;
@@ -739,7 +843,9 @@ do_full_lex_and_parse_async(Async_Context *actx, Data data){
         do_full_lex_and_parse_async__inner(actx, buffer);
     }
 }
+#endif
 
+#if 0
 function void
 do_parse_async__inner(Async_Context *actx, Buffer_ID buffer_id){
     Application_Links *app = actx->app;
@@ -782,19 +888,17 @@ do_parse_async(Async_Context *actx, Data data){
         do_parse_async__inner(actx, buffer);
     }
 }
+#endif
 
 BUFFER_HOOK_SIG(default_begin_buffer){
     ProfileScope(app, "begin buffer");
     
-    b32 treat_as_code = false;
-    
-    String_Const_u8_Array extensions = global_config.code_exts;
-    
     Scratch_Block scratch(app);
     
+    b32 treat_as_code = false;
     String_Const_u8 file_name = push_buffer_file_name(app, scratch, buffer_id);
-    
     if (file_name.size > 0){
+    String_Const_u8_Array extensions = global_config.code_exts;
         String_Const_u8 ext = string_file_extension(file_name);
         for (i32 i = 0; i < extensions.count; ++i){
             if (string_match(ext, extensions.strings[i])){
@@ -870,7 +974,6 @@ BUFFER_HOOK_SIG(default_begin_buffer){
         buffer_map_id = managed_id_declare(app, SCu8("DEFAULT.buffer_map_id"));
         buffer_eol_setting = managed_id_declare(app, SCu8("DEFAULT.buffer_eol_setting"));
         buffer_lex_task = managed_id_declare(app, SCu8("DEFAULT.buffer_lex_task"));
-        buffer_parse_task = managed_id_declare(app, SCu8("DEFAULT.buffer_parse_task"));
     }
     
     Command_Map_ID map_id = (treat_as_code)?(default_code_map):(mapid_file);
@@ -880,8 +983,7 @@ BUFFER_HOOK_SIG(default_begin_buffer){
     *map_id_ptr = map_id;
     
     Line_Ending_Kind setting = guess_line_ending_kind_from_buffer_contents(app, buffer_id);
-    Line_Ending_Kind *eol_setting = scope_attachment(app, scope, buffer_eol_setting,
+    Line_Ending_Kind *eol_setting = scope_attachment(app, scope, buffer_eol_setting, Line_Ending_Kind);
-                                                     Line_Ending_Kind);
     *eol_setting = setting;
     
     // NOTE(allen): Decide buffer settings
@@ -902,7 +1004,7 @@ BUFFER_HOOK_SIG(default_begin_buffer){
     if (use_lexer){
         ProfileBlock(app, "begin buffer kick off lexer");
         Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task);
-        *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_and_parse_async, make_data_struct(&buffer_id));
+        *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_async, make_data_struct(&buffer_id));
     }
     
     if (wrap_lines){
@@ -992,20 +1094,16 @@ BUFFER_EDIT_RANGE_SIG(default_buffer_edit_range){
     
     Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
     Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task);
-    Async_Task *parse_task_ptr = scope_attachment(app, scope, buffer_parse_task, Async_Task);
     
     Base_Allocator *allocator = managed_scope_allocator(app, scope);
     b32 do_full_relex = false;
     
     if (async_task_is_running_or_pending(&global_async_system, *lex_task_ptr)){
         async_task_cancel(&global_async_system, *lex_task_ptr);
+        buffer_unmark_as_modified(buffer_id);
         do_full_relex = true;
         *lex_task_ptr = 0;
     }
-    if (async_task_is_running_or_pending(&global_async_system, *parse_task_ptr)){
-        async_task_cancel(&global_async_system, *parse_task_ptr);
-        *parse_task_ptr = 0;
-    }
     
     Token_Array *ptr = scope_attachment(app, scope, attachment_tokens, Token_Array);
     if (ptr != 0 && ptr->tokens != 0){
@@ -1022,20 +1120,15 @@ BUFFER_EDIT_RANGE_SIG(default_buffer_edit_range){
             Token *token_first = ptr->tokens + token_index_first;
             Token *token_resync = ptr->tokens + token_index_resync_guess;
             
-            Range_i64 relex_range =
+            Range_i64 relex_range = Ii64(token_first->pos, token_resync->pos + token_resync->size + text_shift);
-                Ii64(token_first->pos,
+            String_Const_u8 partial_text = push_buffer_range(app, scratch, buffer_id, relex_range);
-                     token_resync->pos + token_resync->size + text_shift);
-            String_Const_u8 partial_text = push_buffer_range(app, scratch,
-                                                             buffer_id,
-                                                             relex_range);
             
             Token_List relex_list = lex_full_input_cpp(scratch, partial_text);
             if (relex_range.one_past_last < buffer_get_size(app, buffer_id)){
                 token_drop_eof(&relex_list);
             }
             
-            Token_Relex relex = token_relex(relex_list, relex_range.first - text_shift,
+            Token_Relex relex = token_relex(relex_list, relex_range.first - text_shift, ptr->tokens, token_index_first, token_index_resync_guess);
-                                            ptr->tokens, token_index_first, token_index_resync_guess);
             
             ProfileCloseNow(profile_attempt_resync);
             
@@ -1074,14 +1167,13 @@ BUFFER_EDIT_RANGE_SIG(default_buffer_edit_range){
                 ptr->count = new_tokens_count;
                 ptr->max = new_tokens_count;
                 
-                *parse_task_ptr = async_task_no_dep(&global_async_system, do_parse_async,
+                buffer_mark_as_modified(buffer_id);
-                                                    make_data_struct(&buffer_id));
             }
         }
     }
     
     if (do_full_relex){
-        *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_and_parse_async,
+        *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_async,
                                           make_data_struct(&buffer_id));
     }
     
@@ -1095,10 +1187,7 @@ BUFFER_HOOK_SIG(default_end_buffer){
     if (lex_task_ptr != 0){
         async_task_cancel(&global_async_system, *lex_task_ptr);
     }
-    Async_Task *lex_parse_ptr = scope_attachment(app, scope, buffer_parse_task, Async_Task);
+        buffer_unmark_as_modified(buffer_id);
-    if (lex_parse_ptr != 0){
-        async_task_cancel(&global_async_system, *lex_parse_ptr);
-    }
     code_index_lock();
     code_index_erase_file(buffer_id);
     code_index_unlock();
@@ -1111,6 +1200,7 @@ set_all_default_hooks(Application_Links *app){
     set_custom_hook(app, HookID_BufferViewerUpdate, default_view_adjust);
     
     set_custom_hook(app, HookID_ViewEventHandler, default_view_input_handler);
+    set_custom_hook(app, HookID_Tick, default_tick);
     set_custom_hook(app, HookID_RenderCaller, default_render_caller);
 #if 0
     set_custom_hook(app, HookID_DeltaRule, original_delta);

custom/4coder_types.h

@@ -555,6 +555,7 @@ struct User_Input{
 
 typedef i32 Hook_ID;
 enum{
+    HookID_Tick,
     HookID_RenderCaller,
     HookID_DeltaRule,
     HookID_BufferViewerUpdate,
@@ -599,8 +600,9 @@ typedef Rect_f32 Buffer_Region_Function(Application_Links *app, View_ID view_id,
 typedef void New_Clipboard_Contents_Function(Application_Links *app, String_Const_u8 contents);
 #define NEW_CLIPBOARD_CONTENTS_SIG(name) void name(Application_Links *app, String_Const_u8 contents)
 
+typedef void Tick_Function(Application_Links *app, Frame_Info frame_info);
+
 typedef void Render_Caller_Function(Application_Links *app, Frame_Info frame_info, View_ID view);
-#define RENDER_CALLER_SIG(name) void name(Application_Links *app, Frame_Info frame_info, View_ID view)
 
 typedef u32 Layout_Item_Flag;
 enum{