Initial code indexer up and running; more time with global frame mutex free; bugs with layout function selection fixed

4ed_api_implementation.cpp

@@ -365,7 +365,7 @@ buffer_line_y_difference(Application_Links *app, Buffer_ID buffer_id,
     if (api_check_buffer(file)){
         Face *face = font_set_face_from_id(&models->font_set, face_id);
         if (face != 0){
-            Layout_Function *layout_func = models->layout_func;
+            Layout_Function *layout_func = file_get_layout_func(file);
             result = file_line_y_difference(app->tctx, models, file,
                                             layout_func, width, face,
                                             line_a, line_b);
@@ -384,7 +384,7 @@ buffer_line_shift_y(Application_Links *app, Buffer_ID buffer_id,
     if (api_check_buffer(file)){
         Face *face = font_set_face_from_id(&models->font_set, face_id);
         if (face != 0){
-            Layout_Function *layout_func = models->layout_func;
+            Layout_Function *layout_func = file_get_layout_func(file);
             result = file_line_shift_y(app->tctx, models, file,
                                        layout_func, width, face,
                                        line, y_shift);
@@ -403,7 +403,7 @@ buffer_pos_at_relative_xy(Application_Links *app, Buffer_ID buffer_id,
     if (api_check_buffer(file)){
         Face *face = font_set_face_from_id(&models->font_set, face_id);
         if (face != 0){
-            Layout_Function *layout_func = models->layout_func;
+            Layout_Function *layout_func = file_get_layout_func(file);
             result = file_pos_at_relative_xy(app->tctx, models, file,
                                              layout_func, width, face,
                                              base_line, relative_xy);
@@ -422,7 +422,7 @@ buffer_relative_box_of_pos(Application_Links *app, Buffer_ID buffer_id,
     if (api_check_buffer(file)){
         Face *face = font_set_face_from_id(&models->font_set, face_id);
         if (face != 0){
-            Layout_Function *layout_func = models->layout_func;
+            Layout_Function *layout_func = file_get_layout_func(file);
             result = file_relative_box_of_pos(app->tctx, models, file,
                                               layout_func, width, face,
                                               base_line, pos);
@@ -441,7 +441,7 @@ buffer_relative_character_from_pos(Application_Links *app, Buffer_ID buffer_id,
     if (api_check_buffer(file)){
         Face *face = font_set_face_from_id(&models->font_set, face_id);
         if (face != 0){
-            Layout_Function *layout_func = models->layout_func;
+            Layout_Function *layout_func = file_get_layout_func(file);
             result = file_relative_character_from_pos(app->tctx, models, file,
                                                       layout_func, width, face,
                                                       base_line, pos);
@@ -459,7 +459,7 @@ buffer_pos_from_relative_character(Application_Links *app,  Buffer_ID buffer_id,
     if (api_check_buffer(file)){
         Face *face = font_set_face_from_id(&models->font_set, face_id);
         if (face != 0){
-            Layout_Function *layout_func = models->layout_func;
+            Layout_Function *layout_func = file_get_layout_func(file);
             result = file_pos_from_relative_character(app->tctx, models, file,
                                                       layout_func, width, face,
                                                       base_line, relative_character);
@@ -643,14 +643,26 @@ buffer_set_dirty_state(Application_Links *app, Buffer_ID buffer_id, Dirty_State
 }
 
 api(custom) function b32
-buffer_set_layout(Application_Links *app, Buffer_ID buffer_id,
+buffer_set_layout(Application_Links *app, Buffer_ID buffer_id, Layout_Function *layout_func){
-                  Layout_Function *layout_func){
     Models *models = (Models*)app->cmd_context;
     Editing_File *file = imp_get_file(models, buffer_id);
     b32 result = false;
     if (api_check_buffer(file)){
         result = true;
         file->settings.layout_func = layout_func;
+        file_clear_layout_cache(file);
+    }
+    return(result);
+}
+
+api(custom) function b32
+file_clear_layout_cache(Application_Links *app, Buffer_ID buffer_id){
+    Models *models = (Models*)app->cmd_context;
+    Editing_File *file = imp_get_file(models, buffer_id);
+    b32 result = false;
+    if (api_check_buffer(file)){
+        result = true;
+        file_clear_layout_cache(file);
     }
     return(result);
 }
@@ -2774,7 +2786,7 @@ text_layout_create(Application_Links *app, Buffer_ID buffer_id, Rect_f32 rect, B
         
         Gap_Buffer *buffer = &file->state.buffer;
         
-        Layout_Function *layout_func = models->layout_func;
+        Layout_Function *layout_func = file_get_layout_func(file);
         
         Vec2_f32 dim = rect_dim(rect);
         

custom/4coder_async_tasks.cpp

@@ -43,7 +43,6 @@ async_push_node__inner(Async_System *async_system, Async_Task_Function_Type *fun
     node->data.data = (u8*)heap_allocate(&async_system->node_heap, data.size);
     block_copy(node->data.data, data.data, data.size);
     node->data.size = data.size;
-    node->following_task = 0;
     dll_insert_back(&async_system->task_sent, &node->node);
     async_system->task_count += 1;
     system_condition_variable_signal(async_system->cv);
@@ -107,17 +106,7 @@ async_task_thread(void *thread_ptr){
         thread->task = 0;
         thread->cancel_signal = false;
         thread->join_signal = false;
-        {
-            Async_Node *next = node->following_task;
         async_free_node(async_system, node);
-            for (node = next; node != 0; node = next){
-                next = node->following_task;
-                Assert(&node->node != &async_system->task_sent);
-                dll_remove(&node->node);
-                async_system->task_count -= 1;
-                async_free_node(async_system, node);
-            }
-        }
         system_mutex_release(async_system->mutex);
     }
 }
@@ -173,22 +162,6 @@ async_task_no_dep(Async_System *async_system, Async_Task_Function_Type *func,
     return(result);
 }
 
-function Async_Task
-async_task_single_dep(Async_System *async_system, Async_Task_Function_Type *func,
-                      Data data, Async_Task dependency){
-    system_mutex_acquire(async_system->mutex);
-    Async_Node *dep_node = async_get_pending_node(async_system, dependency);
-    if (dep_node == 0){
-        dep_node = async_get_running_node(async_system, dependency);
-    }
-    Async_Node *node = async_push_node__inner(async_system, func, data);
-    if (dep_node != 0){
-        dep_node->following_task = node;
-    }
-    system_mutex_release(async_system->mutex);
-    return(node->task);
-}
-
 function b32
 async_task_is_pending(Async_System *async_system, Async_Task task){
     system_mutex_acquire(async_system->mutex);
@@ -216,6 +189,7 @@ async_task_is_running_or_pending(Async_System *async_system, Async_Task task){
     return(node != 0);
 }
 
+// TODO(allen): ensure that the job is canceled before this returns.
 function void
 async_task_cancel(Async_System *async_system, Async_Task task){
     system_mutex_acquire(async_system->mutex);

custom/4coder_async_tasks.h

@@ -28,7 +28,6 @@ struct Async_Node{
     Async_Thread *thread;
     Async_Task_Function_Type *func;
     Data data;
-    Async_Node *following_task;
 };
 
 struct Async_System{

custom/4coder_code_index.cpp

@@ -4,6 +4,40 @@
 
 // TOP
 
+global Code_Index global_code_index = {};
+
+function void
+code_index_init(void){
+    global_code_index.mutex = system_mutex_make();
+    global_code_index.node_arena = make_arena_system(KB(4));
+    global_code_index.buffer_to_index_file =
+        make_table_u64_u64(global_code_index.node_arena.base_allocator, 500);
+}
+
+function Code_Index_File_Storage*
+code_index__alloc_storage(void){
+    Code_Index_File_Storage *result = global_code_index.free_storage;
+    if (result == 0){
+        result = push_array_zero(&global_code_index.node_arena,
+                                 Code_Index_File_Storage, 1);
+    }
+    else{
+        sll_stack_pop(global_code_index.free_storage);
+    }
+    zdll_push_back(global_code_index.storage_first, global_code_index.storage_last,
+                   result);
+    global_code_index.storage_count += 1;
+    return(result);
+}
+
+function void
+code_index__free_storage(Code_Index_File_Storage *storage){
+    zdll_remove(global_code_index.storage_first, global_code_index.storage_last,
+                storage);
+    global_code_index.storage_count -= 1;
+    sll_stack_push(global_code_index.free_storage, storage);
+}
+
 function void
 code_index_push_nest(Code_Index_Nest_List *list, Code_Index_Nest *nest){
     sll_queue_push(list->first, list->last, nest);
@@ -26,12 +60,133 @@ code_index_nest_ptr_array_from_list(Arena *arena,  Code_Index_Nest_List *list){
 }
 
 function void
-code_index_set_file(Application_Links *app, Buffer_ID buffer, Arena arena, Code_Index_File *index){
+code_index_lock(void){
-    NotImplemented;
+    system_mutex_acquire(global_code_index.mutex);
+}
+
+function void
+code_index_unlock(void){
+    system_mutex_release(global_code_index.mutex);
+}
+
+function void
+code_index_set_file(Buffer_ID buffer, Arena arena, Code_Index_File *index){
+    Code_Index_File_Storage *storage = 0;
+    
+    Table_Lookup lookup = table_lookup(&global_code_index.buffer_to_index_file,
+                                       buffer);
+    if (lookup.found_match){
+        u64 val = 0;
+        table_read(&global_code_index.buffer_to_index_file, lookup, &val);
+        storage = (Code_Index_File_Storage*)IntAsPtr(val);
+        linalloc_clear(&storage->arena);
+    }
+    else{
+        storage = code_index__alloc_storage();
+        table_insert(&global_code_index.buffer_to_index_file, buffer,
+                     (u64)PtrAsInt(storage));
+    }
+    storage->arena = arena;
+    storage->file = index;
+}
+
+function void
+code_index_erase_file(Buffer_ID buffer){
+    Table_Lookup lookup = table_lookup(&global_code_index.buffer_to_index_file,
+                                       buffer);
+    if (lookup.found_match){
+        u64 val = 0;
+        table_read(&global_code_index.buffer_to_index_file, lookup, &val);
+        Code_Index_File_Storage *storage = (Code_Index_File_Storage*)IntAsPtr(val);
+        linalloc_clear(&storage->arena);
+        table_erase(&global_code_index.buffer_to_index_file, lookup);
+        code_index__free_storage(storage);
+    }
+}
+
+function Code_Index_File*
+code_index_get_file(Buffer_ID buffer){
+    Code_Index_File *result = 0;
+    Table_Lookup lookup = table_lookup(&global_code_index.buffer_to_index_file,
+                                       buffer);
+    if (lookup.found_match){
+        u64 val = 0;
+        table_read(&global_code_index.buffer_to_index_file, lookup, &val);
+        result = (Code_Index_File*)IntAsPtr(val);
+    }
+    return(result);
+}
+
+function Code_Index_Nest*
+code_index_get_nest(Code_Index_Nest_Ptr_Array *array, i64 pos){
+    Code_Index_Nest *result = 0;
+    i32 count = array->count;
+    Code_Index_Nest **nest_ptrs = array->ptrs;
+    for (i32 i = 0; i < count; i += 1){
+        Code_Index_Nest *nest = nest_ptrs[i];
+        if (nest->open.max <= pos && pos < nest->close.min){
+            Code_Index_Nest *sub_nest =
+                code_index_get_nest(&nest->nest_array, pos);
+            if (sub_nest != 0){
+                result = sub_nest;
+            }
+            else{
+                result = nest;
+            }
+            break;
+        }
+    }
+    return(result);
+}
+
+function Code_Index_Nest*
+code_index_get_nest(Code_Index_Nest *nest, i64 pos){
+    return(code_index_get_nest(&nest->nest_array, pos));
+}
+
+function Code_Index_Nest*
+code_index_get_nest(Code_Index_File *file, i64 pos){
+    return(code_index_get_nest(&file->nest_array, pos));
 }
 
 ////////////////////////////////
 
+function void
+generic_parse_inc(Generic_Parse_State *state){
+    if (!token_it_inc_all(&state->it)){
+        state->finished = true;
+    }
+}
+
+function void
+generic_parse_skip_soft_tokens(Code_Index_File *index, Generic_Parse_State *state){
+    Token *token = token_it_read(&state->it);
+    for (;token != 0 && !state->finished;){
+        if (token->kind == TokenBaseKind_Comment){
+            state->handle_comment(state->app, state->arena, index, token, state->contents);
+            token_it_inc_non_whitespace(&state->it);
+            token = token_it_read(&state->it);
+        }
+        else if (token->kind == TokenBaseKind_Whitespace){
+            Range_i64 range = Ii64(token);
+            u8 *ptr = state->contents.str + range.one_past_last - 1;
+            for (i64 i = range.one_past_last - 1;
+                 i >= range.first;
+                 i -= 1, ptr -= 1){
+                if (*ptr == '\n'){
+                    state->prev_line_start = ptr + 1;
+                    break;
+                }
+            }
+        }
+        else{
+            break;
+        }
+        generic_parse_inc(state);
+        token = token_it_read(&state->it);
+    }
+}
+
 function void
 generic_parse_init(Application_Links *app, Arena *arena, String_Const_u8 contents, Token_Array *tokens,
                    Generic_Parse_Comment_Function *handle_comment, Generic_Parse_State *state){
@@ -40,93 +195,121 @@ generic_parse_init(Application_Links *app, Arena *arena, String_Const_u8 content
     state->contents = contents;
     state->it = token_iterator(0, tokens);
     state->handle_comment = handle_comment;
-    
+    state->prev_line_start = contents.str;
-    Token *token = token_it_read(&state->it);
-    if (token != 0 && token->kind == TokenBaseKind_Whitespace){
-        token_it_inc_non_whitespace(&state->it);
-    }
-}
-
-function Token*
-generic_parse_read_token(Code_Index_File *index, Generic_Parse_State *state){
-    Token *token = token_it_read(&state->it);
-    for (;token != 0 && token->kind == TokenBaseKind_Comment;){
-        state->handle_comment(state->app, state->arena, index, token, state->contents);
-        token_it_inc_non_whitespace(&state->it);
-        token = token_it_read(&state->it);
-    }
-    return(token);
 }
 
 function Code_Index_Nest*
-generic_parse_parenthical(Code_Index_File *index, Generic_Parse_State *state);
+generic_parse_parenthical(Code_Index_File *index, Generic_Parse_State *state,
+                          i64 indentation);
 
 function Code_Index_Nest*
-generic_parse_scope(Code_Index_File *index, Generic_Parse_State *state){
+generic_parse_scope(Code_Index_File *index, Generic_Parse_State *state,
+                    i64 indentation){
     Token *token = token_it_read(&state->it);
     Code_Index_Nest *result = push_array_zero(state->arena, Code_Index_Nest, 1);
     result->kind = CodeIndexNest_Scope;
     result->open = Ii64(token);
+    result->file = index;
     
+    result->interior_indentation = indentation + 4;
+    result->close_indentation = indentation;
+    indentation = result->interior_indentation;
+    
+    generic_parse_inc(state);
     for (;;){
-        token = generic_parse_read_token(index, state);
+        generic_parse_skip_soft_tokens(index, state);
-        if (token == 0){
+        token = token_it_read(&state->it);
+        if (token == 0 || state->finished){
             break;
         }
         
         if (token->kind == TokenBaseKind_ScopeOpen){
-            Code_Index_Nest *nest = generic_parse_scope(index, state);
+            Code_Index_Nest *nest = generic_parse_scope(index, state, indentation);
+            nest->parent = result;
             code_index_push_nest(&result->nest_list, nest);
         }
         else if (token->kind == TokenBaseKind_ParentheticalOpen){
-            Code_Index_Nest *nest = generic_parse_parenthical(index, state);
+            Code_Index_Nest *nest = generic_parse_parenthical(index, state,
+                                                              indentation);
+            nest->parent = result;
             code_index_push_nest(&result->nest_list, nest);
         }
         else if (token->kind == TokenBaseKind_ScopeClose){
+            result->is_closed = true;
             result->close = Ii64(token);
+            generic_parse_inc(state);
             break;
         }
         else{
-            token_it_inc_non_whitespace(&state->it);
+            generic_parse_inc(state);
         }
     }
     
-    result->nest_array = code_index_nest_ptr_array_from_list(state->arena, &result->nest_list);
+    result->nest_array =
+        code_index_nest_ptr_array_from_list(state->arena, &result->nest_list);
     
     return(result);
 }
 
 function Code_Index_Nest*
-generic_parse_parenthical(Code_Index_File *index, Generic_Parse_State *state){
+generic_parse_parenthical(Code_Index_File *index, Generic_Parse_State *state,
+                          i64 indentation){
     Token *token = token_it_read(&state->it);
     Code_Index_Nest *result = push_array_zero(state->arena, Code_Index_Nest, 1);
     result->kind = CodeIndexNest_Paren;
     result->open = Ii64(token);
+    result->file = index;
     
+    i64 manifested_characters_on_line = 0;
+    {
+        u8 *ptr = state->prev_line_start;
+        u8 *end_ptr = state->contents.str + token->pos;
+        // NOTE(allen): Initial whitespace
+        for (;ptr < end_ptr; ptr += 1){
+            if (!character_is_whitespace(*ptr)){
+                break;
+            }
+        }
+        // NOTE(allen): Manifested characters
+        manifested_characters_on_line = (i64)(end_ptr - ptr) + token->size;
+    }
+    
+    indentation += manifested_characters_on_line;
+    result->interior_indentation = indentation;
+    result->close_indentation = indentation;
+    
+    generic_parse_inc(state);
     for (;;){
-        token = generic_parse_read_token(index, state);
+        generic_parse_skip_soft_tokens(index, state);
-        if (token == 0){
+        token = token_it_read(&state->it);
+        if (token == 0 || state->finished){
             break;
         }
         
         if (token->kind == TokenBaseKind_ScopeOpen){
-            Code_Index_Nest *nest = generic_parse_scope(index, state);
+            Code_Index_Nest *nest = generic_parse_scope(index, state, indentation);
+            nest->parent = result;
             code_index_push_nest(&result->nest_list, nest);
         }
         else if (token->kind == TokenBaseKind_ParentheticalOpen){
-            Code_Index_Nest *nest = generic_parse_parenthical(index, state);
+            Code_Index_Nest *nest = generic_parse_parenthical(index, state,
+                                                              indentation);
+            nest->parent = result;
             code_index_push_nest(&result->nest_list, nest);
         }
         else if (token->kind == TokenBaseKind_ParentheticalClose){
+            result->is_closed = true;
             result->close = Ii64(token);
+            generic_parse_inc(state);
             break;
         }
         else{
-            token_it_inc_non_whitespace(&state->it);
+            generic_parse_inc(state);
         }
     }
     
-    result->nest_array = code_index_nest_ptr_array_from_list(state->arena, &result->nest_list);
+    result->nest_array =
+        code_index_nest_ptr_array_from_list(state->arena, &result->nest_list);
     
     return(result);
 }
@@ -135,26 +318,28 @@ function b32
 generic_parse_full_input_breaks(Code_Index_File *index, Generic_Parse_State *state, i32 limit){
     b32 result = false;
     
+    i64 indentation = 0;
     i64 first_index = token_it_index(&state->it);
     i64 one_past_last_index = first_index + limit;
     for (;;){
-        Token *token = generic_parse_read_token(index, state);
+        generic_parse_skip_soft_tokens(index, state);
+        Token *token = token_it_read(&state->it);
         
-        if (token == 0){
+        if (token == 0 || state->finished){
             result = true;
             break;
         }
         
         if (token->kind == TokenBaseKind_ScopeOpen){
-            Code_Index_Nest *nest = generic_parse_scope(index, state);
+            Code_Index_Nest *nest = generic_parse_scope(index, state, indentation);
             code_index_push_nest(&index->nest_list, nest);
         }
         else if (token->kind == TokenBaseKind_ParentheticalOpen){
-            Code_Index_Nest *nest = generic_parse_parenthical(index, state);
+            Code_Index_Nest *nest = generic_parse_parenthical(index, state, indentation);
             code_index_push_nest(&index->nest_list, nest);
         }
         else{
-            token_it_inc_non_whitespace(&state->it);
+            generic_parse_inc(state);
         }
         
         i64 index = token_it_index(&state->it);
@@ -167,6 +352,11 @@ generic_parse_full_input_breaks(Code_Index_File *index, Generic_Parse_State *sta
         }
     }
     
+    if (result){
+        index->nest_array =
+            code_index_nest_ptr_array_from_list(state->arena, &index->nest_list);
+    }
+    
     return(result);
 }
 

custom/4coder_code_index.h

@@ -32,6 +32,12 @@ struct Code_Index_Nest{
     Range_i64 open;
     Range_i64 close;
     
+    i64 interior_indentation;
+    i64 close_indentation;
+    
+    struct Code_Index_File *file;
+    Code_Index_Nest *parent;
+    
     Code_Index_Nest_List nest_list;
     Code_Index_Nest_Ptr_Array nest_array;
 };
@@ -39,6 +45,24 @@ struct Code_Index_Nest{
 struct Code_Index_File{
     Code_Index_Nest_List nest_list;
     Code_Index_Nest_Ptr_Array nest_array;
+    Buffer_ID buffer;
+};
+
+struct Code_Index_File_Storage{
+    Code_Index_File_Storage *next;
+    Code_Index_File_Storage *prev;
+    Arena arena;
+    Code_Index_File *file;
+};
+
+struct Code_Index{
+    System_Mutex mutex;
+    Arena node_arena;
+    Table_u64_u64 buffer_to_index_file;
+    Code_Index_File_Storage *free_storage;
+    Code_Index_File_Storage *storage_first;
+    Code_Index_File_Storage *storage_last;
+    i32 storage_count;
 };
 
 ////////////////////////////////
@@ -52,6 +76,8 @@ struct Generic_Parse_State{
     String_Const_u8 contents;
     Token_Iterator_Array it;
     Generic_Parse_Comment_Function *handle_comment;
+    u8 *prev_line_start;
+    b32 finished;
 };
 
 #endif

custom/4coder_default_bindings.cpp

@@ -19,6 +19,7 @@ custom_layer_init(Application_Links *app){
     mapping_init(tctx, &framework_mapping);
     setup_default_mapping(&framework_mapping);
     async_task_handler_init(app, &global_async_system);
+    code_index_init();
     
     Profile_Global_List *list = get_core_profile_list(app);
     ProfileThreadName(tctx, list, string_u8_litexpr("main"));

custom/4coder_default_hooks.cpp

@@ -582,7 +582,7 @@ BUFFER_NAME_RESOLVER_SIG(default_buffer_name_resolution){
 }
 
 function void
-do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){
+do_full_lex_and_parse_async__inner(Async_Context *actx, Buffer_ID buffer_id){
     Application_Links *app = actx->app;
     ProfileScope(app, "async lex");
     Thread_Context *tctx = get_thread_context(app);
@@ -593,18 +593,33 @@ do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){
         ProfileBlock(app, "async lex contents (before mutex)");
         system_acquire_global_frame_mutex(tctx);
         ProfileBlock(app, "async lex contents (after mutex)");
+        
+        Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
+        if (scope != 0){
+            Base_Allocator *allocator = managed_scope_allocator(app, scope);
+            Token_Array *tokens_ptr = scope_attachment(app, scope, attachment_tokens,
+                                                       Token_Array);
+            base_free(allocator, tokens_ptr->tokens);
+        }
+        code_index_lock();
+        code_index_erase_file(buffer_id);
+        code_index_unlock();
+        
         contents = push_whole_buffer(app, scratch, buffer_id);
         system_release_global_frame_mutex(tctx);
     }
     
-    Lex_State_Cpp state = {};
+    i32 limit_factor = 10000;
-    lex_full_input_cpp_init(&state, contents);
     
     Token_List list = {};
     b32 canceled = false;
+    
+    {
+        Lex_State_Cpp state = {};
+        lex_full_input_cpp_init(&state, contents);
         for (;;){
             ProfileBlock(app, "async lex block");
-        if (lex_full_input_cpp_breaks(scratch, &list, &state, 10000)){
+            if (lex_full_input_cpp_breaks(scratch, &list, &state, limit_factor)){
                 break;
             }
             if (async_check_canceled(actx)){
@@ -612,7 +627,9 @@ do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){
                 break;
             }
         }
+    }
     
+    Token_Array tokens = {};
     if (!canceled){
         ProfileBlock(app, "async lex save results (before mutex)");
         system_acquire_global_frame_mutex(tctx);
@@ -623,8 +640,6 @@ do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){
             Token_Array *tokens_ptr = scope_attachment(app, scope, attachment_tokens,
                                                        Token_Array);
             base_free(allocator, tokens_ptr->tokens);
-            
-            Token_Array tokens = {};
             tokens.tokens = base_array(allocator, Token, list.total_count);
             tokens.count = list.total_count;
             tokens.max = list.total_count;
@@ -633,48 +648,18 @@ do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){
         }
         system_release_global_frame_mutex(tctx);
     }
-}
-
-function void
-do_full_lex_async(Async_Context *actx, Data data){
-    if (data.size == sizeof(Buffer_ID)){
-        Buffer_ID buffer = *(Buffer_ID*)data.data;
-        do_full_lex_async__inner(actx, buffer);
-    }
-}
-
-function void
-do_full_parse_async__inner(Async_Context *actx, Buffer_ID buffer){
-    Application_Links *app = actx->app;
-    ProfileScope(app, "async parse");
-    
-    Thread_Context *tctx = get_thread_context(app);
-    Scratch_Block scratch(tctx);
-    
-    String_Const_u8 contents = {};
-    Token_Array tokens = {};
-    {
-        ProfileBlock(app, "async parse contents (before mutex)");
-        system_acquire_global_frame_mutex(tctx);
-        ProfileBlock(app, "async parse contents (after mutex)");
-        contents = push_whole_buffer(app, scratch, buffer);
-        Managed_Scope scope = buffer_get_managed_scope(app, buffer);
-        Token_Array *tokens_ptr = scope_attachment(app, scope, attachment_tokens, Token_Array);
-        tokens.count = tokens_ptr->count;
-        tokens.tokens = push_array_write(scratch, Token, tokens.count, tokens_ptr->tokens);
-        system_release_global_frame_mutex(tctx);
-    }
     
+    if (tokens.count > 0){
+        ProfileBlock(app, "async parse");
         Arena arena = make_arena_system(KB(16));
+        Code_Index_File *index = push_array_zero(&arena, Code_Index_File, 1);
+        index->buffer = buffer_id;
         
         Generic_Parse_State state = {};
         generic_parse_init(app, &arena, contents, &tokens, &state);
         
-    Code_Index_File index = {};
-    b32 canceled = false;
         for (;;){
-        ProfileBlock(app, "async parse block");
+            if (generic_parse_full_input_breaks(index, &state, limit_factor)){
-        if (generic_parse_full_input_breaks(&index, &state, 10000)){
                 break;
             }
             if (async_check_canceled(actx)){
@@ -684,16 +669,21 @@ do_full_parse_async__inner(Async_Context *actx, Buffer_ID buffer){
         }
         
         if (!canceled){
-        ProfileBlock(app, "async parse save results");
+            code_index_lock();
-        code_index_set_file(app, buffer, arena, &index);
+            code_index_set_file(buffer_id, arena, index);
+            code_index_unlock();
+        }
+        else{
+            linalloc_clear(&arena);
+        }
     }
 }
 
 function void
-do_full_parse_async(Async_Context *actx, Data data){
+do_full_lex_and_parse_async(Async_Context *actx, Data data){
     if (data.size == sizeof(Buffer_ID)){
         Buffer_ID buffer = *(Buffer_ID*)data.data;
-        do_full_parse_async__inner(actx, buffer);
+        do_full_lex_and_parse_async__inner(actx, buffer);
     }
 }
 
@@ -815,9 +805,7 @@ BUFFER_HOOK_SIG(default_begin_buffer){
     if (use_lexer){
         ProfileBlock(app, "begin buffer kick off lexer");
         Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task);
-        *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_async, make_data_struct(&buffer_id));
+        *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_and_parse_async, make_data_struct(&buffer_id));
-        async_task_single_dep(&global_async_system, do_full_parse_async, make_data_struct(&buffer_id),
-                              *lex_task_ptr);
     }
     
     if (wrap_lines){
@@ -887,9 +875,9 @@ BUFFER_EDIT_RANGE_SIG(default_buffer_edit_range){
     Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task);
     if (async_task_is_running_or_pending(&global_async_system, *lex_task_ptr)){
         async_task_cancel(&global_async_system, *lex_task_ptr);
-        *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_async, make_data_struct(&buffer_id));
+        *lex_task_ptr = async_task_no_dep(&global_async_system,
-        async_task_single_dep(&global_async_system, do_full_parse_async, make_data_struct(&buffer_id),
+                                          do_full_lex_and_parse_async,
-                              *lex_task_ptr);
+                                          make_data_struct(&buffer_id));
     }
     else{
         Token_Array *ptr = scope_attachment(app, scope, attachment_tokens, Token_Array);
@@ -965,13 +953,9 @@ BUFFER_EDIT_RANGE_SIG(default_buffer_edit_range){
             }
             
             if (do_full_relex){
-                base_free(allocator, ptr->tokens);
-                block_zero_struct(ptr);
                 *lex_task_ptr = async_task_no_dep(&global_async_system,
-                                                  do_full_lex_async,
+                                                  do_full_lex_and_parse_async,
                                                   make_data_struct(&buffer_id));
-                async_task_single_dep(&global_async_system, do_full_parse_async, make_data_struct(&buffer_id),
-                                      *lex_task_ptr);
             }
         }
     }
@@ -986,6 +970,9 @@ BUFFER_HOOK_SIG(default_end_buffer){
     if (lex_task_ptr != 0){
         async_task_cancel(&global_async_system, *lex_task_ptr);
     }
+    code_index_lock();
+    code_index_erase_file(buffer_id);
+    code_index_unlock();
     // no meaning for return
     return(0);
 }

custom/generated/custom_api.cpp

@@ -43,6 +43,7 @@ vtable->push_buffer_file_name = push_buffer_file_name;
 vtable->buffer_get_dirty_state = buffer_get_dirty_state;
 vtable->buffer_set_dirty_state = buffer_set_dirty_state;
 vtable->buffer_set_layout = buffer_set_layout;
+vtable->file_clear_layout_cache = file_clear_layout_cache;
 vtable->buffer_get_layout = buffer_get_layout;
 vtable->buffer_get_setting = buffer_get_setting;
 vtable->buffer_set_setting = buffer_set_setting;
@@ -222,6 +223,7 @@ push_buffer_file_name = vtable->push_buffer_file_name;
 buffer_get_dirty_state = vtable->buffer_get_dirty_state;
 buffer_set_dirty_state = vtable->buffer_set_dirty_state;
 buffer_set_layout = vtable->buffer_set_layout;
+file_clear_layout_cache = vtable->file_clear_layout_cache;
 buffer_get_layout = vtable->buffer_get_layout;
 buffer_get_setting = vtable->buffer_get_setting;
 buffer_set_setting = vtable->buffer_set_setting;

custom/generated/custom_api.h

@@ -40,8 +40,9 @@
 #define custom_push_buffer_file_name_sig() String_Const_u8 custom_push_buffer_file_name(Application_Links* app, Arena* arena, Buffer_ID buffer_id)
 #define custom_buffer_get_dirty_state_sig() Dirty_State custom_buffer_get_dirty_state(Application_Links* app, Buffer_ID buffer_id)
 #define custom_buffer_set_dirty_state_sig() b32 custom_buffer_set_dirty_state(Application_Links* app, Buffer_ID buffer_id, Dirty_State dirty_state)
-#define custom_buffer_set_layout_sig() b32 custom_buffer_set_layout(Application_Links* app, Buffer_ID buffer, Layout_Function* layout_func)
+#define custom_buffer_set_layout_sig() b32 custom_buffer_set_layout(Application_Links* app, Buffer_ID buffer_id, Layout_Function* layout_func)
-#define custom_buffer_get_layout_sig() Layout_Function* custom_buffer_get_layout(Application_Links* app, Buffer_ID buffer)
+#define custom_file_clear_layout_cache_sig() b32 custom_file_clear_layout_cache(Application_Links* app, Buffer_ID buffer)
+#define custom_buffer_get_layout_sig() Layout_Function* custom_buffer_get_layout(Application_Links* app, Buffer_ID buffer_id)
 #define custom_buffer_get_setting_sig() b32 custom_buffer_get_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64* value_out)
 #define custom_buffer_set_setting_sig() b32 custom_buffer_set_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64 value)
 #define custom_buffer_get_managed_scope_sig() Managed_Scope custom_buffer_get_managed_scope(Application_Links* app, Buffer_ID buffer_id)
@@ -215,8 +216,9 @@ typedef String_Const_u8 custom_push_buffer_unique_name_type(Application_Links* a
 typedef String_Const_u8 custom_push_buffer_file_name_type(Application_Links* app, Arena* arena, Buffer_ID buffer_id);
 typedef Dirty_State custom_buffer_get_dirty_state_type(Application_Links* app, Buffer_ID buffer_id);
 typedef b32 custom_buffer_set_dirty_state_type(Application_Links* app, Buffer_ID buffer_id, Dirty_State dirty_state);
-typedef b32 custom_buffer_set_layout_type(Application_Links* app, Buffer_ID buffer, Layout_Function* layout_func);
+typedef b32 custom_buffer_set_layout_type(Application_Links* app, Buffer_ID buffer_id, Layout_Function* layout_func);
-typedef Layout_Function* custom_buffer_get_layout_type(Application_Links* app, Buffer_ID buffer);
+typedef b32 custom_file_clear_layout_cache_type(Application_Links* app, Buffer_ID buffer);
+typedef Layout_Function* custom_buffer_get_layout_type(Application_Links* app, Buffer_ID buffer_id);
 typedef b32 custom_buffer_get_setting_type(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64* value_out);
 typedef b32 custom_buffer_set_setting_type(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64 value);
 typedef Managed_Scope custom_buffer_get_managed_scope_type(Application_Links* app, Buffer_ID buffer_id);
@@ -392,6 +394,7 @@ custom_push_buffer_file_name_type *push_buffer_file_name;
 custom_buffer_get_dirty_state_type *buffer_get_dirty_state;
 custom_buffer_set_dirty_state_type *buffer_set_dirty_state;
 custom_buffer_set_layout_type *buffer_set_layout;
+custom_file_clear_layout_cache_type *file_clear_layout_cache;
 custom_buffer_get_layout_type *buffer_get_layout;
 custom_buffer_get_setting_type *buffer_get_setting;
 custom_buffer_set_setting_type *buffer_set_setting;
@@ -568,8 +571,9 @@ internal String_Const_u8 push_buffer_unique_name(Application_Links* app, Arena*
 internal String_Const_u8 push_buffer_file_name(Application_Links* app, Arena* arena, Buffer_ID buffer_id);
 internal Dirty_State buffer_get_dirty_state(Application_Links* app, Buffer_ID buffer_id);
 internal b32 buffer_set_dirty_state(Application_Links* app, Buffer_ID buffer_id, Dirty_State dirty_state);
-internal b32 buffer_set_layout(Application_Links* app, Buffer_ID buffer, Layout_Function* layout_func);
+internal b32 buffer_set_layout(Application_Links* app, Buffer_ID buffer_id, Layout_Function* layout_func);
-internal Layout_Function* buffer_get_layout(Application_Links* app, Buffer_ID buffer);
+internal b32 file_clear_layout_cache(Application_Links* app, Buffer_ID buffer);
+internal Layout_Function* buffer_get_layout(Application_Links* app, Buffer_ID buffer_id);
 internal b32 buffer_get_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64* value_out);
 internal b32 buffer_set_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64 value);
 internal Managed_Scope buffer_get_managed_scope(Application_Links* app, Buffer_ID buffer_id);
@@ -746,6 +750,7 @@ global custom_push_buffer_file_name_type *push_buffer_file_name = 0;
 global custom_buffer_get_dirty_state_type *buffer_get_dirty_state = 0;
 global custom_buffer_set_dirty_state_type *buffer_set_dirty_state = 0;
 global custom_buffer_set_layout_type *buffer_set_layout = 0;
+global custom_file_clear_layout_cache_type *file_clear_layout_cache = 0;
 global custom_buffer_get_layout_type *buffer_get_layout = 0;
 global custom_buffer_get_setting_type *buffer_get_setting = 0;
 global custom_buffer_set_setting_type *buffer_set_setting = 0;

custom/generated/custom_api_master_list.h

@@ -40,8 +40,9 @@ api(custom) function String_Const_u8 push_buffer_unique_name(Application_Links*
 api(custom) function String_Const_u8 push_buffer_file_name(Application_Links* app, Arena* arena, Buffer_ID buffer_id);
 api(custom) function Dirty_State buffer_get_dirty_state(Application_Links* app, Buffer_ID buffer_id);
 api(custom) function b32 buffer_set_dirty_state(Application_Links* app, Buffer_ID buffer_id, Dirty_State dirty_state);
-api(custom) function b32 buffer_set_layout(Application_Links* app, Buffer_ID buffer, Layout_Function* layout_func);
+api(custom) function b32 buffer_set_layout(Application_Links* app, Buffer_ID buffer_id, Layout_Function* layout_func);
-api(custom) function Layout_Function* buffer_get_layout(Application_Links* app, Buffer_ID buffer);
+api(custom) function b32 file_clear_layout_cache(Application_Links* app, Buffer_ID buffer);
+api(custom) function Layout_Function* buffer_get_layout(Application_Links* app, Buffer_ID buffer_id);
 api(custom) function b32 buffer_get_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64* value_out);
 api(custom) function b32 buffer_set_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64 value);
 api(custom) function Managed_Scope buffer_get_managed_scope(Application_Links* app, Buffer_ID buffer_id);

platform_win32/win32_4ed.cpp

@@ -1723,6 +1723,10 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS
                 win32vars.got_useful_event = false;
             }
             
+            // NOTE(allen): while we're doing this (and possibly sleeping)
+            // we can let async processes get there time in.
+            system_mutex_release(win32vars.global_frame_mutex);
+            
             b32 get_more_messages = true;
             do{
                 if (win32vars.got_useful_event == 0){
@@ -1797,6 +1801,8 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS
                     }
                 }
             }while (get_more_messages);
+            
+            system_mutex_acquire(win32vars.global_frame_mutex);
         }
         
         // NOTE(allen): Mouse Out of Window Detection