From fa20d09245ae9caed023a6d02acd64c86fe08476 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Thu, 10 Mar 2016 22:33:19 -0500 Subject: [PATCH] unique file hash system progress --- 4ed.cpp | 60 ++---- 4ed_app_target.cpp | 3 +- 4ed_file.cpp | 95 +++++++- 4ed_file_view.cpp | 38 ++-- 4ed_gui.cpp | 2 +- 4ed_system.h | 13 ++ 4tech_table.cpp | 491 ++++++++++++++++++++++-------------------- test/4cpp_new_lexer.h | 57 ++++- 8 files changed, 441 insertions(+), 318 deletions(-) diff --git a/4ed.cpp b/4ed.cpp index 958da58e..a9eeccc5 100644 --- a/4ed.cpp +++ b/4ed.cpp @@ -452,7 +452,6 @@ COMMAND_DECL(seek_alphanumeric_right){ ProfileMomentFunction(); REQ_READABLE_VIEW(view); REQ_FILE(file, view); - i32 pos = buffer_seek_alphanumeric_right(&file->state.buffer, view->cursor.pos); view_cursor_move(view, pos); } @@ -461,7 +460,6 @@ COMMAND_DECL(seek_alphanumeric_left){ ProfileMomentFunction(); REQ_READABLE_VIEW(view); REQ_FILE(file, view); - i32 pos = buffer_seek_alphanumeric_left(&file->state.buffer, view->cursor.pos); view_cursor_move(view, pos); } @@ -470,7 +468,6 @@ COMMAND_DECL(seek_alphanumeric_or_camel_right){ ProfileMomentFunction(); REQ_READABLE_VIEW(view); REQ_FILE(file, view); - i32 pos = buffer_seek_alphanumeric_or_camel_right(&file->state.buffer, view->cursor.pos); view_cursor_move(view, pos); } @@ -479,7 +476,6 @@ COMMAND_DECL(seek_alphanumeric_or_camel_left){ ProfileMomentFunction(); REQ_READABLE_VIEW(view); REQ_FILE(file, view); - i32 pos = buffer_seek_alphanumeric_or_camel_left(&file->state.buffer, view->cursor.pos); view_cursor_move(view, pos); } @@ -1738,7 +1734,7 @@ COMMAND_DECL(command_line){ file = working_set_get_active_file(working_set, buffer_id); } else if (buffer_name){ - file = working_set_contains(working_set, make_string(buffer_name, buffer_name_len)); + file = working_set_contains(system, working_set, make_string(buffer_name, buffer_name_len)); if (file == 0){ file = working_set_alloc_always(working_set, &models->mem.general); if (file == 0){ @@ -1755,7 +1751,7 @@ COMMAND_DECL(command_line){ file_create_read_only(system, models, file, buffer_name); file->settings.unimportant = 1; - table_add(&working_set->table, file->name.source_path, file->id.id); + working_set_add(system, working_set, file); for (i = 0; i < proc_count; ++i){ if (procs[i].out_file == file){ @@ -2037,15 +2033,12 @@ extern "C"{ Buffer_Summary buffer = {}; Editing_File *file; Working_Set *working_set = &cmd->models->working_set; - File_ID id; - if (table_find(&working_set->table, make_string(filename, len), &id)){ - file = working_set_get_active_file(working_set, id); - if (file){ - fill_buffer_summary(&buffer, file, working_set); - } + file = working_set_contains(cmd->system, working_set, make_string(filename, len)); + if (file && !file->state.is_dummy){ + fill_buffer_summary(&buffer, file, working_set); } - + return(buffer); } @@ -3293,33 +3286,10 @@ App_Init_Sig(app_init){ font_set_add(partition, models->font_set, file_name, name, pt_size); } } - + // NOTE(allen): file setup - { - Editing_File *files, *null_file; - i16 init_count = 128; - - dll_init_sentinel(&models->working_set.free_sentinel); - dll_init_sentinel(&models->working_set.used_sentinel); - - models->working_set.array_max = 128; - models->working_set.file_arrays = push_array(partition, File_Array, models->working_set.array_max); - - files = push_array(partition, Editing_File, init_count); - working_set_extend_memory(&models->working_set, files, init_count); - - null_file = working_set_index(&models->working_set, 0); - dll_remove(&null_file->node); - null_file->state.is_dummy = 1; - ++models->working_set.file_count; - - models->working_set.table.max = models->working_set.file_max * 3 / 2; - models->working_set.table.count = 0; - models->working_set.table.table = push_array( - partition, File_Table_Entry, models->working_set.table.max); - memset(models->working_set.table.table, 0, sizeof(File_Table_Entry) * models->working_set.table.max); - } - + working_set_init(&models->working_set, partition); + // NOTE(allen): clipboard setup models->working_set.clipboard_max_size = ArrayCount(models->working_set.clipboards); models->working_set.clipboard_size = 0; @@ -4143,7 +4113,7 @@ App_Step_Sig(app_step){ filename.str[0] = char_to_lower(filename.str[0]); - result.file = working_set_contains(working_set, filename); + result.file = working_set_contains(system, working_set, filename); if (result.file == 0){ result.is_new = 1; result.file = working_set_alloc_always(working_set, general); @@ -4153,7 +4123,7 @@ App_Step_Sig(app_step){ file_init_strings(result.file); file_set_name(working_set, result.file, filename.str); file_set_to_loading(result.file); - table_add(&working_set->table, result.file->name.source_path, result.file->id.id); + working_set_add(system, working_set, result.file); result.sys_id = file_id; result.file_index = result.file->id.id; @@ -4258,7 +4228,7 @@ App_Step_Sig(app_step){ { Editing_File *file = working_set_alloc_always(working_set, general); file_create_empty(system, models, file, string.str); - table_add(&working_set->table, file->name.source_path, file->id.id); + working_set_add(system, working_set, file); View *view = panel->view; @@ -4276,7 +4246,7 @@ App_Step_Sig(app_step){ file = working_set_lookup_file(working_set, string); if (!file){ - file = working_set_contains(working_set, string); + file = working_set_contains(system, working_set, string); } } @@ -4295,7 +4265,7 @@ App_Step_Sig(app_step){ file = working_set_lookup_file(working_set, string); if (!file){ - file = working_set_contains(working_set, string); + file = working_set_contains(system, working_set, string); } } @@ -4320,7 +4290,7 @@ App_Step_Sig(app_step){ file = working_set_lookup_file(working_set, string); if (!file){ - file = working_set_contains(working_set, string); + file = working_set_contains(system, working_set, string); } } diff --git a/4ed_app_target.cpp b/4ed_app_target.cpp index fe8f3273..aa453217 100644 --- a/4ed_app_target.cpp +++ b/4ed_app_target.cpp @@ -32,7 +32,8 @@ #include "4tech_table.cpp" #define FCPP_LEXER_IMPLEMENTATION -#include "test/4cpp_new_lexer.h" +//#include "test/4cpp_new_lexer.h" +#include "4cpp_lexer.h" #include "4ed_template.cpp" diff --git a/4ed_file.cpp b/4ed_file.cpp index 76470057..409e8e01 100644 --- a/4ed_file.cpp +++ b/4ed_file.cpp @@ -168,6 +168,7 @@ struct Editing_File{ File_ID id; }; +#if 0 struct File_Table_Entry{ String name; u32 hash; @@ -254,6 +255,31 @@ table_remove(File_Table *table, String name){ } return r; } +#endif + +struct File_Table_Entry{ + Unique_Hash key; + File_ID id; +}; + +internal u32 +tbl_file_hash(void *item, void *arg){ + Unique_Hash uhash = *((Unique_Hash*)item); + u32 hash = uhash.d[0] + 101; + hash = ((hash << 6) + hash) + uhash.d[1]; + hash = ((hash << 6) + hash) + uhash.d[2]; + hash = ((hash << 6) + hash) + uhash.d[3]; + hash = ((hash << 6) + hash) + uhash.non_file_id; + return(hash); +} + +internal i32 +tbl_file_compare(void *a, void *b, void *arg){ + Unique_Hash ahash = *((Unique_Hash*)a); + Unique_Hash bhash = *((Unique_Hash*)b); + i32 result = uhash_equal(ahash, bhash); + return(result); +} struct File_Array{ Editing_File *files; @@ -268,7 +294,7 @@ struct Working_Set{ File_Node free_sentinel; File_Node used_sentinel; - File_Table table; + Table table; String clipboards[64]; i32 clipboard_size, clipboard_max_size; @@ -374,12 +400,10 @@ working_set_index(Working_Set *working_set, i32 id){ inline Editing_File* working_set_get_active_file(Working_Set *working_set, File_ID id){ Editing_File *result = 0; - result = working_set_index(working_set, id); if (result && result->state.is_dummy){ result = 0; } - return(result); } @@ -391,17 +415,72 @@ working_set_get_active_file(Working_Set *working_set, i32 id){ } inline Editing_File* -working_set_contains(Working_Set *working_set, String filename){ +working_set_contains(Working_Set *working_set, Unique_Hash file_hash){ Editing_File *result = 0; - File_ID id; + File_Table_Entry *entry = 0; - replace_char(filename, '\\', '/'); - if (table_find(&working_set->table, filename, &id)){ - result = working_set_index(working_set, id); + entry = (File_Table_Entry*)table_find_item( + &working_set->table, &file_hash, 0, tbl_file_hash, tbl_file_compare); + if (entry){ + result = working_set_index(working_set, entry->id); } + return (result); } +inline Editing_File* +working_set_contains(System_Functions *system, Working_Set *working_set, String filename){ + Unique_Hash file_hash; + Editing_File *result; + terminate_with_null(&filename); + file_hash = system->file_unique_hash(filename.str); + result = working_set_contains(working_set, file_hash); + return(result); +} + +internal void +working_set_init(Working_Set *working_set, Partition *partition){ + Editing_File *files, *null_file; + void *mem; + i32 mem_size; + i16 init_count = 128; + + dll_init_sentinel(&working_set->free_sentinel); + dll_init_sentinel(&working_set->used_sentinel); + + working_set->array_max = 128; + working_set->file_arrays = push_array(partition, File_Array, working_set->array_max); + + files = push_array(partition, Editing_File, init_count); + working_set_extend_memory(working_set, files, init_count); + + null_file = working_set_index(working_set, 0); + dll_remove(&null_file->node); + null_file->state.is_dummy = 1; + ++working_set->file_count; + + mem_size = table_required_mem_size(working_set->file_max * 3 / 2, sizeof(File_Table_Entry)); + mem = push_block(partition, mem_size); + memset(mem, 0, mem_size); + table_init_memory(&working_set->table, mem, working_set->file_max * 3 / 2, sizeof(File_Table_Entry)); +} + +internal void +working_set_add(Working_Set *working_set, Unique_Hash key, File_ID file_id){ + File_Table_Entry entry; + entry.key = key; + entry.id = file_id; + + table_add(&working_set->table, &entry, 0, tbl_file_hash, tbl_file_compare); +} + +internal void +working_set_add(System_Functions *system, Working_Set *working_set, Editing_File *file){ + Unique_Hash file_hash; + file_hash = system->file_unique_hash(file->name.source_path.str); + working_set_add(working_set, file_hash, file->id); +} + // TODO(allen): Pick better first options. internal Editing_File* working_set_lookup_file(Working_Set *working_set, String string){ diff --git a/4ed_file_view.cpp b/4ed_file_view.cpp index 1afa9a04..41e08782 100644 --- a/4ed_file_view.cpp +++ b/4ed_file_view.cpp @@ -659,14 +659,13 @@ Job_Callback_Sig(job_full_lex){ tokens.max_count = memory->size / sizeof(Cpp_Token); tokens.count = 0; - Cpp_Lex_Data status; - status = cpp_lex_nonalloc(cpp_file, &tokens); + Cpp_Lex_Data status = cpp_lex_file_nonalloc(cpp_file, &tokens); while (!status.complete){ system->grow_thread_memory(memory); tokens.tokens = (Cpp_Token*)memory->data; tokens.max_count = memory->size / sizeof(Cpp_Token); - status = cpp_lex_nonalloc(cpp_file, &tokens, status); + status = cpp_lex_file_nonalloc(cpp_file, &tokens, status); } i32 new_max = LargeRoundUp(tokens.count+1, Kbytes(1)); @@ -3980,37 +3979,26 @@ search_set_init(General_Memory *general, Search_Set *set, i32 set_count){ internal void search_hits_table_alloc(General_Memory *general, Table *hits, i32 table_size){ - i32 hash_size, mem_size; - - hash_size = table_size * sizeof(u32); - hash_size = (hash_size + 7) & ~7; - mem_size = hash_size + table_size * sizeof(Offset_String); - - hits->hash_array = (u32*)general_memory_allocate(general, mem_size, 0); - hits->data_array = (u8*)hits->hash_array + hash_size; - hits->max = table_size; - - hits->item_size = sizeof(Offset_String); + void *mem; + i32 mem_size; + + mem_size = table_required_mem_size(table_size, sizeof(Offset_String)); + mem = general_memory_reallocate_nocopy(general, hits->hash_array, mem_size, 0); + table_init_memory(hits, mem, table_size, sizeof(Offset_String)); } internal void search_hits_init(General_Memory *general, Table *hits, String_Space *str, i32 table_size, i32 str_size){ - i32 hash_size, mem_size; + void *mem; + i32 mem_size; if (hits->hash_array == 0){ search_hits_table_alloc(general, hits, table_size); } else if (hits->max < table_size){ - hash_size = table_size * sizeof(u32); - hash_size = (hash_size + 7) & ~7; - mem_size = hash_size + table_size * sizeof(Offset_String); - - hits->hash_array = (u32*)general_memory_reallocate_nocopy( - general, hits->hash_array, mem_size, 0); - hits->data_array = (u8*)hits->hash_array + hash_size; - hits->max = table_size; - - hits->item_size = sizeof(Offset_String); + mem_size = table_required_mem_size(table_size, sizeof(Offset_String)); + mem = general_memory_reallocate_nocopy(general, hits->hash_array, mem_size, 0); + table_init_memory(hits, mem, table_size, sizeof(Offset_String)); } if (str->space == 0){ diff --git a/4ed_gui.cpp b/4ed_gui.cpp index 59b4baf9..d672e4b1 100644 --- a/4ed_gui.cpp +++ b/4ed_gui.cpp @@ -1292,7 +1292,7 @@ do_file_list_box(System_Functions *system, UI_State *state, UI_Layout *layout, append(&full_path, filename); terminate_with_null(&full_path); - Editing_File *file = working_set_contains(state->working_set, full_path); + Editing_File *file = working_set_contains(system, state->working_set, full_path); full_path.size = restore_size; b8 is_folder = (info->folder != 0); diff --git a/4ed_system.h b/4ed_system.h index 76e1c760..90d2b39f 100644 --- a/4ed_system.h +++ b/4ed_system.h @@ -15,8 +15,21 @@ struct Plat_Handle{ struct Unique_Hash{ u32 d[4]; + u32 non_file_id; }; +inline Unique_Hash +uhash_zero(){ + Unique_Hash r = {0}; + return(r); +} + +inline int +uhash_equal(Unique_Hash a, Unique_Hash b){ + int result = (memcmp(&a, &b, sizeof(a)) == 0); + return(result); +} + #define Sys_File_Time_Stamp_Sig(name) u64 name(char *filename) typedef Sys_File_Time_Stamp_Sig(System_File_Time_Stamp); diff --git a/4tech_table.cpp b/4tech_table.cpp index e1d92800..613b134b 100644 --- a/4tech_table.cpp +++ b/4tech_table.cpp @@ -1,235 +1,256 @@ -/* - * Mr. 4th Dimention - Allen Webster - * - * 14.02.2016 - * - * 4tech C style genereic hash table - * - */ - -// TOP - -#define TableHashEmpty 0 -#define TableHashDeleted 1 -#define TableHashMin 0x10000000 - -typedef u32 Hash_Function(void *item, void *arg); -typedef i32 Compare_Function(void *key, void *item, void *arg); - -struct Table{ - u32 *hash_array; - u8 *data_array; - i32 count, max; - - i32 item_size; -}; - -internal b32 -table_at_capacity(Table *table){ - b32 result = 1; - if (table->count * 8 < table->max * 7){ - result = 0; - } - return(result); -} - -internal b32 -table_add(Table *table, void *item, void *arg, Hash_Function *hash_func, Compare_Function *comp_func){ - u32 hash, *inspect; - i32 i; - - Assert(table->count * 8 < table->max * 7); - - hash = (hash_func(item, arg) | TableHashMin); - i = hash % table->max; - inspect = table->hash_array + i; - - while (*inspect >= TableHashMin){ - if (*inspect == hash){ - if (comp_func(item, table->data_array + i*table->item_size, arg) == 0){ - return(1); - } - } - ++i; - ++inspect; - if (i == table->max){ - i = 0; - inspect = table->hash_array; - } - } - *inspect = hash; - memcpy(table->data_array + i*table->item_size, item, table->item_size); - ++table->count; - - return(0); -} - -internal b32 -table_find_pos(Table *table, void *search_key, void *arg, i32 *pos, i32 *index, Hash_Function *hash_func, Compare_Function *comp_func){ - u32 hash, *inspect; - i32 i; - - hash = (hash_func(search_key, arg) | TableHashMin); - i = hash % table->max; - inspect = table->hash_array + i; - - while (*inspect != TableHashEmpty){ - if (*inspect == hash){ - if (comp_func(search_key, table->data_array + i*table->item_size, arg) == 0){ - if (pos) *pos = i*table->item_size; - if (index) *index = i; - return(1); - } - } - ++i; - ++inspect; - if (i == table->max){ - i = 0; - inspect = table->hash_array; - } - } - - return(0); -} - -internal void* -table_find_item(Table *table, void *search_key, void *arg, Hash_Function *hash_func, Compare_Function *comp_func){ - i32 pos; - void *result = 0; - if (table_find_pos(table, search_key, arg, &pos, 0, hash_func, comp_func)){ - result = table->data_array + pos; - } - return(result); -} - -internal void -table_remove_index(Table *table, i32 index){ - table->hash_array[index] = TableHashDeleted; - --table->count; -} - -internal b32 -table_remove_match(Table *table, void *search_key, void *arg, Hash_Function *hash_func, Compare_Function *comp_func){ - i32 index; - b32 result = 0; - if (table_find_pos(table, search_key, arg, 0, &index, hash_func, comp_func)){ - table_remove_index(table, index); - result = 1; - } - return(result); -} - -internal void -table_clear(Table *table){ - table->count = 0; - memset(table->hash_array, 0, table->max*sizeof(*table->hash_array)); -} - -internal void -table_rehash(Table *src, Table *dst, void *arg, Hash_Function *hash_func, Compare_Function *comp_func){ - i32 i, c, count, item_size; - u32 *hash_item; - u8 *data_item; - - Assert((dst->count + src->count - 1) * 7 < dst->max * 8); - Assert(dst->item_size == src->item_size); - - count = src->count; - hash_item = src->hash_array; - data_item = src->data_array; - item_size = src->item_size; - for (i = 0, c = 0; c < count; ++i, ++hash_item, data_item += item_size){ - if (*hash_item >= TableHashMin){ - ++c; - table_add(dst, data_item, arg, hash_func, comp_func); - } - } -} - -internal u32 -tbl_string_hash(void *item, void *arg){ - String *string = (String*)item; - char *str; - i32 i,len; - u32 x = 5381; - char c; - (void)arg; - - str = string->str; - len = string->size; - i = 0; - while (i < len){ - c = str[i++]; - x = ((x << 5) + x) + c; - } - - return(x); -} - -internal i32 -tbl_string_compare(void *a, void *b, void *arg){ - String *stra = (String*)a; - String *strb = (String*)b; - i32 result = !match(*stra, *strb); - return(result); -} - -internal u32 -tbl_offset_string_hash(void *item, void *arg){ - Offset_String *string = (Offset_String*)item; - char *str; - i32 i,len; - u32 x = 5381; - char c; - - str = ((char*)arg) + string->offset; - len = string->size; - i = 0; - while (i < len){ - c = str[i++]; - x = ((x << 5) + x) + c; - } - - return(x); -} - -internal i32 -tbl_offset_string_compare(void *a, void *b, void *arg){ - Offset_String *ostra = (Offset_String*)a; - Offset_String *ostrb = (Offset_String*)b; - String stra = make_string((char*)arg + ostra->offset, ostra->size); - String strb = make_string((char*)arg + ostrb->offset, ostrb->size); - i32 result = !match(stra, strb); - return(result); -} - -struct String_Space{ - char *space; - i32 pos, new_pos, max; -}; - -internal Offset_String -strspace_append(String_Space *space, char *str, i32 len){ - Offset_String result = {}; - if (space->new_pos + len <= space->max){ - result.offset = space->new_pos; - result.size = len; - - memcpy(space->space + space->new_pos, str, len); - space->new_pos = space->pos + len; - } - return(result); -} - -internal void -strspace_keep_prev(String_Space *space){ - space->pos = space->new_pos; -} - -internal void -strspace_discard_prev(String_Space *space){ - space->new_pos = space->pos; -} - -// BOTTOM - +/* + * Mr. 4th Dimention - Allen Webster + * + * 14.02.2016 + * + * 4tech C style genereic hash table + * + */ + +// TOP + +#define TableHashEmpty 0 +#define TableHashDeleted 1 +#define TableHashMin 0x10000000 + +typedef u32 Hash_Function(void *item, void *arg); +typedef i32 Compare_Function(void *key, void *item, void *arg); + +struct Table{ + u32 *hash_array; + u8 *data_array; + i32 count, max; + + i32 item_size; +}; + +internal i32 +table_required_mem_size(i32 table_size, i32 item_size){ + i32 mem_size, hash_size; + hash_size = ((table_size * sizeof(u32)) + 7) & ~7; + mem_size = hash_size + table_size * item_size; + return(mem_size); +} + +internal void +table_init_memory(Table *table, void *memory, i32 table_size, i32 item_size){ + i32 hash_size = table_size * sizeof(u32); + hash_size = (hash_size + 7) & ~7; + + table->hash_array = (u32*)memory; + table->data_array = (u8*)(table->hash_array) + hash_size; + + table->count = 0; + table->max = table_size; + table->item_size = item_size; +} + +internal b32 +table_at_capacity(Table *table){ + b32 result = 1; + if (table->count * 8 < table->max * 7){ + result = 0; + } + return(result); +} + +internal b32 +table_add(Table *table, void *item, void *arg, Hash_Function *hash_func, Compare_Function *comp_func){ + u32 hash, *inspect; + i32 i; + + Assert(table->count * 8 < table->max * 7); + + hash = (hash_func(item, arg) | TableHashMin); + i = hash % table->max; + inspect = table->hash_array + i; + + while (*inspect >= TableHashMin){ + if (*inspect == hash){ + if (comp_func(item, table->data_array + i*table->item_size, arg) == 0){ + return(1); + } + } + ++i; + ++inspect; + if (i == table->max){ + i = 0; + inspect = table->hash_array; + } + } + *inspect = hash; + memcpy(table->data_array + i*table->item_size, item, table->item_size); + ++table->count; + + return(0); +} + +internal b32 +table_find_pos(Table *table, void *search_key, void *arg, i32 *pos, i32 *index, Hash_Function *hash_func, Compare_Function *comp_func){ + u32 hash, *inspect; + i32 i; + + hash = (hash_func(search_key, arg) | TableHashMin); + i = hash % table->max; + inspect = table->hash_array + i; + + while (*inspect != TableHashEmpty){ + if (*inspect == hash){ + if (comp_func(search_key, table->data_array + i*table->item_size, arg) == 0){ + if (pos) *pos = i*table->item_size; + if (index) *index = i; + return(1); + } + } + ++i; + ++inspect; + if (i == table->max){ + i = 0; + inspect = table->hash_array; + } + } + + return(0); +} + +internal void* +table_find_item(Table *table, void *search_key, void *arg, Hash_Function *hash_func, Compare_Function *comp_func){ + i32 pos; + void *result = 0; + if (table_find_pos(table, search_key, arg, &pos, 0, hash_func, comp_func)){ + result = table->data_array + pos; + } + return(result); +} + +internal void +table_remove_index(Table *table, i32 index){ + table->hash_array[index] = TableHashDeleted; + --table->count; +} + +internal b32 +table_remove_match(Table *table, void *search_key, void *arg, Hash_Function *hash_func, Compare_Function *comp_func){ + i32 index; + b32 result = 0; + if (table_find_pos(table, search_key, arg, 0, &index, hash_func, comp_func)){ + table_remove_index(table, index); + result = 1; + } + return(result); +} + +internal void +table_clear(Table *table){ + table->count = 0; + memset(table->hash_array, 0, table->max*sizeof(*table->hash_array)); +} + +internal void +table_rehash(Table *src, Table *dst, void *arg, Hash_Function *hash_func, Compare_Function *comp_func){ + i32 i, c, count, item_size; + u32 *hash_item; + u8 *data_item; + + Assert((dst->count + src->count - 1) * 7 < dst->max * 8); + Assert(dst->item_size == src->item_size); + + count = src->count; + hash_item = src->hash_array; + data_item = src->data_array; + item_size = src->item_size; + for (i = 0, c = 0; c < count; ++i, ++hash_item, data_item += item_size){ + if (*hash_item >= TableHashMin){ + ++c; + table_add(dst, data_item, arg, hash_func, comp_func); + } + } +} + +internal u32 +tbl_string_hash(void *item, void *arg){ + String *string = (String*)item; + char *str; + i32 i,len; + u32 x = 5381; + char c; + (void)arg; + + str = string->str; + len = string->size; + i = 0; + while (i < len){ + c = str[i++]; + x = ((x << 5) + x) + c; + } + + return(x); +} + +internal i32 +tbl_string_compare(void *a, void *b, void *arg){ + String *stra = (String*)a; + String *strb = (String*)b; + i32 result = !match(*stra, *strb); + return(result); +} + +internal u32 +tbl_offset_string_hash(void *item, void *arg){ + Offset_String *string = (Offset_String*)item; + char *str; + i32 i,len; + u32 x = 5381; + char c; + + str = ((char*)arg) + string->offset; + len = string->size; + i = 0; + while (i < len){ + c = str[i++]; + x = ((x << 5) + x) + c; + } + + return(x); +} + +internal i32 +tbl_offset_string_compare(void *a, void *b, void *arg){ + Offset_String *ostra = (Offset_String*)a; + Offset_String *ostrb = (Offset_String*)b; + String stra = make_string((char*)arg + ostra->offset, ostra->size); + String strb = make_string((char*)arg + ostrb->offset, ostrb->size); + i32 result = !match(stra, strb); + return(result); +} + +struct String_Space{ + char *space; + i32 pos, new_pos, max; +}; + +internal Offset_String +strspace_append(String_Space *space, char *str, i32 len){ + Offset_String result = {}; + if (space->new_pos + len <= space->max){ + result.offset = space->new_pos; + result.size = len; + + memcpy(space->space + space->new_pos, str, len); + space->new_pos = space->pos + len; + } + return(result); +} + +internal void +strspace_keep_prev(String_Space *space){ + space->pos = space->new_pos; +} + +internal void +strspace_discard_prev(String_Space *space){ + space->new_pos = space->pos; +} + +// BOTTOM + diff --git a/test/4cpp_new_lexer.h b/test/4cpp_new_lexer.h index 94a1b5d3..1622028d 100644 --- a/test/4cpp_new_lexer.h +++ b/test/4cpp_new_lexer.h @@ -84,6 +84,7 @@ enum Lex_State{ LS_char, LS_string, LS_number, + LS_float, LS_comment_pre, LS_comment, LS_comment_block, @@ -103,6 +104,8 @@ enum Lex_State{ LS_star, LS_modulo, LS_caret, + LS_eq, + LS_bang, }; struct Lex_Data{ @@ -191,6 +194,9 @@ cpp_lex_nonalloc(char *chunk, int file_absolute_pos, int size, Cpp_Token_Stack * case '%': state = LS_modulo; break; case '^': state = LS_caret; break; + case '=': state = LS_eq; break; + case '!': state = LS_bang; break; + #define OperCase(op,type) case op: emit_token = 1; break; OperCase('{', CPP_TOKEN_BRACE_OPEN); OperCase('}', CPP_TOKEN_BRACE_CLOSE); @@ -227,8 +233,8 @@ cpp_lex_nonalloc(char *chunk, int file_absolute_pos, int size, Cpp_Token_Stack * if (c >= '0' && c <= '9'){ state = LS_number; } - else if (c == '.'){ - state = LS_float; + else{ + emit_token = 1; } break; @@ -370,6 +376,20 @@ cpp_lex_nonalloc(char *chunk, int file_absolute_pos, int size, Cpp_Token_Stack * default: emit_token = 1; break; } break; + + case LS_eq: + switch (c){ + case '=': emit_token = 1; break; + default: emit_token = 1; break; + } + break; + + case LS_bang: + switch (c){ + case '=': emit_token = 1; break; + default: emit_token = 1; break; + } + break; } } @@ -405,6 +425,13 @@ cpp_lex_nonalloc(char *chunk, int file_absolute_pos, int size, Cpp_Token_Stack * --pos; break; + case LS_number: + token.type = CPP_TOKEN_INTEGER_CONSTANT; + token.flags = 0; + --lex_data.token_end; + --pos; + break; + case LS_comment_pre: token.flags = CPP_TFLAG_IS_OPERATOR; switch (c){ @@ -606,7 +633,7 @@ cpp_lex_nonalloc(char *chunk, int file_absolute_pos, int size, Cpp_Token_Stack * case LS_caret: token.flags = CPP_TFLAG_IS_OPERATOR; switch (c){ - case '^': token.type = CPP_TOKEN_XOREQ; break; + case '=': token.type = CPP_TOKEN_XOREQ; break; default: token.type = CPP_TOKEN_BIT_XOR; --lex_data.token_end; @@ -614,6 +641,30 @@ cpp_lex_nonalloc(char *chunk, int file_absolute_pos, int size, Cpp_Token_Stack * break; } break; + + case LS_eq: + token.flags = CPP_TFLAG_IS_OPERATOR; + switch (c){ + case '=': token.type = CPP_TOKEN_EQEQ; break; + default: + token.type = CPP_TOKEN_EQ; + --lex_data.token_end; + --pos; + break; + } + break; + + case LS_bang: + token.flags = CPP_TFLAG_IS_OPERATOR; + switch (c){ + case '=': token.type = CPP_TOKEN_NOTEQ; break; + default: + token.type = CPP_TOKEN_BIT_NOT; + --lex_data.token_end; + --pos; + break; + } + break; } token.start = lex_data.token_start;