From 3515722fdbd6cff767d3e663cdb4e381df2e1569 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Thu, 23 Mar 2017 18:45:32 -0400 Subject: [PATCH] fixed the utf8 <-> utf16 system in win32 --- 4coder_API/types.h | 4 +- 4coder_API/version.h | 15 +++ 4coder_lib/4coder_utf8.h | 4 +- 4coder_project_commands.cpp | 2 +- 4ed_file_track.h | 2 +- 4ed_system.h | 6 +- 4ed_system_shared.cpp | 53 ++++---- 4ed_system_shared.h | 8 +- internal_4coder_tests.cpp | 2 +- win32_4ed.cpp | 194 ++++++++++++--------------- win32_4ed_file_track.cpp | 30 ++--- win32_utf8.cpp | 258 ++++++++++++++++++++++++++++++++++++ win32_utf8.h | 42 ++++++ 13 files changed, 453 insertions(+), 167 deletions(-) create mode 100644 win32_utf8.cpp create mode 100644 win32_utf8.h diff --git a/4coder_API/types.h b/4coder_API/types.h index 06d45281..6d9a3d6f 100644 --- a/4coder_API/types.h +++ b/4coder_API/types.h @@ -431,9 +431,9 @@ STRUCT File_List{ /* DOC(This field is an array of File_Info structs.) */ File_Info *infos; /* DOC(This field specifies the number of struts in the info array.) */ - int32_t count; + uint32_t count; /* DOC(This field is for internal use.) */ - int32_t block_size; + uint32_t block_size; }; /* DOC(Buffer_Identifier acts as a loosely typed description of a buffer that can either be a name or an id.) */ diff --git a/4coder_API/version.h b/4coder_API/version.h index 496d6f8e..67f7e71c 100644 --- a/4coder_API/version.h +++ b/4coder_API/version.h @@ -2,6 +2,7 @@ #define MINOR 0 #define PATCH 18 +// string #define VN__(a,b,c) #a"."#b"."#c #define VN_(a,b,c) VN__(a,b,c) #define VERSION_NUMBER VN_(MAJOR,MINOR,PATCH) @@ -14,3 +15,17 @@ #endif #define VERSION VERSION_STRING VERSION_TYPE + +// long string +#define L_VN__(a,b,c) L#a L"." L#b L"." L#c +#define L_VN_(a,b,c) L_VN__(a,b,c) +#define L_VERSION_NUMBER L_VN_(MAJOR,MINOR,PATCH) +#define L_VERSION_STRING L"alpha " L_VERSION_NUMBER + +#if defined(FRED_SUPER) +#define L_VERSION_TYPE L" super!" +#else +#define L_VERSION_TYPE +#endif + +#define L_VERSION L_VERSION_STRING L_VERSION_TYPE diff --git a/4coder_lib/4coder_utf8.h b/4coder_lib/4coder_utf8.h index 76d6580d..7d20a7e6 100644 --- a/4coder_lib/4coder_utf8.h +++ b/4coder_lib/4coder_utf8.h @@ -281,10 +281,10 @@ utf8_to_utf16_minimal_checking(u16_4tech *dst, umem_4tech max_wchars, u8_4tech * static umem_4tech utf16_to_utf8_minimal_checking(u8_4tech *dst, umem_4tech max_chars, u16_4tech *src, umem_4tech length, b32_4tech *error){ u16_4tech *s = src; - u16_4tech *s_end = s + max_chars; + u16_4tech *s_end = s + length; u8_4tech *d = dst; - u8_4tech *d_end = d + length; + u8_4tech *d_end = d + max_chars; umem_4tech limit = length; umem_4tech needed_max = 0; diff --git a/4coder_project_commands.cpp b/4coder_project_commands.cpp index c1d96f7a..c9104821 100644 --- a/4coder_project_commands.cpp +++ b/4coder_project_commands.cpp @@ -102,7 +102,7 @@ open_all_files_with_extension_internal(Application_Links *app, String dir, char File_List list = get_file_list(app, dir.str, dir.size); int32_t dir_size = dir.size; - for (int32_t i = 0; i < list.count; ++i){ + for (uint32_t i = 0; i < list.count; ++i){ File_Info *info = list.infos + i; if (info->folder){ if (recursive){ diff --git a/4ed_file_track.h b/4ed_file_track.h index 0ea2ac37..c09ef694 100644 --- a/4ed_file_track.h +++ b/4ed_file_track.h @@ -46,7 +46,7 @@ FILE_TRACK_LINK File_Track_Result expand_track_system_listeners(File_Track_System *system, Partition *scratch, void *mem, i32 size); FILE_TRACK_LINK File_Track_Result -get_change_event(File_Track_System *system, Partition *scratch, u8 *buffer, i32 max); +get_change_event(File_Track_System *system, Partition *scratch, u8 *buffer, i32 max, i32 *size); FILE_TRACK_LINK File_Track_Result shut_down_track_system(File_Track_System *system, Partition *scratch); diff --git a/4ed_system.h b/4ed_system.h index 41eb4a2f..70b92b6e 100644 --- a/4ed_system.h +++ b/4ed_system.h @@ -204,13 +204,13 @@ typedef Sys_Acquire_Lock_Sig(System_Acquire_Lock); typedef Sys_Release_Lock_Sig(System_Release_Lock); // needed for custom layer -#define Sys_Memory_Allocate_Sig(name) void* name(i32 size) +#define Sys_Memory_Allocate_Sig(name) void* name(umem size) typedef Sys_Memory_Allocate_Sig(System_Memory_Allocate); -#define Sys_Memory_Set_Protection_Sig(name) bool32 name(void *ptr, i32 size, u32 flags) +#define Sys_Memory_Set_Protection_Sig(name) bool32 name(void *ptr, umem size, u32 flags) typedef Sys_Memory_Set_Protection_Sig(System_Memory_Set_Protection); -#define Sys_Memory_Free_Sig(name) void name(void *ptr, i32 size) +#define Sys_Memory_Free_Sig(name) void name(void *ptr, umem size) typedef Sys_Memory_Free_Sig(System_Memory_Free); #define Sys_File_Exists_Sig(name) b32 name(char *filename, i32 len) diff --git a/4ed_system_shared.cpp b/4ed_system_shared.cpp index 4bb6d099..b9128811 100644 --- a/4ed_system_shared.cpp +++ b/4ed_system_shared.cpp @@ -32,14 +32,14 @@ global Shared_Vars shared_vars; internal void init_shared_vars(){ umem scratch_size = KB(128); - void *scratch_memory = system_get_memory(scratch_size); - shared_vars.scratch = make_part(scratch_memory, scratch_size); + void *scratch_memory = system_memory_allocate(scratch_size); + shared_vars.scratch = make_part(scratch_memory, (i32)scratch_size); shared_vars.track_table_size = KB(16); - shared_vars.track_table = system_get_memory(shared_vars.track_table_size); + shared_vars.track_table = system_memory_allocate(shared_vars.track_table_size); shared_vars.track_node_size = KB(16); - void *track_nodes = system_get_memory(shared_vars.track_node_size); + void *track_nodes = system_memory_allocate(shared_vars.track_node_size); i32 track_result = init_track_system(&shared_vars.track, &shared_vars.scratch, shared_vars.track_table, shared_vars.track_table_size, track_nodes, shared_vars.track_node_size); @@ -56,9 +56,9 @@ handle_track_out_of_memory(i32 val){ case FileTrack_OutOfTableMemory: { u32 new_table_size = shared_vars.track_table_size*2; - void *new_table = system_get_memory(new_table_size); + void *new_table = system_memory_allocate(new_table_size); move_track_system(&shared_vars.track, &shared_vars.scratch, new_table, new_table_size); - system_free_memory(shared_vars.track_table); + system_memory_free(shared_vars.track_table, shared_vars.track_table_size); shared_vars.track_table_size = new_table_size; shared_vars.track_table = new_table; }break; @@ -66,7 +66,7 @@ handle_track_out_of_memory(i32 val){ case FileTrack_OutOfListenerMemory: { shared_vars.track_node_size *= 2; - void *node_expansion = system_get_memory(shared_vars.track_node_size); + void *node_expansion = system_memory_allocate(shared_vars.track_node_size); expand_track_system_listeners(&shared_vars.track, &shared_vars.scratch, node_expansion, shared_vars.track_node_size); }break; @@ -78,13 +78,13 @@ handle_track_out_of_memory(i32 val){ internal Sys_Add_Listener_Sig(system_add_listener){ - b32 result = 0; + b32 result = false; for (;;){ - i32 track_result = add_listener(&shared_vars.track, &shared_vars.scratch, filename); + i32 track_result = add_listener(&shared_vars.track, &shared_vars.scratch, (u8*)filename); if (handle_track_out_of_memory(track_result)){ if (track_result == FileTrack_Good){ - result = 1; + result = true; } break; } @@ -95,10 +95,10 @@ Sys_Add_Listener_Sig(system_add_listener){ internal Sys_Remove_Listener_Sig(system_remove_listener){ - i32 result = 0; - i32 track_result = remove_listener(&shared_vars.track, &shared_vars.scratch, filename); + b32 result = false; + i32 track_result = remove_listener(&shared_vars.track, &shared_vars.scratch, (u8*)filename); if (track_result == FileTrack_Good){ - result = 1; + result = true; } return(result); } @@ -108,7 +108,7 @@ Sys_Get_File_Change_Sig(system_get_file_change){ b32 result = false; i32 size = 0; - i32 get_result = get_change_event(&shared_vars.track, &shared_vars.scratch, buffer, max, &size); + i32 get_result = get_change_event(&shared_vars.track, &shared_vars.scratch, (u8*)buffer, max, &size); *required_size = size; *mem_too_small = false; @@ -138,14 +138,14 @@ sysshared_load_file(char *filename){ result.got_file = 1; if (size > 0){ result.size = size; - result.data = (char*)system_get_memory(size+1); + result.data = (char*)system_memory_allocate(size+1); if (!result.data){ result = null_file_data; } else{ if (!system_load_file(handle, result.data, size)){ - system_free_memory(result.data); + system_memory_free(result.data, size+1); result = null_file_data; } } @@ -159,21 +159,19 @@ sysshared_load_file(char *filename){ internal b32 usable_ascii(char c){ - b32 result = 1; + b32 result = true; if ((c < ' ' || c > '~') && c != '\n' && c != '\r' && c != '\t'){ - result = 0; + result = false; } return(result); } internal void sysshared_filter_real_files(char **files, i32 *file_count){ - i32 i, j; - i32 end; - - end = *file_count; - for (i = 0, j = 0; i < end; ++i){ - if (system_file_can_be_made(files[i])){ + i32 end = *file_count; + i32 i = 0, j = 0; + for (; i < end; ++i){ + if (system_file_can_be_made((u8*)files[i])){ files[j] = files[i]; ++j; } @@ -183,7 +181,7 @@ sysshared_filter_real_files(char **files, i32 *file_count){ internal Partition sysshared_scratch_partition(i32 size){ - void *data = system_get_memory(size); + void *data = system_memory_allocate((umem)size); Partition part = make_part(data, size); return(part); } @@ -193,10 +191,11 @@ sysshared_partition_grow(Partition *part, i32 new_size){ void *data = 0; if (new_size > part->max){ // TODO(allen): attempt to grow in place by just acquiring next vpages?! - data = system_get_memory(new_size); + data = system_memory_allocate((umem)new_size); memcpy(data, part->base, part->pos); - system_free_memory(part->base); + system_memory_free(part->base, part->max); part->base = (char*)data; + part->max = new_size; } } diff --git a/4ed_system_shared.h b/4ed_system_shared.h index dd52d280..0347f017 100644 --- a/4ed_system_shared.h +++ b/4ed_system_shared.h @@ -24,18 +24,12 @@ struct File_Data{ }; global File_Data null_file_data = {0}; -#define Sys_Get_Memory_Sig(name) void* name(i32 size, i32 line_number, char *file_name) -#define Sys_Free_Memory_Sig(name) void name(void *block) -#define Sys_File_Can_Be_Made_Sig(name) b32 name(char *filename) +#define Sys_File_Can_Be_Made_Sig(name) b32 name(u8 *filename) #define Sys_Get_Binary_Path_Sig(name) i32 name(String *out) -internal Sys_Get_Memory_Sig(system_get_memory_); -internal Sys_Free_Memory_Sig(system_free_memory); internal Sys_File_Can_Be_Made_Sig(system_file_can_be_made); internal Sys_Get_Binary_Path_Sig(system_get_binary_path); -#define system_get_memory(size) system_get_memory_((size), __LINE__, __FILE__) - #endif // BOTTOM diff --git a/internal_4coder_tests.cpp b/internal_4coder_tests.cpp index 59184519..9e1935a9 100644 --- a/internal_4coder_tests.cpp +++ b/internal_4coder_tests.cpp @@ -47,7 +47,7 @@ CUSTOM_COMMAND_SIG(load_lots_of_files){ append_s_char(&str, '/'); int32_t size = str.size; - for (int32_t i = 0; i < list.count; ++i, ++info){ + for (uint32_t i = 0; i < list.count; ++i, ++info){ if (!info->folder){ append_ss(&str, make_string(info->filename, info->filename_len)); Buffer_Summary buffer = create_buffer(app, str.str, str.size, diff --git a/win32_4ed.cpp b/win32_4ed.cpp index 5fbc36e7..311f1f2b 100644 --- a/win32_4ed.cpp +++ b/win32_4ed.cpp @@ -71,7 +71,10 @@ ////////////////////////////// +#include "win32_utf8.h" + #include "4ed_file_track.h" +#include "font/4coder_font_interface_to_os.h" #include "4ed_system_shared.h" #include "win32_4ed_file_track.cpp" @@ -283,41 +286,6 @@ Sys_Memory_Free_Sig(system_memory_free){ VirtualFree(ptr, 0, MEM_RELEASE); } -// TODO(allen): delete -internal -Sys_Get_Memory_Sig(system_get_memory_){ - void *ptr = 0; - if (size > 0){ - ptr = VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); - } - return(ptr); -} - -// TODO(allen): delete -internal -Sys_Free_Memory_Sig(system_free_memory){ - if (block){ - VirtualFree(block, 0, MEM_RELEASE); - } -} - -#define Win32GetMemory(size) system_get_memory_(size, __LINE__, __FILE__) -#define Win32FreeMemory(ptr) system_free_memory(ptr) - -#define Win32ScratchPartition sysshared_scratch_partition -#define Win32ScratchPartitionGrow sysshared_partition_grow -#define Win32ScratchPartitionDouble sysshared_partition_double - -#if 0 -#if FRED_INTERNAL -internal void -INTERNAL_system_debug_message(char *message){ - OutputDebugStringW(message); -} -#endif -#endif - - // // Multithreading // @@ -358,7 +326,7 @@ JobThreadProc(LPVOID lpParameter){ if (thread_memory->size == 0){ i32 new_size = KB(64); - thread_memory->data = Win32GetMemory(new_size); + thread_memory->data = system_memory_allocate(new_size); thread_memory->size = new_size; } @@ -408,7 +376,7 @@ JobThreadProc(LPVOID lpParameter){ internal void initialize_unbounded_queue(Unbounded_Work_Queue *source_queue){ i32 max = 512; - source_queue->jobs = (Full_Job_Data*)system_get_memory(max*sizeof(Full_Job_Data)); + source_queue->jobs = (Full_Job_Data*)system_memory_allocate(max*sizeof(Full_Job_Data)); source_queue->count = 0; source_queue->max = max; source_queue->skip = 0; @@ -518,13 +486,12 @@ Sys_Post_Job_Sig(system_post_job){ u32 result = queue->next_job_id++; while (queue->count >= queue->max){ - i32 new_max = queue->max*2; - Full_Job_Data *new_jobs = (Full_Job_Data*) - system_get_memory(new_max*sizeof(Full_Job_Data)); + u32 new_max = queue->max*2; + Full_Job_Data *new_jobs = (Full_Job_Data*)system_memory_allocate(new_max*sizeof(Full_Job_Data)); memcpy(new_jobs, queue->jobs, queue->count); - system_free_memory(queue->jobs); + system_memory_free(queue->jobs, 0); queue->jobs = new_jobs; queue->max = new_max; @@ -614,11 +581,11 @@ Sys_Grow_Thread_Memory_Sig(system_grow_thread_memory){ void *old_data = memory->data; i32 old_size = memory->size; i32 new_size = l_round_up_i32(memory->size*2, KB(4)); - memory->data = system_get_memory(new_size); + memory->data = system_memory_allocate(new_size); memory->size = new_size; if (old_data){ memcpy(memory->data, old_data, old_size); - system_free_memory(old_data); + system_memory_free(old_data, 0); } system_release_lock(CANCEL_LOCK0 + memory->id - 1); } @@ -744,15 +711,13 @@ Sys_Yield_Coroutine_Sig(system_yield_coroutine){ internal Sys_File_Can_Be_Made_Sig(system_file_can_be_made){ - HANDLE file = CreateFile((char*)filename, FILE_APPEND_DATA, 0, 0, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0); - - if (!file || file == INVALID_HANDLE_VALUE){ - return 0; + HANDLE file = CreateFile_utf8(filename, FILE_APPEND_DATA, 0, 0, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0); + b32 result = false; + if (file != 0 && file != INVALID_HANDLE_VALUE){ + CloseHandle(file); + result = true; } - - CloseHandle(file); - - return(1); + return(result); } internal @@ -764,14 +729,14 @@ Sys_Set_File_List_Sig(system_set_file_list){ append_sc(&dir, directory); terminate_with_null(&dir); - HANDLE dir_handle = CreateFile(dir.str, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, 0); + HANDLE dir_handle = CreateFile_utf8((u8*)dir.str, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, 0); if (dir_handle != INVALID_HANDLE_VALUE){ - DWORD final_length = GetFinalPathNameByHandle(dir_handle, dir_space, sizeof(dir_space), 0); + DWORD final_length = GetFinalPathNameByHandle_utf8(dir_handle, (u8*)dir_space, sizeof(dir_space), 0); CloseHandle(dir_handle); if (final_length < sizeof(dir_space)){ - char *c_str_dir = dir_space; + u8 *c_str_dir = (u8*)dir_space; final_length -= 4; memmove(c_str_dir, c_str_dir+4, final_length); @@ -796,17 +761,18 @@ Sys_Set_File_List_Sig(system_set_file_list){ } WIN32_FIND_DATA find_data; - HANDLE search = FindFirstFile(c_str_dir, &find_data); + HANDLE search = FindFirstFile_utf8(c_str_dir, &find_data); if (search != INVALID_HANDLE_VALUE){ - i32 character_count = 0; - i32 file_count = 0; + u32 character_count = 0; + u32 file_count = 0; BOOL more_files = true; do{ - if (!match_cs(find_data.cFileName, make_lit_string(".")) && - !match_cs(find_data.cFileName, make_lit_string(".."))){ + b32 nav_dir = + (find_data.cFileName[0] == '.' && find_data.cFileName[1] == 0) ||(find_data.cFileName[0] == '.' && find_data.cFileName[1] == '.' && find_data.cFileName[2] == 0); + if (!nav_dir){ ++file_count; - i32 size = 0; + u32 size = 0; for(;find_data.cFileName[size];++size); character_count += size + 1; } @@ -814,41 +780,57 @@ Sys_Set_File_List_Sig(system_set_file_list){ }while(more_files); FindClose(search); - i32 required_size = character_count + file_count * sizeof(File_Info); + u32 remaining_size = character_count*2; + u32 required_size = remaining_size + file_count*sizeof(File_Info); if (file_list->block_size < required_size){ - system_free_memory(file_list->block); - file_list->block = system_get_memory(required_size); + system_memory_free(file_list->block, 0); + file_list->block = system_memory_allocate(required_size); file_list->block_size = required_size; } file_list->infos = (File_Info*)file_list->block; - char *name = (char*)(file_list->infos + file_count); + u8 *name = (u8*)(file_list->infos + file_count); + u32 corrected_file_count = 0; if (file_list->block != 0){ - search = FindFirstFile(c_str_dir, &find_data); + search = FindFirstFile_utf8(c_str_dir, &find_data); if (search != INVALID_HANDLE_VALUE){ File_Info *info = file_list->infos; more_files = true; do{ - if (!match_cs(find_data.cFileName, make_lit_string(".")) && - !match_cs(find_data.cFileName, make_lit_string(".."))){ + b32 nav_dir = + (find_data.cFileName[0] == '.' && find_data.cFileName[1] == 0) ||(find_data.cFileName[0] == '.' && find_data.cFileName[1] == '.' && find_data.cFileName[2] == 0); + + if (!nav_dir){ info->folder = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; - info->filename = name; + info->filename = (char*)name; - i32 length = copy_fast_unsafe_cc(name, find_data.cFileName); - name += length; + u16 *src = (u16*)find_data.cFileName; + u32 src_len = 0; + for (;src[src_len];++src_len); - info->filename_len = length; - *name++ = 0; - String fname = make_string_cap(info->filename, info->filename_len, info->filename_len+1); - replace_char(&fname, '\\', '/'); - ++info; + u8 *dst = name; + u32 max = remaining_size-1; + + b32 error = false; + u32 length = (u32)utf16_to_utf8_minimal_checking(dst, max, src, src_len, &error); + + if (length <= max && !error){ + name += length; + + info->filename_len = length; + *name++ = 0; + String fname = make_string_cap(info->filename, length, length+1); + replace_char(&fname, '\\', '/'); + ++info; + ++corrected_file_count; + } } more_files = FindNextFile(search, &find_data); }while(more_files); FindClose(search); - file_list->count = file_count; + file_list->count = corrected_file_count; clear_list = false; } } @@ -858,7 +840,7 @@ Sys_Set_File_List_Sig(system_set_file_list){ } if (clear_list){ - Win32FreeMemory(file_list->block); + system_memory_free(file_list->block, 0); file_list->block = 0; file_list->block_size = 0; file_list->infos = 0; @@ -875,10 +857,10 @@ win32_canonical_ascii_name(char *src, u32 len, char *dst, u32 max){ memcpy(src_space, src, len); src_space[len] = 0; - HANDLE file = CreateFile(src_space, GENERIC_READ, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + HANDLE file = CreateFile_utf8((u8*)src_space, GENERIC_READ, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); if (file != INVALID_HANDLE_VALUE){ - DWORD final_length = GetFinalPathNameByHandle(file, dst, max, 0); + DWORD final_length = GetFinalPathNameByHandle_utf8(file, (u8*)dst, max, 0); if (final_length < max && final_length >= 4){ if (dst[final_length-1] == 0){ @@ -900,10 +882,10 @@ win32_canonical_ascii_name(char *src, u32 len, char *dst, u32 max){ memcpy(src_space, path_str.str, path_str.size); src_space[path_str.size] = 0; - HANDLE dir = CreateFile(src_space, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, 0); + HANDLE dir = CreateFile_utf8((u8*)src_space, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, 0); if (dir != INVALID_HANDLE_VALUE){ - DWORD final_length = GetFinalPathNameByHandle(dir, dst, max, 0); + DWORD final_length = GetFinalPathNameByHandle_utf8(dir, (u8*)dst, max, 0); if (final_length < max && final_length >= 4){ if (dst[final_length-1] == 0){ @@ -934,12 +916,12 @@ Sys_Get_Canonical_Sig(system_get_canonical){ internal Sys_Load_Handle_Sig(system_load_handle){ - b32 result = 0; - HANDLE file = CreateFile(filename, GENERIC_READ, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + b32 result = false; + HANDLE file = CreateFile_utf8((u8*)filename, GENERIC_READ, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); if (file != INVALID_HANDLE_VALUE){ *(HANDLE*)handle_out = file; - result = 1; + result = true; } return(result); @@ -988,14 +970,14 @@ Sys_Load_Close_Sig(system_load_close){ internal Sys_Save_File_Sig(system_save_file){ - b32 result = 0; - HANDLE file = CreateFile(filename, GENERIC_WRITE, 0, 0, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0); + b32 result = false; + HANDLE file = CreateFile_utf8((u8*)filename, GENERIC_WRITE, 0, 0, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0); if (file != INVALID_HANDLE_VALUE){ DWORD written_total = 0; DWORD written_size = 0; - result = 1; + result = true; while (written_total < size){ if (!WriteFile(file, buffer + written_total, size - written_total, &written_size, 0)){ @@ -1019,14 +1001,14 @@ Sys_Now_Time_Sig(system_now_time){ internal b32 Win32DirectoryExists(char *path){ - DWORD attrib = GetFileAttributes(path); - return (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_DIRECTORY)); + DWORD attrib = GetFileAttributes_utf8((u8*)path); + return(attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_DIRECTORY)); } internal Sys_Get_Binary_Path_Sig(system_get_binary_path){ i32 result = 0; - i32 size = GetModuleFileName(0, out->str, out->memory_size); + i32 size = GetModuleFileName_utf8(0, (u8*)out->str, out->memory_size); if (size < out->memory_size-1){ out->size = size; remove_last_folder(out); @@ -1048,8 +1030,7 @@ Sys_File_Exists_Sig(system_file_exists){ copy_ss(&full_filename, make_string(filename, len)); terminate_with_null(&full_filename); - file = CreateFile(full_filename.str, GENERIC_READ, 0, 0, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + file = CreateFile_utf8((u8*)full_filename.str, GENERIC_READ, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); if (file != INVALID_HANDLE_VALUE){ CloseHandle(file); @@ -1223,10 +1204,7 @@ Sys_CLI_Call_Sig(system_cli_call){ PROCESS_INFORMATION info = {}; Assert(sizeof(Plat_Handle) >= sizeof(HANDLE)); - if (CreateProcess(cmd, command_line, - 0, 0, TRUE, 0, - env_variables, path, - &startup, &info)){ + if (CreateProcess_utf8((u8*)cmd, (u8*)command_line, 0, 0, TRUE, 0, env_variables, (u8*)path, &startup, &info)){ success = 1; CloseHandle(info.hThread); *(HANDLE*)&cli_out->proc = info.hProcess; @@ -2023,7 +2001,7 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS memory_vars.user_memory = VirtualAlloc(base, memory_vars.target_memory_size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); win32vars.target.max = MB(1); - win32vars.target.push_buffer = (char*)system_get_memory(win32vars.target.max); + win32vars.target.push_buffer = (char*)system_memory_allocate(win32vars.target.max); if (memory_vars.vars_memory == 0 || memory_vars.target_memory == 0 || memory_vars.user_memory == 0 || win32vars.target.push_buffer == 0){ exit(1); @@ -2051,8 +2029,8 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS // DWORD required = (GetCurrentDirectory(0, 0)*4) + 1; - char *current_directory_mem = (char*)system_get_memory(required); - DWORD written = GetCurrentDirectory(required, current_directory_mem); + u8 *current_directory_mem = (u8*)system_memory_allocate(required); + DWORD written = GetCurrentDirectory_utf8(required, current_directory_mem); String current_directory = make_string_cap(current_directory_mem, written, required); terminate_with_null(¤t_directory); @@ -2092,15 +2070,14 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS if (win32vars.custom_api.get_alpha_4coder_version == 0 || win32vars.custom_api.get_alpha_4coder_version(MAJOR, MINOR, PATCH) == 0){ - MessageBox(0,"Error: The application and custom version numbers don't match.\n", "Error",0); + MessageBox_utf8(0, (u8*)"Error: The application and custom version numbers don't match.\n", (u8*)"Error",0); exit(1); } - win32vars.custom_api.get_bindings = (Get_Binding_Data_Function*) - GetProcAddress(win32vars.custom, "get_bindings"); + win32vars.custom_api.get_bindings = (Get_Binding_Data_Function*)GetProcAddress(win32vars.custom, "get_bindings"); } if (win32vars.custom_api.get_bindings == 0){ - MessageBox(0,"Error: The custom dll is missing.\n", "Error",0); + MessageBox_utf8(0, (u8*)"Error: The custom dll is missing.\n", (u8*)"Error", 0); exit(1); } @@ -2116,8 +2093,8 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS window_class.style = CS_HREDRAW|CS_VREDRAW; window_class.lpfnWndProc = (WNDPROC)(Win32Callback); window_class.hInstance = hInstance; - window_class.lpszClassName = "4coder-win32-wndclass"; - window_class.hIcon = LoadIcon(hInstance, "main"); + window_class.lpszClassName = L"4coder-win32-wndclass"; + window_class.hIcon = LoadIcon(hInstance, L"main"); if (!RegisterClass(&window_class)){ exit(1); @@ -2138,8 +2115,6 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS // TODO(allen): non-fatal diagnostics } -#define WINDOW_NAME "4coder-window: " VERSION - i32 window_x = CW_USEDEFAULT; i32 window_y = CW_USEDEFAULT; @@ -2153,6 +2128,8 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS window_style |= WS_MAXIMIZE; } +#define WINDOW_NAME L"4coder-window: " L_VERSION + win32vars.window_handle = CreateWindow(window_class.lpszClassName, WINDOW_NAME, window_style, window_x, window_y, window_rect.right - window_rect.left, window_rect.bottom - window_rect.top, 0, 0, hInstance, 0); if (win32vars.window_handle == 0){ @@ -2227,7 +2204,7 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS win32vars.app.init(&win32vars.system, &win32vars.target, &memory_vars, win32vars.clipboard_contents, current_directory, win32vars.custom_api); - system_free_memory(current_directory.str); + system_memory_free(current_directory.str, 0); b32 keep_playing = 1; win32vars.first = 1; @@ -2468,6 +2445,7 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS } #include "font/4coder_font_static_functions.cpp" +#include "win32_utf8.cpp" #if 0 // NOTE(allen): In case I want to switch back to a console diff --git a/win32_4ed_file_track.cpp b/win32_4ed_file_track.cpp index 683663ac..d48e9f8f 100644 --- a/win32_4ed_file_track.cpp +++ b/win32_4ed_file_track.cpp @@ -81,20 +81,19 @@ init_track_system(File_Track_System *system, Partition *scratch, void *table_mem return(result); } -internal umem -internal_utf8_file_to_utf16_parent(u16 *out, u32 max, u8 *name){ +internal i32 +internal_get_parent_name(u8 *out, i32 max, u8 *name){ u8 *ptr = name; for (; *ptr != 0; ++ptr); - umem len = (umem)(ptr - name); + i32 len = (i32)(ptr - name); // TODO(allen): make this system real Assert(len < max); - umem slash_i = len-1; + i32 slash_i = len-1; for (;slash_i > 0 && name[slash_i] != '\\' && name[slash_i] != '/';--slash_i); - b32 error = false; - slash_i = utf8_to_utf16_minimal_checking(out, max-1, name, len, &error); + for (i32 i = 0; i < slash_i; ++i) out[i] = name[i]; out[slash_i] = 0; return(slash_i); @@ -130,11 +129,10 @@ add_listener(File_Track_System *system, Partition *scratch, u8 *filename){ { File_Track_Tables *tables = to_tables(vars); - // TODO(allen): make this real! - u16 dir_name[1024]; - internal_utf8_file_to_utf16_parent(dir_name, ArrayCount(dir_name), filename); + u8 dir_name[1024]; + internal_get_parent_name(dir_name, ArrayCount(dir_name), filename); - HANDLE dir = CreateFile((LPCWSTR)dir_name, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, 0); + HANDLE dir = CreateFile_utf8(dir_name, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, 0); if (dir != INVALID_HANDLE_VALUE){ BY_HANDLE_FILE_INFORMATION dir_info = {0}; @@ -213,10 +211,10 @@ remove_listener(File_Track_System *system, Partition *scratch, u8 *filename){ File_Track_Tables *tables = to_tables(vars); // TODO(allen): make this real! - u16 dir_name[1024]; - internal_utf8_file_to_utf16_parent(dir_name, ArrayCount(dir_name), filename); + u8 dir_name[1024]; + internal_get_parent_name(dir_name, ArrayCount(dir_name), filename); - HANDLE dir = CreateFile((LPCWSTR)dir_name, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, 0); + HANDLE dir = CreateFile_utf8(dir_name, FILE_LIST_DIRECTORY, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, 0); if (dir != INVALID_HANDLE_VALUE){ BY_HANDLE_FILE_INFORMATION dir_info = {0}; @@ -332,12 +330,14 @@ get_change_event(File_Track_System *system, Partition *scratch, u8 *buffer, i32 FILE_NOTIFY_INFORMATION *info = (FILE_NOTIFY_INFORMATION*)(listener.result + offset); i32 len = info->FileNameLength / 2; - i32 dir_len = GetFinalPathNameByHandle(listener.dir, 0, 0, FILE_NAME_NORMALIZED); + i32 dir_len = GetFinalPathNameByHandle_utf8(listener.dir, 0, 0, FILE_NAME_NORMALIZED); i32 req_size = dir_len + 1 + len; *size = req_size; + + // TODO(allen): This check isn't really right, it should rely on the result from GetFinalPathNameByHandle_utf8. if (req_size < max){ - i32 pos = GetFinalPathNameByHandle(listener.dir, buffer, max, FILE_NAME_NORMALIZED); + i32 pos = GetFinalPathNameByHandle_utf8(listener.dir, buffer, max, FILE_NAME_NORMALIZED); buffer[pos++] = '\\'; for (i32 i = 0; i < len; ++i, ++pos){ diff --git a/win32_utf8.cpp b/win32_utf8.cpp new file mode 100644 index 00000000..a11d963f --- /dev/null +++ b/win32_utf8.cpp @@ -0,0 +1,258 @@ +/* + * Mr. 4th Dimention - Allen Webster + * + * 23.03.2017 + * + * UTF8 versions of WIN32 calls. + * + */ + +// TOP + +#if !defined(FRED_WIN32_UTF8_CPP) +#define FRED_WIN32_UTF8_CPP + +internal HANDLE +CreateFile_utf8(u8 *name, DWORD access, DWORD share, LPSECURITY_ATTRIBUTES security, DWORD creation, DWORD flags, HANDLE template_file){ + HANDLE result = INVALID_HANDLE_VALUE; + + Partition *scratch = &shared_vars.scratch; + Temp_Memory temp = begin_temp_memory(scratch); + + u32 len = 0; + for (;name[len];++len); + u32 name_16_max = (len+1)*2; + u16 *name_16 = push_array(scratch, u16, name_16_max); + + b32 convert_error = false; + u32 name_16_len = (u32)utf8_to_utf16_minimal_checking(name_16, name_16_max-1, name, len, &convert_error); + + if (!convert_error){ + name_16[name_16_len] = 0; + result = CreateFileW((LPWSTR)name_16, access, share, security, creation, flags, template_file); + } + + end_temp_memory(temp); + + return(result); +} + +internal DWORD +GetFinalPathNameByHandle_utf8(HANDLE file, u8 *file_path_out, DWORD path_max, DWORD flags){ + DWORD result = 0; + + if (file_path_out == 0){ + result = GetFinalPathNameByHandleW(file, 0, 0, flags); + } + else{ + Partition *scratch = &shared_vars.scratch; + Temp_Memory temp = begin_temp_memory(scratch); + + u32 path_16_max = KB(32); + u16 *path_16 = push_array(scratch, u16, path_16_max); + + DWORD length_16 = GetFinalPathNameByHandleW(file, (LPWSTR)path_16, path_16_max, flags); + + if (length_16 != 0 && length_16 < path_16_max){ + b32 convert_error = false; + u32 path_8_len = (u32)utf16_to_utf8_minimal_checking(file_path_out, path_max-1, path_16, length_16, &convert_error); + if (path_8_len < path_max && !convert_error){ + file_path_out[path_8_len] = 0; + result = path_8_len; + } + } + + end_temp_memory(temp); + } + + return(result); +} + +internal HANDLE +FindFirstFile_utf8(u8 *name, LPWIN32_FIND_DATA find_data){ + HANDLE result = INVALID_HANDLE_VALUE; + + Partition *scratch = &shared_vars.scratch; + Temp_Memory temp = begin_temp_memory(scratch); + + u32 len = 0; + for (;name[len];++len); + u32 name_16_max = (len+1)*2; + u16 *name_16 = push_array(scratch, u16, name_16_max); + + b32 convert_error = false; + u32 name_16_len = (u32)utf8_to_utf16_minimal_checking(name_16, name_16_max-1, name, len, &convert_error); + + if (name_16_len < name_16_max && !convert_error){ + name_16[name_16_len] = 0; + result = FindFirstFileW((LPWSTR)name_16, find_data); + } + + end_temp_memory(temp); + + return(result); +} + +internal DWORD +GetFileAttributes_utf8(u8 *name){ + DWORD result = 0; + + Partition *scratch = &shared_vars.scratch; + Temp_Memory temp = begin_temp_memory(scratch); + + u32 len = 0; + for (;name[len];++len); + u32 name_16_max = (len+1)*2; + u16 *name_16 = push_array(scratch, u16, name_16_max); + + b32 convert_error = false; + u32 name_16_len = (u32)utf8_to_utf16_minimal_checking(name_16, name_16_max-1, name, len, &convert_error); + + if (name_16_len < name_16_max && !convert_error){ + name_16[name_16_len] = 0; + result = GetFileAttributesW((LPWSTR)name_16); + } + + end_temp_memory(temp); + + return(result); +} + +internal DWORD +GetModuleFileName_utf8(HMODULE module, u8 *file_out, DWORD max){ + DWORD result = 0; + + Partition *scratch = &shared_vars.scratch; + Temp_Memory temp = begin_temp_memory(scratch); + + u32 file_16_max = KB(40); + u16 *file_16 = push_array(scratch, u16, file_16_max); + + DWORD file_16_len = GetModuleFileNameW(module, (LPWSTR)file_16, file_16_max); + + if (max > 0){ + b32 convert_error = false; + u32 file_8_len = (u32)utf16_to_utf8_minimal_checking(file_out, max-1, file_16, file_16_len, &convert_error); + result = file_8_len; + if (convert_error || file_8_len >= max){ + result = 0; + } + } + + end_temp_memory(temp); + + return(result); +} + +internal BOOL +CreateProcess_utf8(u8 *app_name, u8 *command, LPSECURITY_ATTRIBUTES security, LPSECURITY_ATTRIBUTES thread, BOOL inherit_handles, DWORD creation, LPVOID environment, u8 *curdir, LPSTARTUPINFO startup, LPPROCESS_INFORMATION process){ + BOOL result = false; + + Partition *scratch = &shared_vars.scratch; + Temp_Memory temp = begin_temp_memory(scratch); + + u32 app_name_len = 0; + for (;app_name[app_name_len];++app_name_len); + + u32 command_len = 0; + for (;command[command_len];++command_len); + + u32 curdir_len = 0; + for (;curdir[curdir_len];++curdir_len); + + u32 app_name_16_max = (app_name_len+1)*2; + u32 command_16_max = (command_len+1)*2; + u32 curdir_16_max = (curdir_len+1)*2; + + u16 *app_name_16 = push_array(scratch, u16, app_name_16_max); + u16 *command_16 = push_array(scratch, u16, command_16_max); + u16 *curdir_16 = push_array(scratch, u16, curdir_16_max); + + b32 error = false; + u32 app_name_16_len = (u32)utf8_to_utf16_minimal_checking(app_name_16, app_name_16_max-1, app_name, app_name_len, &error); + + if (app_name_16_len < app_name_16_max && !error){ + u32 command_16_len = (u32)utf8_to_utf16_minimal_checking(command_16, command_16_max-1, command, command_len, &error); + + if (command_16_len < command_16_max && !error){ + u32 curdir_16_len = (u32)utf8_to_utf16_minimal_checking(curdir_16, curdir_16_max-1, curdir, curdir_len, &error); + + app_name_16[app_name_16_len] = 0; + command_16[command_16_len] = 0; + curdir_16[curdir_16_len] = 0; + + result = CreateProcessW((LPWSTR)app_name_16, (LPWSTR)command_16, security, thread, inherit_handles, creation, environment, (LPWSTR)curdir_16, startup, process); + } + } + + end_temp_memory(temp); + + return(result); +} + +internal DWORD +GetCurrentDirectory_utf8(DWORD max, u8 *buffer){ + DWORD result = 0; + + Partition *scratch = &shared_vars.scratch; + Temp_Memory temp = begin_temp_memory(scratch); + + u32 buffer_16_max = KB(40); + u16 *buffer_16 = push_array(scratch, u16, buffer_16_max); + + DWORD buffer_16_len = GetCurrentDirectoryW(buffer_16_max, (LPWSTR)buffer_16); + + b32 error = false; + u32 buffer_8_len = (u32)utf16_to_utf8_minimal_checking(buffer, max-1, buffer_16, buffer_16_len, &error); + + if (buffer_8_len < max && !error){ + buffer[buffer_8_len] = 0; + result = buffer_8_len; + } + + end_temp_memory(temp); + + return(result); +} + +internal int +MessageBox_utf8(HWND owner, u8 *text, u8 *caption, UINT type){ + int result = 0; + + Partition *scratch = &shared_vars.scratch; + Temp_Memory temp = begin_temp_memory(scratch); + + u32 text_len = 0; + for(;text[text_len];++text_len); + + u32 caption_len = 0; + for(;caption[caption_len];++caption_len); + + u32 text_16_max = (text_len+1)*2; + u32 caption_16_max = (caption_len+1)*2; + + u16 *text_16 = push_array(scratch, u16, text_16_max); + u16 *caption_16 = push_array(scratch, u16, caption_16_max); + + b32 error = false; + u32 text_16_len = (u32)utf8_to_utf16_minimal_checking(text_16, text_16_max-1, text, text_len, &error); + + if (text_16_len < text_16_max && !error){ + u32 caption_16_len = (u32)utf8_to_utf16_minimal_checking(caption_16, caption_16_max-1, caption, caption_len, &error); + + if (text_16_len < text_16_max && !error){ + text_16[text_16_len] = 0; + caption_16[caption_16_len] = 0; + MessageBoxW(owner, (LPWSTR)text_16, (LPWSTR)caption_16, type); + } + } + + end_temp_memory(temp); + + return(result); +} + +#endif + +// BOTTOM + diff --git a/win32_utf8.h b/win32_utf8.h new file mode 100644 index 00000000..b94d663e --- /dev/null +++ b/win32_utf8.h @@ -0,0 +1,42 @@ +/* + * Mr. 4th Dimention - Allen Webster + * + * 23.03.2017 + * + * UTF8 versions of WIN32 calls. + * + */ + +// TOP + +#if !defined(FRED_WIN32_UTF8_H) +#define FRED_WIN32_UTF8_H + +internal HANDLE +CreateFile_utf8(u8 *name, DWORD access, DWORD share, LPSECURITY_ATTRIBUTES security, DWORD creation, DWORD flags, HANDLE template_file); + +internal DWORD +GetFinalPathNameByHandle_utf8(HANDLE file, u8 *file_path_out, DWORD path_max, DWORD flags); + +internal HANDLE +FindFirstFile_utf8(u8 *name, LPWIN32_FIND_DATA find_data); + +internal DWORD +GetFileAttributes_utf8(u8 *name); + +internal DWORD +GetModuleFileName_utf8(HMODULE module, u8 *file_out, DWORD max); + +internal BOOL +CreateProcess_utf8(u8 *app_name, u8 *command, LPSECURITY_ATTRIBUTES security, LPSECURITY_ATTRIBUTES thread, BOOL inherit_handles, DWORD creation, LPVOID environment, u8 *curdir, LPSTARTUPINFO startup, LPPROCESS_INFORMATION process); + +internal DWORD +GetCurrentDirectory_utf8(DWORD max, u8 *buffer); + +internal int +MessageBox_utf8(HWND owner, u8 *text, u8 *caption, UINT type); + +#endif + +// BOTTOM +