From 142fc723570c2f341db7b469f75c1a0b2e376307 Mon Sep 17 00:00:00 2001 From: Peter Slattery Date: Tue, 12 May 2020 18:23:54 -0700 Subject: [PATCH] Created gs_profiler, began using it in gs_meta --- build/build_app_msvc_win32_debug.bat | 5 +- build/build_meta_msvc_win32_debug.bat | 4 +- src/app/generated/foldhaus_nodes_generated.h | 18 - src/app/generated/foldhaus_panels_generated.h | 16 +- .../generated/gs_meta_generated_typeinfo.h | 86 +---- src/gs_libs/gs_language.h | 38 +- src/gs_libs/gs_memory_arena.h | 149 +++++--- src/gs_libs/gs_profiler.h | 258 +++++++++++++ src/gs_libs/gs_radix_sort.h | 28 +- src/meta/foldhaus_meta.cpp | 50 ++- src/meta/gs_meta.cpp | 348 +++++++++++------- src/meta/gs_meta_code_generator.h | 8 +- src/meta/gs_meta_error.h | 7 +- src/meta/gs_meta_type_table.h | 40 +- src/meta/gs_meta_typeinfo_generator.h | 24 +- src/todo.txt | 22 ++ 16 files changed, 704 insertions(+), 397 deletions(-) create mode 100644 src/gs_libs/gs_profiler.h diff --git a/build/build_app_msvc_win32_debug.bat b/build/build_app_msvc_win32_debug.bat index e38e31d..d29b0b1 100644 --- a/build/build_app_msvc_win32_debug.bat +++ b/build/build_app_msvc_win32_debug.bat @@ -4,9 +4,8 @@ SET MyPath=%~dp0 SET MyPath=%MyPath:~0,-1% call %MyPath%\_prebuild_win32.bat app debug msvc -set CommonCompilerFlags=-nologo -DDEBUG=1 -DPLATFORM_WINDOWS -FC -WX -W4 -Z7 -Oi -GR- -EHsc -EHa- -MTd -fp:fast -fp:except- -IC:\programs-dev\gs_libs\src -set CommonCompilerFlags=-wd4127 -wd4702 -wd4101 -wd4505 -wd4100 -wd4189 -wd4244 -wd4201 -wd4996 -I%CommonLibs% -O2 %CommonCompilerFlags% - +set CommonCompilerFlags=-nologo -DDEBUG=1 -DPLATFORM_WINDOWS -FC -WX -W4 -Z7 -Oi -GR- -EHsc -EHa- -MTd -fp:fast -fp:except- +set CommonCompilerFlags=-wd4127 -wd4702 -wd4101 -wd4505 -wd4100 -wd4189 -wd4244 -wd4201 -wd4996 -O2 %CommonCompilerFlags% set CommonLinkerFlags= -opt:ref -incremental:no set DLLExports=/EXPORT:InitializeApplication /EXPORT:UpdateAndRender /EXPORT:CleanupApplication /EXPORT:ReloadStaticData diff --git a/build/build_meta_msvc_win32_debug.bat b/build/build_meta_msvc_win32_debug.bat index a47b2cb..a57de1b 100644 --- a/build/build_meta_msvc_win32_debug.bat +++ b/build/build_meta_msvc_win32_debug.bat @@ -5,8 +5,8 @@ SET MyPath=%MyPath:~0,-1% call %MyPath%\_prebuild_win32.bat meta debug msvc set CommonCompilerFlags=-nologo -DDEBUG=1 -DPLATFORM_WINDOWS -FC -WX -W4 -Z7 -Oi -GR- -EHsc -EHa- -MTd -fp:fast -fp:except- -set CommonCompilerFlags=-wd4127 -wd4702 -wd4101 -wd4505 -wd4100 -wd4189 -wd4244 -wd4201 -wd4996 -I%CommonLibs% -O2 %CommonCompilerFlags% -set CommonLinkerFlags= -opt:ref +set CommonCompilerFlags=-wd4127 -wd4702 -wd4101 -wd4505 -wd4100 -wd4189 -wd4244 -wd4201 -wd4996 -Od %CommonCompilerFlags% +set CommonLinkerFlags= -opt:ref -incremental:no pushd %BuildPath% diff --git a/src/app/generated/foldhaus_nodes_generated.h b/src/app/generated/foldhaus_nodes_generated.h index 8e9f0f8..bc4eea3 100644 --- a/src/app/generated/foldhaus_nodes_generated.h +++ b/src/app/generated/foldhaus_nodes_generated.h @@ -1,32 +1,14 @@ enum node_type { - NodeType_SolidColorProc, - NodeType_RevolvingDiscs, - NodeType_VerticalColorFadeProc, NodeType_Count, }; static node_specification_ NodeSpecifications[] = { -{ NodeType_SolidColorProc, {"SolidColorProc", 14}, gsm_StructType_solid_color_data }, -{ NodeType_RevolvingDiscs, {"RevolvingDiscs", 14}, gsm_StructType_revolving_discs_data }, -{ NodeType_VerticalColorFadeProc, {"VerticalColorFadeProc", 21}, gsm_StructType_vertical_color_fade_data }, }; void CallNodeProc(node_type Type, u8* NodeData) { switch(Type) { - case NodeType_SolidColorProc: - { - SolidColorProc((solid_color_data*)NodeData); - } break; - case NodeType_RevolvingDiscs: - { - RevolvingDiscs((revolving_discs_data*)NodeData); - } break; - case NodeType_VerticalColorFadeProc: - { - VerticalColorFadeProc((vertical_color_fade_data*)NodeData); - } break; } } diff --git a/src/app/generated/foldhaus_panels_generated.h b/src/app/generated/foldhaus_panels_generated.h index 48610ca..1c7d62c 100644 --- a/src/app/generated/foldhaus_panels_generated.h +++ b/src/app/generated/foldhaus_panels_generated.h @@ -1,19 +1,5 @@ enum panel_type { -PanelType_FileView, -PanelType_SculptureView, -PanelType_AnimationTimeline, -PanelType_DMXView, -PanelType_HierarchyView, -PanelType_NodeGraph, -PanelType_ProfilerView, }; -global_variable s32 GlobalPanelDefsCount = 7; +global_variable s32 GlobalPanelDefsCount = 0; global_variable panel_definition GlobalPanelDefs[] = { -{ "File View", 9, FileView_Init, FileView_Cleanup, FileView_Render, FileView_Commands, FileView_CommandsCount }, -{ "Sculpture View", 14, SculptureView_Init, SculptureView_Cleanup, SculptureView_Render, SculptureView_Commands, SculptureView_CommandsCount }, -{ "Animation Timeline", 18, AnimationTimeline_Init, AnimationTimeline_Cleanup, AnimationTimeline_Render, AnimationTimeline_Commands, AnimationTimeline_CommandsCount }, -{ "Dmx View", 8, DMXView_Init, DMXView_Cleanup, DMXView_Render, DMXView_Commands, DMXView_CommandsCount }, -{ "Hierarchy", 9, HierarchyView_Init, HierarchyView_Cleanup, HierarchyView_Render, HierarchyView_Commands, HierarchyView_CommandsCount }, -{ "Node Graph", 10, NodeGraph_Init, NodeGraph_Cleanup, NodeGraph_Render, NodeGraph_Commands, NodeGraph_CommandsCount }, -{ "Profiler", 8, ProfilerView_Init, ProfilerView_Cleanup, ProfilerView_Render, ProfilerView_Commands, ProfilerView_CommandsCount }, }; diff --git a/src/app/generated/gs_meta_generated_typeinfo.h b/src/app/generated/gs_meta_generated_typeinfo.h index e5cc3e0..a96c209 100644 --- a/src/app/generated/gs_meta_generated_typeinfo.h +++ b/src/app/generated/gs_meta_generated_typeinfo.h @@ -35,94 +35,10 @@ gsm_meta_tag MetaTagStrings[] = { }; enum gsm_struct_type { - gsm_StructType_solid_color_data, - gsm_StructType_v4, - gsm_StructType_float, - gsm_StructType_color_buffer, - gsm_StructType_led, - gsm_StructType_s32, - gsm_StructType_pixel, - gsm_StructType_u8, - gsm_StructType_sin_wave_data, - gsm_StructType_r32, - gsm_StructType_revolving_discs_data, - gsm_StructType_vertical_color_fade_data, - gsm_StructType_multiply_patterns_data, gsm_StructType_Count, }; -static gsm_struct_member_type_info StructMembers_v4[] = { -{ "x", 1, (u64)&((v4*)0)->x, {}, 0}, -{ "y", 1, (u64)&((v4*)0)->y, {}, 0}, -{ "z", 1, (u64)&((v4*)0)->z, {}, 0}, -{ "w", 1, (u64)&((v4*)0)->w, {}, 0}, -{ "r", 1, (u64)&((v4*)0)->r, {}, 0}, -{ "g", 1, (u64)&((v4*)0)->g, {}, 0}, -{ "b", 1, (u64)&((v4*)0)->b, {}, 0}, -{ "a", 1, (u64)&((v4*)0)->a, {}, 0}, -{ "E", 1, (u64)&((v4*)0)->E, {}, 0}, -}; -static gsm_struct_member_type_info StructMembers_led[] = { -{ "Index", 5, (u64)&((led*)0)->Index, {}, 0}, -{ "Position", 8, (u64)&((led*)0)->Position, {}, 0}, -}; -static gsm_struct_member_type_info StructMembers_pixel[] = { -{ "R", 1, (u64)&((pixel*)0)->R, {}, 0}, -{ "G", 1, (u64)&((pixel*)0)->G, {}, 0}, -{ "B", 1, (u64)&((pixel*)0)->B, {}, 0}, -{ "Channels", 8, (u64)&((pixel*)0)->Channels, {}, 0}, -}; -static gsm_struct_member_type_info StructMembers_color_buffer[] = { -{ "LEDs", 4, (u64)&((color_buffer*)0)->LEDs, {}, 0}, -{ "Colors", 6, (u64)&((color_buffer*)0)->Colors, {}, 0}, -{ "LEDCount", 8, (u64)&((color_buffer*)0)->LEDCount, {}, 0}, -}; -static gsm_struct_member_type_info StructMembers_solid_color_data[] = { -{ "Color", 5, (u64)&((solid_color_data*)0)->Color, {MetaTag_node_input, }, 1}, -{ "Result", 6, (u64)&((solid_color_data*)0)->Result, {MetaTag_node_output, }, 1}, -}; -static gsm_struct_member_type_info StructMembers_sin_wave_data[] = { -{ "Period", 6, (u64)&((sin_wave_data*)0)->Period, {MetaTag_node_input, }, 1}, -{ "Min", 3, (u64)&((sin_wave_data*)0)->Min, {MetaTag_node_input, }, 1}, -{ "Max", 3, (u64)&((sin_wave_data*)0)->Max, {MetaTag_node_input, }, 1}, -{ "Result", 6, (u64)&((sin_wave_data*)0)->Result, {MetaTag_node_input, }, 1}, -{ "Accumulator", 11, (u64)&((sin_wave_data*)0)->Accumulator, {}, 0}, -}; -static gsm_struct_member_type_info StructMembers_revolving_discs_data[] = { -{ "Rotation", 8, (u64)&((revolving_discs_data*)0)->Rotation, {MetaTag_node_input, }, 1}, -{ "ThetaZ", 6, (u64)&((revolving_discs_data*)0)->ThetaZ, {MetaTag_node_input, }, 1}, -{ "ThetaY", 6, (u64)&((revolving_discs_data*)0)->ThetaY, {MetaTag_node_input, }, 1}, -{ "DiscWidth", 9, (u64)&((revolving_discs_data*)0)->DiscWidth, {MetaTag_node_input, }, 1}, -{ "InnerRadius", 11, (u64)&((revolving_discs_data*)0)->InnerRadius, {MetaTag_node_input, }, 1}, -{ "OuterRadius", 11, (u64)&((revolving_discs_data*)0)->OuterRadius, {MetaTag_node_input, }, 1}, -{ "Color", 5, (u64)&((revolving_discs_data*)0)->Color, {MetaTag_node_input, }, 1}, -{ "Result", 6, (u64)&((revolving_discs_data*)0)->Result, {MetaTag_node_output, }, 1}, -}; -static gsm_struct_member_type_info StructMembers_vertical_color_fade_data[] = { -{ "Color", 5, (u64)&((vertical_color_fade_data*)0)->Color, {MetaTag_node_input, }, 1}, -{ "Min", 3, (u64)&((vertical_color_fade_data*)0)->Min, {MetaTag_node_input, }, 1}, -{ "Max", 3, (u64)&((vertical_color_fade_data*)0)->Max, {MetaTag_node_input, }, 1}, -{ "Result", 6, (u64)&((vertical_color_fade_data*)0)->Result, {MetaTag_node_output, }, 1}, -}; -static gsm_struct_member_type_info StructMembers_multiply_patterns_data[] = { -{ "A", 1, (u64)&((multiply_patterns_data*)0)->A, {MetaTag_node_input, }, 1}, -{ "B", 1, (u64)&((multiply_patterns_data*)0)->B, {MetaTag_node_input, }, 1}, -{ "Result", 6, (u64)&((multiply_patterns_data*)0)->Result, {MetaTag_node_output, }, 1}, -}; static gsm_struct_type_info StructTypes[] = { -{ gsm_StructType_solid_color_data, "solid_color_data", 16, 36, 0, 0, StructMembers_solid_color_data, 2 }, -{ gsm_StructType_v4, "v4", 2, 16, 0, 0, StructMembers_v4, 3 }, -{ gsm_StructType_float, "float", 5, 4, 0, 0, 0, 0 }, -{ gsm_StructType_color_buffer, "color_buffer", 12, 20, 0, 0, StructMembers_color_buffer, 3 }, -{ gsm_StructType_led, "led", 3, 20, 0, 0, StructMembers_led, 2 }, -{ gsm_StructType_s32, "s32", 3, 4, 0, 0, 0, 0 }, -{ gsm_StructType_pixel, "pixel", 5, 3, 0, 0, StructMembers_pixel, 2 }, -{ gsm_StructType_u8, "u8", 2, 1, 0, 0, 0, 0 }, -{ gsm_StructType_sin_wave_data, "sin_wave_data", 13, 20, 0, 0, StructMembers_sin_wave_data, 5 }, -{ gsm_StructType_r32, "r32", 3, 4, 0, 0, 0, 0 }, -{ gsm_StructType_revolving_discs_data, "revolving_discs_data", 20, 60, 0, 0, StructMembers_revolving_discs_data, 8 }, -{ gsm_StructType_vertical_color_fade_data, "vertical_color_fade_data", 24, 44, 0, 0, StructMembers_vertical_color_fade_data, 4 }, -{ gsm_StructType_multiply_patterns_data, "multiply_patterns_data", 22, 60, 0, 0, StructMembers_multiply_patterns_data, 3 }, }; -static gsm_u32 StructTypesCount = 13; +static gsm_u32 StructTypesCount = 0; diff --git a/src/gs_libs/gs_language.h b/src/gs_libs/gs_language.h index 9e12e25..dc86a4f 100644 --- a/src/gs_libs/gs_language.h +++ b/src/gs_libs/gs_language.h @@ -7,7 +7,7 @@ #include #elif defined(__APPLE__) && defined(__MAC__) -// TODO(Peter): +// TODO(Peter): #else // Std lib #include @@ -124,7 +124,7 @@ Test(b32 Result, char* Description, s32* Count) #define DEBUG_TRACK_SCOPE(a) #endif // DEBUG_TRACK_SCOPE #ifndef DEBUG_TRACK_FUNCTION -#define DEBUG_TRACK_FUNCTION +// #define DEBUG_TRACK_FUNCTION #endif // DEBUG_TRACK_FUNCTION #endif // GS_LANGUAGE_NO_PROFILER_DEFINES @@ -268,15 +268,15 @@ GSLerpDef(r32) GSLerpDef(r64) #undef GSLerpDef -static r32 GSSqrt(r32 V) -{ +static r32 GSSqrt(r32 V) +{ r32 Result = _mm_cvtss_f32(_mm_sqrt_ss(_mm_set_ss(V))); - return Result; + return Result; } #if 0 // TODO(Peter): Need a way to split the input into two f32's to supply to _mm_sqrt_sd -static r64 GSSqrt(r64 V) -{ +static r64 GSSqrt(r64 V) +{ r64 Result = _mm_cvtsd_f64(_mm_sqrt_sd(_mm_set_sd(V))); return Result; } @@ -366,9 +366,9 @@ RoundToNearestPowerOfTwo (u8 V) { u8 Result = 0; - if (IsPowerOfTwo(V)) - { - Result = V; + if (IsPowerOfTwo(V)) + { + Result = V; } else { @@ -387,9 +387,9 @@ RoundToNearestPowerOfTwo (u16 V) { u16 Result = 0; - if (IsPowerOfTwo(V)) - { - Result = V; + if (IsPowerOfTwo(V)) + { + Result = V; } else { @@ -409,9 +409,9 @@ RoundToNearestPowerOfTwo (u32 V) { u32 Result = 0; - if (IsPowerOfTwo(V)) - { - Result = V; + if (IsPowerOfTwo(V)) + { + Result = V; } else { @@ -432,9 +432,9 @@ RoundToNearestPowerOfTwo (u64 V) { u64 Result = 0; - if (IsPowerOfTwo(V)) - { - Result = V; + if (IsPowerOfTwo(V)) + { + Result = V; } else { diff --git a/src/gs_libs/gs_memory_arena.h b/src/gs_libs/gs_memory_arena.h index 772d9e3..350268c 100644 --- a/src/gs_libs/gs_memory_arena.h +++ b/src/gs_libs/gs_memory_arena.h @@ -2,8 +2,8 @@ // Description: Single header file library that defines a push-only memory arena // Author: Peter Slattery // Date Created: 2019-12-22 -// -// +// +// // ----------------- // Set Up // ----------------- @@ -16,26 +16,26 @@ // Simply create a memory_arena and use PushSize, PushStruct, or PushArray // to allocate out of it. // See Example Program below. -// +// // While there are options you can set (see Options below), the library adheres // to a 'zero-is-initialization' policy, that is, a memory_arena initialized to -// zero, under all default options, will 'just work'. +// zero, under all default options, will 'just work'. // // Alignment: // By default, the Push functions use 4 byte alignment -// If you need to control the alignment of an allocation, there are variants of the +// If you need to control the alignment of an allocation, there are variants of the // Push functions that allow for this: PushSizeAligned, PushStructAligned, and PushArrayAligned -// These functions simply take a final parameter which specifies the alignment. +// These functions simply take a final parameter which specifies the alignment. // Note: Alignment must be a power of two // // ----------------- // Options // ----------------- -// -// DEBUG: +// +// DEBUG: // Define DEBUG for debug functionality. -// -// To override the default assert function define GSMem_Assert(expression) +// +// To override the default assert function define GSMem_Assert(expression) // before inluding this file. // // GS_MEMORY_NO_STD_LIBS: @@ -61,8 +61,8 @@ // this file // // GS_MEMORY_TRACK_ALLOCATIONS: -// If you want to keep records of each allocation performed in every arena, define -// GS_MEMORY_TRACK_ALLOCATIONS before including this file. +// If you want to keep records of each allocation performed in every arena, define +// GS_MEMORY_TRACK_ALLOCATIONS before including this file. // When defined, memory arenas gain fields that allow them to keep a list of every // allocation they contain. It also adds a footer on the end of each allocation that // can be checked to ensure there are no writes to allocations that overflow their bounds @@ -78,8 +78,8 @@ // Places the characters 'gs' at the end of each allocation. This would allow for an external // function to check that we haven't written past the end of an allocation -void* MallocWrapper(gs_mem_u32 Size) -{ +void* MallocWrapper(gs_mem_u32 Size) +{ int SizeWithFooter = Size + (sizeof(char) * 2); void* Result = malloc(SizeWithFooter); char* Footer = (char*)(Result + Size); @@ -88,17 +88,17 @@ void* MallocWrapper(gs_mem_u32 Size) return Result; } -void* ReallocWrapper(void* Address, gs_mem_u32 Size) -{ - return realloc(Address, Size); +void* ReallocWrapper(void* Address, gs_mem_u32 Size) +{ + return realloc(Address, Size); } -int +int main(int ArgCount, char** Args) { memory_arena Arena = {}; // Uncomment these lines for an example of how you can implement custom allocation functions - // Arena.Alloc = MallocWrapper; + // Arena.Alloc = MallocWrapper; // Arena.Realloc = ReallocWrapper; int ArrayLength = 10; @@ -139,9 +139,9 @@ main(int ArgCount, char** Args) #ifndef GS_MEMORY_NO_STD_LIBS // NOTE(Peter): We use this so that we can fall back on malloc and realloc -// in the event that a memory_arena needs to grow but doesn't have a +// in the event that a memory_arena needs to grow but doesn't have a // alloc or realloc function pointer assigned to it. -// +// // See GrowArena to see where this is used // #include @@ -167,6 +167,21 @@ if((expression)) { \ #define GSMem_Assert(expression) #endif +struct data +{ + u8* Memory; + u64 Size; +}; + +typedef data allocator_alloc(u64 Size); +typedef void allocator_free(u8* Base, u64 Size); + +struct base_allocator +{ + allocator_alloc* Alloc; + allocator_free* Free; +}; + enum gs_memory_expansion_rule { MemoryExpansion_Allowed, // Zero is initialization lets the memory grow on its own @@ -177,13 +192,13 @@ enum gs_memory_expansion_rule // NOTE(Peter): // This rule is only here to allow for taking arena snapshots. The problem this solves -// is if you take a snapshot while there are 'holes' in memory_buffers behind the +// is if you take a snapshot while there are 'holes' in memory_buffers behind the // most recently added memory_buffer, take a snapshot of that arena, then push something // on that fits in one of those holes, we will fill the hole and be unable to track/free -// that addition via the snapshot construct. +// that addition via the snapshot construct. // // By requiring that allocations in a buffer only come from the most recent memory_buffer -// we can very easily rewind the buffer to the correct location. +// we can very easily rewind the buffer to the correct location. // Hence FindAddress_InLastBufferOnly enum gs_memory_find_address_rule { @@ -229,6 +244,8 @@ struct memory_buffer struct memory_arena { + base_allocator Allocator; + memory_buffer* Buffers; gs_mem_u32 BuffersCount; @@ -266,6 +283,30 @@ struct arena_snapshot #endif }; +#define AllocatorAllocArray(alloc,type,count) (type*)(AllocatorAlloc((alloc), sizeof(type) * (count)).Memory) +#define AllocatorAllocStruct(alloc,type) (type*)(AllocatorAlloc((alloc), sizeof(type)).Memory) +static data +AllocatorAlloc(base_allocator Allocator, gs_mem_u64 Size) +{ + return Allocator.Alloc(Size); +} + +static void +AllocatorFree(base_allocator Allocator, gs_mem_u8* Base, gs_mem_u64 Size) +{ + Allocator.Free(Base, Size); +} + +static memory_arena +CreateMemoryArena(base_allocator Allocator) +{ + memory_arena Result = {0}; + Result.Allocator = Allocator; + Result.ExpansionRule = MemoryExpansion_Allowed; + Result.FindAddressRule = FindAddress_InLastBufferOnly; + return Result; +} + static void FreeMemoryArena(memory_arena* Arena, gs_memory_free* Free = 0) { @@ -340,7 +381,7 @@ FindAlignedAddressInBufferWithRoom(memory_arena* Arena, gs_mem_u32 Size, gs_mem_ Result = AddressInCurrentBuffer; break; } - } + } return Result; } @@ -348,22 +389,8 @@ static gs_mem_u8* ArenaAlloc(memory_arena* Arena, gs_mem_u32 Size) { gs_mem_u8* Result = 0; - - if (Arena->Alloc) - { - Result = (gs_mem_u8*)Arena->Alloc(sizeof(gs_mem_u8) * Size); - } - else - { -#ifdef GS_MEMORY_NO_STD_LIBS - // NOTE(Peter): If you specify no std libs AND don't supply a allocation function - // we should assert as this is an invalid codepath - GSMem_Assert(0); -#else - Result = (gs_mem_u8*)malloc(sizeof(gs_mem_u8) * Size); -#endif - } - + data Data = AllocatorAlloc(Arena->Allocator, Size); + Result = Data.Memory; return Result; } @@ -372,21 +399,21 @@ ArenaRealloc(memory_arena* Arena, gs_mem_u8* Head, gs_mem_u32 OldSize, gs_mem_u3 { gs_mem_u8* Result = 0; - if (Arena->Realloc != 0) + GSMem_Assert(Arena->Allocator.Alloc); + GSMem_Assert(Arena->Allocator.Free); + + data Data = AllocatorAlloc(Arena->Allocator, NewSize); + for (gs_mem_u64 i = 0; i < (gs_mem_u64)OldSize; i++) { - Result = (gs_mem_u8*)Arena->Realloc(Head, OldSize, NewSize); - } - else - { -#ifdef GS_MEMORY_NO_STD_LIBS - // NOTE(Peter): If you specify no std libs AND don't supply a reallocation function - // we should assert as this is an invalid codepath - GSMem_Assert(0); -#else - Result = (gs_mem_u8*)realloc(Head, NewSize); -#endif + Data.Memory[i] = Head[i]; } + if (Head != 0) + { + AllocatorFree(Arena->Allocator, Head, OldSize); + } + + Result = Data.Memory; return Result; } @@ -396,7 +423,7 @@ GrowArena(memory_arena* Arena, gs_mem_u32 SizeNeeded) GSMem_Assert(Arena->ExpansionRule != MemoryExpansion_Disallowed); if (Arena->ExpansionRule == MemoryExpansion_OnlyIfFunctionsProvided) { - GSMem_Assert((Arena->Alloc != 0) && (Arena->Realloc != 0)); + GSMem_Assert((Arena->Allocator.Alloc != 0) && (Arena->Allocator.Free != 0)); } gs_mem_u32 NewBuffersCount = (Arena->BuffersCount + 1); @@ -433,8 +460,8 @@ TrackAllocation(memory_arena* Arena, gs_mem_u8* Head, gs_mem_u32 Size, char* Fil if (Arena->AllocationsUsed >= AllocationsMax) { gs_mem_u32 NewAllocationBuffersCount = Arena->AllocationBuffersCount + 1; - Arena->AllocationBuffers = (tracked_allocation_buffer**)ArenaRealloc(Arena, - (gs_mem_u8*)Arena->AllocationBuffers, + Arena->AllocationBuffers = (tracked_allocation_buffer**)ArenaRealloc(Arena, + (gs_mem_u8*)Arena->AllocationBuffers, Arena->AllocationBuffersCount * sizeof(void*), NewAllocationBuffersCount * sizeof(void*)); Arena->AllocationBuffersCount = NewAllocationBuffersCount; @@ -474,8 +501,8 @@ VerifyAllocationNoOverflow (tracked_allocation Allocation) static void AssertAllocationsNoOverflow (memory_arena Arena) { - for (gs_mem_u32 AllocationIndex = 0; - AllocationIndex< Arena.AllocationsUsed; + for (gs_mem_u32 AllocationIndex = 0; + AllocationIndex< Arena.AllocationsUsed; AllocationIndex++) { gs_mem_u32 BufferIndex = AllocationIndex / GS_MEM_TRACKED_ALLOCATION_BUFFER_SIZE; @@ -502,7 +529,7 @@ AssertAllocationsNoOverflow (memory_arena Arena) #define ClearAllocationsUsed(arena) #define ClearAllocationsUsedToSnapshot(arena, snapshot) -#define TrackAllocation(arena, head, size, filename, linenumber) +#define TrackAllocation(arena, head, size, filename, linenumber) #define PushSize(arena, size) PushSize_((arena), (size)) #define PushArray(arena, type, length) (type*)PushSize_((arena), sizeof(type) * length) @@ -517,7 +544,7 @@ static gs_mem_u8* PushSize_(memory_arena* Arena, gs_mem_u32 Size, gs_mem_u32 Alignment = 4, char* Filename = 0, gs_mem_u32 LineNumber = 0) { // ie. Alignment = 4 = 100 (binary) - // 4 - 1 = 3 + // 4 - 1 = 3 // 100 - 1 = 011 which is a mask of the bits we don't want set in the start address GSMem_Assert(IsPowerOfTwo(Alignment)); gs_mem_u32 AlignmentMask = Alignment - 1; @@ -560,7 +587,7 @@ ClearArena(memory_arena* Arena) { memory_buffer* At = Arena->Buffers + i; At->Used = 0; - } + } Arena->TotalUsed = 0; ClearAllocationsUsed(Arena); diff --git a/src/gs_libs/gs_profiler.h b/src/gs_libs/gs_profiler.h new file mode 100644 index 0000000..9ec2006 --- /dev/null +++ b/src/gs_libs/gs_profiler.h @@ -0,0 +1,258 @@ +// +// File: gs_profiler.h +// Author: Peter Slattery +// Creation Date: 2020-05-11 +// +#ifndef GS_PROFILER_H + +internal s64 +GetWallClock() +{ + LARGE_INTEGER Time; + if (!QueryPerformanceCounter(&Time)) + { + s32 Error = GetLastError(); + // TODO(Peter): I'm waiting to see an error actually occur here + // to know what it could possibly be. + InvalidCodePath; + } + return (s64)Time.QuadPart; +} + +internal u64 +HashDJB2ToU64(char* String) +{ + u64 Hash = 5381; + char* C = String; + while(*C) + { + Hash = ((Hash << 5) + Hash) + *C++; + } + return Hash; +} + +struct scope_record +{ + u64 IdentifierHash; + s64 StartTime; + s64 EndTime; +}; + +struct scope_record_list +{ + scope_record* List; + u32 Count; + u32 MaxCount; + scope_record_list* Next; +}; + +struct collated_scope_record +{ + char* Identifier; + u64 CallCount; + u64 TotalCycles; + u64 LongestDuration; +}; + +#define COLLATED_SCOPES_HASH_SIZE 1024 +struct collated_scopes_hash_table +{ + u64* Hashes; + collated_scope_record* Scopes; + collated_scopes_hash_table* Next; +}; + +struct debug_services +{ + memory_arena Arena; + memory_arena Transient; + + u32 CollatedScopesCount; + collated_scopes_hash_table* CollatedScopes; + + scope_record_list* ScopeRecordsHead; + scope_record_list* ScopeRecordsTail; +}; + +global_variable debug_services GlobalDebugServices = {0}; + +internal u64 +RegisterScope(debug_services* Services, char* Identifier) +{ + u64 Hash = HashDJB2ToU64(Identifier); + u32 Index = (u32)(Hash % COLLATED_SCOPES_HASH_SIZE); + + collated_scopes_hash_table* PrevTable = 0; + collated_scopes_hash_table* TableAt = Services->CollatedScopes; + while (TableAt != 0) + { + if (TableAt->Hashes[Index] == Hash || TableAt->Hashes[Index] == 0) + { + break; + } + PrevTable = TableAt; + TableAt = TableAt->Next; + } + + if (!TableAt) + { + collated_scopes_hash_table* NewTable = PushStruct(&Services->Arena, collated_scopes_hash_table); + NewTable->Hashes = PushArray(&Services->Arena, u64, COLLATED_SCOPES_HASH_SIZE); + GSZeroMemory(NewTable->Hashes, sizeof(u64) * COLLATED_SCOPES_HASH_SIZE); + NewTable->Scopes = PushArray(&Services->Arena, collated_scope_record, COLLATED_SCOPES_HASH_SIZE); + NewTable->Next = 0; + + if (PrevTable) + { + PrevTable->Next = NewTable; + } + else + { + Services->CollatedScopes = NewTable; + } + + TableAt = NewTable; + } + + if (TableAt->Hashes[Index] == 0) + { + // New Scope Being Registered + TableAt->Hashes[Index] = Hash; + TableAt->Scopes[Index] = {0}; + TableAt->Scopes[Index].Identifier = Identifier; + Services->CollatedScopesCount++; + } + else if (TableAt->Hashes[Index] != Hash) + { + InvalidCodePath; + } + + return Hash; +} + +internal void +PushScopeOnList(debug_services* Services, char* ScopeName, s64 Start, s64 End) +{ + if (!Services->ScopeRecordsTail || + Services->ScopeRecordsTail->Count >= Services->ScopeRecordsTail->MaxCount) + { + scope_record_list* NewTail = PushStruct(&Services->Arena, scope_record_list); + NewTail->MaxCount = 4096; + NewTail->List = PushArray(&Services->Arena, scope_record, NewTail->MaxCount); + NewTail->Count = 0; + NewTail->Next = 0; + + if (Services->ScopeRecordsHead) + { + Services->ScopeRecordsTail->Next = NewTail; + } + else + { + Services->ScopeRecordsHead = NewTail; + } + Services->ScopeRecordsTail = NewTail; + } + + u32 NewRecordIndex = Services->ScopeRecordsTail->Count++; + + scope_record* NewRecord = &Services->ScopeRecordsTail->List[NewRecordIndex]; + NewRecord->IdentifierHash = RegisterScope(Services, ScopeName); + NewRecord->StartTime = Start; + NewRecord->EndTime = End; +} + +#define DEBUG_TRACK_FUNCTION scope_tracker ScopeTracker ((char*)__func__) +struct scope_tracker +{ + s64 ScopeStart; + char* ScopeName; + + scope_tracker(char* ScopeName) + { + this->ScopeName = ScopeName; + this->ScopeStart = GetWallClock(); + } + + ~scope_tracker() + { + s64 ScopeEnd = GetWallClock(); + PushScopeOnList(&GlobalDebugServices, this->ScopeName, this->ScopeStart, ScopeEnd); + } +}; + +// +// Reporting +// + +internal collated_scope_record* +GetCollatedRecordForScope(debug_services* Services, u64 Hash) +{ + Assert(Services->CollatedScopes); + collated_scope_record* Result = 0; + u64 Index = Hash % COLLATED_SCOPES_HASH_SIZE; + + collated_scopes_hash_table* TableAt = Services->CollatedScopes; + while (TableAt != 0) + { + if (TableAt->Hashes[Index] == Hash) + { + Result = TableAt->Scopes + Index; + break; + } + TableAt = TableAt->Next; + } + return Result; +} + +internal void +CollateFrame(debug_services* Services) +{ + scope_record_list* ListAt = Services->ScopeRecordsHead; + while (ListAt != 0) + { + for (u32 i = 0; i < ListAt->Count; i++) + { + scope_record ScopeAt = ListAt->List[i]; + u64 ScopeAtDuration = ScopeAt.EndTime - ScopeAt.StartTime; + + collated_scope_record* CollatedScopeData = GetCollatedRecordForScope(Services, ScopeAt.IdentifierHash); + Assert(CollatedScopeData != 0); + CollatedScopeData->CallCount++; + CollatedScopeData->TotalCycles += ScopeAtDuration; + CollatedScopeData->LongestDuration = GSMax(ScopeAtDuration, CollatedScopeData->LongestDuration); + } + ListAt = ListAt->Next; + } + + u32 SortListCount = Services->CollatedScopesCount; + gs_radix_entry* SortList = PushArray(&Services->Transient, gs_radix_entry, SortListCount); + + u32 SortListAt = 0; + collated_scopes_hash_table* TableAt = Services->CollatedScopes; + while (TableAt != 0) + { + for (u32 i = 0; i < COLLATED_SCOPES_HASH_SIZE; i++) + { + if (TableAt->Hashes[i] == 0) { continue; } + SortList[SortListAt].ID = TableAt->Hashes[i]; + SortList[SortListAt].Radix = TableAt->Scopes[i].TotalCycles; + SortListAt += 1; + } + TableAt = TableAt->Next; + } + + RadixSortInPlace(SortList, SortListCount); + + for (u32 i = 0; i < SortListCount; i++) + { + u64 Hash = SortList[i].ID; + collated_scope_record* ScopeData = GetCollatedRecordForScope(Services, Hash); + printf("Scope: %s\n", ScopeData->Identifier); + printf(" Call Count: %lld\n", ScopeData->CallCount); + printf(" Total Cycles: %lld\n", ScopeData->TotalCycles); + printf(" Longest Duration: %lld\n", ScopeData->LongestDuration); + } +} + +#define GS_PROFILER_H +#endif // GS_PROFILER_H \ No newline at end of file diff --git a/src/gs_libs/gs_radix_sort.h b/src/gs_libs/gs_radix_sort.h index 8e62a27..ba9fc22 100644 --- a/src/gs_libs/gs_radix_sort.h +++ b/src/gs_libs/gs_radix_sort.h @@ -10,31 +10,31 @@ TODO #if !defined(GSRad_Assert) #define GSRad_Assert(expression) \ if(!(expression)) { \ -*((int *)0) = 5; \ + *((int *)0) = 5; \ } #endif // !defined(GSRad_Assert) #endif // DEBUG -typedef unsigned int gs_rad_u32; +typedef unsigned long long int gs_rad_u64; typedef unsigned int gs_rad_b32; struct gs_radix_entry { - gs_rad_u32 Radix; - gs_rad_u32 ID; + gs_rad_u64 Radix; + gs_rad_u64 ID; }; static void -RadixSortInPlace_ (gs_radix_entry* Data, gs_rad_u32 Start, gs_rad_u32 End, gs_rad_u32 Iteration) +RadixSortInPlace_ (gs_radix_entry* Data, gs_rad_u64 Start, gs_rad_u64 End, gs_rad_u64 Iteration) { - gs_rad_u32 Shift = Iteration; - gs_rad_u32 ZerosBoundary = Start; - gs_rad_u32 OnesBoundary = End - 1; + gs_rad_u64 Shift = Iteration; + gs_rad_u64 ZerosBoundary = Start; + gs_rad_u64 OnesBoundary = End - 1; - for (gs_rad_u32 d = Start; d < End; d++) + for (gs_rad_u64 d = Start; d < End; d++) { gs_radix_entry Entry = Data[ZerosBoundary]; - gs_rad_u32 Place = (Entry.Radix >> Shift) & 0x1; + gs_rad_u64 Place = (Entry.Radix >> Shift) & 0x1; if (Place) { gs_radix_entry Evicted = Data[OnesBoundary]; @@ -56,10 +56,10 @@ RadixSortInPlace_ (gs_radix_entry* Data, gs_rad_u32 Start, gs_rad_u32 End, gs_ra } static void -RadixSortInPlace (gs_radix_entry* Data, gs_rad_u32 Count) +RadixSortInPlace (gs_radix_entry* Data, gs_rad_u64 Count) { - gs_rad_u32 Highest = 0; - for (gs_rad_u32 i = 0; i < Count; i++) + gs_rad_u64 Highest = 0; + for (gs_rad_u64 i = 0; i < Count; i++) { if (Data[i].Radix > Highest) { @@ -67,7 +67,7 @@ RadixSortInPlace (gs_radix_entry* Data, gs_rad_u32 Count) } } - gs_rad_u32 Iterations = 0; + gs_rad_u64 Iterations = 0; while (Highest > 1) { ++Iterations; diff --git a/src/meta/foldhaus_meta.cpp b/src/meta/foldhaus_meta.cpp index 3206405..7991aa2 100644 --- a/src/meta/foldhaus_meta.cpp +++ b/src/meta/foldhaus_meta.cpp @@ -12,7 +12,7 @@ internal void GenerateNodeMetaInfo (gsm_code_generator* NodeTypeGen, string_builder* NodeSpecificationGen, string_builder* CallNodeProcGen, gs_meta_preprocessor Meta) { // TODO(Peter): Create a FilterTypesByTag function to create a contiguous array - // of type_definition** + // of type_definition** WriteF(NodeSpecificationGen, "static node_specification_ NodeSpecifications[] = {\n"); @@ -225,7 +225,7 @@ GeneratePanelMetaInfo(gs_meta_preprocessor Meta, string_builder* PanelEnumGen, s panel_elements* Panel = Panels.GetElementAtIndex(i); string PanelIdentifier = {0}; PanelIdentifier.Max = Panel->PanelIdentifier.Length; - PanelIdentifier.Memory = (char*)malloc(sizeof(char) * PanelIdentifier.Max); + PanelIdentifier.Memory = PushArray(&Meta.Permanent, char, PanelIdentifier.Max); CopyStringTo(Substring(Panel->PanelIdentifier, 11), &PanelIdentifier); MakeReadableIdentifier(&PanelIdentifier); @@ -259,11 +259,11 @@ GeneratePanelMetaInfo(gs_meta_preprocessor Meta, string_builder* PanelEnumGen, s } internal string -AllocAndConcatStrings(string First, string Second) +AllocAndConcatStrings(base_allocator Allocator, string First, string Second) { string Result = {0}; Result.Max = First.Length + Second.Length + 1; - Result.Memory = (char*)malloc(sizeof(char) * Result.Max); + Result.Memory = AllocatorAllocArray(Allocator, char, Result.Max); ConcatString(First, &Result); ConcatString(Second, &Result); NullTerminate(&Result); @@ -271,6 +271,26 @@ AllocAndConcatStrings(string First, string Second) return Result; } +u64 TotalAllocatedSpace = 0; + +internal data +PlatformAlloc(u64 Size) +{ + data Result = {0}; + Result.Memory = (u8*)malloc(Size); + Result.Size = Size; + + TotalAllocatedSpace += Size; + + return Result; +} + +internal void +PlatformFree(u8* Base, u64 Size) +{ + free((void*)Base); +} + int main(int ArgCount, char* Args[]) { if (ArgCount <= 1) @@ -279,22 +299,27 @@ int main(int ArgCount, char* Args[]) return 0; } + base_allocator Allocator = {0}; + Allocator.Alloc = (allocator_alloc*)PlatformAlloc; + Allocator.Free = (allocator_free*)PlatformFree; + + string RootFile = MakeString(Args[1]); s32 IndexOfLastSlash = ReverseSearchForCharInSet(RootFile, "\\/"); string WorkingDirectory = Substring(RootFile, 0, IndexOfLastSlash + 1); string GeneratedDirectoryName = MakeStringLiteral("generated\\"); - string GeneratedFilesDirectory = AllocAndConcatStrings(WorkingDirectory, GeneratedDirectoryName); + string GeneratedFilesDirectory = AllocAndConcatStrings(Allocator, WorkingDirectory, GeneratedDirectoryName); printf("Putting Generated Files In %s\n", GeneratedFilesDirectory.Memory); - gs_meta_preprocessor Meta = PreprocessProgram(Args[1]); + gs_meta_preprocessor Meta = PreprocessProgram(Allocator, Args[1]); - typeinfo_generator TypeGenerator = InitTypeInfoGenerator(Meta.TypeTable); + typeinfo_generator TypeGenerator = InitTypeInfoGenerator(&Meta.TypeTable); GenerateMetaTagList(Meta.TypeTable, &TypeGenerator); GenerateFilteredTypeInfo(MakeStringLiteral("node_struct"), Meta.TypeTable, &TypeGenerator); GenerateFilteredTypeInfo(MakeStringLiteral("gen_type_info"), Meta.TypeTable, &TypeGenerator); FinishGeneratingTypes(&TypeGenerator); - gsm_code_generator NodeTypeGen = BeginEnumGeneration("node_type", "NodeType", false, true); + gsm_code_generator NodeTypeGen = BeginEnumGeneration(&Meta.TypeTable.Arena, "node_type", "NodeType", false, true); string_builder NodeSpecificationGen = {0}; string_builder CallNodeProcGen = {0}; GenerateNodeMetaInfo(&NodeTypeGen, &NodeSpecificationGen, &CallNodeProcGen, Meta); @@ -303,7 +328,7 @@ int main(int ArgCount, char* Args[]) string_builder PanelCodeGen = {0}; GeneratePanelMetaInfo(Meta, &PanelEnumGen, &PanelCodeGen); - string TypeInfoHFilePath = AllocAndConcatStrings(GeneratedFilesDirectory, MakeStringLiteral("gs_meta_generated_typeinfo.h")); + string TypeInfoHFilePath = AllocAndConcatStrings(Allocator, GeneratedFilesDirectory, MakeStringLiteral("gs_meta_generated_typeinfo.h")); FILE* TypeInfoH = fopen(TypeInfoHFilePath.Memory, "w"); if (TypeInfoH) { @@ -319,7 +344,7 @@ int main(int ArgCount, char* Args[]) printf("Error: Unable to open file at %.*s\n", StringExpand(TypeInfoHFilePath)); } - string NodeInfoHFilePath = AllocAndConcatStrings(GeneratedFilesDirectory, MakeStringLiteral("foldhaus_nodes_generated.h")); + string NodeInfoHFilePath = AllocAndConcatStrings(Allocator, GeneratedFilesDirectory, MakeStringLiteral("foldhaus_nodes_generated.h")); FILE* NodeInfoH = fopen(NodeInfoHFilePath.Memory, "w"); if (NodeInfoH) { @@ -333,7 +358,7 @@ int main(int ArgCount, char* Args[]) printf("Error: Unable to open file at %.*s\n", StringExpand(NodeInfoHFilePath)); } - string PanelInfoHFilePath = AllocAndConcatStrings(GeneratedFilesDirectory, MakeStringLiteral("foldhaus_panels_generated.h")); + string PanelInfoHFilePath = AllocAndConcatStrings(Allocator, GeneratedFilesDirectory, MakeStringLiteral("foldhaus_panels_generated.h")); FILE* PanelInfoH = fopen(PanelInfoHFilePath.Memory, "w"); if (PanelInfoH) { @@ -348,6 +373,9 @@ int main(int ArgCount, char* Args[]) FinishMetaprogram(&Meta); + r64 AllocatedMegabytes = (r64)TotalAllocatedSpace / (1024 * 1024); + printf("Total Allocated Space: %lld bytes, or %f MB\n", TotalAllocatedSpace, AllocatedMegabytes); + //__debugbreak(); return 0; } diff --git a/src/meta/gs_meta.cpp b/src/meta/gs_meta.cpp index 36c406a..0c63156 100644 --- a/src/meta/gs_meta.cpp +++ b/src/meta/gs_meta.cpp @@ -9,8 +9,8 @@ // GSMetaTag() to give commands to the meta layer // // Tag Values -// -// breakpoint +// +// breakpoint // will cause the meta layer to break in the debugger when it reaches // that point in processing the file // TODO: specify which stage you want it to break at @@ -21,41 +21,13 @@ #include #include "..\gs_libs\gs_language.h" +#include "..\gs_libs\gs_radix_sort.h" #include "..\gs_libs\gs_bucket.h" #include "..\gs_libs\gs_memory_arena.h" +#include "..\gs_libs\gs_profiler.h" #include "..\gs_libs\gs_string.h" -#include "gs_meta_lexer.h" -#include "gs_meta_error.h" - -#include "gs_meta_type_table.h" - -struct source_code_file -{ - string Path; - s32 FileSize; - string Contents; - - s32 FirstTokenIndex; - s32 LastTokenIndex; -}; - -struct token_iter -{ - gs_bucket* Tokens; - token* TokenAt; - s32 TokenAtIndex; - s32 FirstToken; - s32 LastToken; - -#define TOKEN_ITER_SNAPSHOTS_MAX 64 - u32 SnapshotsUsed; - u32 Snapshots[TOKEN_ITER_SNAPSHOTS_MAX]; - - errors* Errors; -}; - struct gsm_profiler_scope { s64 StartTime; @@ -81,31 +53,10 @@ struct gsm_profiler gs_bucket Categories; }; -struct gs_meta_preprocessor -{ - errors Errors; - - gs_bucket SourceFiles; - gs_bucket Tokens; - - gs_bucket TagList; - - type_table TypeTable; - - // Performance - s64 PreprocessorStartTime; - s64 TokenizeTime; - s64 PreprocTime; - s64 FixupTime; - s64 PreprocessorEndTime; - - gsm_profiler Profiler; -}; - // ------------------------ // Timing / Performance // ------------------------ - +#if 0 internal s64 GetWallClock () { @@ -119,7 +70,7 @@ GetWallClock () } return (s64)Time.QuadPart; } - +#endif internal s64 GetPerformanceFrequency () { @@ -274,6 +225,63 @@ PrintAllCategories(gsm_profiler* Profiler) } } +#include "gs_meta_lexer.h" +#include "gs_meta_error.h" + +#include "gs_meta_type_table.h" + +struct token_array +{ + token* List; + u32 Count; +}; + +struct source_code_file +{ + string Path; + s32 FileSize; + string Contents; + + token_array Tokens; +}; + +struct token_iter +{ + token_array Tokens; + token* TokenAt; + u32 TokenAtIndex; + +#define TOKEN_ITER_SNAPSHOTS_MAX 64 + u32 SnapshotsUsed; + u32 Snapshots[TOKEN_ITER_SNAPSHOTS_MAX]; + + errors* Errors; +}; + +struct gs_meta_preprocessor +{ + memory_arena Permanent; + memory_arena Transient; + + errors Errors; + + gs_bucket SourceFiles; + //gs_bucket Tokens; + + gs_bucket TagList; + + type_table TypeTable; + + // Performance + s64 PreprocessorStartTime; + s64 TokenizeTime; + s64 PreprocTime; + s64 FixupTime; + s64 PreprocessorEndTime; + + gsm_profiler Profiler; +}; + // ------------------------ // Token Iterator // ------------------------ @@ -281,10 +289,10 @@ PrintAllCategories(gsm_profiler* Profiler) internal token* NextToken (token_iter* Iter) { - if (Iter->TokenAtIndex < Iter->LastToken) + DEBUG_TRACK_FUNCTION; + if (Iter->TokenAtIndex < Iter->Tokens.Count) { - Iter->TokenAtIndex++; - Iter->TokenAt = Iter->Tokens->GetElementAtIndex(Iter->TokenAtIndex); + Iter->TokenAt = Iter->Tokens.List + Iter->TokenAtIndex++; } return Iter->TokenAt; @@ -293,6 +301,7 @@ NextToken (token_iter* Iter) internal b32 TokenAtEquals(token_iter* Iter, char* String) { + DEBUG_TRACK_FUNCTION; b32 Result = false; if (StringEqualsCharArray(Iter->TokenAt->Text, String)) { @@ -305,6 +314,7 @@ TokenAtEquals(token_iter* Iter, char* String) internal b32 TokenAtEquals(token_iter* Iter, token_type Type) { + DEBUG_TRACK_FUNCTION; b32 Result = false; if (Iter->TokenAt->Type == Type) { @@ -317,6 +327,7 @@ TokenAtEquals(token_iter* Iter, token_type Type) internal b32 TokenAtEquals(token_iter* Iter, token_type Type, token* Token) { + DEBUG_TRACK_FUNCTION; b32 Result = false; if (Iter->TokenAt->Type == Type) { @@ -330,12 +341,14 @@ TokenAtEquals(token_iter* Iter, token_type Type, token* Token) internal void PushSnapshot (token_iter* Iter) { + DEBUG_TRACK_FUNCTION; Iter->Snapshots[Iter->SnapshotsUsed++] = Iter->TokenAtIndex; } internal void PopSnapshot (token_iter* Iter) { + DEBUG_TRACK_FUNCTION; if (Iter->SnapshotsUsed > 0) { Iter->SnapshotsUsed -= 1; @@ -345,15 +358,17 @@ PopSnapshot (token_iter* Iter) internal void ApplySnapshot (token_iter* Iter) { + DEBUG_TRACK_FUNCTION; u32 SnapshotIndex = Iter->SnapshotsUsed; u32 SnapshotPoint = Iter->Snapshots[SnapshotIndex]; Iter->TokenAtIndex = SnapshotPoint; - Iter->TokenAt = Iter->Tokens->GetElementAtIndex(SnapshotPoint); + Iter->TokenAt = Iter->Tokens.List + Iter->TokenAtIndex; } internal void ApplySnapshotIfNotParsedAndPop(b32 ParseSuccess, token_iter* Iter) { + DEBUG_TRACK_FUNCTION; PopSnapshot(Iter); if (!ParseSuccess) { @@ -365,6 +380,7 @@ ApplySnapshotIfNotParsedAndPop(b32 ParseSuccess, token_iter* Iter) internal s32 GetFileSize (char* FileName) { + DEBUG_TRACK_FUNCTION; s32 Result = 0; FILE* ReadFile = fopen(FileName, "r"); @@ -386,8 +402,9 @@ GetFileSize (char* FileName) // ------------------------- internal s32 -ReadEntireFileAndNullTerminate (source_code_file* File, errors* Errors) +ReadEntireFileAndNullTerminate (memory_arena* Storage, source_code_file* File, errors* Errors) { + DEBUG_TRACK_FUNCTION; s32 LengthRead = 0; FILE* ReadFile = fopen(File->Path.Memory, "r"); @@ -399,7 +416,7 @@ ReadEntireFileAndNullTerminate (source_code_file* File, errors* Errors) Assert(File->Contents.Memory == 0); File->Contents.Max = (s32)FileSize + 1; - File->Contents.Memory = (char*)malloc(File->Contents.Max); + File->Contents.Memory = (char*)PushSize(Storage, File->Contents.Max); size_t ReadSize = fread(File->Contents.Memory, 1, FileSize, ReadFile); File->Contents.Memory[FileSize] = 0; @@ -415,6 +432,7 @@ ReadEntireFileAndNullTerminate (source_code_file* File, errors* Errors) internal b32 FileAlreadyInSource(string Path, gs_bucket SourceFiles) { + DEBUG_TRACK_FUNCTION; b32 Result = false; for (u32 i = 0; i < SourceFiles.Used; i++) @@ -431,19 +449,18 @@ FileAlreadyInSource(string Path, gs_bucket SourceFiles) } internal void -AddFileToSource(string RelativePath, source_code_file CurrentFile, gs_bucket* SourceFiles, errors* Errors) +AddFileToSource(memory_arena* Storage, string RelativePath, source_code_file CurrentFile, gs_bucket* SourceFiles, errors* Errors) { + DEBUG_TRACK_FUNCTION; source_code_file File = {0}; - File.FirstTokenIndex = -1; - File.LastTokenIndex = -1; - u32 PathLength = RelativePath.Length + 1; - File.Path = MakeString((char*)malloc(sizeof(char) * PathLength), 0, PathLength); + char* PathData = PushArray(Storage, char, PathLength); + File.Path = MakeString(PathData, 0, PathLength); CopyStringTo(RelativePath, &File.Path); NullTerminate(&File.Path); - File.FileSize = ReadEntireFileAndNullTerminate(&File, Errors); + File.FileSize = ReadEntireFileAndNullTerminate(Storage, &File, Errors); if (File.FileSize > 0) { @@ -460,8 +477,21 @@ AddFileToSource(string RelativePath, source_code_file CurrentFile, gs_bucket* Tokens) +TokenizeFile (gs_meta_preprocessor* Meta, source_code_file* File) { + DEBUG_TRACK_FUNCTION; + gsm_profiler_scope* Scope = BeginScope(&Meta->Profiler, MakeStringLiteral("tokenize"), File->Path); + + struct token_list + { + token Token; + token_list* Next; + }; + + u32 TokensCount = 0; + token_list* TokensHead = 0; + token_list* TokensTail = 0; + tokenizer Tokenizer = {}; Tokenizer.At = File->Contents.Memory; Tokenizer.Memory = File->Contents.Memory; @@ -470,15 +500,35 @@ TokenizeFile (source_code_file* File, gs_bucket* Tokens) token* LastToken = 0; while(AtValidPosition(Tokenizer)) { - token NewToken = GetNextToken(&Tokenizer); - u32 TokenIndex = Tokens->PushElementOnBucket(NewToken); - if (File->FirstTokenIndex < 0) + token_list* NewToken = PushStruct(&Meta->Transient, token_list); + NewToken->Token = GetNextToken(&Tokenizer); + NewToken->Next = 0; + + TokensCount++; + if (TokensHead != 0) { - File->FirstTokenIndex = (s32)TokenIndex; + TokensTail->Next = NewToken; + TokensTail = NewToken; + } + else + { + TokensHead = NewToken; + TokensTail = NewToken; } } - File->LastTokenIndex = Tokens->Used - 1; + File->Tokens = {0}; + File->Tokens.Count = TokensCount; + File->Tokens.List = PushArray(&Meta->Permanent, token, File->Tokens.Count); + + token_list* At = TokensHead; + for (u32 i = 0; i < TokensCount; i++) + { + Assert(At != 0); + File->Tokens.List[i] = At->Token; + At = At->Next; + } + EndScope(Scope); } // ------------------------ @@ -488,7 +538,8 @@ TokenizeFile (source_code_file* File, gs_bucket* Tokens) internal b32 ParseMetaTag(token_iter* Iter, gs_meta_preprocessor* Meta) { - gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, + DEBUG_TRACK_FUNCTION; + gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseMetaTag")); b32 Result = false; @@ -517,7 +568,7 @@ ParseMetaTag(token_iter* Iter, gs_meta_preprocessor* Meta) if (StringsEqual(MetaIdentifier.Text, MakeStringLiteral("breakpoint"))) { - // NOTE(Peter): This is not a temporary breakpoint. It is + // NOTE(Peter): This is not a temporary breakpoint. It is // used to be able to break the meta program at specific points // throughout execution __debugbreak(); @@ -534,7 +585,8 @@ ParseMetaTag(token_iter* Iter, gs_meta_preprocessor* Meta) internal b32 ParseSignedness (token_iter* Iter) { - // NOTE(Peter): This doesn't really do much at the moment, but + DEBUG_TRACK_FUNCTION; + // NOTE(Peter): This doesn't really do much at the moment, but // I want all signedness parsing to happen in one place in case // we ever need to do anything with it. @@ -552,6 +604,7 @@ ParseSignedness (token_iter* Iter) internal b32 ShortInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -566,8 +619,8 @@ ShortInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTab } ApplySnapshotIfNotParsedAndPop(Result, Iter); - if (Result) - { + if (Result) + { *TypeHandleOut = GetTypeHandle(MakeStringLiteral("short int"), TypeTable); } return Result; @@ -576,6 +629,7 @@ ShortInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTab internal b32 Int (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -586,8 +640,8 @@ Int (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) } ApplySnapshotIfNotParsedAndPop(Result, Iter); - if (Result) - { + if (Result) + { *TypeHandleOut = GetTypeHandle(MakeStringLiteral("int"), TypeTable); } return Result; @@ -596,6 +650,7 @@ Int (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) internal b32 LongInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -610,8 +665,8 @@ LongInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTabl } ApplySnapshotIfNotParsedAndPop(Result, Iter); - if (Result) - { + if (Result) + { *TypeHandleOut = GetTypeHandle(MakeStringLiteral("long int"), TypeTable); } return Result; @@ -620,6 +675,7 @@ LongInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTabl internal b32 LongLongInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -637,8 +693,8 @@ LongLongInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table Type } ApplySnapshotIfNotParsedAndPop(Result, Iter); - if (Result) - { + if (Result) + { *TypeHandleOut = GetTypeHandle(MakeStringLiteral("long long int"), TypeTable); } return Result; @@ -647,6 +703,7 @@ LongLongInt (token_iter* Iter, type_table_handle* TypeHandleOut, type_table Type internal b32 ParseChar(token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -669,6 +726,7 @@ ParseChar(token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTab internal b32 ParseBool(token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -685,6 +743,7 @@ ParseBool(token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTab internal b32 ParseFloat(token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -701,6 +760,7 @@ ParseFloat(token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTa internal b32 ParseDouble(token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -720,7 +780,8 @@ ParseDouble(token_iter* Iter, type_table_handle* TypeHandleOut, type_table TypeT internal b32 ParseType(token_iter* Iter, gs_meta_preprocessor* Meta, type_table_handle* TypeHandleOut) { - gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, + DEBUG_TRACK_FUNCTION; + gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseType")); b32 Result = false; @@ -737,16 +798,16 @@ ParseType(token_iter* Iter, gs_meta_preprocessor* Meta, type_table_handle* TypeH ParseDouble(Iter, TypeHandleOut, Meta->TypeTable)) { Result = true; - } + } else if (StringsEqual(Iter->TokenAt->Text, MakeStringLiteral("void"))) { NextToken(Iter); Result = true; *TypeHandleOut = GetTypeHandle(MakeStringLiteral("void"), Meta->TypeTable); } - else + else { - gsm_profiler_scope* ProfileInnerScope = BeginScope(&Meta->Profiler, + gsm_profiler_scope* ProfileInnerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseTypeInner")); @@ -760,9 +821,9 @@ ParseType(token_iter* Iter, gs_meta_preprocessor* Meta, type_table_handle* TypeH { Result = true; // NOTE(Peter): In this case, we believe we are at a type identifier, - // however, it hasn't been declared yet. This is due to the fact that we + // however, it hasn't been declared yet. This is due to the fact that we // tokenize files, then parse them, then import the files they include, and - // then begin tokenizing, parsing, etc for those files. + // then begin tokenizing, parsing, etc for those files. // In the case that we get an as-of-yet undeclared type, we leave it // up to the calling site to determine what to do with that information // :UndeclaredType @@ -780,6 +841,7 @@ ParseType(token_iter* Iter, gs_meta_preprocessor* Meta, type_table_handle* TypeH internal b32 ParsePointer (token_iter* Iter) { + DEBUG_TRACK_FUNCTION; b32 Result = false; if (TokenAtEquals(Iter, "*")) { @@ -791,6 +853,7 @@ ParsePointer (token_iter* Iter) internal b32 ParseConstVolatile (token_iter* Iter) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -807,7 +870,8 @@ ParseConstVolatile (token_iter* Iter) internal b32 ParseVariableDecl(token_iter* Iter, gs_bucket* VariableList, gs_meta_preprocessor* Meta) { - gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, + DEBUG_TRACK_FUNCTION; + gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseVariableDecl")); b32 Result = false; @@ -842,14 +906,14 @@ ParseVariableDecl(token_iter* Iter, gs_bucket* VariableList, gs_m if (TokenAtEquals(Iter, Token_Identifier, &IdentifierToken)) { // Array Notationg ie r32 x[2]; - // NOTE(Peter): True initially because if there is no array notation, we + // NOTE(Peter): True initially because if there is no array notation, we // are still ok to proceed b32 ArrayParseSuccess = true; u32 ArrayCount = 0; if (TokenAtEquals(Iter, "[")) { // NOTE(Peter): Once we get to this point, we have to complete the entire - // array notation before we have successfully parsed, hence setting + // array notation before we have successfully parsed, hence setting // ArrayParseSucces to false here. ArrayParseSuccess = false; token NumberToken = {}; @@ -934,6 +998,7 @@ ParseVariableDecl(token_iter* Iter, gs_bucket* VariableList, gs_m internal b32 StructOrUnion(token_iter* Iter, type_definition_type* Type) { + DEBUG_TRACK_FUNCTION; b32 Result = false; if (TokenAtEquals(Iter, "struct")) { @@ -954,7 +1019,8 @@ StructOrUnion(token_iter* Iter, type_definition_type* Type) internal b32 ParseStruct(token_iter* Iter, type_table_handle* StructTypeHandleOut, gs_meta_preprocessor* Meta, type_definition* ContainingStruct = 0) { - gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, + DEBUG_TRACK_FUNCTION; + gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseStruct")); b32 Result = false; @@ -988,7 +1054,7 @@ ParseStruct(token_iter* Iter, type_table_handle* StructTypeHandleOut, gs_meta_pr string AnonStructIdentifier = {}; AnonStructIdentifier.Max = 256; - AnonStructIdentifier.Memory = (char*)malloc(sizeof(char) * AnonStructIdentifier.Max); + AnonStructIdentifier.Memory = PushArray(&Meta->Permanent, char, AnonStructIdentifier.Max); PrintF(&AnonStructIdentifier, "%S_%d", ContainingStruct->Identifier, ContainingStruct->Struct.MemberDecls.Used); @@ -1017,11 +1083,11 @@ ParseStruct(token_iter* Iter, type_table_handle* StructTypeHandleOut, gs_meta_pr else if (ParseStruct(Iter, &MemberStructTypeHandle, Meta, &StructDecl)) { // NOTE(Peter): Pretty sure, since we just parsed the struct, that - // MemberStructTypeIndex should never be Invalid (unknown type). + // MemberStructTypeIndex should never be Invalid (unknown type). // Putting this Assert here for now, but remove if there's a valid // reason that you might not be able to find a struct just parsed at // this point. - Assert(TypeHandleIsValid(MemberStructTypeHandle)); + Assert(TypeHandleIsValid(MemberStructTypeHandle)); MemberDecl.TypeHandle = MemberStructTypeHandle; StructDecl.Struct.MemberDecls.PushElementOnBucket(MemberDecl); @@ -1053,6 +1119,7 @@ ParseStruct(token_iter* Iter, type_table_handle* StructTypeHandleOut, gs_meta_pr internal b32 ParseFunctionParameterList (token_iter* Iter, type_definition* FunctionPtrDecl, gs_meta_preprocessor* Meta) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); @@ -1088,16 +1155,17 @@ ParseFunctionParameterList (token_iter* Iter, type_definition* FunctionPtrDecl, internal b32 ParseFunctionDeclaration (token_iter* Iter, token* Identifier, gs_meta_preprocessor* Meta) { + DEBUG_TRACK_FUNCTION; b32 Result = false; PushSnapshot(Iter); type_table_handle ReturnTypeHandle = InvalidTypeTableHandle; if (ParseType(Iter, Meta, &ReturnTypeHandle)) { - if (!TypeHandleIsValid(ReturnTypeHandle)) - { + if (!TypeHandleIsValid(ReturnTypeHandle)) + { ReturnTypeHandle = PushUndeclaredType(Iter->TokenAt->Text, &Meta->TypeTable); - NextToken(Iter); + NextToken(Iter); } b32 IsPointer = ParsePointer(Iter); @@ -1125,17 +1193,18 @@ ParseFunctionDeclaration (token_iter* Iter, token* Identifier, gs_meta_preproces } ApplySnapshotIfNotParsedAndPop(Result, Iter); - if (!Result) - { + if (!Result) + { *Identifier = {0}; } return Result; } -internal b32 +internal b32 ParseTypedef(token_iter* Iter, gs_meta_preprocessor* Meta) { - gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, + DEBUG_TRACK_FUNCTION; + gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseTypedef")); b32 Result = false; @@ -1170,7 +1239,7 @@ ParseTypedef(token_iter* Iter, gs_meta_preprocessor* Meta) NewType.Size = BasisType->Size; CopyMetaTagsAndClear(&Meta->TagList, &NewType.MetaTags); NewType.Type = BasisType->Type; - if (NewType.Type == TypeDef_Struct || + if (NewType.Type == TypeDef_Struct || NewType.Type == TypeDef_Union) { NewType.Struct = BasisType->Struct; @@ -1207,7 +1276,8 @@ ParseTypedef(token_iter* Iter, gs_meta_preprocessor* Meta) internal b32 ParseEnum (token_iter* Iter, gs_meta_preprocessor* Meta) { - gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, + DEBUG_TRACK_FUNCTION; + gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseEnum")); b32 Result = false; @@ -1236,8 +1306,8 @@ ParseEnum (token_iter* Iter, gs_meta_preprocessor* Meta) if (TokenAtEquals(Iter, "=")) { // TODO(Peter): TempValue is just here until we handle all - // const expr that could define an enum value. Its there so - // that if the first token of an expression is a number, + // const expr that could define an enum value. Its there so + // that if the first token of an expression is a number, // we can avoid using anything from the expression. u32 TempValue = EnumAcc; token NumberToken = {}; @@ -1293,7 +1363,9 @@ ParseEnum (token_iter* Iter, gs_meta_preprocessor* Meta) internal b32 ParseFunction (token_iter* Iter, gs_meta_preprocessor* Meta) { - gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, + DEBUG_TRACK_FUNCTION; + + gsm_profiler_scope* ProfilerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseFunction")); b32 Result = false; @@ -1306,7 +1378,7 @@ ParseFunction (token_iter* Iter, gs_meta_preprocessor* Meta) if (TokenAtEquals(Iter, Token_Identifier, &IdentifierToken) && TokenAtEquals(Iter, "(")) { - gsm_profiler_scope* ProfilerInnerScope = BeginScope(&Meta->Profiler, + gsm_profiler_scope* ProfilerInnerScope = BeginScope(&Meta->Profiler, MakeStringLiteral("parse"), MakeStringLiteral("ParseFunctionInner")); type_definition FunctionDecl = {}; @@ -1346,6 +1418,8 @@ ParseFunction (token_iter* Iter, gs_meta_preprocessor* Meta) internal void PrintIndent (u32 Indent) { + DEBUG_TRACK_FUNCTION; + for (u32 i = 0; i < Indent; i++) { printf(" "); @@ -1357,6 +1431,8 @@ internal void PrintStructDecl (type_definition* StructDecl, type_table TypeTable internal void PrintVariableDecl (variable_decl Member, type_table TypeTable, u32 Indent = 0) { + DEBUG_TRACK_FUNCTION; + type_definition* MemberTypeDef = GetTypeDefinition(Member.TypeHandle, TypeTable); if ((MemberTypeDef->Type == TypeDef_Struct || MemberTypeDef->Type == TypeDef_Union) && MemberTypeDef->Identifier.Length == 0) @@ -1389,6 +1465,8 @@ PrintVariableDecl (variable_decl Member, type_table TypeTable, u32 Indent = 0) internal void PrintStructDecl (type_definition* StructDecl, type_table TypeTable, u32 Indent = 0) { + DEBUG_TRACK_FUNCTION; + Assert(StructDecl->Type == TypeDef_Struct || StructDecl->Type == TypeDef_Union); @@ -1422,6 +1500,8 @@ PrintStructDecl (type_definition* StructDecl, type_table TypeTable, u32 Indent = internal void PrintFunctionPtrDecl (type_definition* FnPtrDecl, type_table TypeTable) { + DEBUG_TRACK_FUNCTION; + type_definition* ReturnType = GetTypeDefinition(FnPtrDecl->FunctionPtr.ReturnTypeHandle, TypeTable); printf("%.*s ", StringExpand(ReturnType->Identifier)); @@ -1442,9 +1522,19 @@ PrintFunctionPtrDecl (type_definition* FnPtrDecl, type_table TypeTable) } internal gs_meta_preprocessor -PreprocessProgram (char* SourceFile) +PreprocessProgram (base_allocator Allocator, char* SourceFile) { + GlobalDebugServices.Arena = CreateMemoryArena(Allocator); + GlobalDebugServices.Transient = CreateMemoryArena(Allocator); + + DEBUG_TRACK_FUNCTION; + gs_meta_preprocessor Meta = {}; + Meta.Permanent = CreateMemoryArena(Allocator); + Meta.Transient = CreateMemoryArena(Allocator); + + Meta.Errors.Arena = CreateMemoryArena(Allocator); + Meta.TypeTable.Arena = CreateMemoryArena(Allocator); gsm_profiler_scope* TotalScope = BeginScope(&Meta.Profiler, "total", "total"); @@ -1452,10 +1542,10 @@ PreprocessProgram (char* SourceFile) PopulateTableWithDefaultCPPTypes(&Meta.TypeTable); - string CurrentWorkingDirectory = MakeString((char*)malloc(1024), 0, 1024); + string CurrentWorkingDirectory = MakeString(PushArray(&Meta.Permanent, char, 1024), 0, 1024); string RootFile = MakeString(SourceFile); - AddFileToSource(RootFile, {}, &Meta.SourceFiles, &Meta.Errors); + AddFileToSource(&Meta.Permanent, RootFile, {}, &Meta.SourceFiles, &Meta.Errors); s32 LastSlash = ReverseSearchForCharInSet(RootFile, "\\/"); if (LastSlash <= 0) @@ -1471,28 +1561,22 @@ PreprocessProgram (char* SourceFile) { source_code_file* File = Meta.SourceFiles.GetElementAtIndex(SourceFileIdx); - gsm_profiler_scope* FileScope = BeginScope(&Meta.Profiler, + gsm_profiler_scope* FileScope = BeginScope(&Meta.Profiler, MakeStringLiteral("file"), File->Path); - gsm_profiler_scope* TokenizeScope = BeginScope(&Meta.Profiler, - MakeStringLiteral("tokenize"), - File->Path); - TokenizeFile(File, &Meta.Tokens); - EndScope(TokenizeScope); + TokenizeFile(&Meta, File); - gsm_profiler_scope* PreprocScope = BeginScope(&Meta.Profiler, + gsm_profiler_scope* PreprocScope = BeginScope(&Meta.Profiler, MakeStringLiteral("preproc"), File->Path); token_iter Iter = {}; - Iter.Tokens = &Meta.Tokens; - Iter.FirstToken = File->FirstTokenIndex; - Iter.LastToken = File->LastTokenIndex; - Iter.TokenAtIndex = Iter.FirstToken; - Iter.TokenAt = Meta.Tokens.GetElementAtIndex(Iter.TokenAtIndex); + Iter.Tokens = File->Tokens; + Iter.TokenAtIndex = 0; + Iter.TokenAt = File->Tokens.List + Iter.TokenAtIndex; Iter.Errors = &Meta.Errors; - while (Iter.TokenAtIndex < Iter.LastToken) + while (Iter.TokenAtIndex < Iter.Tokens.Count) { b32 ParseSuccess = false; @@ -1504,19 +1588,19 @@ PreprocessProgram (char* SourceFile) // NOTE(Peter): For now we aren't going in and preprocessing the header files // we include from the system // Token_Operator is used to check if the include is of the form '#include ' - // and skip it. + // and skip it. // TODO(Peter): This is only a rough approximation of ignoring system headers // TODO(Peter): We should actually see what parsing system headers would entail if (IncludeFile->Type != Token_Operator) { string TempFilePath = IncludeFile->Text; - gsm_profiler_scope* IncludeScope = BeginScope(&Meta.Profiler, + gsm_profiler_scope* IncludeScope = BeginScope(&Meta.Profiler, MakeStringLiteral("include"), TempFilePath); // NOTE(Peter): if the path is NOT absolute ie "C:\etc - if (!(IsAlpha(TempFilePath.Memory[0]) && - TempFilePath.Memory[1] == ':' && + if (!(IsAlpha(TempFilePath.Memory[0]) && + TempFilePath.Memory[1] == ':' && TempFilePath.Memory[2] == '\\')) { TempFilePath = CurrentWorkingDirectory; @@ -1527,7 +1611,7 @@ PreprocessProgram (char* SourceFile) ParseSuccess = true; if (!FileAlreadyInSource(TempFilePath, Meta.SourceFiles)) { - AddFileToSource(TempFilePath, *File, &Meta.SourceFiles, &Meta.Errors); + AddFileToSource(&Meta.Permanent, TempFilePath, *File, &Meta.SourceFiles, &Meta.Errors); } EndScope(IncludeScope); } @@ -1600,6 +1684,8 @@ PreprocessProgram (char* SourceFile) internal void FinishMetaprogram(gs_meta_preprocessor* Meta) { + CollateFrame(&GlobalDebugServices); + FinishProfiler(&Meta->Profiler); PrintAllErrors(Meta->Errors); diff --git a/src/meta/gs_meta_code_generator.h b/src/meta/gs_meta_code_generator.h index ba431ad..0986e7f 100644 --- a/src/meta/gs_meta_code_generator.h +++ b/src/meta/gs_meta_code_generator.h @@ -37,12 +37,12 @@ struct gsm_code_generator // --------------- internal gsm_code_generator -BeginEnumGeneration(string EnumIdentifier, string ValuePrefix, b32 StartsWithInvalid, b32 EndsWithCount) +BeginEnumGeneration(memory_arena* Storage, string EnumIdentifier, string ValuePrefix, b32 StartsWithInvalid, b32 EndsWithCount) { gsm_code_generator Gen = {}; // TODO(Peter): TEMP!! - Gen.Builder = (string_builder*)malloc(sizeof(string_builder)); + Gen.Builder = PushStruct(Storage, string_builder); *Gen.Builder = {}; Gen.Type = gsm_CodeGen_Enum; @@ -60,9 +60,9 @@ BeginEnumGeneration(string EnumIdentifier, string ValuePrefix, b32 StartsWithInv } internal gsm_code_generator -BeginEnumGeneration(char* EnumIdentifier, char* ValuePrefix, b32 StartsWithInvalid, b32 EndsWithCount) +BeginEnumGeneration(memory_arena* Storage, char* EnumIdentifier, char* ValuePrefix, b32 StartsWithInvalid, b32 EndsWithCount) { - return BeginEnumGeneration(MakeStringLiteral(EnumIdentifier), + return BeginEnumGeneration(Storage, MakeStringLiteral(EnumIdentifier), MakeStringLiteral(ValuePrefix), StartsWithInvalid, EndsWithCount); diff --git a/src/meta/gs_meta_error.h b/src/meta/gs_meta_error.h index 2648959..057f6ab 100644 --- a/src/meta/gs_meta_error.h +++ b/src/meta/gs_meta_error.h @@ -9,12 +9,13 @@ struct error_buffer #define ERROR_BUFFER_SIZE 256 struct errors { + memory_arena Arena; error_buffer* Buffers; u32 BuffersCount; u32 Used; }; -internal void +internal void PushFError (errors* Errors, char* Format, ...) { if (Errors->Used >= (Errors->BuffersCount * ERROR_BUFFER_SIZE)) @@ -23,8 +24,8 @@ PushFError (errors* Errors, char* Format, ...) Errors->Buffers = (error_buffer*)realloc(Errors->Buffers, sizeof(error_buffer*) * Errors->BuffersCount); error_buffer* NewBuffer = Errors->Buffers + (Errors->BuffersCount - 1); - NewBuffer->Backbuffer = (char*)malloc(sizeof(char) * ERROR_MAX_LENGTH * ERROR_BUFFER_SIZE); - NewBuffer->Contents = (string*)malloc(sizeof(string) * ERROR_BUFFER_SIZE); + NewBuffer->Backbuffer = PushArray(&Errors->Arena, char, ERROR_MAX_LENGTH * ERROR_BUFFER_SIZE); + NewBuffer->Contents = PushArray(&Errors->Arena, string, ERROR_BUFFER_SIZE); for (u32 i = 0; i < ERROR_BUFFER_SIZE; i++) { diff --git a/src/meta/gs_meta_type_table.h b/src/meta/gs_meta_type_table.h index 3eb2140..1411035 100644 --- a/src/meta/gs_meta_type_table.h +++ b/src/meta/gs_meta_type_table.h @@ -10,7 +10,7 @@ enum type_definition_type TypeDef_Invalid, // NOTE(Peter): tokens with this type require fixup later - TypeDef_Unknown, + TypeDef_Unknown, TypeDef_Enum, TypeDef_Struct, TypeDef_Union, @@ -59,7 +59,7 @@ struct meta_tag struct variable_decl { - // NOTE(Peter): Because of the way the tokenizer works, we don't lex and parse + // NOTE(Peter): Because of the way the tokenizer works, we don't lex and parse // at the same time. This means that not all types will be able to be matched // up on the first pass through. A TypeIndex of -1 means we need to fixup that // type at a later time @@ -141,6 +141,8 @@ struct meta_tag_hash_bucket struct type_table { + memory_arena Arena; + type_table_hash_bucket* Types; u32 TypeBucketsCount; @@ -161,7 +163,7 @@ HashIdentifier(string Identifier) u32 IdentHash = HashString(Identifier); if (IdentHash == 0) { - // NOTE(Peter): We are excluding a has of zero so taht + // NOTE(Peter): We are excluding a has of zero so taht // the type_table_handle where BucketIndex and IndexInBucket // are both zero is an invalid handle IdentHash += 1; @@ -292,8 +294,8 @@ FindSlotForTypeIdentifier(u32 IdentHash, type_table* TypeTable) TypeTable->Types = (type_table_hash_bucket*)realloc(TypeTable->Types, NewTypesSize); type_table_hash_bucket* NewBucket = TypeTable->Types + NewTypeBucketIndex; - NewBucket->Keys = (u32*)malloc(sizeof(u32) * TYPE_TABLE_BUCKET_MAX); - NewBucket->Values = (type_definition*)malloc(sizeof(type_definition) * TYPE_TABLE_BUCKET_MAX); + NewBucket->Keys = PushArray(&TypeTable->Arena, u32, TYPE_TABLE_BUCKET_MAX); + NewBucket->Values = PushArray(&TypeTable->Arena, type_definition, TYPE_TABLE_BUCKET_MAX); GSZeroMemory((u8*)NewBucket->Keys, sizeof(u32) * TYPE_TABLE_BUCKET_MAX); GSZeroMemory((u8*)NewBucket->Values, sizeof(type_definition) * TYPE_TABLE_BUCKET_MAX); @@ -301,7 +303,7 @@ FindSlotForTypeIdentifier(u32 IdentHash, type_table* TypeTable) Result.IndexInBucket = Index; } - // NOTE(Peter): Because we are growing the hashtable, this should never be an invalid + // NOTE(Peter): Because we are growing the hashtable, this should never be an invalid // type handle Assert(TypeHandleIsValid(Result)); return Result; @@ -331,8 +333,8 @@ FindSlotForMetaTag(u32 IdentHash, type_table* TypeTable) TypeTable->MetaTags = (meta_tag_hash_bucket*)realloc(TypeTable->MetaTags, NewMetaBucketListSize); meta_tag_hash_bucket* NewBucket = TypeTable->MetaTags + NewMetaBucketIndex; - NewBucket->Keys = (u32*)malloc(sizeof(u32) * TYPE_TABLE_BUCKET_MAX); - NewBucket->Values = (meta_tag*)malloc(sizeof(meta_tag) * TYPE_TABLE_BUCKET_MAX); + NewBucket->Keys = PushArray(&TypeTable->Arena, u32, TYPE_TABLE_BUCKET_MAX); + NewBucket->Values = PushArray(&TypeTable->Arena, meta_tag, TYPE_TABLE_BUCKET_MAX); GSZeroMemory((u8*)NewBucket->Keys, sizeof(u32) * TYPE_TABLE_BUCKET_MAX); GSZeroMemory((u8*)NewBucket->Values, sizeof(meta_tag) * TYPE_TABLE_BUCKET_MAX); @@ -391,7 +393,7 @@ PushMetaTagOnTable(meta_tag Tag, type_table* TypeTable) } // Guaranteed to return a valid result -internal type_definition* +internal type_definition* GetTypeDefinition(type_table_handle Handle, type_table TypeTable) { Assert(TypeHandleIsValid(Handle)); @@ -404,7 +406,7 @@ GetTypeDefinition(type_table_handle Handle, type_table TypeTable) } // May return zero -internal type_definition* +internal type_definition* GetTypeDefinitionUnsafe(type_table_handle Handle, type_table TypeTable) { type_definition* Result = 0; @@ -426,7 +428,7 @@ GetMetaTag(type_table_handle Handle, type_table TypeTable) return Result; } -internal type_definition* +internal type_definition* GetTypeDefinition(string Identifier, type_table TypeTable) { type_definition* Result = 0; @@ -588,21 +590,21 @@ internal void FixupStructMember (variable_decl* Member, type_definition* MemberTypeDef, type_table TypeTable, errors* Errors) { // NOTE(Peter): There are a lot of cases where struct members which are pointers - // to other structs cause interesting behavior here. + // to other structs cause interesting behavior here. // For example: // struct foo { foo* Next; } // could cause infinite loops if we try and fixup all structs with a size of 0 - // which would happen in this case, because we wouldn't have parsed foo's size + // which would happen in this case, because we wouldn't have parsed foo's size // yet, but would begin fixing up foo because of the type of Next // Another example: // typedef struct bar bar; // struct foo { bar* Bar; } // struct bar { foo* Foo; } - // causes the exact same problem, but we cant detect it by just excluding - // fixing up StructIndex recursively. - // + // causes the exact same problem, but we cant detect it by just excluding + // fixing up StructIndex recursively. + // // TL;DR - // The solution I've chosen to go with is just exclude all pointer members from + // The solution I've chosen to go with is just exclude all pointer members from // causing recursive fixups. Those types should be fixed up at some point in the // process, and we already know how big a pointer is in memory, no matter the type if (!Member->Pointer) @@ -617,7 +619,7 @@ FixupStructMember (variable_decl* Member, type_definition* MemberTypeDef, type_t { FixUpUnionSize(Member->TypeHandle, TypeTable, Errors); } - else + else { if (MemberTypeDef->Type == TypeDef_Unknown) { @@ -672,7 +674,7 @@ FixUpStructSize (type_table_handle TypeHandle, type_table TypeTable, errors* Err { // NOTE(Peter): Because its recursive (it makes sure all type sizes become known // if it needs them) we should never get to the end of this function and not have - // the ability to tell how big something is. + // the ability to tell how big something is. // TODO(Peter): We don't parse all types yet however, so for now, this is just an alert, // not an assert; #if 0 diff --git a/src/meta/gs_meta_typeinfo_generator.h b/src/meta/gs_meta_typeinfo_generator.h index c26cf4a..35008e7 100644 --- a/src/meta/gs_meta_typeinfo_generator.h +++ b/src/meta/gs_meta_typeinfo_generator.h @@ -3,9 +3,9 @@ // Author: Peter Slattery // Creation Date: 2020-01-19 // -// Usage +// Usage // TODO -// +// // #ifndef GS_META_TYPEINFO_GENERATOR_H @@ -31,15 +31,15 @@ struct typeinfo_generator #define TypeHandleToIndex(handle) ((handle.BucketIndex * TYPE_TABLE_BUCKET_MAX) + handle.IndexInBucket) internal typeinfo_generator -InitTypeInfoGenerator(type_table TypeTable) +InitTypeInfoGenerator(type_table* TypeTable) { typeinfo_generator Result = {}; - Result.TypesMax = TypeTable.TypeBucketsCount * TYPE_TABLE_BUCKET_MAX; - Result.TypesGeneratedMask = (b8*)malloc(sizeof(b8) * Result.TypesMax); + Result.TypesMax = TypeTable->TypeBucketsCount * TYPE_TABLE_BUCKET_MAX; + Result.TypesGeneratedMask = PushArray(&TypeTable->Arena, b8, Result.TypesMax); GSZeroMemory((u8*)Result.TypesGeneratedMask, Result.TypesMax); - Result.TypeList = BeginEnumGeneration("gsm_struct_type", "gsm_StructType", false, true); + Result.TypeList = BeginEnumGeneration(&TypeTable->Arena, "gsm_struct_type", "gsm_StructType", false, true); WriteF(&Result.TypeDefinitions, "static gsm_struct_type_info StructTypes[] = {\n"); return Result; @@ -73,7 +73,7 @@ internal void GenerateStructMemberInfo (variable_decl* Member, string StructIdentifier, type_table TypeTable, typeinfo_generator* Gen) { WriteF(&Gen->StructMembers, "{ \"%S\", %d, ", Member->Identifier, Member->Identifier.Length); - WriteF(&Gen->StructMembers, "(u64)&((%S*)0)->%S, ", StructIdentifier, Member->Identifier); + WriteF(&Gen->StructMembers, "(u64)&((%S*)0)->%S, ", StructIdentifier, Member->Identifier); GenerateMetaTagInfo(Member->MetaTags, TypeTable, &Gen->StructMembers); WriteF(&Gen->StructMembers, "},\n"); } @@ -81,7 +81,7 @@ GenerateStructMemberInfo (variable_decl* Member, string StructIdentifier, type_t internal void GenerateTypeInfo (type_definition* Type, type_table_handle TypeHandle, type_table TypeTable, typeinfo_generator* Generator) { - // TODO(Peter): + // TODO(Peter): // 1. allocate the full range of the types hash table // 2. use bucketindex * bucket_max + indexinbucket to get the consecutive index Generator->TypesGeneratedMask[TypeHandleToIndex(TypeHandle)] = true; @@ -114,7 +114,7 @@ GenerateTypeInfo (type_definition* Type, type_table_handle TypeHandle, type_tabl } } - if (Type->Type == TypeDef_Struct || + if (Type->Type == TypeDef_Struct || Type->Type == TypeDef_Union) { for (u32 m = 0; m < Type->Struct.MemberDecls.Used; m++) @@ -124,10 +124,10 @@ GenerateTypeInfo (type_definition* Type, type_table_handle TypeHandle, type_tabl if ((MemberType->Type == TypeDef_Struct || MemberType->Type == TypeDef_Union) && - MemberType->Struct.IsAnonymous) - { + MemberType->Struct.IsAnonymous) + { continue; // Don't gen info for anonymous struct and union members - } + } if (Generator->TypesGeneratedMask[TypeHandleToIndex(Member->TypeHandle)]) { continue; } diff --git a/src/todo.txt b/src/todo.txt index a9edcb9..cc8e282 100644 --- a/src/todo.txt +++ b/src/todo.txt @@ -3,6 +3,28 @@ TODO FOLDHAUS STREAM #0: Metaprogramming - Metaprogramming - fix memory layout (remeber to profile before and after) + - use a base_allocator struct to track allocations to begin with + - introduce memory_arenas + - get rid of gs_bucket and gs_list or whatever, just use custom bucket lists + +Profile 0: +Total Allocated Space: 1186150 bytes, or 1.131201 MB +Total Time: 0.062665 seconds + +Profile 1: +Total Allocated Space: 1217890 bytes, or 1.161470 MB +Total Time: 0.061852 seconds + + + + + + + + + + - convert gs_memory_arena to using 64 bit integers + - investigate centralizing type table type creation in a similar place, rather than scattered all over - Make everything truly platform agnostic - Application DLL - math.h: present for trig functions (though this is part of the c-std lib, so it should be everywhere)