#include "..\src\gs_language.h" #include "..\src\gs_string.h" #include "gs_meta_error.h" #include "gs_meta_lexer.h" #ifndef MAX_PATH #define MAX_PATH 512 #endif // MAX_PATH #define META_FILES_ARRAY_INCREMENT_SIZE 32 struct file_contents { u8* Backbuffer; string Buffer; }; #define TOKEN_BUFFER_SIZE 256 struct token_buffer { token* Tokens; s32 Used; s32 Max; token_buffer* Next; }; struct token_list { token_buffer* Head; s32 TotalUsed; s32 TotalMax; }; struct token_list_iterator { token_list* List; token_buffer* BufferAt; token* At; s32 AtTotalIndex; }; enum basic_type { BasicType_Int8, BasicType_Int16, BasicType_Int32, BasicType_Int64, BasicType_UnsignedInt8, BasicType_UnsignedInt16, BasicType_UnsignedInt32, BasicType_UnsignedInt64, BasicType_Char, BasicType_Float32, BasicType_Float64, BasicType_Void, }; struct variable_definition_symbol { string Identifier; basic_type Type; b32 IsArray; s32 Size; }; struct struct_definition_symbol { string Identifier; s32 MemberCount; variable_definition_symbol* Members; }; struct function_definition_symbol { string Identifier; s32 ArgumentCount; variable_definition_symbol* Arguments; // TODO(Peter): An AST of the actual code that makes up the function? }; enum symbol_type { SymbolType_Function, SymbolType_Struct }; struct symbol { symbol_type Type; union { struct_definition_symbol StructDef; function_definition_symbol FunctionDef; }; }; enum parser_symbol_type { ParserSymbol_Invalid, ParserSymbol_Unknown, ParserSymbol_Auto, ParserSymbol_Break, ParserSymbol_Case, ParserSymbol_Char, ParserSymbol_Const, ParserSymbol_Continue, ParserSymbol_Default, ParserSymbol_Do, ParserSymbol_Double, ParserSymbol_Else, ParserSymbol_Enum, ParserSymbol_Extern, ParserSymbol_Float, ParserSymbol_For, ParserSymbol_Goto, ParserSymbol_If, ParserSymbol_Inline, ParserSymbol_Int, ParserSymbol_Long, ParserSymbol_Register, ParserSymbol_Restrict, ParserSymbol_Return, ParserSymbol_Short, ParserSymbol_Signed, ParserSymbol_Sizeof, ParserSymbol_Static, ParserSymbol_Struct, ParserSymbol_Switch, ParserSymbol_Typedef, ParserSymbol_Union, ParserSymbol_Unsigned, ParserSymbol_Void, ParserSymbol_Volatile, ParserSymbol_While, // Assignment Operators ParserSymbol_AssignEquals, ParserSymbol_PlusEquals, ParserSymbol_MinusEquals, ParserSymbol_TimesEquals, ParserSymbol_DividedEquals, ParserSymbol_ModEquals, ParserSymbol_BitAndEquals, ParserSymbol_BitOrEquals, ParserSymbol_BitNotEquals, ParserSymbol_BitShiftUpEquals, ParserSymbol_BitShiftDownEquals, // Increment/Decrement Operators ParserSymbol_Increment, ParserSymbol_Decrement, }; struct parser_symbol_definition { parser_symbol_type Type; string Text; }; global_variable parser_symbol_definition ParserSymbolTable[] = { // Keywords { ParserSymbol_Auto, MakeStringLiteral("auto")}, { ParserSymbol_Break, MakeStringLiteral("break")}, { ParserSymbol_Case, MakeStringLiteral("case")}, { ParserSymbol_Char, MakeStringLiteral("char")}, { ParserSymbol_Const, MakeStringLiteral("const")}, { ParserSymbol_Continue, MakeStringLiteral("continue")}, { ParserSymbol_Default, MakeStringLiteral("default")}, { ParserSymbol_Do, MakeStringLiteral("do")}, { ParserSymbol_Double, MakeStringLiteral("double")}, { ParserSymbol_Else, MakeStringLiteral("else")}, { ParserSymbol_Enum, MakeStringLiteral("enum")}, { ParserSymbol_Extern, MakeStringLiteral("extern")}, { ParserSymbol_Float, MakeStringLiteral("float")}, { ParserSymbol_For, MakeStringLiteral("for")}, { ParserSymbol_Goto, MakeStringLiteral("goto")}, { ParserSymbol_If, MakeStringLiteral("if")}, { ParserSymbol_Inline, MakeStringLiteral("inline")}, { ParserSymbol_Int, MakeStringLiteral("int")}, { ParserSymbol_Long, MakeStringLiteral("long")}, { ParserSymbol_Register, MakeStringLiteral("register")}, { ParserSymbol_Restrict, MakeStringLiteral("restrict")}, { ParserSymbol_Return, MakeStringLiteral("return")}, { ParserSymbol_Short, MakeStringLiteral("short")}, { ParserSymbol_Signed, MakeStringLiteral("signed")}, { ParserSymbol_Sizeof, MakeStringLiteral("sizeof")}, { ParserSymbol_Static, MakeStringLiteral("static")}, { ParserSymbol_Struct, MakeStringLiteral("struct")}, { ParserSymbol_Switch, MakeStringLiteral("switch")}, { ParserSymbol_Typedef, MakeStringLiteral("typedef")}, { ParserSymbol_Union, MakeStringLiteral("union")}, { ParserSymbol_Unsigned, MakeStringLiteral("unsigned")}, { ParserSymbol_Void, MakeStringLiteral("void")}, { ParserSymbol_Volatile, MakeStringLiteral("volatile")}, { ParserSymbol_While, MakeStringLiteral("while")}, // Assignment Operators { ParserSymbol_AssignEquals, MakeStringLiteral("=")}, { ParserSymbol_PlusEquals, MakeStringLiteral("+=")}, { ParserSymbol_MinusEquals, MakeStringLiteral("-=")}, { ParserSymbol_TimesEquals, MakeStringLiteral("*=")}, { ParserSymbol_DividedEquals, MakeStringLiteral("/=")}, { ParserSymbol_ModEquals, MakeStringLiteral("%=")}, { ParserSymbol_BitAndEquals, MakeStringLiteral("&=")}, { ParserSymbol_BitOrEquals, MakeStringLiteral("|=")}, { ParserSymbol_BitNotEquals, MakeStringLiteral("^=")}, { ParserSymbol_BitShiftUpEquals, MakeStringLiteral("<<=")}, { ParserSymbol_BitShiftDownEquals, MakeStringLiteral(">>=")}, // Increment/Decrement Operators { ParserSymbol_Increment, MakeStringLiteral("++")}, { ParserSymbol_Decrement, MakeStringLiteral("--")}, /* // Arithmetic Operators { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, { ParserSymbol_, MakeStringLiteral("")}, */ }; struct gs_meta_processor { error_list ErrorList; u8* FileStringMemory; s32 FileStringMemorySize; string* Files; file_contents* Contents; token_list* FileTokens; s32 FilesUsed; s32 FilesMax; }; internal gs_meta_processor CreateMetaProcessor () { gs_meta_processor Result = {}; return Result; } internal void SetMetaFilesStringMemory (gs_meta_processor* Meta, s32 StartIndex, s32 EndIndex) { char* StringMemoryAt = (char*)(Meta->FileStringMemory + (MAX_PATH * StartIndex)); for (s32 i = StartIndex; i < EndIndex; i++) { string* String = Meta->Files + i; InitializeString(String, StringMemoryAt, MAX_PATH); StringMemoryAt += MAX_PATH; } } internal b32 IsValidFileType(char* Filename) { string FileString = MakeString(Filename); s32 IndexOfDot = LastIndexOfChar(FileString, '.'); string ExtensionString = Substring(FileString, IndexOfDot); b32 Result = false; if (StringsEqual(ExtensionString, MakeStringLiteral("cpp")) || StringsEqual(ExtensionString, MakeStringLiteral("h"))) { Result = true; } return Result; } internal void AddFile(gs_meta_processor* Meta, char* Filename) { if (!IsValidFileType(Filename)) { LogError(&Meta->ErrorList, "AddFile: %s has an extension that cannot be processed.", Filename); } if(!Meta->Files) { Meta->FilesMax = META_FILES_ARRAY_INCREMENT_SIZE; Meta->FilesUsed = 0; Meta->Files = (string*)malloc(sizeof(string) * Meta->FilesMax); Meta->FileStringMemorySize = Meta->FilesMax * MAX_PATH; Meta->FileStringMemory = (u8*)malloc(Meta->FileStringMemorySize); SetMetaFilesStringMemory(Meta, 0, Meta->FilesMax); } if (Meta->Files && Meta->FilesUsed >= Meta->FilesMax) { Meta->FilesMax += META_FILES_ARRAY_INCREMENT_SIZE; Meta->Files = (string*)realloc(Meta->Files, sizeof(string) * Meta->FilesMax); Meta->FileStringMemorySize = Meta->FilesMax * MAX_PATH; Meta->FileStringMemory = (u8*)realloc(Meta->FileStringMemory, Meta->FileStringMemorySize); SetMetaFilesStringMemory(Meta, Meta->FilesUsed, Meta->FilesMax); } string* File = Meta->Files + Meta->FilesUsed++; File->Length = CharArrayLength(Filename); GSMemCopy(Filename, File->Memory, File->Length); NullTerminate(File); } internal void LoadFileContents (file_contents* Contents, string* FileName, error_list* Errors) { FILE* ReadFile = fopen(FileName->Memory, "r"); if (ReadFile) { fseek(ReadFile, 0, SEEK_END); size_t FileSize = ftell(ReadFile); fseek(ReadFile, 0, SEEK_SET); if (FileSize > 0) { Contents->Backbuffer = (u8*)malloc(FileSize + 1); size_t ReadSize = fread(Contents->Backbuffer, 1, FileSize, ReadFile); Contents->Backbuffer[FileSize] = 0; InitializeString(&Contents->Buffer, (char*)Contents->Backbuffer, (s32)FileSize + 1); Contents->Buffer.Length = (s32)FileSize + 1; } else { LogError(Errors, "LoadFileContents: File Size Was Zero for %s", FileName->Memory); } fclose(ReadFile); } else { LogError(Errors, "LoadFileContents: Could Not Read File %s", FileName->Memory); } } internal void LoadAllFiles (gs_meta_processor* Meta) { Meta->Contents = (file_contents*)malloc(sizeof(file_contents) * Meta->FilesUsed); for (s32 FileIdx = 0; FileIdx < Meta->FilesUsed; FileIdx++) { file_contents* Contents = Meta->Contents + FileIdx; string* FileName = Meta->Files + FileIdx; LoadFileContents(Contents, FileName, &Meta->ErrorList); } } internal token* PushTokenOnList(token_list* List) { token* Result = 0; if (!List->Head) { s32 NewBufferSize = sizeof(token_buffer) + (sizeof(token) * TOKEN_BUFFER_SIZE); u8* NewBufferContents = (u8*)malloc(NewBufferSize); List->Head = (token_buffer*)NewBufferContents; List->Head->Used = 0; List->Head->Max = TOKEN_BUFFER_SIZE; List->Head->Tokens = (token*)(NewBufferContents + sizeof(token_buffer)); List->TotalUsed = List->Head->Used; List->TotalMax = List->Head->Max; } else if (List->TotalUsed >= List->TotalMax) { s32 NewBufferSize = sizeof(token_buffer) + (sizeof(token) * TOKEN_BUFFER_SIZE); u8* NewBufferContents = (u8*)malloc(NewBufferSize); token_buffer* NewBuffer = (token_buffer*)NewBufferContents; NewBuffer->Tokens = (token*)(NewBufferContents + sizeof(token_buffer)); NewBuffer->Next = NewBuffer; List->Head = NewBuffer; List->TotalUsed += List->Head->Used; List->TotalMax += List->Head->Max; } Result = List->Head->Tokens + List->Head->Used++; List->TotalUsed++; return Result; } internal void LexFile (file_contents* Contents, token_list* TokensList) { tokenizer FileTokenizer = {}; FileTokenizer.Memory = Contents->Buffer.Memory; FileTokenizer.MemoryLength = Contents->Buffer.Length; FileTokenizer.At = FileTokenizer.Memory; while (AtValidPosition(FileTokenizer)) { token* Token = PushTokenOnList(TokensList); *Token = GetNextToken(&FileTokenizer); } } internal void LexAllFiles (gs_meta_processor* Meta) { LoadAllFiles(Meta); Meta->FileTokens = (token_list*)malloc(sizeof(token_list) * Meta->FilesUsed); for (s32 SourceFile = 0; SourceFile < Meta->FilesUsed; SourceFile++) { Meta->FileTokens[SourceFile] = {}; token_list* FileTokenList = Meta->FileTokens + SourceFile; file_contents* FileContents = Meta->Contents + SourceFile; LexFile (FileContents, FileTokenList); } } internal token_list_iterator GetTokenIterator(token_list* List) { token_list_iterator Result = {}; Result.List = List; Result.BufferAt = List->Head; Result.At = List->Head->Tokens; Result.AtTotalIndex = 0; return Result; } internal b32 IsValid (token_list_iterator Iterator) { b32 Result = false; if (Iterator.At && Iterator.AtTotalIndex < Iterator.List->TotalUsed) { Result = true; } return Result; } internal void Advance (token_list_iterator* Iterator) { if ((Iterator->At - Iterator->BufferAt->Tokens) >= Iterator->BufferAt->Used) { Iterator->BufferAt = Iterator->BufferAt->Next; if (Iterator->BufferAt) { Iterator->At = Iterator->BufferAt->Tokens; Iterator->AtTotalIndex++; } else { Iterator->At = 0; } } else { Iterator->At++; Iterator->AtTotalIndex++; } } internal parser_symbol_type GetSymbolType (string Text, parser_symbol_type Fallback) { parser_symbol_type Result = Fallback; for (s32 i = 0; i < (sizeof(ParserSymbolTable)/sizeof(ParserSymbolTable[0])); i++) { if (StringsEqual(Text, ParserSymbolTable[i].Text)) { Result = ParserSymbolTable[i].Type; break; } } return Result; } internal void ParseFile (token_list* TokenList, gs_meta_processor* Meta) { for (token_list_iterator TokenIter = GetTokenIterator(TokenList); IsValid(TokenIter); Advance(&TokenIter)) { token* At = TokenIter.At; parser_symbol_type SymbolType = GetSymbolType(At->Text, ParserSymbol_Unknown); switch (SymbolType) { case SymbolType_Struct: { printf("Found Struct: %.*s\n", (At + 1)->Text.Length, (At + 1)->Text.Memory); }break; } } } internal void ParseAllFiles (gs_meta_processor* Meta) { for (s32 SourceFile = 0; SourceFile < Meta->FilesUsed; SourceFile++) { token_list* FileTokenList = Meta->FileTokens + SourceFile; ParseFile(FileTokenList, Meta); } } #if 0 #include #define GS_PRINTF(v) OutputDebugStringA(v) #include "gs_string.h" static char DEBUGCharArray[256]; #define GS_DEBUG_PRINTF(format, ...) sprintf(DEBUGCharArray, format, __VA_ARGS__); OutputDebugStringA(DEBUGCharArray); #include "gs_meta_lexer.h" #include "gs_meta_parser.h" #include "gs_meta_generator.h" internal char* ReadEntireFileAndNullTerminate (char* Filename) { char* Result = 0; FILE* ReadFile = fopen(Filename, "r"); if (ReadFile) { fseek(ReadFile, 0, SEEK_END); size_t FileSize = ftell(ReadFile); fseek(ReadFile, 0, SEEK_SET); Result = (char*)malloc(FileSize + 1); size_t ReadSize = fread(Result, 1, FileSize, ReadFile); Result[FileSize] = 0; fclose(ReadFile); } else { GS_DEBUG_PRINTF("Failed to fopen file"); } return Result; } int PrecompileFile (char* FileName) { char* SourceFileContents = ReadEntireFileAndNullTerminate(FileName); if (!SourceFileContents) { GS_DEBUG_PRINTF("Failed to get source file contents"); return 0; } tokenizer Tokenizer = {}; Tokenizer.At = SourceFileContents; s32 TokensGrowSize = 256; s32 TokensMax = TokensGrowSize; s32 TokensCount = 0; token* Tokens = (token*)malloc(TokensGrowSize * sizeof(token)); token* Current = Tokens; while (Tokenizer.At[0]) { *Current = GetNextToken(&Tokenizer); Current->Next = 0; TokensCount++; if (TokensCount >= TokensMax) { token* Addition = (token*)malloc(TokensGrowSize * sizeof(token)); TokensMax += TokensGrowSize; token* Old = Current; Current = Addition; Old->Next = Current; } else if(Tokenizer.At[0]) { token* Old = Current; Current++; Old->Next = Current; } } Current = Tokens; #if 0 // Print Tokens if (Current) { do { GS_DEBUG_PRINTF("%s %.*s\n", TokenNames[(int)Current->Type], Current->TextLength, Current->Text); Current = Current->Next; }while(Current); } #endif ast_node Ast = {}; Ast.Type = ASTNode_Program; ast_node* CurrentNode = Ast.Children; // Preprocessor Defines // TODO(Peter): Actually preprocess the file token* CurrentToken = Tokens; /*while (CurrentToken) { CurrentToken = CurrentToken->Next; } */ // Analyze File CurrentToken = Tokens; while (CurrentToken) { /*GS_DEBUG_PRINTF("%s %.*s\n", TokenNames[(int)CurrentToken->Type], CurrentToken->TextLength, CurrentToken->Text); */ parse_result ParseResult = {}; ParseResult.Parsed = false; if (CurrentToken->Type == Token_Identifier) { if (StringsEqual(CurrentToken->Text, "struct", 6)) { ParseResult = ParseStructDeclaration(CurrentNode, CurrentToken); CurrentToken = ParseResult.NextToken; } else if (IsFunction(CurrentToken)) { ParseResult = ParseFunctionDeclaration(CurrentNode, CurrentToken); CurrentToken = ParseResult.NextToken; } else { CurrentToken = CurrentToken->Next; } if (ParseResult.Parsed) { if (CurrentNode == 0) { Ast.Children = ParseResult.Node; CurrentNode = Ast.Children; } else { CurrentNode->Next = ParseResult.Node; CurrentNode = CurrentNode->Next; } } } else { CurrentToken = CurrentToken->Next; } } string_partition StringPart = {}; { memory_arena StringMemory = {}; InitMemoryArena(&StringMemory, (u8*)malloc(Megabytes(4)), Megabytes(4)); InitStringPartition(&StringPart, &StringMemory); } // Print Structs ast_node* Node = Ast.Children; while (Node) { switch (Node->Type) { case ASTNode_StructDeclaration: { string_buffer* CodeBuffer = GenerateStructCode(Node, &StringPart); PrintStringBuffer(CodeBuffer); }break; case ASTNode_FunctionDeclaration: { string_buffer* CodeBuffer = GenerateFunctionDeclaration(Node, &StringPart); PrintStringBuffer(CodeBuffer); }break; } Node = Node->Next; } return 0; } int main(int ArgCount, char** ArgV) { if (ArgCount <= 1) { printf("Please supply at least one source directory to analyze.\n"); return 0; } TCHAR Win32FileNameBuffer[MAX_PATH]; char FullFilePath[MAX_PATH]; for (int i = 1; i < ArgCount; i++) { WIN32_FIND_DATA FindFileData; StringCchCopy(Win32FileNameBuffer, MAX_PATH, ArgV[i]); StringCchCat(Win32FileNameBuffer, MAX_PATH, TEXT("\\*")); HANDLE CurrentFile = FindFirstFile(Win32FileNameBuffer, &FindFileData); int FileIdx = 0; do { if (FindFileData.dwFileAttributes == FILE_ATTRIBUTE_DIRECTORY) { // TODO(Peter): Recurse? continue; } printf("File %d: %s\n", FileIdx++, FindFileData.cFileName); } while (FindNextFile(CurrentFile, &FindFileData)); } } #endif