Lumenarium/meta/gs_meta.cpp

747 lines
20 KiB
C++

#include "..\src\gs_language.h"
#include "..\src\gs_string.h"
#include "gs_meta_error.h"
#include "gs_meta_lexer.h"
#ifndef MAX_PATH
#define MAX_PATH 512
#endif // MAX_PATH
#define META_FILES_ARRAY_INCREMENT_SIZE 32
struct file_contents
{
u8* Backbuffer;
string Buffer;
};
#define TOKEN_BUFFER_SIZE 256
struct token_buffer
{
token* Tokens;
s32 Used;
s32 Max;
token_buffer* Next;
};
struct token_list
{
token_buffer* Head;
s32 TotalUsed;
s32 TotalMax;
};
struct token_list_iterator
{
token_list* List;
token_buffer* BufferAt;
token* At;
s32 AtTotalIndex;
};
enum basic_type
{
BasicType_Int8,
BasicType_Int16,
BasicType_Int32,
BasicType_Int64,
BasicType_UnsignedInt8,
BasicType_UnsignedInt16,
BasicType_UnsignedInt32,
BasicType_UnsignedInt64,
BasicType_Char,
BasicType_Float32,
BasicType_Float64,
BasicType_Void,
};
struct variable_definition_symbol
{
string Identifier;
basic_type Type;
b32 IsArray;
s32 Size;
};
struct struct_definition_symbol
{
string Identifier;
s32 MemberCount;
variable_definition_symbol* Members;
};
struct function_definition_symbol
{
string Identifier;
s32 ArgumentCount;
variable_definition_symbol* Arguments;
// TODO(Peter): An AST of the actual code that makes up the function?
};
enum symbol_type
{
SymbolType_Function,
SymbolType_Struct
};
struct symbol
{
symbol_type Type;
union
{
struct_definition_symbol StructDef;
function_definition_symbol FunctionDef;
};
};
enum parser_symbol_type
{
ParserSymbol_Invalid,
ParserSymbol_Unknown,
ParserSymbol_Auto,
ParserSymbol_Break,
ParserSymbol_Case,
ParserSymbol_Char,
ParserSymbol_Const,
ParserSymbol_Continue,
ParserSymbol_Default,
ParserSymbol_Do,
ParserSymbol_Double,
ParserSymbol_Else,
ParserSymbol_Enum,
ParserSymbol_Extern,
ParserSymbol_Float,
ParserSymbol_For,
ParserSymbol_Goto,
ParserSymbol_If,
ParserSymbol_Inline,
ParserSymbol_Int,
ParserSymbol_Long,
ParserSymbol_Register,
ParserSymbol_Restrict,
ParserSymbol_Return,
ParserSymbol_Short,
ParserSymbol_Signed,
ParserSymbol_Sizeof,
ParserSymbol_Static,
ParserSymbol_Struct,
ParserSymbol_Switch,
ParserSymbol_Typedef,
ParserSymbol_Union,
ParserSymbol_Unsigned,
ParserSymbol_Void,
ParserSymbol_Volatile,
ParserSymbol_While,
// Assignment Operators
ParserSymbol_AssignEquals,
ParserSymbol_PlusEquals,
ParserSymbol_MinusEquals,
ParserSymbol_TimesEquals,
ParserSymbol_DividedEquals,
ParserSymbol_ModEquals,
ParserSymbol_BitAndEquals,
ParserSymbol_BitOrEquals,
ParserSymbol_BitNotEquals,
ParserSymbol_BitShiftUpEquals,
ParserSymbol_BitShiftDownEquals,
// Increment/Decrement Operators
ParserSymbol_Increment,
ParserSymbol_Decrement,
};
struct parser_symbol_definition
{
parser_symbol_type Type;
string Text;
};
global_variable parser_symbol_definition ParserSymbolTable[] =
{
// Keywords
{ ParserSymbol_Auto, MakeStringLiteral("auto")},
{ ParserSymbol_Break, MakeStringLiteral("break")},
{ ParserSymbol_Case, MakeStringLiteral("case")},
{ ParserSymbol_Char, MakeStringLiteral("char")},
{ ParserSymbol_Const, MakeStringLiteral("const")},
{ ParserSymbol_Continue, MakeStringLiteral("continue")},
{ ParserSymbol_Default, MakeStringLiteral("default")},
{ ParserSymbol_Do, MakeStringLiteral("do")},
{ ParserSymbol_Double, MakeStringLiteral("double")},
{ ParserSymbol_Else, MakeStringLiteral("else")},
{ ParserSymbol_Enum, MakeStringLiteral("enum")},
{ ParserSymbol_Extern, MakeStringLiteral("extern")},
{ ParserSymbol_Float, MakeStringLiteral("float")},
{ ParserSymbol_For, MakeStringLiteral("for")},
{ ParserSymbol_Goto, MakeStringLiteral("goto")},
{ ParserSymbol_If, MakeStringLiteral("if")},
{ ParserSymbol_Inline, MakeStringLiteral("inline")},
{ ParserSymbol_Int, MakeStringLiteral("int")},
{ ParserSymbol_Long, MakeStringLiteral("long")},
{ ParserSymbol_Register, MakeStringLiteral("register")},
{ ParserSymbol_Restrict, MakeStringLiteral("restrict")},
{ ParserSymbol_Return, MakeStringLiteral("return")},
{ ParserSymbol_Short, MakeStringLiteral("short")},
{ ParserSymbol_Signed, MakeStringLiteral("signed")},
{ ParserSymbol_Sizeof, MakeStringLiteral("sizeof")},
{ ParserSymbol_Static, MakeStringLiteral("static")},
{ ParserSymbol_Struct, MakeStringLiteral("struct")},
{ ParserSymbol_Switch, MakeStringLiteral("switch")},
{ ParserSymbol_Typedef, MakeStringLiteral("typedef")},
{ ParserSymbol_Union, MakeStringLiteral("union")},
{ ParserSymbol_Unsigned, MakeStringLiteral("unsigned")},
{ ParserSymbol_Void, MakeStringLiteral("void")},
{ ParserSymbol_Volatile, MakeStringLiteral("volatile")},
{ ParserSymbol_While, MakeStringLiteral("while")},
// Assignment Operators
{ ParserSymbol_AssignEquals, MakeStringLiteral("=")},
{ ParserSymbol_PlusEquals, MakeStringLiteral("+=")},
{ ParserSymbol_MinusEquals, MakeStringLiteral("-=")},
{ ParserSymbol_TimesEquals, MakeStringLiteral("*=")},
{ ParserSymbol_DividedEquals, MakeStringLiteral("/=")},
{ ParserSymbol_ModEquals, MakeStringLiteral("%=")},
{ ParserSymbol_BitAndEquals, MakeStringLiteral("&=")},
{ ParserSymbol_BitOrEquals, MakeStringLiteral("|=")},
{ ParserSymbol_BitNotEquals, MakeStringLiteral("^=")},
{ ParserSymbol_BitShiftUpEquals, MakeStringLiteral("<<=")},
{ ParserSymbol_BitShiftDownEquals, MakeStringLiteral(">>=")},
// Increment/Decrement Operators
{ ParserSymbol_Increment, MakeStringLiteral("++")},
{ ParserSymbol_Decrement, MakeStringLiteral("--")},
/*
// Arithmetic Operators
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
{ ParserSymbol_, MakeStringLiteral("")},
*/
};
struct gs_meta_processor
{
error_list ErrorList;
u8* FileStringMemory;
s32 FileStringMemorySize;
string* Files;
file_contents* Contents;
token_list* FileTokens;
s32 FilesUsed;
s32 FilesMax;
};
internal gs_meta_processor
CreateMetaProcessor ()
{
gs_meta_processor Result = {};
return Result;
}
internal void
SetMetaFilesStringMemory (gs_meta_processor* Meta, s32 StartIndex, s32 EndIndex)
{
char* StringMemoryAt = (char*)(Meta->FileStringMemory + (MAX_PATH * StartIndex));
for (s32 i = StartIndex; i < EndIndex; i++)
{
string* String = Meta->Files + i;
InitializeString(String, StringMemoryAt, MAX_PATH);
StringMemoryAt += MAX_PATH;
}
}
internal b32
IsValidFileType(char* Filename)
{
string FileString = MakeString(Filename);
s32 IndexOfDot = LastIndexOfChar(FileString, '.');
string ExtensionString = Substring(FileString, IndexOfDot);
b32 Result = false;
if (StringsEqual(ExtensionString, MakeStringLiteral("cpp")) ||
StringsEqual(ExtensionString, MakeStringLiteral("h")))
{
Result = true;
}
return Result;
}
internal void
AddFile(gs_meta_processor* Meta, char* Filename)
{
if (!IsValidFileType(Filename))
{
LogError(&Meta->ErrorList, "AddFile: %s has an extension that cannot be processed.", Filename);
}
if(!Meta->Files)
{
Meta->FilesMax = META_FILES_ARRAY_INCREMENT_SIZE;
Meta->FilesUsed = 0;
Meta->Files = (string*)malloc(sizeof(string) * Meta->FilesMax);
Meta->FileStringMemorySize = Meta->FilesMax * MAX_PATH;
Meta->FileStringMemory = (u8*)malloc(Meta->FileStringMemorySize);
SetMetaFilesStringMemory(Meta, 0, Meta->FilesMax);
}
if (Meta->Files && Meta->FilesUsed >= Meta->FilesMax)
{
Meta->FilesMax += META_FILES_ARRAY_INCREMENT_SIZE;
Meta->Files = (string*)realloc(Meta->Files, sizeof(string) * Meta->FilesMax);
Meta->FileStringMemorySize = Meta->FilesMax * MAX_PATH;
Meta->FileStringMemory = (u8*)realloc(Meta->FileStringMemory, Meta->FileStringMemorySize);
SetMetaFilesStringMemory(Meta, Meta->FilesUsed, Meta->FilesMax);
}
string* File = Meta->Files + Meta->FilesUsed++;
File->Length = CharArrayLength(Filename);
GSMemCopy(Filename, File->Memory, File->Length);
NullTerminate(File);
}
internal void
LoadFileContents (file_contents* Contents, string* FileName, error_list* Errors)
{
FILE* ReadFile = fopen(FileName->Memory, "r");
if (ReadFile)
{
fseek(ReadFile, 0, SEEK_END);
size_t FileSize = ftell(ReadFile);
fseek(ReadFile, 0, SEEK_SET);
if (FileSize > 0)
{
Contents->Backbuffer = (u8*)malloc(FileSize + 1);
size_t ReadSize = fread(Contents->Backbuffer, 1, FileSize, ReadFile);
Contents->Backbuffer[FileSize] = 0;
InitializeString(&Contents->Buffer, (char*)Contents->Backbuffer, (s32)FileSize + 1);
Contents->Buffer.Length = (s32)FileSize + 1;
}
else
{
LogError(Errors, "LoadFileContents: File Size Was Zero for %s", FileName->Memory);
}
fclose(ReadFile);
}
else
{
LogError(Errors, "LoadFileContents: Could Not Read File %s", FileName->Memory);
}
}
internal void
LoadAllFiles (gs_meta_processor* Meta)
{
Meta->Contents = (file_contents*)malloc(sizeof(file_contents) * Meta->FilesUsed);
for (s32 FileIdx = 0; FileIdx < Meta->FilesUsed; FileIdx++)
{
file_contents* Contents = Meta->Contents + FileIdx;
string* FileName = Meta->Files + FileIdx;
LoadFileContents(Contents, FileName, &Meta->ErrorList);
}
}
internal token*
PushTokenOnList(token_list* List)
{
token* Result = 0;
if (!List->Head)
{
s32 NewBufferSize = sizeof(token_buffer) + (sizeof(token) * TOKEN_BUFFER_SIZE);
u8* NewBufferContents = (u8*)malloc(NewBufferSize);
List->Head = (token_buffer*)NewBufferContents;
List->Head->Used = 0;
List->Head->Max = TOKEN_BUFFER_SIZE;
List->Head->Tokens = (token*)(NewBufferContents + sizeof(token_buffer));
List->TotalUsed = List->Head->Used;
List->TotalMax = List->Head->Max;
}
else if (List->TotalUsed >= List->TotalMax)
{
s32 NewBufferSize = sizeof(token_buffer) + (sizeof(token) * TOKEN_BUFFER_SIZE);
u8* NewBufferContents = (u8*)malloc(NewBufferSize);
token_buffer* NewBuffer = (token_buffer*)NewBufferContents;
NewBuffer->Tokens = (token*)(NewBufferContents + sizeof(token_buffer));
NewBuffer->Next = NewBuffer;
List->Head = NewBuffer;
List->TotalUsed += List->Head->Used;
List->TotalMax += List->Head->Max;
}
Result = List->Head->Tokens + List->Head->Used++;
List->TotalUsed++;
return Result;
}
internal void
LexFile (file_contents* Contents, token_list* TokensList)
{
tokenizer FileTokenizer = {};
FileTokenizer.Memory = Contents->Buffer.Memory;
FileTokenizer.MemoryLength = Contents->Buffer.Length;
FileTokenizer.At = FileTokenizer.Memory;
while (AtValidPosition(FileTokenizer))
{
token* Token = PushTokenOnList(TokensList);
*Token = GetNextToken(&FileTokenizer);
}
}
internal void
LexAllFiles (gs_meta_processor* Meta)
{
LoadAllFiles(Meta);
Meta->FileTokens = (token_list*)malloc(sizeof(token_list) * Meta->FilesUsed);
for (s32 SourceFile = 0; SourceFile < Meta->FilesUsed; SourceFile++)
{
Meta->FileTokens[SourceFile] = {};
token_list* FileTokenList = Meta->FileTokens + SourceFile;
file_contents* FileContents = Meta->Contents + SourceFile;
LexFile (FileContents, FileTokenList);
}
}
internal token_list_iterator
GetTokenIterator(token_list* List)
{
token_list_iterator Result = {};
Result.List = List;
Result.BufferAt = List->Head;
Result.At = List->Head->Tokens;
Result.AtTotalIndex = 0;
return Result;
}
internal b32
IsValid (token_list_iterator Iterator)
{
b32 Result = false;
if (Iterator.At && Iterator.AtTotalIndex < Iterator.List->TotalUsed)
{
Result = true;
}
return Result;
}
internal void
Advance (token_list_iterator* Iterator)
{
if ((Iterator->At - Iterator->BufferAt->Tokens) >= Iterator->BufferAt->Used)
{
Iterator->BufferAt = Iterator->BufferAt->Next;
if (Iterator->BufferAt)
{
Iterator->At = Iterator->BufferAt->Tokens;
Iterator->AtTotalIndex++;
}
else
{
Iterator->At = 0;
}
}
else
{
Iterator->At++;
Iterator->AtTotalIndex++;
}
}
internal parser_symbol_type
GetSymbolType (string Text, parser_symbol_type Fallback)
{
parser_symbol_type Result = Fallback;
for (s32 i = 0; i < (sizeof(ParserSymbolTable)/sizeof(ParserSymbolTable[0])); i++)
{
if (StringsEqual(Text, ParserSymbolTable[i].Text))
{
Result = ParserSymbolTable[i].Type;
break;
}
}
return Result;
}
internal void
ParseFile (token_list* TokenList, gs_meta_processor* Meta)
{
for (token_list_iterator TokenIter = GetTokenIterator(TokenList);
IsValid(TokenIter);
Advance(&TokenIter))
{
token* At = TokenIter.At;
parser_symbol_type SymbolType = GetSymbolType(At->Text, ParserSymbol_Unknown);
switch (SymbolType)
{
case SymbolType_Struct:
{
printf("Found Struct: %.*s\n", (At + 1)->Text.Length, (At + 1)->Text.Memory);
}break;
}
}
}
internal void
ParseAllFiles (gs_meta_processor* Meta)
{
for (s32 SourceFile = 0; SourceFile < Meta->FilesUsed; SourceFile++)
{
token_list* FileTokenList = Meta->FileTokens + SourceFile;
ParseFile(FileTokenList, Meta);
}
}
#if 0
#include <windows.h>
#define GS_PRINTF(v) OutputDebugStringA(v)
#include "gs_string.h"
static char DEBUGCharArray[256];
#define GS_DEBUG_PRINTF(format, ...) sprintf(DEBUGCharArray, format, __VA_ARGS__); OutputDebugStringA(DEBUGCharArray);
#include "gs_meta_lexer.h"
#include "gs_meta_parser.h"
#include "gs_meta_generator.h"
internal char*
ReadEntireFileAndNullTerminate (char* Filename)
{
char* Result = 0;
FILE* ReadFile = fopen(Filename, "r");
if (ReadFile)
{
fseek(ReadFile, 0, SEEK_END);
size_t FileSize = ftell(ReadFile);
fseek(ReadFile, 0, SEEK_SET);
Result = (char*)malloc(FileSize + 1);
size_t ReadSize = fread(Result, 1, FileSize, ReadFile);
Result[FileSize] = 0;
fclose(ReadFile);
}
else
{
GS_DEBUG_PRINTF("Failed to fopen file");
}
return Result;
}
int PrecompileFile (char* FileName)
{
char* SourceFileContents = ReadEntireFileAndNullTerminate(FileName);
if (!SourceFileContents)
{
GS_DEBUG_PRINTF("Failed to get source file contents");
return 0;
}
tokenizer Tokenizer = {};
Tokenizer.At = SourceFileContents;
s32 TokensGrowSize = 256;
s32 TokensMax = TokensGrowSize;
s32 TokensCount = 0;
token* Tokens = (token*)malloc(TokensGrowSize * sizeof(token));
token* Current = Tokens;
while (Tokenizer.At[0])
{
*Current = GetNextToken(&Tokenizer);
Current->Next = 0;
TokensCount++;
if (TokensCount >= TokensMax)
{
token* Addition = (token*)malloc(TokensGrowSize * sizeof(token));
TokensMax += TokensGrowSize;
token* Old = Current;
Current = Addition;
Old->Next = Current;
}
else if(Tokenizer.At[0])
{
token* Old = Current;
Current++;
Old->Next = Current;
}
}
Current = Tokens;
#if 0 // Print Tokens
if (Current)
{
do
{
GS_DEBUG_PRINTF("%s %.*s\n", TokenNames[(int)Current->Type], Current->TextLength, Current->Text);
Current = Current->Next;
}while(Current);
}
#endif
ast_node Ast = {};
Ast.Type = ASTNode_Program;
ast_node* CurrentNode = Ast.Children;
// Preprocessor Defines
// TODO(Peter): Actually preprocess the file
token* CurrentToken = Tokens;
/*while (CurrentToken)
{
CurrentToken = CurrentToken->Next;
}
*/
// Analyze File
CurrentToken = Tokens;
while (CurrentToken)
{
/*GS_DEBUG_PRINTF("%s %.*s\n", TokenNames[(int)CurrentToken->Type],
CurrentToken->TextLength, CurrentToken->Text);
*/
parse_result ParseResult = {};
ParseResult.Parsed = false;
if (CurrentToken->Type == Token_Identifier)
{
if (StringsEqual(CurrentToken->Text, "struct", 6))
{
ParseResult = ParseStructDeclaration(CurrentNode, CurrentToken);
CurrentToken = ParseResult.NextToken;
}
else if (IsFunction(CurrentToken))
{
ParseResult = ParseFunctionDeclaration(CurrentNode, CurrentToken);
CurrentToken = ParseResult.NextToken;
}
else
{
CurrentToken = CurrentToken->Next;
}
if (ParseResult.Parsed)
{
if (CurrentNode == 0)
{
Ast.Children = ParseResult.Node;
CurrentNode = Ast.Children;
}
else
{
CurrentNode->Next = ParseResult.Node;
CurrentNode = CurrentNode->Next;
}
}
}
else
{
CurrentToken = CurrentToken->Next;
}
}
string_partition StringPart = {};
{
memory_arena StringMemory = {};
InitMemoryArena(&StringMemory, (u8*)malloc(Megabytes(4)), Megabytes(4));
InitStringPartition(&StringPart, &StringMemory);
}
// Print Structs
ast_node* Node = Ast.Children;
while (Node)
{
switch (Node->Type)
{
case ASTNode_StructDeclaration:
{
string_buffer* CodeBuffer = GenerateStructCode(Node, &StringPart);
PrintStringBuffer(CodeBuffer);
}break;
case ASTNode_FunctionDeclaration:
{
string_buffer* CodeBuffer = GenerateFunctionDeclaration(Node, &StringPart);
PrintStringBuffer(CodeBuffer);
}break;
}
Node = Node->Next;
}
return 0;
}
int main(int ArgCount, char** ArgV)
{
if (ArgCount <= 1)
{
printf("Please supply at least one source directory to analyze.\n");
return 0;
}
TCHAR Win32FileNameBuffer[MAX_PATH];
char FullFilePath[MAX_PATH];
for (int i = 1; i < ArgCount; i++)
{
WIN32_FIND_DATA FindFileData;
StringCchCopy(Win32FileNameBuffer, MAX_PATH, ArgV[i]);
StringCchCat(Win32FileNameBuffer, MAX_PATH, TEXT("\\*"));
HANDLE CurrentFile = FindFirstFile(Win32FileNameBuffer, &FindFileData);
int FileIdx = 0;
do {
if (FindFileData.dwFileAttributes == FILE_ATTRIBUTE_DIRECTORY) {
// TODO(Peter): Recurse?
continue;
}
printf("File %d: %s\n", FileIdx++, FindFileData.cFileName);
} while (FindNextFile(CurrentFile, &FindFileData));
}
}
#endif