diff --git a/4coder_custom.cpp b/4coder_custom.cpp index eed6a3aa..f324e9a1 100644 --- a/4coder_custom.cpp +++ b/4coder_custom.cpp @@ -21,6 +21,9 @@ enum My_Maps{ }; HOOK_SIG(my_start){ + exec_command(cmd_context, cmdid_open_panel_hsplit); + exec_command(cmd_context, cmdid_change_active_panel); + exec_command(cmd_context, cmdid_open_panel_vsplit); exec_command(cmd_context, cmdid_change_active_panel); } diff --git a/4cpp_lexer.h b/4cpp_lexer.h index 1e5e80c7..837af0c0 100644 --- a/4cpp_lexer.h +++ b/4cpp_lexer.h @@ -1,1939 +1,1949 @@ -/* "4cpp" Open C++ Parser v0.1: Lexer - no warranty implied; use at your own risk - -NOTES ON USE: - OPTIONS: - Set options by defining macros before including this file. - - FCPP_LEXER_IMPLEMENTATION - causes this file to output function implementations - - this option is unset after use so that future includes of this file - in the same unit do not continue to output implementations - - FCPP_NO_MALLOC - prevent including - FCPP_NO_ASSERT - prevent including - FCPP_NO_STRING - prevent including - FCPP_NO_CRT - FCPP_NO_MALLOC & FCPP_NO_ASSERT & FCPP_NO_STRING - - FCPP_FORBID_MALLOC - one step above *NO_MALLOC with this set 4cpp functions that do allocations - are not allowed to be declared or defined at all, forcing the user to handle - allocation themselves - - implies FCPP_NO_MALLOC - - FCPP_GET_MEMORY - defines how to make allocations, interface of malloc, defaults to malloc - FCPP_FREE_MEMORY - defines how to free memory, interface of ree, defaults to free - (The above must be defined if FCPP_NO_MALLOC is set, unless FCPP_FORBID_MALLOC is set) - - FCPP_ASSERT - defines how to make assertions, interface of assert, defaults to assert - - FCPP_MEM_COPY - defines how to copy blocks of memory, interface of memcpy, defaults to memcpy - FCPP_MEM_MOVE - defines how to move blocks of memory, interface of memmove, defaults to memmove - (The above must be defined if FCPP_NO_STRING is set) - - FCPP_LINK - defines linkage of non-inline functions, defaults to static - FCPP_EXTERN - changes FCPP_LINK default to extern, this option is ignored if FCPP_LINK is defined - - include the file "4cpp_clear_config.h" if you want to undefine all options for some reason - - HIDDDEN DEPENDENCIES: - 4cpp is not a single file include library, there are dependencies between the files. - Be sure to include these dependencies before 4cpp_lexer.h: - - 4cpp_types.h - 4cpp_string.h -*/ - -// TOP -// TODO(allen): -// -// EASE OF USE AND DEPLOYMENT -// - make it easier to locate the list of function declarations -// - more C compatibility -// -// POTENTIAL -// - Experiment with optimizations. Sean's State machine? -// - Reserve 0th token for null? Put a EOF token at the end? -// - Pass Cpp_File and Cpp_Token_Stack by value instead of by pointer? -// -// CURRENT -// - lex in chunks -// - -#include "4coder_config.h" - -#ifndef FCPP_LEXER_INC -#define FCPP_LEXER_INC - -enum Cpp_Token_Type{ - CPP_TOKEN_JUNK, - CPP_TOKEN_COMMENT, - - CPP_TOKEN_KEY_TYPE, - CPP_TOKEN_KEY_MODIFIER, - CPP_TOKEN_KEY_QUALIFIER, - CPP_TOKEN_KEY_OPERATOR, // NOTE(allen): This type is not actually stored in tokens - CPP_TOKEN_KEY_CONTROL_FLOW, - CPP_TOKEN_KEY_CAST, - CPP_TOKEN_KEY_TYPE_DECLARATION, - CPP_TOKEN_KEY_ACCESS, - CPP_TOKEN_KEY_LINKAGE, - CPP_TOKEN_KEY_OTHER, - - CPP_TOKEN_IDENTIFIER, - CPP_TOKEN_INTEGER_CONSTANT, - CPP_TOKEN_CHARACTER_CONSTANT, - CPP_TOKEN_FLOATING_CONSTANT, - CPP_TOKEN_STRING_CONSTANT, - CPP_TOKEN_BOOLEAN_CONSTANT, - - CPP_TOKEN_STATIC_ASSERT, - - CPP_TOKEN_BRACKET_OPEN, - CPP_TOKEN_BRACKET_CLOSE, - CPP_TOKEN_PARENTHESE_OPEN, - CPP_TOKEN_PARENTHESE_CLOSE, - CPP_TOKEN_BRACE_OPEN, - CPP_TOKEN_BRACE_CLOSE, - CPP_TOKEN_SEMICOLON, - CPP_TOKEN_ELLIPSIS, - - // NOTE(allen): Ambiguous tokens, lexer only, - // parser figures out the real meaning - CPP_TOKEN_STAR, - CPP_TOKEN_AMPERSAND, - CPP_TOKEN_TILDE, - CPP_TOKEN_PLUS, - CPP_TOKEN_MINUS, - CPP_TOKEN_INCREMENT, - CPP_TOKEN_DECREMENT, - - // NOTE(allen): Precedence 1, LtoR - CPP_TOKEN_SCOPE, - - // NOTE(allen): Precedence 2, LtoR - CPP_TOKEN_POSTINC, // from increment, parser only - CPP_TOKEN_POSTDEC, // from decrement, parser only - CPP_TOKEN_FUNC_STYLE_CAST, // parser only - CPP_TOKEN_CPP_STYLE_CAST, - CPP_TOKEN_CALL, // from open paren, parser only - CPP_TOKEN_INDEX, // from bracket open, parser only - CPP_TOKEN_DOT, - CPP_TOKEN_ARROW, - - // NOTE(allen): Precedence 3, RtoL - CPP_TOKEN_PREINC, // from increment, parser only - CPP_TOKEN_PREDEC, // from decrement, parser only - CPP_TOKEN_POSITIVE, // from plus, parser only - CPP_TOKEN_NEGAITVE, // from minus, parser only - CPP_TOKEN_NOT, - CPP_TOKEN_BIT_NOT, // from tilde, direct from 'compl' - CPP_TOKEN_CAST, // from open paren, parser only - CPP_TOKEN_DEREF, // from star, parser only - CPP_TOKEN_TYPE_PTR, // from star, parser only - CPP_TOKEN_ADDRESS, // from ampersand, parser only - CPP_TOKEN_TYPE_REF, // from ampersand, parser only - CPP_TOKEN_SIZEOF, - CPP_TOKEN_ALIGNOF, - CPP_TOKEN_DECLTYPE, - CPP_TOKEN_TYPEID, - CPP_TOKEN_NEW, - CPP_TOKEN_DELETE, - CPP_TOKEN_NEW_ARRAY, // from new and bracket open, parser only - CPP_TOKEN_DELETE_ARRAY, // from delete and bracket open, parser only - - // NOTE(allen): Precedence 4, LtoR - CPP_TOKEN_PTRDOT, - CPP_TOKEN_PTRARROW, - - // NOTE(allen): Precedence 5, LtoR - CPP_TOKEN_MUL, // from start, parser only - CPP_TOKEN_DIV, - CPP_TOKEN_MOD, - - // NOTE(allen): Precedence 6, LtoR - CPP_TOKEN_ADD, // from plus, parser only - CPP_TOKEN_SUB, // from minus, parser only - - // NOTE(allen): Precedence 7, LtoR - CPP_TOKEN_LSHIFT, - CPP_TOKEN_RSHIFT, - - // NOTE(allen): Precedence 8, LtoR - CPP_TOKEN_LESS, - CPP_TOKEN_GRTR, - CPP_TOKEN_GRTREQ, - CPP_TOKEN_LESSEQ, - - // NOTE(allen): Precedence 9, LtoR - CPP_TOKEN_EQEQ, - CPP_TOKEN_NOTEQ, - - // NOTE(allen): Precedence 10, LtoR - CPP_TOKEN_BIT_AND, // from ampersand, direct from 'bitand' - - // NOTE(allen): Precedence 11, LtoR - CPP_TOKEN_BIT_XOR, - - // NOTE(allen): Precedence 12, LtoR - CPP_TOKEN_BIT_OR, - - // NOTE(allen): Precedence 13, LtoR - CPP_TOKEN_AND, - - // NOTE(allen): Precedence 14, LtoR - CPP_TOKEN_OR, - - // NOTE(allen): Precedence 15, RtoL - CPP_TOKEN_TERNARY_QMARK, - CPP_TOKEN_COLON, - CPP_TOKEN_THROW, - CPP_TOKEN_EQ, - CPP_TOKEN_ADDEQ, - CPP_TOKEN_SUBEQ, - CPP_TOKEN_MULEQ, - CPP_TOKEN_DIVEQ, - CPP_TOKEN_MODEQ, - CPP_TOKEN_LSHIFTEQ, - CPP_TOKEN_RSHIFTEQ, - CPP_TOKEN_ANDEQ, - CPP_TOKEN_OREQ, - CPP_TOKEN_XOREQ, - - // NOTE(allen): Precedence 16, LtoR - CPP_TOKEN_COMMA, - - CPP_PP_INCLUDE, - CPP_PP_DEFINE, - CPP_PP_UNDEF, - CPP_PP_IF, - CPP_PP_IFDEF, - CPP_PP_IFNDEF, - CPP_PP_ELSE, - CPP_PP_ELIF, - CPP_PP_ENDIF, - CPP_PP_ERROR, - CPP_PP_IMPORT, - CPP_PP_USING, - CPP_PP_LINE, - CPP_PP_PRAGMA, - CPP_PP_STRINGIFY, - CPP_PP_CONCAT, - CPP_PP_UNKNOWN, - CPP_TOKEN_DEFINED, - CPP_TOKEN_INCLUDE_FILE, - CPP_TOKEN_ERROR_MESSAGE, - - // NOTE(allen): used in the parser - CPP_TOKEN_EOF -}; - -struct Cpp_File{ - char *data; - int size; -}; - -struct Cpp_Token{ - Cpp_Token_Type type; - fcpp_i32 start, size; - fcpp_u16 state_flags; - fcpp_u16 flags; -}; - -enum Cpp_Token_Flag{ - CPP_TFLAG_IGNORE = 1 << 0, - CPP_TFLAG_PP_DIRECTIVE = 1 << 1, - CPP_TFLAG_PP_BODY = 1 << 2, - CPP_TFLAG_BAD_ENDING = 1 << 3, - CPP_TFLAG_MULTILINE = 1 << 4, - CPP_TFLAG_PARAMETERIZED = 1 << 5, - CPP_TFLAG_IS_OPERATOR = 1 << 6, - CPP_TFLAG_IS_KEYWORD = 1 << 7 -}; - -enum Cpp_Preprocessor_State{ - CPP_LEX_PP_DEFAULT, - CPP_LEX_PP_IDENTIFIER, - CPP_LEX_PP_MACRO_IDENTIFIER, - CPP_LEX_PP_INCLUDE, - CPP_LEX_PP_BODY, - CPP_LEX_PP_BODY_IF, - CPP_LEX_PP_NUMBER, - CPP_LEX_PP_ERROR, - CPP_LEX_PP_JUNK, - // NEVER ADD BELOW THIS - CPP_LEX_PP_COUNT -}; - -struct Cpp_Lex_Data{ - Cpp_Preprocessor_State pp_state; - fcpp_i32 pos; - fcpp_bool32 complete; -}; - -struct Cpp_Read_Result{ - Cpp_Token token; - fcpp_i32 pos; - fcpp_bool8 newline; - fcpp_bool8 has_result; -}; - -struct Cpp_Token_Stack{ - Cpp_Token *tokens; - int count, max_count; -}; - -struct Cpp_Token_Merge{ - Cpp_Token new_token; - fcpp_bool32 did_merge; -}; - -struct Seek_Result{ - fcpp_i32 pos; - fcpp_bool32 new_line; -}; - -struct Cpp_Get_Token_Result{ - fcpp_i32 token_index; - fcpp_bool32 in_whitespace; -}; - -// TODO(allen): revisit this keyword data declaration system -struct String_And_Flag{ - char *str; - fcpp_u32 flags; -}; - -struct String_List{ - String_And_Flag *data; - int count; -}; - -struct Sub_Match_List_Result{ - int index; - fcpp_i32 new_pos; -}; - -inline fcpp_u16 -cpp_token_set_pp_state(fcpp_u16 bitfield, Cpp_Preprocessor_State state_value){ - return (fcpp_u16)state_value; -} - -inline Cpp_Preprocessor_State -cpp_token_get_pp_state(fcpp_u16 bitfield){ - return (Cpp_Preprocessor_State)(bitfield); -} - -inline String -cpp_get_lexeme(char *str, Cpp_Token *token){ - String result; - result.str = str + token->start; - result.size = token->size; - return result; -} - -inline bool -is_keyword(Cpp_Token_Type type){ - return (type >= CPP_TOKEN_KEY_TYPE && type <= CPP_TOKEN_KEY_OTHER); -} - -FCPP_LINK Sub_Match_List_Result sub_match_list(Cpp_File file, int pos, String_List list, int sub_size); - -FCPP_LINK Seek_Result seek_unescaped_eol(char *data, int size, int pos); -FCPP_LINK Seek_Result seek_unescaped_delim(char *data, int size, int pos, char delim); -FCPP_LINK Seek_Result seek_block_comment_end(char *data, int size, int pos); - -FCPP_LINK Cpp_Read_Result cpp_read_whitespace(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_junk_line(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_operator(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_pp_operator(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body); -inline Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos) { return cpp_read_alpha_numeric(file, pos, 0); } -FCPP_LINK Cpp_Read_Result cpp_read_number(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_string_litteral(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_character_litteral(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_line_comment(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_block_comment(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_preprocessor(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_pp_include_file(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body); -inline Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos) { return cpp_read_pp_default_mode(file, pos, 0); } - -FCPP_LINK Cpp_Token_Merge cpp_attempt_token_merge(Cpp_Token prev, Cpp_Token next); - -FCPP_LINK bool cpp_push_token_no_merge(Cpp_Token_Stack *stack, Cpp_Token token); -FCPP_LINK bool cpp_push_token_nonalloc(Cpp_Token_Stack *stack, Cpp_Token token); - -FCPP_LINK Cpp_Read_Result cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex); - -FCPP_LINK int cpp_lex_file_token_count(Cpp_File file); -FCPP_LINK Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack, Cpp_Lex_Data data); -inline Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack) { return cpp_lex_file_nonalloc(file, stack, {}); } - -FCPP_LINK Cpp_Get_Token_Result cpp_get_token(Cpp_Token_Stack *stack, int pos); - -FCPP_LINK int cpp_get_end_token(Cpp_Token_Stack *stack, int end); -FCPP_LINK void cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token, int amount); - -struct Cpp_Relex_State{ - Cpp_File file; - Cpp_Token_Stack *stack; - int start, end, amount; - int start_token_i; - int end_token_i; - int relex_start; - int tolerance; - int space_request; -}; - -FCPP_LINK Cpp_Relex_State cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance); -FCPP_LINK bool cpp_relex_nonalloc_main(Cpp_Relex_State state, Cpp_Token_Stack *stack); - -#ifndef FCPP_FORBID_MALLOC -FCPP_LINK Cpp_Token_Stack cpp_make_token_stack(int max); -FCPP_LINK void cpp_free_token_stack(Cpp_Token_Stack stack); -FCPP_LINK void cpp_resize_token_stack(Cpp_Token_Stack *stack, int new_max); - -FCPP_LINK void cpp_push_token(Cpp_Token_Stack *stack, Cpp_Token token); -FCPP_LINK void cpp_lex_file(Cpp_File file, Cpp_Token_Stack *stack); -FCPP_LINK bool cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount, int extra_tolerance); -inline void cpp_relex_file(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount) -{ cpp_relex_file_limited(file, stack, start_i, end_i, amount, -1); } -#endif - -#define FCPP_STRING_LIST(x) {x, FCPP_COUNT(x)} - -// TODO(allen): shift towards storing in a context -FCPP_GLOBAL String_And_Flag int_suf_strings[] = { - {"ull"}, {"ULL"}, - {"llu"}, {"LLU"}, - {"ll"}, {"LL"}, - {"l"}, {"L"}, - {"u"}, {"U"} -}; - -FCPP_GLOBAL String_List int_sufs = FCPP_STRING_LIST(int_suf_strings); - -FCPP_GLOBAL String_And_Flag float_suf_strings[] = { - {"f"}, {"F"}, - {"l"}, {"L"} -}; -FCPP_GLOBAL String_List float_sufs = FCPP_STRING_LIST(float_suf_strings); - -FCPP_GLOBAL String_And_Flag bool_lit_strings[] = { - {"true"}, {"false"} -}; -FCPP_GLOBAL String_List bool_lits = FCPP_STRING_LIST(bool_lit_strings); - -FCPP_GLOBAL String_And_Flag keyword_strings[] = { - {"and", CPP_TOKEN_AND}, - {"and_eq", CPP_TOKEN_ANDEQ}, - {"bitand", CPP_TOKEN_BIT_AND}, - {"bitor", CPP_TOKEN_BIT_OR}, - {"or", CPP_TOKEN_OR}, - {"or_eq", CPP_TOKEN_OREQ}, - {"sizeof", CPP_TOKEN_SIZEOF}, - {"alignof", CPP_TOKEN_ALIGNOF}, - {"decltype", CPP_TOKEN_DECLTYPE}, - {"throw", CPP_TOKEN_THROW}, - {"new", CPP_TOKEN_NEW}, - {"delete", CPP_TOKEN_DELETE}, - {"xor", CPP_TOKEN_BIT_XOR}, - {"xor_eq", CPP_TOKEN_XOREQ}, - {"not", CPP_TOKEN_NOT}, - {"not_eq", CPP_TOKEN_NOTEQ}, - {"typeid", CPP_TOKEN_TYPEID}, - {"compl", CPP_TOKEN_BIT_NOT}, - - {"void", CPP_TOKEN_KEY_TYPE}, - {"bool", CPP_TOKEN_KEY_TYPE}, - {"char", CPP_TOKEN_KEY_TYPE}, - {"int", CPP_TOKEN_KEY_TYPE}, - {"float", CPP_TOKEN_KEY_TYPE}, - {"double", CPP_TOKEN_KEY_TYPE}, - - {"long", CPP_TOKEN_KEY_MODIFIER}, - {"short", CPP_TOKEN_KEY_MODIFIER}, - {"unsigned", CPP_TOKEN_KEY_MODIFIER}, - - {"const", CPP_TOKEN_KEY_QUALIFIER}, - {"volatile", CPP_TOKEN_KEY_QUALIFIER}, - - {"asm", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"break", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"case", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"catch", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"continue", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"default", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"do", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"else", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"for", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"goto", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"if", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"return", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"switch", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"try", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"while", CPP_TOKEN_KEY_CONTROL_FLOW}, - {"static_assert", CPP_TOKEN_KEY_CONTROL_FLOW}, - - {"const_cast", CPP_TOKEN_KEY_CAST}, - {"dynamic_cast", CPP_TOKEN_KEY_CAST}, - {"reinterpret_cast", CPP_TOKEN_KEY_CAST}, - {"static_cast", CPP_TOKEN_KEY_CAST}, - - {"class", CPP_TOKEN_KEY_TYPE_DECLARATION}, - {"enum", CPP_TOKEN_KEY_TYPE_DECLARATION}, - {"struct", CPP_TOKEN_KEY_TYPE_DECLARATION}, - {"typedef", CPP_TOKEN_KEY_TYPE_DECLARATION}, - {"union", CPP_TOKEN_KEY_TYPE_DECLARATION}, - {"template", CPP_TOKEN_KEY_TYPE_DECLARATION}, - {"typename", CPP_TOKEN_KEY_TYPE_DECLARATION}, - - {"friend", CPP_TOKEN_KEY_ACCESS}, - {"namespace", CPP_TOKEN_KEY_ACCESS}, - {"private", CPP_TOKEN_KEY_ACCESS}, - {"protected", CPP_TOKEN_KEY_ACCESS}, - {"public", CPP_TOKEN_KEY_ACCESS}, - {"using", CPP_TOKEN_KEY_ACCESS}, - - {"extern", CPP_TOKEN_KEY_LINKAGE}, - {"export", CPP_TOKEN_KEY_LINKAGE}, - {"inline", CPP_TOKEN_KEY_LINKAGE}, - {"static", CPP_TOKEN_KEY_LINKAGE}, - {"virtual", CPP_TOKEN_KEY_LINKAGE}, - - {"alignas", CPP_TOKEN_KEY_OTHER}, - {"explicit", CPP_TOKEN_KEY_OTHER}, - {"noexcept", CPP_TOKEN_KEY_OTHER}, - {"nullptr", CPP_TOKEN_KEY_OTHER}, - {"operator", CPP_TOKEN_KEY_OTHER}, - {"register", CPP_TOKEN_KEY_OTHER}, - {"this", CPP_TOKEN_KEY_OTHER}, - {"thread_local", CPP_TOKEN_KEY_OTHER}, -}; -FCPP_GLOBAL String_List keywords = FCPP_STRING_LIST(keyword_strings); - -FCPP_GLOBAL String_And_Flag op_strings[] = { - {"...", CPP_TOKEN_ELLIPSIS}, - {"<<=", CPP_TOKEN_LSHIFTEQ}, - {">>=", CPP_TOKEN_RSHIFTEQ}, - {"->*", CPP_TOKEN_PTRARROW}, - {"<<", CPP_TOKEN_LSHIFT}, - {">>", CPP_TOKEN_RSHIFT}, - {"&&", CPP_TOKEN_AND}, - {"||", CPP_TOKEN_OR}, - {"->", CPP_TOKEN_ARROW}, - {"++", CPP_TOKEN_INCREMENT}, - {"--", CPP_TOKEN_DECREMENT}, - {"::", CPP_TOKEN_SCOPE}, - {"+=", CPP_TOKEN_ADDEQ}, - {"-=", CPP_TOKEN_SUBEQ}, - {"*=", CPP_TOKEN_MULEQ}, - {"/=", CPP_TOKEN_DIVEQ}, - {"%=", CPP_TOKEN_MODEQ}, - {"&=", CPP_TOKEN_ANDEQ}, - {"|=", CPP_TOKEN_OREQ}, - {"^=", CPP_TOKEN_XOREQ}, - {"==", CPP_TOKEN_EQEQ}, - {">=", CPP_TOKEN_GRTREQ}, - {"<=", CPP_TOKEN_LESSEQ}, - {"!=", CPP_TOKEN_NOTEQ}, - {".*", CPP_TOKEN_PTRDOT}, - {"{", CPP_TOKEN_BRACE_OPEN}, - {"}", CPP_TOKEN_BRACE_CLOSE}, - {"[", CPP_TOKEN_BRACKET_OPEN}, - {"]", CPP_TOKEN_BRACKET_CLOSE}, - {"(", CPP_TOKEN_PARENTHESE_OPEN}, - {")", CPP_TOKEN_PARENTHESE_CLOSE}, - {"<", CPP_TOKEN_LESS}, - {">", CPP_TOKEN_GRTR}, - {"+", CPP_TOKEN_PLUS}, - {"-", CPP_TOKEN_MINUS}, - {"!", CPP_TOKEN_NOT}, - {"~", CPP_TOKEN_TILDE}, - {"*", CPP_TOKEN_STAR}, - {"&", CPP_TOKEN_AMPERSAND}, - {"|", CPP_TOKEN_BIT_OR}, - {"^", CPP_TOKEN_BIT_XOR}, - {"=", CPP_TOKEN_EQ}, - {",", CPP_TOKEN_COMMA}, - {":", CPP_TOKEN_COLON}, - {";", CPP_TOKEN_SEMICOLON}, - {"/", CPP_TOKEN_DIV}, - {"?", CPP_TOKEN_TERNARY_QMARK}, - {"%", CPP_TOKEN_MOD}, - {".", CPP_TOKEN_DOT}, -}; -FCPP_GLOBAL String_List ops = FCPP_STRING_LIST(op_strings); - -FCPP_GLOBAL String_And_Flag pp_op_strings[] = { - {"##", CPP_PP_CONCAT}, - {"#", CPP_PP_STRINGIFY}, -}; -FCPP_GLOBAL String_List pp_ops = FCPP_STRING_LIST(pp_op_strings); - -FCPP_GLOBAL String_And_Flag preprop_strings[] = { - {"include", CPP_PP_INCLUDE}, - {"INCLUDE", CPP_PP_INCLUDE}, - {"ifndef", CPP_PP_IFNDEF}, - {"IFNDEF", CPP_PP_IFNDEF}, - {"define", CPP_PP_DEFINE}, - {"DEFINE", CPP_PP_DEFINE}, - {"import", CPP_PP_IMPORT}, - {"IMPORT", CPP_PP_IMPORT}, - {"pragma", CPP_PP_PRAGMA}, - {"PRAGMA", CPP_PP_PRAGMA}, - {"undef", CPP_PP_UNDEF}, - {"UNDEF", CPP_PP_UNDEF}, - {"endif", CPP_PP_ENDIF}, - {"ENDIF", CPP_PP_ENDIF}, - {"error", CPP_PP_ERROR}, - {"ERROR", CPP_PP_ERROR}, - {"ifdef", CPP_PP_IFDEF}, - {"IFDEF", CPP_PP_IFDEF}, - {"using", CPP_PP_USING}, - {"USING", CPP_PP_USING}, - {"else", CPP_PP_ELSE}, - {"ELSE", CPP_PP_ELSE}, - {"elif", CPP_PP_ELIF}, - {"ELIF", CPP_PP_ELIF}, - {"line", CPP_PP_LINE}, - {"LINE", CPP_PP_LINE}, - {"if", CPP_PP_IF}, - {"IF", CPP_PP_IF}, -}; -FCPP_GLOBAL String_List preprops = FCPP_STRING_LIST(preprop_strings); - -#undef FCPP_STRING_LIST - -#endif // #ifndef FCPP_CPP_LEXER - -#ifdef FCPP_LEXER_IMPLEMENTATION - -#define _Assert FCPP_ASSERT -#define _TentativeAssert FCPP_ASSERT - -FCPP_LINK Sub_Match_List_Result -sub_match_list(Cpp_File file, int pos, String_List list, int sub_size){ - Sub_Match_List_Result result; - String str_main; - char *str_check; - int i,l; - - result.index = -1; - result.new_pos = pos; - str_main = make_string(file.data + pos, file.size - pos); - if (sub_size > 0){ - str_main = substr(str_main, 0, sub_size); - for (i = 0; i < list.count; ++i){ - str_check = list.data[i].str; - if (match(str_main, str_check)){ - result.index = i; - result.new_pos = pos + sub_size; - break; - } - } - } - else{ - for (i = 0; i < list.count; ++i){ - str_check = list.data[i].str; - if (match_part(str_main, str_check, &l)){ - result.index = i; - result.new_pos = pos + l; - break; - } - } - } - return result; -} - -FCPP_LINK Seek_Result -seek_unescaped_eol(char *data, int size, int pos){ - Seek_Result result = {}; - ++pos; - while (pos < size){ - if (data[pos] == '\\'){ - if (pos + 1 < size && - data[pos+1] == '\n'){ - result.new_line = 1; - ++pos; - } - else if (pos + 1 < size && - data[pos+1] == '\r' && - pos + 2 < size && - data[pos+2] == '\n'){ - result.new_line = 1; - pos += 2; - } - } - else if (data[pos] == '\n'){ - break; - } - ++pos; - } - ++pos; - - result.pos = pos; - return result; -} - -FCPP_LINK Seek_Result -seek_unescaped_delim(char *data, int size, int pos, char delim){ - Seek_Result result = {}; - bool escape = 0; - ++pos; - while (pos < size){ - if (data[pos] == '\n'){ - result.new_line = 1; - } - if (escape){ - escape = 0; - } - else{ - if (data[pos] == '\\'){ - escape = 1; - } - else if (data[pos] == delim){ - break; - } - } - ++pos; - } - ++pos; - - result.pos = pos; - return result; -} - -FCPP_LINK Seek_Result -seek_block_comment_end(char *data, int size, int pos){ - Seek_Result result = {}; - pos += 2; - while (pos < size){ - if (data[pos] == '*' && - pos + 1 < size && - data[pos+1] == '/'){ - break; - } - if (data[pos] == '\n'){ - result.new_line = 1; - } - ++pos; - } - pos += 2; - result.pos = pos; - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_whitespace(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - - while (pos < file.size && char_is_whitespace(file.data[pos])){ - if (file.data[pos] == '\n'){ - result.newline = 1; - } - ++pos; - } - - result.pos = pos; - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_junk_line(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - result.token.start = pos; - result.token.type = CPP_TOKEN_JUNK; - - bool comment_end = 0; - while (pos < file.size && file.data[pos] != '\n'){ - if (file.data[pos] == '/' && pos + 1 < file.size){ - if (file.data[pos + 1] == '/' || - file.data[pos + 1] == '*'){ - comment_end = 1; - break; - } - } - ++pos; - } - - if (comment_end){ - result.pos = pos; - result.token.size = pos - result.token.start; - } - else{ - while (pos > 0 && file.data[pos - 1] == '\r'){ - --pos; - } - if (pos > 0 && file.data[pos - 1] == '\\'){ - --pos; - } - result.pos = pos; - result.token.size = pos - result.token.start; - } - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_operator(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - result.pos = pos; - result.token.start = pos; - - Sub_Match_List_Result match; - match = sub_match_list(file, result.token.start, ops, -1); - - if (match.index != -1){ - result.pos = match.new_pos; - result.token.size = result.pos - result.token.start; - result.token.type = (Cpp_Token_Type)ops.data[match.index].flags; - result.token.flags |= CPP_TFLAG_IS_OPERATOR; - } - else{ - result.token.size = 1; - result.token.type = CPP_TOKEN_JUNK; - result.pos = pos + 1; - } - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_pp_operator(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - result.pos = pos; - result.token.start = pos; - - Sub_Match_List_Result match; - match = sub_match_list(file, result.token.start, pp_ops, -1); - - _Assert(match.index != -1); - result.pos = match.new_pos; - result.token.size = result.pos - result.token.start; - result.token.type = (Cpp_Token_Type)pp_ops.data[match.index].flags; - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){ - Cpp_Read_Result result = {}; - result.pos = pos; - result.token.start = pos; - - while (result.pos < file.size && - char_is_alpha_numeric(file.data[result.pos])){ - ++result.pos; - } - - result.token.size = result.pos - result.token.start; - - // TODO(allen): do better - if (in_if_body){ - String word; - word.size = result.token.size; - word.str = file.data + result.token.start; - if (match(word, "defined")){ - result.token.type = CPP_TOKEN_DEFINED; - result.token.flags |= CPP_TFLAG_IS_OPERATOR; - result.token.flags |= CPP_TFLAG_IS_KEYWORD; - } - } - - if (result.token.type == CPP_TOKEN_JUNK){ - Sub_Match_List_Result match; - match = sub_match_list(file, result.token.start, bool_lits, result.token.size); - - if (match.index != -1){ - result.token.type = CPP_TOKEN_BOOLEAN_CONSTANT; - result.token.flags |= CPP_TFLAG_IS_KEYWORD; - } - else{ - match = sub_match_list(file, result.token.start, keywords, result.token.size); - - if (match.index != -1){ - String_And_Flag data = keywords.data[match.index]; - result.token.type = (Cpp_Token_Type)data.flags; - result.token.flags |= CPP_TFLAG_IS_KEYWORD; - } - else{ - result.token.type = CPP_TOKEN_IDENTIFIER; - } - } - } - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_number(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - result.pos = pos; - result.token.start = pos; - - bool is_float = 0; - bool is_integer = 0; - bool is_oct = 0; - bool is_hex = 0; - bool is_zero = 0; - - if (file.data[pos] == '0'){ - if (pos+1 < file.size){ - char next = file.data[pos+1]; - if (next == 'x'){ - is_hex = 1; - is_integer = 1; - } - else if (next == '.'){ - is_float = 1; - ++result.pos; - } - else if (next >= '0' && next <= '9'){ - is_oct = 1; - is_integer = 1; - } - else{ - is_zero = 1; - is_integer = 1; - } - } - else{ - is_zero = 1; - is_integer = 1; - } - } - else if (file.data[pos] == '.'){ - is_float = 1; - } - - if (is_zero){ - ++result.pos; - } - else if (is_hex){ - ++result.pos; - char character; - do{ - ++result.pos; - if (result.pos >= file.size){ - break; - } - character = file.data[result.pos]; - } while(char_is_hex(character)); - } - else if (is_oct){ - char character; - do{ - ++result.pos; - if (result.pos >= file.size){ - break; - } - character = file.data[result.pos]; - }while(char_is_numeric(character)); - } - else{ - if (!is_float){ - is_integer = 1; - while (1){ - ++result.pos; - - if (result.pos >= file.size){ - break; - } - bool is_good = 0; - char character = file.data[result.pos]; - if (character >= '0' && character <= '9'){ - is_good = 1; - } - else if (character == '.'){ - is_integer = 0; - is_float = 1; - } - if (!is_good){ - break; - } - } - } - - if (is_float){ - bool e_mode = 0; - bool e_minus = 0; - bool is_good = 0; - char character; - - while (1){ - ++result.pos; - if (result.pos >= file.size){ - break; - } - is_good = 0; - character = file.data[result.pos]; - if (character >= '0' && character <= '9'){ - is_good = 1; - } - else{ - if (character == 'e' && !e_mode){ - e_mode = 1; - is_good = 1; - } - else if (character == '-' && e_mode && !e_minus){ - e_minus = 1; - is_good = 1; - } - } - if (!is_good){ - break; - } - } - } - } - - if (is_integer){ - Sub_Match_List_Result match = - sub_match_list(file, result.pos, int_sufs, -1); - if (match.index != -1){ - result.pos = match.new_pos; - } - result.token.type = CPP_TOKEN_INTEGER_CONSTANT; - result.token.size = result.pos - result.token.start; - } - else if (is_float){ - Sub_Match_List_Result match = - sub_match_list(file, result.pos, float_sufs, -1); - if (match.index != -1){ - result.pos = match.new_pos; - } - result.token.type = CPP_TOKEN_FLOATING_CONSTANT; - result.token.size = result.pos - result.token.start; - } - else{ - _Assert(!"This shouldn't happen!"); - } - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_string_litteral(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - result.token.start = pos; - - _Assert(file.data[pos] == '"'); - Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '"'); - pos = seek.pos; - if (seek.new_line){ - result.token.flags |= CPP_TFLAG_MULTILINE; - } - - result.token.size = pos - result.token.start; - result.token.type = CPP_TOKEN_STRING_CONSTANT; - result.pos = pos; - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_character_litteral(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - result.token.start = pos; - - _Assert(file.data[pos] == '\''); - Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '\''); - pos = seek.pos; - if (seek.new_line){ - result.token.flags |= CPP_TFLAG_MULTILINE; - } - - result.token.size = pos - result.token.start; - result.token.type = CPP_TOKEN_CHARACTER_CONSTANT; - result.pos = pos; - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_line_comment(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - result.token.start = pos; - - _Assert(file.data[pos] == '/' && file.data[pos + 1] == '/'); - - pos += 2; - while (pos < file.size){ - if (file.data[pos] == '\n'){ - break; - } - if (file.data[pos] == '\\'){ - if (pos + 1 < file.size && - file.data[pos + 1] == '\n'){ - ++pos; - } - else if (pos + 2 < file.size && - file.data[pos + 1] == '\r' && - file.data[pos + 2] == '\n'){ - pos += 2; - } - } - ++pos; - } - if (pos > 0 && file.data[pos-1] == '\r'){ - --pos; - } - result.token.size = pos - result.token.start; - result.token.type = CPP_TOKEN_COMMENT; - result.pos = pos; - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_block_comment(Cpp_File file, int pos){ - Cpp_Read_Result result = {}; - result.token.start = pos; - - _Assert(file.data[pos] == '/' && file.data[pos + 1] == '*'); - pos += 2; - while (pos < file.size){ - if (file.data[pos] == '*' && - pos + 1 < file.size && - file.data[pos+1] == '/'){ - break; - } - ++pos; - } - pos += 2; - result.token.size = pos - result.token.start; - result.token.type = CPP_TOKEN_COMMENT; - result.pos = pos; - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_preprocessor(Cpp_File file, int pos){ - _Assert(file.data[pos] == '#'); - Cpp_Read_Result result = {}; - result.token.start = pos; - result.token.type = CPP_PP_UNKNOWN; - result.token.flags |= CPP_TFLAG_PP_DIRECTIVE; - - ++pos; - while (pos < file.size && - (file.data[pos] == ' ' || - file.data[pos] == '\t')){ - ++pos; - } - - Sub_Match_List_Result match - = sub_match_list(file, pos, preprops, -1); - - if (match.index != -1){ - result.token.size = match.new_pos - result.token.start; - result.token.type = (Cpp_Token_Type)preprops.data[match.index].flags; - result.pos = match.new_pos; - } - else{ - while (pos < file.size && - !char_is_whitespace(file.data[pos])){ - ++pos; - } - result.token.size = pos - result.token.start; - result.pos = pos; - } - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_pp_include_file(Cpp_File file, int pos){ - char start = file.data[pos]; - _Assert(start == '<' || start == '"'); - - Cpp_Read_Result result = {}; - result.token.start = pos; - result.token.type = CPP_TOKEN_INCLUDE_FILE; - result.token.flags |= CPP_TFLAG_PP_BODY; - - char end; - if (start == '<'){ - end = '>'; - } - else{ - end = '"'; - } - - ++pos; - while (pos < file.size && file.data[pos] != end){ - if (file.data[pos] == '\n'){ - result.token.type = CPP_TOKEN_JUNK; - result.token.flags |= CPP_TFLAG_BAD_ENDING; - break; - } - if (file.data[pos] == '\\'){ - // TODO(allen): Not sure that this is 100% correct. - if (pos + 1 < file.size && - file.data[pos + 1] == '\n'){ - ++pos; - result.token.flags |= CPP_TFLAG_MULTILINE; - } - else if (pos + 2 < file.size && - file.data[pos + 1] == '\r' && - file.data[pos + 2] == '\n'){ - pos += 2; - result.token.flags |= CPP_TFLAG_MULTILINE; - } - } - ++pos; - } - - if (result.token.type != CPP_TOKEN_JUNK){ - if (pos < file.size){ - ++pos; - } - } - - result.token.size = pos - result.token.start; - result.pos = pos; - - return result; -} - -FCPP_LINK Cpp_Read_Result -cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body){ - char current = file.data[pos]; - Cpp_Read_Result result; - if (char_is_numeric(current)){ - result = cpp_read_number(file, pos); - } - else if (char_is_alpha(current)){ - result = cpp_read_alpha_numeric(file, pos, in_if_body); - } - else if (current == '.'){ - if (pos + 1 < file.size){ - char next = file.data[pos + 1]; - if (char_is_numeric(next)){ - result = cpp_read_number(file, pos); - } - else{ - result = cpp_read_operator(file, pos); - } - } - else{ - result = cpp_read_operator(file, pos); - } - } - - else if (current == '/'){ - if (pos + 1 < file.size){ - char next = file.data[pos + 1]; - if (next == '/'){ - result = cpp_read_line_comment(file, pos); - } - else if (next == '*'){ - result = cpp_read_block_comment(file, pos); - } - else{ - result = cpp_read_operator(file, pos); - } - } - else{ - result = cpp_read_operator(file, pos); - } - } - else if (current == '"'){ - result = cpp_read_string_litteral(file, pos); - } - else if (current == '\''){ - result = cpp_read_character_litteral(file, pos); - } - else{ - result = cpp_read_operator(file, pos); - } - - return result; -} - -FCPP_LINK Cpp_Token_Merge -cpp_attempt_token_merge(Cpp_Token prev_token, Cpp_Token next_token){ - Cpp_Token_Merge result = {}; - if (next_token.type == CPP_TOKEN_COMMENT && prev_token.type == CPP_TOKEN_COMMENT && - next_token.flags == prev_token.flags && next_token.state_flags == prev_token.state_flags){ - result.did_merge = 1; - prev_token.size = next_token.start + next_token.size - prev_token.start; - result.new_token = prev_token; - } - else if (next_token.type == CPP_TOKEN_JUNK && prev_token.type == CPP_TOKEN_JUNK && - next_token.flags == prev_token.flags && next_token.state_flags == prev_token.state_flags){ - result.did_merge = 1; - prev_token.size = next_token.start + next_token.size - prev_token.start; - result.new_token = prev_token; - } - return result; -} - -FCPP_LINK bool -cpp_push_token_no_merge(Cpp_Token_Stack *token_stack, Cpp_Token token){ - if (token_stack->count >= token_stack->max_count){ - return 0; - } - - token_stack->tokens[token_stack->count++] = token; - return 1; -} - -FCPP_LINK bool -cpp_push_token_nonalloc(Cpp_Token_Stack *token_stack, Cpp_Token token){ - Cpp_Token_Merge merge = {}; - - if (token_stack->count > 0){ - Cpp_Token prev_token = token_stack->tokens[token_stack->count - 1]; - merge = cpp_attempt_token_merge(prev_token, token); - if (merge.did_merge){ - token_stack->tokens[token_stack->count - 1] = merge.new_token; - } - } - - if (!merge.did_merge){ - if (token_stack->count >= token_stack->max_count){ - return 0; - } - - token_stack->tokens[token_stack->count++] = token; - } - - return 1; -} - -FCPP_LINK Cpp_Read_Result -cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ - Cpp_Lex_Data lex = *lex_data; - Cpp_Read_Result result = {}; - bool has_result = 1; - - fcpp_u16 state_flags = cpp_token_set_pp_state(0, lex.pp_state); - - char current = file.data[lex.pos]; - if (char_is_whitespace(current)){ - result = cpp_read_whitespace(file, lex.pos); - lex.pos = result.pos; - if (result.newline && lex.pp_state != CPP_LEX_PP_DEFAULT){ - lex.pp_state = CPP_LEX_PP_DEFAULT; - } - has_result = 0; - } - - else{ - if (lex.pp_state == CPP_LEX_PP_DEFAULT){ - // TODO(allen): Not first hard of the line? Then it's junk. - if (current == '#'){ - result = cpp_read_preprocessor(file, lex.pos); - lex.pos = result.pos; - switch (result.token.type){ - case CPP_PP_INCLUDE: - case CPP_PP_IMPORT: - case CPP_PP_USING: - lex.pp_state = CPP_LEX_PP_INCLUDE; - break; - case CPP_PP_DEFINE: - lex.pp_state = CPP_LEX_PP_MACRO_IDENTIFIER; - break; - case CPP_PP_UNDEF: - case CPP_PP_IFDEF: - case CPP_PP_IFNDEF: - lex.pp_state = CPP_LEX_PP_IDENTIFIER; - break; - case CPP_PP_IF: - case CPP_PP_ELIF: - lex.pp_state = CPP_LEX_PP_BODY_IF; - break; - case CPP_PP_PRAGMA: - lex.pp_state = CPP_LEX_PP_BODY; - break; - case CPP_PP_LINE: - lex.pp_state = CPP_LEX_PP_NUMBER; - break; - case CPP_PP_ERROR: - lex.pp_state = CPP_LEX_PP_ERROR; - break; - - case CPP_PP_UNKNOWN: - case CPP_PP_ELSE: - case CPP_PP_ENDIF: - lex.pp_state = CPP_LEX_PP_JUNK; - break; - } - } - else{ - result = cpp_read_pp_default_mode(file, lex.pos); - lex.pos = result.pos; - } - } - - else{ - if (current == '\\'){ - fcpp_i32 seek = lex.pos; - ++seek; - while (seek < file.size && file.data[seek] == '\r'){ - ++seek; - } - if ((seek < file.size && file.data[seek] == '\n') || seek >= file.size){ - lex.pos = seek + 1; - has_result = 0; - } - else{ - lex.pp_state = CPP_LEX_PP_JUNK; - result.token.type = CPP_TOKEN_JUNK; - result.token.start = lex.pos; - result.token.size = 1; - result.token.flags |= CPP_TFLAG_PP_BODY; - lex.pos = seek; - } - } - - else{ - switch (lex.pp_state){ - case CPP_LEX_PP_IDENTIFIER: - if (!char_is_alpha_numeric(current)){ - has_result = 0; - lex.pp_state = CPP_LEX_PP_JUNK; - } - else{ - result = cpp_read_alpha_numeric(file, lex.pos); - result.token.flags |= CPP_TFLAG_PP_BODY; - lex.pos = result.pos; - lex.pp_state = CPP_LEX_PP_JUNK; - } - break; - - case CPP_LEX_PP_MACRO_IDENTIFIER: - if (!char_is_alpha_numeric(current)){ - has_result = 0; - lex.pp_state = CPP_LEX_PP_JUNK; - } - else{ - result = cpp_read_alpha_numeric(file, lex.pos); - result.token.flags |= CPP_TFLAG_PP_BODY; - lex.pos = result.pos; - lex.pp_state = CPP_LEX_PP_BODY; - } - break; - - case CPP_LEX_PP_INCLUDE: - if (current != '"' && current != '<'){ - has_result = 0; - lex.pp_state = CPP_LEX_PP_JUNK; - } - else{ - result = cpp_read_pp_include_file(file, lex.pos); - lex.pos = result.pos; - lex.pp_state = CPP_LEX_PP_JUNK; - } - break; - - case CPP_LEX_PP_BODY: - if (current == '#'){ - result = cpp_read_pp_operator(file, lex.pos); - } - else{ - result = cpp_read_pp_default_mode(file, lex.pos); - } - lex.pos = result.pos; - result.token.flags |= CPP_TFLAG_PP_BODY; - break; - - case CPP_LEX_PP_BODY_IF: - if (current == '#'){ - result = cpp_read_pp_operator(file, lex.pos); - } - else{ - result = cpp_read_pp_default_mode(file, lex.pos, 1); - } - lex.pos = result.pos; - result.token.flags |= CPP_TFLAG_PP_BODY; - break; - - case CPP_LEX_PP_NUMBER: - if (!char_is_numeric(current)){ - has_result = 0; - lex.pp_state = CPP_LEX_PP_JUNK; - } - else{ - result = cpp_read_number(file, lex.pos); - lex.pos = result.pos; - result.token.flags |= CPP_TFLAG_PP_BODY; - lex.pp_state = CPP_LEX_PP_INCLUDE; - } - break; - - case CPP_LEX_PP_ERROR: - result = cpp_read_junk_line(file, lex.pos); - lex.pos = result.pos; - result.token.type = CPP_TOKEN_ERROR_MESSAGE; - result.token.flags |= CPP_TFLAG_PP_BODY; - break; - - default: - { - bool took_comment = 0; - if (current == '/' && lex.pos + 1 < file.size){ - if (file.data[lex.pos + 1] == '/'){ - result = cpp_read_line_comment(file, lex.pos); - lex.pp_state = CPP_LEX_PP_DEFAULT; - lex.pos = result.pos; - took_comment = 1; - }else if (file.data[lex.pos + 1] == '*'){ - result = cpp_read_block_comment(file, lex.pos); - lex.pos = result.pos; - took_comment = 1; - } - } - - if (!took_comment){ - result = cpp_read_junk_line(file, lex.pos); - lex.pos = result.pos; - result.token.flags |= CPP_TFLAG_PP_BODY; - } - }break; - - } - } - } - } - - result.token.state_flags = state_flags; - result.has_result = has_result; - - *lex_data = lex; - return result; -} - -FCPP_LINK int -cpp_lex_file_token_count(Cpp_File file){ - int count = 0; - Cpp_Lex_Data lex = {}; - Cpp_Token token = {}; - while (lex.pos < file.size){ - Cpp_Read_Result step_result = cpp_lex_step(file, &lex); - - if (step_result.has_result){ - if (count > 0){ - Cpp_Token_Merge merge = cpp_attempt_token_merge(token, step_result.token); - if (merge.did_merge){ - token = merge.new_token; - } - else{ - token = step_result.token; - ++count; - } - } - else{ - token = step_result.token; - ++count; - } - } - } - return count; -} - -FCPP_LINK Cpp_Lex_Data -cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *token_stack_out, Cpp_Lex_Data data){ - while (data.pos < file.size){ - Cpp_Lex_Data prev_lex = data; - Cpp_Read_Result step_result = cpp_lex_step(file, &data); - - if (step_result.has_result){ - if (!cpp_push_token_nonalloc(token_stack_out, step_result.token)){ - data = prev_lex; - return data; - } - } - } - - data.complete = 1; - return data; -} - -FCPP_LINK Cpp_Get_Token_Result -cpp_get_token(Cpp_Token_Stack *token_stack, int pos){ - int first, last; - first = 0; - last = token_stack->count; - - Cpp_Get_Token_Result result = {}; - if (token_stack->count > 0){ - for (;;){ - result.token_index = (first + last)/2; - - int this_start = token_stack->tokens[result.token_index].start; - int next_start; - if (result.token_index + 1 < token_stack->count){ - next_start = token_stack->tokens[result.token_index+1].start; - } - else{ - next_start = this_start + token_stack->tokens[result.token_index].size; - } - if (this_start <= pos && pos < next_start){ - break; - } - else if (pos < this_start){ - last = result.token_index; - } - else{ - first = result.token_index + 1; - } - if (first == last){ - result.token_index = first; - break; - } - } - - if (result.token_index == token_stack->count){ - --result.token_index; - result.in_whitespace = 1; - } - else{ - Cpp_Token *token = token_stack->tokens + result.token_index; - if (token->start + token->size <= pos){ - result.in_whitespace = 1; - } - } - } - else{ - result.token_index = -1; - result.in_whitespace = 1; - } - - return result; -} - -FCPP_LINK int -cpp_get_end_token(Cpp_Token_Stack *stack, int end){ - Cpp_Get_Token_Result result = cpp_get_token(stack, end); - if (result.token_index < 0) result.token_index = 0; - else if (end > stack->tokens[result.token_index].start) ++result.token_index; - return result.token_index; -} - -FCPP_LINK void -cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token_i, int amount){ - int count = stack->count; - Cpp_Token *token = stack->tokens + from_token_i; - for (int i = from_token_i; i < count; ++i, ++token){ - token->start += amount; - } -} - -FCPP_LINK Cpp_Relex_State -cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, - int start, int end, int amount, int tolerance){ - Cpp_Relex_State state; - state.file = file; - state.stack = stack; - state.start = start; - state.end = end; - state.amount = amount; - state.tolerance = tolerance; - - Cpp_Get_Token_Result result = cpp_get_token(stack, start); - if (result.token_index <= 0){ - state.start_token_i = 0; - } - else{ - state.start_token_i = result.token_index-1; - } - - result = cpp_get_token(stack, end); - if (result.token_index < 0) result.token_index = 0; - else if (end > stack->tokens[result.token_index].start) ++result.token_index; - state.end_token_i = result.token_index; - - state.relex_start = stack->tokens[state.start_token_i].start; - if (start < state.relex_start) state.relex_start = start; - - state.space_request = state.end_token_i - state.start_token_i + tolerance + 1; - - return state; -} - -inline Cpp_Token -cpp__get_token(Cpp_Token_Stack *stack, Cpp_Token *tokens, int size, int index){ - Cpp_Token result; - if (index < stack->count){ - result = tokens[index]; - } - else{ - result.start = size; - result.size = 0; - result.type = CPP_TOKEN_EOF; - result.flags = 0; - result.state_flags = 0; - } - return result; -} - -FCPP_LINK bool -cpp_relex_nonalloc_main(Cpp_Relex_State *state, Cpp_Token_Stack *relex_stack, int *relex_end){ - Cpp_Token_Stack *stack = state->stack; - Cpp_Token *tokens = stack->tokens; - - cpp_shift_token_starts(stack, state->end_token_i, state->amount); - - Cpp_Lex_Data lex = {}; - lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags); - lex.pos = state->relex_start; - - int relex_end_i = state->end_token_i; - Cpp_Token match_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); - Cpp_Token end_token = match_token; - bool went_too_far = 0; - - for (;;){ - Cpp_Read_Result read = cpp_lex_step(state->file, &lex); - if (read.has_result){ - if (read.token.start == end_token.start && - read.token.size == end_token.size && - read.token.flags == end_token.flags && - read.token.state_flags == end_token.state_flags){ - break; - } - cpp_push_token_nonalloc(relex_stack, read.token); - - while (lex.pos > end_token.start && relex_end_i < stack->count){ - ++relex_end_i; - end_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); - } - if (relex_stack->count == relex_stack->max_count){ - went_too_far = 1; - break; - } - } - if (lex.pos >= state->file.size) break; - } - - if (!went_too_far){ - if (relex_stack->count > 0){ - if (state->start_token_i > 0){ - Cpp_Token_Merge merge = - cpp_attempt_token_merge(tokens[state->start_token_i - 1], - relex_stack->tokens[0]); - if (merge.did_merge){ - --state->start_token_i; - relex_stack->tokens[0] = merge.new_token; - } - } - - if (relex_end_i < state->stack->count){ - Cpp_Token_Merge merge = - cpp_attempt_token_merge(relex_stack->tokens[relex_stack->count-1], - tokens[relex_end_i]); - if (merge.did_merge){ - ++relex_end_i; - relex_stack->tokens[relex_stack->count-1] = merge.new_token; - } - } - } - - *relex_end = relex_end_i; - } - else{ - cpp_shift_token_starts(stack, state->end_token_i, -state->amount); - } - - return went_too_far; -} - -#ifndef FCPP_FORBID_MALLOC -FCPP_LINK Cpp_Token_Stack -cpp_make_token_stack(int starting_max){ - Cpp_Token_Stack token_stack; - token_stack.count = 0; - token_stack.max_count = starting_max; - token_stack.tokens = (Cpp_Token*)FCPP_GET_MEMORY(sizeof(Cpp_Token)*starting_max); - return token_stack; -} - -FCPP_LINK void -cpp_free_token_stack(Cpp_Token_Stack token_stack){ - FCPP_FREE_MEMORY(token_stack.tokens); -} - -FCPP_LINK void -cpp_resize_token_stack(Cpp_Token_Stack *token_stack, int new_max){ - Cpp_Token *new_tokens = (Cpp_Token*)FCPP_GET_MEMORY(sizeof(Cpp_Token)*new_max); - - if (new_tokens){ - FCPP_MEM_COPY(new_tokens, token_stack->tokens, sizeof(Cpp_Token)*token_stack->count); - FCPP_FREE_MEMORY(token_stack->tokens); - token_stack->tokens = new_tokens; - token_stack->max_count = new_max; - } -} - -FCPP_LINK void -cpp_push_token(Cpp_Token_Stack *token_stack, Cpp_Token token){ - if (!cpp_push_token_nonalloc(token_stack, token)){ - int new_max = 2*token_stack->max_count + 1; - cpp_resize_token_stack(token_stack, new_max); - bool result = cpp_push_token_nonalloc(token_stack, token); - _Assert(result); - } -} - -FCPP_LINK void -cpp_lex_file(Cpp_File file, Cpp_Token_Stack *token_stack_out){ - Cpp_Lex_Data lex = {}; - while (lex.pos < file.size){ - Cpp_Read_Result step_result = cpp_lex_step(file, &lex); - if (step_result.has_result){ - cpp_push_token(token_stack_out, step_result.token); - } - } -} - -FCPP_LINK bool -cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, - int start, int end, int amount, int tolerance){ -#if 0 - int start_token_i, end_token_i; - Cpp_Get_Token_Result get_result = cpp_get_token(token_stack, start_i); - start_token_i = get_result.token_index; - get_result = cpp_get_token(token_stack, end_i); - end_token_i = get_result.token_index; - if (end_token_i == -1){ - end_token_i = 0; - } - else if (end > token_stack->tokens[end_token_i].start){ - ++end_token_i; - } - cpp_shift_token_starts(token_stack, end_token_i, amount); - - int relex_start_i = start_token_i - 1; - if (relex_start_i < 0){ - relex_start_i = 0; - } - - int end_guess_i = end_token_i + 1; - if (end_guess_i > token_stack->count){ - --end_guess_i; - } -#endif - - int relex_start_i; - int end_token_i, end_guess_i; - { - Cpp_Get_Token_Result result = cpp_get_token(stack, start); - if (result.token_index <= 0){ - relex_start_i = 0; - } - else{ - relex_start_i = result.token_index-1; - } - - result = cpp_get_token(stack, end); - if (result.token_index < 0) result.token_index = 0; - else if (end > stack->tokens[result.token_index].start) ++result.token_index; - end_token_i = result.token_index; - end_guess_i = result.token_index+1; - } - - int relex_start = stack->tokens[relex_start_i].start; - if (start < relex_start) relex_start = start; - - cpp_shift_token_starts(stack, end_token_i, amount); - Cpp_Token_Stack relex_stack = cpp_make_token_stack((end_guess_i - relex_start_i + 1) * 3 / 2); - Cpp_Lex_Data lex = {}; - lex.pp_state = cpp_token_get_pp_state(stack->tokens[relex_start_i].state_flags); - lex.pos = relex_start; - bool went_too_far = 0; - - while (1){ - Cpp_Read_Result result = cpp_lex_step(file, &lex); - if (result.has_result){ - if (end_guess_i < stack->count && - result.token.start == stack->tokens[end_guess_i].start && - result.token.size == stack->tokens[end_guess_i].size && - result.token.flags == stack->tokens[end_guess_i].flags && - result.token.state_flags == stack->tokens[end_guess_i].state_flags){ - break; - } - else{ - cpp_push_token(&relex_stack, result.token); - while (lex.pos > stack->tokens[end_guess_i].start && - end_guess_i < stack->count){ - ++end_guess_i; - } - } - } - - if (lex.pos >= file.size){ - break; - } - - if (tolerance >= 0 && relex_stack.count + relex_start_i >= end_guess_i + tolerance){ - went_too_far = 1; - break; - } - } - - if (!went_too_far){ - int relex_end_i = end_guess_i; - - if (relex_stack.count > 0){ - if (relex_start_i > 0){ - Cpp_Token_Merge merge = cpp_attempt_token_merge(stack->tokens[relex_start_i - 1], - relex_stack.tokens[0]); - if (merge.did_merge){ - --relex_start_i; - relex_stack.tokens[0] = merge.new_token; - } - } - - if (relex_end_i < stack->count){ - Cpp_Token_Merge merge = cpp_attempt_token_merge(relex_stack.tokens[relex_stack.count - 1], - stack->tokens[relex_end_i]); - if (merge.did_merge){ - ++relex_end_i; - relex_stack.tokens[relex_stack.count - 1] = merge.new_token; - } - } - } - - int token_delete_amount = relex_end_i - relex_start_i; - int token_shift_amount = relex_stack.count - token_delete_amount; - - if (token_shift_amount != 0){ - int new_token_count = stack->count + token_shift_amount; - if (new_token_count > stack->max_count){ - int new_max = 2*stack->max_count + 1; - while (new_token_count > new_max){ - new_max = 2*new_max + 1; - } - cpp_resize_token_stack(stack, new_max); - } - - if (relex_end_i < stack->count){ - FCPP_MEM_MOVE(stack->tokens + relex_end_i + token_shift_amount, - stack->tokens + relex_end_i, sizeof(Cpp_Token)*(stack->count - relex_end_i)); - } - - stack->count += token_shift_amount; - } - - FCPP_MEM_COPY(stack->tokens + relex_start_i, relex_stack.tokens, sizeof(Cpp_Token)*relex_stack.count); - cpp_free_token_stack(relex_stack); - } - - else{ - cpp_shift_token_starts(stack, end_token_i, -amount); - cpp_free_token_stack(relex_stack); - } - - return went_too_far; -} -#endif - -#undef _Assert -#undef _TentativeAssert - -#undef FCPP_LEXER_IMPLEMENTATION -#endif // #ifdef FCPP_LEXER_IMPLEMENTATION - -// BOTTOM +/* "4cpp" Open C++ Parser v0.1: Lexer + no warranty implied; use at your own risk + +NOTES ON USE: + OPTIONS: + Set options by defining macros before including this file. + + FCPP_LEXER_IMPLEMENTATION - causes this file to output function implementations + - this option is unset after use so that future includes of this file + in the same unit do not continue to output implementations + + FCPP_NO_MALLOC - prevent including + FCPP_NO_ASSERT - prevent including + FCPP_NO_STRING - prevent including + FCPP_NO_CRT - FCPP_NO_MALLOC & FCPP_NO_ASSERT & FCPP_NO_STRING + + FCPP_FORBID_MALLOC - one step above *NO_MALLOC with this set 4cpp functions that do allocations + are not allowed to be declared or defined at all, forcing the user to handle + allocation themselves + - implies FCPP_NO_MALLOC + + FCPP_GET_MEMORY - defines how to make allocations, interface of malloc, defaults to malloc + FCPP_FREE_MEMORY - defines how to free memory, interface of ree, defaults to free + (The above must be defined if FCPP_NO_MALLOC is set, unless FCPP_FORBID_MALLOC is set) + + FCPP_ASSERT - defines how to make assertions, interface of assert, defaults to assert + + FCPP_MEM_COPY - defines how to copy blocks of memory, interface of memcpy, defaults to memcpy + FCPP_MEM_MOVE - defines how to move blocks of memory, interface of memmove, defaults to memmove + (The above must be defined if FCPP_NO_STRING is set) + + FCPP_LINK - defines linkage of non-inline functions, defaults to static + FCPP_EXTERN - changes FCPP_LINK default to extern, this option is ignored if FCPP_LINK is defined + + include the file "4cpp_clear_config.h" if you want to undefine all options for some reason + + HIDDDEN DEPENDENCIES: + 4cpp is not a single file include library, there are dependencies between the files. + Be sure to include these dependencies before 4cpp_lexer.h: + + 4cpp_types.h + 4cpp_string.h +*/ + +// TOP +// TODO(allen): +// +// EASE OF USE AND DEPLOYMENT +// - make it easier to locate the list of function declarations +// - more C compatibility +// +// POTENTIAL +// - Experiment with optimizations. Sean's State machine? +// - Reserve 0th token for null? Put a EOF token at the end? +// - Pass Cpp_File and Cpp_Token_Stack by value instead of by pointer? +// +// CURRENT +// - lex in chunks +// + +#include "4coder_config.h" + +#ifndef FCPP_LEXER_INC +#define FCPP_LEXER_INC + +enum Cpp_Token_Type{ + CPP_TOKEN_JUNK, + CPP_TOKEN_COMMENT, + + CPP_TOKEN_KEY_TYPE, + CPP_TOKEN_KEY_MODIFIER, + CPP_TOKEN_KEY_QUALIFIER, + CPP_TOKEN_KEY_OPERATOR, // NOTE(allen): This type is not actually stored in tokens + CPP_TOKEN_KEY_CONTROL_FLOW, + CPP_TOKEN_KEY_CAST, + CPP_TOKEN_KEY_TYPE_DECLARATION, + CPP_TOKEN_KEY_ACCESS, + CPP_TOKEN_KEY_LINKAGE, + CPP_TOKEN_KEY_OTHER, + + CPP_TOKEN_IDENTIFIER, + CPP_TOKEN_INTEGER_CONSTANT, + CPP_TOKEN_CHARACTER_CONSTANT, + CPP_TOKEN_FLOATING_CONSTANT, + CPP_TOKEN_STRING_CONSTANT, + CPP_TOKEN_BOOLEAN_CONSTANT, + + CPP_TOKEN_STATIC_ASSERT, + + CPP_TOKEN_BRACKET_OPEN, + CPP_TOKEN_BRACKET_CLOSE, + CPP_TOKEN_PARENTHESE_OPEN, + CPP_TOKEN_PARENTHESE_CLOSE, + CPP_TOKEN_BRACE_OPEN, + CPP_TOKEN_BRACE_CLOSE, + CPP_TOKEN_SEMICOLON, + CPP_TOKEN_ELLIPSIS, + + // NOTE(allen): Ambiguous tokens, lexer only, + // parser figures out the real meaning + CPP_TOKEN_STAR, + CPP_TOKEN_AMPERSAND, + CPP_TOKEN_TILDE, + CPP_TOKEN_PLUS, + CPP_TOKEN_MINUS, + CPP_TOKEN_INCREMENT, + CPP_TOKEN_DECREMENT, + + // NOTE(allen): Precedence 1, LtoR + CPP_TOKEN_SCOPE, + + // NOTE(allen): Precedence 2, LtoR + CPP_TOKEN_POSTINC, // from increment, parser only + CPP_TOKEN_POSTDEC, // from decrement, parser only + CPP_TOKEN_FUNC_STYLE_CAST, // parser only + CPP_TOKEN_CPP_STYLE_CAST, + CPP_TOKEN_CALL, // from open paren, parser only + CPP_TOKEN_INDEX, // from bracket open, parser only + CPP_TOKEN_DOT, + CPP_TOKEN_ARROW, + + // NOTE(allen): Precedence 3, RtoL + CPP_TOKEN_PREINC, // from increment, parser only + CPP_TOKEN_PREDEC, // from decrement, parser only + CPP_TOKEN_POSITIVE, // from plus, parser only + CPP_TOKEN_NEGAITVE, // from minus, parser only + CPP_TOKEN_NOT, + CPP_TOKEN_BIT_NOT, // from tilde, direct from 'compl' + CPP_TOKEN_CAST, // from open paren, parser only + CPP_TOKEN_DEREF, // from star, parser only + CPP_TOKEN_TYPE_PTR, // from star, parser only + CPP_TOKEN_ADDRESS, // from ampersand, parser only + CPP_TOKEN_TYPE_REF, // from ampersand, parser only + CPP_TOKEN_SIZEOF, + CPP_TOKEN_ALIGNOF, + CPP_TOKEN_DECLTYPE, + CPP_TOKEN_TYPEID, + CPP_TOKEN_NEW, + CPP_TOKEN_DELETE, + CPP_TOKEN_NEW_ARRAY, // from new and bracket open, parser only + CPP_TOKEN_DELETE_ARRAY, // from delete and bracket open, parser only + + // NOTE(allen): Precedence 4, LtoR + CPP_TOKEN_PTRDOT, + CPP_TOKEN_PTRARROW, + + // NOTE(allen): Precedence 5, LtoR + CPP_TOKEN_MUL, // from start, parser only + CPP_TOKEN_DIV, + CPP_TOKEN_MOD, + + // NOTE(allen): Precedence 6, LtoR + CPP_TOKEN_ADD, // from plus, parser only + CPP_TOKEN_SUB, // from minus, parser only + + // NOTE(allen): Precedence 7, LtoR + CPP_TOKEN_LSHIFT, + CPP_TOKEN_RSHIFT, + + // NOTE(allen): Precedence 8, LtoR + CPP_TOKEN_LESS, + CPP_TOKEN_GRTR, + CPP_TOKEN_GRTREQ, + CPP_TOKEN_LESSEQ, + + // NOTE(allen): Precedence 9, LtoR + CPP_TOKEN_EQEQ, + CPP_TOKEN_NOTEQ, + + // NOTE(allen): Precedence 10, LtoR + CPP_TOKEN_BIT_AND, // from ampersand, direct from 'bitand' + + // NOTE(allen): Precedence 11, LtoR + CPP_TOKEN_BIT_XOR, + + // NOTE(allen): Precedence 12, LtoR + CPP_TOKEN_BIT_OR, + + // NOTE(allen): Precedence 13, LtoR + CPP_TOKEN_AND, + + // NOTE(allen): Precedence 14, LtoR + CPP_TOKEN_OR, + + // NOTE(allen): Precedence 15, RtoL + CPP_TOKEN_TERNARY_QMARK, + CPP_TOKEN_COLON, + CPP_TOKEN_THROW, + CPP_TOKEN_EQ, + CPP_TOKEN_ADDEQ, + CPP_TOKEN_SUBEQ, + CPP_TOKEN_MULEQ, + CPP_TOKEN_DIVEQ, + CPP_TOKEN_MODEQ, + CPP_TOKEN_LSHIFTEQ, + CPP_TOKEN_RSHIFTEQ, + CPP_TOKEN_ANDEQ, + CPP_TOKEN_OREQ, + CPP_TOKEN_XOREQ, + + // NOTE(allen): Precedence 16, LtoR + CPP_TOKEN_COMMA, + + CPP_PP_INCLUDE, + CPP_PP_DEFINE, + CPP_PP_UNDEF, + CPP_PP_IF, + CPP_PP_IFDEF, + CPP_PP_IFNDEF, + CPP_PP_ELSE, + CPP_PP_ELIF, + CPP_PP_ENDIF, + CPP_PP_ERROR, + CPP_PP_IMPORT, + CPP_PP_USING, + CPP_PP_LINE, + CPP_PP_PRAGMA, + CPP_PP_STRINGIFY, + CPP_PP_CONCAT, + CPP_PP_UNKNOWN, + CPP_TOKEN_DEFINED, + CPP_TOKEN_INCLUDE_FILE, + CPP_TOKEN_ERROR_MESSAGE, + + // NOTE(allen): used in the parser + CPP_TOKEN_EOF +}; + +// TODO(allen): This is a dumb redundant type... probably just +// move towards using String for this everywhere eventually. +struct Cpp_File{ + char *data; + int size; +}; + +Cpp_File +data_as_cpp_file(Data data){ + Cpp_File result; + result.data = (char*)data.data; + result.size = data.size; + return(result); +} + +struct Cpp_Token{ + Cpp_Token_Type type; + fcpp_i32 start, size; + fcpp_u16 state_flags; + fcpp_u16 flags; +}; + +enum Cpp_Token_Flag{ + CPP_TFLAG_IGNORE = 1 << 0, + CPP_TFLAG_PP_DIRECTIVE = 1 << 1, + CPP_TFLAG_PP_BODY = 1 << 2, + CPP_TFLAG_BAD_ENDING = 1 << 3, + CPP_TFLAG_MULTILINE = 1 << 4, + CPP_TFLAG_PARAMETERIZED = 1 << 5, + CPP_TFLAG_IS_OPERATOR = 1 << 6, + CPP_TFLAG_IS_KEYWORD = 1 << 7 +}; + +enum Cpp_Preprocessor_State{ + CPP_LEX_PP_DEFAULT, + CPP_LEX_PP_IDENTIFIER, + CPP_LEX_PP_MACRO_IDENTIFIER, + CPP_LEX_PP_INCLUDE, + CPP_LEX_PP_BODY, + CPP_LEX_PP_BODY_IF, + CPP_LEX_PP_NUMBER, + CPP_LEX_PP_ERROR, + CPP_LEX_PP_JUNK, + // NEVER ADD BELOW THIS + CPP_LEX_PP_COUNT +}; + +struct Cpp_Lex_Data{ + Cpp_Preprocessor_State pp_state; + fcpp_i32 pos; + fcpp_bool32 complete; +}; + +struct Cpp_Read_Result{ + Cpp_Token token; + fcpp_i32 pos; + fcpp_bool8 newline; + fcpp_bool8 has_result; +}; + +struct Cpp_Token_Stack{ + Cpp_Token *tokens; + int count, max_count; +}; + +struct Cpp_Token_Merge{ + Cpp_Token new_token; + fcpp_bool32 did_merge; +}; + +struct Seek_Result{ + fcpp_i32 pos; + fcpp_bool32 new_line; +}; + +struct Cpp_Get_Token_Result{ + fcpp_i32 token_index; + fcpp_bool32 in_whitespace; +}; + +// TODO(allen): revisit this keyword data declaration system +struct String_And_Flag{ + char *str; + fcpp_u32 flags; +}; + +struct String_List{ + String_And_Flag *data; + int count; +}; + +struct Sub_Match_List_Result{ + int index; + fcpp_i32 new_pos; +}; + +inline fcpp_u16 +cpp_token_set_pp_state(fcpp_u16 bitfield, Cpp_Preprocessor_State state_value){ + return (fcpp_u16)state_value; +} + +inline Cpp_Preprocessor_State +cpp_token_get_pp_state(fcpp_u16 bitfield){ + return (Cpp_Preprocessor_State)(bitfield); +} + +inline String +cpp_get_lexeme(char *str, Cpp_Token *token){ + String result; + result.str = str + token->start; + result.size = token->size; + return result; +} + +inline bool +is_keyword(Cpp_Token_Type type){ + return (type >= CPP_TOKEN_KEY_TYPE && type <= CPP_TOKEN_KEY_OTHER); +} + +FCPP_LINK Sub_Match_List_Result sub_match_list(Cpp_File file, int pos, String_List list, int sub_size); + +FCPP_LINK Seek_Result seek_unescaped_eol(char *data, int size, int pos); +FCPP_LINK Seek_Result seek_unescaped_delim(char *data, int size, int pos, char delim); +FCPP_LINK Seek_Result seek_block_comment_end(char *data, int size, int pos); + +FCPP_LINK Cpp_Read_Result cpp_read_whitespace(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_junk_line(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_operator(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_pp_operator(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body); +inline Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos) { return cpp_read_alpha_numeric(file, pos, 0); } +FCPP_LINK Cpp_Read_Result cpp_read_number(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_string_litteral(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_character_litteral(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_line_comment(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_block_comment(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_preprocessor(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_pp_include_file(Cpp_File file, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body); +inline Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos) { return cpp_read_pp_default_mode(file, pos, 0); } + +FCPP_LINK Cpp_Token_Merge cpp_attempt_token_merge(Cpp_Token prev, Cpp_Token next); + +FCPP_LINK bool cpp_push_token_no_merge(Cpp_Token_Stack *stack, Cpp_Token token); +FCPP_LINK bool cpp_push_token_nonalloc(Cpp_Token_Stack *stack, Cpp_Token token); + +FCPP_LINK Cpp_Read_Result cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex); + +FCPP_LINK int cpp_lex_file_token_count(Cpp_File file); +FCPP_LINK Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack, Cpp_Lex_Data data); +inline Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack) { return cpp_lex_file_nonalloc(file, stack, {}); } + +FCPP_LINK Cpp_Get_Token_Result cpp_get_token(Cpp_Token_Stack *stack, int pos); + +FCPP_LINK int cpp_get_end_token(Cpp_Token_Stack *stack, int end); +FCPP_LINK void cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token, int amount); + +struct Cpp_Relex_State{ + Cpp_File file; + Cpp_Token_Stack *stack; + int start, end, amount; + int start_token_i; + int end_token_i; + int relex_start; + int tolerance; + int space_request; +}; + +FCPP_LINK Cpp_Relex_State cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance); +FCPP_LINK bool cpp_relex_nonalloc_main(Cpp_Relex_State state, Cpp_Token_Stack *stack); + +#ifndef FCPP_FORBID_MALLOC +FCPP_LINK Cpp_Token_Stack cpp_make_token_stack(int max); +FCPP_LINK void cpp_free_token_stack(Cpp_Token_Stack stack); +FCPP_LINK void cpp_resize_token_stack(Cpp_Token_Stack *stack, int new_max); + +FCPP_LINK void cpp_push_token(Cpp_Token_Stack *stack, Cpp_Token token); +FCPP_LINK void cpp_lex_file(Cpp_File file, Cpp_Token_Stack *stack); +FCPP_LINK bool cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount, int extra_tolerance); +inline void cpp_relex_file(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount) +{ cpp_relex_file_limited(file, stack, start_i, end_i, amount, -1); } +#endif + +#define FCPP_STRING_LIST(x) {x, FCPP_COUNT(x)} + +// TODO(allen): shift towards storing in a context +FCPP_GLOBAL String_And_Flag int_suf_strings[] = { + {"ull"}, {"ULL"}, + {"llu"}, {"LLU"}, + {"ll"}, {"LL"}, + {"l"}, {"L"}, + {"u"}, {"U"} +}; + +FCPP_GLOBAL String_List int_sufs = FCPP_STRING_LIST(int_suf_strings); + +FCPP_GLOBAL String_And_Flag float_suf_strings[] = { + {"f"}, {"F"}, + {"l"}, {"L"} +}; +FCPP_GLOBAL String_List float_sufs = FCPP_STRING_LIST(float_suf_strings); + +FCPP_GLOBAL String_And_Flag bool_lit_strings[] = { + {"true"}, {"false"} +}; +FCPP_GLOBAL String_List bool_lits = FCPP_STRING_LIST(bool_lit_strings); + +FCPP_GLOBAL String_And_Flag keyword_strings[] = { + {"and", CPP_TOKEN_AND}, + {"and_eq", CPP_TOKEN_ANDEQ}, + {"bitand", CPP_TOKEN_BIT_AND}, + {"bitor", CPP_TOKEN_BIT_OR}, + {"or", CPP_TOKEN_OR}, + {"or_eq", CPP_TOKEN_OREQ}, + {"sizeof", CPP_TOKEN_SIZEOF}, + {"alignof", CPP_TOKEN_ALIGNOF}, + {"decltype", CPP_TOKEN_DECLTYPE}, + {"throw", CPP_TOKEN_THROW}, + {"new", CPP_TOKEN_NEW}, + {"delete", CPP_TOKEN_DELETE}, + {"xor", CPP_TOKEN_BIT_XOR}, + {"xor_eq", CPP_TOKEN_XOREQ}, + {"not", CPP_TOKEN_NOT}, + {"not_eq", CPP_TOKEN_NOTEQ}, + {"typeid", CPP_TOKEN_TYPEID}, + {"compl", CPP_TOKEN_BIT_NOT}, + + {"void", CPP_TOKEN_KEY_TYPE}, + {"bool", CPP_TOKEN_KEY_TYPE}, + {"char", CPP_TOKEN_KEY_TYPE}, + {"int", CPP_TOKEN_KEY_TYPE}, + {"float", CPP_TOKEN_KEY_TYPE}, + {"double", CPP_TOKEN_KEY_TYPE}, + + {"long", CPP_TOKEN_KEY_MODIFIER}, + {"short", CPP_TOKEN_KEY_MODIFIER}, + {"unsigned", CPP_TOKEN_KEY_MODIFIER}, + + {"const", CPP_TOKEN_KEY_QUALIFIER}, + {"volatile", CPP_TOKEN_KEY_QUALIFIER}, + + {"asm", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"break", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"case", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"catch", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"continue", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"default", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"do", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"else", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"for", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"goto", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"if", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"return", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"switch", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"try", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"while", CPP_TOKEN_KEY_CONTROL_FLOW}, + {"static_assert", CPP_TOKEN_KEY_CONTROL_FLOW}, + + {"const_cast", CPP_TOKEN_KEY_CAST}, + {"dynamic_cast", CPP_TOKEN_KEY_CAST}, + {"reinterpret_cast", CPP_TOKEN_KEY_CAST}, + {"static_cast", CPP_TOKEN_KEY_CAST}, + + {"class", CPP_TOKEN_KEY_TYPE_DECLARATION}, + {"enum", CPP_TOKEN_KEY_TYPE_DECLARATION}, + {"struct", CPP_TOKEN_KEY_TYPE_DECLARATION}, + {"typedef", CPP_TOKEN_KEY_TYPE_DECLARATION}, + {"union", CPP_TOKEN_KEY_TYPE_DECLARATION}, + {"template", CPP_TOKEN_KEY_TYPE_DECLARATION}, + {"typename", CPP_TOKEN_KEY_TYPE_DECLARATION}, + + {"friend", CPP_TOKEN_KEY_ACCESS}, + {"namespace", CPP_TOKEN_KEY_ACCESS}, + {"private", CPP_TOKEN_KEY_ACCESS}, + {"protected", CPP_TOKEN_KEY_ACCESS}, + {"public", CPP_TOKEN_KEY_ACCESS}, + {"using", CPP_TOKEN_KEY_ACCESS}, + + {"extern", CPP_TOKEN_KEY_LINKAGE}, + {"export", CPP_TOKEN_KEY_LINKAGE}, + {"inline", CPP_TOKEN_KEY_LINKAGE}, + {"static", CPP_TOKEN_KEY_LINKAGE}, + {"virtual", CPP_TOKEN_KEY_LINKAGE}, + + {"alignas", CPP_TOKEN_KEY_OTHER}, + {"explicit", CPP_TOKEN_KEY_OTHER}, + {"noexcept", CPP_TOKEN_KEY_OTHER}, + {"nullptr", CPP_TOKEN_KEY_OTHER}, + {"operator", CPP_TOKEN_KEY_OTHER}, + {"register", CPP_TOKEN_KEY_OTHER}, + {"this", CPP_TOKEN_KEY_OTHER}, + {"thread_local", CPP_TOKEN_KEY_OTHER}, +}; +FCPP_GLOBAL String_List keywords = FCPP_STRING_LIST(keyword_strings); + +FCPP_GLOBAL String_And_Flag op_strings[] = { + {"...", CPP_TOKEN_ELLIPSIS}, + {"<<=", CPP_TOKEN_LSHIFTEQ}, + {">>=", CPP_TOKEN_RSHIFTEQ}, + {"->*", CPP_TOKEN_PTRARROW}, + {"<<", CPP_TOKEN_LSHIFT}, + {">>", CPP_TOKEN_RSHIFT}, + {"&&", CPP_TOKEN_AND}, + {"||", CPP_TOKEN_OR}, + {"->", CPP_TOKEN_ARROW}, + {"++", CPP_TOKEN_INCREMENT}, + {"--", CPP_TOKEN_DECREMENT}, + {"::", CPP_TOKEN_SCOPE}, + {"+=", CPP_TOKEN_ADDEQ}, + {"-=", CPP_TOKEN_SUBEQ}, + {"*=", CPP_TOKEN_MULEQ}, + {"/=", CPP_TOKEN_DIVEQ}, + {"%=", CPP_TOKEN_MODEQ}, + {"&=", CPP_TOKEN_ANDEQ}, + {"|=", CPP_TOKEN_OREQ}, + {"^=", CPP_TOKEN_XOREQ}, + {"==", CPP_TOKEN_EQEQ}, + {">=", CPP_TOKEN_GRTREQ}, + {"<=", CPP_TOKEN_LESSEQ}, + {"!=", CPP_TOKEN_NOTEQ}, + {".*", CPP_TOKEN_PTRDOT}, + {"{", CPP_TOKEN_BRACE_OPEN}, + {"}", CPP_TOKEN_BRACE_CLOSE}, + {"[", CPP_TOKEN_BRACKET_OPEN}, + {"]", CPP_TOKEN_BRACKET_CLOSE}, + {"(", CPP_TOKEN_PARENTHESE_OPEN}, + {")", CPP_TOKEN_PARENTHESE_CLOSE}, + {"<", CPP_TOKEN_LESS}, + {">", CPP_TOKEN_GRTR}, + {"+", CPP_TOKEN_PLUS}, + {"-", CPP_TOKEN_MINUS}, + {"!", CPP_TOKEN_NOT}, + {"~", CPP_TOKEN_TILDE}, + {"*", CPP_TOKEN_STAR}, + {"&", CPP_TOKEN_AMPERSAND}, + {"|", CPP_TOKEN_BIT_OR}, + {"^", CPP_TOKEN_BIT_XOR}, + {"=", CPP_TOKEN_EQ}, + {",", CPP_TOKEN_COMMA}, + {":", CPP_TOKEN_COLON}, + {";", CPP_TOKEN_SEMICOLON}, + {"/", CPP_TOKEN_DIV}, + {"?", CPP_TOKEN_TERNARY_QMARK}, + {"%", CPP_TOKEN_MOD}, + {".", CPP_TOKEN_DOT}, +}; +FCPP_GLOBAL String_List ops = FCPP_STRING_LIST(op_strings); + +FCPP_GLOBAL String_And_Flag pp_op_strings[] = { + {"##", CPP_PP_CONCAT}, + {"#", CPP_PP_STRINGIFY}, +}; +FCPP_GLOBAL String_List pp_ops = FCPP_STRING_LIST(pp_op_strings); + +FCPP_GLOBAL String_And_Flag preprop_strings[] = { + {"include", CPP_PP_INCLUDE}, + {"INCLUDE", CPP_PP_INCLUDE}, + {"ifndef", CPP_PP_IFNDEF}, + {"IFNDEF", CPP_PP_IFNDEF}, + {"define", CPP_PP_DEFINE}, + {"DEFINE", CPP_PP_DEFINE}, + {"import", CPP_PP_IMPORT}, + {"IMPORT", CPP_PP_IMPORT}, + {"pragma", CPP_PP_PRAGMA}, + {"PRAGMA", CPP_PP_PRAGMA}, + {"undef", CPP_PP_UNDEF}, + {"UNDEF", CPP_PP_UNDEF}, + {"endif", CPP_PP_ENDIF}, + {"ENDIF", CPP_PP_ENDIF}, + {"error", CPP_PP_ERROR}, + {"ERROR", CPP_PP_ERROR}, + {"ifdef", CPP_PP_IFDEF}, + {"IFDEF", CPP_PP_IFDEF}, + {"using", CPP_PP_USING}, + {"USING", CPP_PP_USING}, + {"else", CPP_PP_ELSE}, + {"ELSE", CPP_PP_ELSE}, + {"elif", CPP_PP_ELIF}, + {"ELIF", CPP_PP_ELIF}, + {"line", CPP_PP_LINE}, + {"LINE", CPP_PP_LINE}, + {"if", CPP_PP_IF}, + {"IF", CPP_PP_IF}, +}; +FCPP_GLOBAL String_List preprops = FCPP_STRING_LIST(preprop_strings); + +#undef FCPP_STRING_LIST + +#endif // #ifndef FCPP_CPP_LEXER + +#ifdef FCPP_LEXER_IMPLEMENTATION + +#define _Assert FCPP_ASSERT +#define _TentativeAssert FCPP_ASSERT + +FCPP_LINK Sub_Match_List_Result +sub_match_list(Cpp_File file, int pos, String_List list, int sub_size){ + Sub_Match_List_Result result; + String str_main; + char *str_check; + int i,l; + + result.index = -1; + result.new_pos = pos; + str_main = make_string(file.data + pos, file.size - pos); + if (sub_size > 0){ + str_main = substr(str_main, 0, sub_size); + for (i = 0; i < list.count; ++i){ + str_check = list.data[i].str; + if (match(str_main, str_check)){ + result.index = i; + result.new_pos = pos + sub_size; + break; + } + } + } + else{ + for (i = 0; i < list.count; ++i){ + str_check = list.data[i].str; + if (match_part(str_main, str_check, &l)){ + result.index = i; + result.new_pos = pos + l; + break; + } + } + } + return result; +} + +FCPP_LINK Seek_Result +seek_unescaped_eol(char *data, int size, int pos){ + Seek_Result result = {}; + ++pos; + while (pos < size){ + if (data[pos] == '\\'){ + if (pos + 1 < size && + data[pos+1] == '\n'){ + result.new_line = 1; + ++pos; + } + else if (pos + 1 < size && + data[pos+1] == '\r' && + pos + 2 < size && + data[pos+2] == '\n'){ + result.new_line = 1; + pos += 2; + } + } + else if (data[pos] == '\n'){ + break; + } + ++pos; + } + ++pos; + + result.pos = pos; + return result; +} + +FCPP_LINK Seek_Result +seek_unescaped_delim(char *data, int size, int pos, char delim){ + Seek_Result result = {}; + bool escape = 0; + ++pos; + while (pos < size){ + if (data[pos] == '\n'){ + result.new_line = 1; + } + if (escape){ + escape = 0; + } + else{ + if (data[pos] == '\\'){ + escape = 1; + } + else if (data[pos] == delim){ + break; + } + } + ++pos; + } + ++pos; + + result.pos = pos; + return result; +} + +FCPP_LINK Seek_Result +seek_block_comment_end(char *data, int size, int pos){ + Seek_Result result = {}; + pos += 2; + while (pos < size){ + if (data[pos] == '*' && + pos + 1 < size && + data[pos+1] == '/'){ + break; + } + if (data[pos] == '\n'){ + result.new_line = 1; + } + ++pos; + } + pos += 2; + result.pos = pos; + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_whitespace(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + + while (pos < file.size && char_is_whitespace(file.data[pos])){ + if (file.data[pos] == '\n'){ + result.newline = 1; + } + ++pos; + } + + result.pos = pos; + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_junk_line(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + result.token.start = pos; + result.token.type = CPP_TOKEN_JUNK; + + bool comment_end = 0; + while (pos < file.size && file.data[pos] != '\n'){ + if (file.data[pos] == '/' && pos + 1 < file.size){ + if (file.data[pos + 1] == '/' || + file.data[pos + 1] == '*'){ + comment_end = 1; + break; + } + } + ++pos; + } + + if (comment_end){ + result.pos = pos; + result.token.size = pos - result.token.start; + } + else{ + while (pos > 0 && file.data[pos - 1] == '\r'){ + --pos; + } + if (pos > 0 && file.data[pos - 1] == '\\'){ + --pos; + } + result.pos = pos; + result.token.size = pos - result.token.start; + } + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_operator(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + result.pos = pos; + result.token.start = pos; + + Sub_Match_List_Result match; + match = sub_match_list(file, result.token.start, ops, -1); + + if (match.index != -1){ + result.pos = match.new_pos; + result.token.size = result.pos - result.token.start; + result.token.type = (Cpp_Token_Type)ops.data[match.index].flags; + result.token.flags |= CPP_TFLAG_IS_OPERATOR; + } + else{ + result.token.size = 1; + result.token.type = CPP_TOKEN_JUNK; + result.pos = pos + 1; + } + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_pp_operator(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + result.pos = pos; + result.token.start = pos; + + Sub_Match_List_Result match; + match = sub_match_list(file, result.token.start, pp_ops, -1); + + _Assert(match.index != -1); + result.pos = match.new_pos; + result.token.size = result.pos - result.token.start; + result.token.type = (Cpp_Token_Type)pp_ops.data[match.index].flags; + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){ + Cpp_Read_Result result = {}; + result.pos = pos; + result.token.start = pos; + + while (result.pos < file.size && + char_is_alpha_numeric(file.data[result.pos])){ + ++result.pos; + } + + result.token.size = result.pos - result.token.start; + + // TODO(allen): do better + if (in_if_body){ + String word; + word.size = result.token.size; + word.str = file.data + result.token.start; + if (match(word, "defined")){ + result.token.type = CPP_TOKEN_DEFINED; + result.token.flags |= CPP_TFLAG_IS_OPERATOR; + result.token.flags |= CPP_TFLAG_IS_KEYWORD; + } + } + + if (result.token.type == CPP_TOKEN_JUNK){ + Sub_Match_List_Result match; + match = sub_match_list(file, result.token.start, bool_lits, result.token.size); + + if (match.index != -1){ + result.token.type = CPP_TOKEN_BOOLEAN_CONSTANT; + result.token.flags |= CPP_TFLAG_IS_KEYWORD; + } + else{ + match = sub_match_list(file, result.token.start, keywords, result.token.size); + + if (match.index != -1){ + String_And_Flag data = keywords.data[match.index]; + result.token.type = (Cpp_Token_Type)data.flags; + result.token.flags |= CPP_TFLAG_IS_KEYWORD; + } + else{ + result.token.type = CPP_TOKEN_IDENTIFIER; + } + } + } + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_number(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + result.pos = pos; + result.token.start = pos; + + bool is_float = 0; + bool is_integer = 0; + bool is_oct = 0; + bool is_hex = 0; + bool is_zero = 0; + + if (file.data[pos] == '0'){ + if (pos+1 < file.size){ + char next = file.data[pos+1]; + if (next == 'x'){ + is_hex = 1; + is_integer = 1; + } + else if (next == '.'){ + is_float = 1; + ++result.pos; + } + else if (next >= '0' && next <= '9'){ + is_oct = 1; + is_integer = 1; + } + else{ + is_zero = 1; + is_integer = 1; + } + } + else{ + is_zero = 1; + is_integer = 1; + } + } + else if (file.data[pos] == '.'){ + is_float = 1; + } + + if (is_zero){ + ++result.pos; + } + else if (is_hex){ + ++result.pos; + char character; + do{ + ++result.pos; + if (result.pos >= file.size){ + break; + } + character = file.data[result.pos]; + } while(char_is_hex(character)); + } + else if (is_oct){ + char character; + do{ + ++result.pos; + if (result.pos >= file.size){ + break; + } + character = file.data[result.pos]; + }while(char_is_numeric(character)); + } + else{ + if (!is_float){ + is_integer = 1; + while (1){ + ++result.pos; + + if (result.pos >= file.size){ + break; + } + bool is_good = 0; + char character = file.data[result.pos]; + if (character >= '0' && character <= '9'){ + is_good = 1; + } + else if (character == '.'){ + is_integer = 0; + is_float = 1; + } + if (!is_good){ + break; + } + } + } + + if (is_float){ + bool e_mode = 0; + bool e_minus = 0; + bool is_good = 0; + char character; + + while (1){ + ++result.pos; + if (result.pos >= file.size){ + break; + } + is_good = 0; + character = file.data[result.pos]; + if (character >= '0' && character <= '9'){ + is_good = 1; + } + else{ + if (character == 'e' && !e_mode){ + e_mode = 1; + is_good = 1; + } + else if (character == '-' && e_mode && !e_minus){ + e_minus = 1; + is_good = 1; + } + } + if (!is_good){ + break; + } + } + } + } + + if (is_integer){ + Sub_Match_List_Result match = + sub_match_list(file, result.pos, int_sufs, -1); + if (match.index != -1){ + result.pos = match.new_pos; + } + result.token.type = CPP_TOKEN_INTEGER_CONSTANT; + result.token.size = result.pos - result.token.start; + } + else if (is_float){ + Sub_Match_List_Result match = + sub_match_list(file, result.pos, float_sufs, -1); + if (match.index != -1){ + result.pos = match.new_pos; + } + result.token.type = CPP_TOKEN_FLOATING_CONSTANT; + result.token.size = result.pos - result.token.start; + } + else{ + _Assert(!"This shouldn't happen!"); + } + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_string_litteral(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + result.token.start = pos; + + _Assert(file.data[pos] == '"'); + Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '"'); + pos = seek.pos; + if (seek.new_line){ + result.token.flags |= CPP_TFLAG_MULTILINE; + } + + result.token.size = pos - result.token.start; + result.token.type = CPP_TOKEN_STRING_CONSTANT; + result.pos = pos; + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_character_litteral(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + result.token.start = pos; + + _Assert(file.data[pos] == '\''); + Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '\''); + pos = seek.pos; + if (seek.new_line){ + result.token.flags |= CPP_TFLAG_MULTILINE; + } + + result.token.size = pos - result.token.start; + result.token.type = CPP_TOKEN_CHARACTER_CONSTANT; + result.pos = pos; + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_line_comment(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + result.token.start = pos; + + _Assert(file.data[pos] == '/' && file.data[pos + 1] == '/'); + + pos += 2; + while (pos < file.size){ + if (file.data[pos] == '\n'){ + break; + } + if (file.data[pos] == '\\'){ + if (pos + 1 < file.size && + file.data[pos + 1] == '\n'){ + ++pos; + } + else if (pos + 2 < file.size && + file.data[pos + 1] == '\r' && + file.data[pos + 2] == '\n'){ + pos += 2; + } + } + ++pos; + } + if (pos > 0 && file.data[pos-1] == '\r'){ + --pos; + } + result.token.size = pos - result.token.start; + result.token.type = CPP_TOKEN_COMMENT; + result.pos = pos; + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_block_comment(Cpp_File file, int pos){ + Cpp_Read_Result result = {}; + result.token.start = pos; + + _Assert(file.data[pos] == '/' && file.data[pos + 1] == '*'); + pos += 2; + while (pos < file.size){ + if (file.data[pos] == '*' && + pos + 1 < file.size && + file.data[pos+1] == '/'){ + break; + } + ++pos; + } + pos += 2; + result.token.size = pos - result.token.start; + result.token.type = CPP_TOKEN_COMMENT; + result.pos = pos; + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_preprocessor(Cpp_File file, int pos){ + _Assert(file.data[pos] == '#'); + Cpp_Read_Result result = {}; + result.token.start = pos; + result.token.type = CPP_PP_UNKNOWN; + result.token.flags |= CPP_TFLAG_PP_DIRECTIVE; + + ++pos; + while (pos < file.size && + (file.data[pos] == ' ' || + file.data[pos] == '\t')){ + ++pos; + } + + Sub_Match_List_Result match + = sub_match_list(file, pos, preprops, -1); + + if (match.index != -1){ + result.token.size = match.new_pos - result.token.start; + result.token.type = (Cpp_Token_Type)preprops.data[match.index].flags; + result.pos = match.new_pos; + } + else{ + while (pos < file.size && + !char_is_whitespace(file.data[pos])){ + ++pos; + } + result.token.size = pos - result.token.start; + result.pos = pos; + } + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_pp_include_file(Cpp_File file, int pos){ + char start = file.data[pos]; + _Assert(start == '<' || start == '"'); + + Cpp_Read_Result result = {}; + result.token.start = pos; + result.token.type = CPP_TOKEN_INCLUDE_FILE; + result.token.flags |= CPP_TFLAG_PP_BODY; + + char end; + if (start == '<'){ + end = '>'; + } + else{ + end = '"'; + } + + ++pos; + while (pos < file.size && file.data[pos] != end){ + if (file.data[pos] == '\n'){ + result.token.type = CPP_TOKEN_JUNK; + result.token.flags |= CPP_TFLAG_BAD_ENDING; + break; + } + if (file.data[pos] == '\\'){ + // TODO(allen): Not sure that this is 100% correct. + if (pos + 1 < file.size && + file.data[pos + 1] == '\n'){ + ++pos; + result.token.flags |= CPP_TFLAG_MULTILINE; + } + else if (pos + 2 < file.size && + file.data[pos + 1] == '\r' && + file.data[pos + 2] == '\n'){ + pos += 2; + result.token.flags |= CPP_TFLAG_MULTILINE; + } + } + ++pos; + } + + if (result.token.type != CPP_TOKEN_JUNK){ + if (pos < file.size){ + ++pos; + } + } + + result.token.size = pos - result.token.start; + result.pos = pos; + + return result; +} + +FCPP_LINK Cpp_Read_Result +cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body){ + char current = file.data[pos]; + Cpp_Read_Result result; + if (char_is_numeric(current)){ + result = cpp_read_number(file, pos); + } + else if (char_is_alpha(current)){ + result = cpp_read_alpha_numeric(file, pos, in_if_body); + } + else if (current == '.'){ + if (pos + 1 < file.size){ + char next = file.data[pos + 1]; + if (char_is_numeric(next)){ + result = cpp_read_number(file, pos); + } + else{ + result = cpp_read_operator(file, pos); + } + } + else{ + result = cpp_read_operator(file, pos); + } + } + + else if (current == '/'){ + if (pos + 1 < file.size){ + char next = file.data[pos + 1]; + if (next == '/'){ + result = cpp_read_line_comment(file, pos); + } + else if (next == '*'){ + result = cpp_read_block_comment(file, pos); + } + else{ + result = cpp_read_operator(file, pos); + } + } + else{ + result = cpp_read_operator(file, pos); + } + } + else if (current == '"'){ + result = cpp_read_string_litteral(file, pos); + } + else if (current == '\''){ + result = cpp_read_character_litteral(file, pos); + } + else{ + result = cpp_read_operator(file, pos); + } + + return result; +} + +FCPP_LINK Cpp_Token_Merge +cpp_attempt_token_merge(Cpp_Token prev_token, Cpp_Token next_token){ + Cpp_Token_Merge result = {}; + if (next_token.type == CPP_TOKEN_COMMENT && prev_token.type == CPP_TOKEN_COMMENT && + next_token.flags == prev_token.flags && next_token.state_flags == prev_token.state_flags){ + result.did_merge = 1; + prev_token.size = next_token.start + next_token.size - prev_token.start; + result.new_token = prev_token; + } + else if (next_token.type == CPP_TOKEN_JUNK && prev_token.type == CPP_TOKEN_JUNK && + next_token.flags == prev_token.flags && next_token.state_flags == prev_token.state_flags){ + result.did_merge = 1; + prev_token.size = next_token.start + next_token.size - prev_token.start; + result.new_token = prev_token; + } + return result; +} + +FCPP_LINK bool +cpp_push_token_no_merge(Cpp_Token_Stack *token_stack, Cpp_Token token){ + if (token_stack->count >= token_stack->max_count){ + return 0; + } + + token_stack->tokens[token_stack->count++] = token; + return 1; +} + +FCPP_LINK bool +cpp_push_token_nonalloc(Cpp_Token_Stack *token_stack, Cpp_Token token){ + Cpp_Token_Merge merge = {}; + + if (token_stack->count > 0){ + Cpp_Token prev_token = token_stack->tokens[token_stack->count - 1]; + merge = cpp_attempt_token_merge(prev_token, token); + if (merge.did_merge){ + token_stack->tokens[token_stack->count - 1] = merge.new_token; + } + } + + if (!merge.did_merge){ + if (token_stack->count >= token_stack->max_count){ + return 0; + } + + token_stack->tokens[token_stack->count++] = token; + } + + return 1; +} + +FCPP_LINK Cpp_Read_Result +cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ + Cpp_Lex_Data lex = *lex_data; + Cpp_Read_Result result = {}; + bool has_result = 1; + + fcpp_u16 state_flags = cpp_token_set_pp_state(0, lex.pp_state); + + char current = file.data[lex.pos]; + if (char_is_whitespace(current)){ + result = cpp_read_whitespace(file, lex.pos); + lex.pos = result.pos; + if (result.newline && lex.pp_state != CPP_LEX_PP_DEFAULT){ + lex.pp_state = CPP_LEX_PP_DEFAULT; + } + has_result = 0; + } + + else{ + if (lex.pp_state == CPP_LEX_PP_DEFAULT){ + // TODO(allen): Not first hard of the line? Then it's junk. + if (current == '#'){ + result = cpp_read_preprocessor(file, lex.pos); + lex.pos = result.pos; + switch (result.token.type){ + case CPP_PP_INCLUDE: + case CPP_PP_IMPORT: + case CPP_PP_USING: + lex.pp_state = CPP_LEX_PP_INCLUDE; + break; + case CPP_PP_DEFINE: + lex.pp_state = CPP_LEX_PP_MACRO_IDENTIFIER; + break; + case CPP_PP_UNDEF: + case CPP_PP_IFDEF: + case CPP_PP_IFNDEF: + lex.pp_state = CPP_LEX_PP_IDENTIFIER; + break; + case CPP_PP_IF: + case CPP_PP_ELIF: + lex.pp_state = CPP_LEX_PP_BODY_IF; + break; + case CPP_PP_PRAGMA: + lex.pp_state = CPP_LEX_PP_BODY; + break; + case CPP_PP_LINE: + lex.pp_state = CPP_LEX_PP_NUMBER; + break; + case CPP_PP_ERROR: + lex.pp_state = CPP_LEX_PP_ERROR; + break; + + case CPP_PP_UNKNOWN: + case CPP_PP_ELSE: + case CPP_PP_ENDIF: + lex.pp_state = CPP_LEX_PP_JUNK; + break; + } + } + else{ + result = cpp_read_pp_default_mode(file, lex.pos); + lex.pos = result.pos; + } + } + + else{ + if (current == '\\'){ + fcpp_i32 seek = lex.pos; + ++seek; + while (seek < file.size && file.data[seek] == '\r'){ + ++seek; + } + if ((seek < file.size && file.data[seek] == '\n') || seek >= file.size){ + lex.pos = seek + 1; + has_result = 0; + } + else{ + lex.pp_state = CPP_LEX_PP_JUNK; + result.token.type = CPP_TOKEN_JUNK; + result.token.start = lex.pos; + result.token.size = 1; + result.token.flags |= CPP_TFLAG_PP_BODY; + lex.pos = seek; + } + } + + else{ + switch (lex.pp_state){ + case CPP_LEX_PP_IDENTIFIER: + if (!char_is_alpha_numeric(current)){ + has_result = 0; + lex.pp_state = CPP_LEX_PP_JUNK; + } + else{ + result = cpp_read_alpha_numeric(file, lex.pos); + result.token.flags |= CPP_TFLAG_PP_BODY; + lex.pos = result.pos; + lex.pp_state = CPP_LEX_PP_JUNK; + } + break; + + case CPP_LEX_PP_MACRO_IDENTIFIER: + if (!char_is_alpha_numeric(current)){ + has_result = 0; + lex.pp_state = CPP_LEX_PP_JUNK; + } + else{ + result = cpp_read_alpha_numeric(file, lex.pos); + result.token.flags |= CPP_TFLAG_PP_BODY; + lex.pos = result.pos; + lex.pp_state = CPP_LEX_PP_BODY; + } + break; + + case CPP_LEX_PP_INCLUDE: + if (current != '"' && current != '<'){ + has_result = 0; + lex.pp_state = CPP_LEX_PP_JUNK; + } + else{ + result = cpp_read_pp_include_file(file, lex.pos); + lex.pos = result.pos; + lex.pp_state = CPP_LEX_PP_JUNK; + } + break; + + case CPP_LEX_PP_BODY: + if (current == '#'){ + result = cpp_read_pp_operator(file, lex.pos); + } + else{ + result = cpp_read_pp_default_mode(file, lex.pos); + } + lex.pos = result.pos; + result.token.flags |= CPP_TFLAG_PP_BODY; + break; + + case CPP_LEX_PP_BODY_IF: + if (current == '#'){ + result = cpp_read_pp_operator(file, lex.pos); + } + else{ + result = cpp_read_pp_default_mode(file, lex.pos, 1); + } + lex.pos = result.pos; + result.token.flags |= CPP_TFLAG_PP_BODY; + break; + + case CPP_LEX_PP_NUMBER: + if (!char_is_numeric(current)){ + has_result = 0; + lex.pp_state = CPP_LEX_PP_JUNK; + } + else{ + result = cpp_read_number(file, lex.pos); + lex.pos = result.pos; + result.token.flags |= CPP_TFLAG_PP_BODY; + lex.pp_state = CPP_LEX_PP_INCLUDE; + } + break; + + case CPP_LEX_PP_ERROR: + result = cpp_read_junk_line(file, lex.pos); + lex.pos = result.pos; + result.token.type = CPP_TOKEN_ERROR_MESSAGE; + result.token.flags |= CPP_TFLAG_PP_BODY; + break; + + default: + { + bool took_comment = 0; + if (current == '/' && lex.pos + 1 < file.size){ + if (file.data[lex.pos + 1] == '/'){ + result = cpp_read_line_comment(file, lex.pos); + lex.pp_state = CPP_LEX_PP_DEFAULT; + lex.pos = result.pos; + took_comment = 1; + }else if (file.data[lex.pos + 1] == '*'){ + result = cpp_read_block_comment(file, lex.pos); + lex.pos = result.pos; + took_comment = 1; + } + } + + if (!took_comment){ + result = cpp_read_junk_line(file, lex.pos); + lex.pos = result.pos; + result.token.flags |= CPP_TFLAG_PP_BODY; + } + }break; + + } + } + } + } + + result.token.state_flags = state_flags; + result.has_result = has_result; + + *lex_data = lex; + return result; +} + +FCPP_LINK int +cpp_lex_file_token_count(Cpp_File file){ + int count = 0; + Cpp_Lex_Data lex = {}; + Cpp_Token token = {}; + while (lex.pos < file.size){ + Cpp_Read_Result step_result = cpp_lex_step(file, &lex); + + if (step_result.has_result){ + if (count > 0){ + Cpp_Token_Merge merge = cpp_attempt_token_merge(token, step_result.token); + if (merge.did_merge){ + token = merge.new_token; + } + else{ + token = step_result.token; + ++count; + } + } + else{ + token = step_result.token; + ++count; + } + } + } + return count; +} + +FCPP_LINK Cpp_Lex_Data +cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *token_stack_out, Cpp_Lex_Data data){ + while (data.pos < file.size){ + Cpp_Lex_Data prev_lex = data; + Cpp_Read_Result step_result = cpp_lex_step(file, &data); + + if (step_result.has_result){ + if (!cpp_push_token_nonalloc(token_stack_out, step_result.token)){ + data = prev_lex; + return data; + } + } + } + + data.complete = 1; + return data; +} + +FCPP_LINK Cpp_Get_Token_Result +cpp_get_token(Cpp_Token_Stack *token_stack, int pos){ + int first, last; + first = 0; + last = token_stack->count; + + Cpp_Get_Token_Result result = {}; + if (token_stack->count > 0){ + for (;;){ + result.token_index = (first + last)/2; + + int this_start = token_stack->tokens[result.token_index].start; + int next_start; + if (result.token_index + 1 < token_stack->count){ + next_start = token_stack->tokens[result.token_index+1].start; + } + else{ + next_start = this_start + token_stack->tokens[result.token_index].size; + } + if (this_start <= pos && pos < next_start){ + break; + } + else if (pos < this_start){ + last = result.token_index; + } + else{ + first = result.token_index + 1; + } + if (first == last){ + result.token_index = first; + break; + } + } + + if (result.token_index == token_stack->count){ + --result.token_index; + result.in_whitespace = 1; + } + else{ + Cpp_Token *token = token_stack->tokens + result.token_index; + if (token->start + token->size <= pos){ + result.in_whitespace = 1; + } + } + } + else{ + result.token_index = -1; + result.in_whitespace = 1; + } + + return result; +} + +FCPP_LINK int +cpp_get_end_token(Cpp_Token_Stack *stack, int end){ + Cpp_Get_Token_Result result = cpp_get_token(stack, end); + if (result.token_index < 0) result.token_index = 0; + else if (end > stack->tokens[result.token_index].start) ++result.token_index; + return result.token_index; +} + +FCPP_LINK void +cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token_i, int amount){ + int count = stack->count; + Cpp_Token *token = stack->tokens + from_token_i; + for (int i = from_token_i; i < count; ++i, ++token){ + token->start += amount; + } +} + +FCPP_LINK Cpp_Relex_State +cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, + int start, int end, int amount, int tolerance){ + Cpp_Relex_State state; + state.file = file; + state.stack = stack; + state.start = start; + state.end = end; + state.amount = amount; + state.tolerance = tolerance; + + Cpp_Get_Token_Result result = cpp_get_token(stack, start); + if (result.token_index <= 0){ + state.start_token_i = 0; + } + else{ + state.start_token_i = result.token_index-1; + } + + result = cpp_get_token(stack, end); + if (result.token_index < 0) result.token_index = 0; + else if (end > stack->tokens[result.token_index].start) ++result.token_index; + state.end_token_i = result.token_index; + + state.relex_start = stack->tokens[state.start_token_i].start; + if (start < state.relex_start) state.relex_start = start; + + state.space_request = state.end_token_i - state.start_token_i + tolerance + 1; + + return state; +} + +inline Cpp_Token +cpp__get_token(Cpp_Token_Stack *stack, Cpp_Token *tokens, int size, int index){ + Cpp_Token result; + if (index < stack->count){ + result = tokens[index]; + } + else{ + result.start = size; + result.size = 0; + result.type = CPP_TOKEN_EOF; + result.flags = 0; + result.state_flags = 0; + } + return result; +} + +FCPP_LINK bool +cpp_relex_nonalloc_main(Cpp_Relex_State *state, Cpp_Token_Stack *relex_stack, int *relex_end){ + Cpp_Token_Stack *stack = state->stack; + Cpp_Token *tokens = stack->tokens; + + cpp_shift_token_starts(stack, state->end_token_i, state->amount); + + Cpp_Lex_Data lex = {}; + lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags); + lex.pos = state->relex_start; + + int relex_end_i = state->end_token_i; + Cpp_Token match_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); + Cpp_Token end_token = match_token; + bool went_too_far = 0; + + for (;;){ + Cpp_Read_Result read = cpp_lex_step(state->file, &lex); + if (read.has_result){ + if (read.token.start == end_token.start && + read.token.size == end_token.size && + read.token.flags == end_token.flags && + read.token.state_flags == end_token.state_flags){ + break; + } + cpp_push_token_nonalloc(relex_stack, read.token); + + while (lex.pos > end_token.start && relex_end_i < stack->count){ + ++relex_end_i; + end_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); + } + if (relex_stack->count == relex_stack->max_count){ + went_too_far = 1; + break; + } + } + if (lex.pos >= state->file.size) break; + } + + if (!went_too_far){ + if (relex_stack->count > 0){ + if (state->start_token_i > 0){ + Cpp_Token_Merge merge = + cpp_attempt_token_merge(tokens[state->start_token_i - 1], + relex_stack->tokens[0]); + if (merge.did_merge){ + --state->start_token_i; + relex_stack->tokens[0] = merge.new_token; + } + } + + if (relex_end_i < state->stack->count){ + Cpp_Token_Merge merge = + cpp_attempt_token_merge(relex_stack->tokens[relex_stack->count-1], + tokens[relex_end_i]); + if (merge.did_merge){ + ++relex_end_i; + relex_stack->tokens[relex_stack->count-1] = merge.new_token; + } + } + } + + *relex_end = relex_end_i; + } + else{ + cpp_shift_token_starts(stack, state->end_token_i, -state->amount); + } + + return went_too_far; +} + +#ifndef FCPP_FORBID_MALLOC +FCPP_LINK Cpp_Token_Stack +cpp_make_token_stack(int starting_max){ + Cpp_Token_Stack token_stack; + token_stack.count = 0; + token_stack.max_count = starting_max; + token_stack.tokens = (Cpp_Token*)FCPP_GET_MEMORY(sizeof(Cpp_Token)*starting_max); + return token_stack; +} + +FCPP_LINK void +cpp_free_token_stack(Cpp_Token_Stack token_stack){ + FCPP_FREE_MEMORY(token_stack.tokens); +} + +FCPP_LINK void +cpp_resize_token_stack(Cpp_Token_Stack *token_stack, int new_max){ + Cpp_Token *new_tokens = (Cpp_Token*)FCPP_GET_MEMORY(sizeof(Cpp_Token)*new_max); + + if (new_tokens){ + FCPP_MEM_COPY(new_tokens, token_stack->tokens, sizeof(Cpp_Token)*token_stack->count); + FCPP_FREE_MEMORY(token_stack->tokens); + token_stack->tokens = new_tokens; + token_stack->max_count = new_max; + } +} + +FCPP_LINK void +cpp_push_token(Cpp_Token_Stack *token_stack, Cpp_Token token){ + if (!cpp_push_token_nonalloc(token_stack, token)){ + int new_max = 2*token_stack->max_count + 1; + cpp_resize_token_stack(token_stack, new_max); + bool result = cpp_push_token_nonalloc(token_stack, token); + _Assert(result); + } +} + +FCPP_LINK void +cpp_lex_file(Cpp_File file, Cpp_Token_Stack *token_stack_out){ + Cpp_Lex_Data lex = {}; + while (lex.pos < file.size){ + Cpp_Read_Result step_result = cpp_lex_step(file, &lex); + if (step_result.has_result){ + cpp_push_token(token_stack_out, step_result.token); + } + } +} + +FCPP_LINK bool +cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, + int start, int end, int amount, int tolerance){ +#if 0 + int start_token_i, end_token_i; + Cpp_Get_Token_Result get_result = cpp_get_token(token_stack, start_i); + start_token_i = get_result.token_index; + get_result = cpp_get_token(token_stack, end_i); + end_token_i = get_result.token_index; + if (end_token_i == -1){ + end_token_i = 0; + } + else if (end > token_stack->tokens[end_token_i].start){ + ++end_token_i; + } + cpp_shift_token_starts(token_stack, end_token_i, amount); + + int relex_start_i = start_token_i - 1; + if (relex_start_i < 0){ + relex_start_i = 0; + } + + int end_guess_i = end_token_i + 1; + if (end_guess_i > token_stack->count){ + --end_guess_i; + } +#endif + + int relex_start_i; + int end_token_i, end_guess_i; + { + Cpp_Get_Token_Result result = cpp_get_token(stack, start); + if (result.token_index <= 0){ + relex_start_i = 0; + } + else{ + relex_start_i = result.token_index-1; + } + + result = cpp_get_token(stack, end); + if (result.token_index < 0) result.token_index = 0; + else if (end > stack->tokens[result.token_index].start) ++result.token_index; + end_token_i = result.token_index; + end_guess_i = result.token_index+1; + } + + int relex_start = stack->tokens[relex_start_i].start; + if (start < relex_start) relex_start = start; + + cpp_shift_token_starts(stack, end_token_i, amount); + Cpp_Token_Stack relex_stack = cpp_make_token_stack((end_guess_i - relex_start_i + 1) * 3 / 2); + Cpp_Lex_Data lex = {}; + lex.pp_state = cpp_token_get_pp_state(stack->tokens[relex_start_i].state_flags); + lex.pos = relex_start; + bool went_too_far = 0; + + while (1){ + Cpp_Read_Result result = cpp_lex_step(file, &lex); + if (result.has_result){ + if (end_guess_i < stack->count && + result.token.start == stack->tokens[end_guess_i].start && + result.token.size == stack->tokens[end_guess_i].size && + result.token.flags == stack->tokens[end_guess_i].flags && + result.token.state_flags == stack->tokens[end_guess_i].state_flags){ + break; + } + else{ + cpp_push_token(&relex_stack, result.token); + while (lex.pos > stack->tokens[end_guess_i].start && + end_guess_i < stack->count){ + ++end_guess_i; + } + } + } + + if (lex.pos >= file.size){ + break; + } + + if (tolerance >= 0 && relex_stack.count + relex_start_i >= end_guess_i + tolerance){ + went_too_far = 1; + break; + } + } + + if (!went_too_far){ + int relex_end_i = end_guess_i; + + if (relex_stack.count > 0){ + if (relex_start_i > 0){ + Cpp_Token_Merge merge = cpp_attempt_token_merge(stack->tokens[relex_start_i - 1], + relex_stack.tokens[0]); + if (merge.did_merge){ + --relex_start_i; + relex_stack.tokens[0] = merge.new_token; + } + } + + if (relex_end_i < stack->count){ + Cpp_Token_Merge merge = cpp_attempt_token_merge(relex_stack.tokens[relex_stack.count - 1], + stack->tokens[relex_end_i]); + if (merge.did_merge){ + ++relex_end_i; + relex_stack.tokens[relex_stack.count - 1] = merge.new_token; + } + } + } + + int token_delete_amount = relex_end_i - relex_start_i; + int token_shift_amount = relex_stack.count - token_delete_amount; + + if (token_shift_amount != 0){ + int new_token_count = stack->count + token_shift_amount; + if (new_token_count > stack->max_count){ + int new_max = 2*stack->max_count + 1; + while (new_token_count > new_max){ + new_max = 2*new_max + 1; + } + cpp_resize_token_stack(stack, new_max); + } + + if (relex_end_i < stack->count){ + FCPP_MEM_MOVE(stack->tokens + relex_end_i + token_shift_amount, + stack->tokens + relex_end_i, sizeof(Cpp_Token)*(stack->count - relex_end_i)); + } + + stack->count += token_shift_amount; + } + + FCPP_MEM_COPY(stack->tokens + relex_start_i, relex_stack.tokens, sizeof(Cpp_Token)*relex_stack.count); + cpp_free_token_stack(relex_stack); + } + + else{ + cpp_shift_token_starts(stack, end_token_i, -amount); + cpp_free_token_stack(relex_stack); + } + + return went_too_far; +} +#endif + +#undef _Assert +#undef _TentativeAssert + +#undef FCPP_LEXER_IMPLEMENTATION +#endif // #ifdef FCPP_LEXER_IMPLEMENTATION + +// BOTTOM diff --git a/4cpp_preprocessor.cpp b/4cpp_preprocessor.cpp index b3e6adb8..ce825892 100644 --- a/4cpp_preprocessor.cpp +++ b/4cpp_preprocessor.cpp @@ -1,862 +1,19 @@ -/* - * Mr. 4th Dimention - Allen Webster - * - * 30.11.2015 - * - * CPP preprocessor - * - */ - -// TOP - -#define byte unsigned char - -struct Cpp_PP_Step{ - int error_code; - int finished; - int early_out; - - int mem_size; - int mem_type; - - int emit; - int file_index; - int token_index; -}; - -enum Cpp_PP_Expansion_Type{ - PPExp_Normal, - // never below this - PPExp_Count -}; -struct Cpp_PP_Expansion_Header{ - int prev_frame; - int type; -}; -struct Cpp_PP_Expansion_Normal{ - int file_offset; - int current_token; - int end_token; - int file_level; -}; - -struct Cpp_PP_State{ - byte *base; - int top, frame, max; - - int substep; - - struct{ - union{ - int name_index; - int param_count; - int macro_offset; - } define; - } vars; - - Debug(int did_advance); -}; - -internal int -cpp__push_expansion_normal(Cpp_PP_State *state, int file, int start, int end, int file_level){ - Cpp_PP_Expansion_Header *header; - Cpp_PP_Expansion_Normal *expansion; - int added_size; - int prev_frame; - int result; - - added_size = sizeof(Cpp_PP_Expansion_Header) + sizeof(Cpp_PP_Expansion_Normal); - - if (state->top + added_size <= state->max){ - result = 1; - - prev_frame = state->frame; - state->frame = state->top; - state->top += added_size; - - header = (Cpp_PP_Expansion_Header*)(state->base + state->frame); - expansion = (Cpp_PP_Expansion_Normal*)(header + 1); - - header->prev_frame = prev_frame; - header->type = PPExp_Normal; - expansion->file_offset = file; - expansion->current_token = start; - expansion->end_token = end; - expansion->file_level = file_level; - } - else{ - result = 0; - } - - return(result); -} - -internal int -cpp__pop_expansion(Cpp_PP_State *state){ - Cpp_PP_Expansion_Header *header; - int result; - - if (state->top >= sizeof(Cpp_PP_Expansion_Header)){ - Assert(state->top - state->frame >= sizeof(Cpp_PP_Expansion_Header)); - result = 1; - - header = (Cpp_PP_Expansion_Header*)(state->base + state->frame); - state->top = state->frame; - state->frame = header->prev_frame; - } - else{ - result = 0; - } - - return(result); -} - -enum Cpp_PP_Memory_Type{ - PPMem_None, - PPMem_Definition_Items, - PPMem_Definition_Table, - PPMem_Spare_File_String, - PPMem_Spare_File_Tokens, - // never below this - PPMem_Count -}; - -internal void -cpp__pp_memory_request(Cpp_PP_Step *step, int amount, int type){ - Assert(!step->early_out); - step->early_out = 1; - step->mem_size = amount; - step->mem_type = type; -} - -enum Cpp_PP_Table_Entry_Type{ - PPItem_Unset, - PPItem_File, - PPItem_Macro, - // never below this - PPItem_Count -}; -struct Cpp_PP_Table_Entry{ - int name_start, name_size; - int type; - unsigned int hash; - int item_offset; -}; -struct Cpp_PP_Table{ - Cpp_PP_Table_Entry *entries; - unsigned int count, max; -}; - -struct Cpp_PP_File{ - Cpp_File file; - Cpp_Token_Stack tokens; -}; -struct Cpp_PP_Macro{ - int self_start; - int name_start; - int name_size; - int param_count; - int variadic; - int body_start; - int body_end; -}; - -struct Cpp_PP_Definitions{ - byte *items; - int pos, restore_pos, max; - Cpp_PP_File *spare_file; - int spare_pos; - - Cpp_PP_Table table; -}; - -internal unsigned int -cpp__hash(char *name, int size, int type){ - unsigned int x; - int i; - - x = 5381; - x = ((x << 5) + x) ^ ('A' + type); - for (i = 0; i < size; ++i, ++name){ - x = ((x << 5) + x) ^ *name; - } - - return(x); -} - -internal unsigned int -cpp__hash(Cpp_PP_Definitions *definitions, int name_start, int name_size, int type){ - unsigned int x; - char *s; - s = definitions->spare_file->file.data + name_start; - x = cpp__hash(s, name_size, type); - return(x); -} - -internal int -cpp__table_can_add(Cpp_PP_Table *table){ - int result; - result = 0; - if (table->count*8 < table->max*7) result = 1; - return(result); -} - -internal void -cpp__table_add_direct(Cpp_PP_Table *table, Cpp_PP_Table_Entry entry){ - unsigned int i, j; - i = entry.hash % table->max; - j = i - 1; - if (i <= 1) j += table->max; - - for (; i != j; ++i){ - if (i == table->max) i = 0; - if (table->entries[i].type == PPItem_Unset){ - table->entries[i] = entry; - break; - } - } - - Assert(i != j); -} - -internal void -cpp__table_add(Cpp_PP_Definitions *definitions, Cpp_PP_Table *table, - int name_start, int name_size, int type, int offset){ - Cpp_PP_Table_Entry entry; - unsigned int hash; - - Assert(cpp__table_can_add(table)); - - hash = cpp__hash(definitions, name_start, name_size, type); - entry.name_start = name_start; - entry.name_size = name_size; - entry.type = type; - entry.hash = hash; - entry.item_offset = offset; - - cpp__table_add_direct(table, entry); -} - -internal int -cpp__str_match(char *a, char *b, int size){ - char *e; - int result; - result = 0; - for(e = a + size; a < e && *a == *b; ++a, ++b); - if (a == e) result = 1; - return(result); -} - -internal int -cpp__table_find_hash_pos(Cpp_PP_Definitions *definitions, Cpp_PP_Table *table, - char *name, int name_size, int typ e, unsigned int *hash_index){ - Cpp_PP_Table_Entry *entry; - int result; - unsigned int hash; - unsigned int i, j; - - result = 0; - hash = cpp__hash(name, name_size, type); - i = hash % table->max; - j = i - 1; - if (i <= 1) j += table->max; - - for (; i != j; ++i){ - if (i == table->max) i = 0; - entry = table->entries + i; - if (entry->type == PPItem_Unset) break; - if (entry->hash == hash && entry->type == type && entry->name_size == name_size && - cpp__str_match(name, definitions->spare_file->file.data + entry->name_start, name_size)){ - result = 1; - *hash_index = i; - break; - } - } - - return(result); -} - -internal int -cpp__table_find(Cpp_PP_Definitions *definitions, Cpp_PP_Table *table, - char *name, int name_size, int type, int *offset){ - int result; - unsigned int hash_index; - - result = 0; - if (cpp__table_find_hash_pos(definitions, table, name, name_size, type, &hash_index)){ - result = 1; - *offset = table->entries[hash_index].item_offset; - } - - return(result); -} - -internal void -cpp__table_cpy(Cpp_PP_Table *dest, Cpp_PP_Table *src){ - Cpp_PP_Table_Entry *entry; - unsigned int i; - - Assert(dest->max >= src->max); - for (i = 0, entry = src->entries; i < src->max; ++i, ++entry){ - if (entry->type != PPItem_Unset){ - cpp__table_add_direct(dest, *entry); - } - } -} - -internal void -cpp__def_begin_uncertain(Cpp_PP_Definitions *definitions){ - definitions->restore_pos = definitions->pos; -} -internal void -cpp__def_success_uncertain(Cpp_PP_Definitions *definitions){ - definitions->restore_pos = 0; -} -internal void -cpp__def_fail_uncertain(Cpp_PP_Definitions *definitions){ - if (definitions->restore_pos != 0){ - definitions->pos = definitions->restore_pos; - } -} - -internal int -cpp__def_can_save_strings(Cpp_PP_Definitions *definitions, int total_size){ - int result; - Cpp_PP_File *file; - file = definitions->spare_file; - - if (definitions->spare_pos + total_size <= file->file.size) result = 1; - else result = 0; - - return(result); -} - -internal int -cpp__def_can_save_item(Cpp_PP_Definitions *definitions, int size){ - int result; - - if (definitions->pos + size <= definitions->max) result = 1; - else result = 0; - - return(result); -} - -internal int -cpp__def_save_string(Cpp_PP_Step *step, Cpp_PP_Definitions *definitions, - char *str, int size){ - int result; - Cpp_PP_File *file; - file = definitions->spare_file; - - Assert(definitions->spare_pos + size <= file->file.size); - memcpy(file->file.data + definitions->spare_pos, str, size); - result = definitions->spare_pos; - definitions->spare_pos += size; - - return(result); -} - -internal int -cpp__mem_up(int x){ - int xx; - xx = x / 1024; - if (xx*1024 < x){ - xx += 1; - } - xx = xx << 10; - return(xx); -} - -internal int -cpp__def_begin_macro(Cpp_PP_Step *step, Cpp_PP_Definitions *definitions, - char *str, int size){ - int result, str_pos; - Cpp_PP_Macro *macro_item; - - result = 0; - if (cpp__def_can_save_strings(definitions, size)){ - if (cpp__def_can_save_item(definitions, sizeof(Cpp_PP_Macro))){ - str_pos = cpp__def_save_string(step, definitions, str, size); - - Assert(definitions->pos + sizeof(Cpp_PP_Macro) <= definitions->max); - result = definitions->pos; - definitions->pos += sizeof(Cpp_PP_Macro); - - macro_item = (Cpp_PP_Macro*)(definitions->items + result); - *macro_item = {}; - macro_item->self_start = result; - macro_item->name_start = str_pos; - macro_item->name_size = size; - } - else{ - cpp__pp_memory_request(step, cpp__mem_up(definitions->max + 1), PPMem_Definition_Items); - } - } - else{ - cpp__pp_memory_request(step, cpp__mem_up(definitions->spare_file->file.size + size), - PPMem_Spare_File_String); - } - - return(result); -} - -internal void -cpp__def_set_macro_params(Cpp_PP_Definitions *definitions, int macro_offset, int param_count){ - Cpp_PP_Macro *macro_item; - macro_item = (Cpp_PP_Macro*)(definitions->items + macro_offset); - macro_item->param_count = param_count; -} - -internal void -cpp__def_set_macro_variadic(Cpp_PP_Definitions *definitions, int macro_offset, int variadic){ - Cpp_PP_Macro *macro_item; - macro_item = (Cpp_PP_Macro*)(definitions->items + macro_offset); - macro_item->variadic = variadic; -} - -internal void -cpp__def_set_macro_body_start(Cpp_PP_Definitions *definitions, int macro_offset, int body_start){ - Cpp_PP_Macro *macro_item; - macro_item = (Cpp_PP_Macro*)(definitions->items + macro_offset); - macro_item->body_start = body_start; -} - -internal void -cpp__def_set_macro_body_end(Cpp_PP_Definitions *definitions, int macro_offset, int body_end){ - Cpp_PP_Macro *macro_item; - macro_item = (Cpp_PP_Macro*)(definitions->items + macro_offset); - macro_item->body_end = body_end; -} - -internal void -cpp__def_end_macro(Cpp_PP_Step *step, Cpp_PP_Definitions *definitions){ - if (cpp__table_can_add(&definitions->table)){ - Cpp_PP_Macro *macro_item; - macro_item = (Cpp_PP_Macro*)(definitions->items + definitions->restore_pos); - - cpp__table_add(definitions, &definitions->table, - macro_item->name_start, macro_item->name_size, - PPItem_Macro, macro_item->self_start); - } - else{ - cpp__pp_memory_request(step, - cpp__mem_up(definitions->table.max*sizeof(Cpp_PP_Table_Entry)+1), - PPMem_Definition_Table); - } -} - -internal Cpp_PP_File* -cpp_preproc_get_pp_file(Cpp_PP_Definitions *definitions, int file_offset){ - Cpp_PP_File *result; - result = (Cpp_PP_File*)(definitions->items+file_offset); - return(result); -} - -internal Cpp_File* -cpp_preproc_get_file(Cpp_PP_Definitions *definitions, int file_offset){ - Cpp_PP_File *pp_file; - Cpp_File *result; - pp_file = cpp_preproc_get_pp_file(definitions, file_offset); - result = &pp_file->file; - return(result); -} - -internal Cpp_Token* -cpp_preproc_get_token(Cpp_PP_Definitions *definitions, int file_offset, int token_index){ - Cpp_PP_File *pp_file; - Cpp_Token *result; - pp_file = cpp_preproc_get_pp_file(definitions, file_offset); - result = &pp_file->tokens.tokens[token_index]; - return(result); -} - -internal int -cpp__define_file(Cpp_PP_Definitions *definitions, Cpp_File file, Cpp_Token_Stack tokens){ - int result; - Cpp_PP_File *file_item; - - Assert(definitions->pos + sizeof(Cpp_PP_File) <= definitions->max); - result = definitions->pos; - definitions->pos += sizeof(Cpp_PP_File); - - file_item = (Cpp_PP_File*)(definitions->items + result); - file_item->file = file; - file_item->tokens = tokens; - - return(result); -} - -internal void -cpp_preproc_set_spare_space(Cpp_PP_Definitions *definitions, - int str_size, void *str_mem, - int token_size, void *token_mem){ - Cpp_PP_File pp_file; - pp_file.file.size = str_size; - pp_file.file.data = (char*)str_mem; - pp_file.tokens.max_count = token_size; - pp_file.tokens.count = 0; - pp_file.tokens.tokens = (Cpp_Token*)token_mem; - - int pos = cpp__define_file(definitions, pp_file.file, pp_file.tokens); - Cpp_PP_File *spare_file = cpp_preproc_get_pp_file(definitions, pos); - definitions->spare_file = spare_file; - definitions->spare_pos = 0; -} - -internal void -cpp_preproc_target(Cpp_PP_State *state, Cpp_PP_Definitions *definitions, - Cpp_File file, Cpp_Token_Stack tokens){ - int file_index; - - Assert(state->top == 0); - - file_index = cpp__define_file(definitions, file, tokens); - cpp__push_expansion_normal(state, file_index, 0, tokens.count, 1); -} - -internal void* -cpp_preproc_provide_memory(Cpp_PP_State *state, Cpp_PP_Definitions *definitions, - int type, int size, void *memory){ - Cpp_PP_Table new_table; - Cpp_PP_File *spare; - void *result; - result = 0; - - - - switch (type){ - case PPMem_None: Assert(0); break; - - case PPMem_Definition_Items: - Assert(size > definitions->max); - memcpy(memory, definitions->items, definitions->pos); - result = definitions->items; - definitions->items = (byte*)memory; - break; - - case PPMem_Definition_Table: - Assert(size > definitions->table.max * sizeof(*new_table.entries)); - new_table.entries = (Cpp_PP_Table_Entry*)memory; - new_table.max = size/sizeof(*new_table.entries); - new_table.count = 0; - cpp__table_cpy(&new_table, &definitions->table); - result = definitions->table.entries; - definitions->table = new_table; - break; - - case PPMem_Spare_File_String: - spare = definitions->spare_file; - Assert(size > spare->file.size); - memcpy(memory, spare->file.data, definitions->spare_pos); - result = spare->file.data; - spare->file.data = (char*)memory; - break; - - case PPMem_Spare_File_Tokens: - spare = definitions->spare_file; - Assert(size > spare->tokens.count*sizeof(Cpp_Token)); - memcpy(memory, spare->tokens.tokens, spare->tokens.count*sizeof(Cpp_Token)); - result = spare->tokens.tokens; - spare->tokens.tokens = (Cpp_Token*)memory; - break; - - default: Assert(0); break; - } - - return(result); -} - -enum Cpp_PP_Error{ - PPErr_None, - PPErr_Define_Identifier_Missing, - PPErr_Parameter_Identifier_Missing, - PPErr_Preprocessor_Directive_In_Expansion, - PPErr_Expected_Comma_Or_Parenthese, - PPErr_Unfinished_Parameters, - // never below this - PPErr_Count -}; - -internal char* -cpp_preproc_error_str(int error_code){ - char *result = 0; - - switch (error_code){ - case PPErr_None: - result = "no error"; break; - - case PPErr_Define_Identifier_Missing: - result = "define identifier missing"; break; - - case PPErr_Parameter_Identifier_Missing: - result = "parameter identifier missing"; break; - - case PPErr_Preprocessor_Directive_In_Expansion: - result = "preprocessor directive in expansion"; break; - - case PPErr_Expected_Comma_Or_Parenthese: - result = "expected comma or parenthese"; break; - - case PPErr_Unfinished_Parameters: - result = "unfinished parameter list"; break; - - default: - result = "unknown error code"; - } - - return(result); -} - -internal int -cpp_preproc_recommend_termination(int error_code){ - int result = 0; - - return(result); -} - -struct Cpp__PP_Visit{ - int file_offset; - int token_index; -}; - -internal void -cpp__pp_next_token(Cpp_PP_State *state, Cpp_PP_Expansion_Normal *expansion){ - ++expansion->current_token; - Debug(state->did_advance = 1); -} - -enum Cpp_PP_State_Step{ - PPState_None, - PPState_Define, - PPState_Name, - PPState_Define_Body, - PPState_Define_Parameter, - PPState_Define_Parameter_Comma, - PPState_Error_Recovery, - // never below this - PPState_Count -}; - -internal int -cpp__pp_require_identifier(Cpp_PP_State *state, Cpp_Token token){ - int result; - result = PPErr_None; - if ((token.flags & CPP_TFLAG_PP_BODY) == 0){ - result = PPErr_Define_Identifier_Missing; - state->substep = PPState_None; - Debug(state->did_advance = 1); - } - else if (token.type != CPP_TOKEN_IDENTIFIER && - (token.flags & CPP_TFLAG_IS_KEYWORD) == 0){ - result = PPErr_Define_Identifier_Missing; - state->substep = PPState_Error_Recovery; - Debug(state->did_advance = 1); - } - return(result); -} - -internal Cpp_PP_Step -cpp__pp_step_normal(Cpp_PP_State *state, Cpp_PP_Definitions *definitions, - Cpp_PP_Expansion_Normal *expansion){ - Cpp__PP_Visit visit; - Cpp_PP_Step result; - Cpp_Token token; - Cpp_PP_File pp_file; - int void_step; - - visit = {}; - result = {}; - void_step = 0; - - if (expansion->current_token < expansion->end_token){ - visit.file_offset = expansion->file_offset; - visit.token_index = expansion->current_token; - } - else{ - void_step = 1; - cpp__pop_expansion(state); - } - - if (!void_step){ - pp_file = *cpp_preproc_get_pp_file(definitions, visit.file_offset); - Assert(visit.token_index >= 0 && visit.token_index < pp_file.tokens.count); - token = pp_file.tokens.tokens[visit.token_index]; - - - - Debug(state->did_advance = 0); - if (token.type == CPP_TOKEN_COMMENT || token.type == CPP_TOKEN_JUNK){ - cpp__pp_next_token(state, expansion); - } - else{ - switch (state->substep){ - case PPState_None: - { - if (expansion->file_level == 0 && (token.flags & CPP_TFLAG_PP_DIRECTIVE)){ - result.error_code = PPErr_Preprocessor_Directive_In_Expansion; - cpp__pp_next_token(state, expansion); - } - else{ - switch (token.type){ - case CPP_PP_DEFINE: - { - state->substep = PPState_Define; - state->vars = {}; - cpp__pp_next_token(state, expansion); - }break; - - default: - { - result.emit = 1; - result.file_index = visit.file_offset; - result.token_index = visit.token_index; - cpp__pp_next_token(state, expansion); - }break; - - } - } - }break; - - case PPState_Define: - { - result.error_code = cpp__pp_require_identifier(state, token); - if (result.error_code == PPErr_None){ - - cpp__def_begin_uncertain(definitions); - state->vars.define.macro_offset = - cpp__def_begin_macro(&result, definitions, - pp_file.file.data + token.start, - token.size); - if (result.early_out) goto cpp__pp_step_early_out; - - state->substep = PPState_Name; - state->vars.define.name_index = visit.token_index; - cpp__pp_next_token(state, expansion); - } - }break; - - case PPState_Name: - { - if ((token.flags & CPP_TFLAG_PP_BODY) == 0){ - cpp__def_end_macro(&result, definitions); - if (result.early_out) goto cpp__pp_step_early_out; - cpp__def_success_uncertain(definitions); - state->substep = PPState_None; - Debug(state->did_advance = 1); - } - else if (token.type == CPP_TOKEN_PARENTHESE_OPEN){ - state->substep = PPState_Define_Parameter; - cpp__pp_next_token(state, expansion); - } - else{ - state->substep = PPState_Define_Body; - } - }break; - - case PPState_Define_Parameter: - { - if (token.type == CPP_TOKEN_PARENTHESE_CLOSE && - (token.flags & CPP_TFLAG_PP_BODY) && - state->vars.define.param_count == 0){ - state->substep = PPState_Define_Body; - cpp__pp_next_token(state, expansion); - } - else{ - result.error_code = cpp__pp_require_identifier(state, token); - if (result.error_code == PPErr_None){ - state->substep = PPState_Define_Parameter_Comma; - cpp__pp_next_token(state, expansion); - ++state->vars.define.param_count; - } - else{ - result.error_code = PPErr_Parameter_Identifier_Missing; - // NOTE(allen): Here I do not do the work I normally do when - // setting an error, because that work was done earlier, this is - // just translating the error code. - } - } - }break; - - case PPState_Define_Parameter_Comma: - { - if ((token.flags & CPP_TFLAG_PP_BODY) == 0){ - result.error_code = PPErr_Unfinished_Parameters; - state->substep = PPState_None; - Debug(state->did_advance = 1); - } - else{ - if (token.type == CPP_TOKEN_COMMA){ - state->substep = PPState_Define_Parameter; - cpp__pp_next_token(state, expansion); - } - else if (token.type == CPP_TOKEN_PARENTHESE_CLOSE){ - state->substep = PPState_Define_Body; - cpp__pp_next_token(state, expansion); - } - else{ - state->substep = PPState_Error_Recovery; - result.error_code = PPErr_Expected_Comma_Or_Parenthese; - Debug(state->did_advance = 1); - } - } - }break; - - case PPState_Define_Body: - { - if ((token.flags & CPP_TFLAG_PP_BODY) == 0){ - // TODO(allen): define macro - state->substep = PPState_None; - Debug(state->did_advance = 1); - } - else{ - cpp__pp_next_token(state, expansion); - } - }break; - - case PPState_Error_Recovery: - { - if ((token.flags & CPP_TFLAG_PP_BODY) == 0){ - state->substep = PPState_None; - Debug(state->did_advance = 1); - } - else{ - cpp__pp_next_token(state, expansion); - } - }break; - - } - } - - Assert(state->did_advance == 1); - } - -cpp__pp_step_early_out: - return(result); -} - -internal Cpp_PP_Step -cpp_preproc_step_nonalloc(Cpp_PP_State *state, Cpp_PP_Definitions *definitions){ - Cpp_PP_Step result; - Cpp_PP_Expansion_Header *header; - - Assert(state->top >= sizeof(Cpp_PP_Expansion_Header)); - - result = {}; - header = (Cpp_PP_Expansion_Header*)(state->base + state->frame); - switch (header->type){ - case PPExp_Normal: - result = cpp__pp_step_normal(state, definitions, - (Cpp_PP_Expansion_Normal*)(header + 1)); - break; - default: Assert(0); - } - - if (state->top == 0) result.finished = 1; - - return(result); -} - -#undef byte - -// BOTTOM - +/* + * Mr. 4th Dimention - Allen Webster + * + * 30.11.2015 + * + * CPP preprocessor + * + */ + +// TOP + +#define byte unsigned char + + + +#undef byte + +// BOTTOM + diff --git a/4ed.cpp b/4ed.cpp index 83a5f504..dd8d89d8 100644 --- a/4ed.cpp +++ b/4ed.cpp @@ -236,7 +236,7 @@ COMMAND_DECL(write_character){ i32 pos; pos = view->cursor.pos; i32 next_cursor_pos = view->cursor.pos + string.size; - view_replace_range(system, mem, view, layout, pos, pos, (u8*)string.str, string.size, next_cursor_pos); + view_replace_range(system, mem, view, layout, pos, pos, string.str, string.size, next_cursor_pos); view_cursor_move(view, next_cursor_pos); file->state.cursor_pos = view->cursor.pos; } @@ -421,7 +421,7 @@ COMMAND_DECL(search){ view_set_widget(view, FWIDG_SEARCH); view->isearch.str = vars->mini_str; view->isearch.reverse = 0; - view->isearch.pos = view->cursor.pos; + view->isearch.pos = view->cursor.pos - 1; } COMMAND_DECL(rsearch){ @@ -433,7 +433,80 @@ COMMAND_DECL(rsearch){ view_set_widget(view, FWIDG_SEARCH); view->isearch.str = vars->mini_str; view->isearch.reverse = 1; - view->isearch.pos = view->cursor.pos; + view->isearch.pos = view->cursor.pos + 1; +} + +COMMAND_DECL(word_complete){ + ProfileMomentFunction(); + REQ_FILE_VIEW(view); + REQ_FILE(file, view); + USE_LAYOUT(layout); + USE_MEM(mem); + + Partition *part = &mem->part; + + Temp_Memory temp1, temp2; + + Buffer_Type *buffer; + Buffer_Backify_Type loop; + char *data; + i32 end; + + i32 cursor_pos, word_start, word_end; + char c; + + char *str, *spare; + i32 size; + + i32 match_start, match_end, match_size; + + buffer = &file->state.buffer; + word_end = view->cursor.pos; + word_start = word_end; + cursor_pos = word_end - 1; + + // TODO(allen): macros for these buffer loops and some method of breaking out of them. + for (loop = buffer_backify_loop(buffer, cursor_pos, 0); + buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; cursor_pos >= end; --cursor_pos){ + c = data[cursor_pos]; + if (char_is_alpha(c)){ + word_start = cursor_pos; + } + else if (!char_is_numeric(c)){ + goto double_break; + } + } + } + // TODO(allen): figure out how labels are scoped. + double_break:; + + size = word_end - word_start; + + if (size > 0){ + temp1 = begin_temp_memory(part); + + str = (char*)push_array(part, char, size); + buffer_stringify(buffer, word_start, word_end, str); + + temp2 = begin_temp_memory(part); + spare = (char*)push_array(part, char, size); + end_temp_memory(temp2); + + // TODO(allen): find string needs explicit end position + match_start = buffer_find_string(buffer, word_end, str, size, spare); + match_end = buffer_seek_word_right_assume_on_word(buffer, match_start); + match_size = match_end - match_start; + spare = (char*)push_array(part, char, match_size); + buffer_stringify(buffer, match_start, match_end, spare); + + view_replace_range(system, mem, view, layout, word_start, word_end, spare, match_size, word_end); + + end_temp_memory(temp1); + } } COMMAND_DECL(goto_line){ @@ -497,24 +570,29 @@ COMMAND_DECL(paste){ USE_LAYOUT(layout); USE_MEM(mem); + Panel *current_panel; + String *src; + File_View *current_view; + i32 pos_left, next_cursor_pos; + i32 panel_count; + i32 i; + if (working_set->clipboard_size > 0){ view->next_mode.rewrite = 1; - String *src = working_set_clipboard_head(working_set); - i32 pos_left = view->cursor.pos; + src = working_set_clipboard_head(working_set); + pos_left = view->cursor.pos; - i32 next_cursor_pos = pos_left+src->size; - view_replace_range(system, mem, view, layout, pos_left, pos_left, (u8*)src->str, src->size, next_cursor_pos); + next_cursor_pos = pos_left+src->size; + view_replace_range(system, mem, view, layout, pos_left, pos_left, src->str, src->size, next_cursor_pos); view_cursor_move(view, next_cursor_pos); view->mark = pos_left; - Editing_Layout *layout = command->layout; - Panel *panels = layout->panels; - i32 panel_count = layout->panel_count; - for (i32 i = 0; i < panel_count; ++i){ - Panel *current_panel = panels + i; - File_View *current_view = view_to_file_view(current_panel->view); + current_panel = layout->panels; + panel_count = layout->panel_count; + for (i = 0; i < panel_count; ++i, ++current_panel){ + current_view = view_to_file_view(current_panel->view); if (current_view && current_view->file == file){ view_post_paste_effect(current_view, 20, pos_left, src->size, @@ -540,7 +618,7 @@ COMMAND_DECL(paste_next){ i32 next_cursor_pos = range.start+src->size; view_replace_range(system, mem, view, layout, range.start, range.end, - (u8*)src->str, src->size, next_cursor_pos); + src->str, src->size, next_cursor_pos); view_cursor_move(view, next_cursor_pos); view->mark = range.start; @@ -2081,10 +2159,12 @@ setup_file_commands(Command_Map *commands, Partition *part, Key_Codes *codes, Co map_add(commands, 'l', MDFR_CTRL, command_toggle_line_wrap); map_add(commands, '?', MDFR_CTRL, command_toggle_show_whitespace); map_add(commands, '|', MDFR_CTRL, command_toggle_tokens); - map_add(commands, 'u', MDFR_CTRL, command_to_uppercase); - map_add(commands, 'j', MDFR_CTRL, command_to_lowercase); + map_add(commands, 'U', MDFR_CTRL, command_to_uppercase); + map_add(commands, 'u', MDFR_CTRL, command_to_lowercase); map_add(commands, '~', MDFR_CTRL, command_clean_all_lines); map_add(commands, 'f', MDFR_CTRL, command_search); + map_add(commands, 'j', MDFR_CTRL, command_word_complete); + map_add(commands, 'r', MDFR_CTRL, command_rsearch); map_add(commands, 'g', MDFR_CTRL, command_goto_line); @@ -2616,12 +2696,12 @@ App_Read_Command_Line_Sig(app_read_command_line){ vars = app_setup_memory(memory); if (clparams.argc > 1){ init_command_line_settings(&vars->settings, plat_settings, clparams); - *files = vars->settings.init_files; - *file_count = &vars->settings.init_files_count; } else{ vars->settings = {}; } + *files = vars->settings.init_files; + *file_count = &vars->settings.init_files_count; } return(out_size); diff --git a/4ed_color_view.cpp b/4ed_color_view.cpp index a6385e54..391d8916 100644 --- a/4ed_color_view.cpp +++ b/4ed_color_view.cpp @@ -896,7 +896,7 @@ do_font_option(Color_UI *ui, i16 font_id){ b32 result = 0; Font_Info *info = get_font_info(ui->state.font_set, font_id); - i32 sub_id = (i32)(info); + i32 sub_id = (i32)(i64)(info); i32_Rect orect = layout_rect(&ui->layout, info->height); Widget_ID wid = make_sub0(&ui->state, sub_id); diff --git a/4ed_file_view.cpp b/4ed_file_view.cpp index 6ff91341..d28bd8ea 100644 --- a/4ed_file_view.cpp +++ b/4ed_file_view.cpp @@ -2967,7 +2967,7 @@ view_do_white_batch_edit(System_Functions *system, Mem_Options *mem, File_View * inline void view_replace_range(System_Functions *system, Mem_Options *mem, File_View *view, Editing_Layout *layout, - i32 start, i32 end, u8 *str, i32 len, i32 next_cursor){ + i32 start, i32 end, char *str, i32 len, i32 next_cursor){ if (view->locked) return; Edit_Spec spec = {}; spec.step.type = ED_NORMAL; @@ -2976,7 +2976,7 @@ view_replace_range(System_Functions *system, spec.step.edit.len = len; spec.step.pre_pos = view->cursor.pos; spec.step.post_pos = next_cursor; - spec.str = str; + spec.str = (u8*)str; file_do_single_edit(system, mem, view->file, layout, spec, hist_normal); } diff --git a/buffer/4coder_buffer_abstract.cpp b/buffer/4coder_buffer_abstract.cpp index 41cf2c39..2c66f8ce 100644 --- a/buffer/4coder_buffer_abstract.cpp +++ b/buffer/4coder_buffer_abstract.cpp @@ -1,1315 +1,1338 @@ -/* - * Mr. 4th Dimention - Allen Webster - * Four Tech - * - * public domain -- no warranty is offered or implied; use this code at your own risk - * - * 24.10.2015 - * - * Buffer data object - * type - Golden Array - * - */ - -// TOP - -#define Buffer_Init_Type cat_4tech(Buffer_Type, _Init) -#define Buffer_Stringify_Type cat_4tech(Buffer_Type, _Stringify_Loop) -#define Buffer_Backify_Type cat_4tech(Buffer_Type, _Backify_Loop) - -inline_4tech void -buffer_stringify(Buffer_Type *buffer, int start, int end, char *out){ - for (Buffer_Stringify_Type loop = buffer_stringify_loop(buffer, start, end); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - memcpy_4tech(out, loop.data, loop.size); - out += loop.size; - } -} - -inline_4tech void -buffer_backify(Buffer_Type *buffer, int start, int end, char *out){ - for (Buffer_Backify_Type loop = buffer_backify_loop(buffer, end, start); - buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - memcpy_4tech(out, loop.data, loop.size); - out += loop.size; - } -} - -internal_4tech int -buffer_convert_out(Buffer_Type *buffer, char *dest, int max){ - Buffer_Stringify_Type loop; - int size, out_size, pos, result; - - size = buffer_size(buffer); - assert_4tech(size + buffer->line_count < max); - - pos = 0; - for (loop = buffer_stringify_loop(buffer, 0, size); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - result = eol_convert_out(dest + pos, max - pos, loop.data, loop.size, &out_size); - assert_4tech(result); - pos += out_size; - } - - return(pos); -} - -internal_4tech int -buffer_count_newlines(Buffer_Type *buffer, int start, int end){ - Buffer_Stringify_Type loop; - int i; - int count; - - assert_4tech(0 <= start); - assert_4tech(start <= end); - assert_4tech(end <= buffer_size(buffer)); - - count = 0; - - for (loop = buffer_stringify_loop(buffer, start, end); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - for (i = 0; i < loop.size; ++i){ - count += (loop.data[i] == '\n'); - } - } - - return(count); -} - -internal_4tech int -buffer_seek_whitespace_down(Buffer_Type *buffer, int pos){ - Buffer_Stringify_Type loop; - char *data; - int end; - int size; - int no_hard; - int prev_endline; - - size = buffer_size(buffer); - loop = buffer_stringify_loop(buffer, pos, size); - - for (;buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (;pos < end; ++pos){ - if (!is_whitespace(data[pos])) goto buffer_seek_whitespace_down_mid; - } - } - -buffer_seek_whitespace_down_mid: - no_hard = 0; - prev_endline = -1; - for (;buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos < end; ++pos){ - if (data[pos] == '\n'){ - if (no_hard) goto buffer_seek_whitespace_down_end; - else{ - no_hard = 1; - prev_endline = pos; - } - } - else if (!is_whitespace(data[pos])){ - no_hard = 0; - } - } - } - -buffer_seek_whitespace_down_end: - if (prev_endline == -1 || prev_endline+1 >= size) pos = size; - else pos = prev_endline+1; - - return pos; -} - -internal_4tech int -buffer_seek_whitespace_up(Buffer_Type *buffer, int pos){ - Buffer_Backify_Type loop; - char *data; - int end; - int size; - int no_hard; - - size = buffer_size(buffer); - loop = buffer_backify_loop(buffer, pos-1, 1); - - for (;buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - end = loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (;pos >= end; --pos){ - if (!is_whitespace(data[pos])) goto buffer_seek_whitespace_up_mid; - } - } - -buffer_seek_whitespace_up_mid: - no_hard = 0; - for (;buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - end = loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos >= end; --pos){ - if (data[pos] == '\n'){ - if (no_hard) goto buffer_seek_whitespace_up_end; - else no_hard = 1; - } - else if (!is_whitespace(data[pos])){ - no_hard = 0; - } - } - } - -buffer_seek_whitespace_up_end: - if (pos != 0) ++pos; - - return pos; -} - -internal_4tech int -buffer_seek_whitespace_right(Buffer_Type *buffer, int pos){ - Buffer_Stringify_Type loop; - char *data; - int end; - int size; - - size = buffer_size(buffer); - loop = buffer_stringify_loop(buffer, pos, size); - - for (;buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos < end && is_whitespace(data[pos]); ++pos); - if (!is_whitespace(data[pos])) break; - } - - for (;buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos < end && !is_whitespace(data[pos]); ++pos); - if (is_whitespace(data[pos])) break; - } - - return(pos); -} - -internal_4tech int -buffer_seek_whitespace_left(Buffer_Type *buffer, int pos){ - Buffer_Backify_Type loop; - char *data; - int end; - int size; - - --pos; - if (pos > 0){ - size = buffer_size(buffer); - loop = buffer_backify_loop(buffer, pos, 1); - - for (;buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - end = loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos >= end && is_whitespace(data[pos]); --pos); - if (!is_whitespace(data[pos])) break; - } - - for (;buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - end = loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos >= end && !is_whitespace(data[pos]); --pos); - if (is_whitespace(data[pos])) break; - } - - if (pos != 0) ++pos; - } - else{ - pos = 0; - } - - return(pos); -} - -internal_4tech int -buffer_seek_alphanumeric_right(Buffer_Type *buffer, int pos){ - Buffer_Stringify_Type loop; - char *data; - int end; - int size; - - size = buffer_size(buffer); - loop = buffer_stringify_loop(buffer, pos, size); - - for (;buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos < end; ++pos){ - if (is_alphanumeric_true(data[pos])) goto buffer_seek_alphanumeric_right_mid; - } - } - -buffer_seek_alphanumeric_right_mid: - for (;buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos < end; ++pos){ - if (!is_alphanumeric_true(data[pos])) goto buffer_seek_alphanumeric_right_end; - } - } - -buffer_seek_alphanumeric_right_end: - return(pos); -} - -internal_4tech int -buffer_seek_alphanumeric_left(Buffer_Type *buffer, int pos){ - Buffer_Backify_Type loop; - char *data; - int end; - int size; - - --pos; - if (pos >= 0){ - size = buffer_size(buffer); - loop = buffer_backify_loop(buffer, pos, 1); - - for (;buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - end = loop.absolute_pos; - data = loop.data - end; - for (; pos >= end; --pos){ - if (is_alphanumeric_true(data[pos])) goto buffer_seek_alphanumeric_left_mid; - } - } - -buffer_seek_alphanumeric_left_mid: - for (;buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - end = loop.absolute_pos; - data = loop.data - end; - for (; pos >= end; --pos){ - if (!is_alphanumeric_true(data[pos])){ - ++pos; - goto buffer_seek_alphanumeric_left_end; - } - } - } - } - else{ - pos = 0; - } - -buffer_seek_alphanumeric_left_end: - return(pos); -} - -internal_4tech int -buffer_seek_range_camel_right(Buffer_Type *buffer, int pos, int an_pos){ - Buffer_Stringify_Type loop; - char *data; - int end, size; - char ch, prev_ch; - - size = buffer_size(buffer); - assert_4tech(pos <= an_pos); - assert_4tech(an_pos <= size); - - ++pos; - if (pos < an_pos){ - loop = buffer_stringify_loop(buffer, pos, an_pos); - if (buffer_stringify_good(&loop)){ - prev_ch = loop.data[0]; - ++pos; - - for (;buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos < end; ++pos){ - ch = data[pos]; - if (is_upper(ch) && is_lower(prev_ch)) goto buffer_seek_alphanumeric_or_camel_right_end; - prev_ch = ch; - } - } - } - } - else{ - pos = an_pos; - } - -buffer_seek_alphanumeric_or_camel_right_end: - return(pos); -} - -internal_4tech int -buffer_seek_range_camel_left(Buffer_Type *buffer, int pos, int an_pos){ - Buffer_Backify_Type loop; - char *data; - int end, size; - char ch, prev_ch; - - size = buffer_size(buffer); - assert_4tech(an_pos <= pos); - assert_4tech(0 <= an_pos); - - loop = buffer_backify_loop(buffer, pos, an_pos+1); - if (buffer_backify_good(&loop)){ - prev_ch = loop.data[0]; - --pos; - - for (;buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - end = loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos >= end; --pos){ - ch = data[pos]; - if (is_upper(ch) && is_lower(prev_ch)) goto buffer_seek_alphanumeric_or_camel_left_end; - prev_ch = ch; - } - } - } - -buffer_seek_alphanumeric_or_camel_left_end: - return(pos); -} - -internal_4tech int -buffer_seek_alphanumeric_or_camel_right(Buffer_Type *buffer, int pos){ - int an_pos, result; - an_pos = buffer_seek_alphanumeric_right(buffer, pos); - result = buffer_seek_range_camel_right(buffer, pos, an_pos); - return(result); -} - -internal_4tech int -buffer_seek_alphanumeric_or_camel_left(Buffer_Type *buffer, int pos){ - int an_pos, result; - an_pos = buffer_seek_alphanumeric_left(buffer, pos); - result = buffer_seek_range_camel_left(buffer, pos, an_pos); - return(result); -} - -internal_4tech int -buffer_find_hard_start(Buffer_Type *buffer, int line_start, int *all_whitespace, - int *all_space, int *preferred_indent, int tab_width){ - Buffer_Stringify_Type loop; - char *data; - int size, end; - int result; - char c; - - *all_space = 1; - *preferred_indent = 0; - - size = buffer_size(buffer); - - tab_width -= 1; - - result = line_start; - for (loop = buffer_stringify_loop(buffer, line_start, size); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; result < end; ++result){ - c = data[result]; - - if (c == '\n' || c == 0){ - *all_whitespace = 1; - goto buffer_find_hard_start_end; - } - if (c >= '!' && c <= '~') goto buffer_find_hard_start_end; - if (c == '\t') *preferred_indent += tab_width; - if (c != ' ') *all_space = 0; - *preferred_indent += 1; - } - } - -buffer_find_hard_start_end: - return(result); -} - -internal_4tech int -buffer_find_string(Buffer_Type *buffer, int start_pos, char *str, int len, char *spare){ - Buffer_Stringify_Type loop; - char *data; - int size, end; - int pos; - - size = buffer_size(buffer); - - pos = start_pos; - if (len > 0){ - for (loop = buffer_stringify_loop(buffer, start_pos, size - len + 1); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos < end; ++pos){ - if (*str == data[pos]){ - buffer_stringify(buffer, pos, pos + len, spare); - if (is_match(str, spare, len)) - goto buffer_find_string_end; - } - } - } - } - -buffer_find_string_end: - if (pos >= size - len + 1) pos = size; - return(pos); -} - -internal_4tech int -buffer_rfind_string(Buffer_Type *buffer, int start_pos, char *str, int len, char *spare){ - Buffer_Backify_Type loop; - char *data; - int end, size; - int pos; - - size = buffer_size(buffer); - - pos = start_pos; - if (pos > size - len) pos = size - len; - - if (len > 0){ - for (loop = buffer_backify_loop(buffer, start_pos, 0); - buffer_backify_good(&loop); - buffer_backify_next(&loop)){ - end = loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; pos >= end; --pos){ - if (*str == data[pos]){ - buffer_stringify(buffer, pos, pos + len, spare); - if (is_match(str, spare, len)) - goto buffer_rfind_string_end; - } - } - } - } - -buffer_rfind_string_end: - return(pos); -} - -#ifndef NON_ABSTRACT_4TECH -typedef struct Buffer_Measure_Starts{ - int i; - int count; - int start; - float width; -} Buffer_Measure_Starts; -#endif - -internal_4tech int -buffer_measure_starts_widths_(Buffer_Measure_Starts *state, Buffer_Type *buffer, float *advance_data){ - Buffer_Stringify_Type loop; - int *start_ptr, *start_end; - float *width_ptr; - debug_4tech(int widths_max); - debug_4tech(int max); - char *data; - int size, end; - float width; - int start, i; - int result; - char ch; - - size = buffer_size(buffer); - - debug_4tech(max = buffer->line_max); - debug_4tech(widths_max = buffer->widths_max); - assert_4tech(max == widths_max); - - result = 1; - - i = state->i; - start = state->start; - width = state->width; - - start_ptr = buffer->line_starts + state->count; - width_ptr = buffer->line_widths + state->count; - start_end = buffer->line_starts + buffer->line_max; - - for (loop = buffer_stringify_loop(buffer, i, size); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; i < end; ++i){ - ch = data[i]; - if (ch == '\n'){ - if (start_ptr == start_end) goto buffer_measure_starts_widths_end; - - *width_ptr++ = width; - *start_ptr++ = start; - start = i + 1; - width = 0; - } - else{ - width += measure_character(advance_data, ch); - } - } - } - - assert_4tech(i == size); - - if (start_ptr == start_end) goto buffer_measure_starts_widths_end; - *start_ptr++ = start; - *width_ptr++ = 0; - result = 0; - -buffer_measure_starts_widths_end: - state->i = i; - state->count = (int)(start_ptr - buffer->line_starts); - state->start = start; - state->width = width; - - return(result); -} - -internal_4tech int -buffer_measure_starts_zero_widths_(Buffer_Measure_Starts *state, Buffer_Type *buffer){ - Buffer_Stringify_Type loop; - int *start_ptr, *start_end; - float *width_ptr; - debug_4tech(int widths_max); - debug_4tech(int max); - char *data; - int size, end; - int start, i; - int result; - char ch; - - size = buffer_size(buffer); - - debug_4tech(max = buffer->line_max); - debug_4tech(widths_max = buffer->widths_max); - assert_4tech(max == widths_max); - - result = 1; - - i = state->i; - start = state->start; - - start_ptr = buffer->line_starts + state->count; - width_ptr = buffer->line_widths + state->count; - start_end = buffer->line_starts + buffer->line_max; - - for (loop = buffer_stringify_loop(buffer, i, size); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; i < end; ++i){ - ch = data[i]; - if (ch == '\n'){ - if (start_ptr == start_end) goto buffer_measure_starts_zero_widths_end; - - *width_ptr++ = 0; - *start_ptr++ = start; - start = i + 1; - } - } - } - - assert_4tech(i == size); - - if (start_ptr == start_end) goto buffer_measure_starts_zero_widths_end; - *start_ptr++ = start; - *width_ptr++ = 0; - result = 0; - -buffer_measure_starts_zero_widths_end: - state->i = i; - state->count = (int)(start_ptr - buffer->line_starts); - state->start = start; - - return(result); -} - -internal_4tech int -buffer_measure_starts_widths(Buffer_Measure_Starts *state, Buffer_Type *buffer, float *advance_data){ - int result; - - if (advance_data){ - result = buffer_measure_starts_widths_(state, buffer, advance_data); - } - else{ - result = buffer_measure_starts_zero_widths_(state, buffer); - } - - return(result); -} - -internal_4tech void -buffer_remeasure_starts(Buffer_Type *buffer, int line_start, int line_end, int line_shift, int text_shift){ - Buffer_Stringify_Type loop; - int *starts; - int line_count; - char *data; - int size, end; - int line_i, char_i, start; - - starts = buffer->line_starts; - line_count = buffer->line_count; - - assert_4tech(0 <= line_start); - assert_4tech(line_start <= line_end); - assert_4tech(line_end < line_count); - assert_4tech(line_count + line_shift <= buffer->line_max); - - ++line_end; - if (text_shift != 0){ - line_i = line_end; - starts += line_i; - for (; line_i < line_count; ++line_i, ++starts){ - *starts += text_shift; - } - starts = buffer->line_starts; - } - - if (line_shift != 0){ - memmove_4tech(starts + line_end + line_shift, starts + line_end, - sizeof(int)*(line_count - line_end)); - line_count += line_shift; - } - - line_end += line_shift; - size = buffer_size(buffer); - char_i = starts[line_start]; - line_i = line_start; - start = char_i; - - for (loop = buffer_stringify_loop(buffer, char_i, size); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; char_i < end; ++char_i){ - if (data[char_i] == '\n'){ - starts[line_i++] = start; - start = char_i + 1; - if (line_i >= line_end && start == starts[line_i]) goto buffer_remeasure_starts_end; - } - } - } - - if (char_i == size){ - starts[line_i++] = start; - } - -buffer_remeasure_starts_end: - assert_4tech(line_count >= 1); - buffer->line_count = line_count; -} - -internal_4tech void -buffer_remeasure_widths(Buffer_Type *buffer, float *advance_data, - int line_start, int line_end, int line_shift){ - Buffer_Stringify_Type loop; - int *starts; - float *widths; - int line_count; - int widths_count; - char *data; - int size, end; - int i, j; - float width; - char ch; - - starts = buffer->line_starts; - widths = buffer->line_widths; - line_count = buffer->line_count; - widths_count = buffer->widths_count; - - assert_4tech(0 <= line_start); - assert_4tech(line_start <= line_end); - assert_4tech(line_count <= buffer->widths_max); - - if (line_shift != 0){ - memmove_4tech(widths + line_end + line_shift, widths + line_end, - sizeof(float)*(widths_count - line_end)); - } - buffer->widths_count = line_count; - - line_end += line_shift; - i = line_start; - j = starts[i]; - - if (line_end == line_count) size = buffer_size(buffer); - else size = starts[line_end]; - - width = 0; - - for (loop = buffer_stringify_loop(buffer, j, size); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - - for (; j < end; ++j){ - ch = data[j]; - if (ch == '\n'){ - widths[i] = width; - ++i; - assert_4tech(j + 1 == starts[i]); - width = 0; - } - else{ - width += measure_character(advance_data, ch); - } - } - } -} - -#if 0 -inline_4tech void -buffer_measure_widths(Buffer_Type *buffer, void *advance_data){ - assert_4tech(buffer->line_count >= 1); - buffer_remeasure_widths(buffer, advance_data, 0, buffer->line_count-1, 0); -} -#endif - -internal_4tech void -buffer_measure_wrap_y(Buffer_Type *buffer, float *wraps, - float font_height, float max_width){ - float *widths; - float y_pos; - int i, line_count; - - line_count = buffer->line_count; - widths = buffer->line_widths; - y_pos = 0; - - for (i = 0; i < line_count; ++i){ - wraps[i] = y_pos; - if (widths[i] == 0) y_pos += font_height; - else y_pos += font_height*ceil_4tech(widths[i]/max_width); - } -} - -internal_4tech int -buffer_get_line_index_range(Buffer_Type *buffer, int pos, int l_bound, int u_bound){ - int *lines; - int start, end; - int i; - - assert_4tech(0 <= l_bound); - assert_4tech(l_bound <= u_bound); - assert_4tech(u_bound <= buffer->line_count); - - lines = buffer->line_starts; - - start = l_bound; - end = u_bound; - for (;;){ - i = (start + end) >> 1; - if (lines[i] < pos) start = i; - else if (lines[i] > pos) end = i; - else{ - start = i; - break; - } - assert_4tech(start < end); - if (start == end - 1) break; - } - - return(start); -} - -inline_4tech int -buffer_get_line_index(Buffer_Type *buffer, int pos){ - int result; - result = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count); - return(result); -} - -#ifndef NON_ABSTRACT_4TECH -internal_4tech int -buffer_get_line_index_from_wrapped_y(float *wraps, float y, float font_height, int l_bound, int u_bound){ - int start, end, i, result; - start = l_bound; - end = u_bound; - for (;;){ - i = (start + end) / 2; - if (wraps[i]+font_height <= y) start = i; - else if (wraps[i] > y) end = i; - else{ - result = i; - break; - } - if (start >= end - 1){ - result = start; - break; - } - } - return(result); -} -#endif - -#ifndef NON_ABSTRACT_4TECH -typedef struct Seek_State{ - Full_Cursor cursor; - Full_Cursor prev_cursor; -} Seek_State; - -internal_4tech int -cursor_seek_step(Seek_State *state, Buffer_Seek seek, int xy_seek, float max_width, - float font_height, float *advances, int size, char ch){ - Full_Cursor cursor, prev_cursor; - float ch_width; - int result; - float x, px, y; - - cursor = state->cursor; - prev_cursor = state->prev_cursor; - - result = 1; - prev_cursor = cursor; - switch (ch){ - case '\n': - ++cursor.line; - cursor.unwrapped_y += font_height; - cursor.wrapped_y += font_height; - cursor.character = 1; - cursor.unwrapped_x = 0; - cursor.wrapped_x = 0; - break; - - default: - ++cursor.character; - if (ch == '\r') ch_width = *(float*)(advances + '\\') + *(float*)(advances + 'r'); - else ch_width = *(float*)(advances + ch); - - if (cursor.wrapped_x + ch_width >= max_width){ - cursor.wrapped_y += font_height; - cursor.wrapped_x = 0; - prev_cursor = cursor; - } - - cursor.unwrapped_x += ch_width; - cursor.wrapped_x += ch_width; - - break; - } - - ++cursor.pos; - - if (cursor.pos > size){ - cursor = prev_cursor; - result = 0; - goto cursor_seek_step_end; - } - - x = y = px = 0; - - switch (seek.type){ - case buffer_seek_pos: - if (cursor.pos > seek.pos){ - cursor = prev_cursor; - result = 0; - goto cursor_seek_step_end; - }break; - - case buffer_seek_wrapped_xy: - x = cursor.wrapped_x; px = prev_cursor.wrapped_x; - y = cursor.wrapped_y; break; - - case buffer_seek_unwrapped_xy: - x = cursor.unwrapped_x; px = prev_cursor.unwrapped_x; - y = cursor.unwrapped_y; break; - - case buffer_seek_line_char: - if (cursor.line == seek.line && cursor.character >= seek.character){ - result = 0; - goto cursor_seek_step_end; - } - else if (cursor.line > seek.line){ - cursor = prev_cursor; - result = 0; - goto cursor_seek_step_end; - }break; - } - - if (xy_seek){ - if (y > seek.y){ - cursor = prev_cursor; - result = 0; - goto cursor_seek_step_end; - } - - if (y > seek.y - font_height && x >= seek.x){ - if (!seek.round_down){ - if (ch != '\n' && (seek.x - px) < (x - seek.x)) cursor = prev_cursor; - result = 0; - goto cursor_seek_step_end; - } - - if (x > seek.x){ - cursor = prev_cursor; - result = 0; - goto cursor_seek_step_end; - } - } - } - -cursor_seek_step_end: - state->cursor = cursor; - state->prev_cursor = prev_cursor; - return(result); -} -#endif - -internal_4tech Full_Cursor -buffer_cursor_seek(Buffer_Type *buffer, Buffer_Seek seek, float max_width, - float font_height, float *advance_data, Full_Cursor cursor){ - Buffer_Stringify_Type loop; - char *data; - int size, end; - int i; - int result; - - Seek_State state; - int xy_seek; - - state.cursor = cursor; - - if (advance_data){ - - size = buffer_size(buffer); - xy_seek = (seek.type == buffer_seek_wrapped_xy || seek.type == buffer_seek_unwrapped_xy); - - result = 1; - i = cursor.pos; - for (loop = buffer_stringify_loop(buffer, i, size); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - for (; i < end; ++i){ - result = cursor_seek_step(&state, seek, xy_seek, max_width, - font_height, advance_data, size, data[i]); - if (!result) goto buffer_cursor_seek_end; - } - } - if (result){ - result = cursor_seek_step(&state, seek, xy_seek, max_width, - font_height, advance_data, size, 0); - assert_4tech(result == 0); - } - - } - -buffer_cursor_seek_end: - return(state.cursor); -} - -internal_4tech Full_Cursor -buffer_cursor_from_pos(Buffer_Type *buffer, int pos, float *wraps, - float max_width, float font_height, float *advance_data){ - Full_Cursor result; - int line_index; - - line_index = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count); - result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); - result = buffer_cursor_seek(buffer, seek_pos(pos), max_width, font_height, - advance_data, result); - - return(result); -} - -internal_4tech Full_Cursor -buffer_cursor_from_line_character(Buffer_Type *buffer, int line, int character, float *wraps, - float max_width, float font_height, float *advance_data){ - Full_Cursor result; - int line_index; - - line_index = line - 1; - if (line_index >= buffer->line_count) line_index = buffer->line_count - 1; - if (line_index < 0) line_index = 0; - - result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); - result = buffer_cursor_seek(buffer, seek_line_char(line, character), - max_width, font_height, advance_data, result); - - return(result); -} - -internal_4tech Full_Cursor -buffer_cursor_from_unwrapped_xy(Buffer_Type *buffer, float x, float y, int round_down, float *wraps, - float max_width, float font_height, float *advance_data){ - Full_Cursor result; - int line_index; - - line_index = (int)(y / font_height); - if (line_index >= buffer->line_count) line_index = buffer->line_count - 1; - if (line_index < 0) line_index = 0; - - result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); - result = buffer_cursor_seek(buffer, seek_unwrapped_xy(x, y, round_down), - max_width, font_height, advance_data, result); - - return(result); -} - -internal_4tech Full_Cursor -buffer_cursor_from_wrapped_xy(Buffer_Type *buffer, float x, float y, int round_down, float *wraps, - float max_width, float font_height, float *advance_data){ - Full_Cursor result; - int line_index; - - line_index = buffer_get_line_index_from_wrapped_y(wraps, y, font_height, 0, buffer->line_count); - result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); - result = buffer_cursor_seek(buffer, seek_wrapped_xy(x, y, round_down), - max_width, font_height, advance_data, result); - - return(result); -} - -internal_4tech void -buffer_invert_edit_shift(Buffer_Type *buffer, Buffer_Edit edit, Buffer_Edit *inverse, char *strings, - int *str_pos, int max, int shift_amount){ - int pos; - int len; - - pos = *str_pos; - len = edit.end - edit.start; - assert_4tech(pos + len <= max); - *str_pos = pos + len; - - inverse->str_start = pos; - inverse->len = len; - inverse->start = edit.start + shift_amount; - inverse->end = edit.start + edit.len + shift_amount; - buffer_stringify(buffer, edit.start, edit.end, strings + pos); -} - -inline_4tech void -buffer_invert_edit(Buffer_Type *buffer, Buffer_Edit edit, Buffer_Edit *inverse, char *strings, - int *str_pos, int max){ - buffer_invert_edit_shift(buffer, edit, inverse, strings, str_pos, max, 0); -} - -#ifndef NON_ABSTRACT_4TECH -typedef struct Buffer_Invert_Batch{ - int i; - int shift_amount; - int len; -} Buffer_Invert_Batch; -#endif - -internal_4tech int -buffer_invert_batch(Buffer_Invert_Batch *state, Buffer_Type *buffer, Buffer_Edit *edits, int count, - Buffer_Edit *inverse, char *strings, int *str_pos, int max){ - Buffer_Edit *edit, *inv_edit; - int shift_amount; - int result; - int i; - - result = 0; - i = state->i; - shift_amount = state->shift_amount; - - edit = edits + i; - inv_edit = inverse + i; - - for (; i < count; ++i, ++edit, ++inv_edit){ - if (*str_pos + edit->end - edit->start <= max){ - buffer_invert_edit_shift(buffer, *edit, inv_edit, strings, str_pos, max, shift_amount); - shift_amount += (edit->len - (edit->end - edit->start)); - } - else{ - result = 1; - state->len = edit->end - edit->start; - } - } - - state->i = i; - state->shift_amount = shift_amount; - - return(result); -} - -struct Buffer_Render_Options{ - b8 show_slash_t; -}; - -internal_4tech Full_Cursor -buffer_get_start_cursor(Buffer_Type *buffer, float *wraps, float scroll_y, - int wrapped, float width, float *advance_data, float font_height){ - Full_Cursor result; - - if (wrapped){ - result = buffer_cursor_from_wrapped_xy(buffer, 0, scroll_y, 0, wraps, - width, font_height, advance_data); - } - else{ - result = buffer_cursor_from_unwrapped_xy(buffer, 0, scroll_y, 0, wraps, - width, font_height, advance_data); - } - - return(result); -} - -internal_4tech void -buffer_get_render_data(Buffer_Type *buffer, Buffer_Render_Item *items, int max, int *count, - float port_x, float port_y, - float scroll_x, float scroll_y, Full_Cursor start_cursor, - int wrapped, - float width, float height, - float *advance_data, float font_height, - Buffer_Render_Options opts){ - - Buffer_Stringify_Type loop; - Buffer_Render_Item *item; - char *data; - int size, end; - float shift_x, shift_y; - float x, y; - int i, item_i; - float ch_width, ch_width_sub; - char ch; - - size = buffer_size(buffer); - - shift_x = port_x - scroll_x; - shift_y = port_y - scroll_y; - if (wrapped) shift_y += start_cursor.wrapped_y; - else shift_y += start_cursor.unwrapped_y; - - x = shift_x; - y = shift_y; - item_i = 0; - item = items + item_i; - - if (advance_data){ - for (loop = buffer_stringify_loop(buffer, start_cursor.pos, size); - buffer_stringify_good(&loop); - buffer_stringify_next(&loop)){ - - end = loop.size + loop.absolute_pos; - data = loop.data - loop.absolute_pos; - - for (i = loop.absolute_pos; i < end; ++i){ - ch = data[i]; - ch_width = measure_character(advance_data, ch); - - if (ch_width + x > width + shift_x && wrapped){ - x = shift_x; - y += font_height; - } - if (y > height + shift_y) goto buffer_get_render_data_end; - - switch (ch){ - case '\n': - write_render_item_inline(item, i, ' ', x, y, advance_data, font_height); - item->flags = 0; - ++item_i; - ++item; - - x = shift_x; - y += font_height; - break; - - case 0: - ch_width = write_render_item_inline(item, i, '\\', x, y, advance_data, font_height); - item->flags = BRFlag_Special_Character; - ++item_i; - ++item; - x += ch_width; - - ch_width = write_render_item_inline(item, i, '0', x, y, advance_data, font_height); - item->flags = BRFlag_Special_Character; - ++item_i; - ++item; - x += ch_width; - break; - - case '\r': - ch_width = write_render_item_inline(item, i, '\\', x, y, advance_data, font_height); - item->flags = BRFlag_Special_Character; - ++item_i; - ++item; - x += ch_width; - - ch_width = write_render_item_inline(item, i, 'r', x, y, advance_data, font_height); - item->flags = BRFlag_Special_Character; - ++item_i; - ++item; - x += ch_width; - break; - - case '\t': - if (opts.show_slash_t){ - ch_width_sub = write_render_item_inline(item, i, '\\', x, y, advance_data, font_height); - item->flags = BRFlag_Special_Character; - ++item_i; - ++item; - - write_render_item_inline(item, i, 't', x + ch_width_sub, y, advance_data, font_height); - item->flags = BRFlag_Special_Character; - ++item_i; - ++item; - } - else{ - write_render_item_inline(item, i, ' ', x, y, advance_data, font_height); - item->flags = 0; - ++item_i; - ++item; - } - x += ch_width; - break; - - default: - write_render_item(item, i, ch, x, y, ch_width, font_height); - item->flags = 0; - ++item_i; - ++item; - x += ch_width; - - break; - } - if (y > height + shift_y) goto buffer_get_render_data_end; - } - } - -buffer_get_render_data_end: - if (y <= height + shift_y || item == items){ - ch = 0; - ch_width = measure_character(advance_data, ' '); - write_render_item(item, size, ch, x, y, ch_width, font_height); - ++item_i; - ++item; - x += ch_width; - } - } - else{ - ch = 0; - ch_width = 0; - write_render_item(item, size, ch, x, y, ch_width, font_height); - ++item_i; - ++item; - x += ch_width; - } - - // TODO(allen): handle this with a control state - assert_4tech(item_i <= max); - *count = item_i; -} - -#ifndef NON_ABSTRACT_4TECH -#define NON_ABSTRACT_4TECH 1 -#endif -// BOTTOM - +/* + * Mr. 4th Dimention - Allen Webster + * Four Tech + * + * public domain -- no warranty is offered or implied; use this code at your own risk + * + * 24.10.2015 + * + * Buffer data object + * type - Golden Array + * + */ + +// TOP + +#define Buffer_Init_Type cat_4tech(Buffer_Type, _Init) +#define Buffer_Stringify_Type cat_4tech(Buffer_Type, _Stringify_Loop) +#define Buffer_Backify_Type cat_4tech(Buffer_Type, _Backify_Loop) + +inline_4tech void +buffer_stringify(Buffer_Type *buffer, int start, int end, char *out){ + for (Buffer_Stringify_Type loop = buffer_stringify_loop(buffer, start, end); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + memcpy_4tech(out, loop.data, loop.size); + out += loop.size; + } +} + +inline_4tech void +buffer_backify(Buffer_Type *buffer, int start, int end, char *out){ + for (Buffer_Backify_Type loop = buffer_backify_loop(buffer, end, start); + buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + memcpy_4tech(out, loop.data, loop.size); + out += loop.size; + } +} + +internal_4tech int +buffer_convert_out(Buffer_Type *buffer, char *dest, int max){ + Buffer_Stringify_Type loop; + int size, out_size, pos, result; + + size = buffer_size(buffer); + assert_4tech(size + buffer->line_count < max); + + pos = 0; + for (loop = buffer_stringify_loop(buffer, 0, size); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + result = eol_convert_out(dest + pos, max - pos, loop.data, loop.size, &out_size); + assert_4tech(result); + pos += out_size; + } + + return(pos); +} + +internal_4tech int +buffer_count_newlines(Buffer_Type *buffer, int start, int end){ + Buffer_Stringify_Type loop; + int i; + int count; + + assert_4tech(0 <= start); + assert_4tech(start <= end); + assert_4tech(end <= buffer_size(buffer)); + + count = 0; + + for (loop = buffer_stringify_loop(buffer, start, end); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + for (i = 0; i < loop.size; ++i){ + count += (loop.data[i] == '\n'); + } + } + + return(count); +} + +internal_4tech int +buffer_seek_whitespace_down(Buffer_Type *buffer, int pos){ + Buffer_Stringify_Type loop; + char *data; + int end; + int size; + int no_hard; + int prev_endline; + + size = buffer_size(buffer); + loop = buffer_stringify_loop(buffer, pos, size); + + for (;buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (;pos < end; ++pos){ + if (!is_whitespace(data[pos])) goto buffer_seek_whitespace_down_mid; + } + } + +buffer_seek_whitespace_down_mid: + no_hard = 0; + prev_endline = -1; + for (;buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos < end; ++pos){ + if (data[pos] == '\n'){ + if (no_hard) goto buffer_seek_whitespace_down_end; + else{ + no_hard = 1; + prev_endline = pos; + } + } + else if (!is_whitespace(data[pos])){ + no_hard = 0; + } + } + } + +buffer_seek_whitespace_down_end: + if (prev_endline == -1 || prev_endline+1 >= size) pos = size; + else pos = prev_endline+1; + + return pos; +} + +internal_4tech int +buffer_seek_whitespace_up(Buffer_Type *buffer, int pos){ + Buffer_Backify_Type loop; + char *data; + int end; + int size; + int no_hard; + + size = buffer_size(buffer); + loop = buffer_backify_loop(buffer, pos-1, 1); + + for (;buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (;pos >= end; --pos){ + if (!is_whitespace(data[pos])) goto buffer_seek_whitespace_up_mid; + } + } + +buffer_seek_whitespace_up_mid: + no_hard = 0; + for (;buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos >= end; --pos){ + if (data[pos] == '\n'){ + if (no_hard) goto buffer_seek_whitespace_up_end; + else no_hard = 1; + } + else if (!is_whitespace(data[pos])){ + no_hard = 0; + } + } + } + +buffer_seek_whitespace_up_end: + if (pos != 0) ++pos; + + return pos; +} + +internal_4tech int +buffer_seek_whitespace_right(Buffer_Type *buffer, int pos){ + Buffer_Stringify_Type loop; + char *data; + int end; + int size; + + size = buffer_size(buffer); + loop = buffer_stringify_loop(buffer, pos, size); + + for (;buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos < end && is_whitespace(data[pos]); ++pos); + if (!is_whitespace(data[pos])) break; + } + + for (;buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos < end && !is_whitespace(data[pos]); ++pos); + if (is_whitespace(data[pos])) break; + } + + return(pos); +} + +internal_4tech int +buffer_seek_whitespace_left(Buffer_Type *buffer, int pos){ + Buffer_Backify_Type loop; + char *data; + int end; + int size; + + --pos; + if (pos > 0){ + size = buffer_size(buffer); + loop = buffer_backify_loop(buffer, pos, 1); + + for (;buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos >= end && is_whitespace(data[pos]); --pos); + if (!is_whitespace(data[pos])) break; + } + + for (;buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos >= end && !is_whitespace(data[pos]); --pos); + if (is_whitespace(data[pos])) break; + } + + if (pos != 0) ++pos; + } + else{ + pos = 0; + } + + return(pos); +} + +internal_4tech int +buffer_seek_word_right_assume_on_word(Buffer_Type *buffer, int pos){ + Buffer_Stringify_Type loop; + char *data; + int end; + int size; + + size = buffer_size(buffer); + loop = buffer_stringify_loop(buffer, pos, size); + + for (;buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos < end; ++pos){ + if (!is_alphanumeric(data[pos])) goto double_break; + } + } +double_break: + + return(pos); +} + +internal_4tech int +buffer_seek_alphanumeric_right(Buffer_Type *buffer, int pos){ + Buffer_Stringify_Type loop; + char *data; + int end; + int size; + + size = buffer_size(buffer); + loop = buffer_stringify_loop(buffer, pos, size); + + for (;buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos < end; ++pos){ + if (is_alphanumeric_true(data[pos])) goto buffer_seek_alphanumeric_right_mid; + } + } + +buffer_seek_alphanumeric_right_mid: + for (;buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos < end; ++pos){ + if (!is_alphanumeric_true(data[pos])) goto buffer_seek_alphanumeric_right_end; + } + } + +buffer_seek_alphanumeric_right_end: + return(pos); +} + +internal_4tech int +buffer_seek_alphanumeric_left(Buffer_Type *buffer, int pos){ + Buffer_Backify_Type loop; + char *data; + int end; + int size; + + --pos; + if (pos >= 0){ + size = buffer_size(buffer); + loop = buffer_backify_loop(buffer, pos, 1); + + for (;buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - end; + for (; pos >= end; --pos){ + if (is_alphanumeric_true(data[pos])) goto buffer_seek_alphanumeric_left_mid; + } + } + +buffer_seek_alphanumeric_left_mid: + for (;buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - end; + for (; pos >= end; --pos){ + if (!is_alphanumeric_true(data[pos])){ + ++pos; + goto buffer_seek_alphanumeric_left_end; + } + } + } + } + else{ + pos = 0; + } + +buffer_seek_alphanumeric_left_end: + return(pos); +} + +internal_4tech int +buffer_seek_range_camel_right(Buffer_Type *buffer, int pos, int an_pos){ + Buffer_Stringify_Type loop; + char *data; + int end, size; + char ch, prev_ch; + + size = buffer_size(buffer); + assert_4tech(pos <= an_pos); + assert_4tech(an_pos <= size); + + ++pos; + if (pos < an_pos){ + loop = buffer_stringify_loop(buffer, pos, an_pos); + if (buffer_stringify_good(&loop)){ + prev_ch = loop.data[0]; + ++pos; + + for (;buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos < end; ++pos){ + ch = data[pos]; + if (is_upper(ch) && is_lower(prev_ch)) goto buffer_seek_alphanumeric_or_camel_right_end; + prev_ch = ch; + } + } + } + } + else{ + pos = an_pos; + } + +buffer_seek_alphanumeric_or_camel_right_end: + return(pos); +} + +internal_4tech int +buffer_seek_range_camel_left(Buffer_Type *buffer, int pos, int an_pos){ + Buffer_Backify_Type loop; + char *data; + int end, size; + char ch, prev_ch; + + size = buffer_size(buffer); + assert_4tech(an_pos <= pos); + assert_4tech(0 <= an_pos); + + loop = buffer_backify_loop(buffer, pos, an_pos+1); + if (buffer_backify_good(&loop)){ + prev_ch = loop.data[0]; + --pos; + + for (;buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos >= end; --pos){ + ch = data[pos]; + if (is_upper(ch) && is_lower(prev_ch)) goto buffer_seek_alphanumeric_or_camel_left_end; + prev_ch = ch; + } + } + } + +buffer_seek_alphanumeric_or_camel_left_end: + return(pos); +} + +internal_4tech int +buffer_seek_alphanumeric_or_camel_right(Buffer_Type *buffer, int pos){ + int an_pos, result; + an_pos = buffer_seek_alphanumeric_right(buffer, pos); + result = buffer_seek_range_camel_right(buffer, pos, an_pos); + return(result); +} + +internal_4tech int +buffer_seek_alphanumeric_or_camel_left(Buffer_Type *buffer, int pos){ + int an_pos, result; + an_pos = buffer_seek_alphanumeric_left(buffer, pos); + result = buffer_seek_range_camel_left(buffer, pos, an_pos); + return(result); +} + +internal_4tech int +buffer_find_hard_start(Buffer_Type *buffer, int line_start, int *all_whitespace, + int *all_space, int *preferred_indent, int tab_width){ + Buffer_Stringify_Type loop; + char *data; + int size, end; + int result; + char c; + + *all_space = 1; + *preferred_indent = 0; + + size = buffer_size(buffer); + + tab_width -= 1; + + result = line_start; + for (loop = buffer_stringify_loop(buffer, line_start, size); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; result < end; ++result){ + c = data[result]; + + if (c == '\n' || c == 0){ + *all_whitespace = 1; + goto buffer_find_hard_start_end; + } + if (c >= '!' && c <= '~') goto buffer_find_hard_start_end; + if (c == '\t') *preferred_indent += tab_width; + if (c != ' ') *all_space = 0; + *preferred_indent += 1; + } + } + +buffer_find_hard_start_end: + return(result); +} + +internal_4tech int +buffer_find_string(Buffer_Type *buffer, int start_pos, char *str, int len, char *spare){ + Buffer_Stringify_Type loop; + char *data; + int size, end; + int pos; + + size = buffer_size(buffer); + + pos = start_pos; + if (len > 0){ + for (loop = buffer_stringify_loop(buffer, start_pos, size - len + 1); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos < end; ++pos){ + if (*str == data[pos]){ + buffer_stringify(buffer, pos, pos + len, spare); + if (is_match(str, spare, len)) + goto buffer_find_string_end; + } + } + } + } + +buffer_find_string_end: + if (pos >= size - len + 1) pos = size; + return(pos); +} + +internal_4tech int +buffer_rfind_string(Buffer_Type *buffer, int start_pos, char *str, int len, char *spare){ + Buffer_Backify_Type loop; + char *data; + int end, size; + int pos; + + size = buffer_size(buffer); + + pos = start_pos; + if (pos > size - len) pos = size - len; + + if (len > 0){ + for (loop = buffer_backify_loop(buffer, start_pos, 0); + buffer_backify_good(&loop); + buffer_backify_next(&loop)){ + end = loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; pos >= end; --pos){ + if (*str == data[pos]){ + buffer_stringify(buffer, pos, pos + len, spare); + if (is_match(str, spare, len)) + goto buffer_rfind_string_end; + } + } + } + } + +buffer_rfind_string_end: + return(pos); +} + +#ifndef NON_ABSTRACT_4TECH +typedef struct Buffer_Measure_Starts{ + int i; + int count; + int start; + float width; +} Buffer_Measure_Starts; +#endif + +internal_4tech int +buffer_measure_starts_widths_(Buffer_Measure_Starts *state, Buffer_Type *buffer, float *advance_data){ + Buffer_Stringify_Type loop; + int *start_ptr, *start_end; + float *width_ptr; + debug_4tech(int widths_max); + debug_4tech(int max); + char *data; + int size, end; + float width; + int start, i; + int result; + char ch; + + size = buffer_size(buffer); + + debug_4tech(max = buffer->line_max); + debug_4tech(widths_max = buffer->widths_max); + assert_4tech(max == widths_max); + + result = 1; + + i = state->i; + start = state->start; + width = state->width; + + start_ptr = buffer->line_starts + state->count; + width_ptr = buffer->line_widths + state->count; + start_end = buffer->line_starts + buffer->line_max; + + for (loop = buffer_stringify_loop(buffer, i, size); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; i < end; ++i){ + ch = data[i]; + if (ch == '\n'){ + if (start_ptr == start_end) goto buffer_measure_starts_widths_end; + + *width_ptr++ = width; + *start_ptr++ = start; + start = i + 1; + width = 0; + } + else{ + width += measure_character(advance_data, ch); + } + } + } + + assert_4tech(i == size); + + if (start_ptr == start_end) goto buffer_measure_starts_widths_end; + *start_ptr++ = start; + *width_ptr++ = 0; + result = 0; + +buffer_measure_starts_widths_end: + state->i = i; + state->count = (int)(start_ptr - buffer->line_starts); + state->start = start; + state->width = width; + + return(result); +} + +internal_4tech int +buffer_measure_starts_zero_widths_(Buffer_Measure_Starts *state, Buffer_Type *buffer){ + Buffer_Stringify_Type loop; + int *start_ptr, *start_end; + float *width_ptr; + debug_4tech(int widths_max); + debug_4tech(int max); + char *data; + int size, end; + int start, i; + int result; + char ch; + + size = buffer_size(buffer); + + debug_4tech(max = buffer->line_max); + debug_4tech(widths_max = buffer->widths_max); + assert_4tech(max == widths_max); + + result = 1; + + i = state->i; + start = state->start; + + start_ptr = buffer->line_starts + state->count; + width_ptr = buffer->line_widths + state->count; + start_end = buffer->line_starts + buffer->line_max; + + for (loop = buffer_stringify_loop(buffer, i, size); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; i < end; ++i){ + ch = data[i]; + if (ch == '\n'){ + if (start_ptr == start_end) goto buffer_measure_starts_zero_widths_end; + + *width_ptr++ = 0; + *start_ptr++ = start; + start = i + 1; + } + } + } + + assert_4tech(i == size); + + if (start_ptr == start_end) goto buffer_measure_starts_zero_widths_end; + *start_ptr++ = start; + *width_ptr++ = 0; + result = 0; + +buffer_measure_starts_zero_widths_end: + state->i = i; + state->count = (int)(start_ptr - buffer->line_starts); + state->start = start; + + return(result); +} + +internal_4tech int +buffer_measure_starts_widths(Buffer_Measure_Starts *state, Buffer_Type *buffer, float *advance_data){ + int result; + + if (advance_data){ + result = buffer_measure_starts_widths_(state, buffer, advance_data); + } + else{ + result = buffer_measure_starts_zero_widths_(state, buffer); + } + + return(result); +} + +internal_4tech void +buffer_remeasure_starts(Buffer_Type *buffer, int line_start, int line_end, int line_shift, int text_shift){ + Buffer_Stringify_Type loop; + int *starts; + int line_count; + char *data; + int size, end; + int line_i, char_i, start; + + starts = buffer->line_starts; + line_count = buffer->line_count; + + assert_4tech(0 <= line_start); + assert_4tech(line_start <= line_end); + assert_4tech(line_end < line_count); + assert_4tech(line_count + line_shift <= buffer->line_max); + + ++line_end; + if (text_shift != 0){ + line_i = line_end; + starts += line_i; + for (; line_i < line_count; ++line_i, ++starts){ + *starts += text_shift; + } + starts = buffer->line_starts; + } + + if (line_shift != 0){ + memmove_4tech(starts + line_end + line_shift, starts + line_end, + sizeof(int)*(line_count - line_end)); + line_count += line_shift; + } + + line_end += line_shift; + size = buffer_size(buffer); + char_i = starts[line_start]; + line_i = line_start; + start = char_i; + + for (loop = buffer_stringify_loop(buffer, char_i, size); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; char_i < end; ++char_i){ + if (data[char_i] == '\n'){ + starts[line_i++] = start; + start = char_i + 1; + if (line_i >= line_end && start == starts[line_i]) goto buffer_remeasure_starts_end; + } + } + } + + if (char_i == size){ + starts[line_i++] = start; + } + +buffer_remeasure_starts_end: + assert_4tech(line_count >= 1); + buffer->line_count = line_count; +} + +internal_4tech void +buffer_remeasure_widths(Buffer_Type *buffer, float *advance_data, + int line_start, int line_end, int line_shift){ + Buffer_Stringify_Type loop; + int *starts; + float *widths; + int line_count; + int widths_count; + char *data; + int size, end; + int i, j; + float width; + char ch; + + starts = buffer->line_starts; + widths = buffer->line_widths; + line_count = buffer->line_count; + widths_count = buffer->widths_count; + + assert_4tech(0 <= line_start); + assert_4tech(line_start <= line_end); + assert_4tech(line_count <= buffer->widths_max); + + if (line_shift != 0){ + memmove_4tech(widths + line_end + line_shift, widths + line_end, + sizeof(float)*(widths_count - line_end)); + } + buffer->widths_count = line_count; + + line_end += line_shift; + i = line_start; + j = starts[i]; + + if (line_end == line_count) size = buffer_size(buffer); + else size = starts[line_end]; + + width = 0; + + for (loop = buffer_stringify_loop(buffer, j, size); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + + for (; j < end; ++j){ + ch = data[j]; + if (ch == '\n'){ + widths[i] = width; + ++i; + assert_4tech(j + 1 == starts[i]); + width = 0; + } + else{ + width += measure_character(advance_data, ch); + } + } + } +} + +#if 0 +inline_4tech void +buffer_measure_widths(Buffer_Type *buffer, void *advance_data){ + assert_4tech(buffer->line_count >= 1); + buffer_remeasure_widths(buffer, advance_data, 0, buffer->line_count-1, 0); +} +#endif + +internal_4tech void +buffer_measure_wrap_y(Buffer_Type *buffer, float *wraps, + float font_height, float max_width){ + float *widths; + float y_pos; + int i, line_count; + + line_count = buffer->line_count; + widths = buffer->line_widths; + y_pos = 0; + + for (i = 0; i < line_count; ++i){ + wraps[i] = y_pos; + if (widths[i] == 0) y_pos += font_height; + else y_pos += font_height*ceil_4tech(widths[i]/max_width); + } +} + +internal_4tech int +buffer_get_line_index_range(Buffer_Type *buffer, int pos, int l_bound, int u_bound){ + int *lines; + int start, end; + int i; + + assert_4tech(0 <= l_bound); + assert_4tech(l_bound <= u_bound); + assert_4tech(u_bound <= buffer->line_count); + + lines = buffer->line_starts; + + start = l_bound; + end = u_bound; + for (;;){ + i = (start + end) >> 1; + if (lines[i] < pos) start = i; + else if (lines[i] > pos) end = i; + else{ + start = i; + break; + } + assert_4tech(start < end); + if (start == end - 1) break; + } + + return(start); +} + +inline_4tech int +buffer_get_line_index(Buffer_Type *buffer, int pos){ + int result; + result = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count); + return(result); +} + +#ifndef NON_ABSTRACT_4TECH +internal_4tech int +buffer_get_line_index_from_wrapped_y(float *wraps, float y, float font_height, int l_bound, int u_bound){ + int start, end, i, result; + start = l_bound; + end = u_bound; + for (;;){ + i = (start + end) / 2; + if (wraps[i]+font_height <= y) start = i; + else if (wraps[i] > y) end = i; + else{ + result = i; + break; + } + if (start >= end - 1){ + result = start; + break; + } + } + return(result); +} +#endif + +#ifndef NON_ABSTRACT_4TECH +typedef struct Seek_State{ + Full_Cursor cursor; + Full_Cursor prev_cursor; +} Seek_State; + +internal_4tech int +cursor_seek_step(Seek_State *state, Buffer_Seek seek, int xy_seek, float max_width, + float font_height, float *advances, int size, char ch){ + Full_Cursor cursor, prev_cursor; + float ch_width; + int result; + float x, px, y; + + cursor = state->cursor; + prev_cursor = state->prev_cursor; + + result = 1; + prev_cursor = cursor; + switch (ch){ + case '\n': + ++cursor.line; + cursor.unwrapped_y += font_height; + cursor.wrapped_y += font_height; + cursor.character = 1; + cursor.unwrapped_x = 0; + cursor.wrapped_x = 0; + break; + + default: + ++cursor.character; + if (ch == '\r') ch_width = *(float*)(advances + '\\') + *(float*)(advances + 'r'); + else ch_width = *(float*)(advances + ch); + + if (cursor.wrapped_x + ch_width >= max_width){ + cursor.wrapped_y += font_height; + cursor.wrapped_x = 0; + prev_cursor = cursor; + } + + cursor.unwrapped_x += ch_width; + cursor.wrapped_x += ch_width; + + break; + } + + ++cursor.pos; + + if (cursor.pos > size){ + cursor = prev_cursor; + result = 0; + goto cursor_seek_step_end; + } + + x = y = px = 0; + + switch (seek.type){ + case buffer_seek_pos: + if (cursor.pos > seek.pos){ + cursor = prev_cursor; + result = 0; + goto cursor_seek_step_end; + }break; + + case buffer_seek_wrapped_xy: + x = cursor.wrapped_x; px = prev_cursor.wrapped_x; + y = cursor.wrapped_y; break; + + case buffer_seek_unwrapped_xy: + x = cursor.unwrapped_x; px = prev_cursor.unwrapped_x; + y = cursor.unwrapped_y; break; + + case buffer_seek_line_char: + if (cursor.line == seek.line && cursor.character >= seek.character){ + result = 0; + goto cursor_seek_step_end; + } + else if (cursor.line > seek.line){ + cursor = prev_cursor; + result = 0; + goto cursor_seek_step_end; + }break; + } + + if (xy_seek){ + if (y > seek.y){ + cursor = prev_cursor; + result = 0; + goto cursor_seek_step_end; + } + + if (y > seek.y - font_height && x >= seek.x){ + if (!seek.round_down){ + if (ch != '\n' && (seek.x - px) < (x - seek.x)) cursor = prev_cursor; + result = 0; + goto cursor_seek_step_end; + } + + if (x > seek.x){ + cursor = prev_cursor; + result = 0; + goto cursor_seek_step_end; + } + } + } + +cursor_seek_step_end: + state->cursor = cursor; + state->prev_cursor = prev_cursor; + return(result); +} +#endif + +internal_4tech Full_Cursor +buffer_cursor_seek(Buffer_Type *buffer, Buffer_Seek seek, float max_width, + float font_height, float *advance_data, Full_Cursor cursor){ + Buffer_Stringify_Type loop; + char *data; + int size, end; + int i; + int result; + + Seek_State state; + int xy_seek; + + state.cursor = cursor; + + if (advance_data){ + + size = buffer_size(buffer); + xy_seek = (seek.type == buffer_seek_wrapped_xy || seek.type == buffer_seek_unwrapped_xy); + + result = 1; + i = cursor.pos; + for (loop = buffer_stringify_loop(buffer, i, size); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + for (; i < end; ++i){ + result = cursor_seek_step(&state, seek, xy_seek, max_width, + font_height, advance_data, size, data[i]); + if (!result) goto buffer_cursor_seek_end; + } + } + if (result){ + result = cursor_seek_step(&state, seek, xy_seek, max_width, + font_height, advance_data, size, 0); + assert_4tech(result == 0); + } + + } + +buffer_cursor_seek_end: + return(state.cursor); +} + +internal_4tech Full_Cursor +buffer_cursor_from_pos(Buffer_Type *buffer, int pos, float *wraps, + float max_width, float font_height, float *advance_data){ + Full_Cursor result; + int line_index; + + line_index = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count); + result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); + result = buffer_cursor_seek(buffer, seek_pos(pos), max_width, font_height, + advance_data, result); + + return(result); +} + +internal_4tech Full_Cursor +buffer_cursor_from_line_character(Buffer_Type *buffer, int line, int character, float *wraps, + float max_width, float font_height, float *advance_data){ + Full_Cursor result; + int line_index; + + line_index = line - 1; + if (line_index >= buffer->line_count) line_index = buffer->line_count - 1; + if (line_index < 0) line_index = 0; + + result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); + result = buffer_cursor_seek(buffer, seek_line_char(line, character), + max_width, font_height, advance_data, result); + + return(result); +} + +internal_4tech Full_Cursor +buffer_cursor_from_unwrapped_xy(Buffer_Type *buffer, float x, float y, int round_down, float *wraps, + float max_width, float font_height, float *advance_data){ + Full_Cursor result; + int line_index; + + line_index = (int)(y / font_height); + if (line_index >= buffer->line_count) line_index = buffer->line_count - 1; + if (line_index < 0) line_index = 0; + + result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); + result = buffer_cursor_seek(buffer, seek_unwrapped_xy(x, y, round_down), + max_width, font_height, advance_data, result); + + return(result); +} + +internal_4tech Full_Cursor +buffer_cursor_from_wrapped_xy(Buffer_Type *buffer, float x, float y, int round_down, float *wraps, + float max_width, float font_height, float *advance_data){ + Full_Cursor result; + int line_index; + + line_index = buffer_get_line_index_from_wrapped_y(wraps, y, font_height, 0, buffer->line_count); + result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); + result = buffer_cursor_seek(buffer, seek_wrapped_xy(x, y, round_down), + max_width, font_height, advance_data, result); + + return(result); +} + +internal_4tech void +buffer_invert_edit_shift(Buffer_Type *buffer, Buffer_Edit edit, Buffer_Edit *inverse, char *strings, + int *str_pos, int max, int shift_amount){ + int pos; + int len; + + pos = *str_pos; + len = edit.end - edit.start; + assert_4tech(pos + len <= max); + *str_pos = pos + len; + + inverse->str_start = pos; + inverse->len = len; + inverse->start = edit.start + shift_amount; + inverse->end = edit.start + edit.len + shift_amount; + buffer_stringify(buffer, edit.start, edit.end, strings + pos); +} + +inline_4tech void +buffer_invert_edit(Buffer_Type *buffer, Buffer_Edit edit, Buffer_Edit *inverse, char *strings, + int *str_pos, int max){ + buffer_invert_edit_shift(buffer, edit, inverse, strings, str_pos, max, 0); +} + +#ifndef NON_ABSTRACT_4TECH +typedef struct Buffer_Invert_Batch{ + int i; + int shift_amount; + int len; +} Buffer_Invert_Batch; +#endif + +internal_4tech int +buffer_invert_batch(Buffer_Invert_Batch *state, Buffer_Type *buffer, Buffer_Edit *edits, int count, + Buffer_Edit *inverse, char *strings, int *str_pos, int max){ + Buffer_Edit *edit, *inv_edit; + int shift_amount; + int result; + int i; + + result = 0; + i = state->i; + shift_amount = state->shift_amount; + + edit = edits + i; + inv_edit = inverse + i; + + for (; i < count; ++i, ++edit, ++inv_edit){ + if (*str_pos + edit->end - edit->start <= max){ + buffer_invert_edit_shift(buffer, *edit, inv_edit, strings, str_pos, max, shift_amount); + shift_amount += (edit->len - (edit->end - edit->start)); + } + else{ + result = 1; + state->len = edit->end - edit->start; + } + } + + state->i = i; + state->shift_amount = shift_amount; + + return(result); +} + +struct Buffer_Render_Options{ + b8 show_slash_t; +}; + +internal_4tech Full_Cursor +buffer_get_start_cursor(Buffer_Type *buffer, float *wraps, float scroll_y, + int wrapped, float width, float *advance_data, float font_height){ + Full_Cursor result; + + if (wrapped){ + result = buffer_cursor_from_wrapped_xy(buffer, 0, scroll_y, 0, wraps, + width, font_height, advance_data); + } + else{ + result = buffer_cursor_from_unwrapped_xy(buffer, 0, scroll_y, 0, wraps, + width, font_height, advance_data); + } + + return(result); +} + +internal_4tech void +buffer_get_render_data(Buffer_Type *buffer, Buffer_Render_Item *items, int max, int *count, + float port_x, float port_y, + float scroll_x, float scroll_y, Full_Cursor start_cursor, + int wrapped, + float width, float height, + float *advance_data, float font_height, + Buffer_Render_Options opts){ + + Buffer_Stringify_Type loop; + Buffer_Render_Item *item; + char *data; + int size, end; + float shift_x, shift_y; + float x, y; + int i, item_i; + float ch_width, ch_width_sub; + char ch; + + size = buffer_size(buffer); + + shift_x = port_x - scroll_x; + shift_y = port_y - scroll_y; + if (wrapped) shift_y += start_cursor.wrapped_y; + else shift_y += start_cursor.unwrapped_y; + + x = shift_x; + y = shift_y; + item_i = 0; + item = items + item_i; + + if (advance_data){ + for (loop = buffer_stringify_loop(buffer, start_cursor.pos, size); + buffer_stringify_good(&loop); + buffer_stringify_next(&loop)){ + + end = loop.size + loop.absolute_pos; + data = loop.data - loop.absolute_pos; + + for (i = loop.absolute_pos; i < end; ++i){ + ch = data[i]; + ch_width = measure_character(advance_data, ch); + + if (ch_width + x > width + shift_x && wrapped){ + x = shift_x; + y += font_height; + } + if (y > height + shift_y) goto buffer_get_render_data_end; + + switch (ch){ + case '\n': + write_render_item_inline(item, i, ' ', x, y, advance_data, font_height); + item->flags = 0; + ++item_i; + ++item; + + x = shift_x; + y += font_height; + break; + + case 0: + ch_width = write_render_item_inline(item, i, '\\', x, y, advance_data, font_height); + item->flags = BRFlag_Special_Character; + ++item_i; + ++item; + x += ch_width; + + ch_width = write_render_item_inline(item, i, '0', x, y, advance_data, font_height); + item->flags = BRFlag_Special_Character; + ++item_i; + ++item; + x += ch_width; + break; + + case '\r': + ch_width = write_render_item_inline(item, i, '\\', x, y, advance_data, font_height); + item->flags = BRFlag_Special_Character; + ++item_i; + ++item; + x += ch_width; + + ch_width = write_render_item_inline(item, i, 'r', x, y, advance_data, font_height); + item->flags = BRFlag_Special_Character; + ++item_i; + ++item; + x += ch_width; + break; + + case '\t': + if (opts.show_slash_t){ + ch_width_sub = write_render_item_inline(item, i, '\\', x, y, advance_data, font_height); + item->flags = BRFlag_Special_Character; + ++item_i; + ++item; + + write_render_item_inline(item, i, 't', x + ch_width_sub, y, advance_data, font_height); + item->flags = BRFlag_Special_Character; + ++item_i; + ++item; + } + else{ + write_render_item_inline(item, i, ' ', x, y, advance_data, font_height); + item->flags = 0; + ++item_i; + ++item; + } + x += ch_width; + break; + + default: + write_render_item(item, i, ch, x, y, ch_width, font_height); + item->flags = 0; + ++item_i; + ++item; + x += ch_width; + + break; + } + if (y > height + shift_y) goto buffer_get_render_data_end; + } + } + +buffer_get_render_data_end: + if (y <= height + shift_y || item == items){ + ch = 0; + ch_width = measure_character(advance_data, ' '); + write_render_item(item, size, ch, x, y, ch_width, font_height); + ++item_i; + ++item; + x += ch_width; + } + } + else{ + ch = 0; + ch_width = 0; + write_render_item(item, size, ch, x, y, ch_width, font_height); + ++item_i; + ++item; + x += ch_width; + } + + // TODO(allen): handle this with a control state + assert_4tech(item_i <= max); + *count = item_i; +} + +#ifndef NON_ABSTRACT_4TECH +#define NON_ABSTRACT_4TECH 1 +#endif +// BOTTOM + diff --git a/buffer/4coder_shared.cpp b/buffer/4coder_shared.cpp index c99e3676..092a4764 100644 --- a/buffer/4coder_shared.cpp +++ b/buffer/4coder_shared.cpp @@ -451,6 +451,11 @@ is_alphanumeric_true(char c){ return (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'); } +inline_4tech int +is_alphanumeric(char c){ + return (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_'); +} + inline_4tech int is_upper(char c){ return (c >= 'A' && c <= 'Z'); diff --git a/linux_4ed.cpp b/linux_4ed.cpp index c0979dd8..60b97ed0 100644 --- a/linux_4ed.cpp +++ b/linux_4ed.cpp @@ -43,22 +43,82 @@ #include #include #include +#include +#include #include "4ed_internal.h" #include "4ed_linux_keyboard.cpp" +#include "system_shared.h" #include -void* -LinuxGetMemory(i32 size){ +struct Linux_Vars{ + void *app_code; + void *custom; + + Plat_Settings settings; + System_Functions *system; + App_Functions app; + Custom_API custom_api; + b32 first; + +#if FRED_INTERNAL + Sys_Bubble internal_bubble; +#endif +}; + +globalvar Linux_Vars linuxvars; +globalvar Application_Memory memory_vars; +globalvar Exchange exchange_vars; + +internal +Sys_Get_Memory_Sig(system_get_memory_){ // TODO(allen): Implement without stdlib.h - return (malloc(size)); + void *result = 0; + + if (size != 0){ +#if FRED_INTERNAL + Sys_Bubble *bubble; + + result = malloc(size + sizeof(Sys_Bubble)); + bubble = (Sys_Bubble*)result; + bubble->flags = MEM_BUBBLE_SYS_DEBUG; + bubble->line_number = line_number; + bubble->file_name = file_name; + bubble->size = size; + + // TODO(allen): make Sys_Bubble list thread safe + insert_bubble(&linuxvars.internal_bubble, bubble); + result = bubble + 1; + +#else + result = malloc(size); +#endif + } + + return(result); } -void -LinuxFreeMemory(void *block){ +internal +Sys_Free_Memory_Sig(system_free_memory){ // TODO(allen): Implement without stdlib.h - free(block); + + if (block){ +#if FRED_INTERNAL + Sys_Bubble *bubble; + + bubble = (Sys_Bubble*)block; + --bubble; + Assert((bubble->flags & MEM_BUBBLE_DEBUG_MASK) == MEM_BUBBLE_SYS_DEBUG); + + // TODO(allen): make Sys_Bubble list thread safe + remove_bubble(bubble); + + free(bubble); +#else + free(block); +#endif + } } #if (defined(_BSD_SOURCE) || defined(_SVID_SOURCE)) @@ -92,6 +152,7 @@ Sys_File_Time_Stamp_Sig(system_file_time_stamp){ return(result); } +// TODO(allen): DOES THIS AGREE WITH THE FILESTAMP TIMES? Sys_Time_Sig(system_time){ struct timespec spec; u64 result; @@ -125,10 +186,8 @@ Sys_Set_File_List_Sig(system_set_file_list){ required_size = count + file_count * sizeof(File_Info); if (file_list->block_size < required_size){ - if (file_list->block){ - LinuxFreeMemory(file_list->block); - } - file_list->block = LinuxGetMemory(required_size); + system_free_memory(file_list->block); + file_list->block = system_get_memory(required_size); } file_list->infos = (File_Info*)file_list->block; @@ -143,11 +202,13 @@ Sys_Set_File_List_Sig(system_set_file_list){ fname = entry->d_name; cursor_start = cursor; for (; *fname; ) *cursor++ = *fname++; - *cursor++ = 0; - + + // TODO(allen): detect file/folder status + // (also make sure this even GETS folders!!!) info_ptr->folder = 0; info_ptr->filename.str = cursor_start; info_ptr->filename.size = (i32)(cursor - cursor_start); + *cursor++ = 0; info_ptr->filename.memory_size = info_ptr->filename.size + 1; } } @@ -161,6 +222,181 @@ Sys_Post_Clipboard_Sig(system_post_clipboard){ AllowLocal(str); } +Sys_CLI_Call_Sig(system_cli_call){ + // TODO(allen): Implement + AllowLocal(path); + AllowLocal(script_name); + AllowLocal(cli_out); +} + +Sys_CLI_Begin_Update_Sig(system_cli_begin_update){ + // TODO(allen): Implement + AllowLocal(cli); +} + +Sys_CLI_Update_Step_Sig(system_cli_update_step){ + // TODO(allen): Implement + AllowLocal(cli); + AllowLocal(dest); + AllowLocal(max); + AllowLocal(amount); +} + +Sys_CLI_End_Update_Sig(system_cli_end_update){ + // TODO(allen): Implement + AllowLocal(cli); +} + +Sys_Post_Job_Sig(system_post_job){ + // TODO(allen): Implement + AllowLocal(group_id); + AllowLocal(job); +} + +Sys_Cancel_Job_Sig(system_cancel_job){ + // TODO(allen): Implement + AllowLocal(group_id); + AllowLocal(job_id); +} + +Sys_Acquire_Lock_Sig(system_acquire_lock){ + // TODO(allen): Implement + AllowLocal(id); +} + +Sys_Release_Lock_Sig(system_release_lock){ + // TODO(allen): Implement + AllowLocal(id); +} + +Sys_Grow_Thread_Memory_Sig(system_grow_thread_memory){ + void *old_data; + i32 old_size, new_size; + + system_acquire_lock(CANCEL_LOCK0 + memory->id - 1); + old_data = memory->data; + old_size = memory->size; + new_size = LargeRoundUp(memory->size*2, Kbytes(4)); + memory->data = system_get_memory(new_size); + memory->size = new_size; + if (old_data){ + memcpy(memory->data, old_data, old_size); + system_free_memory(old_data); + } + system_release_lock(CANCEL_LOCK0 + memory->id - 1); +} + +INTERNAL_Sys_Sentinel_Sig(internal_sentinel){ + Bubble *result; +#if FRED_INTERNAL + result = &linuxvars.internal_bubble; +#else + result = 0; +#endif + return(result); +} + +INTERNAL_Sys_Get_Thread_States_Sig(internal_get_thread_states){ + // TODO(allen): Implement + AllowLocal(id); + AllowLocal(running); + AllowLocal(pending); +} + +INTERNAL_Sys_Debug_Message_Sig(internal_debug_message){ + printf("%s", message); +} + +DIRECTORY_HAS_FILE_SIG(system_directory_has_file){ + int result = 0; + // TODO(allen): Implement + AllowLocal(dir); + AllowLocal(filename); + return(result); +} + +DIRECTORY_CD_SIG(system_directory_cd){ + int result = 0; + // TODO(allen): Implement + AllowLocal(dir); + AllowLocal(rel_path); + return(result); +} + +internal +Sys_File_Can_Be_Made(system_file_can_be_made){ + // TODO(allen): Implement + AllowLocal(filename); + return(0); +} + +internal +Sys_Load_File_Sig(system_load_file){ + Data result = {}; + // TODO(allen): Implement + AllowLocal(filename); + return(result); +} + +internal +Sys_Save_File_Sig(system_save_file){ + b32 result = 0; + // TODO(allen): Implement + AllowLocal(filename); + AllowLocal(data); + AllowLocal(size); + return(result); +} + +internal b32 +LinuxLoadAppCode(){ + b32 result = 0; + App_Get_Functions *get_funcs = 0; + + linuxvars.app_code = dlopen("./4ed_app.so", RTLD_LAZY); + if (linuxvars.app_code){ + get_funcs = (App_Get_Functions*) + dlsym(linuxvars.app_code, "app_get_functions"); + } + + if (get_funcs){ + result = 1; + linuxvars.app = get_funcs(); + } + + return(result); +} + +internal void +LinuxLoadSystemCode(){ + linuxvars.system->file_time_stamp = system_file_time_stamp; + linuxvars.system->set_file_list = system_set_file_list; + + linuxvars.system->directory_has_file = system_directory_has_file; + linuxvars.system->directory_cd = system_directory_cd; + + linuxvars.system->post_clipboard = system_post_clipboard; + linuxvars.system->time = system_time; + + linuxvars.system->cli_call = system_cli_call; + linuxvars.system->cli_begin_update = system_cli_begin_update; + linuxvars.system->cli_update_step = system_cli_update_step; + linuxvars.system->cli_end_update = system_cli_end_update; + + linuxvars.system->post_job = system_post_job; + linuxvars.system->cancel_job = system_cancel_job; + linuxvars.system->grow_thread_memory = system_grow_thread_memory; + linuxvars.system->acquire_lock = system_acquire_lock; + linuxvars.system->release_lock = system_release_lock; + + linuxvars.system->internal_sentinel = internal_sentinel; + linuxvars.system->internal_get_thread_states = internal_get_thread_states; + linuxvars.system->internal_debug_message = internal_debug_message; +} + +#include "system_shared.cpp" +#include "4ed_rendering.cpp" + // NOTE(allen): Thanks to Casey for providing the linux OpenGL launcher. static bool ctxErrorOccurred = false; static int XInput2OpCode = 0; @@ -490,8 +726,77 @@ InitializeXInput(Display *dpy, Window XWindow) } int -main(int ArgCount, char **Args) +main(int argc, char **argv) { + linuxvars = {}; + exchange_vars = {}; + +#if FRED_INTERNAL + linuxvars.internal_bubble.next = &linuxvars.internal_bubble; + linuxvars.internal_bubble.prev = &linuxvars.internal_bubble; + linuxvars.internal_bubble.flags = MEM_BUBBLE_SYS_DEBUG; +#endif + + if (!LinuxLoadAppCode()){ + // TODO(allen): Failed to load app code, serious problem. + return 99; + } + + System_Functions system_; + System_Functions *system = &system_; + linuxvars.system = system; + LinuxLoadSystemCode(); + + memory_vars.vars_memory_size = Mbytes(2); + memory_vars.vars_memory = system_get_memory(memory_vars.vars_memory_size); + memory_vars.target_memory_size = Mbytes(512); + memory_vars.target_memory = system_get_memory(memory_vars.target_memory_size); + + String current_directory; + i32 curdir_req, curdir_size; + char *curdir_mem; + + curdir_req = (1 << 9); + curdir_mem = (char*)system_get_memory(curdir_req); + for (; getcwd(curdir_mem, curdir_req) == 0 && curdir_req < (1 << 13);){ + system_free_memory(curdir_mem); + curdir_req *= 4; + curdir_mem = (char*)system_get_memory(curdir_req); + } + + if (curdir_req >= (1 << 13)){ + // TODO(allen): fuckin' bullshit string APIs makin' me pissed + return 57; + } + + for (curdir_size = 0; curdir_mem[curdir_size]; ++curdir_size); + + current_directory = make_string(curdir_mem, curdir_size, curdir_req); + + Command_Line_Parameters clparams; + clparams.argv = argv; + clparams.argc = argc; + + char **files; + i32 *file_count; + i32 output_size; + + output_size = + linuxvars.app.read_command_line(system, + &memory_vars, + current_directory, + &linuxvars.settings, + &files, &file_count, + clparams); + + if (output_size > 0){ + // TODO(allen): crt free version + printf("%.*s", output_size, (char*)memory_vars.target_memory); + } + if (output_size != 0) return 0; + + system_filter_real_files(files, file_count); + Display *XDisplay = XOpenDisplay(0); if(XDisplay && GLXSupportsModernContexts(XDisplay)) { diff --git a/system_shared.cpp b/system_shared.cpp new file mode 100644 index 00000000..c18c48c8 --- /dev/null +++ b/system_shared.cpp @@ -0,0 +1,28 @@ +/* + * Mr. 4th Dimention - Allen Webster + * + * 09.02.2016 + * + * Shared system functions + * + */ + +// TOP + +internal void +system_filter_real_files(char **files, i32 *file_count){ + i32 i, j; + i32 end; + + end = *file_count; + for (i = 0, j = 0; i < end; ++i){ + if (system_file_can_be_made(files[i])){ + files[j] = files[i]; + ++j; + } + } + *file_count = j; +} + +// BOTTOM + diff --git a/system_shared.h b/system_shared.h new file mode 100644 index 00000000..2f3dec02 --- /dev/null +++ b/system_shared.h @@ -0,0 +1,32 @@ +/* + * Mr. 4th Dimention - Allen Webster + * + * 09.02.2016 + * + * Shared system functions + * + */ + +// TOP + +// NOTE(allen): This serves as a list of functions to implement +// in addition to those in 4ed_system.h These are not exposed to +// the application code, but system_shared.cpp and 4ed_rendering.cpp +// rely on the functions listed here. + +#define Sys_Get_Memory_Sig(name) void* name(i32 size, i32 line_number, char *file_name) +#define Sys_Free_Memory_Sig(name) void name(void *block) +#define Sys_File_Can_Be_Made(name) b32 name(char *filename) +#define Sys_Load_File_Sig(name) Data name(char *filename) +#define Sys_Save_File_Sig(name) b32 name(char *filename, char *data, i32 size) + +internal Sys_Get_Memory_Sig(system_get_memory_); +internal Sys_Free_Memory_Sig(system_free_memory); +internal Sys_File_Can_Be_Made(system_file_can_be_made); +internal Sys_Load_File_Sig(system_load_file); +internal Sys_Save_File_Sig(system_save_file); + +#define system_get_memory(size) system_get_memory_((size), __LINE__, __FILE__) + +// BOTTOM + diff --git a/test/experiment.cpp b/test/experiment.cpp index eaccba09..c9a27063 100644 --- a/test/experiment.cpp +++ b/test/experiment.cpp @@ -1,157 +1,74 @@ -/* - * Mr. 4th Dimention - Allen Webster - * - * 21.1.2015 - * - * Test for CPP lexer & parser layer for project codename "4ed" - * - */ - -// TOP - -#include "../4ed_meta.h" - -#define Debug(x) x - -#include "../4cpp_types.h" -#define FCPP_STRING_IMPLEMENTATION -#include "../4cpp_string.h" -#define FCPP_LEXER_IMPLEMENTATION -#include "../4cpp_lexer.h" -#include "../4cpp_preprocessor.cpp" - -#include -#include -#include - -internal bool -system_is_absoute_path(char *path){ - bool is_absolute = 0; - char c = 1; - while (c){ - c = *path++; - if (c == ':'){ - is_absolute = 1; - break; - } - } - return is_absolute; -} - -#undef Assert -#undef TentativeAssert - -#define Assert assert -#define TentativeAssert assert - -Cpp_File -quickie_file(char *filename){ - Cpp_File result = {}; - - FILE *file = fopen(filename, "rb"); - if (file){ - fseek(file, 0, SEEK_END); - result.size = ftell(file); - if (result.size > 0){ - fseek(file, 0, SEEK_SET); - result.data = (char*)malloc(result.size); - fread(result.data, 1, result.size, file); - } - fclose(file); - } - - return result; -} - -inline Cpp_File -quickie_file(String filename){ - assert(filename.size < 511); - char buffer[512]; - memcpy(buffer, filename.str, filename.size); - buffer[filename.size] = 0; - return quickie_file(buffer); -} - -internal void -preproc_init(Cpp_PP_State *state, Cpp_PP_Definitions *definitions){ - *state = {}; - state->max = (32 << 10); - state->base = (byte*)malloc(state->max); - memset(state->base, 0, state->max); - - *definitions = {}; - definitions->max = 128 << 10; - definitions->items = (byte*)malloc(definitions->max); - memset(definitions->items, 0, definitions->max); - - definitions->table.count = 0; - definitions->table.max = 4 << 10; - definitions->table.entries = (Cpp_PP_Table_Entry*) - malloc(definitions->table.max*sizeof(Cpp_PP_Table_Entry)); - memset(definitions->table.entries, 0, definitions->table.max*sizeof(Cpp_PP_Table_Entry)); - - int str_size = 16 << 10; - int token_size = 16 << 10; - void *str_mem = malloc(str_size); - void *token_mem = malloc(token_size); - cpp_preproc_set_spare_space(definitions, - str_size, str_mem, - token_size, token_mem); -} - -int main(int argc, char **argv){ - if (argc < 2){ - printf("usage: %s \n", argv[0]); - return 1; - } - char *TEST_FILE = argv[1]; - - Cpp_File target_file; - target_file = quickie_file(TEST_FILE); - - if (target_file.data == 0){ - printf("could not open file %s\n", TEST_FILE); - return 1; - } - - Cpp_Token_Stack target_tokens = {}; - cpp_lex_file(target_file, &target_tokens); - - Cpp_PP_State state = {}; - Cpp_PP_Definitions definitions = {}; - - preproc_init(&state, &definitions); - - cpp_preproc_target(&state, &definitions, target_file, target_tokens); - - for (Cpp_PP_Step step = {}; - step.finished == 0;){ - step = cpp_preproc_step_nonalloc(&state, &definitions); - - if (step.error_code){ - printf("error: %s\n", cpp_preproc_error_str(step.error_code)); - if (cpp_preproc_recommend_termination(step.error_code)){ - return 1; - } - } - - if (step.mem_size != 0){ - void *memory = malloc(step.mem_size); - memory = cpp_preproc_provide_memory(&state, &definitions, - step.mem_type, step.mem_size, memory); - free(memory); - } - - if (step.emit){ - Cpp_File file = *cpp_preproc_get_file(&definitions, step.file_index); - Cpp_Token token = *cpp_preproc_get_token(&definitions, step.file_index, step.token_index); - - printf("TOKEN: %.*s\n", token.size, file.data + token.start); - } - } - - return 0; -} - -// BOTTOM - +/* + * Mr. 4th Dimention - Allen Webster + * + * 21.1.2015 + * + * Test for CPP lexer & parser layer for project codename "4ed" + * + */ + +// TOP + +#include "../4ed_meta.h" + +#define FCPP_STRING_IMPLEMENTATION +#include "../4coder_string.h" + +#include "../4cpp_types.h" +#define FCPP_LEXER_IMPLEMENTATION +#include "../4cpp_lexer.h" +#include "../4cpp_preprocessor.cpp" + +#include +#include + +Data +file_dump(char *filename){ + Data result; + FILE *file; + result = {}; + file = fopen(filename, "rb"); + if (file){ + fseek(file, 0, SEEK_END); + result.size = ftell(file); + fseek(file, 0, SEEK_SET); + result.data = (byte*)malloc(result.size); + fread(result.data, 1, result.size, file); + fclose(file); + } + return(result); +} + +int main(int argc, char **argv){ + Data target_file; + Cpp_File file; + Cpp_Token_Stack tokens; + Cpp_Token *token; + int i; + + if (argc != 2){ + printf("usage: %s \n", argv[0]); + exit(1); + } + + target_file = file_dump(argv[1]); + if (target_file.data == 0){ + printf("couldn't open file %s\n", argv[1]); + exit(1); + } + + tokens = cpp_make_token_stack(1 << 10); + + file = data_as_cpp_file(target_file); + cpp_lex_file(file, &tokens); + + token = tokens.tokens; + for (i = 0; i < tokens.count; ++i, ++token){ + printf("%.*s\n", token->size, file.data + token->start); + } + + return(0); +} + +// BOTTOM + diff --git a/vc120.pdb b/vc120.pdb index dfa67a6e..da601c9f 100644 Binary files a/vc120.pdb and b/vc120.pdb differ diff --git a/win32_4ed.cpp b/win32_4ed.cpp index c0602d6e..c6b42d29 100644 --- a/win32_4ed.cpp +++ b/win32_4ed.cpp @@ -35,6 +35,7 @@ #include "4ed_dll_reader.cpp" #include "4ed_internal.h" #include "4ed_win32_keyboard.cpp" +#include "system_shared.h" #define FPS 30 #define frame_useconds (1000000 / FPS) @@ -59,7 +60,6 @@ struct Thread_Group{ }; #define UseWinDll 1 -#define UseThreadMemory 1 struct Control_Keys{ b8 l_ctrl; @@ -130,9 +130,7 @@ struct Win32_Vars{ HANDLE locks[LOCK_COUNT]; HANDLE DEBUG_sysmem_lock; -#if UseThreadMemory Thread_Memory *thread_memory; -#endif u64 performance_frequency; u64 start_pcount; @@ -179,6 +177,8 @@ INTERNAL_system_debug_message(char *message){ #endif +// TODO(allen): Transition towards using system_shared functions + internal void* Win32GetMemory_(i32 size, i32 line_number, char *file_name){ void *ptr = 0; @@ -202,6 +202,11 @@ Win32GetMemory_(i32 size, i32 line_number, char *file_name){ return ptr; } +internal +Sys_Get_Memory_Sig(system_get_memory_){ + return(Win32GetMemory_(size, line_number, file_name)); +} + #define Win32GetMemory(size) Win32GetMemory_(size, __LINE__, __FILE__) internal void @@ -291,8 +296,6 @@ system_load_file(char *filename){ return result; } -#include "4ed_rendering.cpp" - internal b32 system_save_file(char *filename, void *data, i32 size){ HANDLE file; @@ -392,10 +395,7 @@ Sys_Set_File_List_Sig(system_set_file_list){ i32 required_size = count + file_count * sizeof(File_Info); if (file_list->block_size < required_size){ - if (file_list->block){ - Win32FreeMemory(file_list->block); - } - + Win32FreeMemory(file_list->block); file_list->block = Win32GetMemory(required_size); } @@ -536,15 +536,6 @@ Sys_Release_Lock_Sig(system_release_lock){ ReleaseSemaphore(win32vars.locks[id], 1, 0); } -internal void -Win32RedrawScreen(HDC hdc){ - system_acquire_lock(RENDER_LOCK); - launch_rendering(&win32vars.target); - system_release_lock(RENDER_LOCK); - glFlush(); - SwapBuffers(hdc); -} - internal void Win32RedrawFromUpdate(){ SendMessage( @@ -619,13 +610,10 @@ ThreadProc(LPVOID lpParameter){ if (safe_running_thread == THREAD_NOT_ASSIGNED){ thread->job_id = full_job->id; thread->running = 1; -#if UseThreadMemory Thread_Memory *thread_memory = 0; -#endif // TODO(allen): remove memory_request if (full_job->job.memory_request != 0){ -#if UseThreadMemory thread_memory = win32vars.thread_memory + thread->id - 1; if (thread_memory->size < full_job->job.memory_request){ if (thread_memory->data){ @@ -635,7 +623,6 @@ ThreadProc(LPVOID lpParameter){ thread_memory->data = Win32GetMemory(new_size); thread_memory->size = new_size; } -#endif } full_job->job.callback(win32vars.system, thread, thread_memory, &exchange_vars.thread, full_job->job.data); @@ -708,13 +695,15 @@ Sys_Cancel_Job_Sig(system_cancel_job){ } } -#if UseThreadMemory internal void system_grow_thread_memory(Thread_Memory *memory){ + void *old_data; + i32 old_size, new_size; + system_acquire_lock(CANCEL_LOCK0 + memory->id - 1); - void *old_data = memory->data; - i32 old_size = memory->size; - i32 new_size = LargeRoundUp(memory->size*2, Kbytes(4)); + old_data = memory->data; + old_size = memory->size; + new_size = LargeRoundUp(memory->size*2, Kbytes(4)); memory->data = Win32GetMemory(new_size); memory->size = new_size; if (old_data){ @@ -723,7 +712,6 @@ system_grow_thread_memory(Thread_Memory *memory){ } system_release_lock(CANCEL_LOCK0 + memory->id - 1); } -#endif #if FRED_INTERNAL internal void @@ -911,15 +899,13 @@ Font_Load_Sig(system_draw_font_load){ internal b32 Win32LoadAppCode(){ b32 result = 0; - + App_Get_Functions *get_funcs = 0; + #if UseWinDll win32vars.app_code = LoadLibraryA("4ed_app.dll"); if (win32vars.app_code){ - result = 1; - App_Get_Functions *get_funcs = (App_Get_Functions*) + get_funcs = (App_Get_Functions*) GetProcAddress(win32vars.app_code, "app_get_functions"); - - win32vars.app = get_funcs(); } #else @@ -944,8 +930,7 @@ Win32LoadAppCode(){ PAGE_EXECUTE_READ, &extra_); - result = 1; - App_Get_Functions *get_functions = (App_Get_Functions*) + get_funcs = (App_Get_Functions*) dll_load_function(&win32vars.app_dll, "app_get_functions", 17); } else{ @@ -962,6 +947,11 @@ Win32LoadAppCode(){ #endif + if (get_funcs){ + result = 1; + win32vars.app = get_funcs(); + } + return result; } @@ -992,6 +982,18 @@ Win32LoadSystemCode(){ win32vars.system->internal_debug_message = INTERNAL_system_debug_message; } +#include "system_shared.cpp" +#include "4ed_rendering.cpp" + +internal void +Win32RedrawScreen(HDC hdc){ + system_acquire_lock(RENDER_LOCK); + launch_rendering(&win32vars.target); + system_release_lock(RENDER_LOCK); + glFlush(); + SwapBuffers(hdc); +} + void ex__file_insert(File_Slot *pos, File_Slot *file){ file->next = pos->next; @@ -1564,7 +1566,7 @@ main(int argc, char **argv){ DWORD required = GetCurrentDirectory(0, 0); required += 1; required *= 4; - char *current_directory_mem = (char*)Win32GetMemory(required); + char *current_directory_mem = (char*)system_get_memory(required); DWORD written = GetCurrentDirectory(required, current_directory_mem); String current_directory = make_string(current_directory_mem, written, required); @@ -1590,24 +1592,13 @@ main(int argc, char **argv){ // if (output_size > 0){ - // TODO + // TODO(allen): crt free version printf("%.*s", output_size, memory_vars.target_memory); - } if (output_size != 0) return 0; FreeConsole(); - if (files){ - i32 i, j; - i32 end = *file_count; - for (i = 0, j = 0; i < end; ++i){ - if (system_file_can_be_made(files[i])){ - files[j] = files[i]; - ++j; - } - } - *file_count = j; - } + system_filter_real_files(files, file_count); LARGE_INTEGER lpf; QueryPerformanceFrequency(&lpf); @@ -1646,11 +1637,8 @@ main(int argc, char **argv){ win32vars.groups[BACKGROUND_THREADS].threads = background; win32vars.groups[BACKGROUND_THREADS].count = ArrayCount(background); - -#if UseThreadMemory Thread_Memory thread_memory[ArrayCount(background)]; win32vars.thread_memory = thread_memory; -#endif exchange_vars.thread.queues[BACKGROUND_THREADS].semaphore = Win32GenHandle( @@ -1662,11 +1650,9 @@ main(int argc, char **argv){ Thread_Context *thread = win32vars.groups[BACKGROUND_THREADS].threads + i; thread->id = i + 1; -#if UseThreadMemory Thread_Memory *memory = win32vars.thread_memory + i; *memory = {}; memory->id = thread->id; -#endif thread->queue = &exchange_vars.thread.queues[BACKGROUND_THREADS]; thread->handle = CreateThread(0, 0, ThreadProc, thread, creation_flag, (LPDWORD)&thread->windows_id); @@ -1864,7 +1850,9 @@ main(int argc, char **argv){ &memory_vars, &exchange_vars, &win32vars.key_codes, win32vars.clipboard_contents, current_directory, win32vars.custom_api); - + + system_free_memory(current_directory.str); + win32vars.input_chunk.pers.keep_playing = 1; win32vars.first = 1; timeBeginPeriod(1);