From 7a23c72d112202a5f87b667b6d2a81d77b5469a7 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 16 Sep 2016 10:10:45 -0400 Subject: [PATCH] switched experimental operations over to token streams --- 4coder_default_include.cpp | 14 ++ power/4coder_experiments.cpp | 433 +++++++++++++++++++---------------- 2 files changed, 254 insertions(+), 193 deletions(-) diff --git a/4coder_default_include.cpp b/4coder_default_include.cpp index bbded083..4c406e2b 100644 --- a/4coder_default_include.cpp +++ b/4coder_default_include.cpp @@ -312,6 +312,7 @@ init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary stream->count = count; stream->start = round_down(pos, count); stream->end = round_up(pos, count); + stream->token_count = token_count; app->buffer_read_tokens(app, buffer, stream->start, stream->end, stream->base_tokens); stream->tokens = stream->base_tokens - stream->start; @@ -321,6 +322,19 @@ init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary return(result); } +static Stream_Tokens +begin_temp_stream_token(Stream_Tokens *stream){ + return(*stream); +} + +static void +end_temp_stream_token(Stream_Tokens *stream, Stream_Tokens temp){ + if (stream->start != temp.start || stream->end != temp.end){ + Application_Links *app = stream->app; + app->buffer_read_tokens(app, stream->buffer, stream->start, stream->end, stream->base_tokens); + } +} + static bool32 forward_stream_tokens(Stream_Tokens *stream){ Application_Links *app = stream->app; diff --git a/power/4coder_experiments.cpp b/power/4coder_experiments.cpp index bad55531..6164b822 100644 --- a/power/4coder_experiments.cpp +++ b/power/4coder_experiments.cpp @@ -319,127 +319,147 @@ CUSTOM_COMMAND_SIG(rename_parameter){ Partition *part = &global_part; Temp_Memory temp = begin_temp_memory(part); - Cpp_Token_Array array = buffer_get_all_tokens(app, part, &buffer); - Cpp_Get_Token_Result result = cpp_get_token(&array, view.cursor.pos); - if (!result.in_whitespace && result.token_index < array.count){ - Cpp_Token *token_ptr = &array.tokens[result.token_index]; - Cpp_Token *token_end = array.tokens + array.count - 1; - if (token_ptr->type == CPP_TOKEN_IDENTIFIER){ - char old_lexeme_base[128]; - String old_lexeme = make_fixed_width_string(old_lexeme_base); - if (token_ptr->size < sizeof(old_lexeme_base)){ + + Cpp_Get_Token_Result result; + if (app->buffer_get_token_index(app, &buffer, view.cursor.pos, &result)){ + if (!result.in_whitespace){ + Cpp_Token stream_space[32]; + Stream_Tokens stream = {0}; + + if (init_stream_tokens(&stream, app, &buffer, result.token_index, stream_space, 32)){ + int32_t token_index = result.token_index; + Cpp_Token token = stream.tokens[token_index]; - Cpp_Token original_token = *token_ptr; - old_lexeme.size = token_ptr->size; - app->buffer_read_range(app, &buffer, token_ptr->start, - token_ptr->start+token_ptr->size, - old_lexeme.str); - - int32_t proc_body_found = 0; - for (++token_ptr; token_ptr < token_end; ++token_ptr){ - switch (token_ptr->type){ - case CPP_TOKEN_BRACE_OPEN: - { - proc_body_found = 1; - goto doublebreak; - }break; + if (token.type == CPP_TOKEN_IDENTIFIER){ + + char old_lexeme_base[128]; + String old_lexeme = make_fixed_width_string(old_lexeme_base); + + if (token.size < sizeof(old_lexeme_base)){ + Cpp_Token original_token = token; + old_lexeme.size = token.size; + app->buffer_read_range(app, &buffer, token.start, + token.start+token.size, + old_lexeme.str); - case CPP_TOKEN_BRACE_CLOSE: - case CPP_TOKEN_PARENTHESE_OPEN: - { - goto doublebreak; - }break; - } - } - doublebreak:; - - if (proc_body_found){ - - Query_Bar with; - char with_space[1024]; - with.prompt = make_lit_string("New Name: "); - with.string = make_fixed_width_string(with_space); - if (!query_user_string(app, &with)) return; - - String replace_string = with.string; - - Cpp_Token *token_start_ptr = token_ptr+1; - - Buffer_Edit *edits = (Buffer_Edit*)partition_current(part); - int32_t edit_max = (partition_remaining(part))/sizeof(Buffer_Edit); - int32_t edit_count = 0; - - if (edit_max >= 1){ - Buffer_Edit edit; - edit.str_start = 0; - edit.len = replace_string.size; - edit.start = original_token.start; - edit.end = original_token.start + original_token.size; + int32_t proc_body_found = 0; + int32_t still_looping = 0; - edits[edit_count] = edit; - ++edit_count; - } - - int32_t nesting_level = 0; - int32_t closed_correctly = 0; - token_ptr = token_start_ptr; - for (; token_ptr < token_end; ++token_ptr){ - switch (token_ptr->type){ - case CPP_TOKEN_IDENTIFIER: - { - if (token_ptr->size == old_lexeme.size){ - char other_lexeme_base[128]; - String other_lexeme = make_fixed_width_string(other_lexeme_base); - other_lexeme.size = old_lexeme.size; - app->buffer_read_range(app, &buffer, token_ptr->start, - token_ptr->start+token_ptr->size, - other_lexeme.str); + ++token_index; + do{ + for (; token_index < stream.end; ++token_index){ + Cpp_Token *token_ptr = stream.tokens + token_index; + switch (token_ptr->type){ + case CPP_TOKEN_BRACE_OPEN: + { + proc_body_found = 1; + goto doublebreak; + }break; - if (match(old_lexeme, other_lexeme)){ - Buffer_Edit edit; - edit.str_start = 0; - edit.len = replace_string.size; - edit.start = token_ptr->start; - edit.end = token_ptr->start + token_ptr->size; + case CPP_TOKEN_BRACE_CLOSE: + case CPP_TOKEN_PARENTHESE_OPEN: + { + goto doublebreak; + }break; + } + } + still_looping = forward_stream_tokens(&stream); + }while(still_looping); + doublebreak:; + + if (proc_body_found){ + + Query_Bar with; + char with_space[1024]; + with.prompt = make_lit_string("New Name: "); + with.string = make_fixed_width_string(with_space); + if (!query_user_string(app, &with)) return; + + String replace_string = with.string; + + Buffer_Edit *edits = (Buffer_Edit*)partition_current(part); + int32_t edit_max = (partition_remaining(part))/sizeof(Buffer_Edit); + int32_t edit_count = 0; + + if (edit_max >= 1){ + Buffer_Edit edit; + edit.str_start = 0; + edit.len = replace_string.size; + edit.start = original_token.start; + edit.end = original_token.start + original_token.size; + + edits[edit_count] = edit; + ++edit_count; + } + + int32_t nesting_level = 0; + int32_t closed_correctly = 0; + ++token_index; + still_looping = 0; + do{ + for (; token_index < stream.end; ++token_index){ + Cpp_Token *token_ptr = stream.tokens + token_index; + switch (token_ptr->type){ + case CPP_TOKEN_IDENTIFIER: + { + if (token_ptr->size == old_lexeme.size){ + char other_lexeme_base[128]; + String other_lexeme = make_fixed_width_string(other_lexeme_base); + other_lexeme.size = old_lexeme.size; + app->buffer_read_range(app, &buffer, token_ptr->start, + token_ptr->start+token_ptr->size, + other_lexeme.str); + + if (match(old_lexeme, other_lexeme)){ + Buffer_Edit edit; + edit.str_start = 0; + edit.len = replace_string.size; + edit.start = token_ptr->start; + edit.end = token_ptr->start + token_ptr->size; + + if (edit_count < edit_max){ + edits[edit_count] = edit; + ++edit_count; + } + else{ + goto doublebreak2; + } + } + } + }break; - if (edit_count < edit_max){ - edits[edit_count] = edit; - ++edit_count; - } - else{ - goto doublebreak2; - } + case CPP_TOKEN_BRACE_OPEN: + { + ++nesting_level; + }break; + + case CPP_TOKEN_BRACE_CLOSE: + { + if (nesting_level == 0){ + closed_correctly = 1; + goto doublebreak2; + } + else{ + --nesting_level; + } + }break; } } - }break; + still_looping = forward_stream_tokens(&stream); + }while(still_looping); + doublebreak2:; - case CPP_TOKEN_BRACE_OPEN: - { - ++nesting_level; - }break; - - case CPP_TOKEN_BRACE_CLOSE: - { - if (nesting_level == 0){ - closed_correctly = 1; - goto doublebreak2; - } - else{ - --nesting_level; - } - }break; + if (closed_correctly){ + app->buffer_batch_edit(app, &buffer, replace_string.str, replace_string.size, + edits, edit_count, BatchEdit_Normal); + } } } - doublebreak2:; - - if (closed_correctly){ - app->buffer_batch_edit(app, &buffer, replace_string.str, replace_string.size, - edits, edit_count, BatchEdit_Normal); - } } } } } + end_temp_memory(temp); } @@ -451,108 +471,135 @@ CUSTOM_COMMAND_SIG(write_explicit_enum_values){ Partition *part = &global_part; Temp_Memory temp = begin_temp_memory(part); - Cpp_Token_Array array = buffer_get_all_tokens(app, part, &buffer); - Cpp_Get_Token_Result result = cpp_get_token(&array, view.cursor.pos); - if (!result.in_whitespace && result.token_index < array.count){ - Cpp_Token *token_ptr = &array.tokens[result.token_index]; - Cpp_Token *token_end = array.tokens + array.count; - if (token_ptr->type == CPP_TOKEN_BRACE_OPEN){ + + Cpp_Get_Token_Result result; + if (app->buffer_get_token_index(app, &buffer, view.cursor.pos, &result)){ + if (!result.in_whitespace){ + Cpp_Token stream_space[32]; + Stream_Tokens stream = {0}; - ++token_ptr; - - int32_t closed_correctly = 0; - Cpp_Token *token_seeker = token_ptr; - for (; token_seeker < token_end; ++token_seeker){ - switch (token_seeker->type){ - case CPP_TOKEN_BRACE_CLOSE: - closed_correctly = 1; - goto finished_seek; + if (init_stream_tokens(&stream, app, &buffer, result.token_index, stream_space, 32)){ + int32_t token_index = result.token_index; + Cpp_Token token = stream.tokens[token_index]; + + if (token.type == CPP_TOKEN_BRACE_OPEN){ - case CPP_TOKEN_BRACE_OPEN: - goto finished_seek; - } - } - finished_seek:; - - if (closed_correctly){ - int32_t count_estimate = 1 + (int32_t)(token_seeker - token_ptr)/2; - - Buffer_Edit *edits = push_array(part, Buffer_Edit, count_estimate); - int32_t edit_count = 0; - - char *string_base = (char*)partition_current(part); - String string = make_string(string_base, 0, partition_remaining(part)); - - int32_t value = 0; - closed_correctly = 0; - for (;token_ptr < token_end; ++token_ptr){ - switch (token_ptr->type){ - case CPP_TOKEN_IDENTIFIER: - { - int32_t edit_start = token_ptr->start + token_ptr->size; - int32_t edit_stop = edit_start; - - int32_t edit_is_good = 0; - for (++token_ptr; token_ptr < token_end; ++token_ptr){ + ++token_index; + + int32_t closed_correctly = 0; + int32_t seeker_index = token_index; + Stream_Tokens seek_stream = begin_temp_stream_token(&stream); + + int32_t still_looping = 0; + do{ + for (; seeker_index < seek_stream.end; ++seeker_index){ + Cpp_Token *token_seeker = seek_stream.tokens + seeker_index; + switch (token_seeker->type){ + case CPP_TOKEN_BRACE_CLOSE: + closed_correctly = 1; + goto finished_seek; + + case CPP_TOKEN_BRACE_OPEN: + goto finished_seek; + } + } + still_looping = forward_stream_tokens(&seek_stream); + }while(still_looping); + finished_seek:; + end_temp_stream_token(&stream, seek_stream); + + if (closed_correctly){ + int32_t count_estimate = 1 + (seeker_index - token_index)/2; + + Buffer_Edit *edits = push_array(part, Buffer_Edit, count_estimate); + int32_t edit_count = 0; + + char *string_base = (char*)partition_current(part); + String string = make_string(string_base, 0, partition_remaining(part)); + + int32_t value = 0; + closed_correctly = 0; + still_looping = 0; + do{ + for (;token_index < stream.end; ++token_index){ + Cpp_Token *token_ptr = stream.tokens + token_index; switch (token_ptr->type){ - case CPP_TOKEN_COMMA: + case CPP_TOKEN_IDENTIFIER: { - edit_stop = token_ptr->start; - edit_is_good = 1; - goto good_edit; + int32_t edit_start = token_ptr->start + token_ptr->size; + int32_t edit_stop = edit_start; + + int32_t edit_is_good = 0; + ++token_index; + do{ + for (; token_index < stream.end; ++token_index){ + token_ptr = stream.tokens + token_index; + switch (token_ptr->type){ + case CPP_TOKEN_COMMA: + { + edit_stop = token_ptr->start; + edit_is_good = 1; + goto good_edit; + }break; + + case CPP_TOKEN_BRACE_CLOSE: + { + edit_stop = token_ptr->start; + closed_correctly = 1; + edit_is_good = 1; + goto good_edit; + }break; + } + } + still_looping = forward_stream_tokens(&stream); + }while(still_looping); + + good_edit:; + if (edit_is_good){ + int32_t str_pos = string.size; + + append(&string, " = "); + append_int_to_str(&string, value); + if (closed_correctly){ + append(&string, "\n"); + } + ++value; + + int32_t str_size = string.size - str_pos; + + Buffer_Edit edit; + edit.str_start = str_pos; + edit.len = str_size; + edit.start = edit_start; + edit.end = edit_stop; + + assert(edit_count < count_estimate); + edits[edit_count] = edit; + ++edit_count; + } + if (!edit_is_good || closed_correctly){ + goto finished; + } }break; case CPP_TOKEN_BRACE_CLOSE: { - edit_stop = token_ptr->start; closed_correctly = 1; - edit_is_good = 1; - goto good_edit; + goto finished; }break; } } - good_edit:; - if (edit_is_good){ - int32_t str_pos = string.size; - - append(&string, " = "); - append_int_to_str(&string, value); - if (closed_correctly){ - append(&string, "\n"); - } - ++value; - - int32_t str_size = string.size - str_pos; - - Buffer_Edit edit; - edit.str_start = str_pos; - edit.len = str_size; - edit.start = edit_start; - edit.end = edit_stop; - - assert(edit_count < count_estimate); - edits[edit_count] = edit; - ++edit_count; - } - if (!edit_is_good || closed_correctly){ - goto finished; - } - }break; + still_looping = forward_stream_tokens(&stream); + }while(still_looping); - case CPP_TOKEN_BRACE_CLOSE: - { - closed_correctly = 1; - goto finished; - }break; + finished:; + if (closed_correctly){ + app->buffer_batch_edit(app, &buffer, string_base, string.size, + edits, edit_count, BatchEdit_Normal); + } } } - - finished:; - if (closed_correctly){ - app->buffer_batch_edit(app, &buffer, string_base, string.size, - edits, edit_count, BatchEdit_Normal); - } } } }