ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

relex API allows starting from any chunk, fixed null check bug

This commit is contained in:
Allen Webster 2016-09-18 10:41:15 -04:00
parent 6168528c0e
commit 3a7f4db69e
3 changed files with 101 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
4coder_API.html
View File

File diff suppressed because one or more lines are too long

95
4cpp_lexer.h
View File

@ -800,7 +800,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, int32_t siz
break; break;
} }
if (chunk[S.pos-1] == 0){ if (S.pos > S.chunk_pos && chunk[S.pos-1] == 0){
--S.pos; --S.pos;
} }
@ -1207,6 +1207,40 @@ cpp_relex_init(Cpp_Token_Array *array, int32_t start_pos, int32_t end_pos, int32
return(state); return(state);
} }
FCPP_LINK int32_t
cpp_relex_start_position(Cpp_Relex_Data *S_ptr){
int32_t result = S_ptr->relex_start_position;
return(result);
}
FCPP_LINK void
cpp_relex_declare_first_chunk_position(Cpp_Relex_Data *S_ptr, int32_t position){
S_ptr->lex.chunk_pos = position;
}
FCPP_LINK int32_t
cpp_relex_is_start_chunk(Cpp_Relex_Data *S_ptr, char *chunk, int32_t chunk_size){
int32_t pos = S_ptr->relex_start_position;
int32_t start = S_ptr->lex.chunk_pos;
int32_t end = start + chunk_size;
int32_t good_chunk = 0;
if (start <= pos && pos < end){
good_chunk = 1;
}
else{
if (chunk == 0){
good_chunk = 1;
S_ptr->lex.chunk_pos = pos;
}
else{
S_ptr->lex.chunk_pos += chunk_size;
}
}
return(good_chunk);
}
// duff-routine defines // duff-routine defines
#define DrCase(PC) case PC: goto resumespot_##PC #define DrCase(PC) case PC: goto resumespot_##PC
@ -1232,41 +1266,40 @@ cpp_relex_step(Cpp_Relex_Data *S_ptr, char *chunk, int32_t chunk_size, int32_t f
cpp_shift_token_starts(array, S.end_token_index, S.character_shift_amount); cpp_shift_token_starts(array, S.end_token_index, S.character_shift_amount);
S.end_token = cpp_index_array(array, full_size, S.end_token_index); S.end_token = cpp_index_array(array, full_size, S.end_token_index);
if (S.relex_start_position < full_size){ // TODO(allen): This can be better I suspect.
// TODO(allen): This can be better I suspect. for (;;){
for (;;){ Cpp_Lex_Result step_result =
Cpp_Lex_Result step_result = cpp_lex_nonalloc_no_null_out_limit(&S.lex, chunk, chunk_size, full_size,
cpp_lex_nonalloc_no_null_out_limit(&S.lex, chunk, chunk_size, full_size, relex_array, 1);
relex_array, 1);
switch (step_result){
switch (step_result){ case LexResult_HitTokenLimit:
case LexResult_HitTokenLimit: {
{ Cpp_Token token = relex_array->tokens[relex_array->count-1];
Cpp_Token token = relex_array->tokens[relex_array->count-1]; if (token.type == S.end_token.type &&
if (token.type == S.end_token.type && token.start == S.end_token.start &&
token.start == S.end_token.start && token.size == S.end_token.size &&
token.size == S.end_token.size && token.flags == S.end_token.flags &&
token.flags == S.end_token.flags && token.state_flags == S.end_token.state_flags){
token.state_flags == S.end_token.state_flags){ --relex_array->count;
--relex_array->count; goto double_break;
goto double_break;
}
while (S.lex.pos > S.end_token.start && S.end_token_index < array->count){
++S.end_token_index;
S.end_token = cpp_index_array(array, full_size, S.end_token_index);
}
} }
break;
case LexResult_NeedChunk: DrYield(1, LexResult_NeedChunk); break; while (S.lex.pos > S.end_token.start && S.end_token_index < array->count){
++S.end_token_index;
case LexResult_NeedTokenMemory: DrYield(2, LexResult_NeedTokenMemory); break; S.end_token = cpp_index_array(array, full_size, S.end_token_index);
}
case LexResult_Finished: goto double_break;
} }
break;
case LexResult_NeedChunk: DrYield(1, LexResult_NeedChunk); break;
case LexResult_NeedTokenMemory: DrYield(2, LexResult_NeedTokenMemory); break;
case LexResult_Finished: goto double_break;
} }
} }
double_break:; double_break:;
DrReturn(LexResult_Finished); DrReturn(LexResult_Finished);

37
4ed_file_view.cpp
View File

@ -1276,20 +1276,49 @@ file_relex_parallel(System_Functions *system,
relex_array.tokens = push_array(part, Cpp_Token, relex_array.max_count); relex_array.tokens = push_array(part, Cpp_Token, relex_array.max_count);
i32 size = file->state.buffer.size; i32 size = file->state.buffer.size;
char *spare = push_array(part, char, size); char *spare = push_array(part, char, size+1);
Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, spare); Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, spare);
char *chunk = file->state.buffer.data; char *chunk = file->state.buffer.data;
i32 chunk_size = size; i32 chunk_size = 1024;
i32 chunk_index = 0;
int32_t start_position = cpp_relex_start_position(&state);
if (start_position == size){
chunk = 0;
chunk_size = 0;
cpp_relex_declare_first_chunk_position(&state, size);
}
else{
chunk_index = start_position / chunk_size;
int32_t chunk_start_position = chunk_index*1024;
if (chunk_start_position + chunk_size > size){
chunk_size = size - chunk_start_position;
}
cpp_relex_declare_first_chunk_position(&state, chunk_start_position);
chunk += chunk_start_position;
}
for(;;){ for(;;){
Cpp_Lex_Result lex_result = Cpp_Lex_Result lex_result =
cpp_relex_step(&state, chunk, chunk_size, size, array, &relex_array); cpp_relex_step(&state, chunk, chunk_size, size, array, &relex_array);
switch (lex_result){ switch (lex_result){
case LexResult_NeedChunk: case LexResult_NeedChunk:
Assert(!"There is only one chunk in the current system."); {
break; ++chunk_index;
chunk += chunk_size;
int32_t chunk_start_position = chunk_index*1024;
if (chunk_start_position + chunk_size > size){
chunk_size = size - chunk_start_position;
}
}break;
case LexResult_NeedTokenMemory: case LexResult_NeedTokenMemory:
inline_lex = 0; inline_lex = 0;