From 892fffe5b186b687f35ddd80a0afdf40866fc8be Mon Sep 17 00:00:00 2001
From: Allen Webster <editor@4coder.net>
Date: Tue, 2 May 2017 17:55:47 -0400
Subject: [PATCH] asserts to catch lexer problems

---
 4cpp/4cpp_lexer_types.h |  1 -
 4ed_file_view.cpp       | 59 +++++++++++++++++++++++------------------
 2 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/4cpp/4cpp_lexer_types.h b/4cpp/4cpp_lexer_types.h
index 495c5bf9..13864a53 100644
--- a/4cpp/4cpp_lexer_types.h
+++ b/4cpp/4cpp_lexer_types.h
@@ -18,7 +18,6 @@
 
 /* DOC(A Cpp_Token_Type classifies a token to make parsing easier. Some types are not actually output by the lexer, but exist because parsers will also make use of token types in their own output.) */
 ENUM(uint32_t, Cpp_Token_Type){
-    
     CPP_TOKEN_JUNK = 0,
     CPP_TOKEN_COMMENT = 1,
     
diff --git a/4ed_file_view.cpp b/4ed_file_view.cpp
index d4262f8d..a84338ed 100644
--- a/4ed_file_view.cpp
+++ b/4ed_file_view.cpp
@@ -2152,13 +2152,10 @@ Job_Callback_Sig(job_full_lex){
     // TODO(allen): deduplicate this against relex
     char *chunks[3];
     i32 chunk_sizes[3];
-    
     chunks[0] = buffer->data;
     chunk_sizes[0] = buffer->size1;
-    
     chunks[1] = buffer->data + buffer->size1 + buffer->gap_size;
     chunk_sizes[1] = buffer->size2;
-    
     chunks[2] = 0;
     chunk_sizes[2] = 0;
     
@@ -2172,22 +2169,27 @@ Job_Callback_Sig(job_full_lex){
             cpp_lex_step(&lex, chunk, chunk_size, text_size, &tokens, 2048);
         
         switch (result){
-            case LexResult_NeedChunk: ++chunk_index; break;
+            case LexResult_NeedChunk:
+            {
+                ++chunk_index;
+                Assert(chunk_index < ArrayCount(chunks));
+            }break;
             
             case LexResult_NeedTokenMemory:
-            if (system->check_cancel(thread)){
-                return;
-            }
-            system->grow_thread_memory(memory);
-            tokens.tokens = (Cpp_Token*)(memory->data);
-            tokens.max_count = memory->size / sizeof(Cpp_Token);
-            break;
-            
-            case LexResult_HitTokenLimit:
-            if (system->check_cancel(thread)){
-                return;
-            }
-            break;
+            {
+                if (system->check_cancel(thread)){
+                    return;
+                }
+                system->grow_thread_memory(memory);
+                tokens.tokens = (Cpp_Token*)(memory->data);
+                tokens.max_count = memory->size / sizeof(Cpp_Token);
+                break;
+                
+                case LexResult_HitTokenLimit:
+                if (system->check_cancel(thread)){
+                    return;
+                }
+            }break;
             
             case LexResult_Finished: still_lexing = 0; break;
         }
@@ -2294,13 +2296,10 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){
             // TODO(allen): deduplicate this against relex
             char *chunks[3];
             i32 chunk_sizes[3];
-            
             chunks[0] = buffer->data;
             chunk_sizes[0] = buffer->size1;
-            
             chunks[1] = buffer->data + buffer->size1 + buffer->gap_size;
             chunk_sizes[1] = buffer->size2;
-            
             chunks[2] = 0;
             chunk_sizes[2] = 0;
             
@@ -2315,8 +2314,11 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){
                 i32 result = cpp_lex_step(&lex, chunk, chunk_size, text_size, &tokens, NO_OUT_LIMIT);
                 
                 switch (result){
-                    case LexResult_NeedChunk: ++chunk_index; break;
-                    
+                    case LexResult_NeedChunk:
+                    {
+                        ++chunk_index;
+                        Assert(chunk_index < ArrayCount(chunks));
+                    }break;
                     
                     case LexResult_Finished:
                     case LexResult_NeedTokenMemory:
@@ -2418,6 +2420,7 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi
         
         while (!cpp_relex_is_start_chunk(&state, chunk, chunk_size)){
             ++chunk_index;
+            Assert(chunk_index < ArrayCount(chunks));
             chunk = chunks[chunk_index];
             chunk_size = chunk_sizes[chunk_index];
         }
@@ -2430,6 +2433,7 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi
                 case LexResult_NeedChunk:
                 {
                     ++chunk_index;
+                    Assert(chunk_index < ArrayCount(chunks));
                     chunk = chunks[chunk_index];
                     chunk_size = chunk_sizes[chunk_index];
                 }break;
@@ -2539,6 +2543,7 @@ file_relex_serial(Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i,
     
     while (!cpp_relex_is_start_chunk(&state, chunk, chunk_size)){
         ++chunk_index;
+        Assert(chunk_index < ArrayCount(chunks));
         chunk = chunks[chunk_index];
         chunk_size = chunk_sizes[chunk_index];
     }
@@ -2548,10 +2553,12 @@ file_relex_serial(Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i,
         
         switch (lex_result){
             case LexResult_NeedChunk:
-            ++chunk_index;
-            chunk = chunks[chunk_index];
-            chunk_size = chunk_sizes[chunk_index];
-            break;
+            {
+                ++chunk_index;
+                Assert(chunk_index < ArrayCount(chunks));
+                chunk = chunks[chunk_index];
+                chunk_size = chunk_sizes[chunk_index];
+            }break;
             
             case LexResult_NeedTokenMemory: InvalidCodePath;