ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lexer table generator started

This commit is contained in:
Allen Webster 2016-03-23 21:05:28 -04:00
parent 830a3a86bd
commit 58a0c9e08f
16 changed files with 2253 additions and 1473 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
4coder_custom.cpp
View File

@ -1,12 +1,11 @@
// Set to Custom_None if you're going to drop in your own include/get_bindings call.
// or choose one of the preexisting customizations
// Delete CustomCurrent to define your own customizations or
// set it to one of the preexisting options
#define Custom_Current Custom_Default
#define Custom_None -1
#define Custom_Default 0
#define Custom_Default 1
// The following customization schemes are power users only:
#define Custom_HandmadeHero 1
#define Custom_HandmadeHero 2
// TOP
@ -14,7 +13,7 @@
#if Custom_Current == Custom_Default
# include "4coder_default_bindings.cpp"
#elif Custom_Current == Custom_HandmadeHero
# include "power/4coder_handmade_hero.cpp"
# include "power/4coder_casey.cpp"
#endif
extern "C" GET_BINDING_DATA(get_bindings){
@ -22,7 +21,7 @@ extern "C" GET_BINDING_DATA(get_bindings){
Bind_Helper *context = &context_actual;
#if Custom_Current == Custom_Default
default_get_bindings(context);
default_get_bindings(context, true);
#elif Custom_Current == Custom_HandmadeHero
casey_get_bindings(context);
#endif

7
4coder_custom.h
View File

@ -1,4 +1,7 @@
#ifndef FCODER_CUSTOM_H
#define FCODER_CUSTOM_H
#include "4coder_version.h"
#include "4coder_keycodes.h"
#include "4coder_style.h"
@ -183,7 +186,6 @@ enum Command_ID{
cmdid_interactive_open,
cmdid_reopen,
cmdid_save,
cmdid_interactive_save_as,
cmdid_change_active_panel,
cmdid_interactive_switch_buffer,
cmdid_interactive_kill_buffer,
@ -233,6 +235,7 @@ enum Param_ID{
par_cli_command,
par_clear_blank_lines,
par_use_tabs,
par_save_update_name,
// never below this
par_type_count
@ -599,3 +602,5 @@ struct Binding_Unit{
} hook;
};
};
#endif

5
4coder_default.cpp
View File

@ -212,6 +212,11 @@ CUSTOM_COMMAND_SIG(open_file_in_quotes){
}
}
CUSTOM_COMMAND_SIG(save_as){
push_parameter(app, par_save_update_name, 1);
exec_command(app, cmdid_save);
}
CUSTOM_COMMAND_SIG(goto_line){
int line_number;
Query_Bar bar;

54
4coder_default_bindings.cpp
View File

@ -10,7 +10,11 @@ unsigned char blink_t = 0;
// 2^24 of them so don't be wasteful!
enum My_Maps{
my_code_map,
my_html_map
my_html_map,
// for testing
my_empty_map1,
my_empty_map2,
my_maps_count
};
HOOK_SIG(my_start){
@ -40,13 +44,14 @@ HOOK_SIG(my_start){
HOOK_SIG(my_file_settings){
// NOTE(allen|a4): In hooks that want parameters, such as this file
// created hook. The file created hook is guaranteed to have only
// opened hook. The file created hook is guaranteed to have only
// and exactly one buffer parameter. In normal command callbacks
// there are no parameter buffers.
Buffer_Summary buffer = app->get_parameter_buffer(app, 0);
assert(buffer.exists);
int treat_as_code = 0;
int wrap_lines = 1;
if (buffer.file_name && buffer.size < (16 << 20)){
String ext = file_extension(make_string(buffer.file_name, buffer.file_name_len));
@ -56,6 +61,13 @@ HOOK_SIG(my_file_settings){
else if (match(ext, make_lit_string("hpp"))) treat_as_code = 1;
}
if (treat_as_code){
wrap_lines = 0;
}
if (buffer.file_name[0] == '*'){
wrap_lines = 0;
}
push_parameter(app, par_lex_as_cpp_file, treat_as_code);
push_parameter(app, par_wrap_lines, !treat_as_code);
push_parameter(app, par_key_mapid, (treat_as_code)?((int)my_code_map):((int)mapid_file));
@ -66,8 +78,9 @@ HOOK_SIG(my_file_settings){
}
HOOK_SIG(my_frame){
// NOTE(allen|a4): Please use me sparingly! This get's called roughly once every *33 ms* if everything is going well.
// But if you start doing a lot in here, there's nothing 4codes does to stop you from making it a lot slower.
// NOTE(allen|a4): Please use me sparingly! This get's called roughly once every *33 ms*
// if everything is going well. But if you start doing a lot in here, there's nothing 4codes does
// to stop you from making it a lot slower.
int result = 0;
Theme_Color theme_color_1[] = {
@ -120,7 +133,7 @@ CUSTOM_COMMAND_SIG(write_div){
}
CUSTOM_COMMAND_SIG(begin_html_mode){
push_parameter(app, par_key_mapid, my_html_map);
push_parameter(app, par_key_mapid, my_empty_map1);
exec_command(app, cmdid_set_settings);
}
@ -291,12 +304,14 @@ CUSTOM_COMMAND_SIG(ruin_theme){
app->set_theme_colors(app, colors, count);
}
void default_get_bindings(Bind_Helper *context){
void default_get_bindings(Bind_Helper *context, int set_hooks){
// NOTE(allen|a3.1): Hooks have no loyalties to maps. All hooks are global
// and once set they always apply, regardless of what map is active.
set_hook(context, hook_start, my_start);
set_hook(context, hook_open_file, my_file_settings);
//set_hook(context, hook_frame, my_frame); // Example of a frame hook, but disabled by default.
if (set_hooks){
set_hook(context, hook_start, my_start);
set_hook(context, hook_open_file, my_file_settings);
//set_hook(context, hook_frame, my_frame); // Example of a frame hook, but disabled by default.
}
set_scroll_rule(context, smooth_scroll_rule);
@ -312,6 +327,7 @@ void default_get_bindings(Bind_Helper *context){
bind(context, 'i', MDFR_CTRL, cmdid_interactive_switch_buffer);
bind(context, 'c', MDFR_ALT, cmdid_open_color_tweaker);
bind(context, 'o', MDFR_ALT, open_in_other);
bind(context, 'w', MDFR_CTRL, save_as);
bind(context, 'm', MDFR_ALT, build_search);
bind(context, 'x', MDFR_ALT, execute_arbitrary_command);
@ -326,14 +342,21 @@ void default_get_bindings(Bind_Helper *context){
bind(context, '~', MDFR_ALT, ruin_theme);
end_map(context);
begin_map(context, my_html_map);
inherit_map(context, mapid_file);
bind(context, 'h', MDFR_ALT, write_h);
bind(context, 'd', MDFR_ALT, write_div);
end_map(context);
begin_map(context, my_empty_map1);
inherit_map(context, mapid_nomap);
end_map(context);
begin_map(context, my_empty_map2);
inherit_map(context, mapid_nomap);
end_map(context);
begin_map(context, my_code_map);
@ -438,16 +461,15 @@ void default_get_bindings(Bind_Helper *context){
bind(context, 'K', MDFR_CTRL, cmdid_kill_buffer);
bind(context, 'O', MDFR_CTRL, cmdid_reopen);
bind(context, 'w', MDFR_CTRL, cmdid_interactive_save_as);
bind(context, 's', MDFR_CTRL, cmdid_save);
bind(context, '\n', MDFR_SHIFT, write_and_auto_tab);
bind(context, ' ', MDFR_SHIFT, cmdid_write_character);
bind(context, 'e', MDFR_CTRL, cmdid_center_view);
bind(context, 'T', MDFR_CTRL | MDFR_ALT, begin_html_mode);
end_map(context);
}

1544
4cpp_lexer.h
View File

File diff suppressed because it is too large Load Diff

135
4ed.cpp
View File

@ -94,7 +94,7 @@ app_get_or_add_map_index(Models *models, i32 mapid){
i32 *map_id_table = models->map_id_table;
for (result = 0; result < user_map_count; ++result){
if (map_id_table[result] == mapid) break;
if (map_id_table[result] == 0){
if (map_id_table[result] == -1){
map_id_table[result] = mapid;
break;
}
@ -936,10 +936,11 @@ COMMAND_DECL(save){
USE_FILE(file, view);
Delay *delay = &models->delay1;
char *filename = 0;
int filename_len = 0;
int buffer_id = -1;
int update_names = 0;
Command_Parameter *end = param_stack_end(&command->part);
Command_Parameter *param = param_stack_first(&command->part, end);
@ -952,27 +953,50 @@ COMMAND_DECL(save){
else if (v == par_buffer_id && param->param.value.type == dynamic_type_int){
buffer_id = dynamic_to_int(&param->param.value);
}
else if (v == par_save_update_name){
update_names = dynamic_to_bool(&param->param.value);
}
}
String name = {};
if (filename){
name = make_string(filename, filename_len);
}
else if (file){
name = file->name.source_path;
}
#if 0
#endif
if (name.size != 0){
if (buffer_id == -1){
if (file){
delayed_save(delay, name, file);
if (buffer_id != -1){
file = working_set_get_active_file(&models->working_set, buffer_id);
}
if (update_names){
String name = {};
if (filename){
name = make_string(filename, filename_len);
}
if (file){
if (name.str){
if (!file->state.is_dummy && file_is_ready(file)){
delayed_save_as(delay, name, file);
}
}
else{
view_show_interactive(system, view, &models->map_ui,
IAct_Save_As, IInt_Sys_File_List, make_lit_string("Save As: "));
}
}
else{
file = working_set_get_active_file(&models->working_set, buffer_id);
if (!file->state.is_dummy && file_is_ready(file) && buffer_needs_save(file)){
delayed_save(delay, name, file);
}
else{
String name = {};
if (filename){
name = make_string(filename, filename_len);
}
else if (file){
name = file->name.source_path;
}
if (name.size != 0){
if (file){
if (!file->state.is_dummy && file_is_ready(file)){
delayed_save(delay, name, file);
}
}
else{
delayed_save(delay, name);
@ -981,15 +1005,6 @@ COMMAND_DECL(save){
}
}
COMMAND_DECL(interactive_save_as){
ProfileMomentFunction();
USE_VIEW(view);
USE_MODELS(models);
view_show_interactive(system, view, &models->map_ui,
IAct_Save_As, IInt_Sys_File_List, make_lit_string("Save As: "));
}
COMMAND_DECL(change_active_panel){
ProfileMomentFunction();
USE_MODELS(models);
@ -2301,9 +2316,7 @@ extern "C"{
GET_ACTIVE_VIEW_SIG(external_get_active_view){
Command_Data *cmd = (Command_Data*)app->cmd_context;
View_Summary view = {};
fill_view_summary(&view, cmd->view, &cmd->vars->live_set, &cmd->models->working_set);
return(view);
}
@ -2318,6 +2331,7 @@ extern "C"{
Command_Data *cmd = (Command_Data*)app->cmd_context;
Live_Views *live_set;
View *vptr;
Editing_File *file;
int result = 0;
int view_id;
@ -2326,15 +2340,19 @@ extern "C"{
view_id = view->view_id - 1;
if (view_id >= 0 && view_id < live_set->max){
vptr = live_set->views + view_id;
result = 1;
if (seek.type == buffer_seek_line_char && seek.character <= 0){
seek.character = 1;
file = vptr->file;
if (file && !file->state.is_loading){
result = 1;
if (seek.type == buffer_seek_line_char && seek.character <= 0){
seek.character = 1;
}
vptr->cursor = view_compute_cursor(vptr, seek);
if (set_preferred_x){
vptr->preferred_x = view_get_cursor_x(vptr);
}
fill_view_summary(view, vptr, live_set, &cmd->models->working_set);
file->state.cursor_pos = vptr->cursor.pos;
}
vptr->cursor = view_compute_cursor(vptr, seek);
if (set_preferred_x){
vptr->preferred_x = view_get_cursor_x(vptr);
}
fill_view_summary(view, vptr, live_set, &cmd->models->working_set);
}
}
@ -2658,7 +2676,7 @@ setup_command_table(){
SET(interactive_open);
SET(reopen);
SET(save);
SET(interactive_save_as);
//SET(interactive_save_as);
SET(change_active_panel);
SET(interactive_switch_buffer);
SET(interactive_kill_buffer);
@ -3083,14 +3101,15 @@ App_Read_Command_Line_Sig(app_read_command_line){
}
else{
vars = app_setup_memory(memory);
settings = &vars->models.settings;
*settings = {};
settings->font_size = 16;
if (clparams.argc > 1){
init_command_line_settings(&vars->models.settings, plat_settings, clparams);
}
else{
settings = &vars->models.settings;
*settings = {};
settings->font_size = 16;
}
*files = vars->models.settings.init_files;
*file_count = &vars->models.settings.init_files_count;
}
@ -3209,6 +3228,7 @@ App_Init_Sig(app_init){
models->map_id_table = push_array(
&models->mem.part, i32, user_map_count);
memset(models->map_id_table, -1, user_map_count*sizeof(i32));
models->user_maps = push_array(
&models->mem.part, Command_Map, user_map_count);
@ -4350,29 +4370,9 @@ App_Step_Sig(app_step){
}
}
}break;
case DACT_SAVE_AS:
{
// TODO(allen): deduplicate
Editing_File *file = 0;
if (panel){
file = panel->view->file;
}
else if (string.str && string.size > 0){
file = working_set_lookup_file(working_set, string);
}
if (file){
i32 sys_id = file_save_and_set_names(system, exchange, mem, working_set, file, string.str);
if (sys_id){
app_push_file_binding(vars, sys_id, file->id.id);
}
else{
delayed_action_repush(&models->delay2, act);
}
}
}break;
case DACT_SAVE:
case DACT_SAVE_AS:
{
if (!file){
if (panel){
@ -4386,9 +4386,12 @@ App_Step_Sig(app_step){
}
// TODO(allen): We could handle the case where someone tries to save the same thing
// twice... that would be nice to have under control.
if (file){
if (file && buffer_needs_save(file)){
i32 sys_id = file_save(system, exchange, mem, file, file->name.source_path.str);
if (sys_id){
if (act->type == DACT_SAVE_AS){
file_set_name(working_set, file, string.str);
}
// TODO(allen): This is fishy! Shouldn't we bind it to a file name instead? This file
// might be killed before we get notified that the saving is done!
app_push_file_binding(vars, sys_id, file->id.id);

3
build.bat
View File

@ -1,4 +1,5 @@
@echo off
"w:\4ed\misc\build_all.bat" /DFRED_SUPER /DFRED_NOT_PACKAGE /Zi
"w:\4ed\misc\build_exp.bat" /Zi
REM "w:\4ed\misc\build_all.bat" /DFRED_SUPER /DFRED_NOT_PACKAGE /Zi
REM "w:\4ed\misc\build_all.bat" /O2 /Zi

193
power/4coder_handmade_hero.cpp → power/4coder_casey.cpp
View File

@ -11,48 +11,96 @@
merchantability, fitness for a particular purpose, or non-infringement.
*/
/* TODO(casey): Here are our current issues
/* NOTE(allen): Should be fixed now
- Need file limit to be substantially higher than 128 (65536?)
~ file limit is now over 8 million
- Font size is too big
~ -f N option on command line, default N = 16
- Asking for a buffer to be saved if you have not modified the buffer should not save the
buffer, or perhaps more "safely", it should diff the buffer against the existing on-disk
contents and only save if there is a detected change between them.
- Search:
- Needs to be case-insensitive, or at least have the option to be
- Needs to replace using the case of the thing being replaced, or at least have the option to do so
- Bug with opening too many files where it simply no longer can switch to a buffer at all?
~ I assume this refers to a file limit issue, if not then maybe it's not actually fixed.
- Bug where opening the same buffer with open-file leads to a confusing situation
where you don't know what you're editing or something??
- Bug where replacing v4 with rectangle2 only replaces some instances???
~ For the interested programmer: the range recomputation wasn't working right so it was always
using the original range from cursor to mark. So if the string gets too long later occurances of
v4 get pushed outside of the range.
- Bug in search where extra backspaces after there are no characters yet "remembers"
how many you hit and then eats that many real characters you type?
- Display:
- Bug in scroll callback that seems to always pass the same view id instead of
the correct id for each view?
- Jumping to subsequent errors in a file seems to jump to an unrelated position
then scroll back to the actual position, which results in lots of extra scrolling
- Need a way of highlighting the current line like Emacs does for the benefit
of people on The Stream(TM)
- Some way to recenter the view so that the line containing the cursor becomes the
center line vertically.
~ cmdid_center_view
- Have buffers normalize slashes to always be forward-slash - right now I'm doing this manually
*/
/* TODO(casey): Here are our current issues
- High priority:
- Would like the option to indent to hanging parentheses, equals signs, etc. instead of
always just "one tab in from the previous line".
- Actually, maybe just expose the dirty state, so that the user can decide whether to
save or not? Not sure...
- Replace:
- Needs to be case-insensitive, or at least have the option to be
- Needs to replace using the case of the thing being replaced, or at least have the option to do so
- Auto-complete doesn't pick nearby words first, it seems, which makes it much slower to use?
- Bug with not being able to switch-to-corresponding-file in another buffer
without accidentally bringing up the file open dialog?
- Display:
- Need a word-wrap mode that wraps at word boundaries instead of characters
- Need to be able to set a word wrap length at something other than the window
?FIXED First go-to-line for a file seems to still just go to the beginning of the buffer?
Not sure Allen's right about the slash problem, but either way, we need some
way to fix it.
- Need a way of highlighting the current line like Emacs does for the benefit
of people on The Stream(TM)
- NOTE / IMPORTANT / TODO highlighting? Ability to customize? Whatever.
- Some kind of parentheses highlighting? I can write this myself, but I
would need some way of adding highlight information to the buffer.
- Indentation:
- Multiple // lines don't seem to indent properly. The first one will go to the correct place, but the subsequent ones will go to the first column regardless?
- Would like the option to indent to hanging parentheses, equals signs, etc. instead of
always just "one tab in from the previous line".
- Crash bug with paste-and-indent that sometimes leaves things unindented then crashes
- Need to have better indentation / wrapping control for typing in comments.
Right now it's a bit worse than Emacs, which does automatically put you at
the same margin as the prev. line (4coder just goes back to column 1). It'd
be nice if it go _better_ than Emacs, with no need to manually flow comments,
etc.
- Up/down arrows and mouse clicks on wrapped lines don't seem to work properly after the second wrap
(eg., a line wrapped to more than 2 physical lines on the screen.)
- Buffer management:
- Bug in view iteration such that buffer_id is sometimes set to 0, so you can't find the view
for a buffer?
- Have buffers normalize slashes to always be forward-slash - right now I'm doing this manually
- Buffer management:
- Switch-to-buffer with no typing, just return, should switch to the most recently
used buffer that is not currently displayed in a view.
- Kill-buffer should perform this switch automatically, or it should be easy
to build a custom kill buffer that does
?FIXED Kill-buffer should perform this switch automatically, or it should be easy
to build a custom kill buffer that does
- Seems like there's no way to switch to buffers whose names are substrings of other
buffers' names without using the mouse?
- Also, mouse-clicking on buffers doesn't seem to work reliably? Often it just goes to a
blank window?
- Need auto-complete for things like "arbitrary command", with options listed, etc.,
so this should either be built into 4ed, or the custom DLL should have the ability
to display possible completions and iterate over internal cmdid's, etc. Possibly
the latter, for maximal ability of customizers to add their own commands?
- Default directory for file open / build search should be that of the current
buffer, not tracked separately? Probably I should code this on my own.
- Macro recording/playback
*/
@ -61,19 +109,33 @@
#include <math.h>
#include <stdio.h>
#include "..\4coder_default.cpp"
enum maps{
my_code_map
};
#ifndef Assert
#define internal static
#define Assert assert
#endif
struct Parsed_Error
{
int exists;
String target_file_name;
int target_line_number;
int source_buffer_id;
int source_position;
};
static bool GlobalEditMode;
static char *GlobalCompilationBufferName = "*compilation*";
// TODO(casey): If 4coder gets variables at some point, this would go in a variable.
static char BuildDirectory[4096] = "./";
static int ErrorParsingPosition;
static int ErrorParsingLastJumpLine;
static int ErrorParsingLastBufferID;
enum token_type
{
@ -358,6 +420,8 @@ CUSTOM_COMMAND_SIG(casey_kill_to_end_of_line)
CUSTOM_COMMAND_SIG(casey_paste_and_tab)
{
// NOTE(allen): Paste puts the mark at the beginning and the cursor at
// the end of the pasted chunk, so it is all set for cmdid_auto_tab_range
exec_command(app, cmdid_paste);
exec_command(app, cmdid_auto_tab_range);
}
@ -417,7 +481,7 @@ SwitchToOrLoadFile(struct Application_Links *app, String FileName, bool CreateIf
// to interactive open to tell it to fail if the file isn't there?
exec_command(app, cmdid_interactive_open);
Result.buffer = app->get_buffer_by_name(app, FileName.str, FileName.size);
Result.buffer = app->get_buffer_by_name(app, FileName.str, FileName.size);
Result.Loaded = true;
Result.Switched = true;
@ -433,8 +497,6 @@ CUSTOM_COMMAND_SIG(casey_load_todo)
SwitchToOrLoadFile(app, ToDoFileName, true);
}
inline String Empty() {String Result = {}; return(Result);}
CUSTOM_COMMAND_SIG(casey_build_search)
{
int keep_going = 1;
@ -524,6 +586,18 @@ CUSTOM_COMMAND_SIG(casey_find_corresponding_file)
}
}
CUSTOM_COMMAND_SIG(casey_find_corresponding_file_other_window)
{
View_Summary old_view = app->get_active_view(app);
Buffer_Summary buffer = app->get_buffer(app, old_view.buffer_id);
exec_command(app, cmdid_change_active_panel);
View_Summary new_view = app->get_active_view(app);
app->view_set_buffer(app, &new_view, buffer.buffer_id);
// exec_command(app, casey_find_corresponding_file);
}
CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking)
{
exec_command(app, cmdid_change_active_panel);
@ -541,19 +615,25 @@ CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking)
String dir = make_string(app->memory, 0, app->memory_size);
append(&dir, BuildDirectory);
for(int At = 0;
At < dir.size;
++At)
{
if(dir.str[At] == '/')
{
dir.str[At] = '\\';
}
}
push_parameter(app, par_flags, CLI_OverlapWithConflict);
push_parameter(app, par_name, GlobalCompilationBufferName, (int)strlen(GlobalCompilationBufferName));
push_parameter(app, par_cli_path, dir.str, dir.size);
if(append(&dir, "build"))
if(append(&dir, "build.bat"))
{
push_parameter(app, par_cli_command, dir.str, dir.size);
exec_command(app, cmdid_command_line);
ErrorParsingPosition = 0;
ErrorParsingLastJumpLine = 0;
ErrorParsingLastBufferID = 0;
exec_command(app, cmdid_change_active_panel);
}
else{
@ -561,21 +641,10 @@ CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking)
}
}
struct Parsed_Error
{
int exists;
String target_file_name;
int target_line_number;
int source_buffer_id;
int source_position;
};
internal bool
casey_errors_are_the_same(Parsed_Error a, Parsed_Error b)
{
bool result = ((a.exists == b.exists) && match(a.target_file_name, b.target_file_name) && (a.target_line_number == b.target_line_number));
bool result = ((a.exists == b.exists) && compare(a.target_file_name, b.target_file_name) && (a.target_line_number == b.target_line_number));
return(result);
}
@ -621,6 +690,7 @@ casey_parse_error(Application_Links *app, Buffer_Summary buffer, View_Summary vi
int size = end - start;
char *ParsingRegion = (char *)malloc(size + 1);
// char *ParsingRegion = (char *)app->push_memory(app, size + 1);
app->buffer_read_range(app, &buffer, start, end, ParsingRegion);
ParsingRegion[size] = 0;
tokenizer Tokenizer = {ParsingRegion};
@ -661,14 +731,7 @@ casey_parse_error(Application_Links *app, Buffer_Summary buffer, View_Summary vi
result.target_line_number = line_number;
result.source_buffer_id = buffer.buffer_id;
result.source_position = start + (int)(ColonToken.Text - ParsingRegion);
int start_pos;
for (start_pos = 0;
start_pos < result.target_file_name.size && result.target_file_name.str[start_pos] == ' ';
++start_pos);
result.target_file_name = substr(result.target_file_name, start_pos);
break;
}
}
@ -938,6 +1001,8 @@ CUSTOM_COMMAND_SIG(casey_quick_calc)
internal void
OpenProject(Application_Links *app, char *ProjectFileName)
{
int TotalOpenAttempts = 0;
FILE *ProjectFile = fopen(ProjectFileName, "r");
if(ProjectFile)
{
@ -954,22 +1019,22 @@ OpenProject(Application_Links *app, char *ProjectFileName)
BuildDirectory[BuildDirSize] = 0;
}
char SourceFileDirectoryName[4096];
char FileDirectoryName[4096];
while(fgets(FileDirectoryName, sizeof(FileDirectoryName) - 1, ProjectFile))
while(fgets(SourceFileDirectoryName, sizeof(SourceFileDirectoryName) - 1, ProjectFile))
{
// NOTE(allen|a3.4.4): Here we get the list of files in this directory.
// Notice that we free_file_list at the end.
String dir = make_string(app->memory, 0, app->memory_size);
append(&dir, FileDirectoryName);
if(dir.size && dir.str[dir.size] == '\n')
String dir = make_string(FileDirectoryName, 0, sizeof(FileDirectoryName));
append(&dir, SourceFileDirectoryName);
if(dir.size && dir.str[dir.size-1] == '\n')
{
--dir.size;
}
if(dir.size && dir.str[dir.size] != '/')
if(dir.size && dir.str[dir.size-1] != '/')
{
dir.str[dir.size] = '/';
++dir.size;
dir.str[dir.size++] = '/';
}
File_List list = app->get_file_list(app, dir.str, dir.size);
@ -992,6 +1057,7 @@ OpenProject(Application_Links *app, char *ProjectFileName)
push_parameter(app, par_name, dir.str, dir.size);
push_parameter(app, par_do_in_background, 1);
exec_command(app, cmdid_interactive_open);
++TotalOpenAttempts;
}
}
}
@ -1130,7 +1196,7 @@ DEFINE_MODAL_KEY(modal_t, casey_load_todo);
DEFINE_MODAL_KEY(modal_u, cmdid_undo);
DEFINE_MODAL_KEY(modal_v, casey_switch_buffer_other_window);
DEFINE_MODAL_KEY(modal_w, cmdid_cut);
DEFINE_MODAL_KEY(modal_x, casey_open_file_other_window);
DEFINE_MODAL_KEY(modal_x, casey_find_corresponding_file_other_window);
DEFINE_MODAL_KEY(modal_y, auto_tab_line_at_cursor);
DEFINE_MODAL_KEY(modal_z, cmdid_interactive_open);
@ -1185,7 +1251,9 @@ HOOK_SIG(casey_file_settings)
if(treat_as_project)
{
OpenProject(app, buffer.file_name);
exec_command(app, cmdid_kill_buffer);
// NOTE(casey): Don't actually want to kill this, or you can never edit the project.
// exec_command(app, cmdid_kill_buffer);
}
return(0);
@ -1238,10 +1306,10 @@ struct Casey_Scroll_Velocity
};
Casey_Scroll_Velocity casey_scroll_velocity_[16] = {0};
Casey_Scroll_Velocity *casey_scroll_velocity = casey_scroll_velocity_ - 1;
Casey_Scroll_Velocity *casey_scroll_velocity = casey_scroll_velocity_;
SCROLL_RULE_SIG(casey_smooth_scroll_rule){
float dt = 1.0f/30.0f; // TODO(casey): Why do I not get the timestep here?
float dt = 1.0f/60.0f; // TODO(casey): Why do I not get the timestep here?
Casey_Scroll_Velocity *velocity = casey_scroll_velocity + view_id;
int result = 0;
if(is_new_target)
@ -1265,7 +1333,6 @@ SCROLL_RULE_SIG(casey_smooth_scroll_rule){
velocity->t = 0;
*scroll_x = target_x;
*scroll_y = target_y;
result = 1;
}
return(result);
@ -1294,6 +1361,7 @@ internal BOOL CALLBACK win32_find_4coder_window(HWND Window, LPARAM LParam)
internal void
win32_toggle_fullscreen(void)
{
#if 0
// NOTE(casey): This follows Raymond Chen's prescription
// for fullscreen toggling, see:
// http://blogs.msdn.com/b/oldnewthing/archive/2010/04/12/9994016.aspx
@ -1322,6 +1390,9 @@ win32_toggle_fullscreen(void)
SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER |
SWP_NOOWNERZORDER | SWP_FRAMECHANGED);
}
#else
ShowWindow(GlobalWindowHandle, SW_MAXIMIZE);
#endif
}
HOOK_SIG(casey_start)
@ -1330,7 +1401,7 @@ HOOK_SIG(casey_start)
app->change_theme(app, literal("Handmade Hero"));
app->change_font(app, literal("liberation mono"));
//win32_toggle_fullscreen();
win32_toggle_fullscreen();
return(0);
}

304
power/4coder_chronal.cpp Normal file
View File

@ -0,0 +1,304 @@
// TOP
#include "4coder_default.cpp"
//#include "chr_winutils.h"
#ifndef literal
#define literal(s) (s), (sizeof(s)-1)
#endif
#define rgb_color(r, g, b) (r << 16 + g << 8 + b << 0)
#define hex_color(hex) hex
const int color_margin_normal = 0x341313;
const int color_margin_insert = 0x5a3619;
enum Vim_Maps {
mapid_normal = mapid_global,
mapid_insert = 0,
mapid_replace,
mapid_visual,
// There are a bunch of different chord "starters" that result in keys having
// different behaviors. There's no better way to handle this right now than
// just explicitly creating maps for each one.
//TODO(chronister): Chords can be built up, so this can have potentially huge
//combinatorics... what I *want* is a way to build up an actual stack of commands
//...
mapid_chord_delete,
mapid_chord_yank,
mapid_chord_g,
};
HOOK_SIG(chronal_init){
exec_command(app, cmdid_open_panel_vsplit);
exec_command(app, cmdid_change_active_panel);
app->change_theme(app, literal("4coder"));
app->change_font(app, literal("hack"));
const int color_bg = 0x15100f;
const int color_bar = 0x1c1212;
const int color_bar_hover = 0x261414;
const int color_bar_active = 0x341313;
const int color_text = 0x916550;
const int color_comment = 0x9d5b25;
const int color_string_literal = 0x9c2d21;
const int color_num_literals = 0xc56211;
const int color_keyword = 0xf74402;
Theme_Color colors[] = {
{ Stag_Back, color_bg },
{ Stag_Margin, color_bar },
{ Stag_Margin_Hover, color_bar_hover },
{ Stag_Margin_Active, color_margin_normal },
{ Stag_Bar, color_bar },
{ Stag_Bar_Active, color_bar_active },
{ Stag_Base, color_text },
{ Stag_Default, color_text },
{ Stag_Comment, color_comment },
{ Stag_Int_Constant, color_num_literals },
{ Stag_Float_Constant, color_num_literals },
{ Stag_Str_Constant, color_string_literal },
{ Stag_Char_Constant, color_string_literal },
{ Stag_Bool_Constant, color_keyword },
{ Stag_Keyword, color_keyword },
{ Stag_Special_Character, color_keyword },
{ Stag_Preproc, color_keyword },
};
app->set_theme_colors(app, colors, ArrayCount(colors));
push_parameter(app, par_key_mapid, mapid_normal);
exec_command(app, cmdid_set_settings);
// no meaning for return
return(0);
}
HOOK_SIG(chronal_file_settings){
// NOTE(allen|a4): In hooks that want parameters, such as this file
// created hook. The file created hook is guaranteed to have only
// and exactly one buffer parameter. In normal command callbacks
// there are no parameter buffers.
Buffer_Summary buffer = app->get_parameter_buffer(app, 0);
assert(buffer.exists);
int treat_as_code = 0;
if (buffer.file_name && buffer.size < (16 << 20)){
String ext = file_extension(make_string(buffer.file_name, buffer.file_name_len));
if (match(ext, make_lit_string("cpp"))) treat_as_code = 1;
else if (match(ext, make_lit_string("h"))) treat_as_code = 1;
else if (match(ext, make_lit_string("c"))) treat_as_code = 1;
else if (match(ext, make_lit_string("hpp"))) treat_as_code = 1;
}
push_parameter(app, par_lex_as_cpp_file, treat_as_code);
push_parameter(app, par_wrap_lines, !treat_as_code);
push_parameter(app, par_key_mapid, mapid_normal);
exec_command(app, cmdid_set_settings);
// no meaning for return
return(0);
}
/* *
* Custom commands *
* */
CUSTOM_COMMAND_SIG(do_nothing){
}
CUSTOM_COMMAND_SIG(enter_insert_mode){
push_parameter(app, par_key_mapid, mapid_insert);
exec_command(app, cmdid_set_settings);
Theme_Color colors[] = {
{ Stag_Bar_Active, color_margin_insert },
{ Stag_Margin_Active, color_margin_insert },
};
app->set_theme_colors(app, colors, ArrayCount(colors));
}
CUSTOM_COMMAND_SIG(enter_normal_mode){
push_parameter(app, par_key_mapid, mapid_normal);
exec_command(app, cmdid_set_settings);
Theme_Color colors[] = {
{ Stag_Bar_Active, color_margin_normal },
{ Stag_Margin_Active, color_margin_normal },
};
app->set_theme_colors(app, colors, ArrayCount(colors));
}
CUSTOM_COMMAND_SIG(seek_forward_word_start){
View_Summary view;
view = app->get_active_view(app);
push_parameter(app, par_flags, BoundryToken);
exec_command(app, cmdid_seek_right);
app->refresh_view(app, &view);
}
CUSTOM_COMMAND_SIG(seek_backward_word_start){
View_Summary view;
view = app->get_active_view(app);
push_parameter(app, par_flags, BoundryToken | BoundryWhitespace);
exec_command(app, cmdid_seek_left);
app->refresh_view(app, &view);
}
CUSTOM_COMMAND_SIG(seek_forward_word_end){
View_Summary view;
view = app->get_active_view(app);
push_parameter(app, par_flags, BoundryToken | BoundryWhitespace);
exec_command(app, cmdid_seek_right);
app->refresh_view(app, &view);
}
CUSTOM_COMMAND_SIG(newline_then_insert_before){
exec_command(app, cmdid_seek_beginning_of_line);
write_string(app, make_lit_string("\n"));
exec_command(app, cmdid_move_left);
exec_command(app, enter_insert_mode);
}
CUSTOM_COMMAND_SIG(newline_then_insert_after){
exec_command(app, cmdid_seek_end_of_line);
write_string(app, make_lit_string("\n"));
exec_command(app, enter_insert_mode);
}
CUSTOM_COMMAND_SIG(begin_chord_delete){
push_parameter(app, par_key_mapid, mapid_chord_delete);
exec_command(app, cmdid_set_settings);
}
CUSTOM_COMMAND_SIG(delete_line){
View_Summary view;
Buffer_Summary buffer;
int pos1, pos2;
view = app->get_active_view(app);
exec_command(app, cmdid_seek_beginning_of_line);
app->refresh_view(app, &view);
pos1 = view.cursor.pos;
exec_command(app, cmdid_seek_end_of_line);
app->refresh_view(app, &view);
pos2 = view.cursor.pos;
Range range = make_range(pos1, pos2);
buffer = app->get_buffer(app, view.buffer_id);
app->buffer_replace_range(app, &buffer, range.start, range.end, 0, 0);
}
CUSTOM_COMMAND_SIG(delete_word){
View_Summary view;
Buffer_Summary buffer;
int pos1, pos2;
view = app->get_active_view(app);
exec_command(app, seek_backward_word_start);
app->refresh_view(app, &view);
pos1 = view.cursor.pos;
exec_command(app, seek_forward_word_end);
app->refresh_view(app, &view);
pos2 = view.cursor.pos;
Range range = make_range(pos1, pos2);
buffer = app->get_buffer(app, view.buffer_id);
app->buffer_replace_range(app, &buffer, range.start, range.end, 0, 0);
}
void
chronal_get_bindings(Bind_Helper *context){
set_hook(context, hook_start, chronal_init);
set_hook(context, hook_open_file, chronal_file_settings);
set_scroll_rule(context, smooth_scroll_rule);
/* *
* SECTION: Vim keybindings *
* */
/* Normal mode.
* aka "It's eating all my input, help!" mode.
* Shortcuts for navigation, entering various modes,
* dealing with the editor.
*/
begin_map(context, mapid_normal);
bind_vanilla_keys(context, do_nothing);
bind(context, 'w', MDFR_NONE, seek_forward_word_start);
bind(context, 'e', MDFR_NONE, seek_forward_word_end);
bind(context, 'b', MDFR_NONE, seek_backward_word_start);
bind(context, '$', MDFR_NONE, cmdid_seek_end_of_line);
bind(context, '0', MDFR_NONE, cmdid_seek_beginning_of_line);
bind(context, 'h', MDFR_NONE, cmdid_move_left);
bind(context, 'j', MDFR_NONE, cmdid_move_down);
bind(context, 'k', MDFR_NONE, cmdid_move_up);
bind(context, 'l', MDFR_NONE, cmdid_move_right);
bind(context, 'u', MDFR_CTRL, cmdid_page_up);
bind(context, 'd', MDFR_CTRL, cmdid_page_down);
bind(context, 'x', MDFR_NONE, cmdid_delete);
bind(context, 'u', MDFR_NONE, cmdid_undo);
bind(context, 'r', MDFR_CTRL, cmdid_redo);
bind(context, '/', MDFR_NONE, search);
bind(context, 'i', MDFR_NONE, enter_insert_mode);
bind(context, 'o', MDFR_NONE, newline_then_insert_after);
bind(context, 'O', MDFR_NONE, newline_then_insert_before);
bind(context, 'n', MDFR_CTRL, cmdid_word_complete);
// TEMP (will be replaced later by :statusbar commands)
bind(context, 'o', MDFR_CTRL, cmdid_interactive_open);
bind(context, 'c', MDFR_CTRL, cmdid_open_color_tweaker);
end_map(context);
/* Insert mode
* You type and it goes into the buffer. Nice and simple.
* Escape to exit.
*/
begin_map(context, mapid_insert);
inherit_map(context, mapid_nomap);
bind_vanilla_keys(context, cmdid_write_character);
bind(context, key_back, MDFR_NONE, cmdid_backspace);
bind(context, key_esc, MDFR_NONE, enter_normal_mode);
end_map(context);
#if 1
/* Chord "modes".
* They're not really an explicit mode per-say, but the meaning of key presses
* does change once a chord starts, and is context-dependent.
* TODO(chronister): I want these to properly build on each other.
*/
begin_map(context, mapid_chord_delete);
inherit_map(context, mapid_nomap);
bind(context, 'd', MDFR_NONE, delete_line);
bind(context, 'w', MDFR_NONE, delete_word);
end_map(context);
#endif
}

54
power/4coder_experiments.cpp Normal file
View File

@ -0,0 +1,54 @@
#include "4coder_default_bindings.cpp"
enum Experiment_Maps{
my_experiment_map = my_maps_count
};
HOOK_SIG(my_file_settings){
Buffer_Summary buffer = app->get_parameter_buffer(app, 0);
assert(buffer.exists);
int treat_as_code = 0;
int wrap_lines = 1;
if (buffer.file_name && buffer.size < (16 << 20)){
String ext = file_extension(make_string(buffer.file_name, buffer.file_name_len));
if (match(ext, make_lit_string("cpp"))) treat_as_code = 1;
else if (match(ext, make_lit_string("h"))) treat_as_code = 1;
else if (match(ext, make_lit_string("c"))) treat_as_code = 1;
else if (match(ext, make_lit_string("hpp"))) treat_as_code = 1;
}
if (treat_as_code){
wrap_lines = 0;
}
if (buffer.file_name[0] == '*'){
wrap_lines = 0;
}
push_parameter(app, par_lex_as_cpp_file, treat_as_code);
push_parameter(app, par_wrap_lines, !treat_as_code);
push_parameter(app, par_key_mapid, (treat_as_code)?((int)my_experiment_map):((int)mapid_file));
exec_command(app, cmdid_set_settings);
// no meaning for return
return(0);
}
CUSTOM_COMMAND_SIG(kill_rect){
// TODO
}
void experiments_get_bindings(Bind_Helper *context){
default_get_bindings(context, 0);
set_hook(context, hook_start, my_start);
set_hook(context, hook_open_file, experiment_file_settings);
begin_map(context, my_experiment_map);
inherit_map(my_code_map);
end_map(context);
}

97
test/4cpp_lexer_fsms.h Normal file
View File

@ -0,0 +1,97 @@
/*
* FSMs for 4c++ lexer
*
* 23.03.2016 (dd.mm.yyyy)
*/
// TOP
enum Lex_State{
LS_default,
LS_identifier,
LS_pound,
LS_pp,
LS_char,
LS_char_slashed,
LS_string,
LS_string_slashed,
LS_number,
LS_number0,
LS_float,
LS_crazy_float0,
LS_crazy_float1,
LS_hex,
LS_comment_pre,
LS_comment,
LS_comment_slashed,
LS_comment_block,
LS_comment_block_ending,
LS_dot,
LS_ellipsis,
LS_less,
LS_less_less,
LS_more,
LS_more_more,
LS_minus,
LS_arrow,
LS_and,
LS_or,
LS_plus,
LS_colon,
LS_star,
LS_modulo,
LS_caret,
LS_eq,
LS_bang,
LS_error_message,
//
LS_count
};
enum Lex_Int_State{
LSINT_default,
LSINT_u,
LSINT_l,
LSINT_L,
LSINT_ul,
LSINT_uL,
LSINT_ll,
LSINT_extra
};
enum Lex_INC_State{
LSINC_default,
LSINC_quotes,
LSINC_pointy,
LSINC_junk,
};
enum Lex_PP_State{
LSPP_default,
LSPP_include,
LSPP_macro_identifier,
LSPP_identifier,
LSPP_body_if,
LSPP_body,
LSPP_number,
LSPP_error,
LSPP_junk,
//
LSPP_count
};
struct Whitespace_FSM{
unsigned char pp_state;
unsigned char white_done;
};
struct Lex_FSM{
unsigned char state;
unsigned char int_state;
unsigned char emit_token;
unsigned char multi_line;
};
// BOTTOM

86
test/4cpp_lexer_tables.c Normal file
View File

@ -0,0 +1,86 @@
unsigned char main_fsm_eqclasses[] = {
0,1,1,1,1,1,1,1,1,2,3,4,4,4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,5,6,7,8,9,10,11,8,8,12,13,8,14,15,16,17,18,18,18,18,18,18,18,18,18,19,8,20,21,22,8,8,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,8,25,8,26,24,1,23,23,23,23,27,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,28,24,24,8,29,8,8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
};
const int num_eq_classes = 30;
unsigned char main_fsm_table[] = {
37,0,0,0,0,35,6,2,37,32,27,4,31,29,25,19,14,9,8,30,21,34,23,1,1,37,33,1,1,28,
38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,1,1,38,38,38,38,1,1,38,38,1,1,38,
39,3,2,39,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,3,3,40,40,40,40,3,3,40,40,3,3,40,
41,4,4,4,4,4,4,4,4,4,4,41,4,4,4,4,4,4,4,4,4,4,4,4,4,5,4,4,4,4,
42,4,4,6,5,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
43,6,6,6,6,6,43,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,6,6,6,6,
44,6,6,6,7,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,10,45,8,8,45,45,45,45,45,45,45,45,45,45,45,
46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,10,46,8,8,46,46,46,46,46,46,46,46,46,13,46,
47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,10,10,47,47,47,47,47,47,47,47,11,47,47,
48,48,48,48,48,48,48,48,48,48,48,48,48,48,12,48,48,12,12,48,48,48,48,48,48,48,48,48,48,48,
49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,12,12,49,49,49,49,49,49,49,49,49,49,49,
50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,13,13,50,50,50,50,13,50,50,50,13,50,50,
51,51,51,51,51,51,51,51,51,51,51,51,17,51,51,51,15,51,51,51,51,51,51,51,51,51,51,51,51,51,
52,15,15,52,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,15,15,15,15,
53,15,15,15,16,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,
54,17,17,17,17,17,17,17,17,17,17,17,18,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
55,17,17,17,17,17,17,17,17,17,17,17,18,17,17,17,55,17,17,17,17,17,17,17,17,17,17,17,17,17,
56,56,56,56,56,56,56,56,56,56,56,56,56,56,56,20,56,10,10,56,56,56,56,56,56,56,56,56,56,56,
57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,
58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,22,58,58,58,58,58,58,58,58,58,
59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,
60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,24,60,60,60,60,60,60,60,
61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,
62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,26,62,62,62,62,62,62,62,
63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,
64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,
65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,
66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,
67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,
68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,
69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,
70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,
71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,
72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,
73,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,
};
unsigned char main_fsm_multiline_table[] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
};

514
test/4cpp_new_lexer.h
View File

@ -5,6 +5,8 @@
#define FCPP_NEW_LEXER_INC
#include "../4cpp_lexer_types.h"
#include "4cpp_lexer_fsms.h"
#include "4cpp_lexer_tables.c"
namespace new_lex{
//
@ -332,80 +334,6 @@ cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token_i, int shift_amoun
}
}
enum Lex_State{
LS_default,
LS_identifier,
LS_pound,
LS_pp,
LS_char,
LS_char_slashed,
LS_string,
LS_string_slashed,
LS_number,
LS_number0,
LS_float,
LS_crazy_float0,
LS_crazy_float1,
LS_hex,
LS_comment_pre,
LS_comment,
LS_comment_slashed,
LS_comment_block,
LS_comment_block_ending,
LS_dot,
LS_ellipsis,
LS_less,
LS_less_less,
LS_more,
LS_more_more,
LS_minus,
LS_arrow,
LS_and,
LS_or,
LS_plus,
LS_colon,
LS_star,
LS_modulo,
LS_caret,
LS_eq,
LS_bang,
LS_error_message,
//
LS_count
};
enum Lex_Int_State{
LSINT_default,
LSINT_u,
LSINT_l,
LSINT_L,
LSINT_ul,
LSINT_uL,
LSINT_ll,
LSINT_extra
};
enum Lex_INC_State{
LSINC_default,
LSINC_quotes,
LSINC_pointy,
LSINC_junk,
};
enum Lex_PP_State{
LSPP_default,
LSPP_include,
LSPP_macro_identifier,
LSPP_identifier,
LSPP_body_if,
LSPP_body,
LSPP_number,
LSPP_error,
LSPP_junk,
//
LSPP_count
};
enum Pos_Update_Rule{
PUR_none,
PUR_unget_whitespace
@ -493,19 +421,6 @@ cpp_push_token_nonalloc(Cpp_Token *out_tokens, int *token_i, Cpp_Token token){
}
}
struct Whitespace_FSM{
unsigned char pp_state;
unsigned char white_done;
};
struct Lex_FSM{
unsigned char state;
unsigned char int_state;
unsigned char emit_token;
unsigned char multi_line;
unsigned char completed;
};
struct Lex_Data{
char *tb;
int tb_pos;
@ -600,7 +515,7 @@ int_fsm(Lex_FSM fsm, char c){
}
Lex_FSM
main_fsm(Lex_FSM fsm, unsigned char pp_state, char c){
main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
if (c == 0) fsm.emit_token = 1;
else
switch (pp_state){
@ -1030,14 +945,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
for (; S.wfsm.white_done == 0 && S.pos < end_pos;){
c = chunk[S.pos++];
wfsm = S.wfsm;
{
if (wfsm.pp_state != LSPP_default){
if (c == '\n') wfsm.pp_state = LSPP_default;
}
if (!(c == ' ' || c == '\n' || c == '\t' || c == '\r' || c == '\f' || c == '\v')){
wfsm.white_done = 1;
}
}
wfsm = whitespace_skip_fsm(wfsm, c);
S.wfsm = wfsm;
}
if (S.wfsm.white_done == 0){
@ -1052,401 +960,26 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
S.tb_pos = 0;
S.fsm = {0};
for(;;){
for (; S.fsm.emit_token == 0 && S.pos < end_pos;){
c = chunk[S.pos++];
S.tb[S.tb_pos++] = c;
fsm = S.fsm;
{
if (c == 0) fsm.emit_token = 1;
else
switch (S.pp_state){
case LSPP_error:
fsm.state = LS_error_message;
if (c == '\n') fsm.emit_token = 1;
break;
case LSPP_include:
switch (fsm.state){
case LSINC_default:
switch (c){
case '"': fsm.state = LSINC_quotes; break;
case '<': fsm.state = LSINC_pointy; break;
default: fsm.state = LSINC_junk; break;
}
break;
case LSINC_quotes:
if (c == '"') fsm.emit_token = 1;
break;
case LSINC_pointy:
if (c == '>') fsm.emit_token = 1;
break;
case LSINC_junk:
if (c == '\n') fsm.emit_token = 1;
break;
}
break;
default:
switch (fsm.state){
case LS_default:
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'){
fsm.state = LS_identifier;
}
else if (c >= '1' && c <= '9'){
fsm.state = LS_number;
}
else if (c == '0'){
fsm.state = LS_number0;
}
else switch (c){
case '\'': fsm.state = LS_char; break;
case '"': fsm.state = LS_string; break;
case '/': fsm.state = LS_comment_pre; break;
case '.': fsm.state = LS_dot; break;
case '<': fsm.state = LS_less; break;
case '>': fsm.state = LS_more; break;
case '-': fsm.state = LS_minus; break;
case '&': fsm.state = LS_and; break;
case '|': fsm.state = LS_or; break;
case '+': fsm.state = LS_plus; break;
case ':': fsm.state = LS_colon; break;
case '*': fsm.state = LS_star; break;
case '%': fsm.state = LS_modulo; break;
case '^': fsm.state = LS_caret; break;
case '=': fsm.state = LS_eq; break;
case '!': fsm.state = LS_bang; break;
case '#': fsm.state = LS_pound; break;
#define OperCase(op,type) case op: fsm.emit_token = 1; break;
OperCase('{', CPP_TOKEN_BRACE_OPEN);
OperCase('}', CPP_TOKEN_BRACE_CLOSE);
OperCase('[', CPP_TOKEN_BRACKET_OPEN);
OperCase(']', CPP_TOKEN_BRACKET_CLOSE);
OperCase('(', CPP_TOKEN_PARENTHESE_OPEN);
OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE);
OperCase('~', CPP_TOKEN_TILDE);
OperCase(',', CPP_TOKEN_COMMA);
OperCase(';', CPP_TOKEN_SEMICOLON);
OperCase('?', CPP_TOKEN_TERNARY_QMARK);
OperCase('@', CPP_TOKEN_JUNK);
OperCase('$', CPP_TOKEN_JUNK);
OperCase('\\', CPP_TOKEN_JUNK);
#undef OperCase
}
break;
case LS_identifier:
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
fsm.emit_token = 1;
}
break;
case LS_pound:
if (S.pp_state == LSPP_default){
if (c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'){
fsm.state = LS_pound;
}
else if (c == '\n'){
fsm.emit_token = 1;
}
else{
fsm.state = LS_pp;
}
}
else{
switch (c){
case '#': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
}
break;
case LS_pp:
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
fsm.emit_token = 1;
}
break;
case LS_char:
switch(c){
case '\'': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_char_slashed; break;
}
break;
case LS_char_slashed:
switch (c){
case '\r': case '\f': case '\v': break;
case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
default: fsm.state = LS_char; break;
}
break;
case LS_string:
switch(c){
case '\"': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_string_slashed; break;
}
break;
case LS_string_slashed:
switch (c){
case '\r': case '\f': case '\v': break;
case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
default: fsm.state = LS_string; break;
}
break;
case LS_number:
if (c >= '0' && c <= '9'){
fsm.state = LS_number;
}
else{
switch (c){
case '.': fsm.state = LS_float; break;
default: fsm.emit_token = 1; break;
}
}
break;
case LS_number0:
if (c >= '0' && c <= '9'){
fsm.state = LS_number;
}
else if (c == 'x'){
fsm.state = LS_hex;
}
else if (c == '.'){
fsm.state = LS_float;
}
else{
fsm.emit_token = 1;
}
break;
case LS_float:
if (!(c >= '0' && c <= '9')){
switch (c){
case 'e': fsm.state = LS_crazy_float0; break;
default: fsm.emit_token = 1; break;
}
}
break;
case LS_crazy_float0:
{
if ((c >= '0' && c <= '9') || c == '-'){
fsm.state = LS_crazy_float1;
}
else{
fsm.emit_token = 1;
}
}
break;
case LS_crazy_float1:
{
if (!(c >= '0' && c <= '9')){
fsm.emit_token = 1;
}
}
break;
case LS_hex:
if (!(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F')){
fsm.emit_token = 1;
}
break;
case LS_dot:
if (c >= '0' && c <= '9'){
fsm.state = LS_float;
}
else
switch (c){
case '.': fsm.state = LS_ellipsis; break;
case '*': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_ellipsis: fsm.emit_token = 1; break;
case LS_less:
switch (c){
case '<': fsm.state = LS_less_less; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_less_less:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_more:
switch (c){
case '>': fsm.state = LS_more_more; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_more_more:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_comment_pre:
switch (c){
case '/': fsm.state = LS_comment; break;
case '*': fsm.state = LS_comment_block; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_comment:
switch (c){
case '\\': fsm.state = LS_comment_slashed; break;
case '\n': fsm.emit_token = 1; break;
}
break;
case LS_comment_slashed:
switch (c){
case '\r': case '\f': case '\v': break;
default: fsm.state = LS_comment; break;
}
break;
case LS_comment_block:
switch (c){
case '*': fsm.state = LS_comment_block_ending; break;
}
break;
case LS_comment_block_ending:
switch (c){
case '*': fsm.state = LS_comment_block_ending; break;
case '/': fsm.emit_token = 1; break;
default: fsm.state = LS_comment_block; break;
}
break;
case LS_minus:
switch (c){
case '>': fsm.state = LS_arrow; break;
case '-': fsm.emit_token = 1; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_arrow:
switch (c){
case '*': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_and:
switch (c){
case '&': fsm.emit_token = 1; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_or:
switch (c){
case '|': fsm.emit_token = 1; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_plus:
switch (c){
case '+': fsm.emit_token = 1; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_colon:
switch (c){
case ':': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_star:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_modulo:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_caret:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_eq:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_bang:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
}
break;
}
if (S.pp_state == LSPP_default){
for (; S.fsm.state < LS_count && S.pos < end_pos;){
c = chunk[S.pos++];
S.tb[S.tb_pos++] = c;
int i = S.fsm.state*num_eq_classes + main_fsm_eqclasses[c];
S.fsm.multi_line |= main_fsm_multiline_table[i];
S.fsm.state = main_fsm_table[i];
}
S.fsm.emit_token = (S.fsm.state >= LS_count);
}
else{
for (; S.fsm.emit_token == 0 && S.pos < end_pos;){
c = chunk[S.pos++];
S.tb[S.tb_pos++] = c;
fsm = S.fsm;
fsm = main_fsm(fsm, S.pp_state, c);
S.fsm = fsm;
}
//fsm = main_fsm(fsm, S.pp_state, c);
S.fsm = fsm;
}
if (S.fsm.emit_token == 0){
DrYield(3, 1);
@ -1457,6 +990,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
Assert(S.fsm.emit_token == 1);
if (c != 0){
if (S.fsm.state >= LS_count) S.fsm.state -= LS_count;
pos_update_rule = PUR_none;
if (S.pp_state == LSPP_include){
switch (S.fsm.state){

90
test/experiment.cpp
View File

@ -30,25 +30,25 @@ dump_file(char *filename){
Data data = {};
HANDLE file;
DWORD hi, lo;
file = CreateFile(filename, GENERIC_READ, 0, 0,
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
if (file != INVALID_HANDLE_VALUE){
lo = GetFileSize(file, &hi);
assert(hi == 0);
data.size = (int)lo;
data.data = (byte*)malloc(data.size + 1);
ReadFile(file, data.data, lo, &lo, 0);
data.data[data.size] = 0;
assert((int)lo == data.size);
CloseHandle(file);
}
return(data);
}
@ -60,11 +60,11 @@ typedef struct File_Info{
typedef struct File_List{
// Ignore this, it's for internal stuff.
void *block;
// The list of files and folders.
File_Info *infos;
int count;
// Ignore this, it's for internal stuff.
int block_size;
} File_List;
@ -87,13 +87,13 @@ system_set_file_list(File_List *file_list, String directory){
append(&dir, directory);
char trail_str[] = "\\*";
append(&dir, trail_str);
char *c_str_dir = make_c_str(dir);
WIN32_FIND_DATA find_data;
HANDLE search;
search = FindFirstFileA(c_str_dir, &find_data);
if (search != INVALID_HANDLE_VALUE){
i32 count = 0;
i32 file_count = 0;
@ -116,12 +116,12 @@ system_set_file_list(File_List *file_list, String directory){
file_list->block = Win32GetMemory(required_size);
file_list->block_size = required_size;
}
file_list->infos = (File_Info*)file_list->block;
char *name = (char*)(file_list->infos + file_count);
if (file_list->block){
search = FindFirstFileA(c_str_dir, &find_data);
if (search != INVALID_HANDLE_VALUE){
File_Info *info = file_list->infos;
more_files = 1;
@ -130,7 +130,7 @@ system_set_file_list(File_List *file_list, String directory){
!match(find_data.cFileName, "..")){
info->folder = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
info->filename.str = name;
i32 i = 0;
for(;find_data.cFileName[i];++i) *name++ = find_data.cFileName[i];
info->filename.size = i;
@ -142,9 +142,9 @@ system_set_file_list(File_List *file_list, String directory){
more_files = FindNextFile(search, &find_data);
}while(more_files);
FindClose(search);
file_list->count = file_count;
}else{
Win32FreeMemory(file_list->block);
file_list->block = 0;
@ -210,14 +210,14 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks){
int k, chunk_size, is_last;
extension = file_extension(make_string_slowly(filename));
if (match(extension, "cpp") || match(extension, "h")){
file_data = dump_file(filename);
if (file_data.size < (100 << 10)){
pass = 1;
if (verbose >= 0) printf("testing on file: %s\n", filename);
exp->test_total++;
exp->correct_stack.count = 0;
exp->testing_stack.count = 0;
@ -226,16 +226,16 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks){
file_cpp.data = (char*)file_data.data;
file_cpp.size = file_data.size;
ld.tb = (char*)malloc(file_data.size + 1);
{
i64 start;
start = __rdtsc();
cpp_lex_file_nonalloc(file_cpp, &exp->correct_stack, lex_data);
time.handcoded += (__rdtsc() - start);
start = __rdtsc();
if (chunks){
int relevant_size = file_data.size + 1;
@ -246,19 +246,19 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks){
chunk_size = relevant_size - k;
is_last = 1;
}
int result = new_lex::cpp_lex_nonalloc(&ld, (char*)file_data.data + k, chunk_size, &exp->testing_stack);
if (result == 0 || result == 2) break;
}
}
}
else{
new_lex::cpp_lex_nonalloc(&ld, (char*)file_data.data, file_data.size, &exp->testing_stack);
}
time.fsm += (__rdtsc() - start);
}
free(ld.tb);
if (exp->correct_stack.count != exp->testing_stack.count){
pass = 0;
if (verbose >= 0){
@ -307,7 +307,7 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks){
if (verbose >= 0) printf("test failed, you failed, fix it now!\n\n");
}
}
free(file_data.data);
}
}
@ -334,37 +334,37 @@ show_time(Times t, int repeats, char *type){
#define BASE_DIR "w:/4ed/data/test/"
int main(){
int repeats = 100;
int verbose_level = -1;
int chunk_start = 1;
int chunk_end = 16;
#define TEST_FILE "lexer_test.cpp"
int chunk_start = 0;
int chunk_end = 0;
#define TEST_FILE "autotab.cpp"
#define SINGLE_ITEM 0
int chunks = (chunk_start > 0 && chunk_start <= chunk_end);
int c = 0;
char test_directory[] = BASE_DIR;
File_List all_files = {};
Experiment exp = {};
Experiment chunk_exp = {};
Times exp_t = {};
Times chunk_exp_t = {};
init_test_stack(&exp.correct_stack);
init_test_stack(&exp.testing_stack);
init_test_stack(&chunk_exp.correct_stack);
init_test_stack(&chunk_exp.testing_stack);
AllowLocal(test_directory);
AllowLocal(all_files);
#if SINGLE_ITEM
(void)(repeats);
(void)(verbose_level);
if (chunks){
begin_t(&chunk_exp_t);
printf("With chunks of %d\n", chunks);
@ -373,14 +373,14 @@ int main(){
}
end_t(&chunk_exp_t);
}
begin_t(&exp_t);
printf("Unchunked\n");
run_experiment(&exp, BASE_DIR TEST_FILE, 1, 0);
end_t(&exp_t);
#else
system_set_file_list(&all_files, make_lit_string(test_directory));
for (int j = 0; j < repeats; ++j){
@ -393,7 +393,7 @@ int main(){
}
end_t(&chunk_exp_t);
}
begin_t(&exp_t);
if (verbose_level == -1 && chunks){
for (c = chunk_start; c <= chunk_end; ++c){
@ -413,7 +413,7 @@ int main(){
printf("chunks of sizes %d through %d tested\n", chunk_start, chunk_end);
printf("chunked passed %d / %d tests\n", chunk_exp.passed_total, chunk_exp.test_total);
}
printf("unchunk passed %d / %d tests\n", exp.passed_total, exp.test_total);
if (passed(exp) && (chunks == 0 || passed(chunk_exp))){
@ -421,8 +421,8 @@ int main(){
show_time(chunk_exp_t, repeats, "Chunked");
}
show_time(exp_t, repeats, "Unchunked");
}
}
return(0);
}

592
test/fsm_table_generator.cpp Normal file
View File

@ -0,0 +1,592 @@
/*
* FSM table generator:
* Generate FSM tables as ".c" files from FSM functions.
*
* 23.03.2016 (dd.mm.yyyy)
*/
// TOP
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "4cpp_lexer_fsms.h"
Whitespace_FSM
whitespace_skip_fsm(Whitespace_FSM wfsm, char c){
if (wfsm.pp_state != LSPP_default){
if (c == '\n') wfsm.pp_state = LSPP_default;
}
if (!(c == ' ' || c == '\n' || c == '\t' || c == '\r' || c == '\f' || c == '\v')){
wfsm.white_done = 1;
}
return(wfsm);
}
Lex_FSM
int_fsm(Lex_FSM fsm, char c){
switch (fsm.int_state){
case LSINT_default:
switch (c){
case 'u': case 'U': fsm.int_state = LSINT_u; break;
case 'l': fsm.int_state = LSINT_l; break;
case 'L': fsm.int_state = LSINT_L; break;
default: fsm.emit_token = 1; break;
}
break;
case LSINT_u:
switch (c){
case 'l': fsm.int_state = LSINT_ul; break;
case 'L': fsm.int_state = LSINT_uL; break;
default: fsm.emit_token = 1; break;
}
break;
case LSINT_l:
switch (c){
case 'l': fsm.int_state = LSINT_ll; break;
case 'U': case 'u': fsm.int_state = LSINT_extra; break;
default: fsm.emit_token = 1; break;
}
break;
case LSINT_L:
switch (c){
case 'L': fsm.int_state = LSINT_ll; break;
case 'U': case 'u': fsm.int_state = LSINT_extra; break;
default: fsm.emit_token = 1; break;
}
break;
case LSINT_ul:
switch (c){
case 'l': fsm.int_state = LSINT_extra; break;
default: fsm.emit_token = 1; break;
}
break;
case LSINT_uL:
switch (c){
case 'L': fsm.int_state = LSINT_extra; break;
default: fsm.emit_token = 1; break;
}
break;
case LSINT_ll:
switch (c){
case 'u': case 'U': fsm.int_state = LSINT_extra; break;
default: fsm.emit_token = 1; break;
}
break;
case LSINT_extra:
fsm.emit_token = 1;
break;
}
return(fsm);
}
Lex_FSM
main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
if (c == 0) fsm.emit_token = 1;
else
switch (pp_state){
case LSPP_error:
fsm.state = LS_error_message;
if (c == '\n') fsm.emit_token = 1;
break;
case LSPP_include:
switch (fsm.state){
case LSINC_default:
switch (c){
case '"': fsm.state = LSINC_quotes; break;
case '<': fsm.state = LSINC_pointy; break;
default: fsm.state = LSINC_junk; break;
}
break;
case LSINC_quotes:
if (c == '"') fsm.emit_token = 1;
break;
case LSINC_pointy:
if (c == '>') fsm.emit_token = 1;
break;
case LSINC_junk:
if (c == '\n') fsm.emit_token = 1;
break;
}
break;
default:
switch (fsm.state){
case LS_default:
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'){
fsm.state = LS_identifier;
}
else if (c >= '1' && c <= '9'){
fsm.state = LS_number;
}
else if (c == '0'){
fsm.state = LS_number0;
}
else switch (c){
case '\'': fsm.state = LS_char; break;
case '"': fsm.state = LS_string; break;
case '/': fsm.state = LS_comment_pre; break;
case '.': fsm.state = LS_dot; break;
case '<': fsm.state = LS_less; break;
case '>': fsm.state = LS_more; break;
case '-': fsm.state = LS_minus; break;
case '&': fsm.state = LS_and; break;
case '|': fsm.state = LS_or; break;
case '+': fsm.state = LS_plus; break;
case ':': fsm.state = LS_colon; break;
case '*': fsm.state = LS_star; break;
case '%': fsm.state = LS_modulo; break;
case '^': fsm.state = LS_caret; break;
case '=': fsm.state = LS_eq; break;
case '!': fsm.state = LS_bang; break;
case '#': fsm.state = LS_pound; break;
#define OperCase(op,type) case op: fsm.emit_token = 1; break;
OperCase('{', CPP_TOKEN_BRACE_OPEN);
OperCase('}', CPP_TOKEN_BRACE_CLOSE);
OperCase('[', CPP_TOKEN_BRACKET_OPEN);
OperCase(']', CPP_TOKEN_BRACKET_CLOSE);
OperCase('(', CPP_TOKEN_PARENTHESE_OPEN);
OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE);
OperCase('~', CPP_TOKEN_TILDE);
OperCase(',', CPP_TOKEN_COMMA);
OperCase(';', CPP_TOKEN_SEMICOLON);
OperCase('?', CPP_TOKEN_TERNARY_QMARK);
OperCase('@', CPP_TOKEN_JUNK);
OperCase('$', CPP_TOKEN_JUNK);
OperCase('\\', CPP_TOKEN_JUNK);
#undef OperCase
}
break;
case LS_identifier:
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
fsm.emit_token = 1;
}
break;
case LS_pound:
if (pp_state == LSPP_default){
if (c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'){
fsm.state = LS_pound;
}
else if (c == '\n'){
fsm.emit_token = 1;
}
else{
fsm.state = LS_pp;
}
}
else{
switch (c){
case '#': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
}
break;
case LS_pp:
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
fsm.emit_token = 1;
}
break;
case LS_char:
switch(c){
case '\'': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_char_slashed; break;
}
break;
case LS_char_slashed:
switch (c){
case '\r': case '\f': case '\v': break;
case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
default: fsm.state = LS_char; break;
}
break;
case LS_string:
switch(c){
case '\"': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_string_slashed; break;
}
break;
case LS_string_slashed:
switch (c){
case '\r': case '\f': case '\v': break;
case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
default: fsm.state = LS_string; break;
}
break;
case LS_number:
if (c >= '0' && c <= '9'){
fsm.state = LS_number;
}
else{
switch (c){
case '.': fsm.state = LS_float; break;
default: fsm.emit_token = 1; break;
}
}
break;
case LS_number0:
if (c >= '0' && c <= '9'){
fsm.state = LS_number;
}
else if (c == 'x'){
fsm.state = LS_hex;
}
else if (c == '.'){
fsm.state = LS_float;
}
else{
fsm.emit_token = 1;
}
break;
case LS_float:
if (!(c >= '0' && c <= '9')){
switch (c){
case 'e': fsm.state = LS_crazy_float0; break;
default: fsm.emit_token = 1; break;
}
}
break;
case LS_crazy_float0:
{
if ((c >= '0' && c <= '9') || c == '-'){
fsm.state = LS_crazy_float1;
}
else{
fsm.emit_token = 1;
}
}
break;
case LS_crazy_float1:
{
if (!(c >= '0' && c <= '9')){
fsm.emit_token = 1;
}
}
break;
case LS_hex:
if (!(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F')){
fsm.emit_token = 1;
}
break;
case LS_dot:
if (c >= '0' && c <= '9'){
fsm.state = LS_float;
}
else
switch (c){
case '.': fsm.state = LS_ellipsis; break;
case '*': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_ellipsis: fsm.emit_token = 1; break;
case LS_less:
switch (c){
case '<': fsm.state = LS_less_less; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_less_less:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_more:
switch (c){
case '>': fsm.state = LS_more_more; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_more_more:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_comment_pre:
switch (c){
case '/': fsm.state = LS_comment; break;
case '*': fsm.state = LS_comment_block; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_comment:
switch (c){
case '\\': fsm.state = LS_comment_slashed; break;
case '\n': fsm.emit_token = 1; break;
}
break;
case LS_comment_slashed:
switch (c){
case '\r': case '\f': case '\v': break;
default: fsm.state = LS_comment; break;
}
break;
case LS_comment_block:
switch (c){
case '*': fsm.state = LS_comment_block_ending; break;
}
break;
case LS_comment_block_ending:
switch (c){
case '*': fsm.state = LS_comment_block_ending; break;
case '/': fsm.emit_token = 1; break;
default: fsm.state = LS_comment_block; break;
}
break;
case LS_minus:
switch (c){
case '>': fsm.state = LS_arrow; break;
case '-': fsm.emit_token = 1; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_arrow:
switch (c){
case '*': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_and:
switch (c){
case '&': fsm.emit_token = 1; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_or:
switch (c){
case '|': fsm.emit_token = 1; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_plus:
switch (c){
case '+': fsm.emit_token = 1; break;
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_colon:
switch (c){
case ':': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_star:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_modulo:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_caret:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_eq:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_bang:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
}
break;
}
return(fsm);
}
void
begin_table(FILE *file, char *type, char *table_name){
fprintf(file, "unsigned %s %s[] = {\n", type, table_name);
}
void
do_table_item(FILE *file, unsigned short item){
fprintf(file, "%d,", (int)item);
}
void
end_row(FILE *file){
fprintf(file, "\n");
}
void
end_table(FILE *file){
fprintf(file, "};\n\n");
}
int main(){
FILE *file;
file = fopen("4cpp_lexer_tables.c", "wb");
unsigned char *full_transition_table = (unsigned char*)malloc(LS_count * 256);
unsigned char *marks = (unsigned char*)malloc(LS_count * 256);
unsigned char *eq_class = (unsigned char*)malloc(LS_count * 256);
unsigned char *eq_class_rep = (unsigned char*)malloc(LS_count * 256);
memset(marks, 0, 256);
int i = 0;
Lex_FSM fsm = {0};
Lex_FSM new_fsm;
for (unsigned short c = 0; c < 256; ++c){
for (unsigned char state = 0; state < LS_count; ++state){
fsm.state = state;
fsm.emit_token = 0;
new_fsm = main_fsm(fsm, LSPP_default, (unsigned char)c);
full_transition_table[i++] = new_fsm.state + LS_count*new_fsm.emit_token;
}
}
unsigned char eq_class_counter = 0;
unsigned char *c_line = full_transition_table;
for (unsigned short c = 0; c < 256; ++c){
if (marks[c] == 0){
eq_class[c] = eq_class_counter;
eq_class_rep[eq_class_counter] = (unsigned char)c;
unsigned char *c2_line = c_line + LS_count;
for (unsigned short c2 = c + 1; c2 < 256; ++c2){
if (memcmp(c_line, c2_line, LS_count) == 0){
marks[c2] = 1;
eq_class[c2] = eq_class_counter;
}
c2_line += LS_count;
}
++eq_class_counter;
}
c_line += LS_count;
}
unsigned char *reduced_transition_table = (unsigned char*)malloc(LS_count * eq_class_counter);
unsigned char *reduced_multiline_table = (unsigned char*)malloc(LS_count * eq_class_counter);
i = 0;
for (unsigned char state = 0; state < LS_count; ++state){
fsm.state = state;
for (unsigned short eq = 0; eq < eq_class_counter; ++eq){
fsm.emit_token = 0;
fsm.multi_line = 0;
new_fsm = main_fsm(fsm, LSPP_default, eq_class_rep[eq]);
reduced_transition_table[i] = new_fsm.state + LS_count*new_fsm.emit_token;
reduced_multiline_table[i] = new_fsm.multi_line;
++i;
}
}
begin_table(file, "char", "main_fsm_eqclasses");
for (unsigned short c = 0; c < 256; ++c){
do_table_item(file, eq_class[c]);
}
end_row(file);
end_table(file);
fprintf(file, "const int num_eq_classes = %d;\n\n", eq_class_counter);
i = 0;
begin_table(file, "char", "main_fsm_table");
for (unsigned char state = 0; state < LS_count; ++state){
for (unsigned short c = 0; c < eq_class_counter; ++c){
do_table_item(file, reduced_transition_table[i++]);
}
end_row(file);
}
end_table(file);
i = 0;
begin_table(file, "char", "main_fsm_multiline_table");
for (unsigned char state = 0; state < LS_count; ++state){
for (unsigned short c = 0; c < eq_class_counter; ++c){
do_table_item(file, reduced_multiline_table[i++]);
}
end_row(file);
}
end_table(file);
fclose(file);
return(0);
}
// BOTTOM

35
win32_4ed.cpp
View File

@ -463,26 +463,33 @@ Sys_Set_File_List_Sig(system_set_file_list){
}
}
// TODO(allen): proper "is terminated" check
internal
Sys_File_Unique_Hash_Sig(system_file_unique_hash){
Unique_Hash hash = {0};
BY_HANDLE_FILE_INFORMATION info;
HANDLE handle;
handle = CreateFile(filename.str, GENERIC_READ, 0, 0,
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
*success = 0;
if (handle && handle != INVALID_HANDLE_VALUE){
if (GetFileInformationByHandle(handle, &info)){
hash.d[2] = info.dwVolumeSerialNumber;
hash.d[1] = info.nFileIndexHigh;
hash.d[0] = info.nFileIndexLow;
*success = 1;
}
char space[1024];
String str;
if (filename.size < sizeof(space)){
str = make_fixed_width_string(space);
copy(&str, filename);
terminate_with_null(&str);
CloseHandle(handle);
handle = CreateFile(str.str, GENERIC_READ, 0, 0,
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
*success = 0;
if (handle && handle != INVALID_HANDLE_VALUE){
if (GetFileInformationByHandle(handle, &info)){
hash.d[2] = info.dwVolumeSerialNumber;
hash.d[1] = info.nFileIndexHigh;
hash.d[0] = info.nFileIndexLow;
*success = 1;
}
CloseHandle(handle);
}
}
return(hash);