ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more lexer table generation

This commit is contained in:
Allen Webster 2016-03-24 10:01:53 -04:00
parent 1f415d13fc
commit fa0a06d545
6 changed files with 543 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.bat
View File

@ -1,5 +1,5 @@
@echo off @echo off
REM "w:\4ed\misc\build_exp.bat" /Zi "w:\4ed\misc\build_exp.bat" /Zi
"w:\4ed\misc\build_all.bat" /DFRED_SUPER /DFRED_NOT_PACKAGE /Zi REM "w:\4ed\misc\build_all.bat" /DFRED_SUPER /DFRED_NOT_PACKAGE /Zi
REM "w:\4ed\misc\build_all.bat" /O2 /Zi REM "w:\4ed\misc\build_all.bat" /O2 /Zi

2
test/4cpp_lexer_fsms.h
View File

@ -12,8 +12,10 @@ enum Lex_State{
LS_pound, LS_pound,
LS_pp, LS_pp,
LS_char, LS_char,
LS_char_multiline,
LS_char_slashed, LS_char_slashed,
LS_string, LS_string,
LS_string_multiline,
LS_string_slashed, LS_string_slashed,
LS_number, LS_number,
LS_number0, LS_number0,

404
test/4cpp_lexer_tables.c
View File

@ -1,86 +1,334 @@
unsigned char main_fsm_eqclasses[] = { unsigned short whitespace_fsm_eq_classes[] = {
0,1,1,1,1,1,1,1,1,2,3,4,4,4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,5,6,7,8,9,10,11,8,8,12,13,8,14,15,16,17,18,18,18,18,18,18,18,18,18,19,8,20,21,22,8,8,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,8,25,8,26,24,1,23,23,23,23,27,23,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,28,24,24,8,29,8,8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9,18, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
}; };
const int num_eq_classes = 30; const int num_whitespace_fsm_eq_classes = 3;
unsigned char whitespace_fsm_table[] = {
9,10,11,12,13,14,15,16,17,
0, 1, 2, 3, 4, 5, 6, 7, 8,
0, 0, 0, 0, 0, 0, 0, 0, 0,
};
unsigned char multiline_state_table[] = {
0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
unsigned short main_fsm_eq_classes[] = {
0,39,39,39,39,39,39,39,39,78,117,156,156,156,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,78,195,234,273,312,351,390,429,312,312,468,507,312,546,585,624,663,702,702,702,702,702,702,702,702,702,741,312,780,819,858,312,312,897,897,897,897,897,897,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,312,975,312,1014,936,39,897,897,897,897,1053,897,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,936,1092,936,936,312,1131,312,312,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
};
const int num_main_fsm_eq_classes = 30;
unsigned char main_fsm_table[] = { unsigned char main_fsm_table[] = {
37,0,0,0,0,35,6,2,37,32,27,4,31,29,25,19,14,9,8,30,21,34,23,1,1,37,33,1,1,28, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,1,1,38,38,38,38,1,1,38,38,1,1,38, 0,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,3,2,39,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, 0,40, 2,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,40,3,3,40,40,40,40,3,3,40,40,3,3,40, 0,40,41,42, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
41,4,4,4,4,4,4,4,4,4,4,41,4,4,4,4,4,4,4,4,4,4,4,4,4,5,4,4,4,4, 0,40, 2,42, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
42,4,4,6,5,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, 37,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
43,6,6,6,6,6,43,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,6,6,6,6, 7,40, 3,42, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
44,6,6,6,7,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, 2,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,10,45,8,8,45,45,45,45,45,45,45,45,45,45,45, 39,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,10,46,8,8,46,46,46,46,46,46,46,46,46,13,46, 34,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,10,10,47,47,47,47,47,47,47,47,11,47,47, 29,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
48,48,48,48,48,48,48,48,48,48,48,48,48,48,12,48,48,12,12,48,48,48,48,48,48,48,48,48,48,48, 4,40, 3,42,43,44, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,12,12,49,49,49,49,49,49,49,49,49,49,49, 33,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,19,17,17,20,20,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,13,13,50,50,50,50,13,50,50,50,13,50,50, 31,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
51,51,51,51,51,51,51,51,51,51,51,51,17,51,51,51,15,51,51,51,51,51,51,51,51,51,51,51,51,51, 27,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,14,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
52,15,15,52,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,16,15,15,15,15, 21,40, 3,42, 4, 5, 4, 7, 8, 7,12,12,51,52,53,54,55,17,17,19,19,22,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
53,15,15,15,16,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15, 16,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,17,17,17,19,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
54,17,17,17,17,17,17,17,17,17,17,17,18,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17, 11, 1, 3, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
55,17,17,17,17,17,17,17,17,17,17,17,18,17,17,17,55,17,17,17,17,17,17,17,17,17,17,17,17,17, 10, 1, 3, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
56,56,56,56,56,56,56,56,56,56,56,56,56,56,56,20,56,10,10,56,56,56,56,56,56,56,56,56,56,56, 32,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57, 23,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,24,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,22,58,58,58,58,58,58,58,58,58, 36,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59,59, 25,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,26,65,28,67,68,69,70,71,72,73,74,75,76,38,
60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,24,60,60,60,60,60,60,60, 1, 1, 3, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61, 1, 1, 3, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,26,62,62,62,62,62,62,62, 39,40, 3,42, 6, 6, 4, 9, 9, 7,49,50,51,52,53,54,55,18,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63, 35,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64, 1, 1, 3, 3, 4, 5, 4, 7, 8, 7,49,50,13,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65, 1, 1, 3, 3, 4, 5, 4, 7, 8, 7,49,15,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66, 30,40, 3,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,67,
68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,68,
69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,69,
70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,
71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,
72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,72,
73,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,
}; };
unsigned char main_fsm_multiline_table[] = { unsigned short pp_include_fsm_eq_classes[] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,39,39,39,39,39,39,39,39,39,78,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,117,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,156,39,195,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, };
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, const int num_pp_include_fsm_eq_classes = 6;
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, unsigned char pp_include_fsm_table[] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 3, 1, 2,42, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 1,40, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 3, 1,41, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, };
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, unsigned short pp_macro_fsm_eq_classes[] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,39,39,39,39,39,39,39,39,39,78,117,117,117,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,156,195,234,273,312,351,390,273,273,429,468,273,507,546,585,624,663,663,663,663,663,663,663,663,663,702,273,741,780,819,273,273,858,858,858,858,858,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,273,936,273,975,897,39,858,858,858,858,1014,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,1053,897,897,273,1092,273,273,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, };
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, const int num_pp_macro_fsm_eq_classes = 29;
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, unsigned char pp_macro_fsm_table[] = {
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,40,41,42, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,40,41,42, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 37,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 7,40,41,42, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 2,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 39,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 34,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 29,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 4,40,41,42,43,44, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 33,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,19,17,17,20,20,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 31,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 27,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,14,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 21,40,41,42, 4, 5, 4, 7, 8, 7,12,12,51,52,53,54,55,17,17,19,19,22,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 16,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,17,17,17,19,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
11, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
10, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
32,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
23,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,24,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
36,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
25,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,26,65,28,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 6, 6, 4, 9, 9, 7,49,50,51,52,53,54,55,18,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
35,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,13,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,15,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
30,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
};
unsigned short pp_identifier_fsm_eq_classes[] = {
0,39,39,39,39,39,39,39,39,39,78,117,117,117,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,156,195,234,273,312,351,390,273,273,429,468,273,507,546,585,624,663,663,663,663,663,663,663,663,663,702,273,741,780,819,273,273,858,858,858,858,858,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,273,936,273,975,897,39,858,858,858,858,1014,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,1053,897,897,273,1092,273,273,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
};
const int num_pp_identifier_fsm_eq_classes = 29;
unsigned char pp_identifier_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41,42, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
2,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
34,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
29,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
4,40,41,42,43,44, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
33,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,19,17,17,20,20,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
31,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
27,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,14,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
21,40,41,42, 4, 5, 4, 7, 8, 7,12,12,51,52,53,54,55,17,17,19,19,22,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
16,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,17,17,17,19,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
11, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
10, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
32,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
23,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,24,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
36,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
25,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,26,65,28,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 6, 6, 4, 9, 9, 7,49,50,51,52,53,54,55,18,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
35,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,13,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,15,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
30,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
};
unsigned short pp_body_if_fsm_eq_classes[] = {
0,39,39,39,39,39,39,39,39,39,78,117,117,117,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,156,195,234,273,312,351,390,273,273,429,468,273,507,546,585,624,663,663,663,663,663,663,663,663,663,702,273,741,780,819,273,273,858,858,858,858,858,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,273,936,273,975,897,39,858,858,858,858,1014,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,1053,897,897,273,1092,273,273,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
};
const int num_pp_body_if_fsm_eq_classes = 29;
unsigned char pp_body_if_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41,42, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
2,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
34,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
29,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
4,40,41,42,43,44, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
33,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,19,17,17,20,20,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
31,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
27,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,14,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
21,40,41,42, 4, 5, 4, 7, 8, 7,12,12,51,52,53,54,55,17,17,19,19,22,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
16,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,17,17,17,19,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
11, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
10, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
32,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
23,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,24,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
36,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
25,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,26,65,28,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 6, 6, 4, 9, 9, 7,49,50,51,52,53,54,55,18,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
35,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,13,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,15,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
30,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
};
unsigned short pp_body_fsm_eq_classes[] = {
0,39,39,39,39,39,39,39,39,39,78,117,117,117,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,156,195,234,273,312,351,390,273,273,429,468,273,507,546,585,624,663,663,663,663,663,663,663,663,663,702,273,741,780,819,273,273,858,858,858,858,858,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,273,936,273,975,897,39,858,858,858,858,1014,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,1053,897,897,273,1092,273,273,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
};
const int num_pp_body_fsm_eq_classes = 29;
unsigned char pp_body_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41,42, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
2,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
34,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
29,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
4,40,41,42,43,44, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
33,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,19,17,17,20,20,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
31,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
27,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,14,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
21,40,41,42, 4, 5, 4, 7, 8, 7,12,12,51,52,53,54,55,17,17,19,19,22,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
16,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,17,17,17,19,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
11, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
10, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
32,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
23,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,24,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
36,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
25,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,26,65,28,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 6, 6, 4, 9, 9, 7,49,50,51,52,53,54,55,18,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
35,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,13,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,15,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
30,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
};
unsigned short pp_number_fsm_eq_classes[] = {
0,39,39,39,39,39,39,39,39,39,78,117,117,117,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,156,195,234,273,312,351,390,273,273,429,468,273,507,546,585,624,663,663,663,663,663,663,663,663,663,702,273,741,780,819,273,273,858,858,858,858,858,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,273,936,273,975,897,39,858,858,858,858,1014,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,1053,897,897,273,1092,273,273,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
};
const int num_pp_number_fsm_eq_classes = 29;
unsigned char pp_number_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41,42, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
2,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
34,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
29,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
4,40,41,42,43,44, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
33,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,19,17,17,20,20,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
31,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
27,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,14,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
21,40,41,42, 4, 5, 4, 7, 8, 7,12,12,51,52,53,54,55,17,17,19,19,22,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
16,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,17,17,17,19,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
11, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
10, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
32,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
23,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,24,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
36,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
25,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,26,65,28,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 6, 6, 4, 9, 9, 7,49,50,51,52,53,54,55,18,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
35,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,13,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,15,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
30,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
};
unsigned short pp_error_fsm_eq_classes[] = {
0,39,39,39,39,39,39,39,39,39,78,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
};
const int num_pp_error_fsm_eq_classes = 3;
unsigned char pp_error_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,
77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,
};
unsigned short pp_junk_fsm_eq_classes[] = {
0,39,39,39,39,39,39,39,39,39,78,117,117,117,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,156,195,234,273,312,351,390,273,273,429,468,273,507,546,585,624,663,663,663,663,663,663,663,663,663,702,273,741,780,819,273,273,858,858,858,858,858,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,273,936,273,975,897,39,858,858,858,858,1014,858,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,897,1053,897,897,273,1092,273,273,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,
};
const int num_pp_junk_fsm_eq_classes = 29;
unsigned char pp_junk_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41,42, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41,42, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
2,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
34,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
29,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
4,40,41,42,43,44, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
33,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,19,17,17,20,20,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
31,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
27,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,14,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
21,40,41,42, 4, 5, 4, 7, 8, 7,12,12,51,52,53,54,55,17,17,19,19,22,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
16,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,17,17,17,19,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
11, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
10, 1,41, 3, 4, 5, 4, 7, 8, 7,10,10,12,14,14,15,55,17,17,19,19,12,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
32,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
23,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,24,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
36,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
25,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,26,65,28,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
39,40,41,42, 6, 6, 4, 9, 9, 7,49,50,51,52,53,54,55,18,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
35,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,50,13,52,53,15,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
1, 1,41, 3, 4, 5, 4, 7, 8, 7,49,15,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
30,40,41,42, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
};
unsigned short * get_eq_classes[] = {
main_fsm_eq_classes,
pp_include_fsm_eq_classes,
pp_macro_fsm_eq_classes,
pp_identifier_fsm_eq_classes,
pp_body_if_fsm_eq_classes,
pp_body_fsm_eq_classes,
pp_number_fsm_eq_classes,
pp_error_fsm_eq_classes,
pp_junk_fsm_eq_classes,
};
unsigned char * get_table[] = {
main_fsm_table,
pp_include_fsm_table,
pp_macro_fsm_table,
pp_identifier_fsm_table,
pp_body_if_fsm_table,
pp_body_fsm_table,
pp_number_fsm_table,
pp_error_fsm_table,
pp_junk_fsm_table,
}; };

45
test/4cpp_new_lexer.h
View File

@ -921,8 +921,6 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
int token_i = token_stack_out->count; int token_i = token_stack_out->count;
int max_token_i = token_stack_out->max_count; int max_token_i = token_stack_out->max_count;
Lex_FSM fsm = {0};
Whitespace_FSM wfsm = {0};
Pos_Update_Rule pos_update_rule = PUR_none; Pos_Update_Rule pos_update_rule = PUR_none;
char c = 0; char c = 0;
@ -942,12 +940,13 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
S.wfsm.white_done = 0; S.wfsm.white_done = 0;
S.wfsm.pp_state = S.pp_state; S.wfsm.pp_state = S.pp_state;
for(;;){ for(;;){
for (; S.wfsm.white_done == 0 && S.pos < end_pos;){ for (; S.wfsm.pp_state < LSPP_count && S.pos < end_pos;){
c = chunk[S.pos++]; c = chunk[S.pos++];
wfsm = S.wfsm; int i = S.wfsm.pp_state + whitespace_fsm_eq_classes[c];
wfsm = whitespace_skip_fsm(wfsm, c); S.wfsm.pp_state = whitespace_fsm_table[i];
S.wfsm = wfsm;
} }
S.wfsm.white_done = (S.wfsm.pp_state >= LSPP_count);
if (S.wfsm.white_done == 0){ if (S.wfsm.white_done == 0){
DrYield(4, 1); DrYield(4, 1);
} }
@ -955,32 +954,27 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
} }
--S.pos; --S.pos;
S.pp_state = S.wfsm.pp_state; S.pp_state = S.wfsm.pp_state;
if (S.pp_state >= LSPP_count){
S.pp_state -= LSPP_count;
}
S.token_start = S.pos; S.token_start = S.pos;
S.tb_pos = 0; S.tb_pos = 0;
S.fsm = {0}; S.fsm = {0};
for(;;){ for(;;){
if (S.pp_state == LSPP_default){ unsigned short *eq_classes = get_eq_classes[S.pp_state];
unsigned char *fsm_table = get_table[S.pp_state];
for (; S.fsm.state < LS_count && S.pos < end_pos;){ for (; S.fsm.state < LS_count && S.pos < end_pos;){
c = chunk[S.pos++]; c = chunk[S.pos++];
S.tb[S.tb_pos++] = c; S.tb[S.tb_pos++] = c;
int i = S.fsm.state*num_eq_classes + main_fsm_eqclasses[c]; int i = S.fsm.state + eq_classes[c];
S.fsm.multi_line |= main_fsm_multiline_table[i]; S.fsm.state = fsm_table[i];
S.fsm.state = main_fsm_table[i]; S.fsm.multi_line |= multiline_state_table[S.fsm.state];
} }
S.fsm.emit_token = (S.fsm.state >= LS_count); S.fsm.emit_token = (S.fsm.state >= LS_count);
}
else{
for (; S.fsm.emit_token == 0 && S.pos < end_pos;){
c = chunk[S.pos++];
S.tb[S.tb_pos++] = c;
fsm = S.fsm;
fsm = main_fsm(fsm, S.pp_state, c);
S.fsm = fsm;
}
}
if (S.fsm.emit_token == 0){ if (S.fsm.emit_token == 0){
DrYield(3, 1); DrYield(3, 1);
} }
@ -1180,11 +1174,21 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
S.token.flags = 0; S.token.flags = 0;
break; break;
case LS_char_multiline:
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
S.token.flags = CPP_TFLAG_MULTILINE;
break;
case LS_string: case LS_string:
S.token.type = CPP_TOKEN_STRING_CONSTANT; S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.flags = 0; S.token.flags = 0;
break; break;
case LS_string_multiline:
S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.flags = CPP_TFLAG_MULTILINE;
break;
case LS_comment_pre: case LS_comment_pre:
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
@ -1464,7 +1468,6 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_
else{ else{
S.token.size = S.pos - S.token_start; S.token.size = S.pos - S.token_start;
} }
S.token.flags |= (S.fsm.multi_line)?(CPP_TFLAG_MULTILINE):(0);
if ((S.token.flags & CPP_TFLAG_PP_DIRECTIVE) == 0){ if ((S.token.flags & CPP_TFLAG_PP_DIRECTIVE) == 0){
S.token.flags |= (S.pp_state != LSPP_default)?(CPP_TFLAG_PP_BODY):(0); S.token.flags |= (S.pp_state != LSPP_default)?(CPP_TFLAG_PP_BODY):(0);
} }

1
test/experiment.cpp
View File

@ -334,7 +334,6 @@ show_time(Times t, int repeats, char *type){
#define BASE_DIR "w:/4ed/data/test/" #define BASE_DIR "w:/4ed/data/test/"
int main(){ int main(){
int repeats = 100; int repeats = 100;
int verbose_level = -1; int verbose_level = -1;
int chunk_start = 0; int chunk_start = 0;

226
test/fsm_table_generator.cpp
View File

@ -10,6 +10,9 @@
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <assert.h>
#define ArrayCount(a) (sizeof(a)/sizeof(*a))
#include "4cpp_lexer_fsms.h" #include "4cpp_lexer_fsms.h"
@ -219,6 +222,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
break; break;
case LS_char: case LS_char:
case LS_char_multiline:
switch(c){ switch(c){
case '\'': fsm.emit_token = 1; break; case '\'': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_char_slashed; break; case '\\': fsm.state = LS_char_slashed; break;
@ -228,12 +232,13 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
case LS_char_slashed: case LS_char_slashed:
switch (c){ switch (c){
case '\r': case '\f': case '\v': break; case '\r': case '\f': case '\v': break;
case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break; case '\n': fsm.state = LS_char_multiline; break;
default: fsm.state = LS_char; break; default: fsm.state = LS_char; break;
} }
break; break;
case LS_string: case LS_string:
case LS_string_multiline:
switch(c){ switch(c){
case '\"': fsm.emit_token = 1; break; case '\"': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_string_slashed; break; case '\\': fsm.state = LS_string_slashed; break;
@ -243,7 +248,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
case LS_string_slashed: case LS_string_slashed:
switch (c){ switch (c){
case '\r': case '\f': case '\v': break; case '\r': case '\f': case '\v': break;
case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break; case '\n': fsm.state = LS_string_multiline; break;
default: fsm.state = LS_string; break; default: fsm.state = LS_string; break;
} }
break; break;
@ -477,14 +482,29 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
return(fsm); return(fsm);
} }
void
begin_table(FILE *file, char *type, char *group_name, char *table_name){
fprintf(file, "unsigned %s %s_%s[] = {\n", type, group_name, table_name);
}
void void
begin_table(FILE *file, char *type, char *table_name){ begin_table(FILE *file, char *type, char *table_name){
fprintf(file, "unsigned %s %s[] = {\n", type, table_name); fprintf(file, "unsigned %s %s[] = {\n", type, table_name);
} }
void
begin_ptr_table(FILE *file, char *type, char *table_name){
fprintf(file, "unsigned %s * %s[] = {\n", type, table_name);
}
void void
do_table_item(FILE *file, unsigned short item){ do_table_item(FILE *file, unsigned short item){
fprintf(file, "%d,", (int)item); fprintf(file, "%2d,", (int)item);
}
void
do_table_item_direct(FILE *file, char *item, char *tail){
fprintf(file, "%s%s,", item, tail);
} }
void void
@ -497,93 +517,203 @@ end_table(FILE *file){
fprintf(file, "};\n\n"); fprintf(file, "};\n\n");
} }
int main(){ struct FSM_Tables{
FILE *file; unsigned char *full_transition_table;
file = fopen("4cpp_lexer_tables.c", "wb"); unsigned char *marks;
unsigned char *eq_class;
unsigned char *eq_class_rep;
unsigned char *reduced_transition_table;
unsigned char *full_transition_table = (unsigned char*)malloc(LS_count * 256); unsigned char eq_class_counter;
unsigned char *marks = (unsigned char*)malloc(LS_count * 256); unsigned short state_count;
unsigned char *eq_class = (unsigned char*)malloc(LS_count * 256); };
unsigned char *eq_class_rep = (unsigned char*)malloc(LS_count * 256);
memset(marks, 0, 256); FSM_Tables
generate_whitespace_skip_table(){
unsigned char state_count = LSPP_count;
FSM_Tables table;
table.full_transition_table = (unsigned char*)malloc(state_count * 256);
table.marks = (unsigned char*)malloc(state_count * 256);
table.eq_class = (unsigned char*)malloc(state_count * 256);
table.eq_class_rep = (unsigned char*)malloc(state_count * 256);
table.state_count = state_count;
memset(table.marks, 0, 256);
int i = 0;
Whitespace_FSM wfsm = {0};
Whitespace_FSM new_wfsm;
for (unsigned short c = 0; c < 256; ++c){
for (unsigned char state = 0; state < state_count; ++state){
wfsm.pp_state = state;
wfsm.white_done = 0;
new_wfsm = whitespace_skip_fsm(wfsm, (unsigned char)c);
table.full_transition_table[i++] = new_wfsm.pp_state + state_count*new_wfsm.white_done;
}
}
table.eq_class_counter = 0;
unsigned char *c_line = table.full_transition_table;
for (unsigned short c = 0; c < 256; ++c){
if (table.marks[c] == 0){
table.eq_class[c] = table.eq_class_counter;
table.eq_class_rep[table.eq_class_counter] = (unsigned char)c;
unsigned char *c2_line = c_line + state_count;
for (unsigned short c2 = c + 1; c2 < 256; ++c2){
if (memcmp(c_line, c2_line, state_count) == 0){
table.marks[c2] = 1;
table.eq_class[c2] = table.eq_class_counter;
}
c2_line += state_count;
}
++table.eq_class_counter;
}
c_line += state_count;
}
table.reduced_transition_table = (unsigned char*)malloc(state_count * table.eq_class_counter);
i = 0;
for (unsigned short eq = 0; eq < table.eq_class_counter; ++eq){
for (unsigned char state = 0; state < state_count; ++state){
wfsm.pp_state = state;
wfsm.white_done = 0;
new_wfsm = whitespace_skip_fsm(wfsm, table.eq_class_rep[eq]);
table.reduced_transition_table[i++] = new_wfsm.pp_state + state_count*new_wfsm.white_done;
}
}
return(table);
}
FSM_Tables
generate_fsm_table(unsigned char pp_state){
unsigned char state_count = LS_count;
FSM_Tables table;
table.full_transition_table = (unsigned char*)malloc(state_count * 256);
table.marks = (unsigned char*)malloc(state_count * 256);
table.eq_class = (unsigned char*)malloc(state_count * 256);
table.eq_class_rep = (unsigned char*)malloc(state_count * 256);
table.state_count = state_count;
memset(table.marks, 0, 256);
int i = 0; int i = 0;
Lex_FSM fsm = {0}; Lex_FSM fsm = {0};
Lex_FSM new_fsm; Lex_FSM new_fsm;
for (unsigned short c = 0; c < 256; ++c){ for (unsigned short c = 0; c < 256; ++c){
for (unsigned char state = 0; state < LS_count; ++state){ for (unsigned char state = 0; state < state_count; ++state){
fsm.state = state; fsm.state = state;
fsm.emit_token = 0; fsm.emit_token = 0;
new_fsm = main_fsm(fsm, LSPP_default, (unsigned char)c); new_fsm = main_fsm(fsm, pp_state, (unsigned char)c);
full_transition_table[i++] = new_fsm.state + LS_count*new_fsm.emit_token; table.full_transition_table[i++] = new_fsm.state + state_count*new_fsm.emit_token;
} }
} }
unsigned char eq_class_counter = 0; table.eq_class_counter = 0;
unsigned char *c_line = full_transition_table; unsigned char *c_line = table.full_transition_table;
for (unsigned short c = 0; c < 256; ++c){ for (unsigned short c = 0; c < 256; ++c){
if (marks[c] == 0){ if (table.marks[c] == 0){
eq_class[c] = eq_class_counter; table.eq_class[c] = table.eq_class_counter;
eq_class_rep[eq_class_counter] = (unsigned char)c; table.eq_class_rep[table.eq_class_counter] = (unsigned char)c;
unsigned char *c2_line = c_line + LS_count; unsigned char *c2_line = c_line + state_count;
for (unsigned short c2 = c + 1; c2 < 256; ++c2){ for (unsigned short c2 = c + 1; c2 < 256; ++c2){
if (memcmp(c_line, c2_line, LS_count) == 0){ if (memcmp(c_line, c2_line, state_count) == 0){
marks[c2] = 1; table.marks[c2] = 1;
eq_class[c2] = eq_class_counter; table.eq_class[c2] = table.eq_class_counter;
} }
c2_line += LS_count; c2_line += state_count;
} }
++eq_class_counter; ++table.eq_class_counter;
} }
c_line += LS_count; c_line += state_count;
} }
unsigned char *reduced_transition_table = (unsigned char*)malloc(LS_count * eq_class_counter); table.reduced_transition_table = (unsigned char*)malloc(state_count * table.eq_class_counter);
unsigned char *reduced_multiline_table = (unsigned char*)malloc(LS_count * eq_class_counter);
i = 0; i = 0;
for (unsigned char state = 0; state < LS_count; ++state){ for (unsigned short eq = 0; eq < table.eq_class_counter; ++eq){
for (unsigned char state = 0; state < state_count; ++state){
fsm.state = state; fsm.state = state;
for (unsigned short eq = 0; eq < eq_class_counter; ++eq){
fsm.emit_token = 0; fsm.emit_token = 0;
fsm.multi_line = 0; new_fsm = main_fsm(fsm, pp_state, table.eq_class_rep[eq]);
new_fsm = main_fsm(fsm, LSPP_default, eq_class_rep[eq]); table.reduced_transition_table[i++] = new_fsm.state + state_count*new_fsm.emit_token;
reduced_transition_table[i] = new_fsm.state + LS_count*new_fsm.emit_token;
reduced_multiline_table[i] = new_fsm.multi_line;
++i;
} }
} }
begin_table(file, "char", "main_fsm_eqclasses"); return(table);
}
void
render_fsm_table(FILE *file, FSM_Tables tables, char *group_name){
begin_table(file, "short", group_name, "eq_classes");
for (unsigned short c = 0; c < 256; ++c){ for (unsigned short c = 0; c < 256; ++c){
do_table_item(file, eq_class[c]); do_table_item(file, tables.eq_class[c]*tables.state_count);
} }
end_row(file); end_row(file);
end_table(file); end_table(file);
fprintf(file, "const int num_eq_classes = %d;\n\n", eq_class_counter); fprintf(file, "const int num_%s_eq_classes = %d;\n\n", group_name, tables.eq_class_counter);
i = 0; int i = 0;
begin_table(file, "char", "main_fsm_table"); begin_table(file, "char", group_name, "table");
for (unsigned char state = 0; state < LS_count; ++state){ for (unsigned short c = 0; c < tables.eq_class_counter; ++c){
for (unsigned short c = 0; c < eq_class_counter; ++c){ for (unsigned char state = 0; state < tables.state_count; ++state){
do_table_item(file, reduced_transition_table[i++]); do_table_item(file, tables.reduced_transition_table[i++]);
} }
end_row(file); end_row(file);
} }
end_table(file); end_table(file);
}
i = 0; struct PP_Names{
begin_table(file, "char", "main_fsm_multiline_table"); unsigned char pp_state;
char *name;
};
PP_Names pp_names[] = {
{LSPP_default, "main_fsm"},
{LSPP_include, "pp_include_fsm"},
{LSPP_macro_identifier, "pp_macro_fsm"},
{LSPP_identifier, "pp_identifier_fsm"},
{LSPP_body_if, "pp_body_if_fsm"},
{LSPP_body, "pp_body_fsm"},
{LSPP_number, "pp_number_fsm"},
{LSPP_error, "pp_error_fsm"},
{LSPP_junk, "pp_junk_fsm"},
};
int main(){
FILE *file;
file = fopen("4cpp_lexer_tables.c", "wb");
FSM_Tables wtables = generate_whitespace_skip_table();
render_fsm_table(file, wtables, "whitespace_fsm");
begin_table(file, "char", "multiline_state_table");
for (unsigned char state = 0; state < LS_count; ++state){ for (unsigned char state = 0; state < LS_count; ++state){
for (unsigned short c = 0; c < eq_class_counter; ++c){ do_table_item(file, (state == LS_string_multiline || state == LS_char_multiline));
do_table_item(file, reduced_multiline_table[i++]);
} }
end_row(file); end_row(file);
end_table(file);
for (int i = 0; i < ArrayCount(pp_names); ++i){
assert(i == pp_names[i].pp_state);
FSM_Tables tables = generate_fsm_table(pp_names[i].pp_state);
render_fsm_table(file, tables, pp_names[i].name);
}
begin_ptr_table(file, "short", "get_eq_classes");
for (int i = 0; i < ArrayCount(pp_names); ++i){
do_table_item_direct(file, pp_names[i].name, "_eq_classes");
end_row(file);
}
end_table(file);
begin_ptr_table(file, "char", "get_table");
for (int i = 0; i < ArrayCount(pp_names); ++i){
do_table_item_direct(file, pp_names[i].name, "_table");
end_row(file);
} }
end_table(file); end_table(file);
fclose(file); fclose(file);
return(0); return(0);
} }