ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

experimenting with extended utf8

This commit is contained in:
Allen Webster 2017-03-25 13:05:25 -04:00
parent 7521c2f436
commit a80ff44e04
2 changed files with 93 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
4coder_lib/4coder_utf8.h
View File

@ -58,23 +58,40 @@ utf8_to_u32_length_unchecked(u8_4tech *buffer, u32_4tech *length_out){
*length_out = 1;
}
else if (buffer[0] < 0xE0){
result = ((u32_4tech)((buffer[0])&0x1F)) << 6;
result = ((u32_4tech)((buffer[0])&0x1F)) << 6;
result |= ((u32_4tech)((buffer[1])&0x3F));
*length_out = 2;
}
else if (buffer[0] < 0xF0){
result = ((u32_4tech)((buffer[0])&0x0F)) << 12;
result = ((u32_4tech)((buffer[0])&0x0F)) << 12;
result |= ((u32_4tech)((buffer[1])&0x3F)) << 6;
result |= ((u32_4tech)((buffer[2])&0x3F));
*length_out = 3;
}
else{
result = ((u32_4tech)((buffer[0])&0x07)) << 18;
else if (buffer[0] < 0xF8){
result = ((u32_4tech)((buffer[0])&0x07)) << 18;
result |= ((u32_4tech)((buffer[1])&0x3F)) << 12;
result |= ((u32_4tech)((buffer[2])&0x3F)) << 6;
result |= ((u32_4tech)((buffer[3])&0x3F));
*length_out = 4;
}
else if (buffer[0] < 0xFC){
result = ((u32_4tech)((buffer[0])&0x03)) << 24;
result |= ((u32_4tech)((buffer[1])&0x3F)) << 18;
result |= ((u32_4tech)((buffer[2])&0x3F)) << 12;
result |= ((u32_4tech)((buffer[3])&0x3F)) << 6;
result |= ((u32_4tech)((buffer[4])&0x3F));
*length_out = 5;
}
else{
result = ((u32_4tech)((buffer[0])&0x01)) << 30;
result |= ((u32_4tech)((buffer[1])&0x03)) << 24;
result |= ((u32_4tech)((buffer[2])&0x3F)) << 18;
result |= ((u32_4tech)((buffer[3])&0x3F)) << 12;
result |= ((u32_4tech)((buffer[4])&0x3F)) << 6;
result |= ((u32_4tech)((buffer[5])&0x3F));
*length_out = 6;
}
return(result);
}
@ -208,7 +225,7 @@ utf8_to_utf16_minimal_checking(u16_4tech *dst, umem_4tech max_wchars, u8_4tech *
break;
}
codepoint = ((u32_4tech)((s[0])&0x1F)) << 6;
codepoint = ((u32_4tech)((s[0])&0x1F)) << 6;
codepoint |= ((u32_4tech)((s[1])&0x3F));
utf8_size = 2;
}
@ -218,23 +235,50 @@ utf8_to_utf16_minimal_checking(u16_4tech *dst, umem_4tech max_wchars, u8_4tech *
break;
}
codepoint = ((u32_4tech)((s[0])&0x0F)) << 12;
codepoint = ((u32_4tech)((s[0])&0x0F)) << 12;
codepoint |= ((u32_4tech)((s[1])&0x3F)) << 6;
codepoint |= ((u32_4tech)((s[2])&0x3F));
utf8_size = 3;
}
else{
if (limit > 3){
else if (s[0] < 0xF8){
if (limit <= 3){
*error = true;
break;
}
codepoint = ((u32_4tech)((s[0])&0x07)) << 18;
codepoint = ((u32_4tech)((s[0])&0x07)) << 18;
codepoint |= ((u32_4tech)((s[1])&0x3F)) << 12;
codepoint |= ((u32_4tech)((s[2])&0x3F)) << 6;
codepoint |= ((u32_4tech)((s[3])&0x3F));
utf8_size = 4;
}
else if (s[0] < 0xFC){
if (limit <= 4){
*error = true;
break;
}
codepoint = ((u32_4tech)((s[0])&0x03)) << 24;
codepoint |= ((u32_4tech)((s[1])&0x3F)) << 18;
codepoint |= ((u32_4tech)((s[2])&0x3F)) << 12;
codepoint |= ((u32_4tech)((s[3])&0x3F)) << 6;
codepoint |= ((u32_4tech)((s[4])&0x3F));
utf8_size = 5;
}
else if (s[0] < 0xFE){
if (limit <= 5){
*error = true;
break;
}
codepoint = ((u32_4tech)((s[0])&0x01)) << 30;
codepoint |= ((u32_4tech)((s[1])&0x3F)) << 24;
codepoint |= ((u32_4tech)((s[2])&0x3F)) << 18;
codepoint |= ((u32_4tech)((s[3])&0x3F)) << 12;
codepoint |= ((u32_4tech)((s[4])&0x3F)) << 6;
codepoint |= ((u32_4tech)((s[5])&0x3F));
utf8_size = 6;
}
s += utf8_size;
limit -= utf8_size;
@ -320,7 +364,7 @@ utf16_to_utf8_minimal_checking(u8_4tech *dst, umem_4tech max_chars, u16_4tech *s
s += utf16_size;
limit -= utf16_size;
u8_4tech d_fill[4];
u8_4tech d_fill[6];
u32_4tech d_fill_count = 0;
if (codepoint <= 0x7F){
@ -333,18 +377,35 @@ utf16_to_utf8_minimal_checking(u8_4tech *dst, umem_4tech max_chars, u16_4tech *s
d_fill_count = 2;
}
else if (codepoint <= 0xFFFF){
d_fill[0] = (u8_4tech)(0xE0 | (codepoint >> 12));
d_fill[1] = (u8_4tech)(0x80 | ((codepoint >> 6) & 0x3F));
d_fill[2] = (u8_4tech)(0x80 | (codepoint & 0x3F));
d_fill[0] = (u8_4tech)(0xE0 | ( codepoint >> 12));
d_fill[1] = (u8_4tech)(0x80 | ((codepoint >> 6 ) & 0x3F));
d_fill[2] = (u8_4tech)(0x80 | ((codepoint) & 0x3F));
d_fill_count = 3;
}
else if (codepoint <= 0x10FFFF){
d_fill[0] = (u8_4tech)(0xF0 | (codepoint >> 18));
else if (codepoint <= 0x1FFFFF){
d_fill[0] = (u8_4tech)(0xF0 | ( codepoint >> 18));
d_fill[1] = (u8_4tech)(0x80 | ((codepoint >> 12) & 0x3F));
d_fill[2] = (u8_4tech)(0x80 | ((codepoint >> 6) & 0x3F));
d_fill[3] = (u8_4tech)(0x80 | (codepoint & 0x3F));
d_fill[2] = (u8_4tech)(0x80 | ((codepoint >> 6 ) & 0x3F));
d_fill[3] = (u8_4tech)(0x80 | ((codepoint) & 0x3F));
d_fill_count = 4;
}
else if (codepoint <= 0x3FFFFFF){
d_fill[0] = (u8_4tech)(0xF8 | ( codepoint >> 24));
d_fill[1] = (u8_4tech)(0x80 | ((codepoint >> 18) & 0x3F));
d_fill[2] = (u8_4tech)(0x80 | ((codepoint >> 12) & 0x3F));
d_fill[3] = (u8_4tech)(0x80 | ((codepoint >> 6 ) & 0x3F));
d_fill[4] = (u8_4tech)(0x80 | ((codepoint) & 0x3F));
d_fill_count = 5;
}
else if (codepoint <= 0x7FFFFFFF){
d_fill[0] = (u8_4tech)(0xFC | ( codepoint >> 30));
d_fill[1] = (u8_4tech)(0x80 | ((codepoint >> 24) & 0x3F));
d_fill[2] = (u8_4tech)(0x80 | ((codepoint >> 18) & 0x3F));
d_fill[3] = (u8_4tech)(0x80 | ((codepoint >> 12) & 0x3F));
d_fill[4] = (u8_4tech)(0x80 | ((codepoint >> 6 ) & 0x3F));
d_fill[5] = (u8_4tech)(0x80 | ((codepoint) & 0x3F));
d_fill_count = 6;
}
else{
*error = true;
break;

24
4ed_translation.cpp
View File

@ -12,7 +12,7 @@
#include "4ed_buffer_model.h"
struct Translation_State{
u8 fill_buffer[4];
u8 fill_buffer[6];
u32 fill_start_i;
u8 fill_i;
u8 fill_expected;
@ -40,20 +40,20 @@ struct Translation_Emit_Rule{
};
struct Translation_Emits{
Buffer_Model_Step steps[5];
Buffer_Model_Step steps[7];
u32 step_count;
};
#define SINGLE_BYTE_ERROR_CLASS max_u8
#define CONTINUATION_BYTE max_u8
internal void
translating_consume_byte(Translation_State *tran, u8 ch, u32 i, u32 size, Translation_Byte_Description *desc_out){
desc_out->byte_class = 0;
if ((ch >= ' ' && ch < 0x7F) || ch == '\t' || ch == '\n' || ch == '\r'){
if (ch < 0x80){
desc_out->byte_class = 1;
}
else if (ch < 0xC0){
desc_out->byte_class = SINGLE_BYTE_ERROR_CLASS;
desc_out->byte_class = CONTINUATION_BYTE;
}
else if (ch < 0xE0){
desc_out->byte_class = 2;
@ -61,9 +61,15 @@ translating_consume_byte(Translation_State *tran, u8 ch, u32 i, u32 size, Transl
else if (ch < 0xF0){
desc_out->byte_class = 3;
}
else{
else if (ch < 0xF8){
desc_out->byte_class = 4;
}
else if (ch < 0xFC){
desc_out->byte_class = 5;
}
else{
desc_out->byte_class = 6;
}
desc_out->prelim_emit_type = BufferModelUnit_None;
desc_out->last_byte_handler = TranLBH_None;
@ -75,7 +81,7 @@ translating_consume_byte(Translation_State *tran, u8 ch, u32 i, u32 size, Transl
if (desc_out->byte_class == 1){
desc_out->prelim_emit_type = BufferModelUnit_Codepoint;
}
else if (desc_out->byte_class == 0 || desc_out->byte_class == SINGLE_BYTE_ERROR_CLASS){
else if (desc_out->byte_class == 0 || desc_out->byte_class == CONTINUATION_BYTE){
desc_out->prelim_emit_type = BufferModelUnit_Numbers;
}
else{
@ -83,7 +89,7 @@ translating_consume_byte(Translation_State *tran, u8 ch, u32 i, u32 size, Transl
}
}
else{
if (desc_out->byte_class == SINGLE_BYTE_ERROR_CLASS){
if (desc_out->byte_class == CONTINUATION_BYTE){
tran->fill_buffer[tran->fill_i] = ch;
++tran->fill_i;
@ -92,7 +98,7 @@ translating_consume_byte(Translation_State *tran, u8 ch, u32 i, u32 size, Transl
}
}
else{
if (desc_out->byte_class >= 2 && desc_out->byte_class <= 4){
if (desc_out->byte_class >= 2 && desc_out->byte_class <= 6){
desc_out->last_byte_handler = TranLBH_Rebuffer;
}
else if (desc_out->byte_class == 1){