mirrors/emacs
1
Fork 0
mirror of git://git.sv.gnu.org/emacs.git synced 2025-12-20 04:30:55 -08:00
Code Issues Projects Releases Wiki Activity

(CATEGORY_MASK_ANY): Add CATEGORY_MASK_UTF_16_AUTO.

Browse source 
(CATEGORY_MASK_UTF_16): Likewise.
(detect_coding_utf_16): Add heuristics to reject utf-16 for a
binary file.
(detect_coding): Add null-byte detection for a binary file.
(detect_coding_system): Likewise.
This commit is contained in:
Kenichi Handa 2008-04-03 12:31:27 +00:00
parent 164ce7fab3
commit 2f3cbb326b
2 changed files with 214 additions and 108 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/ChangeLog
View file

@ -1,3 +1,12 @@
2008-04-03 Kenichi Handa <handa@m17n.org>
* coding.c (CATEGORY_MASK_ANY): Add CATEGORY_MASK_UTF_16_AUTO.
(CATEGORY_MASK_UTF_16): Likewise.
(detect_coding_utf_16): Add heuristics to reject utf-16 for a
binary file.
(detect_coding): Add null-byte detection for a binary file.
(detect_coding_system): Likewise.
2008-04-03 Jason Rumney <jasonr@gnu.org> 2008-04-03 Jason Rumney <jasonr@gnu.org>
* ftfont.c (ftfont_get_open_type_spec): Check spec->script, not val. * ftfont.c (ftfont_get_open_type_spec): Check spec->script, not val.

313
src/coding.c
View file

@ -625,6 +625,7 @@ enum coding_category
| CATEGORY_MASK_ISO_7_ELSE \ | CATEGORY_MASK_ISO_7_ELSE \
| CATEGORY_MASK_ISO_8_ELSE \ | CATEGORY_MASK_ISO_8_ELSE \
| CATEGORY_MASK_UTF_8 \ | CATEGORY_MASK_UTF_8 \
| CATEGORY_MASK_UTF_16_AUTO \
| CATEGORY_MASK_UTF_16_BE \ | CATEGORY_MASK_UTF_16_BE \
| CATEGORY_MASK_UTF_16_LE \ | CATEGORY_MASK_UTF_16_LE \
| CATEGORY_MASK_UTF_16_BE_NOSIG \ | CATEGORY_MASK_UTF_16_BE_NOSIG \
@ -657,7 +658,8 @@ enum coding_category
| CATEGORY_MASK_ISO_ELSE) | CATEGORY_MASK_ISO_ELSE)
#define CATEGORY_MASK_UTF_16 \ #define CATEGORY_MASK_UTF_16 \
(CATEGORY_MASK_UTF_16_BE \ (CATEGORY_MASK_UTF_16_AUTO \
| CATEGORY_MASK_UTF_16_BE \
| CATEGORY_MASK_UTF_16_LE \ | CATEGORY_MASK_UTF_16_LE \
| CATEGORY_MASK_UTF_16_BE_NOSIG \ | CATEGORY_MASK_UTF_16_BE_NOSIG \
| CATEGORY_MASK_UTF_16_LE_NOSIG) | CATEGORY_MASK_UTF_16_LE_NOSIG)
@ -1513,11 +1515,44 @@ detect_coding_utf_16 (coding, detect_info)
| CATEGORY_MASK_UTF_16_BE_NOSIG | CATEGORY_MASK_UTF_16_BE_NOSIG
| CATEGORY_MASK_UTF_16_LE_NOSIG); | CATEGORY_MASK_UTF_16_LE_NOSIG);
} }
else if (c1 >= 0 && c2 >= 0) else
{ {
/* We check the dispersion of Eth and Oth bytes where E is even and
O is odd. If both are high, we assume binary data.*/
unsigned char e[256], o[256];
unsigned e_num = 1, o_num = 1;
memset (e, 0, 256);
memset (o, 0, 256);
e[c1] = 1;
o[c2] = 1;
detect_info->rejected detect_info->rejected
|= (CATEGORY_MASK_UTF_16_BE | CATEGORY_MASK_UTF_16_LE); |= (CATEGORY_MASK_UTF_16_BE | CATEGORY_MASK_UTF_16_LE);
while (1)
{
ONE_MORE_BYTE (c1);
ONE_MORE_BYTE (c2);
if (! e[c1])
{
e[c1] = 1;
e_num++;
if (e_num >= 128)
break;
}
if (! o[c2])
{
o[c1] = 1;
o_num++;
if (o_num >= 128)
break;
}
}
detect_info->rejected |= CATEGORY_MASK_UTF_16;
return 0;
} }
no_more_source: no_more_source:
return 1; return 1;
} }
@ -5677,32 +5712,53 @@ detect_coding (coding)
{ {
int c, i; int c, i;
struct coding_detection_info detect_info; struct coding_detection_info detect_info;
int null_byte_found = 0, eight_bit_found = 0;
detect_info.checked = detect_info.found = detect_info.rejected = 0; detect_info.checked = detect_info.found = detect_info.rejected = 0;
for (i = 0, src = coding->source; src < src_end; i++, src++) coding->head_ascii = -1;
for (src = coding->source; src < src_end; src++)
{ {
c = *src; c = *src;
if (c & 0x80) if (c & 0x80)
break;
if (c < 0x20
&& (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
&& ! inhibit_iso_escape_detection
&& ! detect_info.checked)
{ {
coding->head_ascii = src - (coding->source + coding->consumed); eight_bit_found = 1;
if (detect_coding_iso_2022 (coding, &detect_info)) if (coding->head_ascii < 0)
coding->head_ascii = src - coding->source;
if (null_byte_found)
break;
}
else if (c < 0x20)
{
if ((c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
&& ! inhibit_iso_escape_detection
&& ! detect_info.checked)
{ {
/* We have scanned the whole data. */ if (coding->head_ascii < 0)
if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE)) coding->head_ascii = src - coding->source;
/* We didn't find an 8-bit code. */ if (detect_coding_iso_2022 (coding, &detect_info))
src = src_end; {
break; /* We have scanned the whole data. */
if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE))
/* We didn't find an 8-bit code. We may have
found a null-byte, but it's very rare that
a binary file confirm to ISO-2022. */
src = src_end;
break;
}
}
else if (! c)
{
null_byte_found = 1;
if (eight_bit_found)
break;
} }
} }
} }
coding->head_ascii = src - (coding->source + coding->consumed); if (coding->head_ascii < 0)
coding->head_ascii = src - coding->source;
if (coding->head_ascii < coding->src_bytes if (null_byte_found || eight_bit_found
|| coding->head_ascii < coding->src_bytes
|| detect_info.found) || detect_info.found)
{ {
enum coding_category category; enum coding_category category;
@ -5718,48 +5774,58 @@ detect_coding (coding)
break; break;
} }
else else
for (i = 0; i < coding_category_raw_text; i++) {
{ if (null_byte_found)
category = coding_priorities[i];
this = coding_categories + category;
if (this->id < 0)
{
/* No coding system of this category is defined. */
detect_info.rejected |= (1 << category);
}
else if (category >= coding_category_raw_text)
continue;
else if (detect_info.checked & (1 << category))
{
if (detect_info.found & (1 << category))
break;
}
else if ((*(this->detector)) (coding, &detect_info)
&& detect_info.found & (1 << category))
{
if (category == coding_category_utf_16_auto)
{
if (detect_info.found & CATEGORY_MASK_UTF_16_LE)
category = coding_category_utf_16_le;
else
category = coding_category_utf_16_be;
}
break;
}
}
if (i < coding_category_raw_text)
setup_coding_system (CODING_ID_NAME (this->id), coding);
else if (detect_info.rejected == CATEGORY_MASK_ANY)
setup_coding_system (Qraw_text, coding);
else if (detect_info.rejected)
for (i = 0; i < coding_category_raw_text; i++)
if (! (detect_info.rejected & (1 << coding_priorities[i])))
{ {
this = coding_categories + coding_priorities[i]; detect_info.checked |= ~CATEGORY_MASK_UTF_16;
setup_coding_system (CODING_ID_NAME (this->id), coding); detect_info.rejected |= ~CATEGORY_MASK_UTF_16;
break;
} }
for (i = 0; i < coding_category_raw_text; i++)
{
category = coding_priorities[i];
this = coding_categories + category;
if (this->id < 0)
{
/* No coding system of this category is defined. */
detect_info.rejected |= (1 << category);
}
else if (category >= coding_category_raw_text)
continue;
else if (detect_info.checked & (1 << category))
{
if (detect_info.found & (1 << category))
break;
}
else if ((*(this->detector)) (coding, &detect_info)
&& detect_info.found & (1 << category))
{
if (category == coding_category_utf_16_auto)
{
if (detect_info.found & CATEGORY_MASK_UTF_16_LE)
category = coding_category_utf_16_le;
else
category = coding_category_utf_16_be;
}
break;
}
}
if (i < coding_category_raw_text)
setup_coding_system (CODING_ID_NAME (this->id), coding);
else if (null_byte_found)
setup_coding_system (Qno_conversion, coding);
else if ((detect_info.rejected & CATEGORY_MASK_ANY)
== CATEGORY_MASK_ANY)
setup_coding_system (Qraw_text, coding);
else if (detect_info.rejected)
for (i = 0; i < coding_category_raw_text; i++)
if (! (detect_info.rejected & (1 << coding_priorities[i])))
{
this = coding_categories + coding_priorities[i];
setup_coding_system (CODING_ID_NAME (this->id), coding);
break;
}
}
} }
} }
else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding->id))) else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding->id)))
@ -7472,6 +7538,7 @@ detect_coding_system (src, src_chars, src_bytes, highest, multibytep,
int id; int id;
struct coding_detection_info detect_info; struct coding_detection_info detect_info;
enum coding_category base_category; enum coding_category base_category;
int null_byte_found = 0, eight_bit_found = 0;
if (NILP (coding_system)) if (NILP (coding_system))
coding_system = Qundecided; coding_system = Qundecided;
@ -7497,33 +7564,54 @@ detect_coding_system (src, src_chars, src_bytes, highest, multibytep,
struct coding_system *this; struct coding_system *this;
int c, i; int c, i;
coding.head_ascii = -1;
/* Skip all ASCII bytes except for a few ISO2022 controls. */ /* Skip all ASCII bytes except for a few ISO2022 controls. */
for (i = 0; src < src_end; i++, src++) for (; src < src_end; src++)
{ {
c = *src; c = *src;
if (c & 0x80) if (c & 0x80)
break;
if (c < 0x20
&& (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
&& ! inhibit_iso_escape_detection)
{ {
coding.head_ascii = src - coding.source; eight_bit_found = 1;
if (detect_coding_iso_2022 (&coding, &detect_info)) if (coding.head_ascii < 0)
coding.head_ascii = src - coding.source;
if (null_byte_found)
break;
}
if (c < 0x20)
{
if ((c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
&& ! inhibit_iso_escape_detection
&& ! detect_info.checked)
{ {
/* We have scanned the whole data. */ if (coding.head_ascii < 0)
if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE)) coding.head_ascii = src - coding.source;
/* We didn't find an 8-bit code. */ if (detect_coding_iso_2022 (&coding, &detect_info))
src = src_end; {
break; /* We have scanned the whole data. */
if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE))
/* We didn't find an 8-bit code. We may have
found a null-byte, but it's very rare that
a binary file confirm to ISO-2022. */
src = src_end;
break;
}
}
else if (! c)
{
null_byte_found = 1;
if (eight_bit_found)
break;
} }
} }
} }
coding.head_ascii = src - coding.source; if (coding.head_ascii < 0)
coding.head_ascii = src - coding.source;
if (src < src_end if (null_byte_found || eight_bit_found
|| coding.head_ascii < coding.src_bytes
|| detect_info.found) || detect_info.found)
{ {
if (src == src_end) if (coding.head_ascii == coding.src_bytes)
/* As all bytes are 7-bit, we can ignore non-ISO-2022 codings. */ /* As all bytes are 7-bit, we can ignore non-ISO-2022 codings. */
for (i = 0; i < coding_category_raw_text; i++) for (i = 0; i < coding_category_raw_text; i++)
{ {
@ -7533,44 +7621,48 @@ detect_coding_system (src, src_chars, src_bytes, highest, multibytep,
break; break;
} }
else else
for (i = 0; i < coding_category_raw_text; i++) {
{ if (null_byte_found)
category = coding_priorities[i]; {
this = coding_categories + category; detect_info.checked |= ~CATEGORY_MASK_UTF_16;
detect_info.rejected |= ~CATEGORY_MASK_UTF_16;
}
for (i = 0; i < coding_category_raw_text; i++)
{
category = coding_priorities[i];
this = coding_categories + category;
if (this->id < 0) if (this->id < 0)
{ {
/* No coding system of this category is defined. */ /* No coding system of this category is defined. */
detect_info.rejected |= (1 << category); detect_info.rejected |= (1 << category);
} }
else if (category >= coding_category_raw_text) else if (category >= coding_category_raw_text)
continue; continue;
else if (detect_info.checked & (1 << category)) else if (detect_info.checked & (1 << category))
{ {
if (highest if (highest
&& (detect_info.found & (1 << category))) && (detect_info.found & (1 << category)))
break;
}
else
{
if ((*(this->detector)) (&coding, &detect_info)
&& highest
&& (detect_info.found & (1 << category)))
{
if (category == coding_category_utf_16_auto)
{
if (detect_info.found & CATEGORY_MASK_UTF_16_LE)
category = coding_category_utf_16_le;
else
category = coding_category_utf_16_be;
}
break; break;
} }
} else if ((*(this->detector)) (&coding, &detect_info)
} && highest
&& (detect_info.found & (1 << category)))
{
if (category == coding_category_utf_16_auto)
{
if (detect_info.found & CATEGORY_MASK_UTF_16_LE)
category = coding_category_utf_16_le;
else
category = coding_category_utf_16_be;
}
break;
}
}
}
} }
if (detect_info.rejected == CATEGORY_MASK_ANY) if ((detect_info.rejected & CATEGORY_MASK_ANY) == CATEGORY_MASK_ANY)
{ {
detect_info.found = CATEGORY_MASK_RAW_TEXT; detect_info.found = CATEGORY_MASK_RAW_TEXT;
id = coding_categories[coding_category_raw_text].id; id = coding_categories[coding_category_raw_text].id;
@ -7659,8 +7751,13 @@ detect_coding_system (src, src_chars, src_bytes, highest, multibytep,
if (VECTORP (eol_type)) if (VECTORP (eol_type))
{ {
if (detect_info.found & ~CATEGORY_MASK_UTF_16) if (detect_info.found & ~CATEGORY_MASK_UTF_16)
normal_eol = detect_eol (coding.source, src_bytes, {
coding_category_raw_text); if (null_byte_found)
normal_eol = EOL_SEEN_LF;
else
normal_eol = detect_eol (coding.source, src_bytes,
coding_category_raw_text);
}
if (detect_info.found & (CATEGORY_MASK_UTF_16_BE if (detect_info.found & (CATEGORY_MASK_UTF_16_BE
| CATEGORY_MASK_UTF_16_BE_NOSIG)) | CATEGORY_MASK_UTF_16_BE_NOSIG))
utf_16_be_eol = detect_eol (coding.source, src_bytes, utf_16_be_eol = detect_eol (coding.source, src_bytes,