From 96d597b6a4e3f37c4364b16b97fb0e5f6816508f Mon Sep 17 00:00:00 2001 From: Rui Hirokawa Date: Fri, 3 Nov 2006 01:11:35 +0000 Subject: [PATCH] added japanese legacy encoding support (made by moriyama@miraclelinux.com) --- ext/mbstring/config.m4 | 1 + ext/mbstring/libmbfl/filters/Makefile.am | 2 +- .../libmbfl/filters/mbfilter_cp51932.c | 360 ++++++++++++++++++ .../libmbfl/filters/mbfilter_cp51932.h | 43 +++ .../libmbfl/filters/mbfilter_euc_jp_win.c | 3 +- ext/mbstring/libmbfl/filters/mbfilter_jis.c | 154 +++++++- ext/mbstring/libmbfl/filters/mbfilter_jis.h | 4 + .../libmbfl/filters/unicode_table_jis.h | 14 +- ext/mbstring/libmbfl/mbfl/mbfl_convert.c | 5 + ext/mbstring/libmbfl/mbfl/mbfl_encoding.c | 3 + ext/mbstring/libmbfl/mbfl/mbfl_encoding.h | 2 + ext/mbstring/libmbfl/mbfl/mbfl_ident.c | 1 + 12 files changed, 575 insertions(+), 17 deletions(-) create mode 100644 ext/mbstring/libmbfl/filters/mbfilter_cp51932.c create mode 100644 ext/mbstring/libmbfl/filters/mbfilter_cp51932.h diff --git a/ext/mbstring/config.m4 b/ext/mbstring/config.m4 index ee9dddbe713..8cddb62ae8c 100644 --- a/ext/mbstring/config.m4 +++ b/ext/mbstring/config.m4 @@ -189,6 +189,7 @@ AC_DEFUN([PHP_MBSTRING_SETUP_LIBMBFL], [ libmbfl/filters/mbfilter_euc_cn.c libmbfl/filters/mbfilter_euc_jp.c libmbfl/filters/mbfilter_euc_jp_win.c + libmbfl/filters/mbfilter_cp51932.c libmbfl/filters/mbfilter_euc_kr.c libmbfl/filters/mbfilter_euc_tw.c libmbfl/filters/mbfilter_htmlent.c diff --git a/ext/mbstring/libmbfl/filters/Makefile.am b/ext/mbstring/libmbfl/filters/Makefile.am index 0c71c8604b6..9b2fda4c395 100644 --- a/ext/mbstring/libmbfl/filters/Makefile.am +++ b/ext/mbstring/libmbfl/filters/Makefile.am @@ -2,7 +2,7 @@ EXTRA_DIST=Makefile.bcc32 mk_sb_tbl.awk noinst_LTLIBRARIES=libmbfl_filters.la INCLUDES=-I../mbfl libmbfl_filters_la_LDFLAGS=-version-info $(SHLIB_VERSION) -libmbfl_filters_la_SOURCES=mbfilter_cp936.c mbfilter_hz.c mbfilter_euc_tw.c mbfilter_big5.c mbfilter_euc_jp.c mbfilter_jis.c mbfilter_iso8859_1.c mbfilter_iso8859_2.c mbfilter_cp1252.c mbfilter_cp1251.c mbfilter_ascii.c mbfilter_iso8859_3.c mbfilter_iso8859_4.c mbfilter_iso8859_5.c mbfilter_iso8859_6.c mbfilter_iso8859_7.c mbfilter_iso8859_8.c mbfilter_iso8859_9.c mbfilter_iso8859_10.c mbfilter_iso8859_13.c mbfilter_iso8859_14.c mbfilter_iso8859_15.c mbfilter_iso8859_16.c mbfilter_htmlent.c mbfilter_byte2.c mbfilter_byte4.c mbfilter_uuencode.c mbfilter_base64.c mbfilter_sjis.c mbfilter_7bit.c mbfilter_qprint.c mbfilter_ucs4.c mbfilter_ucs2.c mbfilter_utf32.c mbfilter_utf16.c mbfilter_utf8.c mbfilter_utf7.c mbfilter_utf7imap.c mbfilter_euc_jp_win.c mbfilter_cp932.c mbfilter_euc_cn.c mbfilter_euc_kr.c mbfilter_uhc.c mbfilter_iso2022_kr.c mbfilter_cp866.c mbfilter_koi8r.c mbfilter_armscii8.c html_entities.c cp932_table.h html_entities.h mbfilter_7bit.h mbfilter_ascii.h mbfilter_base64.h mbfilter_big5.h mbfilter_byte2.h mbfilter_byte4.h mbfilter_cp1251.h mbfilter_cp1252.h mbfilter_cp866.h mbfilter_cp932.h mbfilter_cp936.h mbfilter_euc_cn.h mbfilter_euc_jp.h mbfilter_euc_jp_win.h mbfilter_euc_kr.h mbfilter_euc_tw.h mbfilter_htmlent.h mbfilter_hz.h mbfilter_iso2022_kr.h mbfilter_iso8859_1.h mbfilter_iso8859_10.h mbfilter_iso8859_13.h mbfilter_iso8859_14.h mbfilter_iso8859_15.h mbfilter_iso8859_16.h mbfilter_iso8859_2.h mbfilter_iso8859_3.h mbfilter_iso8859_4.h mbfilter_iso8859_5.h mbfilter_iso8859_6.h mbfilter_iso8859_7.h mbfilter_iso8859_8.h mbfilter_iso8859_9.h mbfilter_jis.h mbfilter_koi8r.h mbfilter_armscii8.h mbfilter_qprint.h mbfilter_sjis.h mbfilter_ucs2.h mbfilter_ucs4.h mbfilter_uhc.h mbfilter_utf16.h mbfilter_utf32.h mbfilter_utf7.h mbfilter_utf7imap.h mbfilter_utf8.h mbfilter_uuencode.h unicode_prop.h unicode_table_big5.h unicode_table_cns11643.h unicode_table_cp1251.h unicode_table_cp1252.h unicode_table_cp866.h unicode_table_cp932_ext.h unicode_table_cp936.h unicode_table_iso8859_10.h unicode_table_iso8859_13.h unicode_table_iso8859_14.h unicode_table_iso8859_15.h unicode_table_iso8859_16.h unicode_table_iso8859_2.h unicode_table_iso8859_3.h unicode_table_iso8859_4.h unicode_table_iso8859_5.h unicode_table_iso8859_6.h unicode_table_iso8859_7.h unicode_table_iso8859_8.h unicode_table_iso8859_9.h unicode_table_jis.h unicode_table_koi8r.h unicode_table_armscii8.h unicode_table_uhc.h +libmbfl_filters_la_SOURCES=mbfilter_cp936.c mbfilter_hz.c mbfilter_euc_tw.c mbfilter_big5.c mbfilter_euc_jp.c mbfilter_jis.c mbfilter_iso8859_1.c mbfilter_iso8859_2.c mbfilter_cp1252.c mbfilter_cp1251.c mbfilter_ascii.c mbfilter_iso8859_3.c mbfilter_iso8859_4.c mbfilter_iso8859_5.c mbfilter_iso8859_6.c mbfilter_iso8859_7.c mbfilter_iso8859_8.c mbfilter_iso8859_9.c mbfilter_iso8859_10.c mbfilter_iso8859_13.c mbfilter_iso8859_14.c mbfilter_iso8859_15.c mbfilter_iso8859_16.c mbfilter_htmlent.c mbfilter_byte2.c mbfilter_byte4.c mbfilter_uuencode.c mbfilter_base64.c mbfilter_sjis.c mbfilter_7bit.c mbfilter_qprint.c mbfilter_ucs4.c mbfilter_ucs2.c mbfilter_utf32.c mbfilter_utf16.c mbfilter_utf8.c mbfilter_utf7.c mbfilter_utf7imap.c mbfilter_euc_jp_win.c mbfilter_cp932.c mbfilter_cp51932.c mbfilter_euc_cn.c mbfilter_euc_kr.c mbfilter_uhc.c mbfilter_iso2022_kr.c mbfilter_cp866.c mbfilter_koi8r.c mbfilter_armscii8.c html_entities.c cp932_table.h html_entities.h mbfilter_7bit.h mbfilter_ascii.h mbfilter_base64.h mbfilter_big5.h mbfilter_byte2.h mbfilter_byte4.h mbfilter_cp1251.h mbfilter_cp1252.h mbfilter_cp866.h mbfilter_cp932.h mbfilter_cp51932.h mbfilter_cp936.h mbfilter_euc_cn.h mbfilter_euc_jp.h mbfilter_euc_jp_win.h mbfilter_euc_kr.h mbfilter_euc_tw.h mbfilter_htmlent.h mbfilter_hz.h mbfilter_iso2022_kr.h mbfilter_iso8859_1.h mbfilter_iso8859_10.h mbfilter_iso8859_13.h mbfilter_iso8859_14.h mbfilter_iso8859_15.h mbfilter_iso8859_16.h mbfilter_iso8859_2.h mbfilter_iso8859_3.h mbfilter_iso8859_4.h mbfilter_iso8859_5.h mbfilter_iso8859_6.h mbfilter_iso8859_7.h mbfilter_iso8859_8.h mbfilter_iso8859_9.h mbfilter_jis.h mbfilter_koi8r.h mbfilter_armscii8.h mbfilter_qprint.h mbfilter_sjis.h mbfilter_ucs2.h mbfilter_ucs4.h mbfilter_uhc.h mbfilter_utf16.h mbfilter_utf32.h mbfilter_utf7.h mbfilter_utf7imap.h mbfilter_utf8.h mbfilter_uuencode.h unicode_prop.h unicode_table_big5.h unicode_table_cns11643.h unicode_table_cp1251.h unicode_table_cp1252.h unicode_table_cp866.h unicode_table_cp932_ext.h unicode_table_cp936.h unicode_table_iso8859_10.h unicode_table_iso8859_13.h unicode_table_iso8859_14.h unicode_table_iso8859_15.h unicode_table_iso8859_16.h unicode_table_iso8859_2.h unicode_table_iso8859_3.h unicode_table_iso8859_4.h unicode_table_iso8859_5.h unicode_table_iso8859_6.h unicode_table_iso8859_7.h unicode_table_iso8859_8.h unicode_table_iso8859_9.h unicode_table_jis.h unicode_table_koi8r.h unicode_table_armscii8.h unicode_table_uhc.h mbfilter_iso8859_2.c: unicode_table_iso8859_2.h diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp51932.c b/ext/mbstring/libmbfl/filters/mbfilter_cp51932.c new file mode 100644 index 00000000000..2e167e26e94 --- /dev/null +++ b/ext/mbstring/libmbfl/filters/mbfilter_cp51932.c @@ -0,0 +1,360 @@ +/* + * "streamable kanji code filter and converter" + * Copyright (c) 1998-2002 HappySize, Inc. All rights reserved. + * + * LICENSE NOTICES + * + * This file is part of "streamable kanji code filter and converter", + * which is distributed under the terms of GNU Lesser General Public + * License (version 2) as published by the Free Software Foundation. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with "streamable kanji code filter and converter"; + * if not, write to the Free Software Foundation, Inc., 59 Temple Place, + * Suite 330, Boston, MA 02111-1307 USA + * + * The author of this file: + * + */ +/* + * The source code included in this files was separated from mbfilter_ja.c + * by moriyoshi koizumi on 4 dec 2002. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "mbfilter.h" +#include "mbfilter_cp51932.h" + +#include "unicode_table_cp932_ext.h" +#include "unicode_table_jis.h" +#include "cp932_table.h" + +static int mbfl_filt_ident_cp51932(int c, mbfl_identify_filter *filter); + +static const unsigned char mblen_table_eucjp[] = { /* 0xA1-0xFE */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1 +}; + + +static const char *mbfl_encoding_cp51932_aliases[] = {"cp51932", NULL}; + +const struct mbfl_identify_vtbl vtbl_identify_cp51932 = { + mbfl_no_encoding_cp51932, + mbfl_filt_ident_common_ctor, + mbfl_filt_ident_common_dtor, + mbfl_filt_ident_cp51932 +}; + +const mbfl_encoding mbfl_encoding_cp51932 = { + mbfl_no_encoding_cp51932, + "CP51932", + "CP51932", + (const char *(*)[])&mbfl_encoding_cp51932_aliases, + mblen_table_eucjp, + MBFL_ENCTYPE_MBCS +}; + +const struct mbfl_convert_vtbl vtbl_cp51932_wchar = { + mbfl_no_encoding_cp51932, + mbfl_no_encoding_wchar, + mbfl_filt_conv_common_ctor, + mbfl_filt_conv_common_dtor, + mbfl_filt_conv_cp51932_wchar, + mbfl_filt_conv_common_flush +}; + +const struct mbfl_convert_vtbl vtbl_wchar_cp51932 = { + mbfl_no_encoding_wchar, + mbfl_no_encoding_cp51932, + mbfl_filt_conv_common_ctor, + mbfl_filt_conv_common_dtor, + mbfl_filt_conv_wchar_cp51932, + mbfl_filt_conv_common_flush +}; + +#define CK(statement) do { if ((statement) < 0) return (-1); } while (0) + +#define sjistoidx(c1, c2) \ + (((c1) > 0x9f) \ + ? (((c1) - 0xc1) * 188 + (c2) - (((c2) > 0x7e) ? 0x41 : 0x40)) \ + : (((c1) - 0x81) * 188 + (c2) - (((c2) > 0x7e) ? 0x41 : 0x40))) +#define idxtoeuc1(c) (((c) / 94) + 0xa1) +#define idxtoeuc2(c) (((c) % 94) + 0xa1) + +/* + * cp51932 => wchar + */ +int +mbfl_filt_conv_cp51932_wchar(int c, mbfl_convert_filter *filter) +{ + int c1, s, w, n; + + switch (filter->status) { + case 0: + if (c >= 0 && c < 0x80) { /* latin */ + CK((*filter->output_function)(c, filter->data)); + } else if (c > 0xa0 && c < 0xff) { /* CP932 first char */ + filter->status = 1; + filter->cache = c; + } else if (c == 0x8e) { /* kana first char */ + filter->status = 2; + } else { + w = c & MBFL_WCSGROUP_MASK; + w |= MBFL_WCSGROUP_THROUGH; + CK((*filter->output_function)(w, filter->data)); + } + break; + + case 1: /* got first half */ + filter->status = 0; + c1 = filter->cache; + if (c > 0xa0 && c < 0xff) { + w = 0; + s = (c1 - 0xa1)*94 + c - 0xa1; + if (s <= 137) { + if (s == 31) { + w = 0xff3c; /* FULLWIDTH REVERSE SOLIDUS */ + } else if (s == 32) { + w = 0xff5e; /* FULLWIDTH TILDE */ + } else if (s == 33) { + w = 0x2225; /* PARALLEL TO */ + } else if (s == 60) { + w = 0xff0d; /* FULLWIDTH HYPHEN-MINUS */ + } else if (s == 80) { + w = 0xffe0; /* FULLWIDTH CENT SIGN */ + } else if (s == 81) { + w = 0xffe1; /* FULLWIDTH POUND SIGN */ + } else if (s == 137) { + w = 0xffe2; /* FULLWIDTH NOT SIGN */ + } + } + if (w == 0) { + if (s >= cp932ext1_ucs_table_min && s < cp932ext1_ucs_table_max) { /* vendor ext1 (13ku) */ + w = cp932ext1_ucs_table[s - cp932ext1_ucs_table_min]; + } else if (s >= 0 && s < jisx0208_ucs_table_size) { /* X 0208 */ + w = jisx0208_ucs_table[s]; + } else if (s >= cp932ext2_ucs_table_min && s < cp932ext2_ucs_table_max) { /* vendor ext2 (89ku - 92ku) */ + w = cp932ext2_ucs_table[s - cp932ext2_ucs_table_min]; + } + } + if (w <= 0) { + w = ((c1 & 0x7f) << 8) | (c & 0x7f); + w &= MBFL_WCSPLANE_MASK; + w |= MBFL_WCSPLANE_WINCP932; + } + CK((*filter->output_function)(w, filter->data)); + } else if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */ + CK((*filter->output_function)(c, filter->data)); + } else { + w = (c1 << 8) | c; + w &= MBFL_WCSGROUP_MASK; + w |= MBFL_WCSGROUP_THROUGH; + CK((*filter->output_function)(w, filter->data)); + } + break; + + case 2: /* got 0x8e, X0201 kana */ + filter->status = 0; + if (c > 0xa0 && c < 0xe0) { + w = 0xfec0 + c; + CK((*filter->output_function)(w, filter->data)); + } else if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */ + CK((*filter->output_function)(c, filter->data)); + } else { + w = 0x8e00 | c; + w &= MBFL_WCSGROUP_MASK; + w |= MBFL_WCSGROUP_THROUGH; + CK((*filter->output_function)(w, filter->data)); + } + break; + + default: + filter->status = 0; + break; + } + + return c; +} + +int +cp932ext3_to_cp51932(int c) +{ + int idx; + + idx = sjistoidx(c >> 8, c & 0xff); + if (idx >= sjistoidx(0xfa, 0x5c)) + idx -= sjistoidx(0xfa, 0x5c) - sjistoidx(0xed, 0x40); + else if (idx >= sjistoidx(0xfa, 0x55)) + idx -= sjistoidx(0xfa, 0x55) - sjistoidx(0xee, 0xfa); + else if (idx >= sjistoidx(0xfa, 0x40)) + idx -= sjistoidx(0xfa, 0x40) - sjistoidx(0xee, 0xef); + return idxtoeuc1(idx) << 8 | idxtoeuc2(idx); +} + +/* + * wchar => cp51932 + */ +int +mbfl_filt_conv_wchar_cp51932(int c, mbfl_convert_filter *filter) +{ + int c1, c2, s1; + + s1 = 0; + if (c >= ucs_a1_jis_table_min && c < ucs_a1_jis_table_max) { + s1 = ucs_a1_jis_table[c - ucs_a1_jis_table_min]; + } else if (c >= ucs_a2_jis_table_min && c < ucs_a2_jis_table_max) { + s1 = ucs_a2_jis_table[c - ucs_a2_jis_table_min]; + } else if (c >= ucs_i_jis_table_min && c < ucs_i_jis_table_max) { + s1 = ucs_i_jis_table[c - ucs_i_jis_table_min]; + } else if (c >= ucs_r_jis_table_min && c < ucs_r_jis_table_max) { + s1 = ucs_r_jis_table[c - ucs_r_jis_table_min]; + } + if (s1 >= 0x8080) s1 = -1; /* we don't support JIS X0213 */ + if (s1 <= 0) { + c1 = c & ~MBFL_WCSPLANE_MASK; + if (c1 == MBFL_WCSPLANE_WINCP932) { + s1 = c & MBFL_WCSPLANE_MASK; + if (s1 >= ((85 + 0x20) << 8)) { /* 85ku - 120ku */ + s1 = -1; + } + } else if (c1 == MBFL_WCSPLANE_JIS0208) { + s1 = c & MBFL_WCSPLANE_MASK; + if ((s1 >= ((85 + 0x20) << 8) && /* 85ku - 94ku */ + s1 <= ((88 + 0x20) << 8)) ||/* IBM extension */ + (s1 >= ((93 + 0x20) << 8) && /* 89ku - 92ku */ + s1 <= ((94 + 0x20) << 8))) { + s1 = -1; + } + } else if (c == 0xa5) { /* YEN SIGN */ + s1 = 0x216f; /* FULLWIDTH YEN SIGN */ + } else if (c == 0x203e) { /* OVER LINE */ + s1 = 0x2131; /* FULLWIDTH MACRON */ + } else if (c == 0xff3c) { /* FULLWIDTH REVERSE SOLIDUS */ + s1 = 0x2140; + } else if (c == 0xff5e) { /* FULLWIDTH TILDE */ + s1 = 0x2141; + } else if (c == 0x2225) { /* PARALLEL TO */ + s1 = 0x2142; + } else if (c == 0xff0d) { /* FULLWIDTH HYPHEN-MINUS */ + s1 = 0x215d; + } else if (c == 0xffe0) { /* FULLWIDTH CENT SIGN */ + s1 = 0x2171; + } else if (c == 0xffe1) { /* FULLWIDTH POUND SIGN */ + s1 = 0x2172; + } else if (c == 0xffe2) { /* FULLWIDTH NOT SIGN */ + s1 = 0x224c; + } else { + s1 = -1; + c1 = 0; + c2 = cp932ext1_ucs_table_max - cp932ext1_ucs_table_min; + while (c1 < c2) { /* CP932 vendor ext1 (13ku) */ + if (c == cp932ext1_ucs_table[c1]) { + s1 = ((c1/94 + 0x2d) << 8) + (c1%94 + 0x21); + break; + } + c1++; + } + if (s1 < 0) { + c1 = 0; + c2 = cp932ext2_ucs_table_max - cp932ext2_ucs_table_min; + while (c1 < c2) { /* CP932 vendor ext3 (115ku - 119ku) */ + if (c == cp932ext2_ucs_table[c1]) { + s1 = ((c1/94 + 0x79) << 8) +(c1%94 + 0x21); + break; + } + c1++; + } + } + } + if (c == 0) { + s1 = 0; + } else if (s1 <= 0) { + s1 = -1; + } + } + + if (s1 >= 0) { + if (s1 < 0x80) { /* latin */ + CK((*filter->output_function)(s1, filter->data)); + } else if (s1 < 0x100) { /* kana */ + CK((*filter->output_function)(0x8e, filter->data)); + CK((*filter->output_function)(s1, filter->data)); + } else if (s1 < 0x8080) { /* X 0208 */ + CK((*filter->output_function)(((s1 >> 8) & 0xff) | 0x80, filter->data)); + CK((*filter->output_function)((s1 & 0xff) | 0x80, filter->data)); + } else { + if (filter->illegal_mode != MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE) { + CK(mbfl_filt_conv_illegal_output(c, filter)); + } + } + } else { + if (filter->illegal_mode != MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE) { + CK(mbfl_filt_conv_illegal_output(c, filter)); + } + } + + return c; +} + +static int mbfl_filt_ident_cp51932(int c, mbfl_identify_filter *filter) +{ + switch (filter->status) { + case 0: /* latin */ + if (c >= 0 && c < 0x80) { /* ok */ + ; + } else if (c > 0xa0 && c < 0xff) { /* kanji first char */ + filter->status = 1; + } else if (c == 0x8e) { /* kana first char */ + filter->status = 2; + } else { /* bad */ + filter->flag = 1; + } + break; + + case 1: /* got first half */ + if (c < 0xa1 || c > 0xfe) { /* bad */ + filter->flag = 1; + } + filter->status = 0; + break; + + case 2: /* got 0x8e */ + if (c < 0xa1 || c > 0xdf) { /* bad */ + filter->flag = 1; + } + filter->status = 0; + break; + + default: + filter->status = 0; + break; + } + + return c; +} + + diff --git a/ext/mbstring/libmbfl/filters/mbfilter_cp51932.h b/ext/mbstring/libmbfl/filters/mbfilter_cp51932.h new file mode 100644 index 00000000000..be86b822623 --- /dev/null +++ b/ext/mbstring/libmbfl/filters/mbfilter_cp51932.h @@ -0,0 +1,43 @@ +/* + * "streamable kanji code filter and converter" + * Copyright (c) 1998-2002 HappySize, Inc. All rights reserved. + * + * LICENSE NOTICES + * + * This file is part of "streamable kanji code filter and converter", + * which is distributed under the terms of GNU Lesser General Public + * License (version 2) as published by the Free Software Foundation. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with "streamable kanji code filter and converter"; + * if not, write to the Free Software Foundation, Inc., 59 Temple Place, + * Suite 330, Boston, MA 02111-1307 USA + * + * The author of this file: + * + */ +/* + * The source code included in this files was separated from mbfilter_ja.h + * by moriyoshi koizumi on 4 dec 2002. + * + */ + +#ifndef MBFL_MBFILTER_CP51932_H +#define MBFL_MBFILTER_CP51932_H + +#include "mbfilter.h" + +extern const mbfl_encoding mbfl_encoding_cp51932; +extern const struct mbfl_identify_vtbl vtbl_identify_cp51932; +extern const struct mbfl_convert_vtbl vtbl_cp51932_wchar; +extern const struct mbfl_convert_vtbl vtbl_wchar_cp51932; + +int mbfl_filt_conv_cp51932_wchar(int c, mbfl_convert_filter *filter); +int mbfl_filt_conv_wchar_cp51932(int c, mbfl_convert_filter *filter); + +#endif /* MBFL_MBFILTER_CP51932_H */ diff --git a/ext/mbstring/libmbfl/filters/mbfilter_euc_jp_win.c b/ext/mbstring/libmbfl/filters/mbfilter_euc_jp_win.c index e57ae693562..bc07f134385 100644 --- a/ext/mbstring/libmbfl/filters/mbfilter_euc_jp_win.c +++ b/ext/mbstring/libmbfl/filters/mbfilter_euc_jp_win.c @@ -60,7 +60,8 @@ static const unsigned char mblen_table_eucjp[] = { /* 0xA1-0xFE */ }; -static const char *mbfl_encoding_eucjp_win_aliases[] = {"eucJP-open", NULL}; +static const char *mbfl_encoding_eucjp_win_aliases[] = {"eucJP-open", + "eucJP-ms", NULL}; const struct mbfl_identify_vtbl vtbl_identify_eucjpwin = { mbfl_no_encoding_eucjp_win, diff --git a/ext/mbstring/libmbfl/filters/mbfilter_jis.c b/ext/mbstring/libmbfl/filters/mbfilter_jis.c index 58336d4e284..3657658ba85 100644 --- a/ext/mbstring/libmbfl/filters/mbfilter_jis.c +++ b/ext/mbstring/libmbfl/filters/mbfilter_jis.c @@ -58,6 +58,15 @@ const mbfl_encoding mbfl_encoding_2022jp = { MBFL_ENCTYPE_MBCS | MBFL_ENCTYPE_SHFTCODE }; +const mbfl_encoding mbfl_encoding_2022jpms = { + mbfl_no_encoding_2022jpms, + "ISO-2022-JP-MS", + "ISO-2022-JP-MS", + NULL, + NULL, + MBFL_ENCTYPE_MBCS | MBFL_ENCTYPE_SHFTCODE +}; + const struct mbfl_identify_vtbl vtbl_identify_jis = { mbfl_no_encoding_jis, mbfl_filt_ident_common_ctor, @@ -72,6 +81,13 @@ const struct mbfl_identify_vtbl vtbl_identify_2022jp = { mbfl_filt_ident_2022jp }; +const struct mbfl_identify_vtbl vtbl_identify_2022jpms = { + mbfl_no_encoding_2022jpms, + mbfl_filt_ident_common_ctor, + mbfl_filt_ident_common_dtor, + mbfl_filt_ident_2022jp +}; + const struct mbfl_convert_vtbl vtbl_jis_wchar = { mbfl_no_encoding_jis, mbfl_no_encoding_wchar, @@ -108,6 +124,24 @@ const struct mbfl_convert_vtbl vtbl_wchar_2022jp = { mbfl_filt_conv_any_jis_flush }; +const struct mbfl_convert_vtbl vtbl_2022jpms_wchar = { + mbfl_no_encoding_2022jpms, + mbfl_no_encoding_wchar, + mbfl_filt_conv_common_ctor, + mbfl_filt_conv_common_dtor, + mbfl_filt_conv_jis_wchar, + mbfl_filt_conv_common_flush +}; + +const struct mbfl_convert_vtbl vtbl_wchar_2022jpms = { + mbfl_no_encoding_wchar, + mbfl_no_encoding_2022jpms, + mbfl_filt_conv_common_ctor, + mbfl_filt_conv_common_dtor, + mbfl_filt_conv_wchar_jis, + mbfl_filt_conv_any_jis_flush +}; + #define CK(statement) do { if ((statement) < 0) return (-1); } while (0) /* @@ -130,12 +164,10 @@ retry: filter->status += 2; } else if (c == 0x0e) { /* "kana in" */ filter->status = 0x20; + CK((*filter->output_function)(c, filter->data)); } else if (c == 0x0f) { /* "kana out" */ filter->status = 0; - } else if (filter->status == 0x10 && c == 0x5c) { /* YEN SIGN */ - CK((*filter->output_function)(0xa5, filter->data)); - } else if (filter->status == 0x10 && c == 0x7e) { /* OVER LINE */ - CK((*filter->output_function)(0x203e, filter->data)); + CK((*filter->output_function)(c, filter->data)); } else if (filter->status == 0x20 && c > 0x20 && c < 0x60) { /* kana */ CK((*filter->output_function)(0xff40 + c, filter->data)); } else if ((filter->status == 0x80 || filter->status == 0x90) && c > 0x20 && c < 0x7f) { /* kanji first char */ @@ -161,9 +193,34 @@ retry: s = (c1 - 0x21)*94 + c - 0x21; if (filter->status == 0x80) { if (s >= 0 && s < jisx0208_ucs_table_size) { + if ((filter->from)->no_encoding != + mbfl_no_encoding_2022jpms) { + w = jisx0208_ucs_table[s]; + } + else { + if ((c1 - 0x21) == 12) { + w = cp932ext1_ucs_table[s-12*94]; + } + else { + if (c1 >= 0x79 && c1 <= 0x7c) { + w = cp932ext2_ucs_table[s-(0x79-0x21)*94]; + } + else { w = jisx0208_ucs_table[s]; + } + } + } } else { + if ((filter->from)->no_encoding != + mbfl_no_encoding_2022jpms) { + w = 0; + } else { + if (c1 >= 0x79 && c1 <= 0x7c) { + w = cp932ext2_ucs_table[s-(0x79-0x21)*94]; + } else { w = 0; + } + } } if (w <= 0) { w = (c1 << 8) | c; @@ -287,7 +344,7 @@ retry: int mbfl_filt_conv_wchar_jis(int c, mbfl_convert_filter *filter) { - int c1, s; + int c1, c2, s; s = 0; if (c >= ucs_a1_jis_table_min && c < ucs_a1_jis_table_max) { @@ -299,6 +356,18 @@ mbfl_filt_conv_wchar_jis(int c, mbfl_convert_filter *filter) } else if (c >= ucs_r_jis_table_min && c < ucs_r_jis_table_max) { s = ucs_r_jis_table[c - ucs_r_jis_table_min]; } + if (s > 0x8080 && s < 0x10000 && + ((filter->to)->no_encoding == mbfl_no_encoding_2022jpms)) { + c1 = 0; + c2 = cp932ext2_ucs_table_max - cp932ext2_ucs_table_min; + while (c1 < c2) { /* CP932 vendor ext3 (115ku - 119ku) */ + if (c == cp932ext2_ucs_table[c1]) { + s = ((c1/94 + 0x79) << 8) +(c1%94 + 0x21); + break; + } + c1++; + } + } if (s <= 0) { c1 = c & ~MBFL_WCSPLANE_MASK; if (c1 == MBFL_WCSPLANE_JIS0208) { @@ -327,8 +396,30 @@ mbfl_filt_conv_wchar_jis(int c, mbfl_convert_filter *filter) } if (c == 0) { s = 0; - } else if (s <= 0) { + } else if (s <= 0 && ((filter->to)->no_encoding == + mbfl_no_encoding_2022jpms)) { s = -1; + c1 = 0; + c2 = cp932ext1_ucs_table_max - cp932ext1_ucs_table_min; + while (c1 < c2) { /* CP932 vendor ext1 (13ku) */ + if (c == cp932ext1_ucs_table[c1]) { + s = ((c1/94 + 0x2d) << 8) + (c1%94 + 0x21); + break; + } + c1++; + } + if (s < 0 && ((filter->to)->no_encoding == + mbfl_no_encoding_2022jpms)) { + c1 = 0; + c2 = cp932ext2_ucs_table_max - cp932ext2_ucs_table_min; + while (c1 < c2) { /* CP932 vendor ext3 (115ku - 119ku) */ + if (c == cp932ext2_ucs_table[c1]) { + s = ((c1/94 + 0x79) << 8) +(c1%94 + 0x21); + break; + } + c1++; + } + } } } if (s >= 0) { @@ -392,7 +483,7 @@ mbfl_filt_conv_wchar_jis(int c, mbfl_convert_filter *filter) int mbfl_filt_conv_wchar_2022jp(int c, mbfl_convert_filter *filter) { - int s; + int c1, c2, s; s = 0; if (c >= ucs_a1_jis_table_min && c < ucs_a1_jis_table_max) { @@ -428,9 +519,47 @@ mbfl_filt_conv_wchar_2022jp(int c, mbfl_convert_filter *filter) s = 0; } else if (s <= 0) { s = -1; + if ( (filter->to)->no_encoding == + mbfl_no_encoding_2022jpms) { + c1 = 0; + c2 = cp932ext1_ucs_table_max - cp932ext1_ucs_table_min; + while (c1 < c2) { /* CP932 vendor ext1 (13ku) */ + if (c == cp932ext1_ucs_table[c1]) { + s = ((c1/94 + 0x2d) << 8) + (c1%94 + 0x21); + break; + } + c1++; + } + if ((filter->to)->no_encoding == + mbfl_no_encoding_2022jpms) { + c1 = 0; + c2 = cp932ext2_ucs_table_max - cp932ext2_ucs_table_min; + while (c1 < c2) { /* CP932 vendor ext3 (115ku - 119ku) */ + if (c == cp932ext2_ucs_table[c1]) { + s = ((c1/94 + 0x79) << 8) +(c1%94 + 0x21); + break; + } + c1++; + } + } + } } - } else if ((s >= 0x80 && s < 0x2121) || (s > 0x8080)) { + } else if (((s >= 0x80 && s < 0x2121) && + (filter->to)->no_encoding != mbfl_no_encoding_2022jpms) || + (s > 0x8080)) { s = -1; + if ((filter->to)->no_encoding == + mbfl_no_encoding_2022jpms) { + c1 = 0; + c2 = cp932ext2_ucs_table_max - cp932ext2_ucs_table_min; + while (c1 < c2) { /* CP932 vendor ext3 (115ku - 119ku) */ + if (c == cp932ext2_ucs_table[c1]) { + s = ((c1/94 + 0x79) << 8) +(c1%94 + 0x21); + break; + } + c1++; + } + } } if (s >= 0) { if (s < 0x80) { /* ASCII */ @@ -441,6 +570,15 @@ mbfl_filt_conv_wchar_2022jp(int c, mbfl_convert_filter *filter) } filter->status = 0; CK((*filter->output_function)(s, filter->data)); + } else if (s < 0x100 && ((filter->to)->no_encoding == + mbfl_no_encoding_2022jpms)) { /* kana */ + if ((filter->status & 0xff00) != 0x100) { + CK((*filter->output_function)(0x1b, filter->data)); /* ESC */ + CK((*filter->output_function)(0x28, filter->data)); /* '(' */ + CK((*filter->output_function)(0x49, filter->data)); /* 'I' */ + } + filter->status = 0x100; + CK((*filter->output_function)(s & 0x7f, filter->data)); } else if (s < 0x10000) { /* X 0208 */ if ((filter->status & 0xff00) != 0x200) { CK((*filter->output_function)(0x1b, filter->data)); /* ESC */ diff --git a/ext/mbstring/libmbfl/filters/mbfilter_jis.h b/ext/mbstring/libmbfl/filters/mbfilter_jis.h index 03753072637..1ba244a282f 100644 --- a/ext/mbstring/libmbfl/filters/mbfilter_jis.h +++ b/ext/mbstring/libmbfl/filters/mbfilter_jis.h @@ -34,12 +34,16 @@ extern const mbfl_encoding mbfl_encoding_jis; extern const mbfl_encoding mbfl_encoding_2022jp; +extern const mbfl_encoding mbfl_encoding_2022jpms; extern const struct mbfl_identify_vtbl vtbl_identify_2022jp; +extern const struct mbfl_identify_vtbl vtbl_identify_2022jpms; extern const struct mbfl_identify_vtbl vtbl_identify_jis; extern const struct mbfl_convert_vtbl vtbl_jis_wchar; extern const struct mbfl_convert_vtbl vtbl_wchar_jis; extern const struct mbfl_convert_vtbl vtbl_2022jp_wchar; extern const struct mbfl_convert_vtbl vtbl_wchar_2022jp; +extern const struct mbfl_convert_vtbl vtbl_2022jpms_wchar; +extern const struct mbfl_convert_vtbl vtbl_wchar_2022jpms; int mbfl_filt_conv_jis_wchar(int c, mbfl_convert_filter *filter); int mbfl_filt_conv_wchar_jis(int c, mbfl_convert_filter *filter); diff --git a/ext/mbstring/libmbfl/filters/unicode_table_jis.h b/ext/mbstring/libmbfl/filters/unicode_table_jis.h index 5671c4e8513..e87dad93c8a 100644 --- a/ext/mbstring/libmbfl/filters/unicode_table_jis.h +++ b/ext/mbstring/libmbfl/filters/unicode_table_jis.h @@ -36,13 +36,13 @@ static const unsigned short jisx0208_ucs_table[] = { 0xFF1F,0xFF01,0x309B,0x309C,0x00B4,0xFF40,0x00A8,0xFF3E, 0xFFE3,0xFF3F,0x30FD,0x30FE,0x309D,0x309E,0x3003,0x4EDD, 0x3005,0x3006,0x3007,0x30FC,0x2015,0x2010,0xFF0F,0xFF3C, - 0x301C,0x2016,0xFF5C,0x2026,0x2025,0x2018,0x2019,0x201C, + 0xFF5E,0x2225,0xFF5C,0x2026,0x2025,0x2018,0x2019,0x201C, 0x201D,0xFF08,0xFF09,0x3014,0x3015,0xFF3B,0xFF3D,0xFF5B, 0xFF5D,0x3008,0x3009,0x300A,0x300B,0x300C,0x300D,0x300E, - 0x300F,0x3010,0x3011,0xFF0B,0x2212,0x00B1,0x00D7,0x00F7, + 0x300F,0x3010,0x3011,0xFF0B,0xFF0D,0x00B1,0x00D7,0x00F7, 0xFF1D,0x2260,0xFF1C,0xFF1E,0x2266,0x2267,0x221E,0x2234, 0x2642,0x2640,0x00B0,0x2032,0x2033,0x2103,0xFFE5,0xFF04, - 0x00A2,0x00A3,0xFF05,0xFF03,0xFF06,0xFF0A,0xFF20,0x00A7, + 0xFFE0,0xFFE1,0xFF05,0xFF03,0xFF06,0xFF0A,0xFF20,0x00A7, 0x2606,0x2605,0x25CB,0x25CF,0x25CE,0x25C7, /* ku 2 */ @@ -51,7 +51,7 @@ static const unsigned short jisx0208_ucs_table[] = { 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, 0x0000,0x2208,0x220B,0x2286,0x2287,0x2282,0x2283,0x222A, 0x2229,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, - 0x0000,0x2227,0x2228,0x00AC,0x21D2,0x21D4,0x2200,0x2203, + 0x0000,0x2227,0x2228,0xFFE2,0x21D2,0x21D4,0x2200,0x2203, 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, 0x0000,0x0000,0x0000,0x2220,0x22A5,0x2312,0x2202,0x2207, 0x2261,0x2252,0x226A,0x226B,0x221A,0x223D,0x221D,0x2235, @@ -1217,9 +1217,9 @@ static const unsigned short jisx0212_ucs_table[] = { /* ku 2 */ 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x02D8,0x02C7, - 0x00B8,0x02D9,0x02DD,0x00AF,0x02DB,0x02DA,0x007E,0x0384, + 0x00B8,0x02D9,0x02DD,0x00AF,0x02DB,0x02DA,0xFF5E,0x0384, 0x0385,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, - 0x0000,0x00A1,0x00A6,0x00BF,0x0000,0x0000,0x0000,0x0000, + 0x0000,0x00A1,0xFFE4,0x00BF,0x0000,0x0000,0x0000,0x0000, 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, @@ -2471,7 +2471,7 @@ static const unsigned short ucs_a2_jis_table[] = { /* 2100h */ 0x0000,0x0000,0x0000,0x216E,0x0000,0x0000,0x0000,0x0000, 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, - 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0xA2F1,0x0000, + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x2d62,0x0000, 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000, 0x0000,0x0000,0xA2EF,0x0000,0x0000,0x0000,0x0000,0x0000, 0x0000,0x0000,0x0000,0x2272,0x0000,0x0000,0x0000,0x0000, diff --git a/ext/mbstring/libmbfl/mbfl/mbfl_convert.c b/ext/mbstring/libmbfl/mbfl/mbfl_convert.c index 78a995cd25f..bcae8c4340c 100644 --- a/ext/mbstring/libmbfl/mbfl/mbfl_convert.c +++ b/ext/mbstring/libmbfl/mbfl/mbfl_convert.c @@ -51,6 +51,7 @@ #include "filters/mbfilter_euc_kr.h" #include "filters/mbfilter_iso2022_kr.h" #include "filters/mbfilter_sjis.h" +#include "filters/mbfilter_cp51932.h" #include "filters/mbfilter_jis.h" #include "filters/mbfilter_euc_jp.h" #include "filters/mbfilter_euc_jp_win.h" @@ -104,10 +105,14 @@ const struct mbfl_convert_vtbl *mbfl_convert_filter_list[] = { &vtbl_wchar_eucjp, &vtbl_sjis_wchar, &vtbl_wchar_sjis, + &vtbl_cp51932_wchar, + &vtbl_wchar_cp51932, &vtbl_jis_wchar, &vtbl_wchar_jis, &vtbl_2022jp_wchar, &vtbl_wchar_2022jp, + &vtbl_2022jpms_wchar, + &vtbl_wchar_2022jpms, &vtbl_eucjpwin_wchar, &vtbl_wchar_eucjpwin, &vtbl_sjiswin_wchar, diff --git a/ext/mbstring/libmbfl/mbfl/mbfl_encoding.c b/ext/mbstring/libmbfl/mbfl/mbfl_encoding.c index 64d8cb4e476..3c18fca9747 100644 --- a/ext/mbstring/libmbfl/mbfl/mbfl_encoding.c +++ b/ext/mbstring/libmbfl/mbfl/mbfl_encoding.c @@ -57,6 +57,7 @@ #include "filters/mbfilter_euc_kr.h" #include "filters/mbfilter_iso2022_kr.h" #include "filters/mbfilter_sjis.h" +#include "filters/mbfilter_cp51932.h" #include "filters/mbfilter_jis.h" #include "filters/mbfilter_euc_jp.h" #include "filters/mbfilter_euc_jp_win.h" @@ -149,8 +150,10 @@ static const mbfl_encoding *mbfl_encoding_ptr_list[] = { &mbfl_encoding_sjis, &mbfl_encoding_eucjp_win, &mbfl_encoding_sjis_win, + &mbfl_encoding_cp51932, &mbfl_encoding_jis, &mbfl_encoding_2022jp, + &mbfl_encoding_2022jpms, &mbfl_encoding_cp1252, &mbfl_encoding_8859_1, &mbfl_encoding_8859_2, diff --git a/ext/mbstring/libmbfl/mbfl/mbfl_encoding.h b/ext/mbstring/libmbfl/mbfl/mbfl_encoding.h index d38b1d7d4ac..af93b6ea504 100644 --- a/ext/mbstring/libmbfl/mbfl/mbfl_encoding.h +++ b/ext/mbstring/libmbfl/mbfl/mbfl_encoding.h @@ -70,8 +70,10 @@ enum mbfl_no_encoding { mbfl_no_encoding_eucjp_win, mbfl_no_encoding_sjis_win, mbfl_no_encoding_sjis_mac, + mbfl_no_encoding_cp51932, mbfl_no_encoding_jis, mbfl_no_encoding_2022jp, + mbfl_no_encoding_2022jpms, mbfl_no_encoding_cp1252, mbfl_no_encoding_8859_1, mbfl_no_encoding_8859_2, diff --git a/ext/mbstring/libmbfl/mbfl/mbfl_ident.c b/ext/mbstring/libmbfl/mbfl/mbfl_ident.c index 72ccef68d28..40c964c2050 100644 --- a/ext/mbstring/libmbfl/mbfl/mbfl_ident.c +++ b/ext/mbstring/libmbfl/mbfl/mbfl_ident.c @@ -106,6 +106,7 @@ static const struct mbfl_identify_vtbl *mbfl_identify_filter_list[] = { &vtbl_identify_sjiswin, &vtbl_identify_jis, &vtbl_identify_2022jp, + &vtbl_identify_2022jpms, &vtbl_identify_euccn, &vtbl_identify_cp936, &vtbl_identify_hz,