-
Notifications
You must be signed in to change notification settings - Fork 663
/
JISAutoDetect.java
248 lines (207 loc) · 9.07 KB
/
JISAutoDetect.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
/*
* Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package sun.nio.cs.ext;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.MalformedInputException;
import sun.nio.cs.DelegatableDecoder;
import sun.nio.cs.HistoricallyNamedCharset;
import java.security.AccessController;
import java.security.PrivilegedAction;
import sun.nio.cs.*;
import static java.lang.Character.UnicodeBlock;
public class JISAutoDetect
extends Charset
implements HistoricallyNamedCharset
{
private final static int EUCJP_MASK = 0x01;
private final static int SJIS2B_MASK = 0x02;
private final static int SJIS1B_MASK = 0x04;
private final static int EUCJP_KANA1_MASK = 0x08;
private final static int EUCJP_KANA2_MASK = 0x10;
public JISAutoDetect() {
super("x-JISAutoDetect", ExtendedCharsets.aliasesFor("x-JISAutoDetect"));
}
public boolean contains(Charset cs) {
return ((cs.name().equals("US-ASCII"))
|| (cs instanceof SJIS)
|| (cs instanceof EUC_JP)
|| (cs instanceof ISO2022_JP));
}
public boolean canEncode() {
return false;
}
public CharsetDecoder newDecoder() {
return new Decoder(this);
}
public String historicalName() {
return "JISAutoDetect";
}
public CharsetEncoder newEncoder() {
throw new UnsupportedOperationException();
}
// A heuristic algorithm for guessing if EUC-decoded text really
// might be Japanese text. Better heuristics are possible...
private static boolean looksLikeJapanese(CharBuffer cb) {
int hiragana = 0; // Fullwidth Hiragana
int katakana = 0; // Halfwidth Katakana
while (cb.hasRemaining()) {
char c = cb.get();
if (0x3040 <= c && c <= 0x309f && ++hiragana > 1) return true;
if (0xff65 <= c && c <= 0xff9f && ++katakana > 1) return true;
}
return false;
}
private static class Decoder extends CharsetDecoder {
private final static String osName = AccessController.doPrivileged(
(PrivilegedAction<String>) () -> System.getProperty("os.name"));
private final static String SJISName = getSJISName();
private final static String EUCJPName = getEUCJPName();
private DelegatableDecoder detectedDecoder = null;
public Decoder(Charset cs) {
super(cs, 0.5f, 1.0f);
}
private static boolean isPlainASCII(byte b) {
return b >= 0 && b != 0x1b;
}
private static void copyLeadingASCII(ByteBuffer src, CharBuffer dst) {
int start = src.position();
int limit = start + Math.min(src.remaining(), dst.remaining());
int p;
byte b;
for (p = start; p < limit && isPlainASCII(b = src.get(p)); p++)
dst.put((char)(b & 0xff));
src.position(p);
}
private CoderResult decodeLoop(DelegatableDecoder decoder,
ByteBuffer src, CharBuffer dst) {
((CharsetDecoder)decoder).reset();
detectedDecoder = decoder;
return detectedDecoder.decodeLoop(src, dst);
}
protected CoderResult decodeLoop(ByteBuffer src, CharBuffer dst) {
if (detectedDecoder == null) {
copyLeadingASCII(src, dst);
// All ASCII?
if (! src.hasRemaining())
return CoderResult.UNDERFLOW;
// Overflow only if there is still ascii but no out buffer.
if (!dst.hasRemaining() &&
isPlainASCII(src.get(src.position())))
return CoderResult.OVERFLOW;
// We need to perform double, not float, arithmetic; otherwise
// we lose low order bits when src is larger than 2**24.
int cbufsiz = (int)(src.limit() * (double)maxCharsPerByte());
CharBuffer sandbox = CharBuffer.allocate(cbufsiz);
// First try ISO-2022-JP, since there is no ambiguity
Charset cs2022 = Charset.forName("ISO-2022-JP");
DelegatableDecoder dd2022
= (DelegatableDecoder) cs2022.newDecoder();
ByteBuffer src2022 = src.asReadOnlyBuffer();
CoderResult res2022 = dd2022.decodeLoop(src2022, sandbox);
if (! res2022.isError())
return decodeLoop(dd2022, src, dst);
// We must choose between EUC and SJIS
Charset csEUCJ = Charset.forName(EUCJPName);
Charset csSJIS = Charset.forName(SJISName);
DelegatableDecoder ddEUCJ
= (DelegatableDecoder) csEUCJ.newDecoder();
DelegatableDecoder ddSJIS
= (DelegatableDecoder) csSJIS.newDecoder();
ByteBuffer srcEUCJ = src.asReadOnlyBuffer();
sandbox.clear();
CoderResult resEUCJ = ddEUCJ.decodeLoop(srcEUCJ, sandbox);
// If EUC decoding fails, must be SJIS
if (resEUCJ.isError())
return decodeLoop(ddSJIS, src, dst);
ByteBuffer srcSJIS = src.asReadOnlyBuffer();
CharBuffer sandboxSJIS = CharBuffer.allocate(cbufsiz);
CoderResult resSJIS = ddSJIS.decodeLoop(srcSJIS, sandboxSJIS);
// If SJIS decoding fails, must be EUC
if (resSJIS.isError())
return decodeLoop(ddEUCJ, src, dst);
// From here on, we have some ambiguity, and must guess.
// We prefer input that does not appear to end mid-character.
if (srcEUCJ.position() > srcSJIS.position())
return decodeLoop(ddEUCJ, src, dst);
if (srcEUCJ.position() < srcSJIS.position())
return decodeLoop(ddSJIS, src, dst);
// end-of-input is after the first byte of the first char?
if (src.position() == srcEUCJ.position())
return CoderResult.UNDERFLOW;
// Use heuristic knowledge of typical Japanese text
sandbox.flip();
return decodeLoop(looksLikeJapanese(sandbox) ? ddEUCJ : ddSJIS,
src, dst);
}
return detectedDecoder.decodeLoop(src, dst);
}
protected void implReset() {
detectedDecoder = null;
}
protected CoderResult implFlush(CharBuffer out) {
if (detectedDecoder != null)
return detectedDecoder.implFlush(out);
else
return super.implFlush(out);
}
public boolean isAutoDetecting() {
return true;
}
public boolean isCharsetDetected() {
return detectedDecoder != null;
}
public Charset detectedCharset() {
if (detectedDecoder == null)
throw new IllegalStateException("charset not yet detected");
return ((CharsetDecoder) detectedDecoder).charset();
}
/**
* Returned Shift_JIS Charset name is OS dependent
*/
private static String getSJISName() {
if (osName.equals("Solaris") || osName.equals("SunOS"))
return("PCK");
else if (osName.startsWith("Windows"))
return("windows-31J");
else
return("Shift_JIS");
}
/**
* Returned EUC-JP Charset name is OS dependent
*/
private static String getEUCJPName() {
if (osName.equals("Solaris") || osName.equals("SunOS"))
return("x-eucjp-open");
else
return("EUC_JP");
}
}
}