results = detector.detect(tis, metadata, new ParseContext());
if (!results.isEmpty()) {
- return results.get(0).getCharset();
+ Charset detected = results.get(0).getCharset();
+ Charset superset = CharsetSupersets.supersetOf(detected);
+ if (superset != null) {
+ metadata.set(TikaCoreProperties.DECODED_CHARSET, superset.name());
+ return superset;
+ }
+ return detected;
}
- Charset charset = null;
// Try determining the encoding based on hints in document metadata
MediaType type = MediaType.parse(metadata.get(Metadata.CONTENT_TYPE));
diff --git a/tika-core/src/main/java/org/apache/tika/detect/CharsetSupersets.java b/tika-core/src/main/java/org/apache/tika/detect/CharsetSupersets.java
new file mode 100644
index 0000000000..f53c98f847
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/detect/CharsetSupersets.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Maps detected charsets to safer superset charsets for decoding.
+ *
+ * When Tika detects a charset that is a strict subset of a broader encoding,
+ * it is safer to decode with the superset — the superset handles all byte
+ * sequences the subset can produce, plus the extension characters the subset
+ * cannot represent. Decoding with only the subset risks mojibake on any
+ * extension characters present in the document.
+ *
+ * Policy: Content-Type and detected-encoding metadata report the detected
+ * charset. Actual stream decoding uses the superset. The superset used is recorded
+ * in {@link org.apache.tika.metadata.TikaCoreProperties#DECODED_CHARSET}.
+ *
+ * Superset map
+ *
+ * - EUC-KR → x-windows-949 (MS949 is a strict superset: all EUC-KR byte sequences
+ * decode identically, extension chars in x-windows-949 would mojibake under EUC-KR)
+ * - Big5 → Big5-HKSCS (HKSCS adds Hong Kong Supplementary Characters)
+ * - GB2312 → GB18030 (GB18030 is a strict superset of both GB2312 and GBK)
+ * - GBK → GB18030 (GB18030 is a strict superset; enables 4-byte extension sequences)
+ * - Shift_JIS → windows-31j (MS932 is a strict superset with NEC/IBM extensions)
+ *
+ */
+public final class CharsetSupersets {
+
+ /**
+ * Maps detected charset canonical names (case-sensitive, as returned by
+ * {@link Charset#name()}) to their superset charset canonical name.
+ */
+ public static final Map SUPERSET_MAP;
+
+ static {
+ Map m = new HashMap<>();
+ m.put("EUC-KR", "x-windows-949");
+ m.put("Big5", "Big5-HKSCS");
+ m.put("GB2312", "GB18030");
+ m.put("GBK", "GB18030");
+ m.put("Shift_JIS", "windows-31j");
+ SUPERSET_MAP = Collections.unmodifiableMap(m);
+ }
+
+ private CharsetSupersets() {
+ }
+
+ /**
+ * Returns the superset charset to use for decoding, or {@code null} if
+ * {@code detected} has no superset override.
+ *
+ * @param detected the charset returned by the encoding detector
+ * @return superset charset, or {@code null} if none is defined
+ */
+ public static Charset supersetOf(Charset detected) {
+ if (detected == null) {
+ return null;
+ }
+ String supersetName = SUPERSET_MAP.get(detected.name());
+ if (supersetName == null) {
+ return null;
+ }
+ try {
+ return Charset.forName(supersetName);
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
index 6d513a2a67..06e0ce4f2c 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
@@ -437,6 +437,18 @@ public interface TikaCoreProperties {
Property ENCODING_DETECTION_TRACE =
Property.externalText(TIKA_META_PREFIX + "encodingDetectionTrace");
+ /**
+ * The charset actually used to decode the stream when a superset override was applied.
+ * When the detected encoding (reported in Content-Type and {@link #DETECTED_ENCODING}) is
+ * a subset of a safer, broader charset (e.g. EUC-KR is a subset of x-windows-949, or
+ * GB2312 is a subset of GB18030), Tika decodes using the superset charset to avoid
+ * mojibake on extension characters. This field records the superset charset name so
+ * callers know which codec was actually used. Absent when detection and decoding use
+ * the same charset.
+ */
+ Property DECODED_CHARSET =
+ Property.externalText(TIKA_META_PREFIX + "decodedCharset");
+
/**
* General metadata key for the count of non-final versions available within a file. This
* was added initially to support generalizing incremental updates in PDF.
diff --git a/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java b/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java
index f68e6720d2..e9fc080c29 100644
--- a/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java
+++ b/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/CharSoupEncodingDetector.java
@@ -190,7 +190,7 @@ private Charset arbitrate(TikaInputStream tis,
Map candidates = new LinkedHashMap<>();
for (Charset candidate : uniqueCharsets) {
- candidates.put(candidate, stripTags(decode(bytes, candidate)));
+ candidates.put(candidate, HtmlStripper.strip(decode(bytes, candidate)));
}
CharSoupLanguageDetector langDetector = new CharSoupLanguageDetector();
@@ -449,26 +449,6 @@ static String decode(byte[] bytes, Charset charset) {
return cb.toString();
}
- /**
- * Simple tag stripping: removes <...> sequences so that
- * HTML/XML tag names and attributes don't pollute language scoring.
- */
- static String stripTags(String text) {
- StringBuilder sb = new StringBuilder(text.length());
- boolean inTag = false;
- for (int i = 0; i < text.length(); i++) {
- char c = text.charAt(i);
- if (c == '<') {
- inTag = true;
- } else if (c == '>') {
- inTag = false;
- } else if (!inTag) {
- sb.append(c);
- }
- }
- return sb.toString();
- }
-
public int getReadLimit() {
return readLimit;
}
diff --git a/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/HtmlStripper.java b/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/HtmlStripper.java
new file mode 100644
index 0000000000..fd6ef5f78a
--- /dev/null
+++ b/tika-encoding-detectors/tika-encoding-detector-charsoup/src/main/java/org/apache/tika/langdetect/charsoup/HtmlStripper.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.langdetect.charsoup;
+
+/**
+ * HTML/XML markup stripping tuned for language scoring and charset
+ * disambiguation. Not a full HTML parser — purpose-built to feed
+ * character-bigram language detectors a markup-free string that still
+ * carries the page's content language.
+ *
+ * Real-world HTML probes are routinely 95-99% markup by byte count.
+ * Without this pass, a language detector sees the markup as its primary
+ * input — which on any HTML page looks predominantly like ASCII English
+ * regardless of the page's actual content language. Stripping the markup
+ * lets the detector see the actual content.
+ *
+ *
What it does, in one linear pass
+ *
+ * - Removes {@code } and {@code }
+ * block contents — JavaScript identifiers / CSS property names look
+ * strongly like English and would skew language scoring on any
+ * page.
+ * - Removes {@code } comments.
+ * - Removes {@code <...>} tag markup (element names, attribute names,
+ * attribute values).
+ * - Replaces named entity references ({@code &}, {@code },
+ * {@code ©}) with a space — these are nearly always
+ * punctuation/typography with low language signal, and a full
+ * named-entity table would be heavyweight.
+ * - Default ({@link #strip(String)}): drops numeric character
+ * references ({@code Ӓ}, {@code ꯍ}) to a single
+ * space, on the grounds that a single numeric-entity-heavy section
+ * can expand to a very different byte distribution than the raw
+ * probe we are trying to characterise — at charset-detection time
+ * we want to score the raw bytes, not a synthetic Unicode rendering
+ * of them.
+ * - Opt-in ({@link #stripAndDecodeNumeric(String)}): decodes
+ * numeric character references to their actual code points. Useful
+ * where numeric entities carry the page's primary content (e.g.
+ * pages that emit CJK ideographs via {@code &#NNNN;} for
+ * cross-charset compatibility, so the decoded content reaches a
+ * downstream language scorer).
+ *
+ *
+ * What it doesn't do
+ *
+ * - Validate HTML structure. Malformed input, unclosed
+ * {@code "
+ + ""
+ + "real content here