diff --git a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java index 2c46a8db3a..2302f110d4 100644 --- a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java +++ b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/StructuralEncodingRules.java @@ -749,6 +749,43 @@ public static Utf8Result checkUtf8(byte[] bytes, int offset, int length) { return Utf8Result.AMBIGUOUS; } + /** + * True if the sample has a COMPLETE 3-/4-byte UTF-8 sequence (lead + * {@code 0xE0–0xF4}, continuations {@code 0x80–0xBF}). A legacy 2-byte CJK + * pair can fake a 2-byte UTF-8 char but never a wide one — the discriminator + * between real UTF-8 and a CJK coincidence. Tests width only; caller must + * already know the sample is valid UTF-8 (see {@link #checkUtf8}). + */ + public static boolean hasWideUtf8Sequence(byte[] bytes) { + return hasWideUtf8Sequence(bytes, 0, bytes.length); + } + + public static boolean hasWideUtf8Sequence(byte[] bytes, int offset, int length) { + int end = offset + length; + for (int i = offset; i < end; i++) { + int b = bytes[i] & 0xFF; + if (b < 0xE0 || b > 0xF4) { + continue; + } + int seqLen = (b >= 0xF0) ? 4 : 3; + if (i + seqLen > end) { + continue; // truncated lead at probe end — not proof + } + boolean complete = true; + for (int k = 1; k < seqLen; k++) { + int cb = bytes[i + k] & 0xFF; + if (cb < 0x80 || cb > 0xBF) { + complete = false; + break; + } + } + if (complete) { + return true; + } + } + return false; + } + /** * Counts the number of malformed UTF-8 sequences in the sample — * one event per bad lead, orphaned continuation, overlong, surrogate, or diff --git a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java index 76af07b373..46f100e63d 100644 --- a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java +++ b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetector.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -42,6 +43,7 @@ import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.ml.chardetect.AdaptiveProbe; +import org.apache.tika.ml.chardetect.StructuralEncodingRules; import org.apache.tika.parser.ParseContext; import org.apache.tika.quality.TextQualityDetector; @@ -203,6 +205,20 @@ public List detect(TikaInputStream tis, Metadata metadata, // decoded as the author intended. Charset declared = pickDeclarativeWithEquivalentDecode(context, candidates); if (declared != null) { + // A STRUCTURAL proof (valid multi-byte UTF-8, ISO-2022, BOM) outranks a + // DECLARATIVE self-claim on conflict. The equivalent-decode shortcut + // compares tag-stripped text, so a doc whose only high bytes live in + // markup (a attribute, an tag) ties on visible text and + // would honour a declaration the bytes contradict. Defer to the proof + // (CJK-declaration guard in #overridesDeclared). + Charset structural = conflictingStructural(context, candidates, declared); + if (structural != null && overridesDeclared(structural, declared, bytes)) { + context.setArbitrationInfo("junk-filter-prefer-structural"); + LOG.trace("junk-filter -> {} (structural proof overrides declared {})", + structural.name(), declared.name()); + return List.of(new EncodingResult(structural, + context.getTopConfidenceFor(structural))); + } float conf = context.getTopConfidenceFor(declared); context.setArbitrationInfo("junk-filter-prefer-declarative"); LOG.trace("junk-filter -> {} (declarative with equivalent decode)", @@ -581,6 +597,36 @@ private static Charset pickDeclarativeWithEquivalentDecode( return null; } + /** A decoded STRUCTURAL candidate other than the declared charset — evidence + * that can outrank the declaration; {@code null} if none. */ + private static Charset conflictingStructural(EncodingDetectorContext context, + Map candidates, Charset declared) { + for (EncodingDetectorContext.Result r : context.getResults()) { + for (EncodingResult er : r.getEncodingResults()) { + if (er.getResultType() != EncodingResult.ResultType.STRUCTURAL) { + continue; + } + Charset cs = er.getCharset(); + if (!cs.equals(declared) && candidates.containsKey(cs)) { + return cs; + } + } + } + return null; + } + + /** Whether a conflicting STRUCTURAL proof overrides the declaration. UTF-8 vs + * a variable-length CJK declaration is ambiguous on a short probe (a 2-byte CJK + * pair can fake a 2-byte UTF-8 char, never a wide one) → require a wide + * sequence; all other proofs are unambiguous. */ + private static boolean overridesDeclared(Charset structural, Charset declared, + byte[] bytes) { + if (StandardCharsets.UTF_8.equals(structural) && isCjkCharset(declared.name())) { + return StructuralEncodingRules.hasWideUtf8Sequence(bytes); + } + return true; + } + private static boolean allDecodingsIdentical(Map candidates) { String first = null; for (String s : candidates.values()) { diff --git a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java index a0904289b8..f1ae65a856 100644 --- a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java +++ b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java @@ -231,6 +231,96 @@ public void noopWhenAllDecodingsIdentical() throws Exception { } } + /** {@code plain body} with {@code hi} as the attribute's high + * bytes — all non-ASCII lives in markup, so every candidate's tag-stripped + * text ties and the equivalent-decode shortcut engages. */ + private static byte[] markupWithHighBytes(byte[] hi) { + byte[] pre = "plain ascii body text here".getBytes(StandardCharsets.US_ASCII); + byte[] out = new byte[pre.length + hi.length + post.length]; + System.arraycopy(pre, 0, out, 0, pre.length); + System.arraycopy(hi, 0, out, pre.length, hi.length); + System.arraycopy(post, 0, out, pre.length + hi.length, post.length); + return out; + } + + @Test + public void structuralUtf8OverridesDeclaredLatin() throws Exception { + // Real-bug shape (TIKA test.html): declared x-MacRoman but bytes are UTF-8 + // (é = C3 A9) with the only high bytes in markup, so stripped text ties. + // A STRUCTURAL proof must beat the contradicted declaration. + Charset utf8 = StandardCharsets.UTF_8; + Charset macRoman = Charset.forName("x-MacRoman"); + byte[] bytes = markupWithHighBytes(new byte[] {(byte) 0xC3, (byte) 0xA9}); + + ParseContext pc = contextWith( + new EncodingResult(macRoman, 0.9f, "x-MacRoman", + EncodingResult.ResultType.DECLARATIVE), + new EncodingResult(utf8, 0.95f, "UTF-8", + EncodingResult.ResultType.STRUCTURAL)); + + JunkFilterEncodingDetector detector = + new JunkFilterEncodingDetector(new PreferenceStub("UTF-8")); + try (TikaInputStream tis = TikaInputStream.get(bytes)) { + List out = detector.detect(tis, new Metadata(), pc); + assertEquals(1, out.size()); + assertEquals(utf8, out.get(0).getCharset()); + assertEquals("junk-filter-prefer-structural", + pc.get(EncodingDetectorContext.class).getArbitrationInfo()); + } + } + + @Test + public void structuralUtf8DefersToDeclaredCjkWithoutWideSequence() throws Exception { + // Declared Shift_JIS + STRUCTURAL UTF-8, but the only UTF-8 evidence is a + // 2-byte sequence a legacy CJK pair can fake — keep the declaration. + Charset utf8 = StandardCharsets.UTF_8; + Charset sjis = Charset.forName("Shift_JIS"); + byte[] bytes = markupWithHighBytes(new byte[] {(byte) 0xC3, (byte) 0xA9}); + + ParseContext pc = contextWith( + new EncodingResult(sjis, 0.9f, "Shift_JIS", + EncodingResult.ResultType.DECLARATIVE), + new EncodingResult(utf8, 0.95f, "UTF-8", + EncodingResult.ResultType.STRUCTURAL)); + + JunkFilterEncodingDetector detector = + new JunkFilterEncodingDetector(new PreferenceStub("UTF-8")); + try (TikaInputStream tis = TikaInputStream.get(bytes)) { + List out = detector.detect(tis, new Metadata(), pc); + assertEquals(1, out.size()); + assertEquals(sjis, out.get(0).getCharset()); + assertEquals("junk-filter-prefer-declarative", + pc.get(EncodingDetectorContext.class).getArbitrationInfo()); + } + } + + @Test + public void structuralUtf8OverridesDeclaredCjkWithWideSequence() throws Exception { + // Declared Shift_JIS + STRUCTURAL UTF-8 with a 3-byte sequence (日 = + // E6 97 A5): unfakeable by a legacy 2-byte CJK pair, so the proof wins. + Charset utf8 = StandardCharsets.UTF_8; + Charset sjis = Charset.forName("Shift_JIS"); + byte[] bytes = markupWithHighBytes( + new byte[] {(byte) 0xE6, (byte) 0x97, (byte) 0xA5}); + + ParseContext pc = contextWith( + new EncodingResult(sjis, 0.9f, "Shift_JIS", + EncodingResult.ResultType.DECLARATIVE), + new EncodingResult(utf8, 0.95f, "UTF-8", + EncodingResult.ResultType.STRUCTURAL)); + + JunkFilterEncodingDetector detector = + new JunkFilterEncodingDetector(new PreferenceStub("UTF-8")); + try (TikaInputStream tis = TikaInputStream.get(bytes)) { + List out = detector.detect(tis, new Metadata(), pc); + assertEquals(1, out.size()); + assertEquals(utf8, out.get(0).getCharset()); + assertEquals("junk-filter-prefer-structural", + pc.get(EncodingDetectorContext.class).getArbitrationInfo()); + } + } + // NOTE: a full default-constructor integration test (which would load // the bundled JunkDetector via ServiceLoader) is not included here // because JunkDetector currently exposes only static factory methods