iscc-core/iscc_core/code_content_text.py at main · iscc/iscc-core · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
# -*- coding: utf-8 -*-
"""*A similarity preserving hash for plain-text content (soft hash).*

The ISCC Text-Code is generated from plain-text that has been extracted
from a media assets.

!!! Warning

    Plain-text extraction from documents in various formats (especially PDF) may
    yield very diffent results depending on the extraction tools being used.
    The [iscc-sdk](https://github.com/iscc/iscc-sdk) uses [Apache Tika](https://tika.apache.org)
    to extract text from documents for Text-Code generation.

**Algorithm overview**

- Apply [`text_collapse`][iscc_core.code_content_text.text_collapse] function to text input
- Count characters of collapsed text
- Apply [`soft_hash_text_v0`][iscc_core.code_content_text.soft_hash_text_v0] to collapsed text
"""

import unicodedata
import xxhash
import iscc_core as ic

__all__ = [
    "gen_text_code",
    "gen_text_code_v0",
    "soft_hash_text_v0",
    "text_collapse",
]


def gen_text_code(text, bits=ic.core_opts.text_bits):
    # type: (str, int) -> dict
    """
    Create an ISCC Text-Code with the latest standard algorithm.

    :param str text: Plain text for Text-Code creation.
    :param int bits: Bit-length (multiple of 32) for ISCC Code Hash (default 64).
    :return: ISCC schema instance with Text-Code and an aditional property `characters`
    :rtype: dict
    """
    return gen_text_code_v0(text, bits)


def gen_text_code_v0(text, bits=ic.core_opts.text_bits):
    # type: (str, int) -> dict
    """
    Create an ISCC Text-Code with algorithm v0.

    !!! note
        Any markup (like HTML tags or markdown) should be removed from the plain-text
        before passing it to this function.

    :param str text: Text for Text-Code creation
    :param int bits: Bit-length of ISCC Code Hash (default 64)
    :return: ISCC schema instance with Text-Code and an aditional property `characters`
    :rtype: dict
    """

    text = text_collapse(text)
    characters = len(text)
    digest = soft_hash_text_v0(text)

    text_code = ic.encode_component(
        mtype=ic.MT.CONTENT,
        stype=ic.ST_CC.TEXT,
        version=ic.VS.V0,
        bit_length=bits,
        digest=digest,
    )

    iscc = "ISCC:" + text_code

    return dict(iscc=iscc, characters=characters)


def soft_hash_text_v0(text):
    # type: (str) -> bytes
    """
    Creates a 256-bit similarity preserving hash for text input with algorithm v0.

    - Slide over text with a
      [`text_ngram_size`][iscc_core.options.CoreOptions.text_ngram_size] wide window
      and create [`xxh32`](https://cyan4973.github.io/xxHash/) hashes
    - Create a [`minhash_256`][iscc_core.minhash.alg_minhash_256] from the hashes generated
      in the previous step.

    !!! note

        Before passing text to this function it must be:

        - stripped of markup
        - normalized
        - stripped of whitespace
        - lowercased

    :param str text: Plain text to be hashed.
    :return: 256-bit similarity preserving byte hash.
    :rtype: bytes
    """
    ngrams = ic.sliding_window(text, ic.core_opts.text_ngram_size)
    features = [xxhash.xxh32_intdigest(s.encode("utf-8")) for s in ngrams]
    hash_digest = ic.alg_minhash_256(features)
    return hash_digest


def text_collapse(text):
    # type: (str) -> str
    """
    Normalize and simplify text for similarity hashing.

    - Decompose with NFD normalization.
    - Remove all whitespace characters and convert text to lower case
    - Filter control characters, marks (diacritics), and punctuation
    - Recombine with NFKC normalization.

    !!! note

        See: [Unicode normalization](https://unicode.org/reports/tr15/).

    :param str text: Plain text to be collapsed.
    :return: Collapsed plain text.
    :rtype: str
    """

    # Decompose with NFD and convert to lower case
    text = unicodedata.normalize("NFD", text).lower()

    # Remove whitespace and filter characters in one pass
    filtered_chars = []

    for ch in text:
        if not ch.isspace() and unicodedata.category(ch)[0] not in ic.core_opts.text_unicode_filter:
            filtered_chars.append(ch)

    # Recombine
    return unicodedata.normalize("NFKC", "".join(filtered_chars))