-
Notifications
You must be signed in to change notification settings - Fork 1
/
split_wet_file.py
134 lines (104 loc) · 3.22 KB
/
split_wet_file.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
import re
import os
import sys
import gzip
import json
import logging
import dataclasses
from functools import lru_cache
from typing import Optional
import fasttext
from ftlangdetect import detect
# fix fasttext warning message
fasttext.FastText.eprint = lambda x: None
# WET file constants
_PAGE_DELIMITER = "WARC/1.0"
_URL_KEY = "WARC-Target-URI:"
_URL_DATE = "WARC-Date:"
_LANGUAGE = "WARC-Identified-Content-Language:"
_CONTENT_TYPE = "Content-Type:"
_CONTENT_LEN = "Content-Length:"
_METADATA_PREFIXES = ("WARC", "CONTENT-", "Content-")
@dataclasses.dataclass
class PageFeatures:
url: str = ""
normalized_url: str = ""
text: str = ""
timestamp: str = ""
content_length: str = ""
content_type: str = ""
language: Optional[str] = None
langdetect: Optional[dict] = None
def normalize_url(url):
# url = tf.compat.as_text(url)
url = re.sub(r"https?:\/\/(www\.)?", "", url)
url = re.sub(r"\?(utm_|ref|feed).*", "", url)
url = url.rstrip("/")
return url
wet_file_path = sys.argv[1]
line_delimiter = '\n'
logging.info("Splitting file: %s", wet_file_path)
def _validate_features(page):
if page.url and page.text and page.timestamp:
return True
return False
def split_pages(wet_file_path):
with gzip.open(wet_file_path, mode="rt") as f:
page = PageFeatures()
for i, line in enumerate(f.readlines()):
line = line.strip()
if not line:
continue
if line == _PAGE_DELIMITER:
if i > 0 and _validate_features(page):
yield page
page = PageFeatures()
if line.startswith(_URL_KEY):
page.url = line[len(_URL_KEY) :].strip()
page.normalized_url = normalize_url(line[len(_URL_KEY) :].strip())
if line.startswith(_URL_DATE):
page.timestamp = line[len(_URL_DATE) :].strip()
if line.startswith(_LANGUAGE):
page.language = line[len(_LANGUAGE) :].strip()
if line.startswith(_CONTENT_TYPE):
page.content_type = line[len(_CONTENT_TYPE) :].strip()
if line.startswith(_CONTENT_LEN):
page.content_length = line[len(_CONTENT_LEN) :].strip()
if line.startswith(_METADATA_PREFIXES):
continue
if page.text:
page.text += line_delimiter
page.text += line
if _validate_features(page):
yield page
if not os.path.exists(wet_file_path):
print(f"input file not found: {wet_file_path}", file=sys.stderr)
sys.exit(1)
@lru_cache
def lazy_gzip_open():
return gzip.open(wet_file_path[:-len('gz')]+"pages.jsonl.gz", "wt", encoding="utf8")
total = 0
count = 0
for page in split_pages(wet_file_path):
keep = False
total += 1
if page.language:
if page.language.startswith('ara'):
keep = True
count += 1
else:
lines = sorted(page.text.split("\n"), key=lambda x: len(x), reverse=True)
selected = []
length = 0
total = sum(map(lambda x: len(x), lines))
while length / total < 0.5:
this_line = lines.pop(0)
selected.append(this_line)
length += len(this_line)
result = detect(text=" ".join(selected))
page.langdetect = result
if result['lang'] == 'ar':
keep = True
if keep:
lazy_gzip_open().write(json.dumps(dataclasses.asdict(page), ensure_ascii=False) + "\n")
logging.info("%d/%d extracted", count, total)