-
Notifications
You must be signed in to change notification settings - Fork 20
/
qass_preprocess.py
57 lines (43 loc) · 1.31 KB
/
qass_preprocess.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
import argparse
import json
import os
from glob import glob
from tqdm import tqdm
MASK_TOKEN = "[MASK]"
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument("--path", type=str, required=True)
args = parser.parse_args()
return args
def insert_mask_to_qa_end(data_point):
transformed_data_point = data_point.copy()
qas = []
for qa in transformed_data_point["qas"]:
qa = qa.copy()
qa["question"] += f" {MASK_TOKEN}."
qas.append(qa)
transformed_data_point["qas"] = qas
return transformed_data_point
def write_mrqa_single_path(data, path, header):
output_path = os.path.splitext(path)[0] + "_qass.jsonl"
if not os.path.exists(output_path):
with open(output_path, "w") as f:
f.write(header)
for dp in data:
f.write(json.dumps(insert_mask_to_qa_end(dp)) + '\n')
def read_mrqa(path):
data = []
with open(path) as f:
header = f.readline()
for line in f:
data_point = json.loads(line)
data.append(data_point)
return data, header
def main():
args = get_args()
paths = glob(args.path)
for path in tqdm(paths):
data, header = read_mrqa(path)
write_mrqa_single_path(data, path, header)
if __name__ == '__main__':
main()