-
Notifications
You must be signed in to change notification settings - Fork 2
/
demo.py
88 lines (60 loc) · 2.28 KB
/
demo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""
Scripts serves as demo. Supports passing in arbitrary occupation
description strings and additionally showcases performance on
toy dataset.
Example use:
1) python demo.py
2) python demo.py --examples "man who fishes" "trainman"
3) python demo.py --toy-dataset-fn-out toy-dataset-example.csv
"""
import argparse
from histocc import OccCANINE, DATASETS
import pandas as pd
def parse_args() -> argparse.Namespace:
_default_example_strs = [
["tailor of the finest suits"],
["the train's fireman"],
["nurse at the local hospital"],
]
parser = argparse.ArgumentParser()
parser.add_argument('--examples', type=str, nargs='+', default=None)
parser.add_argument('--toy-dataset-fn-out', type=str, default=None)
parser.add_argument('--language', type=str, default='en') # TODO add arg choices based on supported languages
parser.add_argument('--threshold', type=float, default=0.22) # Best F1 for English
args = parser.parse_args()
if args.examples is None:
args.examples = _default_example_strs
return args
def load_toydata() -> pd.DataFrame: # TODO probably move fn within OccCANINE
fn_keys = files('histocc').joinpath('Data/TOYDATA.csv')
with fn_keys.open() as file:
keys = pd.read_csv(file)
return keys
def main():
args = parse_args()
# Load model
model = OccCANINE()
# Loop through single-str example
for example_str in args.examples:
occ_code, prob, occ = model.predict(
example_str,
lang=args.language,
get_dict=True,
threshold=args.threshold,
)[0][0]
print(f'HISCO code: {occ_code}. Occupation: {occ}. Certainty: {prob * 100:.2f}%')
# Predict on toy dataset if filename for output specified
if args.toy_dataset_fn_out is None:
return
print('--toy-dataset-fn-out specified -- predicting codes for toy data')
data = DATASETS['toydata']()
model.verbose = True # Set updates to True
model_prediction = model.predict(
data["occ1"],
lang=args.language,
threshold=args.threshold,
)
print(f'Writing output to {args.toy_dataset_fn_out}')
model_prediction.to_csv(args.toy_dataset_fn_out, index=False)
if __name__ == '__main__':
main()