-
-
Notifications
You must be signed in to change notification settings - Fork 15
/
template_arfi.txt.template
63 lines (50 loc) · 3.38 KB
/
template_arfi.txt.template
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
---
name: ARFI
name_long: All Relevant, Fixed Irrelevant
scripts:
- get_plot.py
- merge_descriptives.py
- merge_metrics.py
- merge_tds.py
docs:
- README.md
---
{# This is a template for the ARFI method #}
# version {{ version }}
# Create folder structure. By default, the folder 'output' is used to store output.
mkdir {{ output_folder }}
mkdir {{ output_folder }}/simulation
mkdir {{ output_folder }}/tables
mkdir {{ output_folder }}/tables/metrics
mkdir {{ output_folder }}/tables/time_to_discovery
mkdir {{ output_folder }}/figures
{% for dataset in datasets %}
##################################
### DATASET: {{ dataset.input_file_stem }}
##################################
# Create output folder
mkdir {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/
mkdir {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/metrics
# Collect descriptives about the dataset
mkdir {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/descriptives
python -m asreview data describe {{ dataset.input_file }} -o {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/descriptives/data_stats_{{ dataset.input_file_stem }}.json
{% if not skip_wordclouds %}
# Generate wordcloud visualizations of all datasets
python -m asreview wordcloud {{ dataset.input_file }} -o {{ output_folder }}/figures/wordcloud_{{ dataset.input_file_stem }}.png --width 800 --height 500
python -m asreview wordcloud {{ dataset.input_file }} -o {{ output_folder }}/figures/wordcloud_relevant_{{ dataset.input_file_stem }}.png --width 800 --height 500 --relevant
python -m asreview wordcloud {{ dataset.input_file }} -o {{ output_folder }}/figures/wordcloud_irrelevant_{{ dataset.input_file_stem }}.png --width 800 --height 500 --irrelevant
{% endif %}
# Simulate runs, collect metrics and create plots
mkdir {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/state_files
{% for prior in dataset.priors %}
python -m asreview simulate {{ dataset.input_file }} -s {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/state_files/sim_{{ dataset.input_file_stem }}_{{ prior[0] }}.asreview --prior_record_id {{ " ".join(prior) }} --seed {{ dataset.model_seed }} -m {{ classifier }} -e {{ feature_extractor }} -q {{ query_strategy }} -b {{ balance_strategy }} --n_instances {{ instances_per_query }} --stop_if {{ stop_if }}
python -m asreview metrics {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/state_files/sim_{{ dataset.input_file_stem }}_{{ prior[0] }}.asreview -o {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/metrics/metrics_sim_{{ dataset.input_file_stem }}_{{ prior[0] }}.json
{% endfor %}
# Generate plot and tables for dataset
python {{ scripts_folder }}/get_plot.py -s {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/state_files/ -o {{ output_folder }}/figures/plot_recall_sim_{{ dataset.input_file_stem }}.png --show_legend model
python {{ scripts_folder }}/merge_metrics.py -s {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/metrics/ -o {{ output_folder }}/tables/metrics/metrics_sim_{{ dataset.input_file_stem }}.csv
python {{ scripts_folder }}/merge_tds.py -s {{ output_folder }}/simulation/{{ dataset.input_file_stem }}/metrics/ -o {{ output_folder }}/tables/time_to_discovery/tds_sim_{{ dataset.input_file_stem }}.csv
{% endfor %}
# Merge descriptives and metrics
python {{ scripts_folder }}/merge_descriptives.py
python {{ scripts_folder }}/merge_metrics.py