-
Notifications
You must be signed in to change notification settings - Fork 0
/
train-distilbert-from-electra.sh
executable file
·152 lines (127 loc) · 3.9 KB
/
train-distilbert-from-electra.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
#!/bin/bash
set -o nounset
set -o errexit
VERBOSE_MODE=0
function error_handler()
{
local STATUS=${1:-1}
[ ${VERBOSE_MODE} == 0 ] && exit ${STATUS}
echo "Exits abnormally at line "`caller 0`
exit ${STATUS}
}
trap "error_handler" ERR
PROGNAME=`basename ${BASH_SOURCE}`
DRY_RUN_MODE=0
function print_usage_and_exit()
{
set +x
local STATUS=$1
echo "Usage: ${PROGNAME} [-v] [-v] [--dry-run] [-h] [--help]"
echo ""
echo " Options -"
echo " -v enables verbose mode 1"
echo " -v -v enables verbose mode 2"
echo " --dry-run show what would have been dumped"
echo " -h, --help shows this help message"
exit ${STATUS:-0}
}
function debug()
{
if [ "$VERBOSE_MODE" != 0 ]; then
echo $@
fi
}
#GETOPT=`getopt -o vh --long dry-run,help -n "${PROGNAME}" -- "$@"`
GETOPT=`getopt vh $*`
if [ $? != 0 ] ; then print_usage_and_exit 1; fi
eval set -- "${GETOPT}"
while true
do case "$1" in
-v) let VERBOSE_MODE+=1; shift;;
--dry-run) DRY_RUN_MODE=1; shift;;
-h|--help) print_usage_and_exit 0;;
--) shift; break;;
*) echo "Internal error!"; exit 1;;
esac
done
if (( VERBOSE_MODE > 1 )); then
set -x
fi
if [ ${#} != 0 ]; then print_usage_and_exit 1; fi
set -o errexit
function readlink()
{
TARGET_FILE=$2
cd `dirname $TARGET_FILE`
TARGET_FILE=`basename $TARGET_FILE`
# Iterate down a (possible) chain of symlinks
while [ -L "$TARGET_FILE" ]
do
TARGET_FILE=`readlink $TARGET_FILE`
cd `dirname $TARGET_FILE`
TARGET_FILE=`basename $TARGET_FILE`
done
# Compute the canonicalized name by finding the physical path
# for the directory we're in and appending the target file.
PHYS_DIR=`pwd -P`
RESULT=$PHYS_DIR/$TARGET_FILE
echo $RESULT
}
export -f readlink
# current dir of this script
CDIR=$(readlink -f $(dirname $(readlink -f ${BASH_SOURCE[0]})))
PDIR=$(readlink -f $(dirname $(readlink -f ${BASH_SOURCE[0]}))/..)
cd ${CDIR}
# binarize
file_path=korean/all.txt
tokenizer_name=koelectra-base-v3-discriminator
dump_file=korean/binarized_text
function binarize {
python distillation/scripts/binarized_data.py \
--file_path ${file_path} \
--tokenizer_type bert \
--tokenizer_name ${tokenizer_name} \
--dump_file ${dump_file}
}
# count occurences
data_file=korean/binarized_text.${tokenizer_name}.pickle
counts_dump=korean/token_counts.${tokenizer_name}.pickle
vocab_size=35000
function token_counts {
python distillation/scripts/token_counts.py \
--data_file ${data_file} \
--token_counts_dump ${counts_dump} \
--vocab_size ${vocab_size}
}
# dump lower layer weights from teacher model
student_pretrained_weights=korean/student_pretrained_weights
function extract_weights {
python distillation/scripts/extract_distilbert.py \
--model_type bert \
--model_name ${tokenizer_name} \
--dump_checkpoint ${student_pretrained_weights} \
--vocab_transform
}
# distillation
student_config=distilbert-base-from-electra.json
dump_path=korean/kor-distil-wp-bert.v1
batch_size=3 # default 5
function train {
python distillation/train.py \
--student_type distilbert \
--student_config ${student_config} \
--student_pretrained_weights ${student_pretrained_weights} \
--teacher_type bert \
--teacher_name ${tokenizer_name} \
--alpha_ce 5.0 --alpha_mlm 2.0 --alpha_cos 1.0 --alpha_clm 0.0 --mlm \
--batch_size ${batch_size} \
--freeze_pos_embs \
--dump_path ${dump_path} \
--data_file ${data_file} \
--token_counts ${counts_dump} \
--force # overwrites the `dump_path` if it already exists.
}
binarize
token_counts
extract_weights
train