X2Static Training

Last time we used the HPC available to our university, we had to run our scripts inside docker images.

This meant that our script had to work end-to-end. It would load the data, clone the repositories, train the model and run the evaluation scripts from start to finish.

Following is the script we used in the end:

#!/bin/bash #SBATCH --container-image ghcr.io#bouncmpe/cuda-python3 #SBATCH --gpus=2 #SBATCH --cpus-per-gpu=16 #SBATCH --mem=100G #SBATCH --time=90:00:00

source /opt/python3/venv/base/bin/activate

echo "bounweb corpus" pip install wget python3 -c "import wget; wget.download('https://tulap.cmpe.boun.edu.tr/server/api/core/bitstreams/150f2e37-1dd3-4229-a37e-111f8a365edf/content')" python3 -c "import zipfile; zipfile.ZipFile('bounwebcorpus.txt.zip', 'r').extractall()" rm bounwebcorpus.txt.zip

echo "huwaei" pip install gdown python3 -m gdown 1PTytZ7yGIl9QvxRxCsfWLlHycU4z1_Vp

echo "vocab" python -m gdown 1Sjfh9c7gMa6lvsjprcnbtVyKYEJ8r2fH

echo "comb" python3 -c "open('combined.txt', 'w', encoding='utf-8').writelines([line for file in ['bounwebcorpus.txt', 'turkish-texts-tokenized.txt'] for line in open(file, 'r', encoding='utf-8')])"

mv bounwebcorpus.txt combined.txt

export REPO=Word-Embeddings-Repository-for-Turkish

git clone https://github.com/epfml/X2Static.git

pip install tqdm torch transformers nltk gensim "tensorflow-gpu==2.8.0" scikit-learn torchmetrics matplotlib

python X2Static/src/make_vocab_dataset.py --dataset_location combined.txt --location_save_vocab_dataset processed_data/

python X2Static/src/make_vocab_dataset.py --dataset_location combined.txt --min_count 10 --max_vocab_size 750000 --location_save_vocab_dataset processed_data/

ls -l processed_data

python X2Static/src/learn_from_bert_ver2_paragraph.py --pretrained_bert_model "dbmdz/bert-base-turkish-128k-cased" --location_dataset processed_data/ --model_folder x2static_model --gpu_id 0 --num_epochs 1 --lr 0.001 --algo SparseAdam --t 5e-6 --word_emb_size 768 --num_negatives 10

python X2Static/src/learn_from_bert_ver2_paragraph.py --gpu_id 0 --num_epochs 1 --lr 0.001 --algo SparseAdam --t 5e-6 --word_emb_size 768 --location_dataset processed_data/ --model_folder model/ --num_negatives 10 --pretrained_bert_model dbmdz/bert-base-turkish-128k-cased

ls -l

echo "model" ls -l /model mv /model/vectors_final.txt /bert-decontextualized-static.wv

pip install protobuf==3.20.*

Loop through each script in the array

git clone https://github.com/Turkish-Word-Embeddings/Word-Embeddings-Repository-for-Turkish.git

echo "NER" cd /$REPO/NLP/NER sed -i "s@FOLDER = "C:/Users/karab/Desktop/Models"@FOLDER = '/'@g" ner.py sed -i 's/"no_header": False/"no_header": True/g' ner.py python3 ner.py -w dc_bert

echo "PoS" cd /$REPO/NLP/PoS sed -i "s@FOLDER = "C:/Users/karab/Desktop/Models"@FOLDER = '/'@g" pos.py sed -i 's/"no_header": False/"no_header": True/g' pos.py python3 pos.py -w dc_bert

echo "SENTIMENT" cd "/$REPO/NLP/Sentiment Analysis/" sed -i 's@"C:/Users/karab/Desktop/Models"@"/"@g' sentiment.py sed -i 's/"no_header": False/"no_header": True/g' sentiment.py python3 sentiment.py -w dc_bert -d 1 -s 7 python3 sentiment.py -w dc_bert -d 1 -s 24 python3 sentiment.py -w dc_bert -d 1 -s 30

python3 sentiment.py -w dc_bert -d 2 -s 7 python3 sentiment.py -w dc_bert -d 2 -s 24 python3 sentiment.py -w dc_bert -d 2 -s 30

python3 sentiment.py -w dc_bert -d 3 -hs 196 -s 7 python3 sentiment.py -w dc_bert -d 3 -hs 196 -s 24 python3 sentiment.py -w dc_bert -d 3 -hs 196 -s 30