preprocess_de_en.sh

#! /bin/bash

# check if calling script has set $base
if [ $# -eq 0 ]; then
  script_dir=`dirname "$0"`
  base=$script_dir/../..
else
  base=$1
fi;

mkdir -p $base/shared_models

data=$base/data
scripts=$base/scripts
shared_models=$base/shared_models

src=de
trg=en

domains="all it koran law medical subtitles"

MOSES=$base/tools/moses-scripts/scripts

bpe_num_operations=32000
bpe_vocab_threshold=10

# mirror final vocab size for Sockeye BPE training
sentencepiece_vocab_size=29218

. $scripts/preprocessing/preprocess_data_generic.sh
. $scripts/preprocessing/preprocess_out_of_domain_test_data_generic.sh