Minor edits.

.gitignore

@@ -22,6 +22,9 @@ project/plugins/project/
 # IntelliJ Specific
 **/.idea/**/*
 
-# Ensime Specific
+# IDEs and Build Tools
+.bloop
+.metals
+.vscode
 .ensime
 .ensime_cache

.jvmopts

@@ -1,5 +1,5 @@
--Xmx8G
--Xms1G
--Xss2M
+-Xmx24G
+-Xms4G
+-Xss4M
 -XX:MaxDirectMemorySize=16384M
 -XX:MaxMetaspaceSize=1024M

configurations/experiments/iwslt15/en-vi/transformer-base-128.conf

@@ -25,6 +25,10 @@ training {
 
   optimization {
     max-grad-norm: 10.0
+
+    learning-rate {
+      value: 2.0
+    }
   }
 }
 

configurations/experiments/iwslt15/en-vi/transformer-base-512.conf

@@ -25,6 +25,10 @@ training {
 
   optimization {
     max-grad-norm: 10.0
+
+    learning-rate {
+      value: 2.0
+    }
   }
 }
 

configurations/experiments/iwslt15/en-vi/transformer-tiny-128-32.conf

@@ -1,11 +1,11 @@
 include "base.conf"
 
 data {
-  train-batch-size: 512
+  train-batch-size: 32
   infer-batch-size: 128
   eval-batch-size: 128
 
-  include "../../../vocabularies/word-count-20k.conf"
+  include "../../../vocabularies/bpe-shared-5k.conf"
 }
 
 model {
@@ -30,6 +30,10 @@ model {
 
 training {
   checkpoint-frequency: 5000
+
+  optimization {
+    optimizer: "adaptive_amsgrad"
+  }
 }
 
 evaluation {

configurations/experiments/iwslt16/fr-en/transformer-base-512.conf

@@ -25,6 +25,10 @@ training {
 
   optimization {
     max-grad-norm: 10.0
+
+    learning-rate {
+      value: 2.0
+    }
   }
 }
 

docs/src/main/paradox/assets/custom.css

@@ -17,16 +17,6 @@
     color: rgb(227, 4, 0);
 }
 
-.md-logo {
-    margin-left: 10px;
-    padding: 0.2rem;
-}
-
-.md-logo img {
-    width: 40px;
-    height: 40px;
-}
-
 .md-header-nav__title {
     font-family: 'Lato', sans-serif;
     font-weight: 400;

docs/src/main/paradox/index.md

@@ -1,38 +1,38 @@
 # Symphony Machine Translation
 
-@@@index
-- [Getting Started](getting-started.md)
-- [Data](data.md)
-- [Models](models.md)
-- [Learning](learning.md)
-- [Experiments](experiments.md)
-@@@
+@@@ index
+
+* [Getting Started](getting-started.md)
+* [Data](data.md)
+* [Models](models.md)
+* [Learning](learning.md)
+* [Experiments](experiments.md)
 
-<!-- - [GitHub Repository]: https://github.com/eaplatanios/symphony-mt -->
+@@@
 
 Symphony MT is a modular and extensible machine translation library that 
 supports both pairwise and multilingual translation, across an arbitrary 
 number of languages. It also supports zero-shot translation using either 
 pivoting/bridging, or by providing models that natively support multilingual 
 translation. It contains modules for:
 
-- Downloading and preprocessing data:
+  - Downloading and preprocessing data:
     - **Datasets:** IWSLT-14, IWSLT-15, IWSLT-16, IWSLT-17, WMT-16, etc.
     - **Processors:** data cleaning, normalization, tokenization, etc.
-- Constructing MT models using various features:
+  - Constructing MT models using various features:
     - Various encoders and decoders (e.g., RNNs, bi-directional RNNs, and
       transformer).
     - Attention models (e.g., Bahdanau, Luong, and multi-head).
     - Greedy and beam search decoding.
-- Training MT models:
+  - Training MT models:
     - Lots of supported optimizers (e.g., SGD, Adam, AMSGrad, and YellowFin).
     - Checkpointing.
     - Monitoring (e.g., TensorBoard support).
     - Distributed (e.g., multi-GPU) training.
-- Evaluating MT models:
+  - Evaluating MT models:
     - Can evaluate both while training and after.
     - Supports various metrics (e.g., BLEU, Rouge, Meteor, and TER).
-- Using trained MT models to perform inference.
+  - Using trained MT models to perform inference.
 
 TODO: Add paper and link.
 

docs/src/main/scala/LoadPairwiseDataset.scala

@@ -4,35 +4,27 @@ import org.platanios.symphony.mt.data.loaders.IWSLT15Loader
 import org.platanios.symphony.mt.data.processors._
 import org.platanios.symphony.mt.vocabulary._
 
-import java.nio.file.{Path, Paths}
+import java.nio.file.Paths
 
 object LoadPairwiseDataset {
 
   // #load_pairwise_dataset_example
   val dataConfig = DataConfig(
     // Loader
-    workingDir = Paths.get("data"),
+    dataDir = Paths.get("data"),
     loaderBufferSize = 8192,
     tokenizer = MosesTokenizer(),
     cleaner = MosesCleaner(),
-    vocabulary = GeneratedVocabulary(
-      SimpleVocabularyGenerator(sizeThreshold = 50000, countThreshold = -1)),
+    vocabulary = GeneratedVocabulary(SimpleVocabularyGenerator(sizeThreshold = 50000, countThreshold = -1)),
     // Corpus
-    parallelPortion = 1.0f,
     trainBatchSize = 128,
     inferBatchSize = 32,
-    evaluateBatchSize = 32,
+    evalBatchSize = 32,
     numBuckets = 5,
     srcMaxLength = 50,
     tgtMaxLength = 50,
-    bufferSize = -1L,
-    numShards = 1,
-    shardIndex = 0,
-    numParallelCalls = 4,
-    // Vocabulary
-    unknownToken = Vocabulary.UNKNOWN_TOKEN,
-    beginOfSequenceToken = Vocabulary.BEGIN_OF_SEQUENCE_TOKEN,
-    endOfSequenceToken = Vocabulary.END_OF_SEQUENCE_TOKEN)
+    shuffleBufferSize = -1L,
+    numParallelCalls = 4)
 
   val loader = IWSLT15Loader(English, Vietnamese, dataConfig)
   // #load_pairwise_dataset_example

mt/src/main/scala/org/platanios/symphony/mt/models/helpers/AdaptiveAMSGrad.scala

@@ -213,7 +213,7 @@ class AdaptiveAMSGrad protected (
       ).castTo[T]
 
       val gradientValuesSquare = gradient.values * gradient.values
-      val beta2 = computeBeta(
+      var beta2 = computeBeta(
         step = iteration.get.toFloat,
         initialBeta = getBeta2(variable).toFloat,
         previousBeta = getSlot[T, Float]("Beta2", variable),
@@ -224,6 +224,9 @@ class AdaptiveAMSGrad protected (
         epsilon = epsilonTensor
       ).castTo[T]
 
+
+      beta2 = tf.print(beta2, Seq(beta1, beta2), variable.name, 1000, 1000)
+
       val epsilon = getEpsilon(variable)
 
       var learningRate = getLearningRate(variable, iteration)
@@ -289,10 +292,7 @@ class AdaptiveAMSGrad protected (
     val logGNorm = tf.log(gNorm + epsilon)
     val logANorm = tf.log(aNorm + epsilon)
     val logBeta = (logGNorm - logANorm + logInitialBeta) / stepsTillNextNonZero
-    val beta = tf.maximum(0.0f, tf.minimum(1.0f, tf.exp(logBeta)))
-    //    beta = tf.print(beta, Seq(gNorm), "GNorm: ", 1000, 1000)
-    //    beta = tf.print(beta, Seq(logANorm), "LogANorm: ", 1000, 1000)
-    //    beta = tf.print(beta, Seq(beta), "Beta: ", 1000, 1000)
+    val beta = tf.maximum(0.001f, tf.minimum(0.9f, tf.exp(logBeta)))
     previousBeta.assignScatter(nonZeroIndices, beta)
   }
 }