Move mnist models into a floder. (#2207)

* Clearup the models for mnist dataset

* Move mnist models into a folder

* Pre-commit

* Fix unit tests with the new directory name

* Reformat codes

* Rename mnist directory

README.md

@@ -31,7 +31,7 @@ with Keras API, train the model distributedly with a command line.
 elasticdl train \
   --image_name=elasticdl:mnist \
   --model_zoo=model_zoo \
-  --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+  --model_def=mnist.mnist_functional_api.custom_model \
   --training_data=/data/mnist/train \
   --job_name=test-mnist \
   --volume="host_path=/data,mount_path=/data"

docs/tutorials/elasticdl_cloud.md

@@ -154,7 +154,7 @@ training docker image will be pushed to google cloud repo.
 elasticdl train \
   --image_name=gcr.io/${PROJECT_ID}/elasticdl:mnist \
   --model_zoo=model_zoo \
-  --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+  --model_def=mnist.mnist_functional_api.custom_model \
   --training_data=/data/mnist/train \
   --validation_data=/data/mnist/test \
   --num_epochs=5 \

docs/tutorials/elasticdl_local.md

@@ -116,7 +116,7 @@ The following command submits a training job:
 elasticdl train \
   --image_name=elasticdl:mnist \
   --model_zoo=model_zoo \
-  --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+  --model_def=mnist.mnist_functional_api.custom_model \
   --training_data=/data/mnist/train \
   --validation_data=/data/mnist/test \
   --num_epochs=2 \

docs/tutorials/elasticdl_on_prem_cluster.md

@@ -39,7 +39,7 @@ We launch a training job with 2 PS pods and 4 worker pods.
 elasticdl train \
   --image_name=${DOCKER_HUB_REPO}/elasticdl:mnist \
   --model_zoo=model_zoo \
-  --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+  --model_def=mnist.mnist_functional_api.custom_model \
   --training_data=/data/mnist/train \
   --validation_data=/data/mnist/test \
   --num_epochs=5 \
@@ -131,7 +131,7 @@ submit a job.
     elasticdl train \
     --image_name=${DOCKER_HUB_REPO}/elasticdl:mnist \
     --model_zoo=model_zoo \
-    --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+    --model_def=mnist.mnist_functional_api.custom_model \
     --training_data=/data/mnist/train \
     --validation_data=/data/mnist/test \
     --num_epochs=5 \

elasticdl/README.md

@@ -130,7 +130,7 @@ docker run --net=host --rm -it -v $EDL_REPO:/edl_dir -w /edl_dir \
     elasticdl:dev \
     bash -c "python -m elasticdl.python.master.main \
           --model_zoo=model_zoo \
-          --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+          --model_def=mnist.mnist_functional_api.custom_model \
           --job_name=test \
           --training_data=/data/mnist/train \
           --validation_data=/data/mnist/test \
@@ -151,7 +151,7 @@ docker run --net=host --rm -it -v $EDL_REPO:/edl_dir -w /edl_dir \
     bash -c "python -m elasticdl.python.worker.main \
           --worker_id=1 \
           --model_zoo=model_zoo \
-          --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+          --model_def=mnist.mnist_functional_api.custom_model \
           --minibatch_size=10 \
           --job_type=training_with_evaluation \
           --master_addr=localhost:50001 \

elasticdl/manifests/elasticdl.yaml

@@ -18,7 +18,7 @@ spec:
         - >
           python -m elasticdl.python.master.main
           --model_zoo=model_zoo
-          --model_def=mnist_functional_api.mnist_functional_api.custom_model
+          --model_def=mnist.mnist_functional_api.custom_model
           --training_data=/data/mnist/train
           --validation_data=/data/mnist/test
           --num_epochs=2

elasticdl/python/tests/example_test.py

@@ -109,8 +109,8 @@ def test_deepfm_functional_evaluate(self):
 
     def test_mnist_train(self):
         model_defs = [
-            "mnist_functional_api.mnist_functional_api.custom_model",
-            "mnist_subclass.mnist_subclass.CustomModel",
+            "mnist.mnist_functional_api.custom_model",
+            "mnist.mnist_subclass.CustomModel",
         ]
 
         model_versions = []
@@ -126,8 +126,8 @@ def test_mnist_train(self):
 
     def test_mnist_evaluate(self):
         model_defs = [
-            "mnist_functional_api.mnist_functional_api.custom_model",
-            "mnist_subclass.mnist_subclass.CustomModel",
+            "mnist.mnist_functional_api.custom_model",
+            "mnist.mnist_subclass.CustomModel",
         ]
         for model_def in model_defs:
             self._test_evaluate([28, 28], model_def)

elasticdl/python/tests/layer_test.py

@@ -40,7 +40,7 @@ def _create_model_instance(model_def):
 
 class FindLayerTest(unittest.TestCase):
     def test_find_layer(self):
-        model_def = "mnist_functional_api.mnist_functional_api.custom_model"
+        model_def = "mnist.mnist_functional_api.custom_model"
         model = _create_model_instance(model_def)
 
         layer_num = {

elasticdl/python/tests/worker_allreduce_strategy_test.py

@@ -26,9 +26,7 @@ def setUp(self):
         self._model_zoo_path = os.path.join(
             os.path.dirname(os.path.realpath(__file__)), "../../../model_zoo"
         )
-        self._model_def = (
-            "mnist_functional_api.mnist_functional_api.custom_model"
-        )
+        self._model_def = "mnist.mnist_functional_api.custom_model"
         self._batch_size = 16
         self._test_steps = 10
         self._workers = []

elasticdl/python/tests/worker_ps_interaction_test.py

@@ -144,7 +144,7 @@ def _test_deepfm_train(self, num_ps, num_worker, stop_step):
             t.join()
 
     def test_worker_pull_embedding(self):
-        model_def = "mnist_functional_api.mnist_functional_api.custom_model"
+        model_def = "mnist.mnist_functional_api.custom_model"
         self._create_pserver(model_def, 2)
         arguments = [
             "--worker_id",
@@ -195,7 +195,7 @@ def test_worker_pull_embedding(self):
             self.assertTrue(np.allclose(expected_result, result_dict[layer]))
 
     def test_compare_onebatch_train(self):
-        model_def = "mnist_functional_api.mnist_functional_api.custom_model"
+        model_def = "mnist.mnist_functional_api.custom_model"
         self._create_pserver(model_def, 2)
         images, labels = get_random_batch(self._batch_size)
         # TODO(yunjian.lmh): test optimizer wrapper
@@ -264,7 +264,7 @@ def test_compare_onebatch_train(self):
             np.testing.assert_array_equal(ps_v.numpy(), v.numpy())
 
     def test_compare_mnist_train(self):
-        model_def = "mnist_functional_api.mnist_functional_api.custom_model"
+        model_def = "mnist.mnist_functional_api.custom_model"
         self._create_pserver(model_def, 2)
         db, test_db = get_mnist_dataset(self._batch_size)
         stop_step = 20
@@ -348,7 +348,7 @@ def test_deepfm_four_worker_train(self):
         self._test_deepfm_train(num_ps, num_worker, stop_step)
 
     def test_restart_ps(self):
-        model_def = "mnist_functional_api.mnist_functional_api.custom_model"
+        model_def = "mnist.mnist_functional_api.custom_model"
         num_data = 8
         training_data = [
             get_random_batch(self._batch_size) for _ in range(num_data)

scripts/client_test.sh

@@ -53,7 +53,7 @@ elif [[ "$JOB_TYPE" == "evaluate" ]]; then
     elasticdl evaluate \
       --image_name=elasticdl:ci \
       --model_zoo=model_zoo \
-      --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+      --model_def=mnist.mnist_functional_api.custom_model \
       --checkpoint_dir_for_init=${MNIST_CKPT_DIR}/version-100  \
       --validation_data=/data/mnist/test \
       --num_epochs=1 \
@@ -77,7 +77,7 @@ elif [[ "$JOB_TYPE" == "predict" ]]; then
     elasticdl predict \
       --image_name=elasticdl:ci \
       --model_zoo=model_zoo \
-      --model_def=mnist_functional_api.mnist_functional_api.custom_model \
+      --model_def=mnist.mnist_functional_api.custom_model \
       --checkpoint_dir_for_init=${MNIST_CKPT_DIR}/version-100 \
       --prediction_data=/data/mnist/test \
       --master_resource_request="cpu=0.2,memory=1024Mi" \