update new reader for resnet, mobilenet; test=develop (#4685)

dygraph/mobilenet/README.md

@@ -2,6 +2,10 @@
 
 图像分类是计算机视觉的重要领域，它的目标是将图像分类到预定义的标签。CNN模型在图像分类领域取得了突破的成果，同时模型复杂度也在不断增加。MobileNet是一种小巧而高效CNN模型，本文介绍如何使PaddlePaddle的动态图MobileNet进行图像分类。
 
+**版本要求**
+该模型在develop版本必须使用develop版本或者paddle 2.0alpha的版本才能够运行
+如果期望在paddle 1.8 上执行，请切换到 release/1.8 分支
+
 **代码结构**
 
     ├── run_mul_v1.sh                 # 多卡训练启动脚本_v1
@@ -55,9 +59,9 @@
 **模型性能**
 
     Model          Top-1(单卡/4卡)    Top-5(单卡/4卡)    收敛时间(单卡/4卡)
-    
+
     MobileNetV1    0.707/0.711        0.897/0.899        116小时/30.9小时
-    
+
     MobileNetV2    0.708/0.724        0.899/0.906        227.8小时/60.8小时
 
 **参考论文**

dygraph/mobilenet/imagenet_dataset.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import cv2
+import math
+import random
+import numpy as np
+
+from paddle.incubate.hapi.datasets import DatasetFolder
+from paddle.incubate.hapi.vision.transforms import transforms
+from paddle import fluid
+
+
+class ImageNetDataset(DatasetFolder):
+    def __init__(self,
+                 path,
+                 mode='train',
+                 image_size=224,
+                 resize_short_size=256):
+        super(ImageNetDataset, self).__init__(path)
+        self.mode = mode
+
+        normalize = transforms.Normalize(
+            mean=[123.675, 116.28, 103.53], std=[58.395, 57.120, 57.375])
+        if self.mode == 'train':
+            self.transform = transforms.Compose([
+                transforms.RandomResizedCrop(image_size),
+                transforms.RandomHorizontalFlip(),
+                transforms.Permute(mode='CHW'), normalize
+            ])
+        else:
+            self.transform = transforms.Compose([
+                transforms.Resize(resize_short_size),
+                transforms.CenterCrop(image_size),
+                transforms.Permute(mode='CHW'), normalize
+            ])
+
+    def __getitem__(self, idx):
+        img_path, label = self.samples[idx]
+        img = cv2.imread(img_path).astype(np.float32)
+        label = np.array([label]).astype(np.int64)
+        return self.transform(img), label
+
+    def __len__(self):
+        return len(self.samples)

dygraph/mobilenet/train.py

@@ -31,6 +31,9 @@
 from mobilenet_v1 import *
 from mobilenet_v2 import *
 
+from imagenet_dataset import ImageNetDataset
+from paddle.io import DataLoader
+
 args = parse_args()
 if int(os.getenv("PADDLE_TRAINER_ID", 0)) == 0:
     print_arguments(args)
@@ -116,14 +119,31 @@ def train_mobilenet():
             optimizer.set_dict(opti_dict)
 
         # 3. reader
-        train_data_loader = utility.create_data_loader(is_train=True, args=args)
         test_data_loader = utility.create_data_loader(is_train=False, args=args)
         num_trainers = int(os.environ.get('PADDLE_TRAINERS_NUM', 1))
         imagenet_reader = reader.ImageNetReader(seed=0, place_num=place_num)
-        train_reader = imagenet_reader.train(settings=args)
-        test_reader = imagenet_reader.val(settings=args)
-        train_data_loader.set_sample_list_generator(train_reader, place)
-        test_data_loader.set_sample_list_generator(test_reader, place)
+
+        train_dataset = ImageNetDataset(
+            os.path.join(args.data_dir, "train"), mode='train')
+
+        train_data_loader = DataLoader(
+            train_dataset,
+            batch_size=args.batch_size,
+            places=place,
+            shuffle=True,
+            drop_last=True,
+            num_workers=10)
+
+        test_dataset = ImageNetDataset(
+            os.path.join(args.data_dir, "val"), mode='val')
+
+        test_data_loader = DataLoader(
+            test_dataset,
+            batch_size=args.batch_size,
+            places=place,
+            shuffle=True,
+            drop_last=True,
+            num_workers=1)
 
         # 4. train loop
         total_batch_num = 0  #this is for benchmark
@@ -184,7 +204,7 @@ def train_mobilenet():
                 total_sample += 1
                 batch_id += 1
                 t_last = time.time()
-               
+
                 # NOTE: used for benchmark
                 total_batch_num = total_batch_num + 1