[WIP] New faster version of the RecordIO iterator

Makefile

@@ -278,6 +278,17 @@ else
 	SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu
 endif
 
+ifeq ($(USE_LIBJPEG_TURBO), 1)
+	ifneq ($(USE_LIBJPEG_TURBO_PATH), NONE)
+		CFLAGS += -I$(USE_LIBJPEG_TURBO_PATH)/include
+		LDFLAGS += -L$(USE_LIBJPEG_TURBO_PATH)/lib
+	endif
+	LDFLAGS += -lturbojpeg
+	CFLAGS += -DMXNET_USE_LIBJPEG_TURBO=1
+else
+	CFLAGS += -DMXNET_USE_LIBJPEG_TURBO=0
+endif
+
 # For quick compile test, used smaller subset
 ALLX_DEP= $(ALL_DEP)
 

amalgamation/dmlc-minimum0.cc

@@ -23,6 +23,7 @@
  */
 #include "../dmlc-core/src/io/line_split.cc"
 #include "../dmlc-core/src/io/recordio_split.cc"
+#include "../dmlc-core/src/io/indexed_recordio_split.cc"
 #include "../dmlc-core/src/io/input_split_base.cc"
 #include "../dmlc-core/src/io/local_filesys.cc"
 #include "../dmlc-core/src/data.cc"

example/image-classification/common/data.py

@@ -23,7 +23,9 @@
 def add_data_args(parser):
     data = parser.add_argument_group('Data', 'the input images')
     data.add_argument('--data-train', type=str, help='the training data')
+    data.add_argument('--data-train-idx', type=str, default='', help='the index of training data')
     data.add_argument('--data-val', type=str, help='the validation data')
+    data.add_argument('--data-val-idx', type=str, default='', help='the index of validation data')
     data.add_argument('--rgb-mean', type=str, default='123.68,116.779,103.939',
                       help='a tuple of size 3 for the mean rgb')
     data.add_argument('--pad-size', type=int, default=0,
@@ -119,6 +121,7 @@ def get_rec_iter(args, kv=None):
     rgb_mean = [float(i) for i in args.rgb_mean.split(',')]
     train = mx.io.ImageRecordIter(
         path_imgrec         = args.data_train,
+        path_imgidx         = args.data_train_idx,
         label_width         = 1,
         mean_r              = rgb_mean[0],
         mean_g              = rgb_mean[1],
@@ -147,6 +150,7 @@ def get_rec_iter(args, kv=None):
         return (train, None)
     val = mx.io.ImageRecordIter(
         path_imgrec         = args.data_val,
+        path_imgidx         = args.data_val_idx,
         label_width         = 1,
         mean_r              = rgb_mean[0],
         mean_g              = rgb_mean[1],

include/mxnet/c_api.h

@@ -1892,6 +1892,14 @@ MXNET_DLL int MXRecordIOReaderReadRecord(RecordIOHandle handle,
 */
 MXNET_DLL int MXRecordIOReaderSeek(RecordIOHandle handle, size_t pos);
 
+/**
+ * \brief Get the current writer pointer position
+ * \param handle handle to RecordIO object
+ * \param pos handle to output position
+ * \return 0 when success, -1 when failure happens
+*/
+MXNET_DLL int MXRecordIOReaderTell(RecordIOHandle handle, size_t *pos);
+
 /**
  * \brief Create a MXRtc object
 */

make/config.mk

@@ -62,6 +62,11 @@ USE_CUDNN = 0
 # imbin iterator
 USE_OPENCV = 1
 
+#whether use libjpeg-turbo for image decode without OpenCV wrapper
+USE_LIBJPEG_TURBO = 0
+#add the path to libjpeg-turbo library
+USE_LIBJPEG_TURBO_PATH = NONE
+
 # use openmp for parallelization
 USE_OPENMP = 1
 

src/c_api/c_api.cc

@@ -1103,6 +1103,14 @@ int MXRecordIOReaderSeek(RecordIOHandle handle, size_t pos) {
   API_END();
 }
 
+int MXRecordIOReaderTell(RecordIOHandle handle, size_t *pos) {
+  API_BEGIN();
+  MXRecordIOContext *context =
+    reinterpret_cast<MXRecordIOContext*>(handle);
+  *pos = context->reader->Tell();
+  API_END();
+}
+
 int MXRtcCreate(char* name, mx_uint num_input, mx_uint num_output,
                 char** input_names, char** output_names,
                 NDArrayHandle* inputs, NDArrayHandle* outputs,

src/io/image_iter_common.h

@@ -102,6 +102,8 @@ struct ImageRecParserParam : public dmlc::Parameter<ImageRecParserParam> {
   std::string path_imglist;
   /*! \brief path to image recordio */
   std::string path_imgrec;
+  /*! \brief path to index file */
+  std::string path_imgidx;
   /*! \brief a sequence of names of image augmenters, seperated by , */
   std::string aug_seq;
   /*! \brief label-width */
@@ -130,6 +132,9 @@ struct ImageRecParserParam : public dmlc::Parameter<ImageRecParserParam> {
     DMLC_DECLARE_FIELD(path_imgrec).set_default("")
         .describe("Path to the image RecordIO (.rec) file or a directory path. "\
                   "Created with tools/im2rec.py.");
+    DMLC_DECLARE_FIELD(path_imgidx).set_default("")
+        .describe("Path to the image RecordIO index (.idx) file. "\
+                  "Created with tools/im2rec.py.");
     DMLC_DECLARE_FIELD(aug_seq).set_default("aug_default")
         .describe("The augmenter names to represent"\
                   " sequence of augmenters to be applied, seperated by comma." \

src/io/inst_vector.h

@@ -101,6 +101,10 @@ class InstVector {
   inline DataInst operator[](size_t i) const {
     DataInst inst;
     inst.index = index_[i];
+    // ImageRecordIter depends on data vector
+    // here having size 2. If you want to
+    // change this assumption here, change it
+    // in there as well (InitBatch section)!
     inst.data.push_back(TBlob(data_[i]));
     inst.data.push_back(TBlob(label_[i]));
     return inst;