Merge pull request #6 from songyuwen0808/paddlebox

根据cvrq需求，修改dump格式，便于cvrq不修改qcal等相关脚本
jack603047588 · Jun 21, 2021 · 1888e2b · 1888e2b
2 parents 204635c + 8d3e4c1
commit 1888e2b
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 3 deletions.
diff --git a/paddle/fluid/framework/device_worker.cc b/paddle/fluid/framework/device_worker.cc
@@ -14,6 +14,8 @@ limitations under the License. */
 
 #include "paddle/fluid/framework/device_worker.h"
 
+DECLARE_bool(lineid_have_extend_info);
+DECLARE_bool(dump_filed_same_as_aibox);
 namespace paddle {
 namespace framework {
 
@@ -151,7 +153,16 @@ void DeviceWorker::DumpField(const Scope& scope, int dump_mode,
       continue;
     }
     hit[i] = true;
-    ars[i] += lineid;
+    if (FLAGS_lineid_have_extend_info) {
+        size_t pos = lineid.find(" ");
+        if (pos != std::string::npos) {
+          ars[i] += lineid.substr(0, pos);
+        } else {
+          ars[i] += lineid;
+        }
+    } else {
+        ars[i] += lineid;
+    }
   }
   for (auto& field : *dump_fields_) {
     Variable* var = scope.FindVar(field);
@@ -183,8 +194,17 @@ void DeviceWorker::DumpField(const Scope& scope, int dump_mode,
         continue;
       }
       auto bound = GetTensorBound(tensor, i);
-      ars[i] = ars[i] + "\t" + field + ":" +
-               std::to_string(bound.second - bound.first);
+      if (FLAGS_dump_filed_same_as_aibox) {
+        size_t pos = field.find(".");
+        std::string new_field = field;
+        if (pos != std::string::npos) {
+          new_field = field.substr(0, pos);
+        }
+        ars[i] = ars[i] + "\t" + new_field;
+      } else {
+        ars[i] = ars[i] + "\t" + field + ":" +
+                 std::to_string(bound.second - bound.first);
+      }
       ars[i] += PrintLodTensor(tensor, bound.first, bound.second);
     }
   }
@@ -193,6 +213,15 @@ void DeviceWorker::DumpField(const Scope& scope, int dump_mode,
     if (ars[i].length() == 0) {
       continue;
     }
+
+    if (FLAGS_lineid_have_extend_info) {
+      const std::string& lineid = device_reader_->GetLineId(i);
+      size_t pos = lineid.find(" ");
+      if (pos != std::string::npos) {
+          ars[i] = ars[i] + "\t" + lineid.substr(pos + 1);
+      }
+    }
+
     writer_ << ars[i];
   }
 }

diff --git a/paddle/fluid/platform/flags.cc b/paddle/fluid/platform/flags.cc
@@ -494,6 +494,11 @@ DEFINE_bool(padbox_dataset_disable_polling, false,
             "if true ,will disable input file list polling");
 DEFINE_bool(padbox_dataset_enable_unrollinstance, false,
             "if true ,will enable unrollinstance");
+DEFINE_bool(lineid_have_extend_info, false,
+            "if true , will split line id by space into 2 part, the second part will dump at the last of line");
+DEFINE_bool(dump_filed_same_as_aibox, false,
+            "if true , will change dump format from abc.tmp0:2:1:1 into abc:1:1, which same as aibox");
+
 /**
  * MKLDNN related FLAG
  * Name: use_mkldnn