From 4b54485590e6547804f3d940c146da0728674fdf Mon Sep 17 00:00:00 2001 From: Summer Deng Date: Sat, 8 Feb 2020 15:58:48 -0800 Subject: [PATCH 1/2] bulk_eval_collect_histograms Differential Revision: D19270090 fbshipit-source-id: df2ddd2e72777b3cc7e629ce0b22cc56da8c7fc6 --- .../activation_distribution_observer.cc | 27 +++++++++++++++++-- .../server/activation_distribution_observer.h | 5 +++- caffe2/quantization/server/pybind.cc | 12 ++++++--- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/caffe2/quantization/server/activation_distribution_observer.cc b/caffe2/quantization/server/activation_distribution_observer.cc index 9b85acc7f37ae..3d1dde31ec95e 100644 --- a/caffe2/quantization/server/activation_distribution_observer.cc +++ b/caffe2/quantization/server/activation_distribution_observer.cc @@ -385,12 +385,30 @@ HistogramNetObserver::HistogramNetObserver( const string& out_file_name, int nbins, int dump_freq, - bool mul_nets) + bool mul_nets, + string op_filter) : NetObserver(subject), dump_freq_(dump_freq), cnt_(0), mul_nets_(mul_nets), + op_filter_(op_filter), out_file_name_(out_file_name) { + net_name_ = subject->Name(); + if (op_filter != "") { + bool has_op = false; + for (auto* op : subject->GetOperators()) { + if (op->debug_def().type() == op_filter) { + has_op = true; + break; + } + } + if (!has_op) { + LOG(INFO) << "Net " << net_name_ << " doesn't include operator " + << op_filter; + return; + } + } + hist_infos_.resize(subject->GetOperators().size()); int i = 0; @@ -414,8 +432,12 @@ HistogramNetObserver::HistogramNetObserver( void HistogramNetObserver::DumpAndReset_( const string& out_file_name, bool print_total_min_max) { + if (hist_infos_.size() == 0) { + return; + } stringstream file_name; file_name << out_file_name; + LOG(INFO) << "Dumping histograms of net " << net_name_ << " in " << this; if (mul_nets_) { file_name << "."; file_name << this; @@ -466,11 +488,12 @@ void HistogramNetObserver::DumpAndReset_( } } } + f.flush(); f.close(); } HistogramNetObserver::~HistogramNetObserver() { - DumpAndReset_(out_file_name_, true); + DumpAndReset_(out_file_name_, false); } void HistogramNetObserver::Stop() { diff --git a/caffe2/quantization/server/activation_distribution_observer.h b/caffe2/quantization/server/activation_distribution_observer.h index 56deb6e8ab9c5..c56f6af464824 100644 --- a/caffe2/quantization/server/activation_distribution_observer.h +++ b/caffe2/quantization/server/activation_distribution_observer.h @@ -135,7 +135,8 @@ class HistogramNetObserver final : public NetObserver { const std::string& out_file_name, int nbins, int dump_freq = -1, - bool mul_nets = false); + bool mul_nets = false, + string op_filter = ""); ~HistogramNetObserver(); private: @@ -150,6 +151,8 @@ class HistogramNetObserver final : public NetObserver { * files for the nets will be appended with netbase addresses. */ bool mul_nets_; + string net_name_; + string op_filter_; const std::string out_file_name_; std::vector> hist_infos_; }; diff --git a/caffe2/quantization/server/pybind.cc b/caffe2/quantization/server/pybind.cc index c012f0f3961f0..067ac79484d88 100644 --- a/caffe2/quantization/server/pybind.cc +++ b/caffe2/quantization/server/pybind.cc @@ -32,16 +32,20 @@ PYBIND11_MODULE(dnnlowp_pybind11, m) { m.def( "ObserveHistogramOfOutput", - [](const string& out_file_name, int dump_freq, bool mul_nets) { + [](const string& out_file_name, + int dump_freq, + bool mul_nets, + string op_filter) { AddGlobalNetObserverCreator( - [out_file_name, dump_freq, mul_nets](NetBase* net) { + [out_file_name, dump_freq, mul_nets, op_filter](NetBase* net) { return make_unique( - net, out_file_name, 2048, dump_freq, mul_nets); + net, out_file_name, 2048, dump_freq, mul_nets, op_filter); }); }, pybind11::arg("out_file_name"), pybind11::arg("dump_freq") = -1, - pybind11::arg("mul_nets") = false); + pybind11::arg("mul_nets") = false, + pybind11::arg("op_filter") = ""); m.def( "AddHistogramObserver", From f8fe782ebab41b3b8ff7f05ab0308872ecfa2398 Mon Sep 17 00:00:00 2001 From: Summer Deng Date: Sat, 8 Feb 2020 15:58:59 -0800 Subject: [PATCH 2/2] Add histogram collection and weight prepacking utils Summary: Provide histogram collection and weights prepacking interface for Dper to auto quantize the Ads models. Test Plan: buck test mode/opt deeplearning/numeric_suite/toolkit/test:int8_static_utils_test buck test mode/opt deeplearning/numeric_suite/toolkit/test:histogram_utils_test Differential Revision: D19794819 fbshipit-source-id: c6d0d5c2dc7246b5c38ec858a39eb4455f85a128 --- .../activation_distribution_observer.cc | 49 +++++++++++-------- .../server/activation_distribution_observer.h | 12 +++-- caffe2/quantization/server/pybind.cc | 45 +++++++++++------ 3 files changed, 67 insertions(+), 39 deletions(-) diff --git a/caffe2/quantization/server/activation_distribution_observer.cc b/caffe2/quantization/server/activation_distribution_observer.cc index 3d1dde31ec95e..8f4b56c673cda 100644 --- a/caffe2/quantization/server/activation_distribution_observer.cc +++ b/caffe2/quantization/server/activation_distribution_observer.cc @@ -151,11 +151,13 @@ void OutputMinMaxObserver::Stop() { OutputMinMaxNetObserver::OutputMinMaxNetObserver( NetBase* subject, const string& out_file_name, - int dump_freq) + int dump_freq, + string delimiter) : NetObserver(subject), dump_freq_(dump_freq), cnt_(0), - out_file_name_(out_file_name) { + out_file_name_(out_file_name), + delimiter_(delimiter) { VLOG(2) << out_file_name; min_max_infos_.resize(subject->GetOperators().size()); int i = 0; @@ -184,15 +186,15 @@ void OutputMinMaxNetObserver::DumpAndReset_( op_info->tensor_infos[i]; ostringstream ost; - ost << op_index << " " << op_info->type << " " << i << " " - << tensor_info.name << " "; + ost << op_index << delimiter_ << op_info->type << delimiter_ << i + << delimiter_ << tensor_info.name << delimiter_; if (print_total_min_max) { - ost << tensor_info.total_min << " " << tensor_info.total_max; + ost << tensor_info.total_min << delimiter_ << tensor_info.total_max; } else { - ost << tensor_info.min << " " << tensor_info.max; + ost << tensor_info.min << delimiter_ << tensor_info.max; } - LOG(INFO) << this << " " << ost.str(); + LOG(INFO) << this << delimiter_ << ost.str(); f << ost.str() << endl; op_info->tensor_infos[i].min = numeric_limits::max(); @@ -386,12 +388,14 @@ HistogramNetObserver::HistogramNetObserver( int nbins, int dump_freq, bool mul_nets, - string op_filter) + string op_filter, + string delimiter) : NetObserver(subject), dump_freq_(dump_freq), cnt_(0), mul_nets_(mul_nets), op_filter_(op_filter), + delimiter_(delimiter), out_file_name_(out_file_name) { net_name_ = subject->Name(); if (op_filter != "") { @@ -469,16 +473,17 @@ void HistogramNetObserver::DumpAndReset_( } ostringstream ost; - ost << op_index << " " << info->min_max_info.type << " " << i << " " - << info->min_max_info.tensor_infos[i].name << " " << hist->Min() - << " " << hist->Max() << " " << hist->GetHistogram()->size(); + ost << op_index << delimiter_ << info->min_max_info.type << delimiter_ + << i << delimiter_ << info->min_max_info.tensor_infos[i].name + << delimiter_ << hist->Min() << delimiter_ << hist->Max() + << delimiter_ << hist->GetHistogram()->size(); for (uint64_t c : *hist->GetHistogram()) { - ost << " " << c; + ost << delimiter_ << c; } if (print_total_min_max) { - LOG(INFO) << this << " " << ost.str(); + LOG(INFO) << this << delimiter_ << ost.str(); } f << ost.str() << endl; @@ -535,12 +540,14 @@ OutputColumnMaxHistogramNetObserver::OutputColumnMaxHistogramNetObserver( const std::vector& observe_column_max_for_blobs, int nbins, int dump_freq, - bool mul_nets) + bool mul_nets, + string delimiter) : NetObserver(subject), dump_freq_(dump_freq), cnt_(0), mul_nets_(mul_nets), - out_file_name_(out_file_name) { + out_file_name_(out_file_name), + delimiter_(delimiter) { if (observe_column_max_for_blobs.size() == 0) { return; } @@ -614,17 +621,17 @@ void OutputColumnMaxHistogramNetObserver::DumpAndReset_( } ostringstream ost; // op_idx, output_idx, blob_name, col, min, max, nbins - ost << it.first << " " << output_idx << " " - << info->min_max_info.tensor_infos[i].name << " " << i << " " - << hist->Min() << " " << hist->Max() << " " - << hist->GetHistogram()->size(); + ost << it.first << delimiter_ << output_idx << delimiter_ + << info->min_max_info.tensor_infos[i].name << delimiter_ << i + << delimiter_ << hist->Min() << delimiter_ << hist->Max() + << delimiter_ << hist->GetHistogram()->size(); // bins for (uint64_t c : *hist->GetHistogram()) { - ost << " " << c; + ost << delimiter_ << c; } if (print_total_min_max) { - LOG(INFO) << this << " " << ost.str(); + LOG(INFO) << this << delimiter_ << ost.str(); } f << ost.str() << endl; if (!print_total_min_max) { diff --git a/caffe2/quantization/server/activation_distribution_observer.h b/caffe2/quantization/server/activation_distribution_observer.h index c56f6af464824..344dfbc245c67 100644 --- a/caffe2/quantization/server/activation_distribution_observer.h +++ b/caffe2/quantization/server/activation_distribution_observer.h @@ -63,7 +63,8 @@ class OutputMinMaxNetObserver final : public NetObserver { explicit OutputMinMaxNetObserver( NetBase* subject, const std::string& out_file_name, - int dump_freq = -1); + int dump_freq = -1, + string delimiter = " "); ~OutputMinMaxNetObserver(); private: @@ -74,6 +75,7 @@ class OutputMinMaxNetObserver final : public NetObserver { int dump_freq_, cnt_; const std::string out_file_name_; + std::string delimiter_; std::vector> min_max_infos_; }; @@ -136,7 +138,8 @@ class HistogramNetObserver final : public NetObserver { int nbins, int dump_freq = -1, bool mul_nets = false, - string op_filter = ""); + string op_filter = "", + string delimiter = " "); ~HistogramNetObserver(); private: @@ -153,6 +156,7 @@ class HistogramNetObserver final : public NetObserver { bool mul_nets_; string net_name_; string op_filter_; + string delimiter_; const std::string out_file_name_; std::vector> hist_infos_; }; @@ -165,7 +169,8 @@ class OutputColumnMaxHistogramNetObserver final : public NetObserver { const std::vector& observe_column_max_for_blobs, int nbins, int dump_freq = -1, - bool mul_nets = false); + bool mul_nets = false, + string delimiter = " "); ~OutputColumnMaxHistogramNetObserver(); private: @@ -176,6 +181,7 @@ class OutputColumnMaxHistogramNetObserver final : public NetObserver { int dump_freq_, cnt_; bool mul_nets_; const std::string out_file_name_; + std::string delimiter_; std::unordered_set col_max_blob_names_; // {op_idx: {output_index: col_hists}} diff --git a/caffe2/quantization/server/pybind.cc b/caffe2/quantization/server/pybind.cc index 067ac79484d88..2de76586ff1c8 100644 --- a/caffe2/quantization/server/pybind.cc +++ b/caffe2/quantization/server/pybind.cc @@ -1,9 +1,9 @@ #include #include #include "activation_distribution_observer.h" +#include "caffe2/opt/custom/fakefp16_transform.h" #include "caffe2_dnnlowp_utils.h" #include "quantization_error_minimization.h" -#include "caffe2/opt/custom/fakefp16_transform.h" namespace caffe2 { namespace python { @@ -20,39 +20,50 @@ PYBIND11_MODULE(dnnlowp_pybind11, m) { m.def( "ObserveMinMaxOfOutput", - [](const string& min_max_file_name, int dump_freq) { + [](const string& min_max_file_name, int dump_freq, string delimiter) { AddGlobalNetObserverCreator( - [dump_freq, min_max_file_name](NetBase* net) { + [dump_freq, min_max_file_name, delimiter](NetBase* net) { return make_unique( - net, min_max_file_name, dump_freq); + net, min_max_file_name, dump_freq, delimiter); }); }, pybind11::arg("min_max_file_name"), - pybind11::arg("dump_freq") = -1); + pybind11::arg("dump_freq") = -1, + pybind11::arg("delimiter") = " "); m.def( "ObserveHistogramOfOutput", [](const string& out_file_name, int dump_freq, bool mul_nets, - string op_filter) { + string op_filter, + string delimiter) { AddGlobalNetObserverCreator( - [out_file_name, dump_freq, mul_nets, op_filter](NetBase* net) { + [out_file_name, dump_freq, mul_nets, op_filter, delimiter]( + NetBase* net) { return make_unique( - net, out_file_name, 2048, dump_freq, mul_nets, op_filter); + net, + out_file_name, + 2048, + dump_freq, + mul_nets, + op_filter, + delimiter); }); }, pybind11::arg("out_file_name"), pybind11::arg("dump_freq") = -1, pybind11::arg("mul_nets") = false, - pybind11::arg("op_filter") = ""); + pybind11::arg("op_filter") = "", + pybind11::arg("delimiter") = " "); m.def( "AddHistogramObserver", [](const string& net_name, const string& out_file_name, int dump_freq, - bool mul_nets) { + bool mul_nets, + string delimiter) { Workspace* gWorkspace = caffe2::python::GetCurrentWorkspace(); CAFFE_ENFORCE(gWorkspace); CAFFE_ENFORCE( @@ -63,7 +74,7 @@ PYBIND11_MODULE(dnnlowp_pybind11, m) { const Observable::Observer* observer = nullptr; observer = net->AttachObserver(make_unique( - net, out_file_name, 2048, dump_freq, mul_nets)); + net, out_file_name, 2048, dump_freq, mul_nets, delimiter)); CAFFE_ENFORCE(observer != nullptr); return pybind11::cast(observer); @@ -71,7 +82,8 @@ PYBIND11_MODULE(dnnlowp_pybind11, m) { pybind11::arg("net_name"), pybind11::arg("out_file_name"), pybind11::arg("dump_freq") = -1, - pybind11::arg("mul_nets") = false); + pybind11::arg("mul_nets") = false, + pybind11::arg("delimiter") = " "); m.def( "AddOutputColumnMaxHistogramObserver", @@ -79,7 +91,8 @@ PYBIND11_MODULE(dnnlowp_pybind11, m) { const string& out_file_name, const std::vector& observe_column_max_for_blobs, int dump_freq, - bool mul_nets) { + bool mul_nets, + string delimiter) { Workspace* gWorkspace = caffe2::python::GetCurrentWorkspace(); CAFFE_ENFORCE(gWorkspace); CAFFE_ENFORCE( @@ -96,7 +109,8 @@ PYBIND11_MODULE(dnnlowp_pybind11, m) { observe_column_max_for_blobs, 2048, dump_freq, - mul_nets)); + mul_nets, + delimiter)); CAFFE_ENFORCE(observer != nullptr); return pybind11::cast(observer); @@ -105,7 +119,8 @@ PYBIND11_MODULE(dnnlowp_pybind11, m) { pybind11::arg("out_file_name"), pybind11::arg("observe_column_max_for_blobs"), pybind11::arg("dump_freq") = -1, - pybind11::arg("mul_nets") = false); + pybind11::arg("mul_nets") = false, + pybind11::arg("delimiter") = " "); m.def( "ChooseQuantizationParams",