Filter layer rebased

include/caffe/common_layers.hpp

@@ -181,6 +181,69 @@ class EltwiseLayer : public Layer<Dtype> {
 };
 
 /**
+ * @brief Takes two+ Blobs, interprets last Blob as a selector and
+ *  filter remaining Blobs accordingly with selector data (0 means that
+ * the corresponding item has to be filtered, non-zero means that corresponding
+ * item needs to stay).
+ */
+template <typename Dtype>
+class FilterLayer : public Layer<Dtype> {
+ public:
+  explicit FilterLayer(const LayerParameter& param)
+      : Layer<Dtype>(param) {}
+  virtual void LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top);
+  virtual void Reshape(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top);
+
+  virtual inline const char* type() const { return "Filter"; }
+  virtual inline int MinBottomBlobs() const { return 2; }
+  virtual inline int MinTopBlobs() const { return 1; }
+
+ protected:
+  /**
+   * @param bottom input Blob vector (length 2+)
+   *   -# @f$ (N \times C \times H \times W) @f$
+   *      the inputs to be filtered @f$ x_1 @f$
+   *   -# ...
+   *   -# @f$ (N \times C \times H \times W) @f$
+   *      the inputs to be filtered @f$ x_K @f$
+   *   -# @f$ (N \times 1 \times 1 \times 1) @f$
+   *      the selector blob
+   * @param top output Blob vector (length 1+)
+   *   -# @f$ (S \times C \times H \times W) @f$ () 
+   *        the filtered output @f$ x_1 @f$ 
+   *        where S is the number of items
+   *        that haven't been filtered
+   *      @f$ (S \times C \times H \times W) @f$
+   *        the filtered output @f$ x_K @f$ 
+   *        where S is the number of items
+   *        that haven't been filtered
+   */
+  virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top);
+  virtual void Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+    const vector<Blob<Dtype>*>& top);
+
+  /**
+   * @brief Computes the error gradient w.r.t. the forwarded inputs.
+   *
+   * @param top output Blob vector (length 1+), providing the error gradient with
+   *        respect to the outputs
+   * @param propagate_down see Layer::Backward.
+   * @param bottom input Blob vector (length 2+), into which the top error
+   *        gradient is copied
+   */
+  virtual void Backward_cpu(const vector<Blob<Dtype>*>& top,
+      const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+  virtual void Backward_gpu(const vector<Blob<Dtype>*>& top,
+    const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom);
+
+  bool first_reshape_;
+  vector<int> indices_to_forward_;
+};
+
+/**
  * @brief Reshapes the input Blob into flat vectors.
  *
  * Note: because this layer does not change the input values -- merely the

src/caffe/layers/filter_layer.cpp

@@ -0,0 +1,128 @@
+#include <algorithm>
+#include <vector>
+
+#include "caffe/layer.hpp"
+#include "caffe/util/math_functions.hpp"
+#include "caffe/vision_layers.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void FilterLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top) {
+  CHECK_EQ(top.size(), bottom.size()-1) <<
+      "Top.size() should be equal to bottom.size() - 1";
+  first_reshape_ = true;
+}
+
+template <typename Dtype>
+void FilterLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top) {
+  // bottom[0...k-1] are the blobs to filter
+  // bottom[last] is the "selector_blob"
+  int selector_index = bottom.size() - 1;
+  for (int i = 1; i < bottom[selector_index]->num_axes(); ++i) {
+    CHECK_EQ(bottom[selector_index]->shape(i), 1)
+        << "Selector blob must have all shapes == 1 (except the first one)";
+  }
+  for (int i = 0; i < bottom.size()-1; i++) {
+    CHECK_EQ(bottom[selector_index]->shape(0), bottom[i]->shape(0)) <<
+        "Each bottom should have the same dimension as the selector blob";
+  }
+
+  const Dtype* bottom_data_selector = bottom[selector_index]->cpu_data();
+  indices_to_forward_.clear();
+
+  // look for non-zero elements in bottom[0]. Items of each bottom that
+  // have the same index as the items in bottom[0] with value == non-zero
+  // will be forwarded
+  for (int item_id = 0; item_id < bottom[selector_index]->shape(0); ++item_id) {
+    // we don't need an offset because item size == 1
+    const Dtype* tmp_data_selector = bottom_data_selector + item_id;
+    if (*tmp_data_selector) {
+      indices_to_forward_.push_back(item_id);
+    }
+  }
+  // only filtered items will be forwarded
+  int new_tops_num = indices_to_forward_.size();
+  // init
+  if (first_reshape_) {
+    new_tops_num = bottom[0]->shape(0);
+    first_reshape_ = false;
+  }
+  for (int t = 0; t < top.size(); t++) {
+    int num_axes = bottom[t]->num_axes();
+    vector<int> shape_top(num_axes);
+    shape_top[0] = new_tops_num;
+    for (int ts = 1; ts < num_axes; ts++)
+      shape_top[ts] = bottom[t]->shape(ts);
+    top[t]->Reshape(shape_top);
+  }
+}
+
+template <typename Dtype>
+void FilterLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top) {
+  int new_tops_num = indices_to_forward_.size();
+  // forward all filtered items for all bottoms but the Selector (bottom[last])
+  for (int t = 0; t < top.size(); t++) {
+    const Dtype* bottom_data = bottom[t]->cpu_data();
+    Dtype* top_data = top[t]->mutable_cpu_data();
+    int dim = bottom[t]->count() / bottom[t]->shape(0);
+    for (int n = 0; n < new_tops_num; n++) {
+      int data_offset_top = top[t]->offset(n);
+      int data_offset_bottom =  bottom[t]->offset(indices_to_forward_[n]);
+      caffe_copy(dim, bottom_data + data_offset_bottom,
+          top_data + data_offset_top);
+    }
+  }
+}
+
+template <typename Dtype>
+void FilterLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
+      const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
+  if (propagate_down[bottom.size() - 1]) {
+    LOG(FATAL) << this->type()
+               << "Layer cannot backpropagate to filter index inputs";
+  }
+  for (int i = 0; i < top.size(); i++) {
+    // bottom[last] is the selector and never needs backpropagation
+    // so we can iterate over top vector because top.size() == bottom.size() -1
+    if (propagate_down[i]) {
+      const int dim = top[i]->count() / top[i]->shape(0);
+      int next_to_backward_offset = 0;
+      int batch_offset = 0;
+      int data_offset_bottom = 0;
+      int data_offset_top = 0;
+      for (int n = 0; n < bottom[i]->shape(0); n++) {
+        data_offset_bottom = bottom[i]->offset(n);
+        if (next_to_backward_offset >= indices_to_forward_.size()) {
+          // we already visited all items that were been forwarded, so
+          // just set to zero remaining ones
+          caffe_set(dim, Dtype(0),
+              bottom[i]->mutable_cpu_diff() + data_offset_bottom);
+        } else {
+          batch_offset = indices_to_forward_[next_to_backward_offset];
+          if (n != batch_offset) {  // this data was not been forwarded
+            caffe_set(dim, Dtype(0),
+                bottom[i]->mutable_cpu_diff() + data_offset_bottom);
+          } else {  // this data was been forwarded
+            data_offset_top = top[i]->offset(next_to_backward_offset);
+            next_to_backward_offset++;  // point to next forwarded item index
+            caffe_copy(dim, top[i]->mutable_cpu_diff() + data_offset_top,
+                bottom[i]->mutable_cpu_diff() + data_offset_bottom);
+          }
+        }
+      }
+    }
+  }
+}
+
+#ifdef CPU_ONLY
+STUB_GPU(FilterLayer);
+#endif
+
+INSTANTIATE_CLASS(FilterLayer);
+REGISTER_LAYER_CLASS(Filter);
+
+}  // namespace caffe

src/caffe/layers/filter_layer.cu

@@ -0,0 +1,70 @@
+#include <vector>
+
+#include "caffe/layer.hpp"
+#include "caffe/util/math_functions.hpp"
+#include "caffe/vision_layers.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void FilterLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top) {
+  int new_tops_num = indices_to_forward_.size();
+  // forward all filtered items for all bottoms but the Selector (bottom[last])
+  for (int t = 0; t < top.size(); t++) {
+    const Dtype* bottom_data = bottom[t]->gpu_data();
+    Dtype* top_data = top[t]->mutable_gpu_data();
+    int dim = bottom[t]->count() / bottom[t]->shape(0);
+    for (int n = 0; n < new_tops_num; n++) {
+      int data_offset_top = top[t]->offset(n);
+      int data_offset_bottom =  bottom[t]->offset(indices_to_forward_[n]);
+      caffe_copy(dim, bottom_data + data_offset_bottom,
+          top_data + data_offset_top);
+    }
+  }
+}
+
+template <typename Dtype>
+void FilterLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
+      const vector<bool>& propagate_down, const vector<Blob<Dtype>*>& bottom) {
+  if (propagate_down[bottom.size() - 1]) {
+    LOG(FATAL) << this->type()
+                << "Layer cannot backpropagate to filter index inputs";
+  }
+  for (int i = 0; i < top.size(); i++) {
+    // bottom[last] is the selector and never needs backpropagation
+    // so we can iterate over top vector because top.size() == bottom.size() -1
+    if (propagate_down[i]) {
+      const int dim = top[i]->count() / top[i]->shape(0);
+      int next_to_backward_offset = 0;
+      int batch_offset = 0;
+      int data_offset_bottom = 0;
+      int data_offset_top = 0;
+      for (int n = 0; n < bottom[i]->shape(0); n++) {
+        if (next_to_backward_offset >= indices_to_forward_.size()) {
+          // we already visited all items that were been forwarded, so
+          // just set to zero remaining ones
+          data_offset_bottom = top[i]->offset(n);
+          caffe_set(dim, Dtype(0),
+              bottom[i]->mutable_gpu_diff() + data_offset_bottom);
+        } else {
+          batch_offset = indices_to_forward_[next_to_backward_offset];
+          data_offset_bottom = top[i]->offset(n);
+          if (n != batch_offset) {  // this data was not been forwarded
+            caffe_set(dim, Dtype(0),
+                bottom[i]->mutable_gpu_diff() + data_offset_bottom);
+          } else {  // this data was been forwarded
+            data_offset_top = top[i]->offset(next_to_backward_offset);
+            next_to_backward_offset++;  // point to next forwarded item index
+            caffe_copy(dim, top[i]->mutable_gpu_diff() + data_offset_top,
+                bottom[i]->mutable_gpu_diff() + data_offset_bottom);
+          }
+        }
+      }
+    }
+  }
+}
+
+INSTANTIATE_LAYER_GPU_FUNCS(FilterLayer);
+
+}  // namespace caffe

src/caffe/test/test_filter_layer.cpp

@@ -0,0 +1,128 @@
+#include <cstring>
+#include <limits>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "caffe/blob.hpp"
+#include "caffe/common.hpp"
+#include "caffe/filler.hpp"
+#include "caffe/vision_layers.hpp"
+
+#include "caffe/test/test_caffe_main.hpp"
+#include "caffe/test/test_gradient_check_util.hpp"
+
+namespace caffe {
+
+template <typename TypeParam>
+class FilterLayerTest : public MultiDeviceTest<TypeParam> {
+  typedef typename TypeParam::Dtype Dtype;
+
+ protected:
+  FilterLayerTest()
+      : blob_bottom_data_(new Blob<Dtype>(4, 3, 6, 4)),
+        blob_bottom_labels_(new Blob<Dtype>(4, 1, 1, 1)),
+        blob_bottom_selector_(new Blob<Dtype>(4, 1, 1, 1)),
+        blob_top_data_(new Blob<Dtype>()),
+        blob_top_labels_(new Blob<Dtype>()) {}
+  virtual void SetUp() {
+    // fill the values
+    Caffe::set_random_seed(1890);
+    FillerParameter filler_param;
+    GaussianFiller<Dtype> filler(filler_param);
+    // fill the selector blob
+    Dtype* bottom_data_selector_ = blob_bottom_selector_->mutable_cpu_data();
+    bottom_data_selector_[0] = 0;
+    bottom_data_selector_[1] = 1;
+    bottom_data_selector_[2] = 1;
+    bottom_data_selector_[3] = 0;
+    // fill the other bottom blobs
+    filler.Fill(blob_bottom_data_);
+    for (int i = 0; i < blob_bottom_labels_->count(); ++i) {
+      blob_bottom_labels_->mutable_cpu_data()[i] = caffe_rng_rand() % 5;
+    }
+    blob_bottom_vec_.push_back(blob_bottom_data_);
+    blob_bottom_vec_.push_back(blob_bottom_labels_);
+    blob_bottom_vec_.push_back(blob_bottom_selector_);
+    blob_top_vec_.push_back(blob_top_data_);
+    blob_top_vec_.push_back(blob_top_labels_);
+  }
+  virtual ~FilterLayerTest() {
+    delete blob_bottom_data_;
+    delete blob_bottom_labels_;
+    delete blob_bottom_selector_;
+    delete blob_top_data_;
+    delete blob_top_labels_;
+  }
+  Blob<Dtype>* const blob_bottom_data_;
+  Blob<Dtype>* const blob_bottom_labels_;
+  Blob<Dtype>* const blob_bottom_selector_;
+  // blobs for the top of FilterLayer
+  Blob<Dtype>* const blob_top_data_;
+  Blob<Dtype>* const blob_top_labels_;
+  vector<Blob<Dtype>*> blob_bottom_vec_;
+  vector<Blob<Dtype>*> blob_top_vec_;
+};
+
+TYPED_TEST_CASE(FilterLayerTest, TestDtypesAndDevices);
+
+TYPED_TEST(FilterLayerTest, TestReshape) {
+  typedef typename TypeParam::Dtype Dtype;
+  LayerParameter layer_param;
+  FilterLayer<Dtype> layer(layer_param);
+  layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+  layer.Reshape(this->blob_bottom_vec_, this->blob_top_vec_);
+  // In the test first and last items should have been filtered
+  // so we just expect 2 remaining items
+  EXPECT_EQ(this->blob_top_data_->shape(0), 2);
+  EXPECT_EQ(this->blob_top_labels_->shape(0), 2);
+  EXPECT_GT(this->blob_bottom_data_->shape(0),
+      this->blob_top_data_->shape(0));
+  EXPECT_GT(this->blob_bottom_labels_->shape(0),
+      this->blob_top_labels_->shape(0));
+  for (int i = 1; i < this->blob_bottom_labels_->num_axes(); i++) {
+    EXPECT_EQ(this->blob_bottom_labels_->shape(i),
+        this->blob_top_labels_->shape(i));
+  }
+}
+
+TYPED_TEST(FilterLayerTest, TestForward) {
+  typedef typename TypeParam::Dtype Dtype;
+  LayerParameter layer_param;
+  FilterLayer<Dtype> layer(layer_param);
+  layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
+  layer.Reshape(this->blob_bottom_vec_, this->blob_top_vec_);
+  layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
+  EXPECT_EQ(this->blob_top_labels_->data_at(0, 0, 0, 0),
+      this->blob_bottom_labels_->data_at(1, 0, 0, 0));
+  EXPECT_EQ(this->blob_top_labels_->data_at(1, 0, 0, 0),
+      this->blob_bottom_labels_->data_at(2, 0, 0, 0));
+
+  int dim = this->blob_top_data_->count() /
+      this->blob_top_data_->shape(0);
+  const Dtype* top_data = this->blob_top_data_->cpu_data();
+  const Dtype* bottom_data = this->blob_bottom_data_->cpu_data();
+  // selector is 0 1 1 0, so we need to compare bottom(1,c,h,w)
+  // with top(0,c,h,w) and bottom(2,c,h,w) with top(1,c,h,w)
+  bottom_data += dim;  // bottom(1,c,h,w)
+  for (size_t n = 0; n < dim; n++)
+    EXPECT_EQ(top_data[n], bottom_data[n]);
+
+  bottom_data += dim;  // bottom(2,c,h,w)
+  top_data += dim;  // top(1,c,h,w)
+  for (size_t n = 0; n < dim; n++)
+    EXPECT_EQ(top_data[n], bottom_data[n]);
+}
+
+TYPED_TEST(FilterLayerTest, TestGradient) {
+  typedef typename TypeParam::Dtype Dtype;
+  LayerParameter layer_param;
+  FilterLayer<Dtype> layer(layer_param);
+  GradientChecker<Dtype> checker(1e-2, 1e-3);
+  // check only input 0 (data) because labels and selector
+  // don't need backpropagation
+  checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_,
+      this->blob_top_vec_, 0);
+}
+
+}  // namespace caffe