DEVOPS-353: Add 'isSparse' attribute to input/output

bindings/py/src/nupic/bindings/regions/PyRegion.py

@@ -123,6 +123,7 @@ def getSpec(cls):
            - ``required`` (bool) whether the input is must be connected
            - ``isDefaultInput`` (bool) must be True for exactly one input
            - ``requireSplitterMap`` (bool) [just set this to False.]
+           - ``isSparse`` (bool) whether the input is sparse
 
       - ``outputs`` (dict) similar structure to inputs. The keys
         are:
@@ -132,6 +133,7 @@ def getSpec(cls):
            - ``count``
            - ``regionLevel``
            - ``isDefaultOutput``
+           - ``isSparse``
 
       - ``parameters`` (dict) of dicts with the following keys:
 

src/nupic/engine/Input.cpp

@@ -36,9 +36,10 @@
 
 namespace nupic {
 
-Input::Input(Region &region, NTA_BasicType dataType, bool isRegionLevel)
+Input::Input(Region &region, NTA_BasicType dataType, bool isRegionLevel,
+             bool isSparse)
     : region_(region), isRegionLevel_(isRegionLevel), initialized_(false),
-      data_(dataType), name_("Unnamed") {}
+      data_(dataType), name_("Unnamed"), isSparse_(isSparse) {}
 
 Input::~Input() {
   uninitialize();
@@ -124,12 +125,16 @@ const Array &Input::getData() const {
   return data_;
 }
 
+NTA_BasicType Input::getDataType() const { return data_.getType(); }
+
 Region &Input::getRegion() { return region_; }
 
 const std::vector<Link *> &Input::getLinks() { return links_; }
 
 bool Input::isRegionLevel() { return isRegionLevel_; }
 
+bool Input::isSparse() { return isSparse_; }
+
 // See header file for documentation
 size_t Input::evaluateLinks() {
   /**
@@ -455,6 +460,12 @@ void Input::initialize() {
         << "was called. Region's dimensions must be specified.";
   }
 
+  if (isSparse_) {
+    NTA_CHECK(isRegionLevel_) << "Sparse data must be region level";
+    NTA_CHECK(data_.getType() == NTA_BasicType_UInt32)
+        << "Sparse data must be uint32";
+  }
+
   // Calculate our size and the offset of each link
   size_t count = 0;
   for (std::vector<Link *>::const_iterator l = links_.begin();
@@ -473,8 +484,7 @@ void Input::initialize() {
   // Zero the inputs (required for inspectors)
   if (count != 0) {
     void *buffer = data_.getBuffer();
-    size_t byteCount = count * BasicType::getSize(data_.getType());
-    ::memset(buffer, 0, byteCount);
+    ::memset(buffer, 0, data_.getBufferSize());
   }
 
   NTA_CHECK(splitterMap_.size() == 0);

src/nupic/engine/Input.hpp

@@ -62,8 +62,11 @@ class Input {
    *        The type of the input, i.e. TODO
    * @param isRegionLevel
    *        Whether the input is region level, i.e. TODO
+   * @param isSparse
+   *        Whether the input is sparse. Default false
    */
-  Input(Region &region, NTA_BasicType type, bool isRegionLevel);
+  Input(Region &region, NTA_BasicType type, bool isRegionLevel,
+        bool isSparse = false);
 
   /**
    *
@@ -156,6 +159,11 @@ class Input {
    */
   const Array &getData() const;
 
+  /**
+   *  Get the data type of the output
+   */
+  NTA_BasicType getDataType() const;
+
   /**
    *
    * Get the Region that the input belongs to.
@@ -238,6 +246,14 @@ class Input {
   template <typename T>
   void getInputForNode(size_t nodeIndex, std::vector<T> &input) const;
 
+  /*
+   * Tells whether the input is sparse.
+   *
+   * @returns
+   *     Whether the input is sparse
+   */
+  bool isSparse();
+
 private:
   Region &region_;
   // buffer is concatenation of input buffers (after prepare), or,
@@ -269,6 +285,9 @@ class Input {
   // Useful for us to know our own name
   std::string name_;
 
+  // Whether or not to use sparse data
+  bool isSparse_;
+
   // Internal methods
 
   /*

src/nupic/engine/Link.cpp

@@ -173,6 +173,17 @@ void Link::initialize(size_t destinationOffset) {
               destD == dest_->getRegion().getDimensions());
   }
 
+  // Validate sparse link
+  if (src_->isSparse() && !dest_->isSparse()) {
+    // Sparse to dense: unit32 -> bool
+    NTA_CHECK(dest_->getDataType() == NTA_BasicType_Bool)
+        << "Sparse to Dense link destination must be boolean";
+  } else if (!src_->isSparse() && dest_->isSparse()) {
+    // Dense to sparse:  NTA_BasicType -> uint32
+    NTA_CHECK(dest_->getDataType() == NTA_BasicType_UInt32)
+        << "Dense to Sparse link destination must be uint32";
+  }
+
   destOffset_ = destinationOffset;
   impl_->initialize();
 

src/nupic/engine/Output.cpp

@@ -34,9 +34,10 @@
 
 namespace nupic {
 
-Output::Output(Region &region, NTA_BasicType type, bool isRegionLevel)
+Output::Output(Region &region, NTA_BasicType type, bool isRegionLevel,
+               bool isSparse)
     : region_(region), isRegionLevel_(isRegionLevel), name_("Unnamed"),
-      nodeOutputElementCount_(0) {
+      nodeOutputElementCount_(0), isSparse_(isSparse) {
   data_ = new Array(type);
 }
 
@@ -57,6 +58,12 @@ void Output::initialize(size_t count) {
   if (data_->getBuffer() != nullptr)
     return;
 
+  if (isSparse_) {
+    NTA_CHECK(isRegionLevel_) << "Sparse data must be region level";
+    NTA_CHECK(data_->getType() == NTA_BasicType_UInt32)
+        << "Sparse data must be uint32";
+  }
+
   nodeOutputElementCount_ = count;
   size_t dataCount;
   if (isRegionLevel_)
@@ -97,6 +104,7 @@ const Array &Output::getData() const { return *data_; }
 bool Output::isRegionLevel() const { return isRegionLevel_; }
 
 Region &Output::getRegion() const { return region_; }
+bool Output::isSparse() const { return isSparse_; }
 
 void Output::setName(const std::string &name) { name_ = name; }
 
@@ -108,4 +116,6 @@ size_t Output::getNodeOutputElementCount() const {
 
 bool Output::hasOutgoingLinks() { return (!links_.empty()); }
 
+NTA_BasicType Output::getDataType() const { return data_->getType(); }
+
 } // namespace nupic

src/nupic/engine/Output.hpp

@@ -50,8 +50,11 @@ class Output {
    *        The type of the output, TODO
    * @param isRegionLevel
    *        Whether the output is region level, i.e. TODO
+   * @param isSparse
+   *        Whether the output is sparse. Default false
    */
-  Output(Region &region, NTA_BasicType type, bool isRegionLevel);
+  Output(Region &region, NTA_BasicType type, bool isRegionLevel,
+         bool isSparse = false);
 
   /**
    * Destructor
@@ -130,11 +133,16 @@ class Output {
    * @returns
    *         A constant reference to the data of the output as an @c Array
    *
-   * @note It's mportant to return a const array so caller can't
+   * @note It's important to return a const array so caller can't
    * reallocate the buffer.
    */
   const Array &getData() const;
 
+  /**
+   *  Get the data type of the output
+   */
+  NTA_BasicType getDataType() const;
+
   /**
    *
    * Tells whether the output is region level.
@@ -161,6 +169,15 @@ class Output {
    */
   size_t getNodeOutputElementCount() const;
 
+  /**
+   *
+   * Tells whether the output is sparse.
+   *
+   * @returns
+   *     Whether the output is sparse.
+   */
+  bool isSparse() const;
+
 private:
   Region &region_; // needed for number of nodes
   Array *data_;
@@ -170,6 +187,8 @@ class Output {
   std::set<Link *> links_;
   std::string name_;
   size_t nodeOutputElementCount_;
+  // Whether or not the output is sparse
+  bool isSparse_;
 };
 
 } // namespace nupic

src/nupic/engine/Region.cpp

@@ -112,7 +112,7 @@ void Region::createInputsAndOutputs_() {
     const std::pair<std::string, OutputSpec> &p = spec_->outputs.getByIndex(i);
     std::string outputName = p.first;
     const OutputSpec &os = p.second;
-    auto output = new Output(*this, os.dataType, os.regionLevel);
+    auto output = new Output(*this, os.dataType, os.regionLevel, os.sparse);
     outputs_[outputName] = output;
     // keep track of name in the output also -- see note in Region.hpp
     output->setName(outputName);
@@ -124,7 +124,7 @@ void Region::createInputsAndOutputs_() {
     std::string inputName = p.first;
     const InputSpec &is = p.second;
 
-    auto input = new Input(*this, is.dataType, is.regionLevel);
+    auto input = new Input(*this, is.dataType, is.regionLevel, is.sparse);
     inputs_[inputName] = input;
     // keep track of name in the input also -- see note in Region.hpp
     input->setName(inputName);

src/nupic/engine/Spec.cpp

@@ -82,15 +82,18 @@ std::string Spec::getDefaultOutputName() const {
 
 InputSpec::InputSpec(std::string description, NTA_BasicType dataType,
                      UInt32 count, bool required, bool regionLevel,
-                     bool isDefaultInput, bool requireSplitterMap)
+                     bool isDefaultInput, bool requireSplitterMap, bool sparse)
     : description(std::move(description)), dataType(dataType), count(count),
       required(required), regionLevel(regionLevel),
-      isDefaultInput(isDefaultInput), requireSplitterMap(requireSplitterMap) {}
+      isDefaultInput(isDefaultInput), requireSplitterMap(requireSplitterMap),
+      sparse(sparse) {}
 
 OutputSpec::OutputSpec(std::string description, NTA_BasicType dataType,
-                       size_t count, bool regionLevel, bool isDefaultOutput)
+                       size_t count, bool regionLevel, bool isDefaultOutput,
+                       bool sparse)
     : description(std::move(description)), dataType(dataType), count(count),
-      regionLevel(regionLevel), isDefaultOutput(isDefaultOutput) {}
+      regionLevel(regionLevel), isDefaultOutput(isDefaultOutput),
+      sparse(sparse) {}
 
 CommandSpec::CommandSpec(std::string description)
     : description(std::move(description)) {}

src/nupic/engine/Spec.hpp

@@ -38,7 +38,7 @@ class InputSpec {
   InputSpec() {}
   InputSpec(std::string description, NTA_BasicType dataType, UInt32 count,
             bool required, bool regionLevel, bool isDefaultInput,
-            bool requireSplitterMap = true);
+            bool requireSplitterMap = true, bool sparse = false);
 
   std::string description;
   NTA_BasicType dataType;
@@ -50,13 +50,15 @@ class InputSpec {
   bool regionLevel;
   bool isDefaultInput;
   bool requireSplitterMap;
+  bool sparse;
 };
 
 class OutputSpec {
 public:
   OutputSpec() {}
   OutputSpec(std::string description, const NTA_BasicType dataType,
-             size_t count, bool regionLevel, bool isDefaultOutput);
+             size_t count, bool regionLevel, bool isDefaultOutput,
+             bool sparse = false);
 
   std::string description;
   NTA_BasicType dataType;
@@ -65,6 +67,7 @@ class OutputSpec {
   size_t count;
   bool regionLevel;
   bool isDefaultOutput;
+  bool sparse;
 };
 
 class CommandSpec {

src/nupic/regions/PyRegion.cpp

@@ -1048,9 +1048,15 @@ void PyRegion::createSpec(const char *nodeType, Spec &ns,
       requireSplitterMap = py::Int(input.getItem("requireSplitterMap")) != 0;
     }
 
+    // make sparse optional and default to false.
+    bool sparse = false;
+    if (input.getItem("sparse") != nullptr) {
+      sparse = py::Int(input.getItem("sparse")) != 0;
+    }
+
     ns.inputs.add(name,
                   InputSpec(description, dataType, count, required, regionLevel,
-                            isDefaultInput, requireSplitterMap));
+                            isDefaultInput, requireSplitterMap, sparse));
   }
 
   // Add outputs
@@ -1102,8 +1108,14 @@ void PyRegion::createSpec(const char *nodeType, Spec &ns,
         << outputMessagePrefix.str() << "isDefaultOutput";
     bool isDefaultOutput = py::Int(output.getItem("isDefaultOutput")) != 0;
 
+    // make sparse optional and default to true.
+    bool sparse = false;
+    if (output.getItem("sparse") != nullptr) {
+      sparse = py::Int(output.getItem("sparse")) != 0;
+    }
+
     ns.outputs.add(name, OutputSpec(description, dataType, count, regionLevel,
-                                    isDefaultOutput));
+                                    isDefaultOutput, sparse));
   }
 
   // Add parameters