NUP-2514: fix traversal limit

src/nupic/bindings/algorithms.i

@@ -42,6 +42,8 @@ else:
   from nupic.proto.SvmProto_capnp import (SvmDenseProto, Svm01Proto)
   from nupic.proto.TemporalMemoryProto_capnp import TemporalMemoryProto
 
+# Capnp reader traveral limit (see capnp::ReaderOptions)
+_TRAVERSAL_LIMIT_IN_WORDS = 1 << 63
 
 _ALGORITHMS = _algorithms
 %}
@@ -338,7 +340,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
 
       :param: Destination SvmDenseProto message builder
       """
-      reader = SvmDenseProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+      reader = SvmDenseProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
       pyBuilder.from_dict(reader.to_dict())  # copy
 
     @classmethod
@@ -450,7 +453,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
 
       :param: Destination Svm01Proto message builder
       """
-      reader = Svm01Proto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+      reader = Svm01Proto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
       pyBuilder.from_dict(reader.to_dict())  # copy
 
     @classmethod
@@ -899,7 +903,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
 
       :param: Destination Cells4Proto message builder
       """
-      reader = Cells4Proto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+      reader = Cells4Proto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
       pyBuilder.from_dict(reader.to_dict())  # copy
 
   %}
@@ -1148,7 +1153,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
 
       :param: Destination SpatialPoolerProto message builder
       """
-      reader = SpatialPoolerProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+      reader = SpatialPoolerProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
       pyBuilder.from_dict(reader.to_dict())  # copy
 
 
@@ -1465,7 +1471,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
 
       :param: Destination SdrClassifierProto message builder
       """
-      reader = SdrClassifierProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+      reader = SdrClassifierProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
       pyBuilder.from_dict(reader.to_dict())  # copy
 
     @classmethod
@@ -1581,7 +1588,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
 
       :param: Destination ConnectionsProto message builder
       """
-      reader = ConnectionsProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+      reader = ConnectionsProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
       pyBuilder.from_dict(reader.to_dict())  # copy
 
 
@@ -1782,7 +1790,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
 
       :param: Destination TemporalMemoryProto message builder
       """
-      reader = TemporalMemoryProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+      reader = TemporalMemoryProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
       pyBuilder.from_dict(reader.to_dict())  # copy
 
 

src/nupic/bindings/engine_internal.i

@@ -35,6 +35,9 @@ except ImportError:
 else:
   from nupic.proto.NetworkProto_capnp import NetworkProto
   from nupic.proto.PyRegionProto_capnp import PyRegionProto
+
+# Capnp reader traveral limit (see capnp::ReaderOptions)
+_TRAVERSAL_LIMIT_IN_WORDS = 1 << 63
 %}
 
 %{
@@ -287,7 +290,8 @@ class IterablePair(object):
 
       :param: Destination NetworkProto message builder
       """
-      reader = NetworkProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+      reader = NetworkProto.from_bytes(self._writeAsCapnpPyBytes(), 
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
       pyBuilder.from_dict(reader.to_dict())  # copy
 
 
@@ -364,7 +368,8 @@ class _PyCapnpHelper(object):
 
     :returns: The deserialized python region instance.
     """
-    pyRegionProto = PyRegionProto.from_bytes(pyRegionProtoBytes)
+    pyRegionProto = PyRegionProto.from_bytes(pyRegionProtoBytes,
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
 
     return getattr(regionCls, methodName)(pyRegionProto)
 

src/nupic/bindings/math.i

@@ -33,6 +33,9 @@ except ImportError:
 else:
   from nupic.proto.RandomProto_capnp import RandomProto
 
+# Capnp reader traveral limit (see capnp::ReaderOptions)
+_TRAVERSAL_LIMIT_IN_WORDS = 1 << 63
+
 _MATH = _math
 %}
 
@@ -195,7 +198,8 @@ def write(self, pyBuilder):
 
   :param: Destination RandomProto message builder
   """
-  reader = RandomProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+  reader = RandomProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
   pyBuilder.from_dict(reader.to_dict())  # copy
 
 

src/nupic/bindings/sparse_matrix.i

@@ -45,6 +45,9 @@ except ImportError:
 else:
   from nupic.proto.SparseMatrixProto_capnp import SparseMatrixProto
   from nupic.proto.SparseBinaryMatrixProto_capnp import SparseBinaryMatrixProto
+
+# Capnp reader traveral limit (see capnp::ReaderOptions)
+_TRAVERSAL_LIMIT_IN_WORDS = 1 << 63
 %}
 
 
@@ -423,7 +426,8 @@ def write(self, pyBuilder):
 
   :param: Destination SparseMatrixProto message builder
   """
-  reader = SparseMatrixProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+  reader = SparseMatrixProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
   pyBuilder.from_dict(reader.to_dict())  # copy
 
 @classmethod
@@ -2973,7 +2977,8 @@ def write(self, pyBuilder):
 
   :param: Destination SparseBinaryMatrixProto message builder
   """
-  reader = SparseBinaryMatrixProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+  reader = SparseBinaryMatrixProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
   pyBuilder.from_dict(reader.to_dict())  # copy
 
 def read(self, proto):
@@ -3484,7 +3489,8 @@ def write(self, pyBuilder):
 
   :param: Destination SparseBinaryMatrixProto message builder
   """
-  reader = SparseBinaryMatrixProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
+  reader = SparseBinaryMatrixProto.from_bytes(self._writeAsCapnpPyBytes(),
+                            traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
   pyBuilder.from_dict(reader.to_dict())  # copy
 
 def read(self, proto):

src/nupic/py_support/PyCapnp.hpp

@@ -111,7 +111,9 @@ class PyCapnpHelper {
     kj::Array<capnp::word> array = kj::heapArray<capnp::word>(srcNumWords);
     memcpy(array.asBytes().begin(), srcBytes, srcNumBytes); // copy
 
-    capnp::FlatArrayMessageReader reader(array.asPtr()); // copy ?
+    capnp::ReaderOptions options;
+    options.traversalLimitInWords = kj::maxValue; // Don't limit.
+    capnp::FlatArrayMessageReader reader(array.asPtr(), options); // copy ?
     typename MessageType::Reader proto = reader.getRoot<MessageType>();
     obj.read(proto);
 #else

src/nupic/regions/PyRegion.cpp

@@ -435,7 +435,9 @@ void PyRegion::write(capnp::AnyPointer::Builder &proto) const {
   kj::Array<capnp::word> array = Helper::serialize(node_);
 
   // Initialize PyRegionProto::Reader from serialized python region
-  capnp::FlatArrayMessageReader reader(array.asPtr());
+  capnp::ReaderOptions options;
+  options.traversalLimitInWords = kj::maxValue; // Don't limit.
+  capnp::FlatArrayMessageReader reader(array.asPtr(), options);
   PyRegionProto::Reader pyRegionReader = reader.getRoot<PyRegionProto>();
 
   // Assign python region's serialization output to the builder

src/nupic/types/Serializable.hpp

@@ -52,8 +52,9 @@ template <class ProtoT> class Serializable {
 
   void read(std::istream &stream) {
     kj::std::StdInputStream in(stream);
-
-    capnp::InputStreamMessageReader message(in);
+    capnp::ReaderOptions options;
+    options.traversalLimitInWords = kj::maxValue; // Don't limit.
+    capnp::InputStreamMessageReader message(in, options);
     typename ProtoT::Reader proto = message.getRoot<ProtoT>();
     read(proto);
   }