Merge pull request #40 from Zbysekz/pandaVis_newInterface

PandaVis interface revamp; simplified, now baking instead of TCP conn.

.gitignore

@@ -21,3 +21,8 @@ pyenv2/
 build/
 dist/
 htm.core/
+
+#panda Vis generated files
+nab/detectors/htmcore/bakedDatabase/*.db
+nab/detectors/htmcore/bakedDatabase/*/*.dump
+nab/detectors/htmcore/logs/*.log

nab/detectors/htmcore/htmcore_detector.py

@@ -41,13 +41,15 @@
 # has been seen so far.
 SPATIAL_TOLERANCE = 0.05
 
-PANDA_VIS_ENABLED = False # if we want to run pandaVis tool (repo at https://github.com/htm-community/HTMpandaVis )
+PANDA_VIS_BAKE_DATA = False # if we want to bake data for pandaVis tool (repo at https://github.com/htm-community/HTMpandaVis )
 
-if PANDA_VIS_ENABLED:
-    from PandaVis.pandaComm.server import PandaServer
-    from PandaVis.pandaComm.dataExchange import ServerData, dataHTMObject, dataLayer, dataInput
+if PANDA_VIS_BAKE_DATA:
+    from pandaBaker.pandaBaker import PandaBaker
+    from pandaBaker.pandaBaker import cLayer, cInput, cDataStream
 
-    pandaServer = PandaServer()
+    BAKE_DATABASE_FILE_PATH = os.path.join(os.getcwd(), 'bakedDatabase', 'htmcore_detector.db')
+
+    pandaBaker = PandaBaker(BAKE_DATABASE_FILE_PATH)
 
 parameters_numenta_comparable = {
   # there are 2 (3) encoders: "value" (RDSE) & "time" (DateTime weekend, timeOfDay)
@@ -143,11 +145,6 @@ def __init__(self, *args, **kwargs):
     self.inputs_ = []
     self.iteration_ = 0
 
-    # initialize pandaVis server
-    if PANDA_VIS_ENABLED:
-        pandaServer.Start()
-        self.BuildPandaSystem(parameters_numenta_comparable)
-
 
   def getAdditionalHeaders(self):
     """Returns a list of strings."""
@@ -241,6 +238,9 @@ def initialize(self):
     # self.predictor = Predictor( steps=[1, 5], alpha=parameters["predictor"]['sdrc_alpha'] )
     # predictor_resolution = 1
 
+    # initialize pandaBaker
+    if PANDA_VIS_BAKE_DATA:
+      self.BuildPandaSystem(self.sp, self.tm, parameters["enc"]["value"]["size"], self.encTimestamp.size)
 
   def modelRun(self, ts, val):
       """
@@ -274,24 +274,19 @@ def modelRun(self, ts, val):
       # 3. Temporal Memory
       # Execute Temporal Memory algorithm over active mini-columns.
 
-      # the tm.compute() execute activateDendrites() - calculateAnomaly()/getPredictiveCells() - activateCells()
-      # but to get insight into system with visTool, we need to have different execution order
-      # Note: pandaVis retrieves synapses etc. by requesting data from sp/tm python objects, so data validity is crucial
-      if PANDA_VIS_ENABLED:
-        # activates cells in columns by TM algorithm (winners, bursting...)
-        self.tm.activateCells(activeColumns, learn=True)
+      # to get predictive cells we need to call activateDendrites & activateCells separately
+      if PANDA_VIS_BAKE_DATA:
         # activateDendrites calculates active segments
         self.tm.activateDendrites(learn=True)
         # predictive cells are calculated directly from active segments
         predictiveCells = self.tm.getPredictiveCells()
+        # activates cells in columns by TM algorithm (winners, bursting...)
+        self.tm.activateCells(activeColumns, learn=True)
       else:
         self.tm.compute(activeColumns, learn=True)
 
       self.tm_info.addData( self.tm.getActiveCells().flatten() )
 
-      if PANDA_VIS_ENABLED:
-        self.PandaUpdateData(ts, val, valueBits, dateBits , activeColumns, predictiveCells)
-
       # 4.1 (optional) Predictor #TODO optional
       #TODO optional: also return an error metric on predictions (RMSE, R2,...)
 
@@ -313,7 +308,7 @@ def modelRun(self, ts, val):
           self.minVal = val
 
       # -temporal (raw)
-      raw = self.tm.anomaly
+      raw= self.tm.anomaly
       temporalAnomaly = raw
 
       if self.useLikelihood:
@@ -331,65 +326,75 @@ def modelRun(self, ts, val):
           # print(self.tm_info)
           pass
 
-      if PANDA_VIS_ENABLED:
-          pandaServer.BlockExecution()
-
+      # 6. panda vis
+      if PANDA_VIS_BAKE_DATA:
+          # ------------------HTMpandaVis----------------------
+          # see more about this structure at https://github.com/htm-community/HTMpandaVis/blob/master/pandaBaker/README.md
+          # fill up values
+          pandaBaker.inputs["Value"].stringValue = "value: {:.2f}".format(val)
+          pandaBaker.inputs["Value"].bits = valueBits.sparse
+
+          pandaBaker.inputs["TimeOfDay"].stringValue = str(ts)
+          pandaBaker.inputs["TimeOfDay"].bits = dateBits.sparse
+
+          pandaBaker.layers["Layer1"].activeColumns = activeColumns.sparse
+          pandaBaker.layers["Layer1"].winnerCells = self.tm.getWinnerCells().sparse
+          pandaBaker.layers["Layer1"].predictiveCells = predictiveCells.sparse
+          pandaBaker.layers["Layer1"].activeCells = self.tm.getActiveCells().sparse
+
+          # customizable datastreams to be show on the DASH PLOTS
+          pandaBaker.dataStreams["rawAnomaly"].value = temporalAnomaly
+          pandaBaker.dataStreams["value"].value = val
+          pandaBaker.dataStreams["numberOfWinnerCells"].value = len(self.tm.getWinnerCells().sparse)
+          pandaBaker.dataStreams["numberOfPredictiveCells"].value = len(predictiveCells.sparse)
+          pandaBaker.dataStreams["valueInput_sparsity"].value = valueBits.getSparsity()
+          pandaBaker.dataStreams["dateInput_sparsity"].value = dateBits.getSparsity()
+
+          pandaBaker.dataStreams["Layer1_SP_overlap_metric"].value = self.sp_info.overlap.overlap
+          pandaBaker.dataStreams["Layer1_TM_overlap_metric"].value = self.sp_info.overlap.overlap
+          pandaBaker.dataStreams["Layer1_SP_activation_frequency"].value = self.sp_info.activationFrequency.mean()
+          pandaBaker.dataStreams["Layer1_TM_activation_frequency"].value = self.tm_info.activationFrequency.mean()
+          pandaBaker.dataStreams["Layer1_SP_entropy"].value = self.sp_info.activationFrequency.mean()
+          pandaBaker.dataStreams["Layer1_TM_entropy"].value = self.tm_info.activationFrequency.mean()
+
+          pandaBaker.StoreIteration(self.iteration_-1)
+          print("ITERATION: " + str(self.iteration_-1))
+
+          # ------------------HTMpandaVis----------------------
 
       return (anomalyScore, raw)
 
-  def BuildPandaSystem(self,modelParams):
-      global serverData
-      serverData = ServerData()
-      serverData.HTMObjects["HTM1"] = dataHTMObject()
-      serverData.HTMObjects["HTM1"].inputs["Value"] = dataInput()
-      serverData.HTMObjects["HTM1"].inputs["TimeOfDay"] = dataInput()
-
-      serverData.HTMObjects["HTM1"].layers["Layer1"] = dataLayer(
-          modelParams["sp"]["columnCount"],
-          modelParams["tm"]["cellsPerColumn"],
-      )
-      serverData.HTMObjects["HTM1"].layers["Layer1"].proximalInputs = ["Value","TimeOfDay"]
-      serverData.HTMObjects["HTM1"].layers["Layer1"].distalInputs = ["Layer1"]
-
+  # with this method, the structure for visualization is defined
+  def BuildPandaSystem(self, sp, tm, consumptionBits_size, dateBits_size):
 
-  def PandaUpdateData(self, timestamp, value, valueSDR, datetimeSDR, activeColumns, predictiveCells):
+      # we have two inputs connected to proximal synapses of Layer1
+      pandaBaker.inputs["Value"] = cInput(consumptionBits_size)
+      pandaBaker.inputs["TimeOfDay"] = cInput(dateBits_size)
 
-    pandaServer.currentIteration = self.iteration_ # update server's iteration number
-    # do not update if we are running GOTO iteration command
-    if (not pandaServer.cmdGotoIteration or (
-            pandaServer.cmdGotoIteration and pandaServer.gotoIteration == pandaServer.currentIteration)):
-      # ------------------HTMpandaVis----------------------
-      # fill up values
-      serverData.iterationNo = pandaServer.currentIteration
-      serverData.HTMObjects["HTM1"].inputs["Value"].stringValue = "VALUE:" + str(value)
-      serverData.HTMObjects["HTM1"].inputs["Value"].bits = valueSDR.sparse
-      serverData.HTMObjects["HTM1"].inputs["Value"].count = valueSDR.size
+      pandaBaker.layers["Layer1"] = cLayer(sp, tm)  # Layer1 has Spatial Pooler & Temporal Memory
+      pandaBaker.layers["Layer1"].proximalInputs = [
+          "Value",
+          "TimeOfDay",
+      ]
+      pandaBaker.layers["Layer1"].distalInputs = ["Layer1"]
 
-      serverData.HTMObjects["HTM1"].inputs["TimeOfDay"].stringValue = "TIME OF DAY:" + str(timestamp)
-      serverData.HTMObjects["HTM1"].inputs["TimeOfDay"].bits = datetimeSDR.sparse
-      serverData.HTMObjects["HTM1"].inputs["TimeOfDay"].count = datetimeSDR.size
+      # data for dash plots
+      streams = ["rawAnomaly", "value", "numberOfWinnerCells", "numberOfPredictiveCells",
+                 "valueInput_sparsity", "dateInput_sparsity", "Layer1_SP_overlap_metric", "Layer1_TM_overlap_metric",
+                 "Layer1_SP_activation_frequency", "Layer1_TM_activation_frequency", "Layer1_SP_entropy",
+                 "Layer1_TM_entropy"
+                 ]
 
-      serverData.HTMObjects["HTM1"].layers["Layer1"].activeColumns = activeColumns.sparse
+      pandaBaker.dataStreams = dict((name, cDataStream()) for name in streams)  # create dicts for more comfortable code
+      # could be also written like: pandaBaker.dataStreams["myStreamName"] = cDataStream()
 
-      serverData.HTMObjects["HTM1"].layers["Layer1"].winnerCells = self.tm.getWinnerCells().sparse
-      serverData.HTMObjects["HTM1"].layers["Layer1"].activeCells = self.tm.getActiveCells().sparse
-      serverData.HTMObjects["HTM1"].layers["Layer1"].predictiveCells = predictiveCells.sparse
+      pandaBaker.PrepareDatabase()
 
-      # print("ACTIVECOLS:"+str(serverData.HTMObjects["HTM1"].layers["SensoryLayer"].activeColumns ))
-      # print("WINNERCELLS:"+str(serverData.HTMObjects["HTM1"].layers["SensoryLayer"].winnerCells))
-      # print("ACTIVECELLS:" + str(serverData.HTMObjects["HTM1"].layers["SensoryLayer"].activeCells))
-      # print("PREDICTCELLS:"+str(serverData.HTMObjects["HTM1"].layers["SensoryLayer"].predictiveCells))
-
-      pandaServer.serverData = serverData
-
-      pandaServer.spatialPoolers["HTM1"] = self.sp
-      pandaServer.temporalMemories["HTM1"] = self.tm
-      pandaServer.NewStateDataReady()
 
 
 # WHILE USING PANDAVIS
 # SPECIFY HERE FOR WHAT DATA YOU WANT TO RUN THIS DETECTOR
-if PANDA_VIS_ENABLED:
+if PANDA_VIS_BAKE_DATA:
   import pandas as pd
   import os.path as path
   from nab.corpus import Corpus
@@ -405,3 +410,5 @@ def PandaUpdateData(self, timestamp, value, valueSDR, datetimeSDR, activeColumns
   detector.initialize()
 
   detector.run()
+
+  pandaBaker.CommitBatch()