From dad217c4972c07f3d924aadc8ebc4ee15ffd972a Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 11:21:27 -0700
Subject: [PATCH 01/30] added mibi_consumer

---
 redis_consumer/consumers/__init__.py          |   2 +
 redis_consumer/consumers/mibi_consumer.py     | 166 +++++++++++
 .../consumers/mibi_consumer_test.py           | 260 ++++++++++++++++++
 redis_consumer/processing.py                  |   4 +
 redis_consumer/settings.py                    |  10 +-
 5 files changed, 438 insertions(+), 4 deletions(-)
 create mode 100644 redis_consumer/consumers/mibi_consumer.py
 create mode 100644 redis_consumer/consumers/mibi_consumer_test.py

diff --git a/redis_consumer/consumers/__init__.py b/redis_consumer/consumers/__init__.py
index 2eb1d90a..3e106a7c 100644
--- a/redis_consumer/consumers/__init__.py
+++ b/redis_consumer/consumers/__init__.py
@@ -35,6 +35,7 @@
 # Custom Workflow consumers
 from redis_consumer.consumers.image_consumer import ImageFileConsumer
 from redis_consumer.consumers.tracking_consumer import TrackingConsumer
+from redis_consumer.consumers.mibi_consumer import MibiConsumer
 # TODO: Import future custom Consumer classes.
 
 
@@ -42,6 +43,7 @@
     'image': ImageFileConsumer,
     'zip': ZipFileConsumer,
     'tracking': TrackingConsumer,
+    'mibi':MibiConsumer,
     # TODO: Add future custom Consumer classes here.
 }
 
diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
new file mode 100644
index 00000000..296e18c5
--- /dev/null
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -0,0 +1,166 @@
+# Copyright 2016-2020 The Van Valen Lab at the California Institute of
+# Technology (Caltech), with support from the Paul Allen Family Foundation,
+# Google, & National Institutes of Health (NIH) under Grant U24CA224309-01.
+# All rights reserved.
+#
+# Licensed under a modified Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.github.com/vanvalenlab/kiosk-redis-consumer/LICENSE
+#
+# The Work provided may be used for non-commercial academic purposes only.
+# For any other use of the Work, including commercial use, please contact:
+# vanvalenlab@gmail.com
+#
+# Neither the name of Caltech nor the names of its contributors may be used
+# to endorse or promote products derived from this software without specific
+# prior written permission.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""ImageFileConsumer class for consuming image segmentation jobs."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import timeit
+
+import numpy as np
+
+from redis_consumer.consumers import TensorFlowServingConsumer
+from redis_consumer import utils
+from redis_consumer import settings
+
+from redis_consumer import processing
+
+class MibiConsumer(TensorFlowServingConsumer):
+    """Consumes image files and uploads the results"""
+
+    def is_valid_hash(self, redis_hash):
+        if redis_hash is None:
+            return False
+
+        fname = str(self.redis.hget(redis_hash, 'input_file_name'))
+        return not fname.lower().endswith('.zip')
+
+    def _consume(self, redis_hash):
+        start = timeit.default_timer()
+        hvals = self.redis.hgetall(redis_hash)
+        # hold on to the redis hash/values for logging purposes
+        self._redis_hash = redis_hash
+        self._redis_values = hvals
+
+        if hvals.get('status') in self.finished_statuses:
+            self.logger.warning('Found completed hash `%s` with status %s.',
+                                redis_hash, hvals.get('status'))
+            return hvals.get('status')
+
+        self.logger.debug('Found hash to process `%s` with status `%s`.',
+                          redis_hash, hvals.get('status'))
+
+        self.update_key(redis_hash, {
+            'status': 'started',
+            'identity_started': self.name,
+        })
+
+        # Get model_name and version
+        model_name, model_version = settings.MIBI_MODEL.split(':')
+        self.logger.debug('Model using is: %s', model_name)
+
+        _ = timeit.default_timer()
+
+        # Load input image
+        with utils.get_tempdir() as tempdir:
+            fname = self.storage.download(hvals.get('input_file_name'), tempdir)
+            image = utils.get_image(fname)
+
+        # Pre-process data before sending to the model
+        self.update_key(redis_hash, {
+            'status': 'pre-processing',
+            'download_time': timeit.default_timer() - _,
+        })
+
+        # Calculate scale of image and rescale
+        scale = hvals.get('scale', '')
+        if not scale:
+            # Detect scale of image (Default to 1, implement SCALE_DETECT here)
+            self.logger.debug('Scale was not given. Defaults to 1')
+            scale = 1
+        else:
+            # Scale was set by user
+            self.logger.debug('Image scale was defined as: %s', scale)
+        scale = float(scale)
+        self.logger.debug('Image scale is: %s', scale)
+
+        # Rescale each channel of the image
+        self.logger.debug('Image shape before scaling is: %s', image.shape)
+        images = []
+        for channel in range(image.shape[0]):
+            images.append(utils.rescale(image[channel, ...], scale))
+        image = np.concatenate(images, -1)
+        self.logger.debug('Image shape after scaling is: %s', image.shape)
+
+        # Preprocess image
+        image = np.expand_dims(image, 0)
+        image = processing.phase_preprocess(image)
+        image = np.squeeze(image)
+        self.logger.debug('Shape after phase_preprocess is: %s', image.shape)
+
+        # Send data to the model
+        self.update_key(redis_hash, {'status': 'predicting'})
+        image = self.predict(image, model_name, model_version)
+
+        # Post-process model results
+        self.update_key(redis_hash, {'status': 'post-processing'})
+        image = np.squeeze(processing.deep_watershed_mibi(image))
+        self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
+
+        # Save the post-processed results to a file
+        _ = timeit.default_timer()
+        self.update_key(redis_hash, {'status': 'saving-results'})
+
+        with utils.get_tempdir() as tempdir:
+            # Save each result channel as an image file
+            save_name = hvals.get('original_name', fname)
+            subdir = os.path.dirname(save_name.replace(tempdir, ''))
+            name = os.path.splitext(os.path.basename(save_name))[0]
+
+            # Rescale image to original size before sending back to user
+            if isinstance(image, list):
+                outpaths = []
+                for i in image:
+                    outpaths.extend(utils.save_numpy_array(
+                        utils.rescale(i, 1 / scale), name=name,
+                        subdir=subdir, output_dir=tempdir))
+            else:
+                outpaths = utils.save_numpy_array(
+                    utils.rescale(image, 1 / scale), name=name,
+                    subdir=subdir, output_dir=tempdir)
+
+            # Save each prediction image as zip file
+            zip_file = utils.zip_files(outpaths, tempdir)
+
+            # Upload the zip file to cloud storage bucket
+            cleaned = zip_file.replace(tempdir, '')
+            subdir = os.path.dirname(settings._strip(cleaned))
+            subdir = subdir if subdir else None
+            dest, output_url = self.storage.upload(zip_file, subdir=subdir)
+
+        # Update redis with the final results
+        t = timeit.default_timer() - start
+        self.update_key(redis_hash, {
+            'status': self.final_status,
+            'output_url': output_url,
+            'upload_time': timeit.default_timer() - _,
+            'output_file_name': dest,
+            'total_jobs': 1,
+            'total_time': t,
+            'finished_at': self.get_current_timestamp()
+        })
+        return self.final_status
diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
new file mode 100644
index 00000000..63e00b66
--- /dev/null
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -0,0 +1,260 @@
+# Copyright 2016-2020 The Van Valen Lab at the California Institute of
+# Technology (Caltech), with support from the Paul Allen Family Foundation,
+# Google, & National Institutes of Health (NIH) under Grant U24CA224309-01.
+# All rights reserved.
+#
+# Licensed under a modified Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.github.com/vanvalenlab/kiosk-data-processing/LICENSE
+#
+# The Work provided may be used for non-commercial academic purposes only.
+# For any other use of the Work, including commercial use, please contact:
+# vanvalenlab@gmail.com
+#
+# Neither the name of Caltech nor the names of its contributors may be used
+# to endorse or promote products derived from this software without specific
+# prior written permission.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Tests for post-processing functions"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import copy
+
+import numpy as np
+from skimage.external import tifffile as tiff
+
+import pytest
+
+from redis_consumer import consumers
+from redis_consumer import utils
+from redis_consumer import settings
+
+
+def _get_image(img_h=300, img_w=300):
+    bias = np.random.rand(img_w, img_h) * 64
+    variance = np.random.rand(img_w, img_h) * (255 - 64)
+    img = np.random.rand(img_w, img_h) * variance + bias
+    return img
+
+
+class Bunch(object):
+    def __init__(self, **kwds):
+        self.__dict__.update(kwds)
+
+
+class DummyRedis(object):
+    # pylint: disable=R0201,W0613
+    def __init__(self, items=None, prefix='predict', status='new'):
+        items = [] if items is None else items
+        self.work_queue = copy.copy(items)
+        self.processing_queue = []
+        self.prefix = '/'.join(x for x in prefix.split('/') if x)
+        self.status = status
+        self._redis_master = self
+        self.keys = [
+            '{}:{}:{}'.format(self.prefix, 'x.tiff', self.status),
+            '{}:{}:{}'.format(self.prefix, 'x.zip', 'other'),
+            '{}:{}:{}'.format('other', 'x.TIFF', self.status),
+            '{}:{}:{}'.format(self.prefix, 'x.ZIP', self.status),
+            '{}:{}:{}'.format(self.prefix, 'x.tiff', 'other'),
+            '{}:{}:{}'.format('other', 'x.zip', self.status),
+        ]
+
+    def rpoplpush(self, src, dst):
+        if src.startswith('processing'):
+            source = self.processing_queue
+            dest = self.work_queue
+        elif src.startswith(self.prefix):
+            source = self.work_queue
+            dest = self.processing_queue
+
+        try:
+            x = source.pop()
+            dest.insert(0, x)
+            return x
+        except IndexError:
+            return None
+
+    def lpush(self, name, *values):
+        self.work_queue = list(reversed(values)) + self.work_queue
+        return len(self.work_queue)
+
+    def lrem(self, name, count, value):
+        self.processing_queue.remove(value)
+        return count
+
+    def llen(self, queue):
+        if queue.startswith('processing'):
+            return len(self.processing_queue)
+        return len(self.work_queue)
+
+    def hmget(self, rhash, *args):
+        return [self.hget(rhash, a) for a in args]
+
+    def hmset(self, rhash, hvals):  # pylint: disable=W0613
+        return hvals
+
+    def expire(self, name, time):  # pylint: disable=W0613
+        return 1
+
+    def hget(self, rhash, field):
+        if field == 'status':
+            return rhash.split(':')[-1]
+        elif field == 'file_name':
+            return rhash.split(':')[1]
+        elif field == 'input_file_name':
+            return rhash.split(':')[1]
+        elif field == 'output_file_name':
+            return rhash.split(':')[1]
+        elif field == 'reason':
+            return 'reason'
+        return False
+
+    def hset(self, rhash, status, value):  # pylint: disable=W0613
+        return {status: value}
+
+    def hgetall(self, rhash):  # pylint: disable=W0613
+        return {
+            'model_name': 'model',
+            'model_version': '0',
+            'postprocess_function': '',
+            'preprocess_function': '',
+            'file_name': rhash.split(':')[1],
+            'input_file_name': rhash.split(':')[1],
+            'output_file_name': rhash.split(':')[1],
+            'status': rhash.split(':')[-1],
+            'children': 'predict:1.tiff:done,predict:2.tiff:failed,predict:3.tiff:new',
+            'children:done': 'predict:4.tiff:done,predict:5.tiff:done',
+            'children:failed': 'predict:6.tiff:failed,predict:7.tiff:failed',
+        }
+
+
+class DummyStorage(object):
+    # pylint: disable=R0201,W0613
+    def __init__(self, num=3):
+        self.num = num
+
+    def download(self, path, dest):
+        if path.lower().endswith('.zip'):
+            paths = []
+            for i in range(self.num):
+                img = _get_image()
+                base, ext = os.path.splitext(path)
+                _path = '{}{}{}'.format(base, i, ext)
+                tiff.imsave(os.path.join(dest, _path), img)
+                paths.append(_path)
+            return utils.zip_files(paths, dest)
+        img = _get_image()
+        tiff.imsave(os.path.join(dest, path), img)
+        return path
+
+    def upload(self, zip_path, subdir=None):
+        return True, True
+
+    def get_public_url(self, zip_path):
+        return True
+
+
+class TestImageFileConsumer(object):
+    # pylint: disable=R0201
+    def test_is_valid_hash(self):
+        items = ['item%s' % x for x in range(1, 4)]
+
+        storage = DummyStorage()
+        redis_client = DummyRedis(items)
+        redis_client.hget = lambda *x: x[0]
+
+        consumer = consumers.MibiConsumer(redis_client, storage, 'mibi')
+        assert consumer.is_valid_hash(None) is False
+        assert consumer.is_valid_hash('file.ZIp') is False
+        assert consumer.is_valid_hash('predict:1234567890:file.ZIp') is False
+        assert consumer.is_valid_hash('track:123456789:file.zip') is False
+        assert consumer.is_valid_hash('predict:123456789:file.zip') is False
+        assert consumer.is_valid_hash('mibi:1234567890:file.tiff') is True
+        assert consumer.is_valid_hash('mibi:1234567890:file.png') is True
+
+    def test__consume(self):
+        # pylint: disable=W0613
+        prefix = 'predict'
+        status = 'new'
+        redis_client = DummyRedis(prefix, status)
+        storage = DummyStorage()
+
+        consumer = consumers.ImageFileConsumer(redis_client, storage, prefix)
+
+        def _handle_error(err, rhash):
+            raise err
+
+        def grpc_image(data, *args, **kwargs):
+            return data
+
+        def grpc_image_multi(data, *args, **kwargs):
+            return np.array(tuple(list(data.shape) + [2]))
+
+        def grpc_image_list(data, *args, **kwargs):  # pylint: disable=W0613
+            return [data, data]
+
+        def make_model_metadata_of_size(model_shape=(-1, 256, 256, 2)):
+
+            def get_model_metadata(model_name, model_version):
+                return [{
+                    'in_tensor_name': 'image',
+                    'in_tensor_dtype': 'DT_FLOAT',
+                    'in_tensor_shape': ','.join(str(s) for s in model_shape),
+                }]
+
+            return get_model_metadata
+
+        dummyhash = '{}:test.tiff:{}'.format(prefix, status)
+
+        model_shapes = [
+            (-1, 512, 512, 2),  # image too small, pad
+            (-1, 256, 256, 2),  # image is exactly the right size
+            (-1, 128, 128, 2),  # image too big, tile
+        ]
+
+        consumer._handle_error = _handle_error
+        consumer.grpc_image = grpc_image
+
+        for model_shape in model_shapes:
+            for grpc_func in (grpc_image, grpc_image_list):
+
+                consumer.grpc_image = grpc_func
+                consumer.get_model_metadata = \
+                    make_model_metadata_of_size(model_shape)
+
+                result = consumer._consume(dummyhash)
+                assert result == consumer.final_status
+                # test with a finished hash
+                result = consumer._consume('{}:test.tiff:{}'.format(
+                    prefix, consumer.final_status))
+                assert result == consumer.final_status
+
+        # test with model_name and model_version
+        redis_client.hgetall = lambda x: {
+            'model_name': 'model',
+            'model_version': '0',
+            'postprocess_function': '',
+            'preprocess_function': '',
+            'file_name': 'test_image.tiff',
+            'input_file_name': 'test_image.tiff',
+            'output_file_name': 'test_image.tiff'
+        }
+        redis_client.hmset = lambda x, y: True
+        consumer = consumers.ImageFileConsumer(redis_client, storage, prefix)
+        consumer._handle_error = _handle_error
+        consumer.get_model_metadata = make_model_metadata_of_size((1, 256, 256, 2))
+        consumer.grpc_image = grpc_image
+        result = consumer._consume(dummyhash)
+        assert result == consumer.final_status
diff --git a/redis_consumer/processing.py b/redis_consumer/processing.py
index 035f840b..6c738685 100644
--- a/redis_consumer/processing.py
+++ b/redis_consumer/processing.py
@@ -41,6 +41,10 @@
 
 from deepcell_toolbox.deep_watershed import deep_watershed
 
+# import mibi pre- and post-processing functions
+from deepcell_toolbox.deep_watershed import deep_watershed_mibi
+from deepcell_toolbox.processing import phase_preprocess
+
 from deepcell_toolbox import retinanet_semantic_to_label_image
 from deepcell_toolbox import retinanet_to_label_image
 
diff --git a/redis_consumer/settings.py b/redis_consumer/settings.py
index ea0c1f84..ba2bcca7 100644
--- a/redis_consumer/settings.py
+++ b/redis_consumer/settings.py
@@ -1,4 +1,4 @@
-# Copyright 2016-2020 The Van Valen Lab at the California Institute of
+ # Copyright 2016-2020 The Van Valen Lab at the California Institute of
 # Technology (Caltech), with support from the Paul Allen Family Foundation,
 # Google, & National Institutes of Health (NIH) under Grant U24CA224309-01.
 # All rights reserved.
@@ -118,12 +118,11 @@ def _strip(x):
 # Pre- and Post-processing settings
 PROCESSING_FUNCTIONS = {
     'pre': {
-        'normalize': processing.normalize
+        'normalize': processing.normalize,
     },
     'post': {
         'deepcell': processing.pixelwise,  # TODO: this is deprecated.
         'pixelwise': processing.pixelwise,
-        'mibi': processing.mibi,
         'watershed': processing.watershed,
         'retinanet': processing.retinanet_to_label_image,
         'retinanet-semantic': processing.retinanet_semantic_to_label_image,
@@ -157,11 +156,14 @@ def _strip(x):
 LABEL_DETECT_MODEL = config('LABEL_DETECT_MODEL', default='LabelDetection:1', cast=str)
 LABEL_DETECT_ENABLED = config('LABEL_DETECT_ENABLED', default=False, cast=bool)
 
+# MIBI model Settings
+MIBI_MODEL = config('MIBI_MODEL', default='NewMIBI:0', cast=str)
+
 # Set default models based on label type
 MODEL_CHOICES = {
     0: config('NUCLEAR_MODEL', default='NuclearSegmentation:0', cast=str),
     1: config('PHASE_MODEL', default='PhaseCytoSegmentation:0', cast=str),
-    2: config('CYTOPLASM_MODEL', default='FluoCytoSegmentation:0', cast=str)
+    2: config('CYTOPLASM_MODEL', default='FluoCytoSegmentation:0', cast=str),
 }
 
 POSTPROCESS_CHOICES = {

From 892f732256399cb460bae757b31a4f264dfc8256 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 12:45:55 -0700
Subject: [PATCH 02/30] fixed pep8 issues

---
 redis_consumer/consumers/__init__.py           | 2 +-
 redis_consumer/consumers/mibi_consumer.py      | 1 +
 redis_consumer/consumers/mibi_consumer_test.py | 6 +++---
 redis_consumer/settings.py                     | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/redis_consumer/consumers/__init__.py b/redis_consumer/consumers/__init__.py
index 3e106a7c..7725b0ed 100644
--- a/redis_consumer/consumers/__init__.py
+++ b/redis_consumer/consumers/__init__.py
@@ -43,7 +43,7 @@
     'image': ImageFileConsumer,
     'zip': ZipFileConsumer,
     'tracking': TrackingConsumer,
-    'mibi':MibiConsumer,
+    'mibi': MibiConsumer,
     # TODO: Add future custom Consumer classes here.
 }
 
diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 296e18c5..2312a780 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -39,6 +39,7 @@
 
 from redis_consumer import processing
 
+
 class MibiConsumer(TensorFlowServingConsumer):
     """Consumes image files and uploads the results"""
 
diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 63e00b66..49897553 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -166,7 +166,7 @@ def get_public_url(self, zip_path):
         return True
 
 
-class TestImageFileConsumer(object):
+class TestMibiConsumer(object):
     # pylint: disable=R0201
     def test_is_valid_hash(self):
         items = ['item%s' % x for x in range(1, 4)]
@@ -191,7 +191,7 @@ def test__consume(self):
         redis_client = DummyRedis(prefix, status)
         storage = DummyStorage()
 
-        consumer = consumers.ImageFileConsumer(redis_client, storage, prefix)
+        consumer = consumers.MibiConsumer(redis_client, storage, prefix)
 
         def _handle_error(err, rhash):
             raise err
@@ -252,7 +252,7 @@ def get_model_metadata(model_name, model_version):
             'output_file_name': 'test_image.tiff'
         }
         redis_client.hmset = lambda x, y: True
-        consumer = consumers.ImageFileConsumer(redis_client, storage, prefix)
+        consumer = consumers.MibiConsumer(redis_client, storage, prefix)
         consumer._handle_error = _handle_error
         consumer.get_model_metadata = make_model_metadata_of_size((1, 256, 256, 2))
         consumer.grpc_image = grpc_image
diff --git a/redis_consumer/settings.py b/redis_consumer/settings.py
index ba2bcca7..8e039405 100644
--- a/redis_consumer/settings.py
+++ b/redis_consumer/settings.py
@@ -1,4 +1,4 @@
- # Copyright 2016-2020 The Van Valen Lab at the California Institute of
+# Copyright 2016-2020 The Van Valen Lab at the California Institute of
 # Technology (Caltech), with support from the Paul Allen Family Foundation,
 # Google, & National Institutes of Health (NIH) under Grant U24CA224309-01.
 # All rights reserved.

From 0d17a05613b3ada3b019aa42caf15db56e5d9db3 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 12:55:46 -0700
Subject: [PATCH 03/30] mibi_consumer test changes

---
 redis_consumer/consumers/mibi_consumer_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 49897553..5a06411d 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -254,7 +254,7 @@ def get_model_metadata(model_name, model_version):
         redis_client.hmset = lambda x, y: True
         consumer = consumers.MibiConsumer(redis_client, storage, prefix)
         consumer._handle_error = _handle_error
-        consumer.get_model_metadata = make_model_metadata_of_size((1, 256, 256, 2))
+        consumer.get_model_metadata = make_model_metadata_of_size((256, 256, 2))
         consumer.grpc_image = grpc_image
         result = consumer._consume(dummyhash)
         assert result == consumer.final_status

From 8e34af15dfa91515277c79f0c884f6401d721b46 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 13:00:18 -0700
Subject: [PATCH 04/30] mibi_consumer test changes v2

---
 redis_consumer/consumers/mibi_consumer.py      | 3 ++-
 redis_consumer/consumers/mibi_consumer_test.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 2312a780..ab91753a 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -108,7 +108,8 @@ def _consume(self, redis_hash):
         self.logger.debug('Image shape after scaling is: %s', image.shape)
 
         # Preprocess image
-        image = np.expand_dims(image, 0)
+        if image.ndims < 4:
+            image = np.expand_dims(image, 0)
         image = processing.phase_preprocess(image)
         image = np.squeeze(image)
         self.logger.debug('Shape after phase_preprocess is: %s', image.shape)
diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 5a06411d..49897553 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -254,7 +254,7 @@ def get_model_metadata(model_name, model_version):
         redis_client.hmset = lambda x, y: True
         consumer = consumers.MibiConsumer(redis_client, storage, prefix)
         consumer._handle_error = _handle_error
-        consumer.get_model_metadata = make_model_metadata_of_size((256, 256, 2))
+        consumer.get_model_metadata = make_model_metadata_of_size((1, 256, 256, 2))
         consumer.grpc_image = grpc_image
         result = consumer._consume(dummyhash)
         assert result == consumer.final_status

From 7f7d0cfe3db8893ec86b35096114d24fbc586e2e Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 13:04:43 -0700
Subject: [PATCH 05/30] mibi_consumer test changes v3

---
 redis_consumer/consumers/mibi_consumer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index ab91753a..1db5891a 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -108,7 +108,7 @@ def _consume(self, redis_hash):
         self.logger.debug('Image shape after scaling is: %s', image.shape)
 
         # Preprocess image
-        if image.ndims < 4:
+        if image.ndim < 4:
             image = np.expand_dims(image, 0)
         image = processing.phase_preprocess(image)
         image = np.squeeze(image)

From 85611b929a37a2b39d8a03197e5fd575f922660b Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 13:13:22 -0700
Subject: [PATCH 06/30] mibi_consumer test changes v4

---
 redis_consumer/consumers/mibi_consumer_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 49897553..f5198e07 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -254,7 +254,7 @@ def get_model_metadata(model_name, model_version):
         redis_client.hmset = lambda x, y: True
         consumer = consumers.MibiConsumer(redis_client, storage, prefix)
         consumer._handle_error = _handle_error
-        consumer.get_model_metadata = make_model_metadata_of_size((1, 256, 256, 2))
+        consumer.get_model_metadata = make_model_metadata_of_size((-1, 256, 256, 2))
         consumer.grpc_image = grpc_image
         result = consumer._consume(dummyhash)
         assert result == consumer.final_status

From 2d2ca528ccfc2ae0d683f172e859044e10d104a8 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 13:21:19 -0700
Subject: [PATCH 07/30] mibi_consumer test changes v5

---
 redis_consumer/consumers/mibi_consumer_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index f5198e07..803bd124 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -219,9 +219,9 @@ def get_model_metadata(model_name, model_version):
         dummyhash = '{}:test.tiff:{}'.format(prefix, status)
 
         model_shapes = [
-            (-1, 512, 512, 2),  # image too small, pad
-            (-1, 256, 256, 2),  # image is exactly the right size
-            (-1, 128, 128, 2),  # image too big, tile
+            (-1, 2, 512, 512),  # image too small, pad
+            (-1, 2, 256, 256),  # image is exactly the right size
+            (-1, 2, 128, 128),  # image too big, tile
         ]
 
         consumer._handle_error = _handle_error

From c974a06e840d5e744f985d4c130f0ca128c689c4 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 13:32:15 -0700
Subject: [PATCH 08/30] mibi_consumer test changes v6

---
 redis_consumer/consumers/mibi_consumer.py      | 2 ++
 redis_consumer/consumers/mibi_consumer_test.py | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 1db5891a..1fbba18a 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -107,6 +107,8 @@ def _consume(self, redis_hash):
         image = np.concatenate(images, -1)
         self.logger.debug('Image shape after scaling is: %s', image.shape)
 
+        print('Image shape is: ', image.shape)
+
         # Preprocess image
         if image.ndim < 4:
             image = np.expand_dims(image, 0)
diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 803bd124..f5198e07 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -219,9 +219,9 @@ def get_model_metadata(model_name, model_version):
         dummyhash = '{}:test.tiff:{}'.format(prefix, status)
 
         model_shapes = [
-            (-1, 2, 512, 512),  # image too small, pad
-            (-1, 2, 256, 256),  # image is exactly the right size
-            (-1, 2, 128, 128),  # image too big, tile
+            (-1, 512, 512, 2),  # image too small, pad
+            (-1, 256, 256, 2),  # image is exactly the right size
+            (-1, 128, 128, 2),  # image too big, tile
         ]
 
         consumer._handle_error = _handle_error

From 87fe5e20bb47638c38d144fd299886475d8ee224 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 13:38:28 -0700
Subject: [PATCH 09/30] mibi_consumer test changes v7

---
 redis_consumer/consumers/mibi_consumer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 1fbba18a..6bd0762f 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -112,6 +112,8 @@ def _consume(self, redis_hash):
         # Preprocess image
         if image.ndim < 4:
             image = np.expand_dims(image, 0)
+        if image.ndim < 4:
+            image = np.expand_dims(image, -1)
         image = processing.phase_preprocess(image)
         image = np.squeeze(image)
         self.logger.debug('Shape after phase_preprocess is: %s', image.shape)

From 2c1a9921d91ace642485716fb61e2a8e1247ee0e Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 13:46:09 -0700
Subject: [PATCH 10/30] mibi_consumer test changes v8

---
 redis_consumer/consumers/mibi_consumer.py      | 2 --
 redis_consumer/consumers/mibi_consumer_test.py | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 6bd0762f..1fbba18a 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -112,8 +112,6 @@ def _consume(self, redis_hash):
         # Preprocess image
         if image.ndim < 4:
             image = np.expand_dims(image, 0)
-        if image.ndim < 4:
-            image = np.expand_dims(image, -1)
         image = processing.phase_preprocess(image)
         image = np.squeeze(image)
         self.logger.debug('Shape after phase_preprocess is: %s', image.shape)
diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index f5198e07..76f4c056 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -41,10 +41,10 @@
 from redis_consumer import settings
 
 
-def _get_image(img_h=300, img_w=300):
+def _get_image(frames=2, img_h=256, img_w=256):
     bias = np.random.rand(img_w, img_h) * 64
     variance = np.random.rand(img_w, img_h) * (255 - 64)
-    img = np.random.rand(img_w, img_h) * variance + bias
+    img = np.random.rand(frames, img_w, img_h) * variance + bias
     return img
 
 

From 103143e980b4c0be6d1cdbc9b5d9380c48b07adb Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 13:56:42 -0700
Subject: [PATCH 11/30] mibi_consumer test changes v9

---
 redis_consumer/consumers/mibi_consumer.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 1fbba18a..7c0e3ddb 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -107,8 +107,6 @@ def _consume(self, redis_hash):
         image = np.concatenate(images, -1)
         self.logger.debug('Image shape after scaling is: %s', image.shape)
 
-        print('Image shape is: ', image.shape)
-
         # Preprocess image
         if image.ndim < 4:
             image = np.expand_dims(image, 0)
@@ -120,6 +118,10 @@ def _consume(self, redis_hash):
         self.update_key(redis_hash, {'status': 'predicting'})
         image = self.predict(image, model_name, model_version)
 
+        print('pred shape is: ', image[0].shape)
+        print('Image length is: ', len(image))
+
+
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
         image = np.squeeze(processing.deep_watershed_mibi(image))

From b7afa91e527db3490b0e63db49f9c10b851ad456 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 14:00:20 -0700
Subject: [PATCH 12/30] mibi_consumer test changes v9

---
 redis_consumer/consumers/mibi_consumer.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 7c0e3ddb..a9ff323f 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -118,8 +118,7 @@ def _consume(self, redis_hash):
         self.update_key(redis_hash, {'status': 'predicting'})
         image = self.predict(image, model_name, model_version)
 
-        print('pred shape is: ', image[0].shape)
-        print('Image length is: ', len(image))
+        print('Image shape is: ', image.shape)
 
 
         # Post-process model results

From 145668809a4984a039148b851c432277c0e46da1 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 14:05:08 -0700
Subject: [PATCH 13/30] mibi_consumer test changes v10

---
 redis_consumer/consumers/mibi_consumer.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index a9ff323f..1bd3f958 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -118,12 +118,10 @@ def _consume(self, redis_hash):
         self.update_key(redis_hash, {'status': 'predicting'})
         image = self.predict(image, model_name, model_version)
 
-        print('Image shape is: ', image.shape)
-
-
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
-        image = np.squeeze(processing.deep_watershed_mibi(image))
+        if isinstance(image, list):
+            image = np.squeeze(processing.deep_watershed_mibi(image))
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
 
         # Save the post-processed results to a file

From 04ff53439dd170451083be0448e045ee14309b82 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 14:09:56 -0700
Subject: [PATCH 14/30] mibi_consumer test changes v11

---
 redis_consumer/consumers/mibi_consumer.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 1bd3f958..86feb44c 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -118,6 +118,8 @@ def _consume(self, redis_hash):
         self.update_key(redis_hash, {'status': 'predicting'})
         image = self.predict(image, model_name, model_version)
 
+        print('image type is: ', type(image))
+
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
         if isinstance(image, list):

From 5cfd2886331887011c989b1cdd1d99c95f31e1d7 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 14:16:09 -0700
Subject: [PATCH 15/30] mibi_consumer test changes v12

---
 redis_consumer/consumers/mibi_consumer.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 86feb44c..ee09bc9c 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -123,7 +123,11 @@ def _consume(self, redis_hash):
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
         if isinstance(image, list):
-            image = np.squeeze(processing.deep_watershed_mibi(image))
+            if len(image) == 4:
+                image = np.squeeze(processing.deep_watershed_mibi(image))
+            else
+                image = np.asarray(image)
+        
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
 
         # Save the post-processed results to a file

From 2bb8cea5a83d831b0cc8dc8f2777370c65231f50 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 14:20:58 -0700
Subject: [PATCH 16/30] mibi_consumer test changes v13

---
 redis_consumer/consumers/mibi_consumer.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index ee09bc9c..7065277d 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -125,9 +125,13 @@ def _consume(self, redis_hash):
         if isinstance(image, list):
             if len(image) == 4:
                 image = np.squeeze(processing.deep_watershed_mibi(image))
-            else
+            else:
+                self.logger.warning('Output length was %s, should have been 4')
                 image = np.asarray(image)
-        
+        else:
+            image = image
+            self.logger.warning('Output was not in the form of a list)
+
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
 
         # Save the post-processed results to a file

From 5f5f33023e387fef4ea826f4c2dd36f02b246b35 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Thu, 25 Jun 2020 14:24:15 -0700
Subject: [PATCH 17/30] mibi_consumer test changes v14

---
 redis_consumer/consumers/mibi_consumer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 7065277d..afce214a 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -130,7 +130,7 @@ def _consume(self, redis_hash):
                 image = np.asarray(image)
         else:
             image = image
-            self.logger.warning('Output was not in the form of a list)
+            self.logger.warning('Output was not in the form of a list')
 
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
 

From eea5aeae5f38b7e244a48c88ac59820ab60baa17 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Fri, 26 Jun 2020 18:46:33 -0700
Subject: [PATCH 18/30] trying to debug

---
 redis_consumer/consumers/mibi_consumer.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index afce214a..93ee77d5 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -108,8 +108,9 @@ def _consume(self, redis_hash):
         self.logger.debug('Image shape after scaling is: %s', image.shape)
 
         # Preprocess image
-        if image.ndim < 4:
-            image = np.expand_dims(image, 0)
+        # Image must be of shape (batch, x, y, channels), but scaling
+        # eliminates batch dim, so we recreate it here
+        image = np.expand_dims(image, 0)
         image = processing.phase_preprocess(image)
         image = np.squeeze(image)
         self.logger.debug('Shape after phase_preprocess is: %s', image.shape)
@@ -118,7 +119,12 @@ def _consume(self, redis_hash):
         self.update_key(redis_hash, {'status': 'predicting'})
         image = self.predict(image, model_name, model_version)
 
-        print('image type is: ', type(image))
+        if isinstance(image, list):
+            print('Predictions are as list, length is: ', len(image))
+
+        else:
+            print('Predictions are instead type: ', type(image))
+            print('Size of ', type(image), ' is: ', image.shape)
 
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
@@ -126,7 +132,7 @@ def _consume(self, redis_hash):
             if len(image) == 4:
                 image = np.squeeze(processing.deep_watershed_mibi(image))
             else:
-                self.logger.warning('Output length was %s, should have been 4')
+                self.logger.warning('Output length was %s, expected 4')
                 image = np.asarray(image)
         else:
             image = image

From d69dcf7277e441ac3fb35a08ce92a1d609db0263 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Fri, 26 Jun 2020 18:49:48 -0700
Subject: [PATCH 19/30] trying to debug

---
 redis_consumer/consumers/mibi_consumer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 93ee77d5..522740a0 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -135,7 +135,8 @@ def _consume(self, redis_hash):
                 self.logger.warning('Output length was %s, expected 4')
                 image = np.asarray(image)
         else:
-            image = image
+            # image = image
+            image = np.squeeze(processing.deep_watershed_mibi(image))
             self.logger.warning('Output was not in the form of a list')
 
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)

From d79da5bacb6f382002d729030ae1b9f5b215c3f3 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Fri, 26 Jun 2020 18:59:08 -0700
Subject: [PATCH 20/30] trying to debug v2

---
 redis_consumer/consumers/mibi_consumer.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 522740a0..59811966 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -128,16 +128,16 @@ def _consume(self, redis_hash):
 
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
-        if isinstance(image, list):
-            if len(image) == 4:
-                image = np.squeeze(processing.deep_watershed_mibi(image))
-            else:
-                self.logger.warning('Output length was %s, expected 4')
-                image = np.asarray(image)
+        #if isinstance(image, list):
+        if len(image) == 4:
+            image = np.squeeze(processing.deep_watershed_mibi(image))
         else:
+            self.logger.warning('Output length was %s, expected 4')
+            image = np.asarray(image)
+        #else:
             # image = image
-            image = np.squeeze(processing.deep_watershed_mibi(image))
-            self.logger.warning('Output was not in the form of a list')
+            #image = np.squeeze(processing.deep_watershed_mibi(image))
+            #self.logger.warning('Output was not in the form of a list')
 
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
 

From b1124fe5a5cd64c82d64916e9ca96406f919fa52 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Fri, 26 Jun 2020 19:08:36 -0700
Subject: [PATCH 21/30] trying to debug v4

---
 redis_consumer/consumers/mibi_consumer.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 59811966..f09e5141 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -115,6 +115,8 @@ def _consume(self, redis_hash):
         image = np.squeeze(image)
         self.logger.debug('Shape after phase_preprocess is: %s', image.shape)
 
+        print('Image shape prior to predict is: ', image.shape)
+
         # Send data to the model
         self.update_key(redis_hash, {'status': 'predicting'})
         image = self.predict(image, model_name, model_version)
@@ -133,6 +135,7 @@ def _consume(self, redis_hash):
             image = np.squeeze(processing.deep_watershed_mibi(image))
         else:
             self.logger.warning('Output length was %s, expected 4')
+            image = np.squeeze(processing.deep_watershed_mibi(image))
             image = np.asarray(image)
         #else:
             # image = image

From 83f9438d9a5585b0d16b8d1a75b6ee1cca9508ab Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Fri, 26 Jun 2020 19:25:51 -0700
Subject: [PATCH 22/30] trying to debug v5

---
 redis_consumer/consumers/mibi_consumer_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 76f4c056..9daaf2de 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -219,9 +219,9 @@ def get_model_metadata(model_name, model_version):
         dummyhash = '{}:test.tiff:{}'.format(prefix, status)
 
         model_shapes = [
-            (-1, 512, 512, 2),  # image too small, pad
+            # (-1, 512, 512, 2),  # image too small, pad
             (-1, 256, 256, 2),  # image is exactly the right size
-            (-1, 128, 128, 2),  # image too big, tile
+            # (-1, 128, 128, 2),  # image too big, tile
         ]
 
         consumer._handle_error = _handle_error

From d8c7d436bd0bfd1fd98c397d70292d7cf3f89401 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Fri, 26 Jun 2020 19:34:22 -0700
Subject: [PATCH 23/30] trying to debug v6

---
 redis_consumer/consumers/mibi_consumer.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index f09e5141..3064ab6e 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -130,17 +130,18 @@ def _consume(self, redis_hash):
 
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
-        #if isinstance(image, list):
-        if len(image) == 4:
-            image = np.squeeze(processing.deep_watershed_mibi(image))
+        if isinstance(image, list):
+            if len(image) == 4:
+                image = np.squeeze(processing.deep_watershed_mibi(image))
+            else:
+                print('length of image was %s', len(image))
+                self.logger.warning('Output length was %s, expected 4', len(image))
+                image = np.squeeze(processing.deep_watershed_mibi(image))
+                image = np.asarray(image)
         else:
-            self.logger.warning('Output length was %s, expected 4')
-            image = np.squeeze(processing.deep_watershed_mibi(image))
-            image = np.asarray(image)
-        #else:
-            # image = image
+            image = image
             #image = np.squeeze(processing.deep_watershed_mibi(image))
-            #self.logger.warning('Output was not in the form of a list')
+            self.logger.warning('Output was not in the form of a list')
 
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
 

From e4d594eff7ec784feb7e2c6a74b0ee0f59a717f6 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Fri, 26 Jun 2020 21:03:21 -0700
Subject: [PATCH 24/30] testing v7

---
 redis_consumer/consumers/mibi_consumer.py     | 22 ++++++-------
 .../consumers/mibi_consumer_test.py           | 32 +++++++++----------
 2 files changed, 26 insertions(+), 28 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 3064ab6e..9be9cea3 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -130,18 +130,18 @@ def _consume(self, redis_hash):
 
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
-        if isinstance(image, list):
-            if len(image) == 4:
-                image = np.squeeze(processing.deep_watershed_mibi(image))
-            else:
-                print('length of image was %s', len(image))
-                self.logger.warning('Output length was %s, expected 4', len(image))
-                image = np.squeeze(processing.deep_watershed_mibi(image))
-                image = np.asarray(image)
-        else:
-            image = image
+        # if isinstance(image, list):
+        #    if len(image) == 4:
+        image = np.squeeze(processing.deep_watershed_mibi(image))
+        #    else:
+        #        print('length of image was %s', len(image))
+        #        self.logger.warning('Output length was %s, expected 4', len(image))
+        #        image = np.squeeze(processing.deep_watershed_mibi(image))
+        #        image = np.asarray(image)
+        # else:
+        #    image = image
             #image = np.squeeze(processing.deep_watershed_mibi(image))
-            self.logger.warning('Output was not in the form of a list')
+        #    self.logger.warning('Output was not in the form of a list')
 
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
 
diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 9daaf2de..8ebeb829 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -186,10 +186,10 @@ def test_is_valid_hash(self):
 
     def test__consume(self):
         # pylint: disable=W0613
-        prefix = 'predict'
+        prefix = 'mibi'
         status = 'new'
-        redis_client = DummyRedis(prefix, status)
         storage = DummyStorage()
+        redis_client = DummyRedis(prefix, status)
 
         consumer = consumers.MibiConsumer(redis_client, storage, prefix)
 
@@ -216,30 +216,28 @@ def get_model_metadata(model_name, model_version):
 
             return get_model_metadata
 
-        dummyhash = '{}:test.tiff:{}'.format(prefix, status)
+        dummyhash = '{}:new.tiff:{}'.format(prefix, status)
 
         model_shapes = [
-            # (-1, 512, 512, 2),  # image too small, pad
+            (-1, 512, 512, 2),  # image too small, pad
             (-1, 256, 256, 2),  # image is exactly the right size
-            # (-1, 128, 128, 2),  # image too big, tile
+            (-1, 128, 128, 2),  # image too big, tile
         ]
 
         consumer._handle_error = _handle_error
         consumer.grpc_image = grpc_image
 
         for model_shape in model_shapes:
-            for grpc_func in (grpc_image, grpc_image_list):
-
-                consumer.grpc_image = grpc_func
-                consumer.get_model_metadata = \
-                    make_model_metadata_of_size(model_shape)
-
-                result = consumer._consume(dummyhash)
-                assert result == consumer.final_status
-                # test with a finished hash
-                result = consumer._consume('{}:test.tiff:{}'.format(
-                    prefix, consumer.final_status))
-                assert result == consumer.final_status
+
+            consumer.get_model_metadata = \
+                make_model_metadata_of_size(model_shape)
+
+            result = consumer._consume(dummyhash)
+            assert result == consumer.final_status
+            # test with a finished hash
+            result = consumer._consume('{}:test.tiff:{}'.format(
+                prefix, consumer.final_status))
+            assert result == consumer.final_status
 
         # test with model_name and model_version
         redis_client.hgetall = lambda x: {

From 7053b98eb42f11698808fd855ecbec3eac56b5ee Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Sat, 27 Jun 2020 11:49:18 -0700
Subject: [PATCH 25/30] redid testing

---
 redis_consumer/consumers/mibi_consumer.py      |  6 +++++-
 redis_consumer/consumers/mibi_consumer_test.py | 12 ++++++------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 9be9cea3..96926381 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -90,7 +90,11 @@ def _consume(self, redis_hash):
         # Calculate scale of image and rescale
         scale = hvals.get('scale', '')
         if not scale:
-            # Detect scale of image (Default to 1, implement SCALE_DETECT here)
+            # Detect scale of image (Default to 1)
+            # TODO implement SCALE_DETECT here)
+            # scale = self.detect_scale(image)
+            # self.logger.debug('Image scale detected: %s', scale)
+            # self.update_key(redis_hash, {'scale': scale})
             self.logger.debug('Scale was not given. Defaults to 1')
             scale = 1
         else:
diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 8ebeb829..8483d41d 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -197,7 +197,11 @@ def _handle_error(err, rhash):
             raise err
 
         def grpc_image(data, *args, **kwargs):
-            return data
+            inner = np.zeros(1, 256, 256, 1)
+            outer = np.zeros(1, 256, 256, 1)
+            fgbg = np.zeros(1, 256, 256, 2)
+            feature = np.zeros(1, 256, 256, 3)
+            return [inner, outer, fgbg, feature]
 
         def grpc_image_multi(data, *args, **kwargs):
             return np.array(tuple(list(data.shape) + [2]))
@@ -218,11 +222,7 @@ def get_model_metadata(model_name, model_version):
 
         dummyhash = '{}:new.tiff:{}'.format(prefix, status)
 
-        model_shapes = [
-            (-1, 512, 512, 2),  # image too small, pad
-            (-1, 256, 256, 2),  # image is exactly the right size
-            (-1, 128, 128, 2),  # image too big, tile
-        ]
+        model_shapes = (-1, 256, 256, 2)
 
         consumer._handle_error = _handle_error
         consumer.grpc_image = grpc_image

From a43eb009e1ee5870c2ef0a4e0159d2ae76f44aee Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Sat, 27 Jun 2020 11:56:06 -0700
Subject: [PATCH 26/30] redid testing

---
 redis_consumer/consumers/mibi_consumer_test.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 8483d41d..ddf70df1 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -222,7 +222,11 @@ def get_model_metadata(model_name, model_version):
 
         dummyhash = '{}:new.tiff:{}'.format(prefix, status)
 
-        model_shapes = (-1, 256, 256, 2)
+        model_shapes = [
+#            (-1, 512, 512, 2),  # image too small, pad
+            (-1, 256, 256, 2),  # image is exactly the right size
+#            (-1, 128, 128, 2),  # image too big, tile
+        ]
 
         consumer._handle_error = _handle_error
         consumer.grpc_image = grpc_image

From ae76d0a53a88cdc188611806b3a8b15caa06894d Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Sat, 27 Jun 2020 11:59:49 -0700
Subject: [PATCH 27/30] redid testing

---
 redis_consumer/consumers/mibi_consumer_test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index ddf70df1..e75c99ba 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -197,10 +197,10 @@ def _handle_error(err, rhash):
             raise err
 
         def grpc_image(data, *args, **kwargs):
-            inner = np.zeros(1, 256, 256, 1)
-            outer = np.zeros(1, 256, 256, 1)
-            fgbg = np.zeros(1, 256, 256, 2)
-            feature = np.zeros(1, 256, 256, 3)
+            inner = np.zeros((1, 256, 256, 1))
+            outer = np.zeros((1, 256, 256, 1))
+            fgbg = np.zeros((1, 256, 256, 2))
+            feature = np.zeros((1, 256, 256, 3))
             return [inner, outer, fgbg, feature]
 
         def grpc_image_multi(data, *args, **kwargs):

From 218c0136024ee892c2218b6cd78f4729b3052272 Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Sat, 27 Jun 2020 12:07:04 -0700
Subject: [PATCH 28/30] redid testing

---
 redis_consumer/consumers/mibi_consumer.py     | 21 -------------------
 .../consumers/mibi_consumer_test.py           |  4 ++--
 2 files changed, 2 insertions(+), 23 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer.py b/redis_consumer/consumers/mibi_consumer.py
index 96926381..9d532474 100644
--- a/redis_consumer/consumers/mibi_consumer.py
+++ b/redis_consumer/consumers/mibi_consumer.py
@@ -119,34 +119,13 @@ def _consume(self, redis_hash):
         image = np.squeeze(image)
         self.logger.debug('Shape after phase_preprocess is: %s', image.shape)
 
-        print('Image shape prior to predict is: ', image.shape)
-
         # Send data to the model
         self.update_key(redis_hash, {'status': 'predicting'})
         image = self.predict(image, model_name, model_version)
 
-        if isinstance(image, list):
-            print('Predictions are as list, length is: ', len(image))
-
-        else:
-            print('Predictions are instead type: ', type(image))
-            print('Size of ', type(image), ' is: ', image.shape)
-
         # Post-process model results
         self.update_key(redis_hash, {'status': 'post-processing'})
-        # if isinstance(image, list):
-        #    if len(image) == 4:
         image = np.squeeze(processing.deep_watershed_mibi(image))
-        #    else:
-        #        print('length of image was %s', len(image))
-        #        self.logger.warning('Output length was %s, expected 4', len(image))
-        #        image = np.squeeze(processing.deep_watershed_mibi(image))
-        #        image = np.asarray(image)
-        # else:
-        #    image = image
-            #image = np.squeeze(processing.deep_watershed_mibi(image))
-        #    self.logger.warning('Output was not in the form of a list')
-
         self.logger.debug('Shape after deep_watershed_mibi is: %s', image.shape)
 
         # Save the post-processed results to a file
diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index e75c99ba..20ad26ef 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -223,9 +223,9 @@ def get_model_metadata(model_name, model_version):
         dummyhash = '{}:new.tiff:{}'.format(prefix, status)
 
         model_shapes = [
-#            (-1, 512, 512, 2),  # image too small, pad
+            # (-1, 512, 512, 2),  # image too small, pad
             (-1, 256, 256, 2),  # image is exactly the right size
-#            (-1, 128, 128, 2),  # image too big, tile
+            # (-1, 128, 128, 2),  # image too big, tile
         ]
 
         consumer._handle_error = _handle_error

From de78d19210118459d5872999cafa0b55827ef32e Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Sat, 27 Jun 2020 12:10:46 -0700
Subject: [PATCH 29/30] redid testing

---
 redis_consumer/consumers/mibi_consumer_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 20ad26ef..4537c974 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -223,9 +223,9 @@ def get_model_metadata(model_name, model_version):
         dummyhash = '{}:new.tiff:{}'.format(prefix, status)
 
         model_shapes = [
-            # (-1, 512, 512, 2),  # image too small, pad
+            (-1, 512, 512, 2),  # image too small, pad
             (-1, 256, 256, 2),  # image is exactly the right size
-            # (-1, 128, 128, 2),  # image too big, tile
+            (-1, 128, 128, 2),  # image too big, tile
         ]
 
         consumer._handle_error = _handle_error

From 8495e5136d51f112eee37a17dc88692577b7f11d Mon Sep 17 00:00:00 2001
From: cpavelchek <cpavelch@ucsd.edu>
Date: Sat, 27 Jun 2020 12:17:20 -0700
Subject: [PATCH 30/30] redid testing

---
 .../consumers/mibi_consumer_test.py           | 24 +++++++------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/redis_consumer/consumers/mibi_consumer_test.py b/redis_consumer/consumers/mibi_consumer_test.py
index 4537c974..587099ee 100644
--- a/redis_consumer/consumers/mibi_consumer_test.py
+++ b/redis_consumer/consumers/mibi_consumer_test.py
@@ -222,26 +222,20 @@ def get_model_metadata(model_name, model_version):
 
         dummyhash = '{}:new.tiff:{}'.format(prefix, status)
 
-        model_shapes = [
-            (-1, 512, 512, 2),  # image too small, pad
-            (-1, 256, 256, 2),  # image is exactly the right size
-            (-1, 128, 128, 2),  # image too big, tile
-        ]
+        model_shape = (-1, 256, 256, 2)
 
         consumer._handle_error = _handle_error
         consumer.grpc_image = grpc_image
 
-        for model_shape in model_shapes:
-
-            consumer.get_model_metadata = \
-                make_model_metadata_of_size(model_shape)
+        consumer.get_model_metadata = \
+            make_model_metadata_of_size(model_shape)
 
-            result = consumer._consume(dummyhash)
-            assert result == consumer.final_status
-            # test with a finished hash
-            result = consumer._consume('{}:test.tiff:{}'.format(
-                prefix, consumer.final_status))
-            assert result == consumer.final_status
+        result = consumer._consume(dummyhash)
+        assert result == consumer.final_status
+        # test with a finished hash
+        result = consumer._consume('{}:test.tiff:{}'.format(
+            prefix, consumer.final_status))
+        assert result == consumer.final_status
 
         # test with model_name and model_version
         redis_client.hgetall = lambda x: {