From 3a7aa74373de3b7cffdcba57fc13532ae97ecef4 Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Mon, 27 Mar 2017 01:24:12 +0900 Subject: [PATCH 1/9] FCNObjectSegmentation with PyTorch backend --- .../node_scripts/fcn_object_segmentation.py | 115 ++++++++++++++---- jsk_perception/scripts/install_pytorch.sh | 15 +++ 2 files changed, 105 insertions(+), 25 deletions(-) create mode 100755 jsk_perception/scripts/install_pytorch.sh diff --git a/jsk_perception/node_scripts/fcn_object_segmentation.py b/jsk_perception/node_scripts/fcn_object_segmentation.py index deaa69c77a..22ffe4efdd 100755 --- a/jsk_perception/node_scripts/fcn_object_segmentation.py +++ b/jsk_perception/node_scripts/fcn_object_segmentation.py @@ -1,14 +1,9 @@ #!/usr/bin/env python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - import chainer from chainer import cuda import chainer.serializers as S import fcn -import fcn.models import cv_bridge from jsk_topic_tools import ConnectionBasedTransport @@ -19,45 +14,80 @@ from sensor_msgs.msg import Image -def softmax(w, t=1.0): - e = np.exp(w) - dist = e / np.sum(e, axis=0) - return dist +is_torch_available = True +try: + import torch + import torchfcn +except ImportError: + is_torch_available = False + + +def assert_torch_available(): + if not is_torch_available: + raise RuntimeError( + 'PyTorch is unavailable. Please install it by:' + ' rosrun jsk_perception install_pytorch.sh') class FCNObjectSegmentation(ConnectionBasedTransport): def __init__(self): super(self.__class__, self).__init__() + self.backend = rospy.get_param('~backend', 'chainer') + self.gpu = rospy.get_param('~gpu', -1) # -1 is cpu mode + self.target_names = rospy.get_param('~target_names') self.bg_label = rospy.get_param('~bg_label', 0) self.proba_threshold = rospy.get_param('~proba_threshold', 0.0) + self.mean_bgr = np.array([104.00698793, 116.66876762, 122.67891434]) self._load_model() self.pub = self.advertise('~output', Image, queue_size=1) self.pub_proba = self.advertise( '~output/proba_image', Image, queue_size=1) def _load_model(self): - self.gpu = rospy.get_param('~gpu', -1) # -1 is cpu mode - self.model_name = rospy.get_param('~model_name') - model_h5 = rospy.get_param('~model_h5') - self.target_names = rospy.get_param('~target_names') - self.mean_bgr = np.array([104.00698793, 116.66876762, 122.67891434]) + if self.backend == 'chainer': + self._load_chainer_model() + elif self.backend == 'torch': + assert_torch_available() + self._load_torch_model() + else: + raise RuntimeError('Unsupported backend: %s', self.backend) + def _load_model_chainer_backend(self): + model_name = rospy.get_param('~model_name') + model_h5 = rospy.get_param('~model_h5') n_class = len(self.target_names) - if self.model_name == 'fcn32s': + if model_name == 'fcn32s': self.model = fcn.models.FCN32s(n_class=n_class) - elif self.model_name == 'fcn16s': + elif model_name == 'fcn16s': self.model = fcn.models.FCN16s(n_class=n_class) - elif self.model_name == 'fcn8s': + elif model_name == 'fcn8s': self.model = fcn.models.FCN8s(n_class=n_class) else: - rospy.logerr('Unsupported ~model_name: {0}' - .format(self.model_name)) + raise ValueError('Unsupported ~model_name: {}'.format(model_name)) jsk_loginfo('Loading trained model: {0}'.format(model_h5)) S.load_hdf5(model_h5, self.model) jsk_loginfo('Finished loading trained model: {0}'.format(model_h5)) if self.gpu != -1: self.model.to_gpu(self.gpu) + self.model.train = False + + def _load_torch_model(self): + n_class = len(self.target_names) + model_file = rospy.get_param('~model_file') + model_name = rospy.get_param('~model_name') + if model_name == 'fcn32s': + self.model = torchfcn.models.FCN32s(n_class=n_class) + elif model_name == 'fcn32s_bilinear': + self.model = torchfcn.models.FCN32s(n_class=n_class, deconv=False) + else: + raise ValueError('Unsupported ~model_name: {0}'.format(model_name)) + jsk_loginfo('Loading trained model: %s' % model_file) + self.model.load_state_dict(torch.load(model_file)) + jsk_loginfo('Finished loading trained model: %s' % model_file) + if self.gpu >= 0: + self.model = self.model.cuda(self.gpu) + self.model.eval() def subscribe(self): use_mask = rospy.get_param('~use_mask', False) @@ -92,6 +122,8 @@ def _cb_with_mask(self, img_msg, mask_msg): img = br.imgmsg_to_cv2(img_msg, desired_encoding='bgr8') mask = br.imgmsg_to_cv2(mask_msg, desired_encoding='mono8') label, proba_img = self.segment(img) + if label is None or proba_img is None: + return label[mask == 0] = 0 proba_img[:, :, 0][mask != 0] = 1 proba_img[:, :, 1:][mask != 0] = 0 @@ -106,6 +138,8 @@ def _cb(self, img_msg): br = cv_bridge.CvBridge() img = br.imgmsg_to_cv2(img_msg, desired_encoding='bgr8') label, proba_img = self.segment(img) + if label is None or proba_img is None: + return label_msg = br.cv2_to_imgmsg(label.astype(np.int32), '32SC1') label_msg.header = img_msg.header self.pub.publish(label_msg) @@ -114,22 +148,53 @@ def _cb(self, img_msg): self.pub_proba.publish(proba_msg) def segment(self, bgr): + if self.backend == 'chainer': + return self._segment_chainer_backend(bgr) + elif self.backend == 'torch': + assert_torch_available() + return self._segment_torch_backend(bgr) + raise ValueError('Unsupported backend: {0}'.format(self.backend)) + + def _segment_chainer_backend(self, bgr): blob = (bgr - self.mean_bgr).transpose((2, 0, 1)) x_data = np.array([blob], dtype=np.float32) if self.gpu != -1: x_data = cuda.to_gpu(x_data, device=self.gpu) x = chainer.Variable(x_data, volatile=True) - self.model.train = False self.model(x) - pred = self.model.score - score = cuda.to_cpu(pred.data)[0] - proba_img = softmax(score).transpose((1, 2, 0)) - max_proba_img = np.max(proba_img, axis=-1) - label = np.argmax(score, axis=0) + proba_img = chainer.functions.softmax(self.model.score) + proba_img = chainer.functions.transpose(proba_img, (1, 2, 0)) + max_proba_img = chainer.functions.max(proba_img, axis=-1) + label = chainer.functions.argmax(self.model.score, axis=1) + # gpu -> cpu + proba_img = cuda.to_cpu(proba_img.data)[0] + max_proba_img = cuda.to_cpu(max_proba_img.data)[0] + label = cuda.to_cpu(label.data)[0] # uncertain because the probability is low label[max_proba_img < self.proba_threshold] = self.bg_label return label, proba_img + def _segment_torch_backend(self, bgr): + blob = (bgr - self.mean_bgr).transpose((2, 0, 1)) + x_data = np.array([blob], dtype=np.float32) + x_data = torch.from_numpy(x_data) + if self.gpu >= 0: + x_data = x_data.cuda(self.gpu) + x = torch.autograd.Variable(x_data, volatile=True) + score = self.model(x) + proba = torch.nn.functional.softmax(score) + max_proba, label = torch.max(proba, 1) + # uncertain because the probability is low + label[max_proba < self.proba_threshold] = self.bg_label + # gpu -> cpu + score = score.permute(0, 2, 3, 1).data.cpu().numpy()[0] + proba = proba.permute(0, 2, 3, 1).data.cpu().numpy()[0] + max_proba = max_proba.data.cpu().numpy().squeeze((0, 1)) + label = label.data.cpu().numpy().squeeze((0, 1)) + # uncertain because the probability is low + label[max_proba < self.proba_threshold] = self.bg_label + return label, proba + if __name__ == '__main__': rospy.init_node('fcn_object_segmentation') diff --git a/jsk_perception/scripts/install_pytorch.sh b/jsk_perception/scripts/install_pytorch.sh new file mode 100755 index 0000000000..f7b2eb53f6 --- /dev/null +++ b/jsk_perception/scripts/install_pytorch.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Install script of PyTorch. See http://pytorch.org for detail. + +set -x + +if [ -e /usr/local/cuda-7.5/include/cuda.h ]; then + pip install -q http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl +elif [ -e /usr/local/cuda-8.0/include/cuda.h ]; then + pip install -q http://download.pytorch.org/whl/cu80/torch-0.1.10.post2-cp27-none-linux_x86_64.whl +else + pip install -q http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl +fi + +set +x From dca5e31687a1378bca3a57a11992361c980ef7ac Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Tue, 28 Mar 2017 18:16:26 +0900 Subject: [PATCH 2/9] Remove not needed lines --- jsk_perception/node_scripts/fcn_object_segmentation.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/jsk_perception/node_scripts/fcn_object_segmentation.py b/jsk_perception/node_scripts/fcn_object_segmentation.py index 22ffe4efdd..02be644333 100755 --- a/jsk_perception/node_scripts/fcn_object_segmentation.py +++ b/jsk_perception/node_scripts/fcn_object_segmentation.py @@ -122,8 +122,6 @@ def _cb_with_mask(self, img_msg, mask_msg): img = br.imgmsg_to_cv2(img_msg, desired_encoding='bgr8') mask = br.imgmsg_to_cv2(mask_msg, desired_encoding='mono8') label, proba_img = self.segment(img) - if label is None or proba_img is None: - return label[mask == 0] = 0 proba_img[:, :, 0][mask != 0] = 1 proba_img[:, :, 1:][mask != 0] = 0 @@ -138,8 +136,6 @@ def _cb(self, img_msg): br = cv_bridge.CvBridge() img = br.imgmsg_to_cv2(img_msg, desired_encoding='bgr8') label, proba_img = self.segment(img) - if label is None or proba_img is None: - return label_msg = br.cv2_to_imgmsg(label.astype(np.int32), '32SC1') label_msg.header = img_msg.header self.pub.publish(label_msg) @@ -191,8 +187,6 @@ def _segment_torch_backend(self, bgr): proba = proba.permute(0, 2, 3, 1).data.cpu().numpy()[0] max_proba = max_proba.data.cpu().numpy().squeeze((0, 1)) label = label.data.cpu().numpy().squeeze((0, 1)) - # uncertain because the probability is low - label[max_proba < self.proba_threshold] = self.bg_label return label, proba From 23efa116615f0fb7cf79a17a430c7656c3ad3034 Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Tue, 28 Mar 2017 18:20:36 +0900 Subject: [PATCH 3/9] Add instruction of installing torchfcn --- jsk_perception/node_scripts/fcn_object_segmentation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jsk_perception/node_scripts/fcn_object_segmentation.py b/jsk_perception/node_scripts/fcn_object_segmentation.py index 02be644333..8bc16f398d 100755 --- a/jsk_perception/node_scripts/fcn_object_segmentation.py +++ b/jsk_perception/node_scripts/fcn_object_segmentation.py @@ -17,7 +17,6 @@ is_torch_available = True try: import torch - import torchfcn except ImportError: is_torch_available = False @@ -73,6 +72,11 @@ def _load_model_chainer_backend(self): self.model.train = False def _load_torch_model(self): + try: + import torchfcn + except ImportError as e: + rospy.logerr('Please install torchfcn by pip.') + raise ImportError(e) n_class = len(self.target_names) model_file = rospy.get_param('~model_file') model_name = rospy.get_param('~model_name') From c0829ff17cacc0cdcb46e2d4084c0e5656cf5448 Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Fri, 31 Mar 2017 15:56:07 +0900 Subject: [PATCH 4/9] Install PyTorch for CUDA8.0 with rosdep --- jsk_perception/package.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/jsk_perception/package.xml b/jsk_perception/package.xml index 17ae7be4e8..5a80564f11 100644 --- a/jsk_perception/package.xml +++ b/jsk_perception/package.xml @@ -85,6 +85,7 @@ python-fcn-pip python-sklearn + python-torch-cuda80-pip rosbag roscpp roseus From 40e068fc6788087c3a11f914269e93a4538be72e Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Tue, 4 Apr 2017 12:39:08 +0900 Subject: [PATCH 5/9] Install packages to devel space - new file: install_pytorch.py - deleted: install_pytorch.sh --- jsk_perception/scripts/install_pytorch.py | 42 +++++++++++++++++++++++ jsk_perception/scripts/install_pytorch.sh | 15 -------- 2 files changed, 42 insertions(+), 15 deletions(-) create mode 100755 jsk_perception/scripts/install_pytorch.py delete mode 100755 jsk_perception/scripts/install_pytorch.sh diff --git a/jsk_perception/scripts/install_pytorch.py b/jsk_perception/scripts/install_pytorch.py new file mode 100755 index 0000000000..ecd1357ea4 --- /dev/null +++ b/jsk_perception/scripts/install_pytorch.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +"""Install script of PyTorch. See http://pytorch.org for detail.""" + +import os +import platform +import shlex +import subprocess +import sys + + +def install_packages(pkgs): + if not os.environ.get('CMAKE_PREFIX_PATH'): + print('[ERROR] CMAKE_PREFIX_PATH is not set') + sys.exit(1) + + prefix = os.environ['CMAKE_PREFIX_PATH'].split(':')[0] + target = os.path.join(prefix, 'lib/python2.7/dist-packages') + + for pkg in pkgs: + cmd = 'pip install -q --target %s %s' % (target, pkg) + print('+ %s' % cmd) + subprocess.call(shlex.split(cmd)) + + +def main(): + pkgs = [] + + # PyTorch + if platform.platform().split('-')[0] == 'Linux': + if os.path.exists('/usr/local/cuda-7.5/include/cuda.h'): + pkgs.append('http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl') # NOQA + elif os.path.exists('/usr/local/cuda-8.0/include/cuda.h'): + pkgs.append('http://download.pytorch.org/whl/cu80/torch-0.1.10.post2-cp27-none-linux_x86_64.whl') # NOQA + else: + pkgs.append('http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl') # NOQA + + install_packages(pkgs) + + +if __name__ == '__main__': + main() diff --git a/jsk_perception/scripts/install_pytorch.sh b/jsk_perception/scripts/install_pytorch.sh deleted file mode 100755 index f7b2eb53f6..0000000000 --- a/jsk_perception/scripts/install_pytorch.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Install script of PyTorch. See http://pytorch.org for detail. - -set -x - -if [ -e /usr/local/cuda-7.5/include/cuda.h ]; then - pip install -q http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl -elif [ -e /usr/local/cuda-8.0/include/cuda.h ]; then - pip install -q http://download.pytorch.org/whl/cu80/torch-0.1.10.post2-cp27-none-linux_x86_64.whl -else - pip install -q http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl -fi - -set +x From e12c500e2875a0fd9f9fb942b123166f62ae3246 Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Thu, 6 Apr 2017 14:59:25 +0900 Subject: [PATCH 6/9] Fix method --- jsk_perception/node_scripts/fcn_object_segmentation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jsk_perception/node_scripts/fcn_object_segmentation.py b/jsk_perception/node_scripts/fcn_object_segmentation.py index 8bc16f398d..f48d8efcdb 100755 --- a/jsk_perception/node_scripts/fcn_object_segmentation.py +++ b/jsk_perception/node_scripts/fcn_object_segmentation.py @@ -52,7 +52,7 @@ def _load_model(self): else: raise RuntimeError('Unsupported backend: %s', self.backend) - def _load_model_chainer_backend(self): + def _load_chainer_model(self): model_name = rospy.get_param('~model_name') model_h5 = rospy.get_param('~model_h5') n_class = len(self.target_names) @@ -162,7 +162,7 @@ def _segment_chainer_backend(self, bgr): x_data = cuda.to_gpu(x_data, device=self.gpu) x = chainer.Variable(x_data, volatile=True) self.model(x) - proba_img = chainer.functions.softmax(self.model.score) + proba_img = chainer.functions.softmax(self.model.score)[0] proba_img = chainer.functions.transpose(proba_img, (1, 2, 0)) max_proba_img = chainer.functions.max(proba_img, axis=-1) label = chainer.functions.argmax(self.model.score, axis=1) From 1a1b09da3e14efa6924b67abd1d87dd80d390b1b Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Wed, 12 Apr 2017 04:00:54 +0900 Subject: [PATCH 7/9] Revert "Install packages to devel space" This reverts commit 40e068fc6788087c3a11f914269e93a4538be72e. --- jsk_perception/scripts/install_pytorch.py | 42 ----------------------- jsk_perception/scripts/install_pytorch.sh | 15 ++++++++ 2 files changed, 15 insertions(+), 42 deletions(-) delete mode 100755 jsk_perception/scripts/install_pytorch.py create mode 100755 jsk_perception/scripts/install_pytorch.sh diff --git a/jsk_perception/scripts/install_pytorch.py b/jsk_perception/scripts/install_pytorch.py deleted file mode 100755 index ecd1357ea4..0000000000 --- a/jsk_perception/scripts/install_pytorch.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python - -"""Install script of PyTorch. See http://pytorch.org for detail.""" - -import os -import platform -import shlex -import subprocess -import sys - - -def install_packages(pkgs): - if not os.environ.get('CMAKE_PREFIX_PATH'): - print('[ERROR] CMAKE_PREFIX_PATH is not set') - sys.exit(1) - - prefix = os.environ['CMAKE_PREFIX_PATH'].split(':')[0] - target = os.path.join(prefix, 'lib/python2.7/dist-packages') - - for pkg in pkgs: - cmd = 'pip install -q --target %s %s' % (target, pkg) - print('+ %s' % cmd) - subprocess.call(shlex.split(cmd)) - - -def main(): - pkgs = [] - - # PyTorch - if platform.platform().split('-')[0] == 'Linux': - if os.path.exists('/usr/local/cuda-7.5/include/cuda.h'): - pkgs.append('http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl') # NOQA - elif os.path.exists('/usr/local/cuda-8.0/include/cuda.h'): - pkgs.append('http://download.pytorch.org/whl/cu80/torch-0.1.10.post2-cp27-none-linux_x86_64.whl') # NOQA - else: - pkgs.append('http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl') # NOQA - - install_packages(pkgs) - - -if __name__ == '__main__': - main() diff --git a/jsk_perception/scripts/install_pytorch.sh b/jsk_perception/scripts/install_pytorch.sh new file mode 100755 index 0000000000..f7b2eb53f6 --- /dev/null +++ b/jsk_perception/scripts/install_pytorch.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Install script of PyTorch. See http://pytorch.org for detail. + +set -x + +if [ -e /usr/local/cuda-7.5/include/cuda.h ]; then + pip install -q http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl +elif [ -e /usr/local/cuda-8.0/include/cuda.h ]; then + pip install -q http://download.pytorch.org/whl/cu80/torch-0.1.10.post2-cp27-none-linux_x86_64.whl +else + pip install -q http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl +fi + +set +x From a76b52e7124a71d39f744442a4054a928cc52bc6 Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Wed, 12 Apr 2017 04:01:18 +0900 Subject: [PATCH 8/9] Remove install_pytorch.sh --- jsk_perception/scripts/install_pytorch.sh | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100755 jsk_perception/scripts/install_pytorch.sh diff --git a/jsk_perception/scripts/install_pytorch.sh b/jsk_perception/scripts/install_pytorch.sh deleted file mode 100755 index f7b2eb53f6..0000000000 --- a/jsk_perception/scripts/install_pytorch.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Install script of PyTorch. See http://pytorch.org for detail. - -set -x - -if [ -e /usr/local/cuda-7.5/include/cuda.h ]; then - pip install -q http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl -elif [ -e /usr/local/cuda-8.0/include/cuda.h ]; then - pip install -q http://download.pytorch.org/whl/cu80/torch-0.1.10.post2-cp27-none-linux_x86_64.whl -else - pip install -q http://download.pytorch.org/whl/cu75/torch-0.1.10.post2-cp27-none-linux_x86_64.whl -fi - -set +x From 29c99f4374bfd9a7739f35b04131011a54d76d76 Mon Sep 17 00:00:00 2001 From: Kentaro Wada Date: Wed, 12 Apr 2017 04:10:34 +0900 Subject: [PATCH 9/9] Raise error for unavailable torch & torchfcn --- jsk_perception/node_scripts/fcn_object_segmentation.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/jsk_perception/node_scripts/fcn_object_segmentation.py b/jsk_perception/node_scripts/fcn_object_segmentation.py index f48d8efcdb..b1644441de 100755 --- a/jsk_perception/node_scripts/fcn_object_segmentation.py +++ b/jsk_perception/node_scripts/fcn_object_segmentation.py @@ -23,9 +23,8 @@ def assert_torch_available(): if not is_torch_available: - raise RuntimeError( - 'PyTorch is unavailable. Please install it by:' - ' rosrun jsk_perception install_pytorch.sh') + url = 'http://download.pytorch.org/whl/cu80/torch-0.1.11.post4-cp27-none-linux_x86_64.whl' # NOQA + raise RuntimeError('Please install pytorch: pip install %s' % url) class FCNObjectSegmentation(ConnectionBasedTransport): @@ -75,8 +74,7 @@ def _load_torch_model(self): try: import torchfcn except ImportError as e: - rospy.logerr('Please install torchfcn by pip.') - raise ImportError(e) + raise ImportError('Please install torchfcn: pip install torchfcn') n_class = len(self.target_names) model_file = rospy.get_param('~model_file') model_name = rospy.get_param('~model_name')