Close the loop - whole game flow first iteration (#29)

dashboard/src/subscribers/guesswhat.js

@@ -6,6 +6,18 @@ import $ from 'jquery';
 const cons = new LogConsole("GuessWhat", "#00bc8c");
 const guesswhatCheckbox = $("#guesswhat_checkbox");
 
+const state_listener = new ROSLIB.Topic({
+  ros: ros,
+  name: '/guesswhat_state',
+  messageType: 'std_msgs/String'
+});
+
+const category_listener = new ROSLIB.Topic({
+  ros: ros,
+  name: '/found_category',
+  messageType: 'std_msgs/String'
+});
+
 const question_listener = new ROSLIB.Topic({
   ros: ros,
   name: '/question',
@@ -21,6 +33,12 @@ const answer_listener = new ROSLIB.Topic({
 guesswhatCheckbox.on("change", function () {
   $('#guesswhat_ask_btn').prop('disabled', !this.checked);
   if (this.checked) {
+    state_listener.subscribe(function (message) {
+      cons.log(`State: ${message.data}`)
+    });
+    category_listener.subscribe(function (message) {
+      cons.log(`<b>Game ended!</b> Found: ${message.data}`)
+    });
     answer_listener.subscribe(function (message) {
       cons.log(`Answer: ${message.data}`)
     });
@@ -31,6 +49,8 @@ guesswhatCheckbox.on("change", function () {
   } else {
     answer_listener.removeAllListeners();
     question_listener.removeAllListeners();
+    category_listener.removeAllListeners();
+    state_listener.removeAllListeners();
     cons.log("Unsubscribed");
   }
 });

dashboard/src/subscribers/snips.js

@@ -9,14 +9,14 @@ const snipsCheckbox = $("#snips_checkbox");
 const answer_listener = new ROSLIB.Topic({
   ros: ros,
   latch: true,
-  name: '/snips_answer',
+  name: '/answer',
   messageType: 'std_msgs/String'
 });
 
 const ask_listener = new ROSLIB.Topic({
   ros: ros,
   latch: true,
-  name: '/snips_ask',
+  name: '/question',
   messageType: 'std_msgs/String'
 });
 

dashboard/src/vars.js

@@ -5,8 +5,6 @@ export const devineTopics = [
     "/vgg16_features",
     "/rcnn_segmentation",
     "/devine/image",
-    "/snips_answer",
-    "/snips_ask",
     "/answer",
     "/question",
 ];
@@ -27,4 +25,4 @@ export const distinctColors = [
     "#000080",
     "#808080",
     "#000000",
-];
+];

dashboard/views/publisher.html

@@ -21,7 +21,7 @@ <h3 class="card-header">
                         <div role="separator" class="dropdown-divider all-topics"></div>
                     </div>
                 </div>
-                <input id="publisher_topic" placeholder="/snips_ask" type="text" class="form-control" style="border: none;">
+                <input id="publisher_topic" placeholder="/question" type="text" class="form-control" style="border: none;">
             </div>
         </li>
         <li class="list-group-item">
@@ -36,4 +36,4 @@ <h3 class="card-header">
             </div>
         </li>
     </ul>
-</div>
+</div>

devine.launch

@@ -1,19 +1,30 @@
 <launch>
+  <arg name="robot_control" default="true"/>
   <arg name="kinect" default="true"/>
   <arg name="snips" default="true"/>
   <arg name="gameSystem" default="true"/>
   <arg name="dashboard" default="true" />
   <arg name="guesswhat" default="true" />
+  <arg name="image_processing" default="true" />
+
+  <group if="$(arg robot_control)">
+    <include file="$(find jn0_gazebo)/launch/jn0_empty_world.launch" />
+    <include file="$(find irl_control)/src/irl_control/irl_control.launch" />
+  </group>
   <group if="$(arg kinect)">
-    <include file="$(find openni_launch)/launch/openni.launch" />
-    <node name="img_throttle" pkg="topic_tools" type="throttle" args="messages /camera/rgb/image_color/compressed 0.03 /devine/image" />
-    <node name="pos_lib" pkg="devine_kinect" type="pos_lib.py" respawn="true" />
-    <node name="features_extraction" pkg="devine_image_processing" type="features_extraction.py" respawn="true" />
-    <node name="segmentation" pkg="devine_image_processing" type="segmentation.py" respawn="true" />
+    <include file="$(find openni_launch)/launch/openni.launch">
+      <!-- <arg name="camera" value="openni"/> -->
+    </include>
+    <node name="img_throttle" pkg="topic_tools" type="throttle" args="messages /camera/rgb/image_color/compressed 0.008 /devine/image" />
+  <node name="pos_lib" pkg="devine_kinect" type="pos_lib.py" respawn="true" />
   </group>
   <group if="$(arg snips)">
     <node name="snips" pkg="snips" type="snips.py" respawn="true" />
   </group>
+  <group if="$(arg image_processing)">
+    <node name="features_extraction" pkg="devine_image_processing" type="features_extraction.py" respawn="true" />
+    <node name="image_segmentation" pkg="devine_image_processing" type="segmentation.py" respawn="true" />
+  </group>
   <group if="$(arg gameSystem)">
     <node name="game_system" pkg="game_system" type="client.py" respawn="true" />
   </group>

guesswhat/.gitignore

@@ -0,0 +1,2 @@
+devine_guesswhat/data/*
+weights.*

guesswhat/devine_guesswhat/src/guesswhat_node.py

@@ -33,9 +33,11 @@
 SEGMENTATION_TOPIC = '/rcnn_segmentation'
 FEATURES_TOPIC = '/vgg16_features'
 OBJECT_TOPIC = '/object_found'
+CATEGORY_TOPIC = '/found_category'
+STATE_TOPIC = '/guesswhat_state'
 
-segmentations = Queue(2)
-features = Queue(2)
+segmentations = Queue(1)
+features = Queue(1)
 
 class ImgFeaturesLoader():
     '''Loads image from memory'''
@@ -74,20 +76,28 @@ def open_config(path):
 
 def segmentation_callback(data):
     '''Callback for the segmantion topic'''
+    if segmentations.full():
+        segmentations.get()
+
     try:
         segmentations.put(json.loads(data.data))
     except json.JSONDecodeError:
         rospy.logerr('Garbage on {}, expects json'.format(SEGMENTATION_TOPIC))
 
 def features_callback(data):
     '''Callback for the features topic'''
+    if features.full():
+        features.get()
+
     features.put(np.array(data.data))
 
 if __name__ == '__main__':
     rospy.init_node('guesswhat')
     rospy.Subscriber(SEGMENTATION_TOPIC, String, segmentation_callback)
     rospy.Subscriber(FEATURES_TOPIC, Float64MultiArray, features_callback)
     object_found = rospy.Publisher(OBJECT_TOPIC, Int32MultiArray, queue_size=1)
+    category = rospy.Publisher(CATEGORY_TOPIC, String, queue_size=1)
+    state = rospy.Publisher(STATE_TOPIC, String, queue_size=1, latch=True)
 
     eval_config = open_config(EVAL_CONF_PATH)
     guesser_config = open_config(GUESS_CONF_PATH)
@@ -119,6 +129,7 @@ def features_callback(data):
 
         batchifier = LooperBatchifier(tokenizer, generate_new_games=False)
 
+        state.publish('Waiting for image processing')
         while not rospy.is_shutdown():
             try:
                 seg = segmentations.get(timeout=1)
@@ -127,7 +138,7 @@ def features_callback(data):
                 continue
 
             rospy.loginfo('Starting new game')
-            img = {'id': 0, 'width': 10, 'height': 10, 'coco_url': ''}
+            img = {'id': 0, 'width': 640, 'height': 480, 'coco_url': ''}
             game = Game(id=0,
                         object_id=0,
                         objects=seg['objects'],
@@ -154,3 +165,6 @@ def features_callback(data):
             object_found.publish(Int32MultiArray(data=[int(choice_bbox.x_center),
                                                        int(choice_bbox.y_center)]))
             rospy.loginfo('Game over')
+            category.publish(seg['objects'][choice_index]['category'])
+
+            state.publish('Waiting for image processing')

guesswhat/devine_guesswhat/src/modelwrappers.py

@@ -16,7 +16,7 @@ class GuesserROSWrapper(GuesserWrapper):
     '''Wraps the guesser model and publishes confidence levels'''
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.confidence = rospy.Publisher(CONFIDENCE_TOPIC, Float64MultiArray, queue_size=1)
+        self.confidence = rospy.Publisher(CONFIDENCE_TOPIC, Float64MultiArray, queue_size=1, latch=True)
 
     def find_object(self, *args, **kwargs):
         '''find_object interface for the looper'''
@@ -32,7 +32,7 @@ class OracleROSWrapper(object):
     def __init__(self, tokenizer):
         self.tokenizer = tokenizer
         self.answers = Queue(1)
-        self.questions = rospy.Publisher(QUESTION_TOPIC, String, queue_size=1)
+        self.questions = rospy.Publisher(QUESTION_TOPIC, String, queue_size=1, latch=True)
         rospy.Subscriber(ANSWER_TOPIC, String, self.answer_callback)
 
     def initialize(self, sess):

guesswhat/install_package.bash

@@ -3,5 +3,12 @@
 CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/devine_guesswhat"
 echo "export ROS_PACKAGE_PATH=$CURDIR:\$ROS_PACKAGE_PATH" >> ~/.bashrc
 
+wget https://github.com/projetdevine/static/releases/download/v0.0.1/weights.zip
+unzip weights.zip -d devine_guesswhat/data
+
+# Adding guesswhat repo to pythonpath
+CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/guesswhat/src"
+echo "export PYTHONPATH=$CURDIR:\$PYTHONPATH" >> ~/.bashrc
+
 # Reload env
 exec bash

image_processing/.gitignore

@@ -1 +1,2 @@
 mask_rcnn_coco.h5
+vgg_16.ckpt

image_processing/devine_image_processing/src/features_extraction.py

@@ -26,7 +26,7 @@
 
 if __name__ == '__main__':
     rospy.init_node('features_extraction')
-    features = rospy.Publisher(FEATURES_TOPIC, Float64MultiArray, queue_size=1)
+    features = rospy.Publisher(FEATURES_TOPIC, Float64MultiArray, queue_size=1, latch=True)
 
     image_queue = Queue(2)
 
@@ -56,7 +56,7 @@ def image_callback(data):
                 img = Image.open(BytesIO(img))
                 img = img.resize((IMAGE_SIZE, IMAGE_SIZE), resample=Image.BILINEAR)
                 img = np.array(img, dtype=np.float32)
-                img -= channel[None, None, :]
+                img -= CHANNEL_MEAN[None, None, :]
 
                 ft_output = end_points['vgg_16/fc8']
                 feat = sess.run(ft_output, feed_dict={holder: np.array([img])})

image_processing/devine_image_processing/src/segmentation.py

@@ -65,8 +65,8 @@ def correct_array(self, bbox, im_height):
             width = original_array[3] - original_array[1]
             height = original_array[2] - original_array[0]
             left = original_array[1]
-            top = im_height - original_array[2]
-            correct_boxes[counter] = np.array([left, top, height, width])
+            top = original_array[0]
+            correct_boxes[counter] = np.array([left, top, width, height])
         return correct_boxes
 
     def segment(self, img):
@@ -132,7 +132,7 @@ def __init__(self):
         rospy.init_node('image_segmentation')
         rospy.Subscriber(IMAGE_TOPIC, CompressedImage,
                          self.image_received_callback, queue_size=1)
-        self.publisher = rospy.Publisher(SEGMENTATION_TOPIC, String, queue_size=10)
+        self.publisher = rospy.Publisher(SEGMENTATION_TOPIC, String, queue_size=10, latch=True)
 
     def image_received_callback(self, data):
         '''Callback when a new image is received from the topic'''