Display image detections better on panorama images (not perfect yet)

src/org/openstreetmap/josm/plugins/mapillary/gui/MapillaryImageDisplay.java

@@ -21,6 +21,7 @@
 import java.awt.event.MouseWheelEvent;
 import java.awt.event.MouseWheelListener;
 import java.awt.geom.AffineTransform;
+import java.awt.geom.PathIterator;
 import java.awt.geom.Rectangle2D;
 import java.awt.image.BufferedImage;
 import java.util.ArrayList;
@@ -31,6 +32,7 @@
 import org.openstreetmap.josm.plugins.mapillary.MapillaryLayer;
 import org.openstreetmap.josm.plugins.mapillary.actions.MapillaryDownloadAction;
 import org.openstreetmap.josm.plugins.mapillary.gui.panorama.CameraPlane;
+import org.openstreetmap.josm.plugins.mapillary.gui.panorama.UVMapping;
 import org.openstreetmap.josm.plugins.mapillary.model.ImageDetection;
 import org.openstreetmap.josm.plugins.mapillary.model.MapObject;
 import org.openstreetmap.josm.plugins.mapillary.utils.MapillaryColorScheme;
@@ -463,7 +465,7 @@ public void paintComponent(Graphics g) {
           (int) ((size.width - noImageSize.getWidth()) / 2),
           (int) ((size.height - noImageSize.getHeight()) / 2));
     } else {
-      Rectangle target;
+      final Rectangle target;
       if (this.pano) {
         cameraPlane.mapping(image, offscreenImage);
         target = new Rectangle(0, 0, offscreenImage.getWidth(null), offscreenImage.getHeight(null));
@@ -494,26 +496,61 @@ public void paintComponent(Graphics g) {
       }
 
       if (MapillaryProperties.SHOW_DETECTED_SIGNS.get()) {
-        Point upperLeft = img2compCoord(visibleRect, 0, 0);
-        Point lowerRight = img2compCoord(visibleRect, getImage().getWidth(), getImage().getHeight());
-
-        // Transformation, which can convert you a Shape relative to the unit square to a Shape relative to the Component
-        AffineTransform unit2compTransform = AffineTransform.getTranslateInstance(upperLeft.getX(), upperLeft.getY());
-        unit2compTransform.concatenate(AffineTransform.getScaleInstance(lowerRight.getX() - upperLeft.getX(), lowerRight.getY() - upperLeft.getY()));
-
-        final Graphics2D g2d = (Graphics2D) g;
-        g2d.setStroke(new BasicStroke(2));
-        for (ImageDetection d : detections) {
-          final Shape shape = d.getShape().createTransformedShape(unit2compTransform);
-          g2d.setColor(d.isTrafficSign() ? MapillaryColorScheme.IMAGEDETECTION_TRAFFICSIGN : MapillaryColorScheme.IMAGEDETECTION_UNKNOWN);
-          g2d.draw(shape);
-          if (d.isTrafficSign()) {
-            g2d.drawImage(
-              MapObject.getIcon(d.getValue()).getImage(),
-              shape.getBounds().x, shape.getBounds().y,
-              shape.getBounds().width, shape.getBounds().height,
-              null
-            );
+        if (g instanceof Graphics2D) {
+          final Graphics2D g2d = (Graphics2D) g;
+          g2d.setStroke(new BasicStroke(2));
+          if (pano) {
+            for (final ImageDetection d : detections) {
+              g2d.setColor(d.isTrafficSign() ? MapillaryColorScheme.IMAGEDETECTION_TRAFFICSIGN : MapillaryColorScheme.IMAGEDETECTION_UNKNOWN);
+              final PathIterator pathIt = d.getShape().getPathIterator(null);
+              Point prevPoint = null;
+              int pointIndex;
+              while (!pathIt.isDone()) {
+                final double[] buffer = new double[6];
+                final int segmentType = pathIt.currentSegment(buffer);
+
+                if (segmentType == PathIterator.SEG_LINETO || segmentType == PathIterator.SEG_QUADTO || segmentType == PathIterator.SEG_CUBICTO) {
+                  // Takes advantage of the fact that SEG_LINETO=1, SEG_QUADTO=2, SEG_CUBICTO=3 and currentSegment() returns 1, 2 and 3 points for each of these segment types
+                  final Point curPoint = cameraPlane.getPoint(UVMapping.getVector(buffer[2 * (segmentType - 1)], buffer[2 * (segmentType - 1) + 1]));
+                  if (prevPoint != null && curPoint != null) {
+                    g2d.drawLine(prevPoint.x, prevPoint.y, curPoint.x, curPoint.y);
+                  }
+                  prevPoint = curPoint;
+                } else if (segmentType == PathIterator.SEG_MOVETO) {
+                  prevPoint = cameraPlane.getPoint(UVMapping.getVector(buffer[0], buffer[1]));
+                } else {
+                  prevPoint = null;
+                }
+                pathIt.next();
+              }
+            }
+          } else {
+            final Point upperLeft = img2compCoord(visibleRect, 0, 0);
+            final Point lowerRight = img2compCoord(visibleRect, getImage().getWidth(), getImage().getHeight());
+            final AffineTransform unit2CompTransform = AffineTransform.getTranslateInstance(upperLeft.getX(), upperLeft.getY());
+            unit2CompTransform.concatenate(AffineTransform.getScaleInstance(
+                lowerRight.getX() - upperLeft.getX(),
+                lowerRight.getY() - upperLeft.getY()
+            ));
+
+            for (final ImageDetection d : detections) {
+              final Shape shape = d.getShape().createTransformedShape(unit2CompTransform);
+              g2d.setColor(
+                  d.isTrafficSign()
+                      ? MapillaryColorScheme.IMAGEDETECTION_TRAFFICSIGN
+                      : MapillaryColorScheme.IMAGEDETECTION_UNKNOWN
+              );
+              g2d.draw(shape);
+              if (d.isTrafficSign()) {
+                final Rectangle bounds = shape.getBounds();
+                g2d.drawImage(
+                    MapObject.getIcon(d.getValue()).getImage(),
+                    bounds.x, bounds.y,
+                    bounds.width, bounds.height,
+                    null
+                );
+              }
+            }
           }
         }
       }

src/org/openstreetmap/josm/plugins/mapillary/gui/panorama/CameraPlane.java

@@ -2,10 +2,15 @@
 package org.openstreetmap.josm.plugins.mapillary.gui.panorama;
 
 import java.awt.Point;
+import java.awt.geom.Point2D;
 import java.awt.image.BufferedImage;
 import java.util.stream.IntStream;
 
 public class CameraPlane {
+  private final int width;
+  private final int height;
+  private final double distance;
+
   private Vector3D[][] vectors;
   private double theta;
   private double sinTheta;
@@ -14,28 +19,42 @@ public class CameraPlane {
   private double sinPhi;
   private double cosPhi;
 
-  public CameraPlane(int width, int height, double d) {
+  public CameraPlane(int width, int height, double distance) {
+    this.width = width;
+    this.height = height;
+    this.distance = distance;
     setRotation(0.0, 0.0);
     vectors = new Vector3D[width][height];
-    IntStream.range(0, height).forEach(y -> {
-      IntStream.range(0, width).forEach(x -> {
-        double vecX = x - width / 2;
-        double vecY = y - height / 2;
-        double vecZ = d;
-        double invVecLength = 1 / Math.sqrt(vecX * vecX + vecY * vecY + vecZ * vecZ);
-        vectors[x][y] = new Vector3D(vecX * invVecLength, vecY * invVecLength, vecZ * invVecLength);
+    IntStream.range(0, height).parallel().forEach(y -> {
+      IntStream.range(0, width).parallel().forEach(x -> {
+        vectors[x][y] = new Vector3D(x - width / 2d, y - height / 2d, distance).normalize();
       });
     });
   }
 
-  public Vector3D getVector3D(final Point p) {
-    return getVector3D(p.x, p.y);
+  /**
+   * @param vector the vector for which the corresponding point on the camera plane will be returned
+   * @return the point on the camera plane to which the given vector is mapped, nullable
+   */
+  public Point getPoint(final Vector3D vector) {
+    final Vector3D rotatedVector = rotate(vector, -1);
+    if (rotatedVector.getZ() < 0) {
+      return null; // Ignores any points "behind the back", so they don't get painted a second time on the other side of the sphere
+    }
+    return new Point(
+      (int) Math.max(Integer.MIN_VALUE, Math.min(Integer.MAX_VALUE, Math.round(
+        rotatedVector.getX() / rotatedVector.getZ() * distance + width / 2d
+      ))),
+      (int) Math.max(Integer.MIN_VALUE, Math.min(Integer.MAX_VALUE, Math.round(
+        rotatedVector.getY() / rotatedVector.getZ() * distance + height / 2d
+      )))
+    );
   }
 
-  public Vector3D getVector3D(final int x, final int y) {
+  Vector3D getVector3D(final Point p) {
     Vector3D res;
     try {
-      res = rotate(vectors[x][y]);
+      res = rotate(vectors[p.x][p.y]);
     } catch (Exception e) {
       res = new Vector3D(0, 0, 1);
     }
@@ -50,10 +69,6 @@ public void setRotation(final Point p) {
     setRotation(getVector3D(p));
   }
 
-  public void setRotation(final int x, final int y) {
-    setRotation(getVector3D(x, y));
-  }
-
   public void setRotationFromDelta(final Point from, final Point to) {
     Vector3D f1 = vectors[from.x][from.y];
     Vector3D t1 = vectors[to.x][to.y];
@@ -85,7 +100,7 @@ Vector3D getRotation() {
     return new Vector3D(sinTheta, sinPhi, cosPhi * cosTheta);
   }
 
-  void setRotation(double theta, double phi) {
+  synchronized void setRotation(double theta, double phi) {
     this.theta = theta;
     this.sinTheta = Math.sin(theta);
     this.cosTheta = Math.cos(theta);
@@ -94,34 +109,28 @@ void setRotation(double theta, double phi) {
     this.cosPhi = Math.cos(phi);
   }
 
-  Vector3D rotate(Vector3D vec) {
+  private Vector3D rotate(final Vector3D vec) {
+    return rotate(vec, 1);
+  }
+
+  private Vector3D rotate(final Vector3D vec, final int rotationFactor) {
     double vecX, vecY, vecZ;
     vecZ = vec.getZ() * cosPhi - vec.getY() * sinPhi;
     vecY = vec.getZ() * sinPhi + vec.getY() * cosPhi;
-    vecX = vecZ * sinTheta + vec.getX() * cosTheta;
-    vecZ = vecZ * cosTheta - vec.getX() * sinTheta;
+    vecX = vecZ * sinTheta * rotationFactor + vec.getX() * cosTheta;
+    vecZ = vecZ * cosTheta - vec.getX() * sinTheta * rotationFactor;
     return new Vector3D(vecX, vecY, vecZ);
   }
 
   public void mapping(BufferedImage sourceImage, BufferedImage targetImage) {
-    int height = targetImage.getHeight();
-    int width = targetImage.getWidth();
-    IntStream.range(0, height).parallel().forEach(y -> {
-      IntStream.range(0, width).forEach(x -> {
-        Vector3D vec = getVector3D(new Point(x, y));
-        Point p = mapping(vec, sourceImage.getWidth(), sourceImage.getHeight());
-        int color = sourceImage.getRGB(p.x, p.y);
-        targetImage.setRGB(x, y, color);
+    IntStream.range(0, targetImage.getHeight()).parallel().forEach(y -> {
+      IntStream.range(0, targetImage.getWidth()).forEach(x -> {
+        final Vector3D vec = getVector3D(new Point(x, y));
+        final Point2D.Double p = UVMapping.getTextureCoordinate(vec);
+        targetImage.setRGB(x, y,
+            sourceImage.getRGB((int) (p.x * (sourceImage.getWidth() - 1)), (int) (p.y * (sourceImage.getHeight() - 1)))
+        );
       });
     });
   }
-
-  Point mapping(Vector3D vec, int width, int height) {
-    // https://en.wikipedia.org/wiki/UV_mapping
-    double u = 0.5 + (Math.atan2(vec.getX(), vec.getZ()) / (2 * Math.PI));
-    double v = 0.5 + (Math.asin(vec.getY()) / Math.PI);
-    int tx = (int) ((width - 1) * u);
-    int ty = (int) ((height - 1) * v);
-    return new Point(tx, ty);
-  }
 }

src/org/openstreetmap/josm/plugins/mapillary/gui/panorama/UVMapping.java

@@ -0,0 +1,37 @@
+package org.openstreetmap.josm.plugins.mapillary.gui.panorama;
+
+import java.awt.geom.Point2D;
+
+public final class UVMapping {
+  private UVMapping() {
+    // Private constructor to avoid instantiation
+  }
+  /**
+   * Returns the point of the texture image that is mapped to the given point in 3D space (given as {@link Vector3D})
+   * See <a href="https://en.wikipedia.org/wiki/UV_mapping">the Wikipedia article on UV mapping</a>.
+   * @param vector the vector to which the texture point is mapped
+   * @return a point on the texture image somewhere in the rectangle between (0, 0) and (1, 1)
+   */
+  public static Point2D.Double getTextureCoordinate(final Vector3D vector) {
+    final double u = 0.5 + (Math.atan2(vector.getX(), vector.getZ()) / (2 * Math.PI));
+    final double v = 0.5 + (Math.asin(vector.getY()) / Math.PI);
+    return new Point2D.Double(u, v);
+  }
+
+  /**
+   * For a given point of the texture (i.e. the image), return the point in 3D space where the point
+   * of the texture is mapped to (as {@link Vector3D}).
+   *
+   * @param u x-coordinate of the point on the texture (in the range between 0 and 1, from left to right)
+   * @param v y-coordinate of the point on the texture (in the range between 0 and 1, from top to bottom)
+   * @return the vector from the origin to where the point of the texture is mapped on the sphere
+   */
+  public static Vector3D getVector(final double u, final double v) {
+    final double vectorY = Math.cos(v * Math.PI);
+    return new Vector3D(
+        -Math.sin(2 * Math.PI * u) * Math.sqrt(1 - vectorY * vectorY),
+        -vectorY,
+        -Math.cos(2 * Math.PI * u) * Math.sqrt(1 - vectorY * vectorY)
+    );
+  }
+}

src/org/openstreetmap/josm/plugins/mapillary/gui/panorama/Vector3D.java

@@ -1,25 +1,33 @@
 // License: GPL. For details, see LICENSE file.
 package org.openstreetmap.josm.plugins.mapillary.gui.panorama;
 
-public class Vector3D {
+final class Vector3D {
 
     private double x;
     private double y;
     private double z;
 
-    public Vector3D(double x, double y, double z) {
+    Vector3D(double x, double y, double z) {
         this.x = x;
         this.y = y;
         this.z = z;
     }
 
-    public double getX() {
+    double getX() {
         return x;
     }
-    public double getY() {
+    double getY() {
         return y;
     }
-    public double getZ() {
+    double getZ() {
         return z;
     }
+
+    synchronized Vector3D normalize() {
+      final double length = Math.sqrt(x*x + y*y + z*z);
+      x /= length;
+      y /= length;
+      z /= length;
+      return this;
+    }
 }

test/unit/org/openstreetmap/josm/plugins/mapillary/gui/panorama/CameraPlaneTest.java

@@ -4,6 +4,7 @@
 import static org.junit.Assert.assertEquals;
 
 import java.awt.Point;
+import java.awt.geom.Point2D;
 
 import org.junit.Rule;
 import org.junit.Test;
@@ -36,7 +37,7 @@ public void testGetVector3D() {
     cameraPlane = new CameraPlane(800, 600, CAMERA_PLANE_DISTANCE);
     Vector3D vec = new Vector3D(0, 0, 1);
     cameraPlane.setRotation(vec);
-    Vector3D out = cameraPlane.getVector3D(400, 300);
+    Vector3D out = cameraPlane.getVector3D(new Point(400, 300));
     assertEquals(0.0, out.getX(), 1.0E-04);
     assertEquals(0.0, out.getY(), 1.0E-04);
     assertEquals(1.0, out.getZ(), 1.0E-04);
@@ -47,10 +48,10 @@ public void testMapping() {
     cameraPlane = new CameraPlane(800, 600, CAMERA_PLANE_DISTANCE);
     Vector3D vec = new Vector3D(0, 0, 1);
     cameraPlane.setRotation(vec);
-    Vector3D out = cameraPlane.getVector3D(300, 200);
-    Point map = cameraPlane.mapping(out, 2048, 1024);
-    assertEquals(911, map.getX(), 1);
-    assertEquals(405, map.getY(), 1);
+    Vector3D out = cameraPlane.getVector3D(new Point(300, 200));
+    Point2D map = UVMapping.getTextureCoordinate(out);
+    assertEquals(0.44542099, map.getX(), 1e-8);
+    assertEquals(0.39674936, map.getY(), 1e-8);
   }
 }