Merge pull request #59 from asus4/fix-camera-rotation

Fix camera rotation
asus4 · Aug 24, 2020 · 7a6136c · 7a6136c
2 parents 2a94cb7 + 1e99494
commit 7a6136c
Show file tree

Hide file tree

Showing 20 changed files with 306 additions and 105 deletions.
diff --git a/Assets/Samples/BlazePose/BlazePose.unity b/Assets/Samples/BlazePose/BlazePose.unity
@@ -566,9 +566,9 @@ MonoBehaviour:
   m_UVRect:
     serializedVersion: 2
     x: 0
-    y: 1
+    y: 0
     width: 1
-    height: -1
+    height: 1
 --- !u!222 &1302103929
 CanvasRenderer:
   m_ObjectHideFlags: 0
@@ -692,9 +692,9 @@ MonoBehaviour:
   m_UVRect:
     serializedVersion: 2
     x: 0
-    y: 1
+    y: 0
     width: 1
-    height: -1
+    height: 1
 --- !u!222 &2074584279
 CanvasRenderer:
   m_ObjectHideFlags: 0

diff --git a/Assets/Samples/BlazePose/BlazePoseSample.cs b/Assets/Samples/BlazePose/BlazePoseSample.cs
@@ -130,6 +130,8 @@ void DrawJoints(Vector3[] joints)
             // FIXME: Flipping on iPhone. Need to be fixed
             p.x = 1.0f - p.x; 
 #endif
+            p.y = 1.0f - p.y;
+
             p = MathTF.Leap3(min, max, p);
             p.z += (joints[i].z - 0.5f) * zScale;
             var mtx = Matrix4x4.TRS(p, rotation, scale);

diff --git a/Assets/Samples/BlazePose/PoseLandmarkDetect.cs b/Assets/Samples/BlazePose/PoseLandmarkDetect.cs
@@ -49,18 +49,22 @@ public void Invoke(Texture inputTex, PoseDetect.Result pose)
                 ? TextureResizer.ModifyOptionForWebcam(resizeOptions, (WebCamTexture)inputTex)
                 : resizeOptions;
 
-            float rotation = CalcRotationDegree(ref pose);
-            var mat =RectTransformationCalculator.CalcMatrix(new RectTransformationCalculator.Options(){
-                rect = pose.rect,
-                rotationDegree = rotation,
-                shift = PoseShift,
-                scale = PoseScale,
-                cameraRotationDegree = -options.rotationDegree,
-            });
-            cropMatrix = resizer.VertexTransfrom = mat;
-            resizer.UVRect = TextureResizer.GetTextureST(inputTex, options);
-            RenderTexture rt = resizer.ApplyResize(inputTex, options.width, options.height, true);
-            ToTensor(rt, input0, false);
+            // float rotation = CalcRotationDegree(ref pose);
+            // var mat = RectTransformationCalculator.CalcMatrix(new RectTransformationCalculator.Options()
+            // {
+            //     rect = pose.rect,
+            //     rotationDegree = rotation,
+            //     shift = PoseShift,
+            //     scale = PoseScale,
+            //     cameraRotationDegree = -options.rotationDegree,
+            // });
+            // cropMatrix = resizer.VertexTransfrom = mat;
+            // resizer.UVRect = TextureResizer.GetTextureST(inputTex, options);
+            // RenderTexture rt = resizer.ApplyResize(inputTex, options.width, options.height, true);
+            // ToTensor(rt, input0, false);
+
+            cropMatrix = Matrix4x4.identity;
+            ToTensor(inputTex, input0);
 
             interpreter.SetInputTensorData(0, input0);
             interpreter.Invoke();
@@ -95,7 +99,7 @@ private static float CalcRotationDegree(ref PoseDetect.Result detection)
             // Calc rotation based on 
             // Center of Hip and Center of shoulder
             const float RAD_90 = 90f * Mathf.PI / 180f;
-            var vec = detection.keypoints[2] - detection.keypoints[0];
+            var vec = detection.keypoints[0] - detection.keypoints[2];
             return -(RAD_90 + Mathf.Atan2(vec.y, vec.x)) * Mathf.Rad2Deg;
         }
 

diff --git a/Assets/Samples/Common/BaseImagePredictor.cs b/Assets/Samples/Common/BaseImagePredictor.cs
@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using UnityEngine;
 using Unity.Collections;
-using Unity.Collections.LowLevel.Unsafe;
 
 namespace TensorFlowLite
 {
@@ -57,7 +56,7 @@ public BaseImagePredictor(string modelPath, bool useGPU = true)
                 aspectMode = TextureResizer.AspectMode.Fill,
                 rotationDegree = 0,
                 mirrorHorizontal = false,
-                mirrorVertical = true,
+                mirrorVertical = false,
                 width = width,
                 height = height,
             };

diff --git a/Assets/Samples/Common/RectTransformationCalculator.cs b/Assets/Samples/Common/RectTransformationCalculator.cs
@@ -32,14 +32,14 @@ public static Matrix4x4 CalcMatrix(Options options)
 
             Quaternion rotation = Quaternion.Euler(0, 0, options.rotationDegree);
             Vector2 size = Vector2.Scale(options.rect.size, options.scale);
-            if (options.mirrorHorizontal)
-            {
-                size.x *= -1;
-            }
-            if (options.mirrorVertiacal)
-            {
-                size.y *= -1;
-            }
+            // if (options.mirrorHorizontal)
+            // {
+            //     size.x *= -1;
+            // }
+            // if (options.mirrorVertiacal)
+            // {
+            //     size.y *= -1;
+            // }
 
             Vector2 shift = options.shift;
             // if (options.mirrorHorizontal)

diff --git a/Assets/Samples/Common/TextureResizer.cs b/Assets/Samples/Common/TextureResizer.cs
@@ -157,24 +157,31 @@ private static Matrix4x4 GetVertTransform(float rotation, bool mirrorHorizontal,
             return PUSH_MATRIX * trs * POP_MATRIX;
         }
 
-        public static ResizeOptions ModifyOptionForWebcam(ResizeOptions options, WebCamTexture texture)
+        public static ResizeOptions ModifyOptionForWebcam(ResizeOptions origOptions, WebCamTexture texture)
         {
-            if (options.rotationDegree - texture.videoRotationAngle < 0)
+            ResizeOptions newOptions = origOptions; // copy
+            int videoRotationAngle = texture.videoRotationAngle;
+            if (origOptions.rotationDegree - videoRotationAngle < 0)
             {
-                options.rotationDegree = 360f + options.rotationDegree - texture.videoRotationAngle;
+                newOptions.rotationDegree = 360f + origOptions.rotationDegree - videoRotationAngle;
             }
             else
             {
-                options.rotationDegree = options.rotationDegree - texture.videoRotationAngle;
+                newOptions.rotationDegree = origOptions.rotationDegree - videoRotationAngle;
+            }
+
+            bool needFlip90 = videoRotationAngle == 90 || videoRotationAngle == 270;
+            if (needFlip90)
+            {
+                newOptions.mirrorVertical = origOptions.mirrorHorizontal;
+                newOptions.mirrorHorizontal = origOptions.mirrorVertical;
             }
 
             if (texture.videoVerticallyMirrored)
             {
-                // ? Not sure
-                // options.mirrorHorizontal = !options.mirrorHorizontal;
-                options.mirrorVertical = !options.mirrorVertical;
+                newOptions.mirrorVertical = !newOptions.mirrorVertical;
             }
-            return options;
+            return newOptions;
         }
 
     }

diff --git a/Assets/Samples/Common/TextureToTensor.cs b/Assets/Samples/Common/TextureToTensor.cs
@@ -38,9 +38,10 @@ public void ToTensor(RenderTexture texture, sbyte[,,] inputs)
         {
             var pixels = FetchToTexture2D(texture).GetRawTextureData<Color32>();
             int width = texture.width;
+            int height = texture.height - 1;
             for (int i = 0; i < pixels.Length; i++)
             {
-                int y = i / width;
+                int y = height - i / width;
                 int x = i % width;
                 inputs[y, x, 0] = (sbyte)pixels[i].r;
                 inputs[y, x, 1] = (sbyte)pixels[i].g;
@@ -67,9 +68,10 @@ public void ToTensor(RenderTexture texture, float[,,] inputs, float offset, floa
             // TODO: optimize this
             var pixels = FetchToTexture2D(texture).GetRawTextureData<Color32>();
             int width = texture.width;
+            int height = texture.height - 1;
             for (int i = 0; i < pixels.Length; i++)
             {
-                int y = i / width;
+                int y = height - i / width;
                 int x = i % width;
                 inputs[y, x, 0] = (pixels[i].r - offset) * scale;
                 inputs[y, x, 1] = (pixels[i].g - offset) * scale;
@@ -82,10 +84,11 @@ void ToTensorCPU(RenderTexture texture, float[,,] inputs)
             var pixels = FetchToTexture2D(texture).GetRawTextureData<Color32>();
 
             int width = texture.width;
+            int height = texture.height - 1;
             const float scale = 255f;
             for (int i = 0; i < pixels.Length; i++)
             {
-                int y = i / width;
+                int y = height - i / width;
                 int x = i % width;
                 inputs[y, x, 0] = (float)(pixels[i].r) / scale;
                 inputs[y, x, 1] = (float)(pixels[i].g) / scale;

diff --git a/Assets/Samples/DeepLab/DeepLab.cs b/Assets/Samples/DeepLab/DeepLab.cs
@@ -123,7 +123,7 @@ public Texture2D GetResultTexture2D()
                 for (int x = 0; x < cols; x++)
                 {
                     int argmax = ArgMaxZ(outputs0, y, x, labels);
-                    labelPixels[y * cols + x] = COLOR_TABLE[argmax];
+                    labelPixels[(rows - 1 - y) * cols + x] = COLOR_TABLE[argmax];
                 }
             }
 

diff --git a/Assets/Samples/DeepLab/DeepLab.unity b/Assets/Samples/DeepLab/DeepLab.unity
@@ -96,6 +96,7 @@ LightmapSettings:
     m_PVRFilteringAtrousPositionSigmaAO: 1
     m_ExportTrainingData: 0
     m_TrainingDataDestination: TrainingData
+    m_LightProbeSampleCountMultiplier: 4
   m_LightingDataAsset: {fileID: 0}
   m_UseShadowmask: 1
 --- !u!196 &4
@@ -467,16 +468,17 @@ MonoBehaviour:
   m_Material: {fileID: 0}
   m_Color: {r: 1, g: 1, b: 1, a: 0.7764706}
   m_RaycastTarget: 0
+  m_Maskable: 1
   m_OnCullStateChanged:
     m_PersistentCalls:
       m_Calls: []
   m_Texture: {fileID: 0}
   m_UVRect:
     serializedVersion: 2
     x: 0
-    y: 1
+    y: 0
     width: 1
-    height: -1
+    height: 1
 --- !u!222 &1931683986
 CanvasRenderer:
   m_ObjectHideFlags: 0
@@ -539,16 +541,17 @@ MonoBehaviour:
   m_Material: {fileID: 0}
   m_Color: {r: 1, g: 1, b: 1, a: 1}
   m_RaycastTarget: 0
+  m_Maskable: 1
   m_OnCullStateChanged:
     m_PersistentCalls:
       m_Calls: []
   m_Texture: {fileID: 0}
   m_UVRect:
     serializedVersion: 2
     x: 0
-    y: 1
+    y: 0
     width: 1
-    height: -1
+    height: 1
 --- !u!222 &2074584279
 CanvasRenderer:
   m_ObjectHideFlags: 0

diff --git a/Assets/Samples/DeepLab/DeepLabSample.cs b/Assets/Samples/DeepLab/DeepLabSample.cs
@@ -37,13 +37,13 @@ void OnDestroy()
 
     void Update()
     {
-        var resizeOptions = deepLab.ResizeOptions;
-        resizeOptions.rotationDegree = webcamTexture.videoRotationAngle;
-        deepLab.ResizeOptions = resizeOptions;
-
         deepLab.Invoke(webcamTexture);
+        // Slow works on mobile
         outputView.texture = deepLab.GetResultTexture2D();
 
+        // Fast but errors on mobile. Need to be fixed 
+        // outputView.texture = deepLab.GetResultTexture();
+
         cameraView.material = deepLab.transformMat;
     }
 

diff --git a/Assets/Samples/HandTracking/HandLandmarkDetect.cs b/Assets/Samples/HandTracking/HandLandmarkDetect.cs
@@ -27,7 +27,7 @@ public enum Dimension
         private Matrix4x4 cropMatrix;
 
         public Dimension Dim { get; private set; }
-        public Vector2 PalmShift { get; set; } = new Vector2(0, -0.2f);
+        public Vector2 PalmShift { get; set; } = new Vector2(0, 0.2f);
         public Vector2 PalmScale { get; set; } = new Vector2(2.8f, 2.8f);
         public Matrix4x4 CropMatrix => cropMatrix;
 
@@ -62,27 +62,20 @@ public override void Invoke(Texture inputTex)
         public void Invoke(Texture inputTex, PalmDetect.Result palm)
         {
             var options = resizeOptions;
-            options.mirrorVertical = false;
             if (inputTex is WebCamTexture)
             {
                 options = TextureResizer.ModifyOptionForWebcam(options, (WebCamTexture)inputTex);
             }
 
-            // options.mirrorVertical = true;
-            // options.rotationDegree = (Time.time * 50) % 360.0f;
-            // options.mirrorHorizontal = true;
-            Debug.Log($"options.rotation: {options.rotationDegree}");
-
             float rotation = CalcHandRotation(ref palm) * Mathf.Rad2Deg;
             var mat = RectTransformationCalculator.CalcMatrix(new RectTransformationCalculator.Options()
             {
                 rect = palm.rect,
                 rotationDegree = rotation,
                 shift = PalmShift,
                 scale = PalmScale,
-                // cameraRotationDegree = -options.rotationDegree,
                 cameraRotationDegree = 0,
-                mirrorHorizontal = options.mirrorHorizontal,
+                mirrorHorizontal = !options.mirrorHorizontal,
                 mirrorVertiacal = options.mirrorVertical,
             });
             cropMatrix = resizer.VertexTransfrom = mat;
@@ -111,7 +104,7 @@ public Result GetResult()
                 {
                     result.joints[i] = mtx.MultiplyPoint3x4(new Vector3(
                         output0[i * 2] * SCALE,
-                        output0[i * 2 + 1] * SCALE,
+                        1f - output0[i * 2 + 1] * SCALE,
                         0
                     ));
                 }
@@ -122,7 +115,7 @@ public Result GetResult()
                 {
                     result.joints[i] = mtx.MultiplyPoint3x4(new Vector3(
                         output0[i * 3] * SCALE,
-                        output0[i * 3 + 1] * SCALE,
+                        1f - output0[i * 3 + 1] * SCALE,
                         output0[i * 3 + 2] * SCALE
                     ));
                 }
@@ -134,7 +127,7 @@ private static float CalcHandRotation(ref PalmDetect.Result detection)
         {
             // Rotation based on Center of wrist - Middle finger
             const float RAD_90 = 90f * Mathf.PI / 180f;
-            var vec = detection.keypoints[2] - detection.keypoints[0];
+            var vec = detection.keypoints[0] - detection.keypoints[2];
             return -(RAD_90 + Mathf.Atan2(vec.y, vec.x));
         }
     }

diff --git a/Assets/Samples/HandTracking/HandTracking.unity b/Assets/Samples/HandTracking/HandTracking.unity
@@ -557,18 +557,18 @@ MonoBehaviour:
   m_EditorClassIdentifier: 
   m_Material: {fileID: 0}
   m_Color: {r: 1, g: 1, b: 1, a: 1}
-  m_RaycastTarget: 1
-  m_Maskable: 1
+  m_RaycastTarget: 0
+  m_Maskable: 0
   m_OnCullStateChanged:
     m_PersistentCalls:
       m_Calls: []
   m_Texture: {fileID: 0}
   m_UVRect:
     serializedVersion: 2
     x: 0
-    y: 1
+    y: 0
     width: 1
-    height: -1
+    height: 1
 --- !u!222 &1302103929
 CanvasRenderer:
   m_ObjectHideFlags: 0
@@ -684,17 +684,17 @@ MonoBehaviour:
   m_Material: {fileID: 0}
   m_Color: {r: 1, g: 1, b: 1, a: 1}
   m_RaycastTarget: 0
-  m_Maskable: 1
+  m_Maskable: 0
   m_OnCullStateChanged:
     m_PersistentCalls:
       m_Calls: []
   m_Texture: {fileID: 0}
   m_UVRect:
     serializedVersion: 2
     x: 0
-    y: 1
+    y: 0
     width: 1
-    height: -1
+    height: 1
 --- !u!222 &2074584279
 CanvasRenderer:
   m_ObjectHideFlags: 0