diff --git a/VRCFaceTracking/Params/Eye/EyeTrackingParams.cs b/VRCFaceTracking/Params/Eye/EyeTrackingParams.cs index d7d0bc7c..7726dec7 100644 --- a/VRCFaceTracking/Params/Eye/EyeTrackingParams.cs +++ b/VRCFaceTracking/Params/Eye/EyeTrackingParams.cs @@ -16,11 +16,17 @@ public static class EyeTrackingParams new EParam(v2 => v2.Left.Widen > v2.Right.Widen ? v2.Left.Widen : v2.Right.Widen, "EyesWiden"), new EParam(v2 => v2.Left.Widen, "LeftEyeWiden"), new EParam(v2 => v2.Right.Widen, "RightEyeWiden"), - + #endregion - + + #region Squint + new EParam(v2 => v2.Left.Squint > v2.Right.Squint ? v2.Left.Squint : v2.Right.Squint, "EyesSquint"), + new EParam(v2 => v2.Left.Squint, "LeftEyeSquint"), + new EParam(v2 => v2.Right.Squint, "RightEyeSquint"), + #endregion + #region Squeeze - + new EParam(v2 => v2.Combined.Squeeze, "EyesSqueeze"), new EParam(v2 => v2.Left.Squeeze, "LeftEyeSqueeze"), new EParam(v2 => v2.Right.Squeeze, "RightEyeSqueeze"), @@ -46,21 +52,21 @@ public static class EyeTrackingParams new EParam((v2, eye) => { - if (v2.Left.Widen > 0) + if (v2.Left.Widen > 1-v2.Left.Openness) return NormalizeFloat(0, 1, 0.8f, 1, v2.Left.Widen); return NormalizeFloat(0, 1, 0, 0.8f, v2.Left.Openness); }, "LeftEyeLidExpanded", 0.5f, true), new EParam((v2, eye) => { - if (v2.Right.Widen > 0) + if (v2.Right.Widen > 1-v2.Right.Openness) return NormalizeFloat(0, 1, 0.8f, 1, v2.Right.Widen); return NormalizeFloat(0, 1, 0, 0.8f, v2.Right.Openness); }, "RightEyeLidExpanded", 0.5f, true), new EParam((v2, eye) => { - if (v2.Combined.Widen > 0) + if (v2.Combined.Widen > 1-v2.Combined.Openness) return NormalizeFloat(0, 1, 0.8f, 1, v2.Combined.Widen); return NormalizeFloat(0, 1, 0, 0.8f, v2.Combined.Openness); }, "CombinedEyeLidExpanded", 0.5f, true), @@ -71,27 +77,27 @@ public static class EyeTrackingParams new EParam((v2, eye) => { - if (v2.Left.Widen > 0) + if (v2.Left.Widen > 1-v2.Left.Openness) return NormalizeFloat(0, 1, 0.8f, 1, v2.Left.Widen); - if (v2.Left.Squeeze > 0) + if (v2.Left.Squeeze > 1-v2.Left.Openness) return v2.Left.Squeeze * -1; return NormalizeFloat(0, 1, 0, 0.8f, v2.Left.Openness); } ,"LeftEyeLidExpandedSqueeze", 0.5f, true), new EParam((v2, eye) => { - if (v2.Right.Widen > 0) + if (v2.Right.Widen > 1-v2.Right.Openness) return NormalizeFloat(0, 1, 0.8f, 1, v2.Right.Widen); - if (v2.Right.Squeeze > 0) + if (v2.Right.Squeeze > 1-v2.Right.Openness) return v2.Right.Squeeze * -1; return NormalizeFloat(0, 1, 0, 0.8f, v2.Right.Openness); } ,"RightEyeLidExpandedSqueeze", 0.5f, true), new EParam((v2, eye) => { - if (v2.Combined.Widen > 0) + if (v2.Combined.Widen > 1-v2.Combined.Openness) return NormalizeFloat(0, 1, 0.8f, 1, v2.Combined.Widen); - if (v2.Combined.Squeeze > 0) + if (v2.Combined.Squeeze > 1-v2.Combined.Openness) return v2.Combined.Squeeze * -1; return NormalizeFloat(0, 1, 0, 0.8f, v2.Combined.Openness); } ,"CombinedEyeLidExpandedSqueeze", 0.5f, true), @@ -102,21 +108,21 @@ public static class EyeTrackingParams new BinaryParameter((v2, eye) => { - if (v2.Left.Widen > 0) + if (v2.Left.Widen > 1-v2.Left.Openness) return v2.Left.Widen; return v2.Left.Openness; }, "LeftEyeLidExpanded"), new BinaryParameter((v2, eye) => { - if (v2.Right.Widen > 0) + if (v2.Right.Widen > 1-v2.Right.Openness) return v2.Right.Widen; return v2.Right.Openness; }, "RightEyeLidExpanded"), new BinaryParameter((v2, eye) => { - if (v2.Combined.Widen > 0) + if (v2.Combined.Widen > 1-v2.Combined.Openness) return v2.Combined.Widen; return v2.Combined.Openness; }, "CombinedEyeLidExpanded"), @@ -127,27 +133,27 @@ public static class EyeTrackingParams new BinaryParameter(v2 => { - if (v2.Left.Widen > 0) + if (v2.Left.Widen > 1-v2.Left.Openness) return v2.Left.Widen; - if (v2.Left.Squeeze > 0) + if (v2.Left.Squeeze > 1-v2.Left.Openness) return v2.Left.Squeeze; return v2.Left.Openness; }, "LeftEyeLidExpandedSqueeze"), new BinaryParameter(v2 => { - if (v2.Right.Widen > 0) + if (v2.Right.Widen > 1-v2.Right.Openness) return v2.Right.Widen; - if (v2.Right.Squeeze > 0) + if (v2.Right.Squeeze > 1-v2.Right.Openness) return v2.Right.Squeeze; return v2.Right.Openness; }, "RightEyeLidExpandedSqueeze"), new BinaryParameter((v2, eye) => { - if (v2.Combined.Widen > 0) + if (v2.Combined.Widen > 1-v2.Combined.Openness) return v2.Combined.Widen; - if (v2.Combined.Squeeze > 0) + if (v2.Combined.Squeeze > 1-v2.Combined.Openness) return v2.Combined.Squeeze; return v2.Combined.Openness; }, "CombinedEyeLidExpandedSqueeze"), @@ -158,14 +164,30 @@ public static class EyeTrackingParams // These parameters are used to distinguish when EyeLidExpanded / EyeLidExpandedSqueeze // is returning a value as a Widen or Squeeze. Intended for the Bool or Binary param variant. - new BoolParameter(v2 => v2.Left.Widen > 0, "LeftEyeWidenToggle"), - new BoolParameter(v2 => v2.Right.Widen > 0, "RightEyeWidenToggle"), - new BoolParameter(v2 => v2.Combined.Widen > 0, "EyesWidenToggle"), + new BoolParameter(v2 => v2.Left.Widen > 1-v2.Left.Openness, "LeftEyeWidenToggle"), + new BoolParameter(v2 => v2.Right.Widen > 1-v2.Right.Openness, "RightEyeWidenToggle"), + new BoolParameter(v2 => v2.Combined.Widen > 1-v2.Combined.Openness, "EyesWidenToggle"), + + new BoolParameter(v2 => v2.Left.Squeeze > 1-v2.Left.Openness, "LeftEyeSqueezeToggle"), + new BoolParameter(v2 => v2.Right.Squeeze > 1-v2.Right.Openness, "RightEyeSqueezeToggle"), + new BoolParameter(v2 => v2.Combined.Squeeze > 1-v2.Combined.Openness, "EyesSqueezeToggle"), + + #endregion - new BoolParameter(v2 => v2.Left.Squeeze > 0, "LeftEyeSqueezeToggle"), - new BoolParameter(v2 => v2.Right.Squeeze > 0, "RightEyeSqueezeToggle"), - new BoolParameter(v2 => v2.Combined.Squeeze > 0, "EyesSqueezeToggle"), + #region EyeBrow + new EParam(v2 => v2.Left.Brow.InnerUp > v2.Right.Brow.InnerUp ? v2.Left.Brow.InnerUp : v2.Right.Brow.InnerUp, "BrowsInnerUp"), + new EParam(v2 => v2.Left.Brow.InnerUp, "BrowInnerUpLeft"), + new EParam(v2 => v2.Right.Brow.InnerUp, "BrowInnerUpRight"), + + new EParam(v2 => v2.Left.Brow.OuterUp > v2.Right.Brow.InnerUp ? v2.Left.Brow.OuterUp : v2.Right.Brow.OuterUp, "BrowsOuterUp"), + new EParam(v2 => v2.Left.Brow.OuterUp, "BrowOuterUpLeft"), + new EParam(v2 => v2.Right.Brow.OuterUp, "BrowOuterUpRight"), + + new EParam(v2 => v2.Left.Brow.InnerDown > v2.Right.Brow.InnerUp ? v2.Left.Brow.InnerDown : v2.Right.Brow.InnerDown, "BrowsDown"), + new EParam(v2 => v2.Left.Brow.InnerDown, "BrowDownLeft"), + new EParam(v2 => v2.Right.Brow.InnerDown, "BrowDownRight"), + #endregion #region Status @@ -179,4 +201,4 @@ public static class EyeTrackingParams private static float NormalizeFloat(float minInput, float maxInput, float minOutput, float maxOutput, float value) => (maxOutput - minOutput) / (maxInput - minInput) * (value - maxInput) + maxOutput; } -} \ No newline at end of file +} diff --git a/VRCFaceTracking/Params/Lip/LipShapeConversion.cs b/VRCFaceTracking/Params/Lip/LipShapeConversion.cs index 4cbc37cb..bcb7dcec 100644 --- a/VRCFaceTracking/Params/Lip/LipShapeConversion.cs +++ b/VRCFaceTracking/Params/Lip/LipShapeConversion.cs @@ -15,7 +15,7 @@ public class PositiveNegativeShape : ICombinedShape private float _positiveCache, _negativeCache; private bool _steps; - public PositiveNegativeShape(LipShape_v2 positiveShape, LipShape_v2 negativeShape, bool steps = false) + public PositiveNegativeShape(UnifiedExpression positiveShape, UnifiedExpression negativeShape, bool steps = false) { _positiveShape = (int)positiveShape; _negativeShape = (int)negativeShape; @@ -37,7 +37,7 @@ public class PositiveNegativeAveragedShape : ICombinedShape private readonly int _positiveCount, _negativeCount; private readonly bool _useMax; - public PositiveNegativeAveragedShape(LipShape_v2[] positiveShapes, LipShape_v2[] negativeShapes) + public PositiveNegativeAveragedShape(UnifiedExpression[] positiveShapes, UnifiedExpression[] negativeShapes) { _positiveShapes = positiveShapes.Select(s => (int)s).ToArray(); _negativeShapes = negativeShapes.Select(s => (int)s).ToArray(); @@ -47,7 +47,7 @@ public PositiveNegativeAveragedShape(LipShape_v2[] positiveShapes, LipShape_v2[] _negativeCount = negativeShapes.Length; } - public PositiveNegativeAveragedShape(LipShape_v2[] positiveShapes, LipShape_v2[] negativeShapes, bool useMax) + public PositiveNegativeAveragedShape(UnifiedExpression[] positiveShapes, UnifiedExpression[] negativeShapes, bool useMax) { _positiveShapes = positiveShapes.Select(s => (int)s).ToArray(); _negativeShapes = negativeShapes.Select(s => (int)s).ToArray(); diff --git a/VRCFaceTracking/Params/Lip/LipShapeMerger.cs b/VRCFaceTracking/Params/Lip/LipShapeMerger.cs index 81e22f69..ed1d47e2 100644 --- a/VRCFaceTracking/Params/Lip/LipShapeMerger.cs +++ b/VRCFaceTracking/Params/Lip/LipShapeMerger.cs @@ -11,143 +11,143 @@ public static class LipShapeMerger private static readonly Dictionary MergedShapes = new Dictionary { - {"JawX", new PositiveNegativeShape(LipShape_v2.JawRight, LipShape_v2.JawLeft)}, - {"MouthUpper", new PositiveNegativeShape(LipShape_v2.MouthUpperRight, LipShape_v2.MouthUpperLeft)}, - {"MouthLower", new PositiveNegativeShape(LipShape_v2.MouthLowerRight, LipShape_v2.MouthLowerLeft)}, - {"MouthX", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperRight, LipShape_v2.MouthLowerRight}, new LipShape_v2[]{LipShape_v2.MouthUpperLeft, LipShape_v2.MouthLowerLeft}, true)}, - {"SmileSadRight", new PositiveNegativeShape(LipShape_v2.MouthSmileRight, LipShape_v2.MouthSadRight)}, - {"SmileSadLeft", new PositiveNegativeShape(LipShape_v2.MouthSmileLeft, LipShape_v2.MouthSadLeft)}, - {"SmileSad", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileLeft, LipShape_v2.MouthSmileRight}, new LipShape_v2[]{LipShape_v2.MouthSadLeft, LipShape_v2.MouthSadRight})}, - {"TongueY", new PositiveNegativeShape(LipShape_v2.TongueUp, LipShape_v2.TongueDown)}, - {"TongueX", new PositiveNegativeShape(LipShape_v2.TongueRight, LipShape_v2.TongueLeft)}, - {"PuffSuckRight", new PositiveNegativeShape(LipShape_v2.CheekPuffRight, LipShape_v2.CheekSuck)}, - {"PuffSuckLeft", new PositiveNegativeShape(LipShape_v2.CheekPuffLeft, LipShape_v2.CheekSuck)}, - {"PuffSuck", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.CheekPuffLeft, LipShape_v2.CheekPuffRight}, new LipShape_v2[]{LipShape_v2.CheekSuck}, true)}, + {"JawX", new PositiveNegativeShape(UnifiedExpression.JawRight, UnifiedExpression.JawLeft)}, + {"MouthUpper", new PositiveNegativeShape(UnifiedExpression.MouthUpperRight, UnifiedExpression.MouthUpperLeft)}, + {"MouthLower", new PositiveNegativeShape(UnifiedExpression.MouthLowerRight, UnifiedExpression.MouthLowerLeft)}, + {"MouthX", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperRight, UnifiedExpression.MouthLowerRight}, new UnifiedExpression[]{UnifiedExpression.MouthUpperLeft, UnifiedExpression.MouthLowerLeft}, true)}, + {"SmileSadRight", new PositiveNegativeShape(UnifiedExpression.MouthSmileRight, UnifiedExpression.MouthSadRight)}, + {"SmileSadLeft", new PositiveNegativeShape(UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthSadLeft)}, + {"SmileSad", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthSmileRight}, new UnifiedExpression[]{UnifiedExpression.MouthSadLeft, UnifiedExpression.MouthSadRight})}, + {"TongueY", new PositiveNegativeShape(UnifiedExpression.TongueUp, UnifiedExpression.TongueDown)}, + {"TongueX", new PositiveNegativeShape(UnifiedExpression.TongueRight, UnifiedExpression.TongueLeft)}, + {"PuffSuckRight", new PositiveNegativeShape(UnifiedExpression.CheekPuffRight, UnifiedExpression.CheekSuck)}, + {"PuffSuckLeft", new PositiveNegativeShape(UnifiedExpression.CheekPuffLeft, UnifiedExpression.CheekSuck)}, + {"PuffSuck", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.CheekPuffLeft, UnifiedExpression.CheekPuffRight}, new UnifiedExpression[]{UnifiedExpression.CheekSuck}, true)}, //Additional combined shapes created with the help of the VRCFT Discord! //JawOpen based params - {"JawOpenApe", new PositiveNegativeShape(LipShape_v2.JawOpen, LipShape_v2.MouthApeShape)}, - {"JawOpenPuff", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.JawOpen}, new LipShape_v2[]{LipShape_v2.CheekPuffLeft, LipShape_v2.CheekPuffRight})}, - {"JawOpenPuffRight", new PositiveNegativeShape(LipShape_v2.JawOpen, LipShape_v2.CheekPuffRight)}, - {"JawOpenPuffLeft", new PositiveNegativeShape(LipShape_v2.JawOpen, LipShape_v2.CheekPuffLeft)}, - {"JawOpenSuck", new PositiveNegativeShape(LipShape_v2.JawOpen, LipShape_v2.CheekSuck)}, - {"JawOpenForward", new PositiveNegativeShape(LipShape_v2.JawOpen, LipShape_v2.JawForward)}, - {"JawOpenOverlay", new PositiveNegativeShape(LipShape_v2.JawOpen, LipShape_v2.MouthLowerOverlay)}, + {"JawOpenApe", new PositiveNegativeShape(UnifiedExpression.JawOpen, UnifiedExpression.MouthApeShape)}, + {"JawOpenPuff", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.JawOpen}, new UnifiedExpression[]{UnifiedExpression.CheekPuffLeft, UnifiedExpression.CheekPuffRight})}, + {"JawOpenPuffRight", new PositiveNegativeShape(UnifiedExpression.JawOpen, UnifiedExpression.CheekPuffRight)}, + {"JawOpenPuffLeft", new PositiveNegativeShape(UnifiedExpression.JawOpen, UnifiedExpression.CheekPuffLeft)}, + {"JawOpenSuck", new PositiveNegativeShape(UnifiedExpression.JawOpen, UnifiedExpression.CheekSuck)}, + {"JawOpenForward", new PositiveNegativeShape(UnifiedExpression.JawOpen, UnifiedExpression.JawForward)}, + {"JawOpenOverlay", new PositiveNegativeShape(UnifiedExpression.JawOpen, UnifiedExpression.MouthLowerOverlay)}, //MouthUpperUpRight based params - {"MouthUpperUpRightUpperInside", new PositiveNegativeShape(LipShape_v2.MouthUpperUpRight, LipShape_v2.MouthUpperInside)}, - {"MouthUpperUpRightPuffRight", new PositiveNegativeShape(LipShape_v2.MouthUpperUpRight, LipShape_v2.CheekPuffRight)}, - {"MouthUpperUpRightApe", new PositiveNegativeShape(LipShape_v2.MouthUpperUpRight, LipShape_v2.MouthApeShape)}, - {"MouthUpperUpRightPout", new PositiveNegativeShape(LipShape_v2.MouthUpperUpRight, LipShape_v2.MouthPout)}, - {"MouthUpperUpRightOverlay", new PositiveNegativeShape(LipShape_v2.MouthUpperUpRight, LipShape_v2.MouthLowerOverlay)}, - {"MouthUpperUpRightSuck", new PositiveNegativeShape(LipShape_v2.MouthUpperUpRight, LipShape_v2.CheekSuck)}, + {"MouthUpperUpRightUpperInside", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpRight, UnifiedExpression.MouthUpperInside)}, + {"MouthUpperUpRightPuffRight", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpRight, UnifiedExpression.CheekPuffRight)}, + {"MouthUpperUpRightApe", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpRight, UnifiedExpression.MouthApeShape)}, + {"MouthUpperUpRightPout", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpRight, UnifiedExpression.MouthPout)}, + {"MouthUpperUpRightOverlay", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpRight, UnifiedExpression.MouthLowerOverlay)}, + {"MouthUpperUpRightSuck", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpRight, UnifiedExpression.CheekSuck)}, //MouthUpperUpLeft based params - {"MouthUpperUpLeftUpperInside", new PositiveNegativeShape(LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperInside)}, - {"MouthUpperUpLeftPuffLeft", new PositiveNegativeShape(LipShape_v2.MouthUpperUpLeft, LipShape_v2.CheekPuffLeft)}, - {"MouthUpperUpLeftApe", new PositiveNegativeShape(LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthApeShape)}, - {"MouthUpperUpLeftPout", new PositiveNegativeShape(LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthPout)}, - {"MouthUpperUpLeftOverlay", new PositiveNegativeShape(LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthLowerOverlay)}, - {"MouthUpperUpLeftSuck", new PositiveNegativeShape(LipShape_v2.MouthUpperUpLeft, LipShape_v2.CheekSuck)}, + {"MouthUpperUpLeftUpperInside", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperInside)}, + {"MouthUpperUpLeftPuffLeft", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.CheekPuffLeft)}, + {"MouthUpperUpLeftApe", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthApeShape)}, + {"MouthUpperUpLeftPout", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthPout)}, + {"MouthUpperUpLeftOverlay", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthLowerOverlay)}, + {"MouthUpperUpLeftSuck", new PositiveNegativeShape(UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.CheekSuck)}, // MouthUpperUp Left+Right base params - {"MouthUpperUpUpperInside", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.MouthUpperInside })}, - {"MouthUpperUpInside", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.MouthUpperInside, LipShape_v2.MouthLowerInside}, true)}, - {"MouthUpperUpPuff", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.CheekPuffLeft, LipShape_v2.CheekPuffRight})}, - {"MouthUpperUpPuffLeft", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.CheekPuffLeft})}, - {"MouthUpperUpPuffRight", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.CheekPuffRight})}, - {"MouthUpperUpApe", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.MouthApeShape})}, - {"MouthUpperUpPout", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.MouthPout})}, - {"MouthUpperUpOverlay", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.MouthLowerOverlay})}, - {"MouthUpperUpSuck", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthUpperUpLeft, LipShape_v2.MouthUpperUpRight}, new LipShape_v2[]{LipShape_v2.CheekSuck})}, + {"MouthUpperUpUpperInside", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.MouthUpperInside })}, + {"MouthUpperUpInside", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.MouthUpperInside, UnifiedExpression.MouthLowerInside}, true)}, + {"MouthUpperUpPuff", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.CheekPuffLeft, UnifiedExpression.CheekPuffRight})}, + {"MouthUpperUpPuffLeft", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.CheekPuffLeft})}, + {"MouthUpperUpPuffRight", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.CheekPuffRight})}, + {"MouthUpperUpApe", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.MouthApeShape})}, + {"MouthUpperUpPout", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.MouthPout})}, + {"MouthUpperUpOverlay", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.MouthLowerOverlay})}, + {"MouthUpperUpSuck", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthUpperUpLeft, UnifiedExpression.MouthUpperUpRight}, new UnifiedExpression[]{UnifiedExpression.CheekSuck})}, //MouthLowerDownRight based params - {"MouthLowerDownRightLowerInside", new PositiveNegativeShape(LipShape_v2.MouthLowerDownRight, LipShape_v2.MouthLowerInside)}, - {"MouthLowerDownRightPuffRight", new PositiveNegativeShape(LipShape_v2.MouthLowerDownRight, LipShape_v2.CheekPuffRight)}, - {"MouthLowerDownRightApe", new PositiveNegativeShape(LipShape_v2.MouthLowerDownRight, LipShape_v2.MouthApeShape)}, - {"MouthLowerDownRightPout", new PositiveNegativeShape(LipShape_v2.MouthLowerDownRight, LipShape_v2.MouthPout)}, - {"MouthLowerDownRightOverlay", new PositiveNegativeShape(LipShape_v2.MouthLowerDownRight, LipShape_v2.MouthLowerOverlay)}, - {"MouthLowerDownRightSuck", new PositiveNegativeShape(LipShape_v2.MouthLowerDownRight, LipShape_v2.CheekSuck)}, + {"MouthLowerDownRightLowerInside", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownRight, UnifiedExpression.MouthLowerInside)}, + {"MouthLowerDownRightPuffRight", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownRight, UnifiedExpression.CheekPuffRight)}, + {"MouthLowerDownRightApe", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownRight, UnifiedExpression.MouthApeShape)}, + {"MouthLowerDownRightPout", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownRight, UnifiedExpression.MouthPout)}, + {"MouthLowerDownRightOverlay", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownRight, UnifiedExpression.MouthLowerOverlay)}, + {"MouthLowerDownRightSuck", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownRight, UnifiedExpression.CheekSuck)}, //MouthLowerDownLeft based params - {"MouthLowerDownLeftLowerInside", new PositiveNegativeShape(LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerInside)}, - {"MouthLowerDownLeftPuffLeft", new PositiveNegativeShape(LipShape_v2.MouthLowerDownLeft, LipShape_v2.CheekPuffLeft)}, - {"MouthLowerDownLeftApe", new PositiveNegativeShape(LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthApeShape)}, - {"MouthLowerDownLeftPout", new PositiveNegativeShape(LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthPout)}, - {"MouthLowerDownLeftOverlay", new PositiveNegativeShape(LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerOverlay)}, - {"MouthLowerDownLeftSuck", new PositiveNegativeShape(LipShape_v2.MouthLowerDownLeft, LipShape_v2.CheekSuck)}, + {"MouthLowerDownLeftLowerInside", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerInside)}, + {"MouthLowerDownLeftPuffLeft", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.CheekPuffLeft)}, + {"MouthLowerDownLeftApe", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthApeShape)}, + {"MouthLowerDownLeftPout", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthPout)}, + {"MouthLowerDownLeftOverlay", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerOverlay)}, + {"MouthLowerDownLeftSuck", new PositiveNegativeShape(UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.CheekSuck)}, // MouthLowerDown Left+Right base params - {"MouthLowerDownLowerInside", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.MouthLowerInside})}, - {"MouthLowerDownInside", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.MouthUpperInside, LipShape_v2.MouthLowerInside}, true)}, - {"MouthLowerDownPuff", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.CheekPuffLeft, LipShape_v2.CheekPuffRight})}, - {"MouthLowerDownPuffLeft", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.CheekPuffLeft})}, - {"MouthLowerDownPuffRight", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.CheekPuffRight})}, - {"MouthLowerDownApe", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.MouthApeShape})}, - {"MouthLowerDownPout", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.MouthPout})}, - {"MouthLowerDownOverlay", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.MouthLowerOverlay})}, - {"MouthLowerDownSuck", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthLowerDownLeft, LipShape_v2.MouthLowerDownRight}, new LipShape_v2[]{LipShape_v2.CheekSuck})}, + {"MouthLowerDownLowerInside", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.MouthLowerInside})}, + {"MouthLowerDownInside", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.MouthUpperInside, UnifiedExpression.MouthLowerInside}, true)}, + {"MouthLowerDownPuff", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.CheekPuffLeft, UnifiedExpression.CheekPuffRight})}, + {"MouthLowerDownPuffLeft", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.CheekPuffLeft})}, + {"MouthLowerDownPuffRight", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.CheekPuffRight})}, + {"MouthLowerDownApe", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.MouthApeShape})}, + {"MouthLowerDownPout", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.MouthPout})}, + {"MouthLowerDownOverlay", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.MouthLowerOverlay})}, + {"MouthLowerDownSuck", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthLowerDownLeft, UnifiedExpression.MouthLowerDownRight}, new UnifiedExpression[]{UnifiedExpression.CheekSuck})}, // MouthInsideOverturn based params - {"MouthUpperInsideOverturn", new PositiveNegativeShape(LipShape_v2.MouthUpperInside, LipShape_v2.MouthUpperOverturn)}, - {"MouthLowerInsideOverturn", new PositiveNegativeShape(LipShape_v2.MouthLowerInside, LipShape_v2.MouthLowerOverturn)}, + {"MouthUpperInsideOverturn", new PositiveNegativeShape(UnifiedExpression.MouthUpperInside, UnifiedExpression.MouthUpperOverturn)}, + {"MouthLowerInsideOverturn", new PositiveNegativeShape(UnifiedExpression.MouthLowerInside, UnifiedExpression.MouthLowerOverturn)}, //SmileRight based params; Recommend using these if you already have SmileSadLeft setup! - {"SmileRightUpperOverturn", new PositiveNegativeShape(LipShape_v2.MouthSmileRight, LipShape_v2.MouthUpperOverturn)}, - {"SmileRightLowerOverturn", new PositiveNegativeShape(LipShape_v2.MouthSmileRight, LipShape_v2.MouthLowerOverturn)}, - {"SmileRightOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileRight}, new LipShape_v2[]{LipShape_v2.MouthUpperOverturn, LipShape_v2.MouthLowerOverturn})}, - {"SmileRightApe", new PositiveNegativeShape(LipShape_v2.MouthSmileRight, LipShape_v2.MouthApeShape)}, - {"SmileRightOverlay", new PositiveNegativeShape(LipShape_v2.MouthSmileRight, LipShape_v2.MouthLowerOverlay)}, - {"SmileRightPout", new PositiveNegativeShape(LipShape_v2.MouthSmileRight, LipShape_v2.MouthPout)}, + {"SmileRightUpperOverturn", new PositiveNegativeShape(UnifiedExpression.MouthSmileRight, UnifiedExpression.MouthUpperOverturn)}, + {"SmileRightLowerOverturn", new PositiveNegativeShape(UnifiedExpression.MouthSmileRight, UnifiedExpression.MouthLowerOverturn)}, + {"SmileRightOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileRight}, new UnifiedExpression[]{UnifiedExpression.MouthUpperOverturn, UnifiedExpression.MouthLowerOverturn})}, + {"SmileRightApe", new PositiveNegativeShape(UnifiedExpression.MouthSmileRight, UnifiedExpression.MouthApeShape)}, + {"SmileRightOverlay", new PositiveNegativeShape(UnifiedExpression.MouthSmileRight, UnifiedExpression.MouthLowerOverlay)}, + {"SmileRightPout", new PositiveNegativeShape(UnifiedExpression.MouthSmileRight, UnifiedExpression.MouthPout)}, //SmileLeft based params; Recommend using these if you already have SmileSadRight setup! - {"SmileLeftUpperOverturn", new PositiveNegativeShape(LipShape_v2.MouthSmileLeft, LipShape_v2.MouthUpperOverturn)}, - {"SmileLeftLowerOverturn", new PositiveNegativeShape(LipShape_v2.MouthSmileLeft, LipShape_v2.MouthLowerOverturn)}, - {"SmileLeftOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileLeft}, new LipShape_v2[]{LipShape_v2.MouthUpperOverturn, LipShape_v2.MouthLowerOverturn})}, - {"SmileLeftApe", new PositiveNegativeShape(LipShape_v2.MouthSmileLeft, LipShape_v2.MouthApeShape)}, - {"SmileLeftOverlay", new PositiveNegativeShape(LipShape_v2.MouthSmileLeft, LipShape_v2.MouthLowerOverlay)}, - {"SmileLeftPout", new PositiveNegativeShape(LipShape_v2.MouthSmileLeft, LipShape_v2.MouthPout)}, + {"SmileLeftUpperOverturn", new PositiveNegativeShape(UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthUpperOverturn)}, + {"SmileLeftLowerOverturn", new PositiveNegativeShape(UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthLowerOverturn)}, + {"SmileLeftOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileLeft}, new UnifiedExpression[]{UnifiedExpression.MouthUpperOverturn, UnifiedExpression.MouthLowerOverturn})}, + {"SmileLeftApe", new PositiveNegativeShape(UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthApeShape)}, + {"SmileLeftOverlay", new PositiveNegativeShape(UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthLowerOverlay)}, + {"SmileLeftPout", new PositiveNegativeShape(UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthPout)}, //Smile Left+Right based params - {"SmileUpperOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileLeft, LipShape_v2.MouthSmileRight}, new LipShape_v2[]{LipShape_v2.MouthUpperOverturn})}, - {"SmileLowerOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileLeft, LipShape_v2.MouthSmileRight}, new LipShape_v2[]{LipShape_v2.MouthLowerOverturn})}, - {"SmileOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileLeft, LipShape_v2.MouthSmileRight}, new LipShape_v2[]{LipShape_v2.MouthUpperOverturn, LipShape_v2.MouthLowerOverturn})}, - {"SmileApe", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileLeft, LipShape_v2.MouthSmileRight}, new LipShape_v2[]{LipShape_v2.MouthApeShape})}, - {"SmileOverlay", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileLeft, LipShape_v2.MouthSmileRight}, new LipShape_v2[]{LipShape_v2.MouthLowerOverlay})}, - {"SmilePout", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.MouthSmileLeft, LipShape_v2.MouthSmileRight}, new LipShape_v2[]{LipShape_v2.MouthPout})}, + {"SmileUpperOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthSmileRight}, new UnifiedExpression[]{UnifiedExpression.MouthUpperOverturn})}, + {"SmileLowerOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthSmileRight}, new UnifiedExpression[]{UnifiedExpression.MouthLowerOverturn})}, + {"SmileOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthSmileRight}, new UnifiedExpression[]{UnifiedExpression.MouthUpperOverturn, UnifiedExpression.MouthLowerOverturn})}, + {"SmileApe", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthSmileRight}, new UnifiedExpression[]{UnifiedExpression.MouthApeShape})}, + {"SmileOverlay", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthSmileRight}, new UnifiedExpression[]{UnifiedExpression.MouthLowerOverlay})}, + {"SmilePout", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.MouthSmileLeft, UnifiedExpression.MouthSmileRight}, new UnifiedExpression[]{UnifiedExpression.MouthPout})}, //CheekPuffRight based params - {"PuffRightUpperOverturn", new PositiveNegativeShape(LipShape_v2.CheekPuffRight, LipShape_v2.MouthUpperOverturn)}, - {"PuffRightLowerOverturn", new PositiveNegativeShape(LipShape_v2.CheekPuffRight, LipShape_v2.MouthLowerOverturn)}, - {"PuffRightOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.CheekPuffRight}, new LipShape_v2[]{LipShape_v2.MouthUpperOverturn, LipShape_v2.MouthLowerOverturn}, true)}, + {"PuffRightUpperOverturn", new PositiveNegativeShape(UnifiedExpression.CheekPuffRight, UnifiedExpression.MouthUpperOverturn)}, + {"PuffRightLowerOverturn", new PositiveNegativeShape(UnifiedExpression.CheekPuffRight, UnifiedExpression.MouthLowerOverturn)}, + {"PuffRightOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.CheekPuffRight}, new UnifiedExpression[]{UnifiedExpression.MouthUpperOverturn, UnifiedExpression.MouthLowerOverturn}, true)}, //CheekPuffLeft based params - {"PuffLeftUpperOverturn", new PositiveNegativeShape(LipShape_v2.CheekPuffLeft, LipShape_v2.MouthUpperOverturn)}, - {"PuffLeftLowerOverturn", new PositiveNegativeShape(LipShape_v2.CheekPuffLeft, LipShape_v2.MouthLowerOverturn)}, - {"PuffLeftOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.CheekPuffLeft}, new LipShape_v2[]{LipShape_v2.MouthUpperOverturn, LipShape_v2.MouthLowerOverturn}, true)}, + {"PuffLeftUpperOverturn", new PositiveNegativeShape(UnifiedExpression.CheekPuffLeft, UnifiedExpression.MouthUpperOverturn)}, + {"PuffLeftLowerOverturn", new PositiveNegativeShape(UnifiedExpression.CheekPuffLeft, UnifiedExpression.MouthLowerOverturn)}, + {"PuffLeftOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.CheekPuffLeft}, new UnifiedExpression[]{UnifiedExpression.MouthUpperOverturn, UnifiedExpression.MouthLowerOverturn}, true)}, //CheekPuff Left+Right based params - {"PuffUpperOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.CheekPuffRight, LipShape_v2.CheekPuffLeft}, new LipShape_v2[]{LipShape_v2.MouthUpperOverturn})}, - {"PuffLowerOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.CheekPuffRight, LipShape_v2.CheekPuffLeft}, new LipShape_v2[]{LipShape_v2.MouthLowerOverturn})}, - {"PuffOverturn", new PositiveNegativeAveragedShape(new LipShape_v2[]{LipShape_v2.CheekPuffRight, LipShape_v2.CheekPuffLeft}, new LipShape_v2[]{LipShape_v2.MouthUpperOverturn, LipShape_v2.MouthLowerOverturn}, true)}, + {"PuffUpperOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.CheekPuffRight, UnifiedExpression.CheekPuffLeft}, new UnifiedExpression[]{UnifiedExpression.MouthUpperOverturn})}, + {"PuffLowerOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.CheekPuffRight, UnifiedExpression.CheekPuffLeft}, new UnifiedExpression[]{UnifiedExpression.MouthLowerOverturn})}, + {"PuffOverturn", new PositiveNegativeAveragedShape(new UnifiedExpression[]{UnifiedExpression.CheekPuffRight, UnifiedExpression.CheekPuffLeft}, new UnifiedExpression[]{UnifiedExpression.MouthUpperOverturn, UnifiedExpression.MouthLowerOverturn}, true)}, //Combine both TongueSteps (-1 fully in, 0 on edge, 1 fully out) - {"TongueSteps", new PositiveNegativeShape(LipShape_v2.TongueLongStep1, LipShape_v2.TongueLongStep2, true)}, + {"TongueSteps", new PositiveNegativeShape(UnifiedExpression.TongueLongStep1, UnifiedExpression.TongueLongStep2, true)}, }; // Make a list called LipParameters containing the results from both GetOptimizedLipParameters and GetAllLipParameters, and add GetLipActivatedStatus public static readonly IParameter[] AllLipParameters = GetAllLipShapes().Union(GetOptimizedLipParameters()).Union(GetLipActivatedStatus()).ToArray(); - public static bool IsLipShapeName(string name) => MergedShapes.ContainsKey(name) || Enum.TryParse(name, out LipShape_v2 shape); + public static bool IsLipShapeName(string name) => MergedShapes.ContainsKey(name) || Enum.TryParse(name, out UnifiedExpression shape); private static IEnumerable GetOptimizedLipParameters() => MergedShapes .Select(shape => new EParam((eye, lip) => shape.Value.GetBlendedLipShape(lip.LatestShapes), shape.Key, 0.0f)); private static IEnumerable GetAllLipShapes() => - ((LipShape_v2[]) Enum.GetValues(typeof(LipShape_v2))).ToList().Select(shape => + ((UnifiedExpression[]) Enum.GetValues(typeof(UnifiedExpression))).ToList().Select(shape => new EParam((eye, lip) => lip.LatestShapes[(int)shape], shape.ToString(), 0.0f)); diff --git a/VRCFaceTracking/Params/Lip/LipShapes.cs b/VRCFaceTracking/Params/Lip/LipShapes.cs new file mode 100644 index 00000000..197b7d09 --- /dev/null +++ b/VRCFaceTracking/Params/Lip/LipShapes.cs @@ -0,0 +1,73 @@ +namespace VRCFaceTracking.Params.Lip +{ + // TODO: Create expressions map to relate legacy shapes to new ones + public enum UnifiedExpression + { + JawRight, // +JawX + JawLeft, // -JawX + JawForward, + JawOpen, + MouthApeShape, + MouthUpperRight, // +MouthUpper + MouthUpperLeft, // -MouthUpper + MouthLowerRight, // +MouthLower + MouthLowerLeft, // -MouthLower + MouthUpperOverturn, + MouthLowerOverturn, + MouthPout, + MouthSmileRight, // +SmileSadRight + MouthSmileLeft, // +SmileSadLeft + MouthSadRight, // -SmileSadRight + MouthSadLeft, // -SmileSadLeft + CheekPuffRight, + CheekPuffLeft, + CheekSuck, + MouthUpperUpRight, + MouthUpperUpLeft, + MouthLowerDownRight, + MouthLowerDownLeft, + MouthUpperInside, + MouthLowerInside, + MouthLowerOverlay, + TongueLongStep1, + TongueLongStep2, + TongueDown, // -TongueY + TongueUp, // +TongueY + TongueRight, // +TongueX + TongueLeft, // -TongueX + TongueRoll, + TongueUpLeftMorph, + TongueUpRightMorph, + TongueDownLeftMorph, + TongueDownRightMorph, + SRanipalMax, + + CheekRaiserLeft, + CheekRaiserRight, + ChinRaiserBottom, + ChinRaiserTop, + DimplerLeft, + DimplerRight, + LipFunnelerLeftBottom, + LipFunnelerLeftTop, + LipFunnelerRightBottom, + LipFunnelerRightTop, + LipPressorLeft, + LipPressorRight, + LipPuckerLeft, // Mouth Pout but not combined + LipPuckerRight, // Mouth Pout but not combined + LipStretcherLeft, + LipStretcherRight, + LipSuckLeftBottom, // Cheek Suck but not combined + LipSuckLeftTop, // Cheek Suck but not combined + LipSuckRightBottom, // Cheek Suck but not combined + LipSuckRightTop, // Cheek Suck but not combined + LipTightenerLeft, + LipTightenerRight, + LowerLipDepressorLeft, + LowerLipDepressorRight, + NoseWrinklerLeft, + NoseWrinklerRight, + Max + } +} \ No newline at end of file diff --git a/VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_LipData_v2.cs b/VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_LipData_v2.cs index de2a42a6..fc1ed4d8 100644 --- a/VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_LipData_v2.cs +++ b/VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_LipData_v2.cs @@ -8,48 +8,7 @@ namespace anipal { namespace Lip { - public enum LipShape_v2 - { - //None = -1, - JawRight = 0, // +JawX - JawLeft = 1, // -JawX - JawForward = 2, - JawOpen = 3, - MouthApeShape = 4, - MouthUpperRight = 5, // +MouthUpper - MouthUpperLeft = 6, // -MouthUpper - MouthLowerRight = 7, // +MouthLower - MouthLowerLeft = 8, // -MouthLower - MouthUpperOverturn = 9, - MouthLowerOverturn = 10, - MouthPout = 11, - MouthSmileRight = 12, // +SmileSadRight - MouthSmileLeft = 13, // +SmileSadLeft - MouthSadRight = 14, // -SmileSadRight - MouthSadLeft = 15, // -SmileSadLeft - CheekPuffRight = 16, - CheekPuffLeft = 17, - CheekSuck = 18, - MouthUpperUpRight = 19, - MouthUpperUpLeft = 20, - MouthLowerDownRight = 21, - MouthLowerDownLeft = 22, - MouthUpperInside = 23, - MouthLowerInside = 24, - MouthLowerOverlay = 25, - TongueLongStep1 = 26, - TongueLongStep2 = 32, - TongueDown = 30, // -TongueY - TongueUp = 29, // +TongueY - TongueRight = 28, // +TongueX - TongueLeft = 27, // -TongueX - TongueRoll = 31, - TongueUpLeftMorph = 34, - TongueUpRightMorph = 33, - TongueDownLeftMorph = 36, - TongueDownRightMorph = 35, - //Max = 37, - } + // LipShape_v2 => VRCFTLipShape moved to LipShapes [StructLayout(LayoutKind.Sequential)] public struct PredictionData_v2 diff --git a/VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_Lip_v2.cs b/VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_Lip_v2.cs index 24569289..e3c3fed3 100644 --- a/VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_Lip_v2.cs +++ b/VRCFaceTracking/TrackingLibs/SRanipal/Lip/SRanipal_Lip_v2.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Runtime.InteropServices; +using VRCFaceTracking.Params.Lip; namespace ViveSR { @@ -17,13 +18,13 @@ public class SRanipal_Lip_v2 public const int WeightingCount = 37; private static Error LastUpdateResult = Error.FAILED; public static LipData_v2 LipData; - private static Dictionary Weightings; + private static Dictionary Weightings; static SRanipal_Lip_v2() { LipData.image = Marshal.AllocCoTaskMem(ImageWidth * ImageHeight * ImageChannel); - Weightings = new Dictionary(); - for (int i = 0; i < WeightingCount; ++i) Weightings.Add((LipShape_v2)i, 0.0f); + Weightings = new Dictionary(); + for (int i = 0; i < WeightingCount; ++i) Weightings.Add((UnifiedExpression)i, 0.0f); } private static unsafe bool UpdateData() @@ -33,7 +34,7 @@ private static unsafe bool UpdateData() { for (int i = 0; i < WeightingCount; ++i) { - Weightings[(LipShape_v2)i] = LipData.prediction_data.blend_shape_weight[i]; + Weightings[(UnifiedExpression)i] = LipData.prediction_data.blend_shape_weight[i]; } } return LastUpdateResult == Error.WORK; @@ -44,7 +45,7 @@ private static unsafe bool UpdateData() /// /// Weighting values obtained from anipal's Lip module. /// Indicates whether the values received are new. - public static bool GetLipWeightingsAndImage(out Dictionary shapes, out IntPtr image) + public static bool GetLipWeightingsAndImage(out Dictionary shapes, out IntPtr image) { bool update = UpdateData(); shapes = Weightings; diff --git a/VRCFaceTracking/TrackingLibs/SRanipal/SRanipalTrackingInterface.cs b/VRCFaceTracking/TrackingLibs/SRanipal/SRanipalTrackingInterface.cs index a055ac1b..79db6134 100644 --- a/VRCFaceTracking/TrackingLibs/SRanipal/SRanipalTrackingInterface.cs +++ b/VRCFaceTracking/TrackingLibs/SRanipal/SRanipalTrackingInterface.cs @@ -1,222 +1,801 @@ using System; using System.Diagnostics; -using System.Runtime.InteropServices; +using System.IO; +using System.IO.MemoryMappedFiles; using System.Threading; -using ViveSR; -using ViveSR.anipal; -using ViveSR.anipal.Eye; -using ViveSR.anipal.Lip; -using VRCFaceTracking.Assets.UI; +using System.Runtime.InteropServices; +using VRCFaceTracking; +using VRCFaceTracking.Params; +using VRCFaceTracking.Params.Lip; + +using System.Net; +using System.Net.Sockets; +using System.Reflection; namespace VRCFaceTracking.SRanipal { - public class SRanipalExtTrackingInterface : ExtTrackingModule + // Treating this as a static class instead of an enum so we don't need to cast + public static class FBExpression { - LipData_v2 lipData = default; - EyeData_v2 eyeData = default; - private static CancellationTokenSource _cancellationToken; - - public override (bool SupportsEye, bool SupportsLip) Supported => (true, true); + public const int Brow_Lowerer_L = 0; + public const int Brow_Lowerer_R = 1; + public const int Cheek_Puff_L = 2; + public const int Cheek_Puff_R = 3; + public const int Cheek_Raiser_L = 4; + public const int Cheek_Raiser_R = 5; + public const int Cheek_Suck_L = 6; + public const int Cheek_Suck_R = 7; + public const int Chin_Raiser_B = 8; + public const int Chin_Raiser_T = 9; + public const int Dimpler_L = 10; + public const int Dimpler_R = 11; + public const int Eyes_Closed_L = 12; + public const int Eyes_Closed_R = 13; + public const int Eyes_Look_Down_L = 14; + public const int Eyes_Look_Down_R = 15; + public const int Eyes_Look_Left_L = 16; + public const int Eyes_Look_Left_R = 17; + public const int Eyes_Look_Right_L = 18; + public const int Eyes_Look_Right_R = 19; + public const int Eyes_Look_Up_L = 20; + public const int Eyes_Look_Up_R = 21; + public const int Inner_Brow_Raiser_L = 22; + public const int Inner_Brow_Raiser_R = 23; + public const int Jaw_Drop = 24; + public const int Jaw_Sideways_Left = 25; + public const int Jaw_Sideways_Right = 26; + public const int Jaw_Thrust = 27; + public const int Lid_Tightener_L = 28; + public const int Lid_Tightener_R = 29; + public const int Lip_Corner_Depressor_L = 30; + public const int Lip_Corner_Depressor_R = 31; + public const int Lip_Corner_Puller_L = 32; + public const int Lip_Corner_Puller_R = 33; + public const int Lip_Funneler_LB = 34; + public const int Lip_Funneler_LT = 35; + public const int Lip_Funneler_RB = 36; + public const int Lip_Funneler_RT = 37; + public const int Lip_Pressor_L = 38; + public const int Lip_Pressor_R = 39; + public const int Lip_Pucker_L = 40; + public const int Lip_Pucker_R = 41; + public const int Lip_Stretcher_L = 42; + public const int Lip_Stretcher_R = 43; + public const int Lip_Suck_LB = 44; + public const int Lip_Suck_LT = 45; + public const int Lip_Suck_RB = 46; + public const int Lip_Suck_RT = 47; + public const int Lip_Tightener_L = 48; + public const int Lip_Tightener_R = 49; + public const int Lips_Toward = 50; + public const int Lower_Lip_Depressor_L = 51; + public const int Lower_Lip_Depressor_R = 52; + public const int Mouth_Left = 53; + public const int Mouth_Right = 54; + public const int Nose_Wrinkler_L = 55; + public const int Nose_Wrinkler_R = 56; + public const int Outer_Brow_Raiser_L = 57; + public const int Outer_Brow_Raiser_R = 58; + public const int Upper_Lid_Raiser_L = 59; + public const int Upper_Lid_Raiser_R = 60; + public const int Upper_Lip_Raiser_L = 61; + public const int Upper_Lip_Raiser_R = 62; + public const int Max = 63; + } - public override (bool eyeSuccess, bool lipSuccess) Initialize(bool eye, bool lip) + public class FBData + { + [StructLayout(LayoutKind.Sequential)] + public struct AllData { - Error eyeError = Error.UNDEFINED, lipError = Error.UNDEFINED; + public EyeData eyeData; + public FaceData faceData; - if (eye) - // Only try to init if we're actually using the only headset that supports SRanipal eye tracking - eyeError = SRanipal_API.Initial(SRanipal_Eye_v2.ANIPAL_TYPE_EYE_V2, IntPtr.Zero); - - if (lip) - lipError = SRanipal_API.Initial(SRanipal_Lip_v2.ANIPAL_TYPE_LIP_V2, IntPtr.Zero); + public override string ToString() + { + return "EyeData: " + eyeData.ToString() + " FaceData: " + faceData.ToString(); + } + } - var (eyeEnabled, lipEnabled) = HandleSrErrors(eyeError, lipError); + [StructLayout(LayoutKind.Sequential)] + public struct EyeData + { + public Eye leftEye; + public Eye rightEye; - if (eyeEnabled && Utils.HasAdmin) + public override string ToString() { - var found = false; - int tries = 0; - while (!found && tries < 15) - { - tries++; - found = Attach(); - Thread.Sleep(250); - } - - if (found) - { - // Find the EyeCameraDevice.dll module inside sr_runtime, get it's offset and add hex 19190 to it for the image stream. - foreach (ProcessModule module in _process.Modules) - if (module.ModuleName == "EyeCameraDevice.dll") - _offset = module.BaseAddress + (_process.MainModule.FileVersionInfo.FileVersion == "1.3.2.0" ? 0x19190 : 0x19100); - - UnifiedTrackingData.LatestEyeData.SupportsImage = true; - UnifiedTrackingData.LatestEyeData.ImageSize = (200, 100); - } + return "LeftEye: " + leftEye.ToString() + " RightData: " + rightEye.ToString(); } - - if (lipEnabled) + } + + [StructLayout(LayoutKind.Sequential)] + public struct Eye + { + public float confidence; + public Vec3 position; + public Quat rotation; + + public override string ToString() { - UnifiedTrackingData.LatestLipData.SupportsImage = true; - UnifiedTrackingData.LatestLipData.ImageSize = (SRanipal_Lip_v2.ImageWidth, SRanipal_Lip_v2.ImageHeight); - UnifiedTrackingData.LatestLipData.ImageData = new byte[UnifiedTrackingData.LatestLipData.ImageSize.x * - UnifiedTrackingData.LatestLipData.ImageSize.y]; - lipData.image = Marshal.AllocCoTaskMem(UnifiedTrackingData.LatestLipData.ImageSize.x * - UnifiedTrackingData.LatestLipData.ImageSize.y); + return "Confidence: " + confidence + ", Position: " + position.ToString() + ", Rotation: " + rotation.ToString(); } - - return (eyeEnabled, lipEnabled); } - private static (bool eyeSuccess, bool lipSuccess) HandleSrErrors(Error eyeError, Error lipError) + [StructLayout(LayoutKind.Sequential)] + public struct Vec3 { - bool eyeEnabled = false, lipEnabled = false; - - if (eyeError == Error.WORK) - eyeEnabled = true; + public float x; + public float y; + public float z; - if (lipError == Error.FOXIP_SO) - while (lipError == Error.FOXIP_SO) - lipError = SRanipal_API.Initial(SRanipal_Lip_v2.ANIPAL_TYPE_LIP_V2, IntPtr.Zero); - - if (lipError == Error.WORK) - lipEnabled = true; + public override string ToString() + { + return $"Vector3: X: {x}, Y: {y}, Z: {z})"; + } + } - return (eyeEnabled, lipEnabled); + [StructLayout(LayoutKind.Sequential)] + public struct Quat + { + public float x; + public float y; + public float z; + public float w; + + public override string ToString() + { + return $"Quaternion: X: {x}, Y: {y}, Z: {z}, W: {w})"; + } } - - public override void Teardown() + + [StructLayout(LayoutKind.Sequential)] + public struct FaceData { - _cancellationToken.Cancel(); - _cancellationToken.Dispose(); - - Thread.Sleep(2000); + public float FaceRegionConfidenceLower; + public float FaceRegionConfidenceUpper; + public float BrowLowererL; + public float BrowLowererR; + public float CheekPuffL; + public float CheekPuffR; + public float CheekRaiserL; + public float CheekRaiserR; + public float CheekSuckL; + public float CheekSuckR; + public float ChinRaiserB; + public float ChinRaiserT; + public float DimplerL; + public float DimplerR; + public float EyesClosedL; + public float EyesClosedR; + public float EyesLookDownL; + public float EyesLookDownR; + public float EyesLookLeftL; + public float EyesLookLeftR; + public float EyesLookRightL; + public float EyesLookRightR; + public float EyesLookUpL; + public float EyesLookUpR; + public float InnerBrowRaiserL; + public float InnerBrowRaiserR; + public float JawDrop; + public float JawSidewaysLeft; + public float JawSidewaysRight; + public float JawThrust; + public float LidTightenerL; + public float LidTightenerR; + public float LipCornerDepressorL; + public float LipCornerDepressorR; + public float LipCornerPullerL; + public float LipCornerPullerR; + public float LipFunnelerLB; + public float LipFunnelerLT; + public float LipFunnelerRB; + public float LipFunnelerRT; + public float LipPressorL; + public float LipPressorR; + public float LipPuckerL; + public float LipPuckerR; + public float LipStretcherL; + public float LipStretcherR; + public float LipSuckLB; + public float LipSuckLT; + public float LipSuckRB; + public float LipSuckRT; + public float LipTightenerL; + public float LipTightenerR; + public float LipsToward; + public float LowerLipDepressorL; + public float LowerLipDepressorR; + public float MouthLeft; + public float MouthRight; + public float NoseWrinklerL; + public float NoseWrinklerR; + public float OuterBrowRaiserL; + public float OuterBrowRaiserR; + public float UpperLidRaiserL; + public float UpperLidRaiserR; + public float UpperLipRaiserL; + public float UpperLipRaiserR; - if (Status.EyeState > ModuleState.Uninitialized) + public override string ToString() { - Logger.Msg("Teardown: Releasing Eye"); - // Attempt to release this module and give up after 10 seconds because Vive Moment - var killThread = new Thread(() => SRanipal_API.Release(SRanipal_Eye_v2.ANIPAL_TYPE_EYE_V2)); - killThread.Start(); - if (!killThread.Join(new TimeSpan(0, 0, 5))) - { - killThread.Abort(); - if (killThread.IsAlive) - killThread.Interrupt(); - } + return + "FaceData: " + + "\nFaceRegionConfidenceLower: " + FaceRegionConfidenceLower + + "\nFaceRegionConfidenceUpper: " + FaceRegionConfidenceUpper + + "\nBrowLowererL: " + BrowLowererL + + "\nBrowLowererR: " + BrowLowererR + + "\nCheekPuffL: " + CheekPuffL + + "\nCheekPuffR: " + CheekPuffR + + "\nCheekRaiserL: " + CheekRaiserL + + "\nCheekRaiserR: " + CheekRaiserR + + "\nCheekSuckL: " + CheekSuckL + + "\nCheekSuckR: " + CheekSuckR + + "\nChinRaiserB: " + ChinRaiserB + + "\nChinRaiserT: " + ChinRaiserT + + "\nDimplerL: " + DimplerL + + "\nDimplerR: " + DimplerR + + "\nEyesClosedL: " + EyesClosedL + + "\nEyesClosedR: " + EyesClosedR + + "\nEyesLookDownL: " + EyesLookDownL + + "\nEyesLookDownR: " + EyesLookDownR + + "\nEyesLookLeftL: " + EyesLookLeftL + + "\nEyesLookLeftR: " + EyesLookLeftR + + "\nEyesLookRightL: " + EyesLookRightL + + "\nEyesLookRightR: " + EyesLookRightR + + "\nEyesLookUpL: " + EyesLookUpL + + "\nEyesLookUpR: " + EyesLookUpR + + "\nInnerBrowRaiserL: " + InnerBrowRaiserL + + "\nInnerBrowRaiserR: " + InnerBrowRaiserR + + "\nJawDrop: " + JawDrop + + "\nJawSidewaysLeft: " + JawSidewaysLeft + + "\nJawSidewaysRight: " + JawSidewaysRight + + "\nJawThrust: " + JawThrust + + "\nLidTightenerL: " + LidTightenerL + + "\nLidTightenerR: " + LidTightenerR + + "\nLipCornerDepressorL: " + LipCornerDepressorL + + "\nLipCornerDepressorR: " + LipCornerDepressorR + + "\nLipCornerPullerL: " + LipCornerPullerL + + "\nLipCornerPullerR: " + LipCornerPullerR + + "\nLipFunnelerLB: " + LipFunnelerLB + + "\nLipFunnelerLT: " + LipFunnelerLT + + "\nLipFunnelerRB: " + LipFunnelerRB + + "\nLipFunnelerRT: " + LipFunnelerRT + + "\nLipPressorL: " + LipPressorL + + "\nLipPressorR: " + LipPressorR + + "\nLipPuckerL: " + LipPuckerL + + "\nLipPuckerR: " + LipPuckerR + + "\nLipStretcherL: " + LipStretcherL + + "\nLipStretcherR: " + LipStretcherR + + "\nLipSuckLB: " + LipSuckLB + + "\nLipSuckLT: " + LipSuckLT + + "\nLipSuckRB: " + LipSuckRB + + "\nLipSuckRT: " + LipSuckRT + + "\nLipTightenerL: " + LipTightenerL + + "\nLipTightenerR: " + LipTightenerR + + "\nLipsToward: " + LipsToward + + "\nLowerLipDepressorL: " + LowerLipDepressorL + + "\nLowerLipDepressorR: " + LowerLipDepressorR + + "\nMouthLeft: " + MouthLeft + + "\nMouthRight: " + MouthRight + + "\nNoseWrinklerL: " + NoseWrinklerL + + "\nNoseWrinklerR: " + NoseWrinklerR + + "\nOuterBrowRaiserL: " + OuterBrowRaiserL + + "\nOuterBrowRaiserR: " + OuterBrowRaiserR + + "\nUpperLidRaiserL: " + UpperLidRaiserL + + "\nUpperLidRaiserR: " + UpperLidRaiserR + + "\nUpperLipRaiserL: " + UpperLipRaiserL + + "\nUpperLipRaiserR: " + UpperLipRaiserR; + } + } + } + + public class SRanipalExtTrackingInterface : ExtTrackingModule + { + public IPAddress localAddr; + public int PORT = 13191; + + private TcpClient client; + private NetworkStream stream; + private bool connected = false; + + private const int expressionsSize = 63; + private byte[] rawExpressions = new byte[expressionsSize * 4 + (8 * 2 * 4)]; + private float[] expressions = new float[expressionsSize + (8 * 2)]; + + private double pitch_L, yaw_L, pitch_R, yaw_R; // eye rotations + + public override (bool SupportsEye, bool SupportsLip) Supported => (true, true); + + public override (bool eyeSuccess, bool lipSuccess) Initialize(bool eye, bool lip) + { + string configPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "questProIP.txt"); + if (!File.Exists(configPath)) + { + Logger.Msg("Failed to find config JSON! Please maker sure it is present in the same directory as the DLL."); + return (false, false); } - if (Status.LipState > ModuleState.Uninitialized) + string text = File.ReadAllText(configPath).Trim(); + + if (!IPAddress.TryParse(text, out localAddr)) { - Logger.Msg("Teardown: Releasing Lip"); - // Same for lips - var killThread = new Thread(() => SRanipal_API.Release(SRanipal_Lip_v2.ANIPAL_TYPE_LIP_V2)); - killThread.Start(); - if (!killThread.Join(new TimeSpan(0,0,5))) - killThread.Abort(); + Logger.Error("The IP provided in questProIP.txt is not valid. Please check the file and try again."); + return (false, false); } + + ConnectToTCP(); + + Logger.Msg("ALXR handshake successful! Data will be broadcast to VRCFaceTracking."); + return (true, true); } - #region Update + private bool ConnectToTCP() + { + try + { + client = new TcpClient(); + Logger.Msg($"Trying to establish a Quest Pro connection at {localAddr}:{PORT}..."); + + client.Connect(localAddr, PORT); + Logger.Msg("Connected to Quest Pro!"); + + stream = client.GetStream(); + connected = true; + + return true; + } + catch (Exception e) + { + Logger.Error(e.Message); + return false; + } + } - public override Action GetUpdateThreadFunc() { - _cancellationToken = new CancellationTokenSource(); return () => { - while (!_cancellationToken.IsCancellationRequested) + while (true) + { + Update(); + //Thread.Sleep(10); // blocked by IO + } + }; + } + + private void Update() + { + try + { + // Attempt reconnection if needed + if (!connected || stream == null) + { + ConnectToTCP(); + } + + if (stream == null) + { + Logger.Warning("Can't read from network stream just yet! Trying again soon..."); + return; + } + + if (!stream.CanRead) + { + Logger.Warning("Can't read from network stream just yet! Trying again soon..."); + return; + } + + int offset = 0; + int readBytes; + do { - if (Status.LipState == ModuleState.Active && UpdateMouth() != Error.WORK) + readBytes = stream.Read(rawExpressions, offset, rawExpressions.Length - offset); + offset += readBytes; + } + while (readBytes > 0 && offset < rawExpressions.Length); + + if (offset < rawExpressions.Length && connected) + { + // TODO Reconnect to the server if we lose connection + Logger.Warning("End of stream! Reconnecting..."); + Thread.Sleep(1000); + connected = false; + try { - Logger.Msg("An error occured while getting lip data. This might be a wireless crash."); - Logger.Msg("Waiting 30 seconds before reinitializing to account for wireless users."); - Thread.Sleep(30000); - UnifiedLibManager.Initialize(); - return; + stream.Close(); } - - if (Status.EyeState == ModuleState.Active && UpdateEye() != Error.WORK) + catch (SocketException e) { - Logger.Msg("An error occured while getting eye data. This might be a wireless crash."); - Logger.Msg("Waiting 30 seconds before reinitializing to account for wireless users."); - Thread.Sleep(30000); - UnifiedLibManager.Initialize(); - return; + Logger.Error(e.Message); + Thread.Sleep(1000); } - - Thread.Sleep(10); } - }; - } - private static Process _process; - private static IntPtr _processHandle; - private IntPtr _offset; + // We receive information from the stream as a byte array 63*4 bytes long, since floats are 32 bits long and we have 63 expressions. + // We then need to convert this byte array to a float array. Thankfully, this can all be done in a single line of code. + Buffer.BlockCopy(rawExpressions, 0, expressions, 0, expressionsSize * 4 + (8 * 2 * 4)); - private static bool Attach() - { - if (Process.GetProcessesByName("sr_runtime").Length <= 0) return false; - _process = Process.GetProcessesByName("sr_runtime")[0]; - _processHandle = - Utils.OpenProcess(Utils.PROCESS_VM_READ, - false, _process.Id); - return true; - } + + // temp + //Logger.Msg(" "); + //Logger.Msg("Inner_Brow_Raiser_R: " + expressions[FBExpression.Inner_Brow_Raiser_R].ToString()); + //Logger.Msg("Outer_Brow_Raiser_R: " + expressions[FBExpression.Outer_Brow_Raiser_R].ToString()); + //Logger.Msg("Brow_Lowerer_R: " + expressions[FBExpression.Brow_Lowerer_R].ToString()); + //Logger.Msg(expressions[FBExpression.Brow_Lowerer_R].ToString()); + + //Logger.Msg("DownPre: " + expressions[FBExpression.Eyes_Look_Down_L].ToString()); + //Logger.Msg("UpPre: " + expressions[FBExpression.Eyes_Look_Up_L].ToString()); + //Logger.Msg("ClosedPre: " + expressions[FBExpression.Eyes_Closed_L].ToString()); + + double q_x = expressions[64]; + double q_y = expressions[65]; + double q_z = expressions[66]; + double q_w = expressions[67]; + + double yaw = Math.Atan2(2.0 * (q_y * q_z + q_w * q_x), q_w * q_w - q_x * q_x - q_y * q_y + q_z * q_z); + double pitch = Math.Asin(-2.0 * (q_x * q_z - q_w * q_y)); + //double roll = Math.Atan2(2.0 * (q_x * q_y + q_w * q_z), q_w * q_w + q_x * q_x - q_y * q_y - q_z * q_z); - private static byte[] ReadMemory(IntPtr offset, int size) { - var buffer = new byte[size]; + pitch_L = (180.0 / Math.PI) * pitch; // from radians + yaw_L = (180.0 / Math.PI) * yaw; - var bytesRead = 0; - Utils.ReadProcessMemory((int) _processHandle, offset, buffer, size, ref bytesRead); + q_x = expressions[72]; + q_y = expressions[73]; + q_z = expressions[74]; + q_w = expressions[75]; - return bytesRead != size ? null : buffer; + yaw = Math.Atan2(2.0 * (q_y * q_z + q_w * q_x), q_w * q_w - q_x * q_x - q_y * q_y + q_z * q_z); + pitch = Math.Asin(-2.0 * (q_x * q_z - q_w * q_y)); + + pitch_R = (180.0 / Math.PI) * pitch; // from radians + yaw_R = (180.0 / Math.PI) * yaw; + + //Logger.Msg(expressions[63].ToString()); // confidence + //Logger.Msg(((180.0 / Math.PI) * pitch).ToString()); + //Logger.Msg(((180.0 / Math.PI) * yaw).ToString()); + //Logger.Msg(((180.0 / Math.PI) * roll).ToString()); + + //Logger.Msg(expressions[68].ToString()); + //Logger.Msg(expressions[69].ToString()); + //Logger.Msg(expressions[70].ToString()); + + //Logger.Msg(pitch_L.ToString()); + //Logger.Msg(yaw_L.ToString()); + + //Logger.Msg(pitch_R.ToString()); + //Logger.Msg(yaw_R.ToString()); + + //Logger.Msg(expressions[69].ToString()); + //Logger.Msg(expressions[70].ToString()); + //Logger.Msg(expressions[71].ToString()); + + + //Logger.Msg(expressions[63].ToString()); + //Logger.Msg(expressions[64].ToString()); + //Logger.Msg(expressions[65].ToString()); + //Logger.Msg(expressions[66].ToString()); + //Logger.Msg(expressions[67].ToString()); + //Logger.Msg(expressions[68].ToString()); + //Logger.Msg(expressions[69].ToString()); + + PrepareUpdate(); + UpdateExpressions(); + } + catch (SocketException e) + { + Logger.Error(e.Message); + Thread.Sleep(1000); + } } - - private Error UpdateEye() + + + // Preprocess our expressions per the Meta Documentation + private void PrepareUpdate() { - var updateResult = SRanipal_Eye_API.GetEyeData_v2(ref eyeData); - UnifiedTrackingData.LatestEyeData.UpdateData(eyeData); - - if (!MainWindow.IsEyePageVisible || _processHandle == IntPtr.Zero || !UnifiedTrackingData.LatestEyeData.SupportsImage) return updateResult; - - // Read 20000 image bytes from the predefined offset. 10000 bytes per eye. - var imageBytes = ReadMemory(_offset, 20000); + //expressions[FBExpression.Eyes_Closed_L] = (( + // (expressions[FBExpression.Eyes_Closed_L] - + // Math.Min(expressions[FBExpression.Eyes_Look_Down_L], + // expressions[FBExpression.Eyes_Look_Down_R])))); // temporary + //expressions[FBExpression.Eyes_Closed_R] = (( + // (expressions[FBExpression.Eyes_Closed_R] - + // Math.Min(expressions[FBExpression.Eyes_Look_Down_L], + // expressions[FBExpression.Eyes_Look_Down_R])))); // temporary + + if(expressions[FBExpression.Eyes_Look_Down_L] == expressions[FBExpression.Eyes_Look_Up_L] && expressions[FBExpression.Eyes_Closed_L] > 0.25f) + { // wonky eyelid case, eyes are actually closed now + expressions[FBExpression.Eyes_Closed_L] = 0; //0.9f - (expressions[FBExpression.Lid_Tightener_L] * 3); + } + else + { + expressions[FBExpression.Eyes_Closed_L] = 0.9f - ((expressions[FBExpression.Eyes_Closed_L] * 3) / (1+ expressions[FBExpression.Eyes_Look_Down_L] * 3)); + } + + if (expressions[FBExpression.Eyes_Look_Down_R] == expressions[FBExpression.Eyes_Look_Up_R] && expressions[FBExpression.Eyes_Closed_R] > 0.25f) + { // wonky eyelid case, eyes are actually closed now + expressions[FBExpression.Eyes_Closed_R] = 0; //0.9f - (expressions[FBExpression.Lid_Tightener_R] * 3); + } + else + { + expressions[FBExpression.Eyes_Closed_R] = 0.9f - ((expressions[FBExpression.Eyes_Closed_R] * 3) / (1 + expressions[FBExpression.Eyes_Look_Down_R] * 3)); + } + + //expressions[FBExpression.Lid_Tightener_L] = 0.8f-expressions[FBExpression.Eyes_Closed_L]; // sad: fix combined param instead + //expressions[FBExpression.Lid_Tightener_R] = 0.8f-expressions[FBExpression.Eyes_Closed_R]; // sad: fix combined param instead + + if (1 - expressions[FBExpression.Eyes_Closed_L] < expressions[FBExpression.Lid_Tightener_L]) + expressions[FBExpression.Lid_Tightener_L] = (1 - expressions[FBExpression.Eyes_Closed_L]) - 0.01f; + + if (1 - expressions[FBExpression.Eyes_Closed_R] < expressions[FBExpression.Lid_Tightener_R]) + expressions[FBExpression.Lid_Tightener_R] = (1 - expressions[FBExpression.Eyes_Closed_R]) - 0.01f; + + //expressions[FBExpression.Lid_Tightener_L] = Math.Max(0, expressions[FBExpression.Lid_Tightener_L] - 0.15f); + //expressions[FBExpression.Lid_Tightener_R] = Math.Max(0, expressions[FBExpression.Lid_Tightener_R] - 0.15f); + + expressions[FBExpression.Upper_Lid_Raiser_L] = Math.Max(0, expressions[FBExpression.Upper_Lid_Raiser_L] - 0.5f); + expressions[FBExpression.Upper_Lid_Raiser_R] = Math.Max(0, expressions[FBExpression.Upper_Lid_Raiser_R] - 0.5f); + + expressions[FBExpression.Lid_Tightener_L] = Math.Max(0, expressions[FBExpression.Lid_Tightener_L] - 0.5f); + expressions[FBExpression.Lid_Tightener_R] = Math.Max(0, expressions[FBExpression.Lid_Tightener_R] - 0.5f); + + expressions[FBExpression.Inner_Brow_Raiser_L] = Math.Min(1, expressions[FBExpression.Inner_Brow_Raiser_L] * 3f);// * 4; + expressions[FBExpression.Brow_Lowerer_L] = Math.Min(1, expressions[FBExpression.Brow_Lowerer_L] * 3f);// * 4; + expressions[FBExpression.Outer_Brow_Raiser_L] = Math.Min(1, expressions[FBExpression.Outer_Brow_Raiser_L] * 3f);// * 4; + + expressions[FBExpression.Inner_Brow_Raiser_R] = Math.Min(1, expressions[FBExpression.Inner_Brow_Raiser_R] * 3f);// * 4; + expressions[FBExpression.Brow_Lowerer_R] = Math.Min(1, expressions[FBExpression.Brow_Lowerer_R] * 3f);// * 4; + expressions[FBExpression.Outer_Brow_Raiser_R] = Math.Min(1, expressions[FBExpression.Outer_Brow_Raiser_R] * 3f);// * 4; + - // Concatenate the two images side by side instead of one after the other - byte[] leftEye = new byte[10000]; - Array.Copy(imageBytes, 0, leftEye, 0, 10000); - byte[] rightEye = new byte[10000]; - Array.Copy(imageBytes, 10000, rightEye, 0, 10000); + expressions[FBExpression.Eyes_Look_Up_L] = expressions[FBExpression.Eyes_Look_Up_L] * 0.55f; + expressions[FBExpression.Eyes_Look_Up_R] = expressions[FBExpression.Eyes_Look_Up_R] * 0.55f; + expressions[FBExpression.Eyes_Look_Down_L] = expressions[FBExpression.Eyes_Look_Down_L] * 1.5f; + expressions[FBExpression.Eyes_Look_Down_R] = expressions[FBExpression.Eyes_Look_Down_R] * 1.5f; + + expressions[FBExpression.Eyes_Look_Left_L] = expressions[FBExpression.Eyes_Look_Left_L] * 0.85f; + expressions[FBExpression.Eyes_Look_Right_L] = expressions[FBExpression.Eyes_Look_Right_L] * 0.85f; + expressions[FBExpression.Eyes_Look_Left_R] = expressions[FBExpression.Eyes_Look_Left_R] * 0.85f; + expressions[FBExpression.Eyes_Look_Right_R] = expressions[FBExpression.Eyes_Look_Right_R] * 0.85f; + - for (var i = 0; i < 100; i++) // 100 lines of 200 bytes + + + // hack: turn rots to looks + //pitch = 29(left)-- > -29(right) + //yaw = -27(down)-- > 27(up) + + float sranimul = 0.75f; + + if (pitch_L > 0) { - // Add 100 bytes from the left eye to the left side of the image - int leftIndex = i * 100 * 2; - Array.Copy(leftEye,i*100, imageBytes, leftIndex, 100); + expressions[FBExpression.Eyes_Look_Left_L] = Math.Min(1, (float)(pitch_L / 29.0)) * sranimul; + expressions[FBExpression.Eyes_Look_Right_L] = 0; + } + else + { + expressions[FBExpression.Eyes_Look_Left_L] = 0; + expressions[FBExpression.Eyes_Look_Right_L] = Math.Min(1, (float)((-pitch_L) / 29.0)) * sranimul; + } + if(yaw_L > 0) + { + expressions[FBExpression.Eyes_Look_Up_L] = Math.Min(1, (float)(yaw_L / 27.0)) * sranimul; + expressions[FBExpression.Eyes_Look_Down_L] = 0; + } + else + { + expressions[FBExpression.Eyes_Look_Up_L] = 0; + expressions[FBExpression.Eyes_Look_Down_L] = Math.Min(1, (float)((-yaw_L) / 27.0)) * sranimul; + } - // Add 100 bytes from the right eye to the right side of the image - Array.Copy(rightEye, i*100, imageBytes, leftIndex + 100, 100); + + if (pitch_R > 0) + { + expressions[FBExpression.Eyes_Look_Left_R] = Math.Min(1, (float)(pitch_R / 29.0)) * sranimul; + expressions[FBExpression.Eyes_Look_Right_R] = 0; + } + else + { + expressions[FBExpression.Eyes_Look_Left_R] = 0; + expressions[FBExpression.Eyes_Look_Right_R] = Math.Min(1, (float)((-pitch_R) / 29.0)) * sranimul; + } + if (yaw_R > 0) + { + expressions[FBExpression.Eyes_Look_Up_R] = Math.Min(1, (float)(yaw_R / 27.0)) * sranimul; + expressions[FBExpression.Eyes_Look_Down_R] = 0; + } + else + { + expressions[FBExpression.Eyes_Look_Up_R] = 0; + expressions[FBExpression.Eyes_Look_Down_R] = Math.Min(1, (float)((-yaw_R) / 27.0)) * sranimul; } - // Write the image to the latest eye data - UnifiedTrackingData.LatestEyeData.ImageData = imageBytes; - return updateResult; } - private Error UpdateMouth() + // Thank you @adjerry on the VRCFT discord for these conversions! https://docs.google.com/spreadsheets/d/118jo960co3Mgw8eREFVBsaJ7z0GtKNr52IB4Bz99VTA/edit#gid=0 + private void UpdateExpressions() { - var updateResult = SRanipal_Lip_API.GetLipData_v2(ref lipData); - UnifiedTrackingData.LatestLipData.UpdateData(lipData); - - if (!MainWindow.IsLipPageVisible || lipData.image == IntPtr.Zero || !UnifiedTrackingData.LatestLipData.SupportsImage) return updateResult; - - Marshal.Copy(lipData.image, UnifiedTrackingData.LatestLipData.ImageData, 0, UnifiedTrackingData.LatestLipData.ImageSize.x * - UnifiedTrackingData.LatestLipData.ImageSize.y); + UnifiedTrackingData.LatestEyeData.Left = MakeEye + ( + LookLeft: expressions[FBExpression.Eyes_Look_Left_L], + LookRight: expressions[FBExpression.Eyes_Look_Right_L], + LookUp: expressions[FBExpression.Eyes_Look_Up_L], + LookDown: expressions[FBExpression.Eyes_Look_Down_L], + Openness: expressions[FBExpression.Eyes_Closed_L], + Squint: expressions[FBExpression.Lid_Tightener_L], + Squeeze: expressions[FBExpression.Lid_Tightener_L], + Widen: expressions[FBExpression.Upper_Lid_Raiser_L], + InnerUp: expressions[FBExpression.Inner_Brow_Raiser_L], + InnerDown: expressions[FBExpression.Brow_Lowerer_L], + OuterUp: expressions[FBExpression.Outer_Brow_Raiser_L], + OuterDown: expressions[FBExpression.Brow_Lowerer_L] + ); + + UnifiedTrackingData.LatestEyeData.Right = MakeEye + ( + LookLeft: expressions[FBExpression.Eyes_Look_Left_R], + LookRight: expressions[FBExpression.Eyes_Look_Right_R], + LookUp: expressions[FBExpression.Eyes_Look_Up_R], + LookDown: expressions[FBExpression.Eyes_Look_Down_R], + Openness: expressions[FBExpression.Eyes_Closed_R], + Squint: expressions[FBExpression.Lid_Tightener_R], + Squeeze: expressions[FBExpression.Lid_Tightener_R], + Widen: expressions[FBExpression.Upper_Lid_Raiser_R], + InnerUp: expressions[FBExpression.Inner_Brow_Raiser_R], + InnerDown: expressions[FBExpression.Brow_Lowerer_R], + OuterUp: expressions[FBExpression.Outer_Brow_Raiser_R], + OuterDown: expressions[FBExpression.Brow_Lowerer_R] + ); + + + + UnifiedTrackingData.LatestEyeData.Combined = MakeEye + ( + LookLeft: (expressions[FBExpression.Eyes_Look_Left_R]+expressions[FBExpression.Eyes_Look_Left_R])/2.0f, + LookRight: (expressions[FBExpression.Eyes_Look_Right_R]+expressions[FBExpression.Eyes_Look_Right_R]) / 2.0f, + LookUp: (expressions[FBExpression.Eyes_Look_Up_R]+expressions[FBExpression.Eyes_Look_Up_R]) / 2.0f, + LookDown: (expressions[FBExpression.Eyes_Look_Down_R]+expressions[FBExpression.Eyes_Look_Down_R]) / 2.0f, + Openness: (expressions[FBExpression.Eyes_Closed_R]+expressions[FBExpression.Eyes_Closed_R]) / 2.0f, + Squint: (expressions[FBExpression.Lid_Tightener_R]+expressions[FBExpression.Lid_Tightener_R]) / 2.0f, + Squeeze: (expressions[FBExpression.Lid_Tightener_R]+expressions[FBExpression.Lid_Tightener_R]) / 2.0f, + Widen: (expressions[FBExpression.Upper_Lid_Raiser_R]+expressions[FBExpression.Upper_Lid_Raiser_R]) / 2.0f, + InnerUp: (expressions[FBExpression.Inner_Brow_Raiser_R]+expressions[FBExpression.Inner_Brow_Raiser_R]) / 2.0f, + InnerDown: (expressions[FBExpression.Brow_Lowerer_R]+expressions[FBExpression.Brow_Lowerer_R]) / 2.0f, + OuterUp: (expressions[FBExpression.Outer_Brow_Raiser_R]+expressions[FBExpression.Outer_Brow_Raiser_R]) / 2.0f, + OuterDown: (expressions[FBExpression.Brow_Lowerer_R]+expressions[FBExpression.Brow_Lowerer_R]) / 2.0f + ); + + UnifiedTrackingData.LatestEyeData.EyesDilation = 0.5f; + UnifiedTrackingData.LatestEyeData.EyesPupilDiameter = 0.0035f; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.CheekPuffLeft] = expressions[FBExpression.Cheek_Puff_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.CheekPuffRight] = expressions[FBExpression.Cheek_Puff_R]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.CheekSuck] = (expressions[FBExpression.Cheek_Suck_L] + expressions[FBExpression.Cheek_Suck_R]) / 2; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.JawOpen] = expressions[FBExpression.Jaw_Drop]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.JawLeft] = expressions[FBExpression.Jaw_Sideways_Left]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.JawRight] = expressions[FBExpression.Jaw_Sideways_Right]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.JawForward] = expressions[FBExpression.Jaw_Thrust]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthPout] = (expressions[FBExpression.Lip_Pucker_L] + expressions[FBExpression.Lip_Pucker_R]) / 3; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperLeft] = expressions[FBExpression.Mouth_Left]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerLeft] = (expressions[FBExpression.Mouth_Left] + expressions[FBExpression.Lip_Funneler_LB]) / 2; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperRight] = expressions[FBExpression.Mouth_Right]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerRight] = (expressions[FBExpression.Mouth_Right] + expressions[FBExpression.Lip_Funneler_RB]) / 2; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileLeft] = Math.Min(1, expressions[FBExpression.Lip_Corner_Puller_L] * 1.2f); + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileRight] = Math.Min(1, expressions[FBExpression.Lip_Corner_Puller_R] * 1.2f); + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadLeft] = Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_L] + expressions[FBExpression.Lip_Stretcher_L]) * 0.75f);//Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_L]) * 1.5f); + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadRight] = Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_R] + expressions[FBExpression.Lip_Stretcher_R]) * 0.75f);//Math.Min(1, (expressions[FBExpression.Lip_Corner_Depressor_R]) * 1.5f); + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperOverturn] = (expressions[FBExpression.Lips_Toward] + expressions[FBExpression.Lip_Funneler_LT] + expressions[FBExpression.Lip_Funneler_RT]) / 3; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerOverturn] = (expressions[FBExpression.Lips_Toward] + expressions[FBExpression.Lip_Funneler_LB] + expressions[FBExpression.Lip_Funneler_RB]) / 3; + + + // solve issue with smilesad combined blendshape + if (UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileLeft] > UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadLeft]) + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadLeft] /= 1 + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileLeft]; + else if (UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileLeft] < UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadLeft]) + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileLeft] /= 1 + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadLeft]; + + if (UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileRight] > UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadRight]) + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadRight] /= 1 + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileRight]; + else if (UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileRight] < UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadRight]) + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSmileRight] /= 1 + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthSadRight]; + + //UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperUpLeft] = (expressions[FBExpression.Upper_Lip_Raiser_L] + expressions[FBExpression.Lip_Funneler_LT]) / 2; + //UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperUpRight] = (expressions[FBExpression.Upper_Lip_Raiser_R] + expressions[FBExpression.Lip_Funneler_RT]) / 2; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipFunnelerLeftBottom] = expressions[FBExpression.Lip_Funneler_LB]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipFunnelerRightBottom] = expressions[FBExpression.Lip_Funneler_RB]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipFunnelerLeftTop] = expressions[FBExpression.Lip_Funneler_LT]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipFunnelerRightTop] = expressions[FBExpression.Lip_Funneler_RT]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.CheekRaiserLeft] = expressions[FBExpression.Cheek_Raiser_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.CheekRaiserRight] = expressions[FBExpression.Cheek_Raiser_R]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.ChinRaiserBottom] = expressions[FBExpression.Chin_Raiser_B]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.ChinRaiserTop] = expressions[FBExpression.Chin_Raiser_T]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.DimplerLeft] = expressions[FBExpression.Dimpler_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.DimplerRight] = expressions[FBExpression.Dimpler_R]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipPressorLeft] = expressions[FBExpression.Lip_Pressor_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipPressorRight] = expressions[FBExpression.Lip_Pressor_R]; - return updateResult; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipPuckerLeft] = expressions[FBExpression.Lip_Pucker_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipPuckerRight] = expressions[FBExpression.Lip_Pucker_R]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipStretcherLeft] = expressions[FBExpression.Lip_Stretcher_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipStretcherRight] = expressions[FBExpression.Lip_Stretcher_R]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipSuckLeftBottom] = expressions[FBExpression.Lip_Suck_LB]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipSuckLeftTop] = expressions[FBExpression.Lip_Suck_LT]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipSuckRightBottom] = expressions[FBExpression.Lip_Suck_RB]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipSuckRightTop] = expressions[FBExpression.Lip_Suck_RT]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipTightenerLeft] = expressions[FBExpression.Lip_Tightener_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LipTightenerRight] = expressions[FBExpression.Lip_Tightener_R]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LowerLipDepressorLeft] = expressions[FBExpression.Lower_Lip_Depressor_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.LowerLipDepressorRight] = expressions[FBExpression.Lower_Lip_Depressor_R]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.NoseWrinklerLeft] = expressions[FBExpression.Nose_Wrinkler_L]; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.NoseWrinklerRight] = expressions[FBExpression.Nose_Wrinkler_R]; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthApeShape] = Math.Min(1, expressions[FBExpression.Lips_Toward] * 1.4f * (0.5f+expressions[FBExpression.Jaw_Drop])); + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperInside] = (expressions[FBExpression.Lip_Suck_LT] + expressions[FBExpression.Lip_Suck_RT]) / 2; + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerInside] = (expressions[FBExpression.Lip_Suck_LB] + expressions[FBExpression.Lip_Suck_RB]) / 2; + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerDownLeft] = Math.Min(1, (expressions[FBExpression.Lip_Funneler_LB] + expressions[FBExpression.Lower_Lip_Depressor_L])); + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerDownRight] = Math.Min(1, (expressions[FBExpression.Lip_Funneler_RB] + expressions[FBExpression.Lower_Lip_Depressor_R])); + + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperUpLeft] = Math.Min(1, (expressions[FBExpression.Lip_Funneler_LT] + expressions[FBExpression.Upper_Lip_Raiser_L]*2)); + UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperUpRight] = Math.Min(1, (expressions[FBExpression.Lip_Funneler_RT] + expressions[FBExpression.Upper_Lip_Raiser_R]*2)); + + + // Possible matches + //UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerOverlay = FBData.faceData.ChinRaiserB; + //UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthUpperInside = FBData.faceData.ChinRaiserB; + //UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerInside = FBData.faceData.ChinRaiserT; + //UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerDownLeft = FBData.faceData.LowerLipDepressorL; + //UnifiedTrackingData.LatestLipData.LatestShapes[(int)UnifiedExpression.MouthLowerDownRight = FBData.faceData.LowerLipDepressorR; + + // Expressions reported by @aero that look the best - would be best to get a side by side comparison + //LeftEyeX = data[E["EYESLOOKRIGHTL"] - data[E["EYESLOOKLEFTL"]] + //RightEyeX = data[E["EYESLOOKRIGHTR"] - data[E["EYESLOOKLEFTR"]] + //EyesY = (((data[E["EYESLOOKUPL"] + data[E["EYESLOOKUPR"]) / 1.6) - (data[E["EYESLOOKDOWNL"] + data[E["EYESLOOKDOWNR"])) / 2 + //LeftEyeLid = data[E["EYESCLOSEDL"] + min(data[E["EYESLOOKDOWNL"], data[E["EYESLOOKDOWNR"]) + //RightEyeLid = data[E["EYESCLOSEDR"] + min(data[E["EYESLOOKDOWNL"], data[E["EYESLOOKDOWNR"]) + //JawOpen = data[E["JAWDROP"]] + //MouthPout = (data[E["LIPPUCKERL"] + data[E["LIPPUCKERR"]) / 2 + //JawX = (data[E["MOUTHRIGHT"] - data[E["MOUTHLEFT"]) * 2 + //JawForward = data[E["JAWTHRUST"]] + //CheekPuffLeft = data[E["CHEEKPUFFL"] * 4 + //CheekPuffRight = data[E["CHEEKPUFFR"] * 4 + //SmileSadLeft = (data[E["DIMPLERL"] + (data[E["UPPERLIPRAISERL"] * 2)) - ((data[E["LOWERLIPDEPRESSORL"]) * 4) + //SmileSadRight = (data[E["DIMPLERR"] + (data[E["UPPERLIPRAISERR"] * 2)) - ((data[E["LOWERLIPDEPRESSORR"]) * 4) } - #endregion + private Eye MakeEye(float LookLeft, float LookRight, float LookUp, float LookDown, float Openness, float Squint, float Squeeze, float Widen, float InnerUp, float InnerDown, float OuterUp, float OuterDown) + { + return new Eye() + { + Look = new Vector2(LookRight - LookLeft, LookUp - LookDown), + Openness = Openness, + Squeeze = Squeeze, + Squint = Squint, + Widen = Widen, + Brow = new Eye.EyeBrow() + { + InnerUp = InnerUp, + InnerDown = InnerDown, + OuterUp = OuterUp, + OuterDown = OuterDown, + } + }; + } + + public override void Teardown() + { + + } } -} \ No newline at end of file +} diff --git a/VRCFaceTracking/UnifiedTrackingData.cs b/VRCFaceTracking/UnifiedTrackingData.cs index bfbfd026..2ea23ef5 100644 --- a/VRCFaceTracking/UnifiedTrackingData.cs +++ b/VRCFaceTracking/UnifiedTrackingData.cs @@ -5,6 +5,7 @@ using ViveSR.anipal.Lip; using VRCFaceTracking.Params; using VRCFaceTracking.Params.Eye; +using VRCFaceTracking.Params.Lip; using VRCFaceTracking.Params.LipMerging; using Vector2 = VRCFaceTracking.Params.Vector2; @@ -14,22 +15,29 @@ namespace VRCFaceTracking public struct Eye { public Vector2 Look; + public EyeBrow Brow; public float Openness; public float Widen, Squeeze; + public float Squint; - public void Update(SingleEyeData eyeData, SingleEyeExpression? expression = null) { if (eyeData.GetValidity(SingleEyeDataValidity.SINGLE_EYE_DATA_GAZE_DIRECTION_VALIDITY)) Look = eyeData.gaze_direction_normalized.Invert(); Openness = eyeData.eye_openness; - + if (expression == null) return; // This is null when we use this as a combined eye, so don't try read data from it - + Widen = expression.Value.eye_wide; Squeeze = expression.Value.eye_squeeze; } + + public struct EyeBrow + { + public float InnerUp, InnerDown; + public float OuterUp, OuterDown; + } } public class EyeTrackingData @@ -102,8 +110,9 @@ public class LipTrackingData public byte[] ImageData; public bool SupportsImage; - public float[] LatestShapes = new float[SRanipal_Lip_v2.WeightingCount]; + public float[] LatestShapes = new float[(int) UnifiedExpression.Max + 1]; + // Updates only SRanipal_Lip_v2 data. Shapes beyond SRanipal are exposed to module developers, and must be updated manually public void UpdateData(LipData_v2 lipData) { unsafe diff --git a/VRCFaceTracking/VRCFaceTracking.csproj b/VRCFaceTracking/VRCFaceTracking.csproj index 0b820e45..242e3889 100644 --- a/VRCFaceTracking/VRCFaceTracking.csproj +++ b/VRCFaceTracking/VRCFaceTracking.csproj @@ -60,6 +60,7 @@ +