Skip to content

Commit

Permalink
Merge pull request #1 from mrmr32/eyetracking
Browse files Browse the repository at this point in the history 
Eyetracking
  • Loading branch information
@mrmr32
mrmr32 committed Jun 11, 2023
2 parents b6e260e + a8bf87c commit 469b5dd
Show file tree
Hide file tree
Showing 5 changed files with 155 additions and 12 deletions.
4 changes: 4 additions & 0 deletions FUNDING.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# These are supported funding model platforms

github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: mrmr32
52 changes: 49 additions & 3 deletions lib/DAZMorphLibrary.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@ public class DAZMorphLibrary
public static DAZMorph JawChew;
public static DAZMorph LipTopDown;
public static DAZMorph LipBottomIn;
public static DAZMorph EyeBlink_R;
public static DAZMorph EyeBlink_L;
public static DAZMorph EyeSquint_R;
public static DAZMorph EyeSquint_L;
public static DAZMorph BrowDown_L;
public static DAZMorph BrowDown_R;
public static DAZMorph BrowInnerUp;
public static DAZMorph BrowOuterUp_L;
public static DAZMorph BrowOuterUp_R;


public static DAZMorph TongueSideSide;
Expand Down Expand Up @@ -133,13 +142,50 @@ public DAZMorphLibrary(Atom containingAtom, float defaultMorphValueParam, Boolea
JawChew = _initMorph(
"Jackaroo.JarModularExpressions.2:/Custom/Atom/Person/Morphs/female/Jackaroo/JaRExpressions1.2/JawChew.vmi",
"JawChew"
);
);

LipTopDown = _initMorph("Lip Top Down");

LipBottomIn = _initMorph("Lip Bottom In");



EyeBlink_R = _initMorph("Eyes Closed Right");

EyeBlink_L = _initMorph("Eyes Closed Left");

EyeSquint_R = _initMorph(
"Jackaroo.JarModularExpressions.2:/Custom/Atom/Person/Morphs/female/Jackaroo/JaRExpressions1.2/EyeSquint_R.vmi",
"EyeSquint_R"
);

EyeSquint_L = _initMorph(
"Jackaroo.JarModularExpressions.2:/Custom/Atom/Person/Morphs/female/Jackaroo/JaRExpressions1.2/EyeSquint_L.vmi",
"EyeSquint_L"
);

BrowDown_L = _initMorph(
"Jackaroo.JarModularExpressions.2:/Custom/Atom/Person/Morphs/female/Jackaroo/JaRExpressions1.2/BrowD_L.vmi",
"EyeSquint_L"
);

BrowDown_R = _initMorph(
"Jackaroo.JarModularExpressions.2:/Custom/Atom/Person/Morphs/female/Jackaroo/JaRExpressions1.2/BrowD_R.vmi",
"EyeSquint_L"
);

BrowInnerUp = _initMorph(
"Jackaroo.JarModularExpressions.2:/Custom/Atom/Person/Morphs/female/Jackaroo/JaRExpressions1.2/BrowU_C.vmi",
"EyeSquint_L"
);

BrowOuterUp_L = _initMorph(
"Jackaroo.JarModularExpressions.2:/Custom/Atom/Person/Morphs/female/Jackaroo/JaRExpressions1.2/BrowU_L.vmi",
"EyeSquint_L"
);

BrowOuterUp_R = _initMorph(
"Jackaroo.JarModularExpressions.2:/Custom/Atom/Person/Morphs/female/Jackaroo/JaRExpressions1.2/BrowU_R.vmi",
"EyeSquint_L"
);
}
catch (Exception e)
{
Expand Down
76 changes: 74 additions & 2 deletions lib/MorphMappers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
using System.Linq;
using System.Text;
using SimpleJSON;
using UnityEngine;
using System.Collections;

namespace FacialTrackerVamPlugin
{
Expand All @@ -20,6 +22,13 @@ public class MorphMappers
private static float factorDivisorMouthPouty = 2;
private static float factorGlobal = 1f;

private MVRScript script;

private EyesControl _eyeBehavior;
private FreeControllerV3 _eyeTarget;
private DAZMeshEyelidControl _eyelidBehavior;
private MotionAnimationControl _head;


public static void JawRight()
{
Expand Down Expand Up @@ -159,18 +168,78 @@ public static void LipBottomIn()
_setMorphValue(DAZMorphLibrary.LipBottomIn, SRanipalMorphLibrary.Mouth_Upper_Inside * factorGlobal *1.4f);
}

// TODO as we're looking at a static position we should update this function every time the `containingAtom` moves
public void Eye()
{
float ySeePlane = 0.5f;
Vector3 seeVector = new Vector3(SRanipalMorphLibrary.Eye_X_Right, SRanipalMorphLibrary.Eye_Y_Right, ySeePlane);
/*if (Mathf.Abs(SRanipalMorphLibrary.Eye_X_Right) < 0.05f && Mathf.Abs(SRanipalMorphLibrary.Eye_Y_Right) < 0.05f &&
(Mathf.Abs(SRanipalMorphLibrary.Eye_X_Left) > 0.05f || Mathf.Abs(SRanipalMorphLibrary.Eye_Y_Left) > 0.05f)) {
// wink with the right eye
seeVector = new Vector3(-SRanipalMorphLibrary.Eye_X_Left, SRanipalMorphLibrary.Eye_Y_Left, ySeePlane); // the left eye needs to be mirrored
}*/

_eyeTarget.control.position = this._head.transform.position + this._head.transform.rotation * seeVector;
}

// @see https://github.com/mrmr32/TriggerIncrementer/blob/main/TriggerIncrementer.cs#L175
private MotionAnimationControl GetHead() {
foreach (MotionAnimationControl mac in this.script.containingAtom.motionAnimationControls) { // TODO get head inside linkableRigidbodies?
if (!mac.name.Equals("headControl")) continue;
return mac;
}

return null; // not found
}

public static void Blink()
{
_setMorphValue(DAZMorphLibrary.EyeBlink_L, SRanipalMorphLibrary.Eye_Blink_Left * factorGlobal);
_setMorphValue(DAZMorphLibrary.EyeBlink_R, SRanipalMorphLibrary.Eye_Blink_Right * factorGlobal);

_setMorphValue(DAZMorphLibrary.EyeSquint_L, SRanipalMorphLibrary.Eye_Squint_Left * factorGlobal);
_setMorphValue(DAZMorphLibrary.EyeSquint_R, SRanipalMorphLibrary.Eye_Squint_Right * factorGlobal);
}

public static void Brow()
{
_setMorphValue(DAZMorphLibrary.BrowDown_L, SRanipalMorphLibrary.Brow_Down_Left * factorGlobal);
_setMorphValue(DAZMorphLibrary.BrowDown_R, SRanipalMorphLibrary.Brow_Down_Right * factorGlobal);
_setMorphValue(DAZMorphLibrary.BrowInnerUp, SRanipalMorphLibrary.Brow_Inner_Up * factorGlobal);
_setMorphValue(DAZMorphLibrary.BrowOuterUp_L, SRanipalMorphLibrary.Brow_Outer_Up_Left * factorGlobal);
_setMorphValue(DAZMorphLibrary.BrowOuterUp_R, SRanipalMorphLibrary.Brow_Outer_Up_Right * factorGlobal);
}

// Class constructor
public MorphMappers(Atom containingAtom, float defaultMorphValue, Boolean ignoreMissingMorphs)
public MorphMappers(Atom containingAtom, MVRScript script, float defaultMorphValue, Boolean ignoreMissingMorphs)
{
this.script = script;

dazMorphLibrary = new DAZMorphLibrary(containingAtom, defaultMorphValue, ignoreMissingMorphs);
sranipalMorphLibrary = new SRanipalMorphLibrary();

// @ref https://github.com/acidbubbles/vam-glance/blob/master/Glance.cs
_eyeTarget = containingAtom.freeControllers.FirstOrDefault(fc => fc.name == "eyeTargetControl");
if (_eyeTarget == null) throw new NullReferenceException(nameof(_eyeTarget));
_eyeBehavior = (EyesControl) containingAtom.GetStorableByID("Eyes");
if (_eyeBehavior == null) throw new NullReferenceException(nameof(_eyeBehavior));
_eyelidBehavior = (DAZMeshEyelidControl) containingAtom.GetStorableByID("EyelidControl");
if (_eyelidBehavior == null) throw new NullReferenceException(nameof(_eyelidBehavior));

// TODO this should be re-runed every time the atom changes
_eyeTarget.hidden = true;
_eyeBehavior.currentLookMode = EyesControl.LookMode.Target;
// TODO add a property for the people that are not using an eyetracking device?
_eyelidBehavior.SetBoolParamValue("blinkEnabled", false); // I'm the one who blinks

if ((this._head = GetHead()) == null) throw new InvalidOperationException("Head not found in added object");
}

// Function to run all mappers at once
public void _runAll(JSONNode newSranipalMorphValues = null)
{
// If new SRanipal morph values were passed, update morph library
if (newSranipalMorphValues != null) sranipalMorphLibrary._updateFromJsonNode(newSranipalMorphValues);
if (newSranipalMorphValues != null) sranipalMorphLibrary._updateFromJsonNode(newSranipalMorphValues.AsObject);

// Call mapper functions
JawRight();
Expand All @@ -191,6 +260,9 @@ public void _runAll(JSONNode newSranipalMorphValues = null)
JawDown();
LipTopDown();
LipBottomIn();
Eye();
Blink();
Brow();

// ... any new mapper functions should be called here

Expand Down
33 changes: 27 additions & 6 deletions lib/SRanipalMorphLibrary.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,29 @@ public class SRanipalMorphLibrary
public static float Tongue_UpLeft_Morph = 0;
public static float Tongue_DownRight_Morph = 0;
public static float Tongue_DownLeft_Morph = 0;
public static float Eye_X_Left = 0;
public static float Eye_Y_Left = 0;
public static float Eye_X_Right = 0;
public static float Eye_Y_Right = 0;
public static float Eye_Blink_Left = 0;
public static float Eye_Blink_Right = 0;
public static float Eye_Squint_Left = 0;
public static float Eye_Squint_Right = 0;
public static float Brow_Down_Left = 0;
public static float Brow_Down_Right = 0;
public static float Brow_Inner_Up = 0;
public static float Brow_Outer_Up_Left = 0;
public static float Brow_Outer_Up_Right = 0;

public SRanipalMorphLibrary()
{

}

public void _updateFromJsonNode(JSONNode sranipalValues)
public void _updateFromJsonNode(JSONClass sranipalValues)
{

try
{

float.TryParse(sranipalValues["Jaw_Right"], out Jaw_Right);
float.TryParse(sranipalValues["Jaw_Left"], out Jaw_Left);
float.TryParse(sranipalValues["Jaw_Forward"], out Jaw_Forward);
Expand Down Expand Up @@ -95,16 +106,26 @@ public void _updateFromJsonNode(JSONNode sranipalValues)
float.TryParse(sranipalValues["Tongue_UpLeft_Morph"], out Tongue_UpLeft_Morph); // NOT CURRENTLY USED
float.TryParse(sranipalValues["Tongue_DownRight_Morph"], out Tongue_DownRight_Morph); // NOT CURRENTLY USED
float.TryParse(sranipalValues["Tongue_DownLeft_Morph"], out Tongue_DownLeft_Morph); // NOT CURRENTLY USED

if (sranipalValues.HasKey("Eye_X_Left")) float.TryParse(sranipalValues["Eye_X_Left"], out Eye_X_Left);
if (sranipalValues.HasKey("Eye_Y_Left")) float.TryParse(sranipalValues["Eye_Y_Left"], out Eye_Y_Left);
if (sranipalValues.HasKey("Eye_X_Right")) float.TryParse(sranipalValues["Eye_X_Right"], out Eye_X_Right);
if (sranipalValues.HasKey("Eye_Y_Right")) float.TryParse(sranipalValues["Eye_Y_Right"], out Eye_Y_Right);
if (sranipalValues.HasKey("Eye_Blink_Left")) float.TryParse(sranipalValues["Eye_Blink_Left"], out Eye_Blink_Left);
if (sranipalValues.HasKey("Eye_Blink_Right")) float.TryParse(sranipalValues["Eye_Blink_Right"], out Eye_Blink_Right);
if (sranipalValues.HasKey("Eye_Squint_Left")) float.TryParse(sranipalValues["Eye_Squint_Left"], out Eye_Squint_Left);
if (sranipalValues.HasKey("Eye_Squint_Right")) float.TryParse(sranipalValues["Eye_Squint_Right"], out Eye_Squint_Right);
if (sranipalValues.HasKey("Brow_Down_Left")) float.TryParse(sranipalValues["Brow_Down_Left"], out Brow_Down_Left);
if (sranipalValues.HasKey("Brow_Down_Right")) float.TryParse(sranipalValues["Brow_Down_Right"], out Brow_Down_Right);
if (sranipalValues.HasKey("Brow_Inner_Up")) float.TryParse(sranipalValues["Brow_Inner_Up"], out Brow_Inner_Up);
if (sranipalValues.HasKey("Brow_Outer_Up_Left")) float.TryParse(sranipalValues["Brow_Outer_Up_Left"], out Brow_Outer_Up_Left);
if (sranipalValues.HasKey("Brow_Outer_Up_Right")) float.TryParse(sranipalValues["Brow_Outer_Up_Right"], out Brow_Outer_Up_Right);
}
catch (Exception e)
{
SuperController.LogError($"Unable to retrieve SRanipal morph values from JSON message");
throw e;
}

}

}

}
2 changes: 1 addition & 1 deletion lib/main.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public override void Init()
if (containingAtom.type != "Person") throw new System.Exception("Plugin must be attached to a Person atom");

// Set up MorphMappers class, which will take care of building morph libraries for us
morphMappers = new MorphMappers(containingAtom, DEFAULT_MORPH_VALUE, IGNORE_MISSING_MORPHS);
morphMappers = new MorphMappers(containingAtom, this, DEFAULT_MORPH_VALUE, IGNORE_MISSING_MORPHS);

// Set up UDP server socket so we can listen for messages
startServer();
Expand Down

0 comments on commit 469b5dd

Please sign in to comment.