Merge pull request #12492 from AdmiralCurtiss/wiimote-ir-passthrough

Implement IR passthrough for emulated Wiimotes

Data/Sys/Profiles/Wiimote/Wii Remote with MotionPlus Pointing.ini

@@ -7,6 +7,19 @@ Buttons/2 = `2`
 Buttons/- = `-`
 Buttons/+ = `+`
 Buttons/Home = `HOME`
+IRPassthrough/Enabled = False
+IRPassthrough/Object 1 X = `IR Object 1 X`
+IRPassthrough/Object 1 Y = `IR Object 1 Y`
+IRPassthrough/Object 1 Size = `IR Object 1 Size`
+IRPassthrough/Object 2 X = `IR Object 2 X`
+IRPassthrough/Object 2 Y = `IR Object 2 Y`
+IRPassthrough/Object 2 Size = `IR Object 2 Size`
+IRPassthrough/Object 3 X = `IR Object 3 X`
+IRPassthrough/Object 3 Y = `IR Object 3 Y`
+IRPassthrough/Object 3 Size = `IR Object 3 Size`
+IRPassthrough/Object 4 X = `IR Object 4 X`
+IRPassthrough/Object 4 Y = `IR Object 4 Y`
+IRPassthrough/Object 4 Size = `IR Object 4 Size`
 IMUAccelerometer/Up = `Accel Up`
 IMUAccelerometer/Down = `Accel Down`
 IMUAccelerometer/Left = `Accel Left`

Source/Core/Core/HW/WiimoteCommon/DataReport.cpp

@@ -125,13 +125,14 @@ struct NoExt : virtual DataReportManipulator
   u8* GetExtDataPtr() override { return nullptr; }
 };
 
-template <u32 Offset, u32 Length, u32 DataOffset = 0>
+template <IRReportFormat Format, u32 Offset, u32 Length, u32 DataOffset = 0>
 struct IncludeIR : virtual DataReportManipulator
 {
   u32 GetIRDataSize() const override { return Length; }
   const u8* GetIRDataPtr() const override { return data_ptr + Offset; }
   u8* GetIRDataPtr() override { return data_ptr + Offset; }
   u32 GetIRDataFormatOffset() const override { return DataOffset; }
+  IRReportFormat GetIRReportFormat() const override { return Format; }
 };
 
 struct NoIR : virtual DataReportManipulator
@@ -140,6 +141,7 @@ struct NoIR : virtual DataReportManipulator
   const u8* GetIRDataPtr() const override { return nullptr; }
   u8* GetIRDataPtr() override { return nullptr; }
   u32 GetIRDataFormatOffset() const override { return 0; }
+  IRReportFormat GetIRReportFormat() const override { return IRReportFormat::None; }
 };
 
 #ifdef _MSC_VER
@@ -162,7 +164,10 @@ struct ReportCoreExt8 : IncludeCore, NoAccel, NoIR, IncludeExt<2, 8>
 {
 };
 
-struct ReportCoreAccelIR12 : IncludeCore, IncludeAccel, IncludeIR<5, 12>, NoExt
+struct ReportCoreAccelIR12 : IncludeCore,
+                             IncludeAccel,
+                             IncludeIR<IRReportFormat::Extended, 5, 12>,
+                             NoExt
 {
   u32 GetDataSize() const override { return 17; }
 };
@@ -175,19 +180,25 @@ struct ReportCoreAccelExt16 : IncludeCore, IncludeAccel, NoIR, IncludeExt<5, 16>
 {
 };
 
-struct ReportCoreIR10Ext9 : IncludeCore, NoAccel, IncludeIR<2, 10>, IncludeExt<12, 9>
+struct ReportCoreIR10Ext9 : IncludeCore,
+                            NoAccel,
+                            IncludeIR<IRReportFormat::Basic, 2, 10>,
+                            IncludeExt<12, 9>
 {
 };
 
-struct ReportCoreAccelIR10Ext6 : IncludeCore, IncludeAccel, IncludeIR<5, 10>, IncludeExt<15, 6>
+struct ReportCoreAccelIR10Ext6 : IncludeCore,
+                                 IncludeAccel,
+                                 IncludeIR<IRReportFormat::Basic, 5, 10>,
+                                 IncludeExt<15, 6>
 {
 };
 
 struct ReportExt21 : NoCore, NoAccel, NoIR, IncludeExt<0, 21>
 {
 };
 
-struct ReportInterleave1 : IncludeCore, IncludeIR<3, 18, 0>, NoExt
+struct ReportInterleave1 : IncludeCore, IncludeIR<IRReportFormat::Full1, 3, 18, 0>, NoExt
 {
   // FYI: Only 8-bits of precision in this report, and no Y axis.
   void GetAccelData(AccelData* accel) const override
@@ -220,7 +231,7 @@ struct ReportInterleave1 : IncludeCore, IncludeIR<3, 18, 0>, NoExt
   u32 GetDataSize() const override { return 21; }
 };
 
-struct ReportInterleave2 : IncludeCore, IncludeIR<3, 18, 18>, NoExt
+struct ReportInterleave2 : IncludeCore, IncludeIR<IRReportFormat::Full2, 3, 18, 18>, NoExt
 {
   // FYI: Only 8-bits of precision in this report, and no X axis.
   void GetAccelData(AccelData* accel) const override
@@ -372,6 +383,11 @@ u32 DataReportBuilder::GetIRDataFormatOffset() const
   return m_manip->GetIRDataFormatOffset();
 }
 
+IRReportFormat DataReportBuilder::GetIRReportFormat() const
+{
+  return m_manip->GetIRReportFormat();
+}
+
 void DataReportBuilder::GetCoreData(CoreData* core) const
 {
   m_manip->GetCoreData(core);

Source/Core/Core/HW/WiimoteCommon/DataReport.h

@@ -39,6 +39,7 @@ class DataReportManipulator
   virtual const u8* GetIRDataPtr() const = 0;
   virtual u32 GetIRDataSize() const = 0;
   virtual u32 GetIRDataFormatOffset() const = 0;
+  virtual IRReportFormat GetIRReportFormat() const = 0;
 
   virtual u8* GetExtDataPtr() = 0;
   virtual const u8* GetExtDataPtr() const = 0;
@@ -76,6 +77,7 @@ class DataReportBuilder
   u32 GetExtDataSize() const;
 
   u32 GetIRDataFormatOffset() const;
+  IRReportFormat GetIRReportFormat() const;
 
   void GetCoreData(CoreData*) const;
   void GetAccelData(AccelData*) const;

Source/Core/Core/HW/WiimoteCommon/WiimoteConstants.h

@@ -50,6 +50,15 @@ enum class OutputReportID : u8
   IRLogicEnable2 = 0x1a,
 };
 
+enum class IRReportFormat : u8
+{
+  None,
+  Basic,     // from ReportCoreIR10Ext9 or ReportCoreAccelIR10Ext6
+  Extended,  // from ReportCoreAccelIR12
+  Full1,     // from ReportInterleave1
+  Full2,     // from ReportInterleave2
+};
+
 enum class LED : u8
 {
   None = 0x00,

Source/Core/Core/HW/WiimoteEmu/Camera.cpp

@@ -59,7 +59,7 @@ CameraLogic::GetCameraPoints(const Common::Matrix44& transform, Common::Vec2 fie
   using Common::Vec3;
   using Common::Vec4;
 
-  const std::array<Vec3, NUM_POINTS> leds{
+  const std::array<Vec3, 2> leds{
       Vec3{-SENSOR_BAR_LED_SEPARATION / 2, 0, 0},
       Vec3{SENSOR_BAR_LED_SEPARATION / 2, 0, 0},
   };
@@ -68,7 +68,7 @@ CameraLogic::GetCameraPoints(const Common::Matrix44& transform, Common::Vec2 fie
       Matrix44::Perspective(field_of_view.y, field_of_view.x / field_of_view.y, 0.001f, 1000) *
       Matrix44::FromMatrix33(Matrix33::RotateX(float(MathUtil::TAU / 4))) * transform;
 
-  std::array<CameraPoint, leds.size()> camera_points;
+  std::array<CameraPoint, CameraLogic::NUM_POINTS> camera_points;
 
   std::transform(leds.begin(), leds.end(), camera_points.begin(), [&](const Vec3& v) {
     const auto point = camera_view * Vec4(v, 1.0);

Source/Core/Core/HW/WiimoteEmu/Camera.h

@@ -126,7 +126,7 @@ class CameraLogic : public I2CSlave
 
   // FYI: A real wiimote normally only returns 1 point for each LED cluster (2 total).
   // Sending all 4 points can actually cause some stuttering issues.
-  static constexpr int NUM_POINTS = 2;
+  static constexpr int NUM_POINTS = 4;
 
   // Range from 0-15. Small values (2-4) seem to be very typical.
   // This is reduced based on distance from sensor bar.

Source/Core/Core/HW/WiimoteEmu/DesiredWiimoteState.cpp

@@ -82,7 +82,7 @@ SerializedWiimoteState SerializeDesiredState(const DesiredWiimoteState& state)
 
   if (has_camera)
   {
-    for (size_t i = 0; i < 2; ++i)
+    for (size_t i = 0; i < state.camera_points.size(); ++i)
     {
       const u16 camera_x = state.camera_points[i].position.x;  // 10 bits
       const u16 camera_y = state.camera_points[i].position.y;  // 10 bits
@@ -178,7 +178,7 @@ bool DeserializeDesiredState(DesiredWiimoteState* state, const SerializedWiimote
     else if (has_accel)
       s += 4;
     if (has_camera)
-      s += 6;
+      s += 12;
     if (has_motion_plus)
       s += 6;
     switch (extension)
@@ -260,7 +260,7 @@ bool DeserializeDesiredState(DesiredWiimoteState* state, const SerializedWiimote
 
   if (has_camera)
   {
-    for (size_t i = 0; i < 2; ++i)
+    for (size_t i = 0; i < state->camera_points.size(); ++i)
     {
       const u8 camera_misc = d[pos];
       const u8 camera_x_high = d[pos + 1];

Source/Core/Core/HW/WiimoteEmu/DesiredWiimoteState.h

@@ -21,11 +21,12 @@ struct DesiredWiimoteState
       {Wiimote::ACCEL_ZERO_G << 2, Wiimote::ACCEL_ZERO_G << 2, Wiimote::ACCEL_ONE_G << 2});
 
   // No light detected by the IR camera.
-  static constexpr std::array<CameraPoint, 2> DEFAULT_CAMERA = {CameraPoint(), CameraPoint()};
+  static constexpr std::array<CameraPoint, 4> DEFAULT_CAMERA = {CameraPoint(), CameraPoint(),
+                                                                CameraPoint(), CameraPoint()};
 
   WiimoteCommon::ButtonData buttons{};  // non-button state in this is ignored
   WiimoteCommon::AccelData acceleration = DEFAULT_ACCELERATION;
-  std::array<CameraPoint, 2> camera_points = DEFAULT_CAMERA;
+  std::array<CameraPoint, 4> camera_points = DEFAULT_CAMERA;
   std::optional<MotionPlus::DataFormat::Data> motion_plus = std::nullopt;
   DesiredExtensionState extension;
 };
@@ -34,7 +35,7 @@ struct DesiredWiimoteState
 struct SerializedWiimoteState
 {
   u8 length;
-  std::array<u8, 24> data;  // 12 bytes Wiimote, 6 bytes MotionPlus, 6 bytes Extension
+  std::array<u8, 30> data;  // 18 bytes Wiimote, 6 bytes MotionPlus, 6 bytes Extension
 };
 
 SerializedWiimoteState SerializeDesiredState(const DesiredWiimoteState& state);

Source/Core/Core/HW/WiimoteEmu/WiimoteEmu.cpp

@@ -49,6 +49,7 @@
 #include "InputCommon/ControllerEmu/ControlGroup/IMUAccelerometer.h"
 #include "InputCommon/ControllerEmu/ControlGroup/IMUCursor.h"
 #include "InputCommon/ControllerEmu/ControlGroup/IMUGyroscope.h"
+#include "InputCommon/ControllerEmu/ControlGroup/IRPassthrough.h"
 #include "InputCommon/ControllerEmu/ControlGroup/ModifySettingsButton.h"
 #include "InputCommon/ControllerEmu/ControlGroup/Tilt.h"
 
@@ -250,6 +251,8 @@ Wiimote::Wiimote(const unsigned int index) : m_index(index), m_bt_device_index(i
                         _trans("Camera field of view (affects sensitivity of pointing).")},
                        fov_default.y, 0.01, 180);
 
+  groups.emplace_back(m_ir_passthrough = new ControllerEmu::IRPassthrough(
+                          IR_PASSTHROUGH_GROUP, _trans("Point (Passthrough)")));
   groups.emplace_back(m_imu_accelerometer = new ControllerEmu::IMUAccelerometer(
                           ACCELEROMETER_GROUP, _trans("Accelerometer")));
   groups.emplace_back(m_imu_gyroscope =
@@ -360,6 +363,8 @@ ControllerEmu::ControlGroup* Wiimote::GetWiimoteGroup(WiimoteGroup group) const
     return m_imu_gyroscope;
   case WiimoteGroup::IMUPoint:
     return m_imu_ir;
+  case WiimoteGroup::IRPassthrough:
+    return m_ir_passthrough;
   default:
     ASSERT(false);
     return nullptr;
@@ -447,6 +452,33 @@ void Wiimote::UpdateButtonsStatus(const DesiredWiimoteState& target_state)
   m_status.buttons.hex = target_state.buttons.hex & ButtonData::BUTTON_MASK;
 }
 
+static std::array<CameraPoint, CameraLogic::NUM_POINTS>
+GetPassthroughCameraPoints(ControllerEmu::IRPassthrough* ir_passthrough)
+{
+  std::array<CameraPoint, CameraLogic::NUM_POINTS> camera_points;
+  for (size_t i = 0; i < camera_points.size(); ++i)
+  {
+    const ControlState size = ir_passthrough->GetObjectSize(i);
+    if (size <= 0.0f)
+      continue;
+
+    const ControlState x = ir_passthrough->GetObjectPositionX(i);
+    const ControlState y = ir_passthrough->GetObjectPositionY(i);
+
+    camera_points[i].position.x =
+        std::clamp(std::lround(x * ControlState(CameraLogic::CAMERA_RES_X - 1)), long(0),
+                   long(CameraLogic::CAMERA_RES_X - 1));
+    camera_points[i].position.y =
+        std::clamp(std::lround(y * ControlState(CameraLogic::CAMERA_RES_Y - 1)), long(0),
+                   long(CameraLogic::CAMERA_RES_Y - 1));
+    camera_points[i].size =
+        std::clamp(std::lround(size * ControlState(CameraLogic::MAX_POINT_SIZE)), long(0),
+                   long(CameraLogic::MAX_POINT_SIZE));
+  }
+
+  return camera_points;
+}
+
 void Wiimote::BuildDesiredWiimoteState(DesiredWiimoteState* target_state,
                                        SensorBarState sensor_bar_state)
 {
@@ -470,7 +502,11 @@ void Wiimote::BuildDesiredWiimoteState(DesiredWiimoteState* target_state,
       ConvertAccelData(GetTotalAcceleration(), ACCEL_ZERO_G << 2, ACCEL_ONE_G << 2);
 
   // Calculate IR camera state.
-  if (sensor_bar_state == SensorBarState::Enabled)
+  if (m_ir_passthrough->enabled)
+  {
+    target_state->camera_points = GetPassthroughCameraPoints(m_ir_passthrough);
+  }
+  else if (sensor_bar_state == SensorBarState::Enabled)
   {
     target_state->camera_points = CameraLogic::GetCameraPoints(
         GetTotalTransformation(),
@@ -762,6 +798,12 @@ void Wiimote::LoadDefaults(const ControllerInterface& ciface)
   m_imu_gyroscope->SetControlExpression(3, "`Gyro Roll Right`");
   m_imu_gyroscope->SetControlExpression(4, "`Gyro Yaw Left`");
   m_imu_gyroscope->SetControlExpression(5, "`Gyro Yaw Right`");
+  for (int i = 0; i < 4; ++i)
+  {
+    m_ir_passthrough->SetControlExpression(i * 3 + 0, fmt::format("`IR Object {} X`", i + 1));
+    m_ir_passthrough->SetControlExpression(i * 3 + 1, fmt::format("`IR Object {} Y`", i + 1));
+    m_ir_passthrough->SetControlExpression(i * 3 + 2, fmt::format("`IR Object {} Size`", i + 1));
+  }
 #endif
 
   // Enable Nunchuk:

Source/Core/Core/HW/WiimoteEmu/WiimoteEmu.h

@@ -34,6 +34,7 @@ class Force;
 class IMUAccelerometer;
 class IMUGyroscope;
 class IMUCursor;
+class IRPassthrough;
 class ModifySettingsButton;
 class Output;
 class Tilt;
@@ -59,6 +60,7 @@ enum class WiimoteGroup
   IMUAccelerometer,
   IMUGyroscope,
   IMUPoint,
+  IRPassthrough,
 };
 
 enum class NunchukGroup;
@@ -121,6 +123,7 @@ class Wiimote : public ControllerEmu::EmulatedController, public WiimoteCommon::
   static constexpr const char* ACCELEROMETER_GROUP = "IMUAccelerometer";
   static constexpr const char* GYROSCOPE_GROUP = "IMUGyroscope";
   static constexpr const char* IR_GROUP = "IR";
+  static constexpr const char* IR_PASSTHROUGH_GROUP = "IRPassthrough";
 
   static constexpr const char* A_BUTTON = "A";
   static constexpr const char* B_BUTTON = "B";
@@ -300,6 +303,7 @@ class Wiimote : public ControllerEmu::EmulatedController, public WiimoteCommon::
   ControllerEmu::IMUAccelerometer* m_imu_accelerometer;
   ControllerEmu::IMUGyroscope* m_imu_gyroscope;
   ControllerEmu::IMUCursor* m_imu_ir;
+  ControllerEmu::IRPassthrough* m_ir_passthrough;
 
   ControllerEmu::SettingValue<bool> m_sideways_setting;
   ControllerEmu::SettingValue<bool> m_upright_setting;

Source/Core/DolphinLib.props

@@ -504,6 +504,7 @@
     <ClInclude Include="InputCommon\ControllerEmu\ControlGroup\IMUAccelerometer.h" />
     <ClInclude Include="InputCommon\ControllerEmu\ControlGroup\IMUCursor.h" />
     <ClInclude Include="InputCommon\ControllerEmu\ControlGroup\IMUGyroscope.h" />
+    <ClInclude Include="InputCommon\ControllerEmu\ControlGroup\IRPassthrough.h" />
     <ClInclude Include="InputCommon\ControllerEmu\ControlGroup\MixedTriggers.h" />
     <ClInclude Include="InputCommon\ControllerEmu\ControlGroup\ModifySettingsButton.h" />
     <ClInclude Include="InputCommon\ControllerEmu\ControlGroup\Slider.h" />
@@ -1162,6 +1163,7 @@
     <ClCompile Include="InputCommon\ControllerEmu\ControlGroup\IMUAccelerometer.cpp" />
     <ClCompile Include="InputCommon\ControllerEmu\ControlGroup\IMUCursor.cpp" />
     <ClCompile Include="InputCommon\ControllerEmu\ControlGroup\IMUGyroscope.cpp" />
+    <ClCompile Include="InputCommon\ControllerEmu\ControlGroup\IRPassthrough.cpp" />
     <ClCompile Include="InputCommon\ControllerEmu\ControlGroup\MixedTriggers.cpp" />
     <ClCompile Include="InputCommon\ControllerEmu\ControlGroup\ModifySettingsButton.cpp" />
     <ClCompile Include="InputCommon\ControllerEmu\ControlGroup\Slider.cpp" />

Source/Core/DolphinQt/Config/Mapping/WiimoteEmuMotionControlIMU.cpp

@@ -48,6 +48,8 @@ void WiimoteEmuMotionControlIMU::CreateMainLayout()
   auto* groups_layout = new QHBoxLayout();
   groups_layout->addWidget(
       CreateGroupBox(Wiimote::GetWiimoteGroup(GetPort(), WiimoteEmu::WiimoteGroup::IMUPoint)));
+  groups_layout->addWidget(
+      CreateGroupBox(Wiimote::GetWiimoteGroup(GetPort(), WiimoteEmu::WiimoteGroup::IRPassthrough)));
   groups_layout->addWidget(CreateGroupBox(
       Wiimote::GetWiimoteGroup(GetPort(), WiimoteEmu::WiimoteGroup::IMUAccelerometer)));
   groups_layout->addWidget(

Source/Core/InputCommon/CMakeLists.txt

@@ -37,6 +37,8 @@ add_library(inputcommon
   ControllerEmu/ControlGroup/IMUCursor.h
   ControllerEmu/ControlGroup/IMUGyroscope.cpp
   ControllerEmu/ControlGroup/IMUGyroscope.h
+  ControllerEmu/ControlGroup/IRPassthrough.cpp
+  ControllerEmu/ControlGroup/IRPassthrough.h
   ControllerEmu/ControlGroup/MixedTriggers.cpp
   ControllerEmu/ControlGroup/MixedTriggers.h
   ControllerEmu/ControlGroup/ModifySettingsButton.cpp

Source/Core/InputCommon/ControllerEmu/ControlGroup/ControlGroup.h

@@ -49,7 +49,8 @@ enum class GroupType
   Shake,
   IMUAccelerometer,
   IMUGyroscope,
-  IMUCursor
+  IMUCursor,
+  IRPassthrough,
 };
 
 class ControlGroup