Implementation of Native Touch Passthrough and Better Gesture Recognizing

.gitignore

@@ -3,3 +3,4 @@
 **/xcschemes/*.xcscheme
 Build
 DerivedData
+*.xccheckout

Limelight/Database/DataManager.h

@@ -19,6 +19,11 @@
                            width:(NSInteger)width
                 audioConfig:(NSInteger)audioConfig
                 onscreenControls:(NSInteger)onscreenControls
+            keyboardToggleFingers:(NSInteger)keyboardToggleFingers
+             swipeExitScreenEdge:(NSInteger)swipeExitScreenEdge
+             swipeToExitDistance:(CGFloat)swipeToExitDistance
+       liftStreamViewForKeyboard:(BOOL)liftStreamViewForKeyboard
+             showKeyboardToolbar:(BOOL)showKeyboardToolbar
                    optimizeGames:(BOOL)optimizeGames
                  multiController:(BOOL)multiController
                  swapABXYButtons:(BOOL)swapABXYButtons

Limelight/Database/DataManager.m

@@ -58,6 +58,11 @@ - (void) saveSettingsWithBitrate:(NSInteger)bitrate
                            width:(NSInteger)width
                      audioConfig:(NSInteger)audioConfig
                 onscreenControls:(NSInteger)onscreenControls
+           keyboardToggleFingers:(NSInteger)keyboardToggleFingers
+             swipeExitScreenEdge:(NSInteger)swipeExitScreenEdge
+             swipeToExitDistance:(CGFloat)swipeToExitDistance
+       liftStreamViewForKeyboard:(BOOL)liftStreamViewForKeyboard
+             showKeyboardToolbar:(BOOL)showKeyboardToolbar
                    optimizeGames:(BOOL)optimizeGames
                  multiController:(BOOL)multiController
                  swapABXYButtons:(BOOL)swapABXYButtons
@@ -77,6 +82,11 @@ - (void) saveSettingsWithBitrate:(NSInteger)bitrate
         settingsToSave.width = [NSNumber numberWithInteger:width];
         settingsToSave.audioConfig = [NSNumber numberWithInteger:audioConfig];
         settingsToSave.onscreenControls = [NSNumber numberWithInteger:onscreenControls];
+        settingsToSave.keyboardToggleFingers = [NSNumber numberWithInteger:(uint16_t)keyboardToggleFingers];
+        settingsToSave.swipeExitScreenEdge = [NSNumber numberWithInteger:(uint32_t)swipeExitScreenEdge];
+        settingsToSave.swipeToExitDistance = [NSNumber numberWithFloat:swipeToExitDistance];
+        settingsToSave.liftStreamViewForKeyboard = liftStreamViewForKeyboard;
+        settingsToSave.showKeyboardToolbar = showKeyboardToolbar;
         settingsToSave.optimizeGames = optimizeGames;
         settingsToSave.multiController = multiController;
         settingsToSave.swapABXYButtons = swapABXYButtons;

Limelight/Database/TemporarySettings.h

@@ -18,6 +18,9 @@
 @property (nonatomic, retain) NSNumber * width;
 @property (nonatomic, retain) NSNumber * audioConfig;
 @property (nonatomic, retain) NSNumber * onscreenControls;
+@property (nonatomic, retain) NSNumber * keyboardToggleFingers;
+@property (nonatomic, retain) NSNumber * swipeExitScreenEdge;
+@property (nonatomic, retain) NSNumber * swipeToExitDistance;
 @property (nonatomic, retain) NSString * uniqueId;
 @property (nonatomic) enum {
     CODEC_PREF_AUTO,
@@ -34,6 +37,9 @@
 @property (nonatomic) BOOL btMouseSupport;
 @property (nonatomic) BOOL absoluteTouchMode;
 @property (nonatomic) BOOL statsOverlay;
+@property (nonatomic) BOOL liftStreamViewForKeyboard;
+@property (nonatomic) BOOL showKeyboardToolbar;
+
 
 - (id) initFromSettings:(Settings*)settings;
 

Limelight/Database/TemporarySettings.m

@@ -85,6 +85,11 @@ - (id) initFromSettings:(Settings*)settings {
     self.btMouseSupport = settings.btMouseSupport;
     self.absoluteTouchMode = settings.absoluteTouchMode;
     self.statsOverlay = settings.statsOverlay;
+    self.keyboardToggleFingers = settings.keyboardToggleFingers;
+    self.swipeExitScreenEdge = settings.swipeExitScreenEdge;
+    self.swipeToExitDistance = settings.swipeToExitDistance;
+    self.liftStreamViewForKeyboard = settings.liftStreamViewForKeyboard;
+    self.showKeyboardToolbar = settings.showKeyboardToolbar;
 #endif
     self.uniqueId = settings.uniqueId;
 

Limelight/Input/AbsoluteTouchHandler.m

@@ -38,83 +38,85 @@ - (id)initWithView:(StreamView*)view {
     return self;
 }
 
-- (void)onLongPressStart:(NSTimer*)timer {
-    // Raise the left click and start a right click
-    LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT);
-    LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_RIGHT);
-}
-
-- (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
-    // Ignore touch down events with more than one finger
-    if ([[event allTouches] count] > 1) {
-        return;
-    }
-
-    UITouch* touch = [touches anyObject];
-    CGPoint touchLocation = [touch locationInView:view];
-
-    // Don't reposition for finger down events within the deadzone. This makes double-clicking easier.
-    if (touch.timestamp - lastTouchUp.timestamp > DOUBLE_TAP_DEAD_ZONE_DELAY ||
-        sqrt(pow((touchLocation.x / view.bounds.size.width) - (lastTouchUpLocation.x / view.bounds.size.width), 2) +
-             pow((touchLocation.y / view.bounds.size.height) - (lastTouchUpLocation.y / view.bounds.size.height), 2)) > DOUBLE_TAP_DEAD_ZONE_DELTA) {
-        [view updateCursorLocation:touchLocation isMouse:NO];
-    }
-
-    // Press the left button down
-    LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT);
-
-    // Start the long press timer
-    longPressTimer = [NSTimer scheduledTimerWithTimeInterval:LONG_PRESS_ACTIVATION_DELAY
-                                                      target:self
-                                                    selector:@selector(onLongPressStart:)
-                                                    userInfo:nil
-                                                     repeats:NO];
-
-    lastTouchDown = touch;
-    lastTouchDownLocation = touchLocation;
-}
-
-- (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event {
-    // Ignore touch move events with more than one finger
-    if ([[event allTouches] count] > 1) {
-        return;
-    }
-
-    UITouch* touch = [touches anyObject];
-    CGPoint touchLocation = [touch locationInView:view];
-
-    if (sqrt(pow((touchLocation.x / view.bounds.size.width) - (lastTouchDownLocation.x / view.bounds.size.width), 2) +
-             pow((touchLocation.y / view.bounds.size.height) - (lastTouchDownLocation.y / view.bounds.size.height), 2)) > LONG_PRESS_ACTIVATION_DELTA) {
-        // Moved too far since touch down. Cancel the long press timer.
-        [longPressTimer invalidate];
-        longPressTimer = nil;
-    }
-
-    [view updateCursorLocation:[[touches anyObject] locationInView:view] isMouse:NO];
-}
-
-- (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
-    // Only fire this logic if all touches have ended
-    if ([[event allTouches] count] == [touches count]) {
-        // Cancel the long press timer
-        [longPressTimer invalidate];
-        longPressTimer = nil;
-
-        // Left button up on finger up
-        LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT);
-
-        // Raise right button too in case we triggered a long press gesture
-        LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_RIGHT);
-
-        // Remember this last touch for touch-down deadzoning
-        lastTouchUp = [touches anyObject];
-        lastTouchUpLocation = [lastTouchUp locationInView:view];
-    }
-}
-
-- (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event {
-    // Treat this as a normal touchesEnded event
-    [self touchesEnded:touches withEvent:event];
-}
+//- (void)onLongPressStart:(NSTimer*)timer {
+//    // Raise the left click and start a right click
+//    LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT);
+//    LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_RIGHT);
+//}
+//
+//- (void)touchesBegan:(NSSet *)touches withEvent:(UIEvent *)event {
+//    // Ignore touch down events with more than one finger
+//    return;
+//    if ([[event allTouches] count] > 1) {
+//        return;
+//    }
+//    
+//    UITouch* touch = [touches anyObject];
+//    CGPoint touchLocation = [touch locationInView:view];
+//    
+//    // Don't reposition for finger down events within the deadzone. This makes double-clicking easier.
+//    if (touch.timestamp - lastTouchUp.timestamp > DOUBLE_TAP_DEAD_ZONE_DELAY ||
+//        sqrt(pow((touchLocation.x / view.bounds.size.width) - (lastTouchUpLocation.x / view.bounds.size.width), 2) +
+//             pow((touchLocation.y / view.bounds.size.height) - (lastTouchUpLocation.y / view.bounds.size.height), 2)) > DOUBLE_TAP_DEAD_ZONE_DELTA) {
+//        [view updateCursorLocation:touchLocation isMouse:NO];
+//    }
+//    
+//    // Press the left button down
+//    LiSendMouseButtonEvent(BUTTON_ACTION_PRESS, BUTTON_LEFT);
+//    
+//    // Start the long press timer
+//    longPressTimer = [NSTimer scheduledTimerWithTimeInterval:LONG_PRESS_ACTIVATION_DELAY
+//                                                      target:self
+//                                                    selector:@selector(onLongPressStart:)
+//                                                    userInfo:nil
+//                                                     repeats:NO];
+//    
+//    lastTouchDown = touch;
+//    lastTouchDownLocation = touchLocation;
+//}
+//
+//- (void)touchesMoved:(NSSet *)touches withEvent:(UIEvent *)event {
+//    // Ignore touch move events with more than one finger
+//    return;
+//    if ([[event allTouches] count] > 1) {
+//        return;
+//    }
+//    
+//    UITouch* touch = [touches anyObject];
+//    CGPoint touchLocation = [touch locationInView:view];
+//    
+//    if (sqrt(pow((touchLocation.x / view.bounds.size.width) - (lastTouchDownLocation.x / view.bounds.size.width), 2) +
+//             pow((touchLocation.y / view.bounds.size.height) - (lastTouchDownLocation.y / view.bounds.size.height), 2)) > LONG_PRESS_ACTIVATION_DELTA) {
+//        // Moved too far since touch down. Cancel the long press timer.
+//        [longPressTimer invalidate];
+//        longPressTimer = nil;
+//    }
+//    
+//    [view updateCursorLocation:[[touches anyObject] locationInView:view] isMouse:NO];
+//}
+//
+//- (void)touchesEnded:(NSSet *)touches withEvent:(UIEvent *)event {
+//    // Only fire this logic if all touches have ended
+//    if ([[event allTouches] count] == [touches count]) {
+//        // Cancel the long press timer
+//        [longPressTimer invalidate];
+//        longPressTimer = nil;
+//
+//        // Left button up on finger up
+//        LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_LEFT);
+//
+//        // Raise right button too in case we triggered a long press gesture
+//        LiSendMouseButtonEvent(BUTTON_ACTION_RELEASE, BUTTON_RIGHT);
+//        
+//        // Remember this last touch for touch-down deadzoning
+//        lastTouchUp = [touches anyObject];
+//        lastTouchUpLocation = [lastTouchUp locationInView:view];
+//    }
+//}
+//
+//- (void)touchesCancelled:(NSSet *)touches withEvent:(UIEvent *)event {
+//    // Treat this as a normal touchesEnded event
+//    [self touchesEnded:touches withEvent:event];
+//}
 
 @end

Limelight/Input/CustomTapGestureRecognizer.h

@@ -0,0 +1,22 @@
+//
+//  CustomTapGestureRecognizer.h
+//  Moonlight
+//
+//  Created by ZWM on 2024/5/15.
+//  Copyright © 2024 Moonlight Game Streaming Project. All rights reserved.
+//
+
+#ifndef CustomTapGestureRecognizer_h
+#define CustomTapGestureRecognizer_h
+
+
+
+@interface CustomTapGestureRecognizer : UIGestureRecognizer
+
+@property (nonatomic, assign) uint8_t numberOfTouchesRequired;
+@property (nonatomic, assign) double tapDownTimeThreshold;
+@property (atomic, readonly) CGFloat lowestTouchPointHeight;
+
+
+@end
+#endif /* CustomTapGestureRecognizer_h */

Limelight/Input/CustomTapGestureRecognizer.m

@@ -0,0 +1,57 @@
+//
+//  CustomTapGestureRecognizer.m
+//  Moonlight
+//
+//  Created by ZWM on 2024/5/15.
+//  Copyright © 2024 Moonlight Game Streaming Project. All rights reserved.
+//
+
+#import <Foundation/Foundation.h>
+#import <UIKit/UIGestureRecognizerSubclass.h>
+#import "CustomTapGestureRecognizer.h"
+
+@implementation CustomTapGestureRecognizer
+
+static NSTimeInterval multiFingerDownTime;
+static bool multiFingerDown;
+static CGFloat screenHeightInPoints;
+static CGFloat lowestTouchPointYCoord = 0.0;
+
+- (instancetype)initWithTarget:(nullable id)target action:(nullable SEL)action {
+    self = [super initWithTarget:target action:action];
+    screenHeightInPoints = CGRectGetHeight([[UIScreen mainScreen] bounds]);
+    return self;
+}
+
+- (void)touchesBegan:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
+    // Check if the number of touches and taps meets the required criteria
+    if ([[event allTouches] count] == _numberOfTouchesRequired) {
+        multiFingerDownTime = CACurrentMediaTime();
+        multiFingerDown = true;
+
+        for(UITouch *touch in [event allTouches]){
+            if(lowestTouchPointYCoord < [touch locationInView:self.view].y) lowestTouchPointYCoord = [touch locationInView:self.view].y;
+        }
+        _lowestTouchPointHeight = screenHeightInPoints - lowestTouchPointYCoord;
+        self.state = UIGestureRecognizerStatePossible;
+    }
+}
+
+- (void)touchesEnded:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
+    // [super touchesEnded:touches withEvent:event];
+    if(multiFingerDown && [[event allTouches] count] == [touches count]){
+        multiFingerDown = false;
+        if((CACurrentMediaTime() - multiFingerDownTime) < _tapDownTimeThreshold / 1000){
+            lowestTouchPointYCoord = 0.0; //reset for next recognition
+            self.state = UIGestureRecognizerStateRecognized;
+        }
+    }
+}
+
++ (CGFloat)lowestTouchPointHeight{
+    @synchronized (self){
+        return self.lowestTouchPointHeight;
+    }
+}
+
+@end

Limelight/Input/NativeTouchHandler.h

@@ -0,0 +1,27 @@
+//
+//  NativeTouchHandler.h
+//  Moonlight
+//
+//  Created by ZWM on 2024/5/14.
+//  Copyright © 2024 Moonlight Game Streaming Project. All rights reserved.
+//
+
+#import "StreamView.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface NativeTouchHandler : UIResponder
+
++ (NSMutableDictionary* )initializePointerIdDict;
++ (NSMutableSet* )initializePointerIdSet;
++ (void)populatePointerId:(UITouch*)event;
++ (void)removePointerId:(UITouch*)event;
++ (uint32_t) retrievePointerIdFromDict:(UITouch*)event;
+
+
+-(id)initWithView:(StreamView*)view;
+
+@end
+
+NS_ASSUME_NONNULL_END
+