fix(common): move cursor to the bottom right corner via nircmd

fixes #374

src/common/node/robot/nircmd.ts

@@ -1,5 +1,5 @@
 import { sleep } from '@/common';
-import { BreachProtocolRobotKeys } from './robot';
+import { BreachProtocolRobotKeys, RobotSettings } from './robot';
 import { WindowsRobot } from './win32';
 
 export class NirCmdRobot extends WindowsRobot {
@@ -8,6 +8,15 @@ export class NirCmdRobot extends WindowsRobot {
 
   protected readonly binPath = './resources/win32/nircmd/nircmd.exe';
 
+  constructor(
+    settings: RobotSettings,
+    private readonly scaling: number,
+    private readonly width: number,
+    private readonly height: number
+  ) {
+    super(settings);
+  }
+
   async activateGameWindow() {
     await this.bin(`win activate stitle ${this.gameWindowTitle}`);
     // Wait extra time as nircmd will not wait for window to be actually
@@ -25,9 +34,8 @@ export class NirCmdRobot extends WindowsRobot {
       await this.moveAway();
     }
 
-    const scaling = this.settings.useScaling ? this.scaling : 1;
-    const sX = (x - this.x) / scaling;
-    const sY = (y - this.y) / scaling;
+    const sX = x - this.x;
+    const sY = y - this.y;
     const r = await this.moveRelative(sX, sY);
 
     this.x = x;
@@ -37,17 +45,21 @@ export class NirCmdRobot extends WindowsRobot {
   }
 
   moveAway() {
-    this.x = 0;
-    this.y = 0;
+    this.x = this.width;
+    this.y = this.height;
 
-    return this.moveRelative(-9999, -9999);
+    return this.moveRelative(this.width, this.height);
   }
 
   pressKey(key: BreachProtocolRobotKeys) {
     return this.bin(`sendkeypress 0x${this.getMappedKey(key)}`);
   }
 
   private moveRelative(x: number, y: number) {
-    return this.bin(`sendmouse move ${x} ${y}`);
+    return this.bin(`sendmouse move ${this.scale(x)} ${this.scale(y)}`);
+  }
+
+  private scale(value: number) {
+    return Math.round(value / this.scaling);
   }
 }

src/common/node/robot/robot.ts

@@ -43,10 +43,7 @@ export abstract class BreachProtocolRobot {
 
   protected gameWindowTitle = 'Cyberpunk 2077';
 
-  constructor(
-    protected readonly settings: RobotSettings,
-    protected readonly scaling: number = 1
-  ) {}
+  constructor(protected readonly settings: RobotSettings) {}
 
   protected abstract getMappedKey(key: BreachProtocolRobotKeys): string;
 

src/core/ocr/bouding-box.test.ts

@@ -0,0 +1,64 @@
+import { BoudingBox } from './bounding-box';
+
+describe('BoudingBox', () => {
+  const horizontal = [
+    [1024, 768],
+    [1024, 1280],
+    [1152, 864],
+    [1280, 768],
+    [1280, 800],
+    [1280, 960],
+    [1280, 1024],
+    [1600, 1024],
+    [1600, 1200],
+    [1680, 1050],
+    [1920, 1200],
+    [1920, 1440],
+  ];
+
+  const vertical = [
+    [2560, 1080],
+    [3440, 1440],
+    [3840, 1080],
+  ];
+
+  const regular = [
+    [1280, 720],
+    [1360, 768],
+    [1366, 768],
+    [1600, 900],
+    [1920, 1080],
+    [2560, 1440],
+    [3840, 2160],
+  ];
+
+  it.each(horizontal)('should crop horizontal black bars(%ix%i)', (x, y) => {
+    const box = new BoudingBox(x, y);
+
+    expect(box.aspectRatio).toBe(BoudingBox.ASPECT_RATIO);
+    expect(box.width).toBe(x);
+    expect(box.height).toBe(y - 2 * box.top);
+    expect(box.left).toBe(0);
+    expect(box.top).toBe((y - box.height) / 2);
+  });
+
+  it.each(vertical)('should crop vertical black bars(%ix%i)', (x, y) => {
+    const box = new BoudingBox(x, y);
+
+    expect(box.aspectRatio).toBe(BoudingBox.ASPECT_RATIO);
+    expect(box.width).toBe(x - 2 * box.left);
+    expect(box.height).toBe(y);
+    expect(box.left).toBe((x - box.width) / 2);
+    expect(box.top).toBe(0);
+  });
+
+  it.each(regular)('should not crop 16:9 resolutions(%ix%i)', (x, y) => {
+    const box = new BoudingBox(x, y);
+
+    expect(box.aspectRatio).toBe(BoudingBox.ASPECT_RATIO);
+    expect(box.width).toBe(x);
+    expect(box.height).toBe(y);
+    expect(box.left).toBe(0);
+    expect(box.top).toBe(0);
+  });
+});

src/core/ocr/bounding-box.ts

@@ -0,0 +1,34 @@
+export class BoudingBox {
+  /** Expected aspect ratio of breach protocol. */
+  static readonly ASPECT_RATIO = 16 / 9;
+
+  private readonly ratio =
+    this.getAspectRatio(this.x, this.y) / BoudingBox.ASPECT_RATIO;
+
+  /** Width of the breach protocol. */
+  readonly width = this.ratio > 1 ? this.y * BoudingBox.ASPECT_RATIO : this.x;
+
+  /** Height of the breach protocol. */
+  readonly height = this.ratio < 1 ? this.x / BoudingBox.ASPECT_RATIO : this.y;
+
+  /** Distance in pixels from left edge to breach protocol. */
+  readonly left = (this.x - this.width) / 2;
+
+  /** Distance in pixels from top edge to breach protocol. */
+  readonly top = (this.y - this.height) / 2;
+
+  /** Aspect ratio of breach protocol. */
+  readonly aspectRatio = this.getAspectRatio(this.width, this.height);
+
+  constructor(public readonly x: number, public readonly y: number) {}
+
+  private getAspectRatio(x: number, y: number) {
+    // WXGA, very close to 16:9
+    // https://en.wikipedia.org/wiki/Graphics_display_resolution#WXGA
+    if (y === 768 && (x === 1366 || x === 1360)) {
+      return BoudingBox.ASPECT_RATIO;
+    }
+
+    return x / y;
+  }
+}

src/core/ocr/fragments/base.ts

@@ -1,5 +1,6 @@
 import { chunk, getClosest, Point, unique } from '@/common';
 import { HexCode, HEX_CODES } from '../../common';
+import { BoudingBox } from '../bounding-box';
 import { FragmentContainer, ImageContainer } from '../image-container';
 import {
   BreachProtocolRecognizer,
@@ -102,7 +103,8 @@ export abstract class BreachProtocolFragment<
 
   protected getFragmentBoundingBox() {
     const { p1, p2 } = this;
-    const { width, height, left, top } = this.container.getCroppedBoundingBox();
+    const { width: x, height: y } = this.container.dimensions;
+    const { width, height, left, top } = new BoudingBox(x, y);
 
     return {
       left: Math.round(p1.x * width + left),

src/core/ocr/image-container.ts

@@ -38,40 +38,11 @@ export interface FragmentContainer<T> {
 }
 
 export abstract class ImageContainer<T> {
-  /** Aspect ratio of breach protocol. */
-  static readonly ASPECT_RATIO = 16 / 9;
-
   abstract readonly instance: T;
 
   abstract readonly dimensions: Dimensions;
 
   abstract toFragmentContainer(
     config: FragmentContainerConfig
   ): FragmentContainer<T>;
-
-  /** Return aspect ratio for given resolution and handle edge cases. */
-  getAspectRatio(x: number, y: number) {
-    // WXGA, very close to 16:9
-    // https://en.wikipedia.org/wiki/Graphics_display_resolution#WXGA
-    if (y === 768 && (x === 1366 || x === 1360)) {
-      return ImageContainer.ASPECT_RATIO;
-    }
-
-    return x / y;
-  }
-
-  getCroppedBoundingBox() {
-    const { width: x, height: y } = this.dimensions;
-    // Resolution with ratio less than one have horizontal black
-    // bars, and ratio greater than one have vertical.
-    // Resolutions with ratio equal to 1 are in 16:9 aspect ratio
-    // and do not require cropping.
-    const ratio = this.getAspectRatio(x, y) / ImageContainer.ASPECT_RATIO;
-    const width = ratio > 1 ? y * ImageContainer.ASPECT_RATIO : x;
-    const height = ratio < 1 ? x / ImageContainer.ASPECT_RATIO : y;
-    const left = (x - width) / 2;
-    const top = (y - height) / 2;
-
-    return { width, height, left, top };
-  }
 }

src/core/ocr/index.ts

@@ -3,3 +3,4 @@ export * from './image-container';
 export * from './ocr';
 export * from './recognizer';
 export * from './result';
+export { BoudingBox } from './bounding-box';

src/core/ocr/ocr.test.ts

@@ -42,73 +42,6 @@ type Resolution =
   | '3440x1440'
   | '3840x2160';
 
-describe('image container', () => {
-  const aspectRatio = ImageContainer.ASPECT_RATIO;
-  const horizontal = [
-    [1024, 768],
-    [1024, 1280],
-    [1152, 864],
-    [1280, 768],
-    [1280, 800],
-    [1280, 960],
-    [1280, 1024],
-    [1600, 1024],
-    [1600, 1200],
-    [1680, 1050],
-    [1920, 1200],
-    [1920, 1440],
-  ];
-
-  const vertical = [
-    [2560, 1080],
-    [3440, 1440],
-    [3840, 1080],
-  ];
-
-  const regular = [
-    [1280, 720],
-    [1360, 768],
-    [1366, 768],
-    [1600, 900],
-    [1920, 1080],
-    [2560, 1440],
-    [3840, 2160],
-  ];
-
-  it.each(horizontal)('should crop horizontal black bars(%ix%i)', (x, y) => {
-    const container = new NoopImageContainer({ width: x, height: y });
-    const { width, height, left, top } = container.getCroppedBoundingBox();
-
-    expect(container.getAspectRatio(width, height)).toBe(aspectRatio);
-    expect(width).toBe(x);
-    expect(height).toBe(y - 2 * top);
-    expect(left).toBe(0);
-    expect(top).toBe((y - height) / 2);
-  });
-
-  it.each(vertical)('should crop vertical black bars(%ix%i)', (x, y) => {
-    const container = new NoopImageContainer({ width: x, height: y });
-    const { width, height, left, top } = container.getCroppedBoundingBox();
-
-    expect(container.getAspectRatio(width, height)).toBe(aspectRatio);
-    expect(width).toBe(x - 2 * left);
-    expect(height).toBe(y);
-    expect(left).toBe((x - width) / 2);
-    expect(top).toBe(0);
-  });
-
-  it.each(regular)('should not crop 16:9 resolutions(%ix%i)', (x, y) => {
-    const container = new NoopImageContainer({ width: x, height: y });
-    const { width, height, left, top } = container.getCroppedBoundingBox();
-
-    expect(container.getAspectRatio(width, height)).toBe(aspectRatio);
-    expect(width).toBe(x);
-    expect(height).toBe(y);
-    expect(left).toBe(0);
-    expect(top).toBe(0);
-  });
-});
-
 describe('raw data validation', () => {
   let recognizer: TestBreachProtocolRecognizer;
   let container: NoopImageContainer;

src/electron/worker/worker.ts

@@ -7,6 +7,7 @@ import {
   XDoToolRobot,
 } from '@/common/node';
 import {
+  BoudingBox,
   BreachProtocolBufferSizeFragment,
   BreachProtocolDaemonsFragment,
   BreachProtocolFragmentOptions,
@@ -310,12 +311,14 @@ export class BreachProtocolWorker {
         case 'ahk':
           return new AutoHotkeyRobot(this.settings);
         case 'nircmd':
-          const { activeDisplayId } = this.settings;
-          const { dpiScale } = this.displays.find(
+          const { activeDisplayId, useScaling } = this.settings;
+          const { dpiScale, width, height } = this.displays.find(
             (d) => d.id === activeDisplayId
           );
+          const scaling = useScaling ? dpiScale : 1;
+          const box = new BoudingBox(width, height);
 
-          return new NirCmdRobot(this.settings, dpiScale);
+          return new NirCmdRobot(this.settings, scaling, box.width, box.height);
         default:
           throw new Error(`Invalid engine "${this.settings.engine}" selected!`);
       }