fix: Update simulator preferences handling

.azure-pipelines.yml

@@ -19,8 +19,8 @@ parameters:
     xcodeVersion: 10.3
     iosDeviceName: iPhone X
     vmImage: macOS-10.14
-  - iosVersion: 13
-    xcodeVersion: 11
+  - iosVersion: 13.2
+    xcodeVersion: 11.2
     iosDeviceName: iPhone 11
     vmImage: macOS-10.15
   - iosVersion: 13.7

azure-templates/base_unit_test_job.yml

@@ -14,6 +14,8 @@ jobs:
     - task: NodeTool@0
       inputs:
         versionSpec: "$(DEFAULT_NODE_VERSION)"
+    - script: ls /Applications
+      displayName: List Apps
     - script: sudo xcode-select --switch "/Applications/Xcode_$(XCODE_VERSION).app/Contents/Developer"
       displayName: Prepare Env
     - script: xcrun simctl list

lib/defaults-utils.js

@@ -0,0 +1,154 @@
+import _ from 'lodash';
+import { DOMParser, XMLSerializer } from 'xmldom';
+import { exec } from 'teen_process';
+import B from 'bluebird';
+import log from './logger';
+
+/**
+ * Serializes the given value to plist-compatible
+ * XML representation, which is ready for further usage
+ * with `defaults` command line tool arguments
+ *
+ * @param {*} value The value to be serialized
+ * @param {boolean} serialize [true] Whether to serialize the resulting
+ * XML to string or to return raw HTMLElement instance
+ * @returns {HTMLElement|string} Either string or raw node representation of
+ * the given value
+ * @throws {TypeError} If it is not known how to serialize the given value
+ */
+function toXmlArg (value, serialize = true) {
+  let xmlDoc = null;
+
+  if (_.isPlainObject(value)) {
+    xmlDoc = new DOMParser().parseFromString('<dict></dict>', 'text/xml');
+    for (const [subKey, subValue] of _.toPairs(value)) {
+      const keyEl = xmlDoc.createElement('key');
+      const keyTextEl = xmlDoc.createTextNode(subKey);
+      keyEl.appendChild(keyTextEl);
+      xmlDoc.documentElement.appendChild(keyEl);
+      const subValueEl = xmlDoc.importNode(toXmlArg(subValue, false), true);
+      xmlDoc.documentElement.appendChild(subValueEl);
+    }
+  } else if (_.isArray(value)) {
+    xmlDoc = new DOMParser().parseFromString('<array></array>', 'text/xml');
+    for (const subValue of value) {
+      const subValueEl = xmlDoc.importNode(toXmlArg(subValue, false), true);
+      xmlDoc.documentElement.appendChild(subValueEl);
+    }
+  } else if (_.isBoolean(value)) {
+    xmlDoc = new DOMParser().parseFromString(value ? '<true/>' : '<false/>', 'text/xml');
+  } else if (_.isInteger(value)) {
+    xmlDoc = new DOMParser().parseFromString(`<integer>${value}</integer>`, 'text/xml');
+  } else if (_.isNumber(value)) {
+    xmlDoc = new DOMParser().parseFromString(`<real>${value}</real>`, 'text/xml');
+  } else if (_.isString(value)) {
+    xmlDoc = new DOMParser().parseFromString(`<string></string>`, 'text/xml');
+    const valueTextEl = xmlDoc.createTextNode(value);
+    xmlDoc.documentElement.appendChild(valueTextEl);
+  }
+
+  if (!xmlDoc) {
+    throw new TypeError(`The defaults value ${JSON.stringify(value)} cannot be written, ` +
+      `because it is not known how to handle its type`);
+  }
+
+  return serialize
+    ? new XMLSerializer().serializeToString(xmlDoc.documentElement)
+    : xmlDoc.documentElement;
+}
+
+/**
+ * Generates command line args for the `defaults`
+ * command line utility based on the given preference values mapping.
+ * See https://shadowfile.inode.link/blog/2018/06/advanced-defaults1-usage/
+ * for more details.
+ *
+ * @param {Object} valuesMap Preferences mapping
+ * @returns {Array<Array<string>>} Each item in the array
+ * is the `defaults write <plist>` command suffix
+ */
+function generateUpdateCommandArgs (valuesMap) {
+  const resultArgs = [];
+  for (const [key, value] of _.toPairs(valuesMap)) {
+    try {
+      if (_.isPlainObject(value)) {
+        const dictArgs = [key, '-dict-add'];
+        for (const [subKey, subValue] of _.toPairs(value)) {
+          dictArgs.push(subKey, toXmlArg(subValue));
+        }
+        resultArgs.push(dictArgs);
+      } else if (_.isArray(value)) {
+        const arrayArgs = [key, '-array-add'];
+        for (const subValue of arrayArgs) {
+          arrayArgs.push(toXmlArg(subValue));
+        }
+        resultArgs.push(arrayArgs);
+      } else {
+        resultArgs.push([key, toXmlArg(value)]);
+      }
+    } catch (e) {
+      if (e instanceof TypeError) {
+        log.warn(e.message);
+      } else {
+        throw e;
+      }
+    }
+  }
+  return resultArgs;
+}
+
+
+class NSUserDefaults {
+  constructor (plist) {
+    this.plist = plist;
+  }
+
+  /**
+   * Reads the content of the given plist file using plutil command line tool
+   * and serializes it to a JSON representation
+   *
+   * @returns {Object} The serialized plist content
+   * @throws {Error} If there was an error during serialization
+   */
+  async asJson () {
+    try {
+      const {stdout} = await exec('plutil', ['-convert', 'json', '-o', '-', this.plist]);
+      return JSON.parse(stdout);
+    } catch (e) {
+      throw new Error(`'${this.plist}' cannot be converted to JSON. Original error: ${e.stderr || e.message}`);
+    }
+  }
+
+  /**
+   * Updates the content of the given plist file.
+   * If the plist does not exist yet then it is going to be created.
+   *
+   * @param {Object} valuesMap Mapping of preference values to update.
+   * If any of item values are of dictionary type then only the first level dictionary gets
+   * updated. Everything below this level will be replaced. This is the known limitation
+   * of the `defaults` command line tool. A workaround for it would be to read the current
+   * preferences mapping first and merge it with this value.
+   * @throws {Error} If there was an error while updating the plist
+   */
+  async update (valuesMap) {
+    if (!_.isPlainObject(valuesMap)) {
+      throw new TypeError(`plist values must be a map. '${valuesMap}' is given instead`);
+    }
+    if (_.isEmpty(valuesMap)) {
+      return;
+    }
+
+    const commandArgs = generateUpdateCommandArgs(valuesMap);
+    try {
+      await B.all(commandArgs.map((args) => exec('defaults', ['write', this.plist, ...args])));
+    } catch (e) {
+      throw new Error(`Could not write defaults into '${this.plist}'. Original error: ${e.stderr || e.message}`);
+    }
+  }
+}
+
+
+export {
+  NSUserDefaults,
+  toXmlArg, generateUpdateCommandArgs,
+};

lib/simulator-xcode-6.js

@@ -470,18 +470,8 @@ class SimulatorXcode6 extends EventEmitter {
    * @property {?string} scaleFactor [null] - Defines the window scale value for the UI client window for the current Simulator.
    *   Equals to null by default, which keeps the current scale unchanged.
    *   It should be one of ['1.0', '0.75', '0.5', '0.33', '0.25'].
-   * @property {boolean} connectHardwareKeyboard [false] - Whether to connect the hardware keyboard to the
-   *   Simulator UI client. Defaults to false.
-   * @property {string} pasteboardAutomaticSync ['off'] - Whether to disable pasteboard sync with the
-   *   Simulator UI client or respect the system wide preference. 'on', 'off', or 'system' is available.
-   *   The sync increases launching simulator process time, but it allows system to sync pasteboard
-   *   with simulators. Follows system-wide preference if the value is 'system'.
-   *   Defaults to 'off'.
    * @property {number} startupTimeout [60000] - Number of milliseconds to wait until Simulator booting
    *   process is completed. The default timeout will be used if not set explicitly.
-   * @property {?boolean} tracePointer [false] - Whether to highlight touches on Simulator
-   *   screen. This is helpful while debugging automated tests or while observing the automation
-   *   recordings.
    */
 
   /**
@@ -492,17 +482,13 @@ class SimulatorXcode6 extends EventEmitter {
     opts = _.cloneDeep(opts);
     _.defaultsDeep(opts, {
       scaleFactor: null,
-      connectHardwareKeyboard: false,
-      pasteboardAutomaticSync: 'off',
-      tracePointer: false,
       startupTimeout: this.startupTimeout,
     });
 
     const simulatorApp = path.resolve(await getXcodePath(), 'Applications', this.simulatorApp);
     const args = [
       '-Fn', simulatorApp,
       '--args', '-CurrentDeviceUDID', this.udid,
-      '-RotateWindowWhenSignaledByGuest', '1',
     ];
 
     if (opts.scaleFactor) {
@@ -512,36 +498,6 @@ class SimulatorXcode6 extends EventEmitter {
       args.push(argumentName, opts.scaleFactor);
     }
 
-    if (_.isBoolean(opts.connectHardwareKeyboard)) {
-      args.push('-ConnectHardwareKeyboard', `${+opts.connectHardwareKeyboard}`);
-    }
-
-    if (opts.tracePointer === true) {
-      args.push(
-        '-ShowSingleTouches', '1',
-        '-ShowPinches', '1',
-        '-ShowPinchPivotPoint', '1',
-        '-HighlightEdgeGestures', '1'
-      );
-    }
-
-    switch (_.lowerCase(opts.pasteboardAutomaticSync)) {
-      case 'on':
-        args.push('-PasteboardAutomaticSync', '1');
-        break;
-      case 'off':
-        // Improve launching simulator performance
-        // https://github.com/WebKit/webkit/blob/master/Tools/Scripts/webkitpy/xcode/simulated_device.py#L413
-        args.push('-PasteboardAutomaticSync', '0');
-        break;
-      case 'system':
-        // Do not add -PasteboardAutomaticSync
-        break;
-      default:
-        log.warn(`['on', 'off' or 'system'] are available as the pasteboard automatic sync option. Defaulting to 'off'.`);
-        args.push('-PasteboardAutomaticSync', '0');
-    }
-
     log.info(`Starting Simulator UI with command: open ${args.join(' ')}`);
     try {
       await exec('open', args, {timeout: opts.startupTimeout});
@@ -684,7 +640,7 @@ class SimulatorXcode6 extends EventEmitter {
     log.debug('Attempting to launch and quit the simulator, to create directory structure');
     log.debug(`Will launch with Safari? ${safari}`);
 
-    await this.run(startupTimeout);
+    await this.run({startupTimeout});
 
     if (safari) {
       await this.openUrl('http://www.appium.io');

lib/simulator-xcode-9.js

@@ -1,11 +1,12 @@
 import SimulatorXcode8 from './simulator-xcode-8';
 import _ from 'lodash';
 import path from 'path';
-import { fs, plist, timing } from 'appium-support';
+import { fs, timing } from 'appium-support';
 import AsyncLock from 'async-lock';
 import log from './logger';
 import { waitForCondition, retryInterval } from 'asyncbox';
 import { toBiometricDomainComponent, getDeveloperRoot } from './utils.js';
+import { NSUserDefaults } from './defaults-utils';
 
 const SIMULATOR_SHUTDOWN_TIMEOUT = 15 * 1000;
 const startupLock = new AsyncLock();
@@ -53,6 +54,11 @@ class SimulatorXcode9 extends SimulatorXcode8 {
    * @property {?boolean} tracePointer [false] - Whether to highlight touches on Simulator
    * screen. This is helpful while debugging automated tests or while observing the automation
    * recordings.
+   * @property {string} pasteboardAutomaticSync ['off'] - Whether to disable pasteboard sync with the
+   * Simulator UI client or respect the system wide preference. 'on', 'off', or 'system' is available.
+   * The sync increases launching simulator process time, but it allows system to sync pasteboard
+   * with simulators. Follows system-wide preference if the value is 'system'.
+   * Defaults to 'off'.
    * @property {DevicePreferences} devicePreferences: preferences of the newly created Simulator
    * device
    */
@@ -71,6 +77,7 @@ class SimulatorXcode9 extends SimulatorXcode8 {
       isHeadless: false,
       startupTimeout: this.startupTimeout,
     });
+
     if (opts.scaleFactor) {
       opts.devicePreferences.SimulatorWindowLastScale = parseFloat(opts.scaleFactor);
     }
@@ -79,13 +86,34 @@ class SimulatorXcode9 extends SimulatorXcode8 {
     const commonPreferences = {
       RotateWindowWhenSignaledByGuest: true
     };
-    if (_.isBoolean(opts.connectHardwareKeyboard)) {
-      opts.devicePreferences.ConnectHardwareKeyboard = opts.connectHardwareKeyboard;
-      commonPreferences.ConnectHardwareKeyboard = opts.connectHardwareKeyboard;
+    if (_.isBoolean(opts.connectHardwareKeyboard) || _.isNil(opts.connectHardwareKeyboard)) {
+      opts.devicePreferences.ConnectHardwareKeyboard = opts.connectHardwareKeyboard ?? false;
+      commonPreferences.ConnectHardwareKeyboard = opts.connectHardwareKeyboard ?? false;
+    }
+    if (_.isBoolean(opts.tracePointer)) {
+      commonPreferences.ShowSingleTouches = opts.tracePointer;
+      commonPreferences.ShowPinches = opts.tracePointer;
+      commonPreferences.ShowPinchPivotPoint = opts.tracePointer;
+      commonPreferences.HighlightEdgeGestures = opts.tracePointer;
     }
-    if (!_.isEmpty(opts.devicePreferences) || !_.isEmpty(commonPreferences)) {
-      await this.updatePreferences(opts.devicePreferences, commonPreferences);
+    switch (_.lowerCase(opts.pasteboardAutomaticSync)) {
+      case 'on':
+        commonPreferences.PasteboardAutomaticSync = true;
+        break;
+      case 'off':
+        // Improve launching simulator performance
+        // https://github.com/WebKit/webkit/blob/master/Tools/Scripts/webkitpy/xcode/simulated_device.py#L413
+        commonPreferences.PasteboardAutomaticSync = false;
+        break;
+      case 'system':
+        // Do not add -PasteboardAutomaticSync
+        break;
+      default:
+        log.info(`['on', 'off' or 'system'] are available as the pasteboard automatic sync option. Defaulting to 'off'`);
+        commonPreferences.PasteboardAutomaticSync = false;
     }
+    await this.updatePreferences(opts.devicePreferences, commonPreferences);
+
     const timer = new timing.Timer().start();
     const shouldWaitForBoot = await startupLock.acquire(this.uiClientBundleId, async () => {
       const isServerRunning = await this.isRunning();
@@ -96,7 +124,7 @@ class SimulatorXcode9 extends SimulatorXcode8 {
           return false;
         }
         if (await this.killUIClient({pid: uiClientPid})) {
-          log.info(`Detected the Simulator UI client was running and killed it. Verifying the current Simulator state...`);
+          log.info(`Detected the Simulator UI client was running and killed it. Verifying the current Simulator state`);
         }
         try {
           // Stopping the UI client kills all running servers for some early XCode versions. This is a known bug
@@ -110,7 +138,8 @@ class SimulatorXcode9 extends SimulatorXcode8 {
           }
           return false;
         }
-        log.info(`Booting Simulator with UDID '${this.udid}' in headless mode. All UI-related capabilities are going to be ignored`);
+        log.info(`Booting Simulator with UDID '${this.udid}' in headless mode. ` +
+          `All UI-related capabilities are going to be ignored`);
         await this.boot();
       } else {
         if (isServerRunning && uiClientPid) {
@@ -242,25 +271,33 @@ class SimulatorXcode9 extends SimulatorXcode8 {
     }
     this.verifyDevicePreferences(devicePrefs);
     const plistPath = path.resolve(homeFolderPath, 'Library', 'Preferences', 'com.apple.iphonesimulator.plist');
-    if (!await fs.hasAccess(plistPath)) {
-      log.warn(`Simulator preferences file '${plistPath}' is not accessible. ` +
-        `Ignoring Simulator preferences update.`);
-      return false;
-    }
-    let newPrefs = {};
-    if (!_.isEmpty(devicePrefs)) {
-      newPrefs.DevicePreferences = {[this.udid.toUpperCase()]: devicePrefs};
-    }
-    newPrefs = _.merge(newPrefs, commonPrefs);
     return await preferencesPlistGuard.acquire(SimulatorXcode9.name, async () => {
+      const defaults = new NSUserDefaults(plistPath);
+      const prefsToUpdate = _.clone(commonPrefs);
       try {
-        const currentPlistContent = await plist.parsePlistFile(plistPath);
-        await plist.updatePlistFile(plistPath, _.merge(currentPlistContent, newPrefs), true);
-        log.debug(`Updated ${this.udid} Simulator preferences at '${plistPath}' with ${JSON.stringify(newPrefs)}`);
+        if (!_.isEmpty(devicePrefs)) {
+          let existingDevicePrefs;
+          const udidKey = this.udid.toUpperCase();
+          if (await fs.exists(plistPath)) {
+            const currentPlistContent = await defaults.asJson();
+            if (_.isPlainObject(currentPlistContent.DevicePreferences)
+                && _.isPlainObject(currentPlistContent.DevicePreferences[udidKey])) {
+              existingDevicePrefs = currentPlistContent.DevicePreferences[udidKey];
+            }
+          }
+          Object.assign(prefsToUpdate, {
+            DevicePreferences: {
+              [udidKey]: Object.assign({}, existingDevicePrefs || {}, devicePrefs)
+            }
+          });
+        }
+        await defaults.update(prefsToUpdate);
+        log.debug(`Updated ${this.udid} Simulator preferences at '${plistPath}' with ` +
+          JSON.stringify(prefsToUpdate));
         return true;
       } catch (e) {
         log.warn(`Cannot update ${this.udid} Simulator preferences at '${plistPath}'. ` +
-                 `Try to delete the file manually in order to reset it. Original error: ${e.message}`);
+          `Try to delete the file manually in order to reset it. Original error: ${e.message}`);
         return false;
       }
     });