add RunInDomain option to JSTestSettings (#61)

* add RunInDomain option to JSTestSettings

* add tests and fix tests

* fix arg tests

* fix no domain test

README.md

@@ -7,11 +7,13 @@
 jstestadapter is a JavaScript test adapter extension for [Visual Studio Test Platform](https://github.com/Microsoft/vstest). jstest with vstest can be used as a command line tool to run tests written in mocha, jasmine or jest.
 
 ### Install
+
 ```bash
 npm install --save-dev jstestadapter
 ```
 
 ### Usage
+
 ```bash
 # Testing with default test framework, Jasmine
 path/to/vstest.console.exe --TestAdapterPath:./node_modules/jstestadapter/ path/to/test.1.js path/to/test.2.js
@@ -28,21 +30,26 @@ path/to/vstest.console.exe --TestAdapterPath:./node_modules/jstestadapter/ path/
 Option                  |  Usage                                                                                | Default
 ----------------------- | ------------------------------------------------------------------------------------- | --------
 TestFramework           | One of the following test frameworks for execution: Jasmine/Mocha/Jest                | Jasmine
+RunInDomain             | Run JavaScript tests in a node [Domain](https://nodejs.org/api/domain.html). _Note: If set to false the test must complete all execution before returning control. i.e. all callbacks such as setTimeout must be completed before the test completes execution._ | true
 DebugLogs               | Enable debug logs for JavaScript test runner                                          | false
 DebugFilePath           | Path for diagnostic logs                                                              | ""
 TestFrameworkConfigJson | Override test framework configurations (Specific to the testframework) in json format | {}
-NodeModulesPath         | Custom path to node_modules                                                         | ""
+NodeModulesPath         | Custom path to node_modules                                                           | ""
+
+#### RunSettings can be provided through the the vstest cli itself
 
-#### RunSettings can be provided through the the vstest cli itself:
 ```bash
 vstest.console.exe --Isolation --TestAdapterPath:<path> <files> -- JSTest.DebugLogs=true JSTest.TestFramework=mocha
 ```
 
-#### Using RunSettings xml defined for vstest:
+#### Using RunSettings xml defined for vstest
+
 ```bash
 vstest.console.exe --Settings:RunSettings.xml --TestAdapterPath:<path> <files>
 ```
+
 With RunSettings.xml:
+
 ```xml
 <RunSettings>
     <JSTest>
@@ -61,6 +68,7 @@ With RunSettings.xml:
 For uploading test result attachments along with tests checkout [karanjitsingh/jstestcontext](https://github.com/karanjitsingh/jstestcontext)
 
 ### Building from source
+
 ```bash
 # Build binaries and javascript to `./artifacts/Debug/net451/` along with the package tarball in `./artifacts/Debug`
 .\scripts\build.ps1
@@ -94,10 +102,8 @@ For uploading test result attachments along with tests checkout [karanjitsingh/j
 | -test          | "test filter"                      | Test filter                             | -       |
 | -vstest        | \path\to\custom\vstest.console.exe | Path to vstest.console.exe              | D:\vstest\artifacts\Debug\net451\win7-x64\vstest.console.exe  |
 
-
 ---
 
-
 ##### Tested with framework versions
 
 ![npm (tag)](https://img.shields.io/npm/v/jasmine/latest.svg?label=jasmine%40latest)<br />

src/JSTest.Runner/ArgProcessor.ts

@@ -5,12 +5,16 @@ import { CLIArgs } from './TestRunner/CLIArgs';
 
      export function processCLIArgs(env: IEnvironment): CLIArgs {
 
+        let runInDomain = false;
         let debugEnabled = false;
         let debugFilePath = '';
 
          for (let i = 4; i < env.argv.length; i++) {
             if (env.argv[i].startsWith('--')) {
                 switch (env.argv[i].substr(2).toLowerCase()) {
+                    case 'runindomain':
+                        runInDomain = true;
+                        break;
                     case 'diag':
                         debugEnabled = true;
                         if (env.argv[++i]) {
@@ -29,7 +33,8 @@ import { CLIArgs } from './TestRunner/CLIArgs';
             ip: env.argv[2],
             port: Number(env.argv[3]),
             traceEnabled: debugEnabled,
-            traceFilePath: debugFilePath
+            traceFilePath: debugFilePath,
+            runInDomain: runInDomain
         };
     }
 } 

src/JSTest.Runner/ObjectModel/JSTestSettings.ts

@@ -10,6 +10,7 @@ export class JSTestSettings {
     public readonly TestFrameworkConfigJson: JSON;
     public readonly AttachmentsFolder: string;
     public readonly CoverageEnabled: boolean;
+    public readonly RunInDomain: boolean;
 
     constructor(json: any, environment: IEnvironment) {
         this.JavaScriptTestFramework = -1;
@@ -37,6 +38,7 @@ export class JSTestSettings {
         }
 
         this.CoverageEnabled = json.CodeCoverageEnabled === true;
+        this.RunInDomain = json.RunInDomain === true;
 
         // Set attachments folder and environment variable
         // TODO: we should probably get the env key from jstestcontext package

src/JSTest.Runner/TestRunner/CLIArgs.ts

@@ -3,4 +3,5 @@ export interface CLIArgs {
     port: number;
     traceEnabled: boolean;
     traceFilePath: string;
+    runInDomain: boolean;
 }

src/JSTest.Runner/TestRunner/ExecutionManagers/TestSessionManager.ts

@@ -17,22 +17,24 @@ export class TestSessionManager {
     private testSessionIterator: IterableIterator<TestSession>;
     private sessionCompleteCount: number;
     private sessionCount: number;
+    protected runInDomain: boolean;
     public onAllSessionsComplete: IEvent<IEventArgs>;
 
     public static instance: TestSessionManager;
 
-    protected constructor(environment: IEnvironment) {
+    protected constructor(environment: IEnvironment, runInDomain: boolean) {
         this.sessionCount = 0;
         this.sessionCompleteCount = 0;
         this.onAllSessionsComplete = environment.createEvent();
         this.testSessionBucket = new Map();
         this.testSessionIterator = this.testSessionBucket.values();
+        this.runInDomain = runInDomain;
     }
 
-    public static INITIALIZE(environment: IEnvironment) {
+    public static INITIALIZE(environment: IEnvironment, runInDomain: boolean) {
         if (!TestSessionManager.instance) {
             EqtTrace.info(`TestSessionManager: initializing`);
-            TestSessionManager.instance = new TestSessionManager(environment);
+            TestSessionManager.instance = new TestSessionManager(environment, runInDomain);
         }
     }
 
@@ -75,7 +77,7 @@ export class TestSessionManager {
     }
 
     public executeJobs() {
-        this.runSessionInDomain(this.testSessionIterator.next().value);
+        this.runSession(this.testSessionIterator.next().value);
     }
 
     public updateSessionEventArgs(args: TestSessionEventArgs) {
@@ -88,6 +90,24 @@ export class TestSessionManager {
         return this.testSessionBucket.get(sessionId).TestSessionEventArgs;
     }
 
+    protected runSession(testSession: TestSession) {
+        if (this.runInDomain) {
+            return this.runSessionInDomain(testSession);
+        } else {
+            return this.runSessionNoDomain(testSession);
+        }
+    }
+
+    protected runSessionNoDomain(testSession: TestSession) {
+        try {
+            EqtTrace.info(`TestSessionManager: Executing session with no domain`);
+            testSession.Job();
+        } catch (err) {
+            EqtTrace.error('TestSessionManager: error running session outside of domain', err);
+            this.sessionError(testSession, err);
+        }
+    }
+
     protected runSessionInDomain(testSession: TestSession) {
         // tslint:disable-next-line:no-require-imports
         const domain = require('domain');

src/JSTest.Runner/TestRunner/TestRunner.ts

@@ -125,6 +125,6 @@ export class TestRunner {
 
     private initializeSingletons() {
         TestFrameworkFactory.INITIALIZE(this.environment);
-        TestSessionManager.INITIALIZE(this.environment);
+        TestSessionManager.INITIALIZE(this.environment, this.cliArgs.runInDomain);
     }
 }

src/JSTest/JSProcess.cs

@@ -50,9 +50,11 @@ public JSProcess()
         }
 
         public bool IsAlive => this.process != null && !this.process.HasExited;
-        
+
         public string DebugFilePath { get; set; }
 
+        public bool RunInDomain { get; set; }
+
         public void EnableDebugLogs(string debugFilePath)
         {
             this.debugEnabled = true;
@@ -62,7 +64,7 @@ public void EnableDebugLogs(string debugFilePath)
         public bool LaunchProcess(TestProcessStartInfo startInfo, ProcessCallbacks processStreamCallbacks)
         {
             var endpoint = this.InitializeChannel();
-            
+
             var process = new Process();
             try
             {
@@ -156,10 +158,11 @@ private void InitializeStartInfo(Process process, TestProcessStartInfo startInfo
         {
             process.StartInfo.FileName = startInfo.FileName;
             process.StartInfo.WorkingDirectory = startInfo.WorkingDirectory;
-            process.StartInfo.Arguments = string.Format("{0} {1} {2} {3}",
+            process.StartInfo.Arguments = string.Format("{0} {1} {2} {3} {4}",
                                                         startInfo.Arguments,
                                                         endPoint.Address,
                                                         endPoint.Port,
+                                                        this.GetTestSessionArgs(),
                                                         this.GetDebugArg());
 
             foreach (var entry in startInfo.EnvironmentVariables)
@@ -189,13 +192,24 @@ private IPEndPoint InitializeChannel()
             return endpoint;
         }
 
+        private string GetTestSessionArgs()
+        {
+            if (!this.RunInDomain)
+            {
+                return string.Empty;
+            }
+
+            return "--runInDomain";
+        }
+
         private string GetDebugArg()
         {
             if (this.debugEnabled)
             {
                 var arg = "--diag";
 
-                try {
+                try
+                {
                     if (File.Exists(this.debugFilePath) || Directory.Exists(Path.GetDirectoryName(this.debugFilePath)) || Directory.Exists(this.debugFilePath))
                     {
                         arg += " " + Uri.EscapeDataString(this.debugFilePath);

src/JSTest/RuntimeManager.cs

@@ -40,6 +40,8 @@ public TestRuntimeManager(JSTestSettings settings, TestRunEvents testRunEvents)
             this.settings = settings;
             this.testRunEvents = testRunEvents;
 
+            this.jsProcess.RunInDomain = this.settings.RunInDomain;
+
             if (this.settings.DebugLogs)
             {
                 this.jsProcess.EnableDebugLogs(this.settings.DebugFilePath);
@@ -73,13 +75,13 @@ public int GetProcessId()
         private Action<object, string> ProcessErrorReceived => (process, data) =>
         {
             EqtTrace.Error("RuntimeManager: Node {0} StdErr: {1}", this.jsProcess.ProcessId, data);
-            
+
             if (!string.IsNullOrEmpty(data))
             {
                 Console.WriteLine("JSTest: {0} StdErr: {1}", this.jsProcess.ProcessId, data);
             }
         };
-        
+
         public Task CleanProcessAsync()
         {
             try
@@ -150,7 +152,7 @@ private void InitializeCommunication(CancellationToken cancellationToken)
             {
                 EqtTrace.Verbose("RuntimeManager: Initializing communication with client process.");
                 var connectionStopwatch = Stopwatch.StartNew();
-                
+
                 // Start the message loop
                 Task.Run(() => { this.MessageLoopAsync(this.jsProcess.CommunicationChannel, cancellationToken); });
                 this.jsProcess.CommunicationChannel.SendMessage(MessageType.TestRunSettings, settings);

src/JSTest/Settings/JSTestSettings.cs

@@ -16,6 +16,8 @@ public class JSTestSettings
 
         #region JS Runner Specific Settings
 
+        public bool RunInDomain { get; set; }
+
         public bool DebugLogs { get; set; }
 
         public string DebugFilePath { get; set; }
@@ -98,6 +100,7 @@ public JSTestSettings()
             this.JavaScriptTestFramework = JSTestFramework.Jasmine;
             this.Discovery = false;
             this.RunInParallel = true;
+            this.RunInDomain = true;
             this.DebugLogs = false;
             this.NodePath = string.Empty;
             this.NodeModulesPath = string.Empty;

test/JSTest.Runner.UnitTests/TestRunner/ArgProcessorTests.ts

@@ -1,4 +1,5 @@
 import * as assert from 'assert';
+
 import { ArgProcessor } from '../../../src/JSTest.Runner/ArgProcessor';
 import { CLIArgs } from '../../../src/JSTest.Runner/TestRunner/CLIArgs';
 
@@ -16,7 +17,8 @@ describe('ArgProcessor Suite', () => {
             ip: '192.168.1.1',
             port: '9999',
             traceEnabled: false,
-            traceFilePath: ''
+            traceFilePath: '',
+            runInDomain: false
         });
     });
 
@@ -28,7 +30,8 @@ describe('ArgProcessor Suite', () => {
             ip: '192.168.1.1',
             port: '9999',
             traceEnabled: true,
-            traceFilePath: ''
+            traceFilePath: '',
+            runInDomain: false
         });
     });
 
@@ -40,7 +43,21 @@ describe('ArgProcessor Suite', () => {
             ip: '192.168.1.1',
             port: '9999',
             traceEnabled: true,
-            traceFilePath: 'file'
+            traceFilePath: 'file',
+            runInDomain: false
+        });
+    });
+
+    it('Will enable running in domain for logs for --runInDomain', () => {
+        mockEnv.argv = [process.env[0], process.env[1], '192.168.1.1', '9999', '--runInDomain'];
+        const args: CLIArgs = ArgProcessor.processCLIArgs(mockEnv);
+
+        assert.deepEqual(args, {
+            ip: '192.168.1.1',
+            port: '9999',
+            traceEnabled: false,
+            traceFilePath: '',
+            runInDomain: true
         });
     });
 

test/JSTest.Runner.UnitTests/TestRunner/ExecutionManager/BaseExecutionManagerTests.ts

@@ -1,16 +1,17 @@
+import * as Assert from 'assert';
+
 import { Environment } from '../../../../src/JSTest.Runner/Environment/Node/Environment';
+import { Event } from '../../../../src/JSTest.Runner/Events/Event';
 import { TestFrameworkFactory } from '../../../../src/JSTest.Runner/TestRunner/TestFrameworks/TestFrameworkFactory';
 import { TestSessionManager } from '../../../../src/JSTest.Runner/TestRunner/ExecutionManagers/TestSessionManager';
-import { Event } from '../../../../src/JSTest.Runner/Events/Event';
-import * as Assert from 'assert';
 import { TestableBaseExecutionManager } from './Testable';
 
 describe('BaseExecutionManager Suite', () => {
     let executionManager: TestableBaseExecutionManager;
     const environment = new Environment();
 
     TestFrameworkFactory.INITIALIZE(environment);
-    TestSessionManager.INITIALIZE(environment);
+    TestSessionManager.INITIALIZE(environment, true);
 
     before(() => {
         executionManager = new TestableBaseExecutionManager(environment);

test/JSTest.Runner.UnitTests/TestRunner/ExecutionManager/DiscoveryManagerTests.ts

@@ -26,7 +26,7 @@ describe('DiscoveryManager Suite', () => {
     const environment = new Environment();
 
     const testableTestFrameworkFactory = new TestableTestFrameworkFactory(environment);
-    const testableTestSessionManager = new TestableTestSessionManager(environment);
+    const testableTestSessionManager = new TestableTestSessionManager(environment, true);
 
     before(() => {
         mockFactory = Mock.ofInstance(testableTestFrameworkFactory);

test/JSTest.Runner.UnitTests/TestRunner/ExecutionManager/ExecutionManagerTests.ts

@@ -35,7 +35,7 @@ describe('ExecutionManager Suite', () => {
     const environment = new Environment();
 
     const testableTestFrameworkFactory = new TestableTestFrameworkFactory(environment);
-    const testableTestSessionManager = new TestableTestSessionManager(environment);
+    const testableTestSessionManager = new TestableTestSessionManager(environment, true);
 
     before(() => {
         mockFactory = Mock.ofInstance(testableTestFrameworkFactory);