diff --git a/src/reporter/index.js b/src/reporter/index.js index ee43c10684..64dbe5e794 100644 --- a/src/reporter/index.js +++ b/src/reporter/index.js @@ -45,6 +45,7 @@ export default class Reporter { testRunIds: [], screenshotPath: null, screenshots: [], + videos: [], quarantine: null, errs: [], warnings: [], @@ -72,6 +73,7 @@ export default class Reporter { unstable: reportItem.unstable, screenshotPath: reportItem.screenshotPath, screenshots: reportItem.screenshots, + videos: reportItem.videos, quarantine: reportItem.quarantine, skipped: reportItem.test.skip }; @@ -107,6 +109,9 @@ export default class Reporter { reportItem.screenshots = this.task.screenshots.getScreenshotsInfo(testRun.test); } + if (this.task.videos) + reportItem.videos = this.task.videos.getTestVideos(reportItem.test); + if (testRun.quarantine) { reportItem.quarantine = testRun.quarantine.attempts.reduce((result, errors, index) => { const passed = !errors.length; diff --git a/src/runner/task.js b/src/runner/task.js index 263dcb31a4..2fc1acd968 100644 --- a/src/runner/task.js +++ b/src/runner/task.js @@ -3,10 +3,10 @@ import moment from 'moment'; import AsyncEventEmitter from '../utils/async-event-emitter'; import BrowserJob from './browser-job'; import Screenshots from '../screenshots'; -import VideoRecorder from '../video-recorder'; import WarningLog from '../notifications/warning-log'; import FixtureHookController from './fixture-hook-controller'; import * as clientScriptsRouting from '../custom-client-scripts/routing'; +import Videos from '../video-recorder/videos'; export default class Task extends AsyncEventEmitter { constructor (tests, browserConnectionGroups, proxy, opts) { @@ -32,7 +32,7 @@ export default class Task extends AsyncEventEmitter { this.testStructure = this._prepareTestStructure(tests); if (this.opts.videoPath) - this.videoRecorders = this._createVideoRecorders(this.pendingBrowserJobs); + this.videos = new Videos(this.pendingBrowserJobs, this.opts, this.warningLog, this.timeStamp); } _assignBrowserJobEventHandlers (job) { @@ -109,12 +109,6 @@ export default class Task extends AsyncEventEmitter { }); } - _createVideoRecorders (browserJobs) { - const videoOptions = { timeStamp: this.timeStamp, ...this.opts.videoOptions }; - - return browserJobs.map(browserJob => new VideoRecorder(browserJob, this.opts.videoPath, videoOptions, this.opts.videoEncodingOptions, this.warningLog)); - } - unRegisterClientScriptRouting () { clientScriptsRouting.unRegister(this.proxy, this.clientScriptRoutes); } diff --git a/src/video-recorder/index.js b/src/video-recorder/index.js index 57c3346971..51b568e076 100644 --- a/src/video-recorder/index.js +++ b/src/video-recorder/index.js @@ -9,14 +9,17 @@ import WARNING_MESSAGES from '../notifications/warning-message'; import { getPluralSuffix, getConcatenatedValuesString, getToBeInPastTense } from '../utils/string'; import TestRunVideoRecorder from './test-run-video-recorder'; +import { EventEmitter } from 'events'; const DEBUG_LOGGER = debug('testcafe:video-recorder'); const VIDEO_EXTENSION = 'mp4'; const TEMP_DIR_PREFIX = 'video'; -export default class VideoRecorder { +export default class VideoRecorder extends EventEmitter { constructor (browserJob, basePath, opts, encodingOpts, warningLog) { + super(); + this.browserJob = browserJob; this.basePath = basePath; this.failedOnly = opts.failedOnly; @@ -160,12 +163,14 @@ export default class VideoRecorder { if (this.failedOnly && !testRunRecorder.hasErrors) return; - await this._saveFiles(testRunRecorder); - } - - async _saveFiles (testRunRecorder) { const videoPath = this._getTargetVideoPath(testRunRecorder); + await this._saveFiles(testRunRecorder, videoPath); + + this.emit('test-run-video-saved', { testRun: testRunRecorder.testRun, videoPath, singleFile: !!this.singleFile }); + } + + async _saveFiles (testRunRecorder, videoPath) { await makeDir(dirname(videoPath)); if (this.singleFile) diff --git a/src/video-recorder/videos.js b/src/video-recorder/videos.js new file mode 100644 index 0000000000..18c32fc1eb --- /dev/null +++ b/src/video-recorder/videos.js @@ -0,0 +1,42 @@ +import VideoRecorder from './index'; + +export default class Videos { + constructor (browserJobs, { videoPath, videoOptions, videoEncodingOptions }, warningLog, timeStamp) { + const options = { timeStamp: timeStamp, ...videoOptions }; + + this.recordings = {}; + + browserJobs.forEach(browserJob => { + const recorder = this._createVideoRecorder(browserJob, videoPath, options, videoEncodingOptions, warningLog); + + recorder.on('test-run-video-saved', args => this._addTestRunVideoInfo(args)); + }); + } + + getTestVideos (test) { + const rec = this.recordings[test.id]; + + return rec ? rec.runs : []; + } + + _createVideoRecorder (browserJob, videoPath, options, videoEncodingOptions, warningLog) { + return new VideoRecorder(browserJob, videoPath, options, videoEncodingOptions, warningLog); + } + + _addTestRunVideoInfo ({ testRun, videoPath, singleFile }) { + const testId = testRun.test.id; + let rec = this.recordings[testId]; + + if (!rec) { + rec = { runs: [] }; + + this.recordings[testId] = rec; + } + + rec.runs.push({ + testRunId: testRun.id, + videoPath, + singleFile + }); + } +} diff --git a/test/functional/fixtures/video-recording/test.js b/test/functional/fixtures/video-recording/test.js index ddd4be30ad..c3e670a773 100644 --- a/test/functional/fixtures/video-recording/test.js +++ b/test/functional/fixtures/video-recording/test.js @@ -1,19 +1,61 @@ -const expect = require('chai').expect; -const config = require('../../config'); +const expect = require('chai').expect; +const path = require('path'); +const { uniq } = require('lodash'); +const config = require('../../config'); const assertionHelper = require('../../assertion-helper.js'); +function customReporter (errs, videos) { + return () => { + return { + async reportTaskStart () { + }, + async reportFixtureStart () { + }, + async reportTestDone (name, testRunInfo) { + testRunInfo.errs.forEach(err => { + errs[err.errMsg] = true; + }); + + testRunInfo.videos.forEach(video => { + videos.push(video); + }); + }, + async reportTaskDone () { + } + }; + }; +} + +function checkVideoPaths (videoLog, videoPaths) { + const testRunIds = uniq(videoLog.map(video => video.testRunId)); + + expect(videoLog.length).eql(testRunIds.length); + + const loggedPaths = uniq(videoLog.map(video => video.videoPath)).sort(); + const actualPaths = [...videoPaths].sort(); + + expect(loggedPaths.length).eql(videoPaths.length); + + for (let i = 0; i < loggedPaths.length; i++) + expect(path.relative(actualPaths[i], loggedPaths[i])).eql(''); +} if (config.useLocalBrowsers) { describe('Video Recording', () => { afterEach(assertionHelper.removeVideosDir); it('Should record video without options', () => { + const errs = {}; + const videos = []; + return runTests('./testcafe-fixtures/index-test.js', '', { only: 'chrome,firefox', setVideoPath: true, - shouldFail: true + reporter: customReporter(errs, videos) }) - .catch(errors => { + .then(() => { + const errors = Object.keys(errs); + expect(errors.length).to.equal(2); expect(errors[0]).to.match(/^Error: Error 1/); expect(errors[1]).to.match(/^Error: Error 2/); @@ -21,48 +63,66 @@ if (config.useLocalBrowsers) { .then(assertionHelper.getVideoFilesList) .then(videoFiles => { expect(videoFiles.length).to.equal(3 * config.browsers.length); + + checkVideoPaths(videos, videoFiles); }); }); - it('Should record video in a single file', ()=> { + it('Should record video in a single file', () => { + const errs = {}; + const videos = []; + return runTests('./testcafe-fixtures/index-test.js', '', { only: 'chrome,firefox', shouldFail: true, setVideoPath: true, + reporter: customReporter(errs, videos), videoOptions: { singleFile: true } }) - .catch(assertionHelper.getVideoFilesList) - .catch(errors => { + .then(() => { + const errors = Object.keys(errs); + expect(errors.length).to.equal(2); expect(errors[0]).to.match(/^Error: Error 1/); expect(errors[1]).to.match(/^Error: Error 2/); }) + .then(assertionHelper.getVideoFilesList) .then(videoFiles => { expect(videoFiles.length).to.equal(1 * config.browsers.length); + + checkVideoPaths(videos, videoFiles); }); }); it('Should record only failed tests', () => { + const errs = {}; + const videos = []; + return runTests('./testcafe-fixtures/index-test.js', '', { only: 'chrome,firefox', shouldFail: true, setVideoPath: true, + reporter: customReporter(errs, videos), videoOptions: { failedOnly: true } }) - .catch(assertionHelper.getVideoFilesList) - .catch(errors => { + .then(() => { + const errors = Object.keys(errs); + expect(errors.length).to.equal(2); expect(errors[0]).to.match(/^Error: Error 1/); expect(errors[1]).to.match(/^Error: Error 2/); }) + .then(assertionHelper.getVideoFilesList) .then(videoFiles => { expect(videoFiles.length).to.equal(2 * config.browsers.length); + + checkVideoPaths(videos, videoFiles); }); }); @@ -89,14 +149,20 @@ if (config.useLocalBrowsers) { }); it('Should record video with quarantine mode enabled', () => { + const errs = {}; + const videos = []; + return runTests('./testcafe-fixtures/quarantine-test.js', '', { only: 'chrome', quarantineMode: true, - setVideoPath: true + setVideoPath: true, + reporter: customReporter(errs, videos), }) .then(assertionHelper.getVideoFilesList) .then(videoFiles => { expect(videoFiles.length).to.equal(2); + + checkVideoPaths(videos, videoFiles); }); }); diff --git a/test/server/reporter-test.js b/test/server/reporter-test.js index 6bcf8925ff..988cd7657d 100644 --- a/test/server/reporter-test.js +++ b/test/server/reporter-test.js @@ -2,6 +2,7 @@ const { expect } = require('chai'); const { chunk, random } = require('lodash'); const Reporter = require('../../lib/reporter'); const Task = require('../../lib/runner/task'); +const Videos = require('../../lib/video-recorder/videos'); const delay = require('../../lib/utils/delay'); describe('Reporter', () => { @@ -346,6 +347,16 @@ describe('Reporter', () => { } } + class VideosMock extends Videos { + constructor (recordings) { + super([], { videoPath: '' }); + + + this.recordings = recordings; + } + } + + class TaskMock extends Task { constructor () { super(testMocks, chunk(browserConnectionMocks, 1), {}, { stopOnFirstFail: false }); @@ -362,6 +373,7 @@ describe('Reporter', () => { } _createBrowserJobs () { + return []; } } @@ -384,60 +396,60 @@ describe('Reporter', () => { }, randomDelay()); } - function createReporter (taskMock) { - return new Reporter({ - reportTaskStart: function (...args) { - expect(args[0]).to.be.a('date'); + beforeEach(() => { + log = []; + }); - // NOTE: replace startTime - args[0] = new Date('Thu Jan 01 1970 00:00:00 UTC'); + it('Should analyze task progress and call appropriate plugin methods', function () { + this.timeout(30000); - return delay(1000) - .then(() => log.push({ method: 'reportTaskStart', args: args })); - }, + const taskMock = new TaskMock(); - reportFixtureStart: function () { - return delay(1000) - .then(() => log.push({ method: 'reportFixtureStart', args: Array.prototype.slice.call(arguments) })); - }, + function createReporter () { + return new Reporter({ + reportTaskStart: function (...args) { + expect(args[0]).to.be.a('date'); - reportTestStart: function (...args) { - expect(args[0]).to.be.an('string'); + // NOTE: replace startTime + args[0] = new Date('Thu Jan 01 1970 00:00:00 UTC'); - return delay(1000) - .then(() => log.push({ method: 'reportTestStart', args: args })); - }, + return delay(1000) + .then(() => log.push({ method: 'reportTaskStart', args: args })); + }, - reportTestDone: function (...args) { - expect(args[1].durationMs).to.be.an('number'); + reportFixtureStart: function () { + return delay(1000) + .then(() => log.push({ method: 'reportFixtureStart', args: Array.prototype.slice.call(arguments) })); + }, - // NOTE: replace durationMs - args[1].durationMs = 74000; + reportTestStart: function (...args) { + expect(args[0]).to.be.an('string'); - return delay(1000) - .then(() => log.push({ method: 'reportTestDone', args: args })); - }, + return delay(1000) + .then(() => log.push({ method: 'reportTestStart', args: args })); + }, - reportTaskDone: function (...args) { - expect(args[0]).to.be.a('date'); + reportTestDone: function (...args) { + expect(args[1].durationMs).to.be.an('number'); - // NOTE: replace endTime - args[0] = new Date('Thu Jan 01 1970 00:15:25 UTC'); + // NOTE: replace durationMs + args[1].durationMs = 74000; - return delay(1000) - .then(() => log.push({ method: 'reportTaskDone', args: args })); - } - }, taskMock); - } + return delay(1000) + .then(() => log.push({ method: 'reportTestDone', args: args })); + }, - beforeEach(() => { - log = []; - }); + reportTaskDone: function (...args) { + expect(args[0]).to.be.a('date'); - it('Should analyze task progress and call appropriate plugin methods', function () { - this.timeout(30000); + // NOTE: replace endTime + args[0] = new Date('Thu Jan 01 1970 00:15:25 UTC'); - const taskMock = new TaskMock(); + return delay(1000) + .then(() => log.push({ method: 'reportTaskDone', args: args })); + } + }, taskMock); + } const expectedLog = [ { @@ -566,7 +578,8 @@ describe('Reporter', () => { userAgent: 'chrome', takenOnFail: false, quarantineAttempt: 2 - }] + }], + videos: [] }, { run: 'run-001' @@ -630,7 +643,8 @@ describe('Reporter', () => { userAgent: 'chrome', takenOnFail: true, quarantineAttempt: null - }] + }], + videos: [] }, { run: 'run-001' @@ -668,7 +682,8 @@ describe('Reporter', () => { skipped: false, quarantine: null, screenshotPath: null, - screenshots: [] + screenshots: [], + videos: [] }, { run: 'run-001' @@ -716,7 +731,8 @@ describe('Reporter', () => { skipped: false, quarantine: null, screenshotPath: null, - screenshots: [] + screenshots: [], + videos: [] }, { run: 'run-001' @@ -754,7 +770,8 @@ describe('Reporter', () => { skipped: false, quarantine: null, screenshotPath: null, - screenshots: [] + screenshots: [], + videos: [] }, { run: 'run-001' @@ -806,7 +823,8 @@ describe('Reporter', () => { skipped: false, quarantine: null, screenshotPath: null, - screenshots: [] + screenshots: [], + videos: [] }, { run: 'run-001' @@ -844,7 +862,8 @@ describe('Reporter', () => { skipped: true, quarantine: null, screenshotPath: null, - screenshots: [] + screenshots: [], + videos: [] }, { run: 'run-001' @@ -882,7 +901,8 @@ describe('Reporter', () => { skipped: false, quarantine: null, screenshotPath: null, - screenshots: [] + screenshots: [], + videos: [] }, { run: 'run-001' @@ -928,4 +948,65 @@ describe('Reporter', () => { expect(reporter.plugin.chalk.enabled).to.be.false; }); + + it('Should provide videos info to the reporter', () => { + const videoLog = []; + const taskMock = new TaskMock(); + + taskMock.videos = new VideosMock({ + 'idf1t1': { + runs: [{ + testRunId: 'f1t1-id1', + videoPath: 'f1t1-path1' + }, { + testRunId: 'f1t1-id2', + videoPath: 'f1t1-path2' + }] + }, + 'idf1t2': { + runs: [{ + testRunId: 'f1t2-id1', + videoPath: 'f1t2-path1' + }, { + testRunId: 'f1t2-id2', + videoPath: 'f1t2-path2' + }] + } + }); + + function createReporter () { + return new Reporter({ + reportTaskStart: function () { + }, + reportTaskDone: function () { + }, + reportFixtureStart: function () { + }, + reportTestStart: function () { + }, + reportTestDone: function (name, testRunInfo) { + videoLog.push(testRunInfo.videos); + } + }, taskMock); + } + + createReporter(); + + return Promise.all([ + emulateBrowserJob(taskMock, chromeTestRunMocks.slice(0, 2)), + emulateBrowserJob(taskMock, firefoxTestRunMocks.slice(0, 2)) + ]) + .then(() => { + expect(videoLog).eql([ + [ + { testRunId: 'f1t1-id1', videoPath: 'f1t1-path1' }, + { testRunId: 'f1t1-id2', videoPath: 'f1t1-path2' } + ], + [ + { testRunId: 'f1t2-id1', videoPath: 'f1t2-path1' }, + { testRunId: 'f1t2-id2', videoPath: 'f1t2-path2' } + ]] + ); + }); + }); }); diff --git a/test/server/video-recorder-test.js b/test/server/video-recorder-test.js index fab4f378bc..f4a9c18b71 100644 --- a/test/server/video-recorder-test.js +++ b/test/server/video-recorder-test.js @@ -1,7 +1,10 @@ + + const { expect } = require('chai'); const { noop } = require('lodash'); const TestRun = require('../../lib/test-run/index'); const VideoRecorder = require('../../lib/video-recorder'); +const Videos = require('../../lib/video-recorder/videos'); const TestRunVideoRecorder = require('../../lib/video-recorder/test-run-video-recorder'); const AsyncEmitter = require('../../lib/utils/async-event-emitter'); const WarningLog = require('../../lib/notifications/warning-log'); @@ -78,15 +81,29 @@ class VideoRecorderMock extends VideoRecorder { return super._onTestRunCreate(options); } + _getTargetVideoPath (testRunRecorder) { + return `path-${testRunRecorder.testRun.id}`; + } + _saveFiles () { } } +class VideosMock extends Videos { + constructor (browserJobs, options, warningLog, timeStamp) { + super(browserJobs, options, warningLog, timeStamp); + } + + _createVideoRecorder (browserJob, videoPath, options, videoEncodingOptions, warningLog) { + return new VideoRecorderMock(browserJob, videoPath, options, videoEncodingOptions, warningLog); + } +} + function createTestRunMock (warningLog) { - const testRun = TestRun.prototype; + const testRun = Object.create(TestRun.prototype); Object.assign(testRun, { - test: { name: 'Test' }, + test: { name: 'Test', id: 'test-id' }, debugLog: { command: noop }, _enqueueCommand: noop, browserManipulationQueue: [], @@ -191,4 +208,98 @@ describe('Video Recorder', () => { expect(videoRecorder.log.includes('The browser window was resized during the "Test" test while TestCafe recorded a video. TestCafe cannot adjust the video resolution during recording. As a result, the video content may appear broken. Do not resize the browser window when TestCafe records a video.')).to.be.true; }); }); + + it('Should build correct video info object', () => { + const browserJobMock = new AsyncEmitter(); + + const videos1 = new VideosMock([browserJobMock], { videoPath: VIDEOS_BASE_PATH, videoOptions: { singleFile: true } }); + const videos2 = new VideosMock([browserJobMock], { videoPath: VIDEOS_BASE_PATH, videoOptions: { singleFile: false } }); + + const testRunMock1 = createTestRunMock(); + const testRunMock2 = createTestRunMock(); + const testRunMock3 = createTestRunMock(); + const testRunMock4 = createTestRunMock(); + + const test1 = { name: 'Test1', id: 'test-1' }; + const test2 = { name: 'Test2', id: 'test-2' }; + + Object.assign(testRunMock1.testRun, { session: { id: 'test-run-1' }, test: test1 }); + Object.assign(testRunMock2.testRun, { session: { id: 'test-run-2' }, test: test1 }); + Object.assign(testRunMock3.testRun, { session: { id: 'test-run-3' }, test: test2 }); + Object.assign(testRunMock4.testRun, { session: { id: 'test-run-4' }, test: test2 }); + + testRunMock1.index = 0; + testRunMock2.index = 1; + testRunMock3.index = 2; + testRunMock4.index = 3; + + const expectedLog1 = { + recordings: { + 'test-1': { + runs: [{ + testRunId: 'test-run-1', + videoPath: 'path-test-run-1', + singleFile: true + }, { + testRunId: 'test-run-2', + videoPath: 'path-test-run-2', + singleFile: true + }] + }, + 'test-2': { + runs: [{ + testRunId: 'test-run-3', + videoPath: 'path-test-run-3', + singleFile: true + }, { + testRunId: 'test-run-4', + videoPath: 'path-test-run-4', + singleFile: true + }] + } + } + }; + + const expectedLog2 = { + recordings: { + 'test-1': { + runs: [{ + testRunId: 'test-run-1', + videoPath: 'path-test-run-1', + singleFile: false + }, { + testRunId: 'test-run-2', + videoPath: 'path-test-run-2', + singleFile: false + }] + }, + 'test-2': { + runs: [{ + testRunId: 'test-run-3', + videoPath: 'path-test-run-3', + singleFile: false + }, { + testRunId: 'test-run-4', + videoPath: 'path-test-run-4', + singleFile: false + }] + } + } + }; + + + return browserJobMock.emit('start') + .then(() => browserJobMock.emit('test-run-create', testRunMock1)) + .then(() => browserJobMock.emit('test-run-before-done', testRunMock1)) + .then(() => browserJobMock.emit('test-run-create', testRunMock2)) + .then(() => browserJobMock.emit('test-run-before-done', testRunMock2)) + .then(() => browserJobMock.emit('test-run-create', testRunMock3)) + .then(() => browserJobMock.emit('test-run-before-done', testRunMock3)) + .then(() => browserJobMock.emit('test-run-create', testRunMock4)) + .then(() => browserJobMock.emit('test-run-before-done', testRunMock4)) + .then(() => { + expect(videos1).eql(expectedLog1); + expect(videos2).eql(expectedLog2); + }); + }); });