From 620b614a49d1b8b7cb933742655f894b5edf1c95 Mon Sep 17 00:00:00 2001 From: Karthik Nadig Date: Fri, 5 Jun 2020 14:56:48 -0700 Subject: [PATCH] Cherry-pick ExP platform work from master (#12160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * User cannot belong to all experiments in an experiment group (#11945) * 10790 prep - Create an experiments/ folder (#11980) * experiments.ts -> experiments/manager.ts * eexperimentGroups -> experiments/experimentGroups * test/experiments.u.t -> test/experiments/manager.u.t * experimentGroups -> groups * Whoops committed one file too many * Add support for VS Code's experiment service (#11979) * Add npm package * News entry * experiments.ts -> experiments/manager.ts * Wrong issue number * eexperimentGroups -> experiments/experimentGroups * Move experiments.unit.tests.ts -> experiments/ * Commit new files * Merge gone sideways * Add types * Add opt in/out handling * Activation (service registry) * Don't pin tas-client to one version * Unit tests + fixes * Lol forgot to remove a comment + add headers * Forgot 'use strict' in service.ts * Use IApplicationEnvironment instead of IExtensions * Apply suggestions from code review Co-authored-by: Don Jayamanne * Remove unnecessary formatted props * n e v e r m i n d * Aight fixed it * flight -> experiment * Check stub calls instead of ctor impl * removed getExperimentService stub check * Set shared properties for all telemetry events * Add test for shared properties Co-authored-by: Don Jayamanne * Fix merge issues * Fix index unit tests * Revert "Fix index unit tests" This reverts commit 2fb61fc366c2c55b5c4b9c34e766ffbbcd50be74. * Make MPLS and vscode-extension-telemetry work together 🤝 (#11823) * Revert "Revert vscode-extension-telemetry changes for the release (#11602)" This reverts commit 71d17932d3268fb20c55e745a82f2da8bf1f3d0c. * Remove entry from changelog + add new news entry * Update LS code to use periods instead of slashes * Revert "Update LS code to use periods instead of slashes" This reverts commit 13566513a86bbe63365cfc16c387fe4c504bfc10. * Replace slashes before sending telemetry instead * Too fast too furious * Fix more merge issues Co-authored-by: Don Jayamanne Co-authored-by: Kim-Adeline Miguel <51720070+kimadeline@users.noreply.github.com> --- gulpfile.js | 14 +- news/1 Enhancements/10790.md | 1 + news/2 Fixes/11943.md | 1 + news/3 Code Health/11597.md | 1 + package-lock.json | 131 +++++++-- package.json | 3 +- src/client/activation/aaTesting.ts | 2 +- src/client/activation/activationManager.ts | 2 +- src/client/activation/activationService.ts | 2 +- src/client/activation/extensionSurvey.ts | 2 +- .../activation/languageClientMiddleware.ts | 7 +- .../languageServer/languageServerProxy.ts | 7 +- .../activation/node/languageServerProxy.ts | 7 +- src/client/api.ts | 2 +- .../checks/macPythonInterpreter.ts | 2 +- .../checks/pythonPathDeprecated.ts | 2 +- .../diagnostics/checks/upgradeCodeRunner.ts | 2 +- src/client/common/configSettings.ts | 2 +- src/client/common/configuration/service.ts | 2 +- .../groups.ts} | 0 .../manager.ts} | 38 ++- src/client/common/experiments/service.ts | 91 +++++++ src/client/common/experiments/telemetry.ts | 24 ++ src/client/common/serviceRegistry.ts | 6 +- src/client/common/types.ts | 8 + src/client/common/utils/cacheUtils.ts | 2 +- .../datascience/data-viewing/dataViewer.ts | 2 +- .../interactive-common/interactiveBase.ts | 2 +- .../ipywidgets/ipywidgetHandler.ts | 2 +- .../datascience/jupyter/jupyterDebugger.ts | 2 +- .../datascience/jupyter/jupyterVariables.ts | 2 +- src/client/datascience/plotting/plotViewer.ts | 2 +- .../raw-kernel/rawNotebookProvider.ts | 2 +- .../debugger/extension/adapter/factory.ts | 2 +- .../adapter/outdatedDebuggerPrompt.ts | 2 +- .../configuration/resolvers/attach.ts | 2 +- .../resolvers/launchConfigExperiment.ts | 2 +- .../wrapperEnvironmentActivationService.ts | 2 +- .../autoSelection/rules/settings.ts | 2 +- .../autoSelection/rules/workspaceEnv.ts | 2 +- .../interpreterSelector.ts | 2 +- .../pythonPathUpdaterServiceFactory.ts | 2 +- src/client/interpreter/interpreterService.ts | 2 +- src/client/startupTelemetry.ts | 2 +- src/client/telemetry/index.ts | 133 ++++----- .../telemetry/vscode-extension-telemetry.d.ts | 33 --- src/client/testing/main.ts | 2 +- src/test/activation/aaTesting.unit.test.ts | 2 +- .../activation/activationManager.unit.test.ts | 4 +- .../activation/activationService.unit.test.ts | 2 +- .../activation/extensionSurvey.unit.test.ts | 2 +- .../languageServer/manager.unit.test.ts | 2 +- src/test/api.functional.test.ts | 2 +- .../checks/macPythonInterpreter.unit.test.ts | 2 +- .../checks/pythonPathDeprecated.unit.test.ts | 2 +- .../checks/upgradeCodeRunner.unit.test.ts | 2 +- .../configSettings.pythonPath.unit.test.ts | 2 +- .../common/configuration/service.unit.test.ts | 2 +- .../manager.unit.test.ts} | 39 ++- .../common/experiments/service.unit.test.ts | 256 ++++++++++++++++++ .../common/experiments/telemetry.unit.test.ts | 62 +++++ src/test/common/installer.test.ts | 5 +- src/test/common/moduleInstaller.test.ts | 5 +- src/test/common/serviceRegistry.unit.test.ts | 2 +- .../terminalActivation.testvirtualenvs.ts | 2 +- .../interpreterSelector.unit.test.ts | 2 +- .../datascience/dataScienceIocContainer.ts | 4 +- .../datascience/debugger.functional.test.tsx | 2 +- .../datascience/notebook.functional.test.ts | 2 +- .../uiTests/ipywidget.ui.functional.test.ts | 2 +- .../variableexplorer.functional.test.tsx | 2 +- src/test/debugger/attach.ptvsd.test.ts | 2 +- .../extension/adapter/factory.unit.test.ts | 4 +- .../outdatedDebuggerPrompt.unit.test.ts | 4 +- .../resolvers/attach.unit.test.ts | 2 +- .../launchConfigExperiments.unit.test.ts | 4 +- ...rEnvironmentActivationService.unit.test.ts | 2 +- .../autoSelection/rules/settings.unit.test.ts | 4 +- .../rules/workspaceEnv.unit.test.ts | 4 +- .../interpreterService.unit.test.ts | 2 +- .../pythonPathUpdaterFactory.unit.test.ts | 2 +- src/test/startupTelemetry.unit.test.ts | 2 +- src/test/telemetry/index.unit.test.ts | 185 ++++++------- src/test/testing/helper.ts | 2 +- src/test/testing/main.unit.test.ts | 4 +- 85 files changed, 863 insertions(+), 334 deletions(-) create mode 100644 news/1 Enhancements/10790.md create mode 100644 news/2 Fixes/11943.md create mode 100644 news/3 Code Health/11597.md rename src/client/common/{experimentGroups.ts => experiments/groups.ts} (100%) rename src/client/common/{experiments.ts => experiments/manager.ts} (92%) create mode 100644 src/client/common/experiments/service.ts create mode 100644 src/client/common/experiments/telemetry.ts delete mode 100644 src/client/telemetry/vscode-extension-telemetry.d.ts rename src/test/common/{experiments.unit.test.ts => experiments/manager.unit.test.ts} (96%) create mode 100644 src/test/common/experiments/service.unit.test.ts create mode 100644 src/test/common/experiments/telemetry.unit.test.ts diff --git a/gulpfile.js b/gulpfile.js index f2f91140595b..a290b648282a 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -287,7 +287,9 @@ function getAllowedWarningsForWebPack(buildConfig) { 'WARNING in ./node_modules/@jupyterlab/services/node_modules/ws/lib/validation.js', 'WARNING in ./node_modules/any-promise/register.js', 'WARNING in ./node_modules/log4js/lib/appenders/index.js', - 'WARNING in ./node_modules/log4js/lib/clustering.js' + 'WARNING in ./node_modules/log4js/lib/clustering.js', + 'WARNING in ./node_modules/diagnostic-channel-publishers/dist/src/azure-coretracing.pub.js', + 'WARNING in ./node_modules/applicationinsights/out/AutoCollection/NativePerformance.js' ]; case 'extension': return [ @@ -300,10 +302,16 @@ function getAllowedWarningsForWebPack(buildConfig) { 'remove-files-plugin@1.4.0:', 'WARNING in ./node_modules/@jupyterlab/services/node_modules/ws/lib/buffer-util.js', 'WARNING in ./node_modules/@jupyterlab/services/node_modules/ws/lib/validation.js', - 'WARNING in ./node_modules/@jupyterlab/services/node_modules/ws/lib/Validation.js' + 'WARNING in ./node_modules/@jupyterlab/services/node_modules/ws/lib/Validation.js', + 'WARNING in ./node_modules/diagnostic-channel-publishers/dist/src/azure-coretracing.pub.js', + 'WARNING in ./node_modules/applicationinsights/out/AutoCollection/NativePerformance.js' ]; case 'debugAdapter': - return ['WARNING in ./node_modules/vscode-uri/lib/index.js']; + return [ + 'WARNING in ./node_modules/vscode-uri/lib/index.js', + 'WARNING in ./node_modules/diagnostic-channel-publishers/dist/src/azure-coretracing.pub.js', + 'WARNING in ./node_modules/applicationinsights/out/AutoCollection/NativePerformance.js' + ]; default: throw new Error('Unknown WebPack Configuration'); } diff --git a/news/1 Enhancements/10790.md b/news/1 Enhancements/10790.md new file mode 100644 index 000000000000..1da59c36dc77 --- /dev/null +++ b/news/1 Enhancements/10790.md @@ -0,0 +1 @@ +Integrate VS Code experiment framework in the extension. diff --git a/news/2 Fixes/11943.md b/news/2 Fixes/11943.md new file mode 100644 index 000000000000..32ba8b829d59 --- /dev/null +++ b/news/2 Fixes/11943.md @@ -0,0 +1 @@ +Ensure user cannot belong to all experiments in an experiment group. diff --git a/news/3 Code Health/11597.md b/news/3 Code Health/11597.md new file mode 100644 index 000000000000..4d5ba9f74640 --- /dev/null +++ b/news/3 Code Health/11597.md @@ -0,0 +1 @@ +Update telemetry on errors and exceptions to use [vscode-extension-telemetry](https://www.npmjs.com/package/vscode-extension-telemetry). diff --git a/package-lock.json b/package-lock.json index aae95237a16e..a3a2300e5f2b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4657,13 +4657,14 @@ } }, "applicationinsights": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/applicationinsights/-/applicationinsights-1.0.6.tgz", - "integrity": "sha512-VQT3kBpJVPw5fCO5n+WUeSx0VHjxFtD7znYbILBlVgOS9/cMDuGFmV2Br3ObzFyZUDGNbEfW36fD1y2/vAiCKw==", + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/applicationinsights/-/applicationinsights-1.7.4.tgz", + "integrity": "sha512-XFLsNlcanpjFhHNvVWEfcm6hr7lu9znnb6Le1Lk5RE03YUV9X2B2n2MfM4kJZRrUdV+C0hdHxvWyv+vWoLfY7A==", "requires": { + "cls-hooked": "^4.2.2", + "continuation-local-storage": "^3.2.1", "diagnostic-channel": "0.2.0", - "diagnostic-channel-publishers": "0.2.1", - "zone.js": "0.7.6" + "diagnostic-channel-publishers": "^0.3.3" } }, "aproba": { @@ -5230,11 +5231,28 @@ "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==", "dev": true }, + "async-hook-jl": { + "version": "1.7.6", + "resolved": "https://registry.npmjs.org/async-hook-jl/-/async-hook-jl-1.7.6.tgz", + "integrity": "sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg==", + "requires": { + "stack-chain": "^1.3.7" + } + }, "async-limiter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" }, + "async-listener": { + "version": "0.6.10", + "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", + "integrity": "sha512-gpuo6xOyF4D5DE5WvyqZdPA3NGhiT6Qf07l7DCB0wwDEsLvDIbCr6j9S5aj5Ch96dLace5tXVzWBZkxU/c5ohw==", + "requires": { + "semver": "^5.3.0", + "shimmer": "^1.1.0" + } + }, "async-settle": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-settle/-/async-settle-1.0.0.tgz", @@ -5318,6 +5336,14 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, + "axios": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", + "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "requires": { + "follow-redirects": "1.5.10" + } + }, "azure-devops-node-api": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-7.2.0.tgz", @@ -6807,6 +6833,16 @@ } } }, + "cls-hooked": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/cls-hooked/-/cls-hooked-4.2.2.tgz", + "integrity": "sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw==", + "requires": { + "async-hook-jl": "^1.7.6", + "emitter-listener": "^1.0.1", + "semver": "^5.4.1" + } + }, "clsx": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.0.4.tgz", @@ -7148,6 +7184,15 @@ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, + "continuation-local-storage": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", + "integrity": "sha512-jx44cconVqkCEEyLSKWwkvUXwO561jXMa3LPjTPsm5QR22PA0/mhe33FT4Xb5y74JDvt/Cq+5lm8S8rskLv9ZA==", + "requires": { + "async-listener": "^0.6.0", + "emitter-listener": "^1.1.1" + } + }, "convert-source-map": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.6.0.tgz", @@ -8690,9 +8735,9 @@ } }, "diagnostic-channel-publishers": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/diagnostic-channel-publishers/-/diagnostic-channel-publishers-0.2.1.tgz", - "integrity": "sha1-ji1geottef6IC1SLxYzGvrKIxPM=" + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/diagnostic-channel-publishers/-/diagnostic-channel-publishers-0.3.4.tgz", + "integrity": "sha512-SZ1zMfFiEabf4Qx0Og9V1gMsRoqz3O+5ENkVcNOfI+SMJ3QhQsdEoKX99r0zvreagXot2parPxmrwwUM/ja8ug==" }, "diagnostics": { "version": "1.1.1", @@ -9059,6 +9104,14 @@ "minimalistic-crypto-utils": "^1.0.0" } }, + "emitter-listener": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", + "integrity": "sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==", + "requires": { + "shimmer": "^1.2.0" + } + }, "emoji-regex": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", @@ -10320,6 +10373,29 @@ } } }, + "follow-redirects": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "requires": { + "debug": "=3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, "font-awesome": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/font-awesome/-/font-awesome-4.7.0.tgz", @@ -20668,6 +20744,11 @@ } } }, + "shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" + }, "shortid": { "version": "2.2.14", "resolved": "https://registry.npmjs.org/shortid/-/shortid-2.2.14.tgz", @@ -21260,6 +21341,11 @@ "figgy-pudding": "^3.5.1" } }, + "stack-chain": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/stack-chain/-/stack-chain-1.3.7.tgz", + "integrity": "sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU=" + }, "stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -22033,6 +22119,14 @@ } } }, + "tas-client": { + "version": "0.0.762", + "resolved": "https://registry.npmjs.org/tas-client/-/tas-client-0.0.762.tgz", + "integrity": "sha512-i4dcYHkk2rnmBHr8RC1IoZVHU9wQT+OmDDnUeRW/vbpjDSCqgL5Qh8KeHs3DizJgqP9MWjLK/Kmqfm8VbD6g3g==", + "requires": { + "axios": "^0.19.0" + } + }, "teeny-request": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-6.0.1.tgz", @@ -24563,11 +24657,11 @@ "integrity": "sha512-+OMm11R1bGYbpIJ5eQIkwoDGFF4GvBz3Ztl6/VM+/RNNb2Gjk2c0Ku+oMmfhlTmTlPCpgHBsH4JqVCbUYhu5bA==" }, "vscode-extension-telemetry": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.1.0.tgz", - "integrity": "sha512-WVCnP+uLxlqB6UD98yQNV47mR5Rf79LFxpuZhSPhEf0Sb4tPZed3a63n003/dchhOwyCTCBuNN4n8XKJkLEI1Q==", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/vscode-extension-telemetry/-/vscode-extension-telemetry-0.1.4.tgz", + "integrity": "sha512-9U2pUZ/YwZBfA8CkBrHwMxjnq9Ab+ng8daJWJzEQ6CAxlZyRhmck23bx2lqqpEwGWJCiuceQy4k0Me6llEB4zw==", "requires": { - "applicationinsights": "1.0.6" + "applicationinsights": "1.7.4" } }, "vscode-jsonrpc": { @@ -24620,6 +24714,14 @@ "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0-next.1.tgz", "integrity": "sha512-tZFUSbyjUcrh+qQf13ALX4QDdOfDX0cVaBFgy7ktJ0VwS7AW/yRKgGPSxVqqP9OCMNPdqP57O5q47w2pEwfaUg==" }, + "vscode-tas-client": { + "version": "0.0.757", + "resolved": "https://registry.npmjs.org/vscode-tas-client/-/vscode-tas-client-0.0.757.tgz", + "integrity": "sha512-IXP+vFTIE1HpvYxqm7SuFe/K5N3cPoF4TTH6uYGAeuuYxV586tjYFtK9UmF00ajzcqvLGvsPgxmfgLeX6vUUtA==", + "requires": { + "tas-client": "0.0.762" + } + }, "vscode-test": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/vscode-test/-/vscode-test-1.2.3.tgz", @@ -25681,11 +25783,6 @@ } } } - }, - "zone.js": { - "version": "0.7.6", - "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.7.6.tgz", - "integrity": "sha1-+7w50+AmHQmG8boGMG6zrrDSIAk=" } } } diff --git a/package.json b/package.json index 4b8dc7e22bb9..dd7ca7b54133 100644 --- a/package.json +++ b/package.json @@ -2986,11 +2986,12 @@ "untildify": "^3.0.2", "vscode-debugadapter": "^1.28.0", "vscode-debugprotocol": "^1.28.0", - "vscode-extension-telemetry": "0.1.0", + "vscode-extension-telemetry": "0.1.4", "vscode-jsonrpc": "^5.0.1", "vscode-languageclient": "^6.2.0-next.2", "vscode-languageserver": "^6.2.0-next.2", "vscode-languageserver-protocol": "^3.16.0-next.2", + "vscode-tas-client": "^0.0.757", "vsls": "^0.3.1291", "winreg": "^1.2.4", "winston": "^3.2.1", diff --git a/src/client/activation/aaTesting.ts b/src/client/activation/aaTesting.ts index e37ddb31e4d0..1a424e5f2606 100644 --- a/src/client/activation/aaTesting.ts +++ b/src/client/activation/aaTesting.ts @@ -4,7 +4,7 @@ 'use strict'; import { inject, injectable } from 'inversify'; -import { ValidateABTesting } from '../common/experimentGroups'; +import { ValidateABTesting } from '../common/experiments/groups'; import { IExperimentsManager } from '../common/types'; import { IExtensionSingleActivationService } from './types'; diff --git a/src/client/activation/activationManager.ts b/src/client/activation/activationManager.ts index f570eb05aa3a..bf53a1d18103 100644 --- a/src/client/activation/activationManager.ts +++ b/src/client/activation/activationManager.ts @@ -8,7 +8,7 @@ import { TextDocument } from 'vscode'; import { IApplicationDiagnostics } from '../application/types'; import { IActiveResourceService, IDocumentManager, IWorkspaceService } from '../common/application/types'; import { DEFAULT_INTERPRETER_SETTING, PYTHON_LANGUAGE } from '../common/constants'; -import { DeprecatePythonPath } from '../common/experimentGroups'; +import { DeprecatePythonPath } from '../common/experiments/groups'; import { traceDecorators } from '../common/logger'; import { IFileSystem } from '../common/platform/types'; import { IDisposable, IExperimentsManager, IInterpreterPathService, Resource } from '../common/types'; diff --git a/src/client/activation/activationService.ts b/src/client/activation/activationService.ts index 3cc2d7f18b80..77c7e9e0d412 100644 --- a/src/client/activation/activationService.ts +++ b/src/client/activation/activationService.ts @@ -9,7 +9,7 @@ import { LSNotSupportedDiagnosticServiceId } from '../application/diagnostics/ch import { IDiagnosticsService } from '../application/diagnostics/types'; import { IApplicationShell, ICommandManager, IWorkspaceService } from '../common/application/types'; import { STANDARD_OUTPUT_CHANNEL } from '../common/constants'; -import { LSControl, LSEnabled } from '../common/experimentGroups'; +import { LSControl, LSEnabled } from '../common/experiments/groups'; import { traceError } from '../common/logger'; import { IConfigurationService, diff --git a/src/client/activation/extensionSurvey.ts b/src/client/activation/extensionSurvey.ts index 67ec7e68e9d6..b6c0847407cc 100644 --- a/src/client/activation/extensionSurvey.ts +++ b/src/client/activation/extensionSurvey.ts @@ -6,7 +6,7 @@ import { inject, injectable, optional } from 'inversify'; import * as querystring from 'querystring'; import { IApplicationEnvironment, IApplicationShell } from '../common/application/types'; -import { ShowExtensionSurveyPrompt } from '../common/experimentGroups'; +import { ShowExtensionSurveyPrompt } from '../common/experiments/groups'; import '../common/extensions'; import { traceDecorators } from '../common/logger'; import { IPlatformService } from '../common/platform/types'; diff --git a/src/client/activation/languageClientMiddleware.ts b/src/client/activation/languageClientMiddleware.ts index 287ce9c47ef8..512bb5e4eac3 100644 --- a/src/client/activation/languageClientMiddleware.ts +++ b/src/client/activation/languageClientMiddleware.ts @@ -60,7 +60,7 @@ import { import { ProvideDeclarationSignature } from 'vscode-languageclient/lib/declaration'; import { HiddenFilePrefix } from '../common/constants'; -import { CollectLSRequestTiming, CollectNodeLSRequestTiming } from '../common/experimentGroups'; +import { CollectLSRequestTiming, CollectNodeLSRequestTiming } from '../common/experiments/groups'; import { IConfigurationService, IExperimentsManager, IPythonExtensionBanner } from '../common/types'; import { StopWatch } from '../common/utils/stopWatch'; import { sendTelemetryEvent } from '../telemetry'; @@ -444,9 +444,12 @@ function captureTelemetryForLSPMethod(method: string, debounceMilliseconds: numb this.lastCaptured.set(method, now); this.eventCount += 1; + // Replace all slashes in the method name so it doesn't get scrubbed by vscode-extension-telemetry. + const formattedMethod = method.replace(/\//g, '.'); + const properties = { lsVersion: this.serverVersion || 'unknown', - method: method + method: formattedMethod }; const stopWatch = new StopWatch(); diff --git a/src/client/activation/languageServer/languageServerProxy.ts b/src/client/activation/languageServer/languageServerProxy.ts index 29d2e318eb96..ce69f971abc7 100644 --- a/src/client/activation/languageServer/languageServerProxy.ts +++ b/src/client/activation/languageServer/languageServerProxy.ts @@ -73,7 +73,12 @@ export class DotNetLanguageServerProxy implements ILanguageServerProxy { if (settings.downloadLanguageServer) { this.languageClient.onTelemetry((telemetryEvent) => { const eventName = telemetryEvent.EventName || EventName.PYTHON_LANGUAGE_SERVER_TELEMETRY; - sendTelemetryEvent(eventName, telemetryEvent.Measurements, telemetryEvent.Properties); + const formattedProperties = { + ...telemetryEvent.Properties, + // Replace all slashes in the method name so it doesn't get scrubbed by vscode-extension-telemetry. + method: telemetryEvent.Properties.method?.replace(/\//g, '.') + }; + sendTelemetryEvent(eventName, telemetryEvent.Measurements, formattedProperties); }); } await this.registerTestServices(); diff --git a/src/client/activation/node/languageServerProxy.ts b/src/client/activation/node/languageServerProxy.ts index 17f25cbd97e7..ef1ecbb08310 100644 --- a/src/client/activation/node/languageServerProxy.ts +++ b/src/client/activation/node/languageServerProxy.ts @@ -99,7 +99,12 @@ export class NodeLanguageServerProxy implements ILanguageServerProxy { if (settings.downloadLanguageServer) { this.languageClient.onTelemetry((telemetryEvent) => { const eventName = telemetryEvent.EventName || EventName.LANGUAGE_SERVER_TELEMETRY; - sendTelemetryEvent(eventName, telemetryEvent.Measurements, telemetryEvent.Properties); + const formattedProperties = { + ...telemetryEvent.Properties, + // Replace all slashes in the method name so it doesn't get scrubbed by vscode-extension-telemetry. + method: telemetryEvent.Properties.method?.replace(/\//g, '.') + }; + sendTelemetryEvent(eventName, telemetryEvent.Measurements, formattedProperties); }); } await this.registerTestServices(); diff --git a/src/client/api.ts b/src/client/api.ts index 61328de72ae6..0b37a5da101c 100644 --- a/src/client/api.ts +++ b/src/client/api.ts @@ -4,7 +4,7 @@ 'use strict'; import { isTestExecution } from './common/constants'; -import { DebugAdapterNewPtvsd } from './common/experimentGroups'; +import { DebugAdapterNewPtvsd } from './common/experiments/groups'; import { traceError } from './common/logger'; import { IConfigurationService, IExperimentsManager, Resource } from './common/types'; import { diff --git a/src/client/application/diagnostics/checks/macPythonInterpreter.ts b/src/client/application/diagnostics/checks/macPythonInterpreter.ts index 26344f41a607..b3c8434c7b30 100644 --- a/src/client/application/diagnostics/checks/macPythonInterpreter.ts +++ b/src/client/application/diagnostics/checks/macPythonInterpreter.ts @@ -6,7 +6,7 @@ import { inject, injectable } from 'inversify'; import { ConfigurationChangeEvent, DiagnosticSeverity, Uri } from 'vscode'; import { IWorkspaceService } from '../../../common/application/types'; -import { DeprecatePythonPath } from '../../../common/experimentGroups'; +import { DeprecatePythonPath } from '../../../common/experiments/groups'; import '../../../common/extensions'; import { IPlatformService } from '../../../common/platform/types'; import { diff --git a/src/client/application/diagnostics/checks/pythonPathDeprecated.ts b/src/client/application/diagnostics/checks/pythonPathDeprecated.ts index 80dd0d5b0cbc..0a2754cca958 100644 --- a/src/client/application/diagnostics/checks/pythonPathDeprecated.ts +++ b/src/client/application/diagnostics/checks/pythonPathDeprecated.ts @@ -6,7 +6,7 @@ import { inject, named } from 'inversify'; import { ConfigurationTarget, DiagnosticSeverity } from 'vscode'; import { IWorkspaceService } from '../../../common/application/types'; -import { DeprecatePythonPath } from '../../../common/experimentGroups'; +import { DeprecatePythonPath } from '../../../common/experiments/groups'; import { IDisposableRegistry, IExperimentsManager, Resource } from '../../../common/types'; import { Common, Diagnostics } from '../../../common/utils/localize'; import { IServiceContainer } from '../../../ioc/types'; diff --git a/src/client/application/diagnostics/checks/upgradeCodeRunner.ts b/src/client/application/diagnostics/checks/upgradeCodeRunner.ts index c2f43939ce6e..6418e3636e7b 100644 --- a/src/client/application/diagnostics/checks/upgradeCodeRunner.ts +++ b/src/client/application/diagnostics/checks/upgradeCodeRunner.ts @@ -7,7 +7,7 @@ import { inject, named } from 'inversify'; import { DiagnosticSeverity } from 'vscode'; import { IWorkspaceService } from '../../../common/application/types'; import { CODE_RUNNER_EXTENSION_ID } from '../../../common/constants'; -import { DeprecatePythonPath } from '../../../common/experimentGroups'; +import { DeprecatePythonPath } from '../../../common/experiments/groups'; import { IDisposableRegistry, IExperimentsManager, IExtensions, Resource } from '../../../common/types'; import { Common, Diagnostics } from '../../../common/utils/localize'; import { IServiceContainer } from '../../../ioc/types'; diff --git a/src/client/common/configSettings.ts b/src/client/common/configSettings.ts index 89aa79e5f7eb..d5464ddcc762 100644 --- a/src/client/common/configSettings.ts +++ b/src/client/common/configSettings.ts @@ -21,7 +21,7 @@ import { sendSettingTelemetry } from '../telemetry/envFileTelemetry'; import { IWorkspaceService } from './application/types'; import { WorkspaceService } from './application/workspace'; import { DEFAULT_INTERPRETER_SETTING, isTestExecution } from './constants'; -import { DeprecatePythonPath } from './experimentGroups'; +import { DeprecatePythonPath } from './experiments/groups'; import { ExtensionChannels } from './insidersBuild/types'; import { IS_WINDOWS } from './platform/constants'; import * as internalPython from './process/internal/python'; diff --git a/src/client/common/configuration/service.ts b/src/client/common/configuration/service.ts index 00a8c789f204..b922254d8b05 100644 --- a/src/client/common/configuration/service.ts +++ b/src/client/common/configuration/service.ts @@ -11,7 +11,7 @@ import { IServiceContainer } from '../../ioc/types'; import { IWorkspaceService } from '../application/types'; import { PythonSettings } from '../configSettings'; import { isUnitTestExecution } from '../constants'; -import { DeprecatePythonPath } from '../experimentGroups'; +import { DeprecatePythonPath } from '../experiments/groups'; import { IConfigurationService, IExperimentsManager, IInterpreterPathService, IPythonSettings } from '../types'; @injectable() diff --git a/src/client/common/experimentGroups.ts b/src/client/common/experiments/groups.ts similarity index 100% rename from src/client/common/experimentGroups.ts rename to src/client/common/experiments/groups.ts diff --git a/src/client/common/experiments.ts b/src/client/common/experiments/manager.ts similarity index 92% rename from src/client/common/experiments.ts rename to src/client/common/experiments/manager.ts index f4655f63f36f..8ac58bb32278 100644 --- a/src/client/common/experiments.ts +++ b/src/client/common/experiments/manager.ts @@ -8,24 +8,26 @@ import { inject, injectable, named, optional } from 'inversify'; import { parse } from 'jsonc-parser'; import * as path from 'path'; -import { IConfigurationService, IHttpClient, IPythonSettings } from '../common/types'; -import { sendTelemetryEvent } from '../telemetry'; -import { EventName } from '../telemetry/constants'; -import { IApplicationEnvironment } from './application/types'; -import { EXTENSION_ROOT_DIR, STANDARD_OUTPUT_CHANNEL } from './constants'; -import { traceDecorators, traceError } from './logger'; -import { IFileSystem } from './platform/types'; +import { sendTelemetryEvent } from '../../telemetry'; +import { EventName } from '../../telemetry/constants'; +import { IApplicationEnvironment } from '../application/types'; +import { EXTENSION_ROOT_DIR, STANDARD_OUTPUT_CHANNEL } from '../constants'; +import { traceDecorators, traceError } from '../logger'; +import { IFileSystem } from '../platform/types'; import { ABExperiments, + IConfigurationService, ICryptoUtils, IExperimentsManager, + IHttpClient, IOutputChannel, IPersistentState, - IPersistentStateFactory -} from './types'; -import { sleep } from './utils/async'; -import { swallowExceptions } from './utils/decorators'; -import { Experiments } from './utils/localize'; + IPersistentStateFactory, + IPythonSettings +} from '../types'; +import { sleep } from '../utils/async'; +import { swallowExceptions } from '../utils/decorators'; +import { Experiments } from '../utils/localize'; const EXPIRY_DURATION_MS = 30 * 60 * 1000; export const isDownloadedStorageValidKey = 'IS_EXPERIMENTS_STORAGE_VALID_KEY'; @@ -149,6 +151,8 @@ export class ExperimentsManager implements IExperimentsManager { public populateUserExperiments(): void { this.cleanUpExperimentsOptList(); if (Array.isArray(this.experimentStorage.value)) { + const remainingExpriments: ABExperiments = []; + // First process experiments in order of user preference (if they have opted out or opted in). for (const experiment of this.experimentStorage.value) { try { if ( @@ -168,13 +172,19 @@ export class ExperimentsManager implements IExperimentsManager { expNameOptedInto: experiment.name }); this.userExperiments.push(experiment); - } else if (this.isUserInRange(experiment.min, experiment.max, experiment.salt)) { - this.userExperiments.push(experiment); + } else { + remainingExpriments.push(experiment); } } catch (ex) { traceError(`Failed to populate experiment list for experiment '${experiment.name}'`, ex); } } + + // Add users (based on algorithm) to experiments they haven't already opted out of or opted into. + remainingExpriments + .filter((experiment) => this.isUserInRange(experiment.min, experiment.max, experiment.salt)) + .filter((experiment) => !this.userExperiments.some((existing) => existing.salt === experiment.salt)) + .forEach((experiment) => this.userExperiments.push(experiment)); } } diff --git a/src/client/common/experiments/service.ts b/src/client/common/experiments/service.ts new file mode 100644 index 000000000000..68e01e5b2ea3 --- /dev/null +++ b/src/client/common/experiments/service.ts @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, named } from 'inversify'; +import { Memento } from 'vscode'; +import { getExperimentationService, IExperimentationService, TargetPopulation } from 'vscode-tas-client'; +import { sendTelemetryEvent } from '../../telemetry'; +import { EventName } from '../../telemetry/constants'; +import { IApplicationEnvironment } from '../application/types'; +import { GLOBAL_MEMENTO, IConfigurationService, IExperimentService, IMemento, IPythonSettings } from '../types'; +import { ExperimentationTelemetry } from './telemetry'; + +export class ExperimentService implements IExperimentService { + /** + * Experiments the user requested to opt into manually. + */ + public _optInto: string[] = []; + /** + * Experiments the user requested to opt out from manually. + */ + public _optOutFrom: string[] = []; + + private readonly experimentationService?: IExperimentationService; + private readonly settings: IPythonSettings; + + constructor( + @inject(IConfigurationService) readonly configurationService: IConfigurationService, + @inject(IApplicationEnvironment) private readonly appEnvironment: IApplicationEnvironment, + @inject(IMemento) @named(GLOBAL_MEMENTO) readonly globalState: Memento + ) { + this.settings = configurationService.getSettings(undefined); + + // Users can only opt in or out of experiment groups, not control groups. + const optInto = this.settings.experiments.optInto; + const optOutFrom = this.settings.experiments.optOutFrom; + this._optInto = optInto.filter((exp) => !exp.endsWith('control')); + this._optOutFrom = optOutFrom.filter((exp) => !exp.endsWith('control')); + + // Don't initialize the experiment service if the extension's experiments setting is disabled. + const enabled = this.settings.experiments.enabled; + if (!enabled) { + return; + } + + let targetPopulation: TargetPopulation; + + if (this.appEnvironment.channel === 'insiders') { + targetPopulation = TargetPopulation.Insiders; + } else { + targetPopulation = TargetPopulation.Public; + } + + const telemetryReporter = new ExperimentationTelemetry(); + + this.experimentationService = getExperimentationService( + this.appEnvironment.extensionName, + this.appEnvironment.packageJson.version!, + targetPopulation, + telemetryReporter, + globalState + ); + } + + public async inExperiment(experiment: string): Promise { + if (!this.experimentationService) { + return false; + } + + // Currently the service doesn't support opting in and out of experiments, + // so we need to perform these checks and send the corresponding telemetry manually. + if (this._optOutFrom.includes('All') || this._optOutFrom.includes(experiment)) { + sendTelemetryEvent(EventName.PYTHON_EXPERIMENTS_OPT_IN_OUT, undefined, { + expNameOptedOutOf: experiment + }); + + return false; + } + + if (this._optInto.includes('All') || this._optInto.includes(experiment)) { + sendTelemetryEvent(EventName.PYTHON_EXPERIMENTS_OPT_IN_OUT, undefined, { + expNameOptedInto: experiment + }); + + return true; + } + + return this.experimentationService.isCachedFlightEnabled(experiment); + } +} diff --git a/src/client/common/experiments/telemetry.ts b/src/client/common/experiments/telemetry.ts new file mode 100644 index 000000000000..f6e13c9b8947 --- /dev/null +++ b/src/client/common/experiments/telemetry.ts @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { IExperimentationTelemetry } from 'vscode-tas-client'; +import { sendTelemetryEvent, setSharedProperty } from '../../telemetry'; + +export class ExperimentationTelemetry implements IExperimentationTelemetry { + public setSharedProperty(name: string, value: string): void { + // Add the shared property to all telemetry being sent, not just events being sent by the experimentation package. + setSharedProperty(name, value); + } + + public postEvent(eventName: string, properties: Map): void { + const formattedProperties: { [key: string]: string } = {}; + properties.forEach((value, key) => { + formattedProperties[key] = value; + }); + + // tslint:disable-next-line: no-any + sendTelemetryEvent(eventName as any, undefined, formattedProperties); + } +} diff --git a/src/client/common/serviceRegistry.ts b/src/client/common/serviceRegistry.ts index 3105e902c3d9..0fd06c872ec5 100644 --- a/src/client/common/serviceRegistry.ts +++ b/src/client/common/serviceRegistry.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import { IExtensionSingleActivationService } from '../activation/types'; -import { IFileDownloader, IHttpClient, IInterpreterPathService } from '../common/types'; +import { IExperimentService, IFileDownloader, IHttpClient, IInterpreterPathService } from '../common/types'; import { LiveShareApi } from '../datascience/liveshare/liveshare'; import { INotebookExecutionLogger } from '../datascience/types'; import { IServiceManager } from '../ioc/types'; @@ -39,7 +39,8 @@ import { AsyncDisposableRegistry } from './asyncDisposableRegistry'; import { ConfigurationService } from './configuration/service'; import { CryptoUtils } from './crypto'; import { EditorUtils } from './editor'; -import { ExperimentsManager } from './experiments'; +import { ExperimentsManager } from './experiments/manager'; +import { ExperimentService } from './experiments/service'; import { FeatureDeprecationManager } from './featureDeprecationManager'; import { ExtensionInsidersDailyChannelRule, @@ -146,6 +147,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(ILiveShareApi, LiveShareApi); serviceManager.addSingleton(ICryptoUtils, CryptoUtils); serviceManager.addSingleton(IExperimentsManager, ExperimentsManager); + serviceManager.addSingleton(IExperimentService, ExperimentService); serviceManager.addSingleton(ITerminalHelper, TerminalHelper); serviceManager.addSingleton( diff --git a/src/client/common/types.ts b/src/client/common/types.ts index ff33f32e19f0..044f433562bd 100644 --- a/src/client/common/types.ts +++ b/src/client/common/types.ts @@ -618,6 +618,14 @@ export interface IExperimentsManager { sendTelemetryIfInExperiment(experimentName: string): void; } +/** + * Experiment service leveraging VS Code's experiment framework. + */ +export const IExperimentService = Symbol('IExperimentService'); +export interface IExperimentService { + inExperiment(experimentName: string): Promise; +} + export type InterpreterConfigurationScope = { uri: Resource; configTarget: ConfigurationTarget }; export type InspectInterpreterSettingType = { globalValue?: string; diff --git a/src/client/common/utils/cacheUtils.ts b/src/client/common/utils/cacheUtils.ts index 66fe0c35f626..0f57303d75bf 100644 --- a/src/client/common/utils/cacheUtils.ts +++ b/src/client/common/utils/cacheUtils.ts @@ -9,7 +9,7 @@ import { Uri } from 'vscode'; import '../../common/extensions'; import { IServiceContainer } from '../../ioc/types'; import { DEFAULT_INTERPRETER_SETTING } from '../constants'; -import { DeprecatePythonPath } from '../experimentGroups'; +import { DeprecatePythonPath } from '../experiments/groups'; import { IExperimentsManager, IInterpreterPathService, Resource } from '../types'; type VSCodeType = typeof import('vscode'); diff --git a/src/client/datascience/data-viewing/dataViewer.ts b/src/client/datascience/data-viewing/dataViewer.ts index 25a3538c08dc..9c5985fd30c7 100644 --- a/src/client/datascience/data-viewing/dataViewer.ts +++ b/src/client/datascience/data-viewing/dataViewer.ts @@ -9,7 +9,7 @@ import { ViewColumn } from 'vscode'; import { IApplicationShell, IWebPanelProvider, IWorkspaceService } from '../../common/application/types'; import { EXTENSION_ROOT_DIR, UseCustomEditorApi } from '../../common/constants'; -import { WebHostNotebook } from '../../common/experimentGroups'; +import { WebHostNotebook } from '../../common/experiments/groups'; import { traceError } from '../../common/logger'; import { IConfigurationService, IDisposable, IExperimentsManager, Resource } from '../../common/types'; import * as localize from '../../common/utils/localize'; diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 0ea429d5a72a..cda47d3e2216 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -35,7 +35,7 @@ import { } from '../../common/application/types'; import { CancellationError } from '../../common/cancellation'; import { EXTENSION_ROOT_DIR, isTestExecution, PYTHON_LANGUAGE } from '../../common/constants'; -import { RunByLine, WebHostNotebook } from '../../common/experimentGroups'; +import { RunByLine, WebHostNotebook } from '../../common/experiments/groups'; import { traceError, traceInfo, traceWarning } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService, IDisposableRegistry, IExperimentsManager } from '../../common/types'; diff --git a/src/client/datascience/ipywidgets/ipywidgetHandler.ts b/src/client/datascience/ipywidgets/ipywidgetHandler.ts index 3a22f0a79462..c2f569070d80 100644 --- a/src/client/datascience/ipywidgets/ipywidgetHandler.ts +++ b/src/client/datascience/ipywidgets/ipywidgetHandler.ts @@ -12,7 +12,7 @@ import { LoadIPyWidgetClassLoadAction, NotifyIPyWidgeWidgetVersionNotSupportedAction } from '../../../datascience-ui/interactive-common/redux/reducers/types'; -import { EnableIPyWidgets } from '../../common/experimentGroups'; +import { EnableIPyWidgets } from '../../common/experiments/groups'; import { traceError, traceInfo } from '../../common/logger'; import { IDisposableRegistry, IExperimentsManager, IOutputChannel } from '../../common/types'; import * as localize from '../../common/utils/localize'; diff --git a/src/client/datascience/jupyter/jupyterDebugger.ts b/src/client/datascience/jupyter/jupyterDebugger.ts index 29413db006e7..e9d88dea98d2 100644 --- a/src/client/datascience/jupyter/jupyterDebugger.ts +++ b/src/client/datascience/jupyter/jupyterDebugger.ts @@ -9,7 +9,7 @@ import { DebugConfiguration } from 'vscode'; import * as vsls from 'vsls/vscode'; import { concatMultilineStringOutput } from '../../../datascience-ui/common'; import { IApplicationShell, ICommandManager, IDebugService, IWorkspaceService } from '../../common/application/types'; -import { DebugAdapterNewPtvsd } from '../../common/experimentGroups'; +import { DebugAdapterNewPtvsd } from '../../common/experiments/groups'; import { traceError, traceInfo, traceWarning } from '../../common/logger'; import { IPlatformService } from '../../common/platform/types'; import { IConfigurationService, IExperimentsManager, Version } from '../../common/types'; diff --git a/src/client/datascience/jupyter/jupyterVariables.ts b/src/client/datascience/jupyter/jupyterVariables.ts index f1ec50fbffc0..578c9abb465f 100644 --- a/src/client/datascience/jupyter/jupyterVariables.ts +++ b/src/client/datascience/jupyter/jupyterVariables.ts @@ -6,7 +6,7 @@ import { inject, injectable, named } from 'inversify'; import { Event, EventEmitter } from 'vscode'; import { IDebugService } from '../../common/application/types'; -import { RunByLine } from '../../common/experimentGroups'; +import { RunByLine } from '../../common/experiments/groups'; import { IDisposableRegistry, IExperimentsManager } from '../../common/types'; import { captureTelemetry } from '../../telemetry'; import { Identifiers, Telemetry } from '../constants'; diff --git a/src/client/datascience/plotting/plotViewer.ts b/src/client/datascience/plotting/plotViewer.ts index 6f17e0d9bf91..89e395907e63 100644 --- a/src/client/datascience/plotting/plotViewer.ts +++ b/src/client/datascience/plotting/plotViewer.ts @@ -11,7 +11,7 @@ import { traceInfo } from '../../../client/common/logger'; import { createDeferred } from '../../../client/common/utils/async'; import { IApplicationShell, IWebPanelProvider, IWorkspaceService } from '../../common/application/types'; import { EXTENSION_ROOT_DIR, UseCustomEditorApi } from '../../common/constants'; -import { WebHostNotebook } from '../../common/experimentGroups'; +import { WebHostNotebook } from '../../common/experiments/groups'; import { traceError } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService, IDisposable, IExperimentsManager, Resource } from '../../common/types'; diff --git a/src/client/datascience/raw-kernel/rawNotebookProvider.ts b/src/client/datascience/raw-kernel/rawNotebookProvider.ts index 22e222a3e208..f9c863b8710f 100644 --- a/src/client/datascience/raw-kernel/rawNotebookProvider.ts +++ b/src/client/datascience/raw-kernel/rawNotebookProvider.ts @@ -6,7 +6,7 @@ import * as uuid from 'uuid/v4'; import { Event, EventEmitter, Uri } from 'vscode'; import { CancellationToken } from 'vscode-jsonrpc'; import { ILiveShareApi } from '../../common/application/types'; -import { LocalZMQKernel } from '../../common/experimentGroups'; +import { LocalZMQKernel } from '../../common/experiments/groups'; import '../../common/extensions'; import { traceError, traceInfo } from '../../common/logger'; import { IAsyncDisposableRegistry, IConfigurationService, IExperimentsManager, Resource } from '../../common/types'; diff --git a/src/client/debugger/extension/adapter/factory.ts b/src/client/debugger/extension/adapter/factory.ts index c0339f9e4705..23d428b18470 100644 --- a/src/client/debugger/extension/adapter/factory.ts +++ b/src/client/debugger/extension/adapter/factory.ts @@ -13,7 +13,7 @@ import { WorkspaceFolder } from 'vscode'; import { IApplicationShell } from '../../../common/application/types'; -import { DebugAdapterNewPtvsd } from '../../../common/experimentGroups'; +import { DebugAdapterNewPtvsd } from '../../../common/experiments/groups'; import { traceVerbose } from '../../../common/logger'; import { IExperimentsManager } from '../../../common/types'; import { EXTENSION_ROOT_DIR } from '../../../constants'; diff --git a/src/client/debugger/extension/adapter/outdatedDebuggerPrompt.ts b/src/client/debugger/extension/adapter/outdatedDebuggerPrompt.ts index f149e0958dbd..457ac2c18a77 100644 --- a/src/client/debugger/extension/adapter/outdatedDebuggerPrompt.ts +++ b/src/client/debugger/extension/adapter/outdatedDebuggerPrompt.ts @@ -7,7 +7,7 @@ import { inject, injectable } from 'inversify'; import { DebugAdapterTracker, DebugAdapterTrackerFactory, DebugSession, ProviderResult } from 'vscode'; import { DebugProtocol } from 'vscode-debugprotocol'; import { IApplicationShell } from '../../../common/application/types'; -import { DebugAdapterNewPtvsd } from '../../../common/experimentGroups'; +import { DebugAdapterNewPtvsd } from '../../../common/experiments/groups'; import { IBrowserService, IExperimentsManager } from '../../../common/types'; import { Common, OutdatedDebugger } from '../../../common/utils/localize'; import { IPromptShowState } from './types'; diff --git a/src/client/debugger/extension/configuration/resolvers/attach.ts b/src/client/debugger/extension/configuration/resolvers/attach.ts index bcfdd4734e79..7fb951cb386b 100644 --- a/src/client/debugger/extension/configuration/resolvers/attach.ts +++ b/src/client/debugger/extension/configuration/resolvers/attach.ts @@ -6,7 +6,7 @@ import { inject, injectable } from 'inversify'; import { CancellationToken, Uri, WorkspaceFolder } from 'vscode'; import { IDocumentManager, IWorkspaceService } from '../../../../common/application/types'; -import { DebugAdapterNewPtvsd } from '../../../../common/experimentGroups'; +import { DebugAdapterNewPtvsd } from '../../../../common/experiments/groups'; import { IPlatformService } from '../../../../common/platform/types'; import { IConfigurationService, IExperimentsManager } from '../../../../common/types'; import { Diagnostics } from '../../../../common/utils/localize'; diff --git a/src/client/debugger/extension/configuration/resolvers/launchConfigExperiment.ts b/src/client/debugger/extension/configuration/resolvers/launchConfigExperiment.ts index d6416950adf0..66abc44ecbd3 100644 --- a/src/client/debugger/extension/configuration/resolvers/launchConfigExperiment.ts +++ b/src/client/debugger/extension/configuration/resolvers/launchConfigExperiment.ts @@ -4,7 +4,7 @@ 'use strict'; import { inject, injectable } from 'inversify'; -import { DebugAdapterNewPtvsd, WebAppReload } from '../../../../common/experimentGroups'; +import { DebugAdapterNewPtvsd, WebAppReload } from '../../../../common/experiments/groups'; import { traceInfo } from '../../../../common/logger'; import { IExperimentsManager } from '../../../../common/types'; import { sendTelemetryEvent } from '../../../../telemetry'; diff --git a/src/client/interpreter/activation/wrapperEnvironmentActivationService.ts b/src/client/interpreter/activation/wrapperEnvironmentActivationService.ts index b8fc7317caeb..4fb5d8ea2e29 100644 --- a/src/client/interpreter/activation/wrapperEnvironmentActivationService.ts +++ b/src/client/interpreter/activation/wrapperEnvironmentActivationService.ts @@ -5,7 +5,7 @@ import { inject, injectable } from 'inversify'; import * as path from 'path'; -import { UseTerminalToGetActivatedEnvVars } from '../../common/experimentGroups'; +import { UseTerminalToGetActivatedEnvVars } from '../../common/experiments/groups'; import '../../common/extensions'; import { traceError } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; diff --git a/src/client/interpreter/autoSelection/rules/settings.ts b/src/client/interpreter/autoSelection/rules/settings.ts index bbd79abe6f0c..eae85e915ca2 100644 --- a/src/client/interpreter/autoSelection/rules/settings.ts +++ b/src/client/interpreter/autoSelection/rules/settings.ts @@ -5,7 +5,7 @@ import { inject, injectable } from 'inversify'; import { IWorkspaceService } from '../../../common/application/types'; -import { DeprecatePythonPath } from '../../../common/experimentGroups'; +import { DeprecatePythonPath } from '../../../common/experiments/groups'; import { IFileSystem } from '../../../common/platform/types'; import { IExperimentsManager, IInterpreterPathService, IPersistentStateFactory, Resource } from '../../../common/types'; import { AutoSelectionRule, IInterpreterAutoSelectionService } from '../types'; diff --git a/src/client/interpreter/autoSelection/rules/workspaceEnv.ts b/src/client/interpreter/autoSelection/rules/workspaceEnv.ts index d033b087c451..53bf0cb6f769 100644 --- a/src/client/interpreter/autoSelection/rules/workspaceEnv.ts +++ b/src/client/interpreter/autoSelection/rules/workspaceEnv.ts @@ -6,7 +6,7 @@ import { inject, injectable, named } from 'inversify'; import { Uri } from 'vscode'; import { IWorkspaceService } from '../../../common/application/types'; -import { DeprecatePythonPath } from '../../../common/experimentGroups'; +import { DeprecatePythonPath } from '../../../common/experiments/groups'; import { traceVerbose } from '../../../common/logger'; import { IFileSystem, IPlatformService } from '../../../common/platform/types'; import { IExperimentsManager, IInterpreterPathService, IPersistentStateFactory, Resource } from '../../../common/types'; diff --git a/src/client/interpreter/configuration/interpreterSelector/interpreterSelector.ts b/src/client/interpreter/configuration/interpreterSelector/interpreterSelector.ts index f7c95fb7e0e1..6afc9ace5b45 100644 --- a/src/client/interpreter/configuration/interpreterSelector/interpreterSelector.ts +++ b/src/client/interpreter/configuration/interpreterSelector/interpreterSelector.ts @@ -5,7 +5,7 @@ import { inject, injectable } from 'inversify'; import { Disposable, Uri } from 'vscode'; -import { DeprecatePythonPath } from '../../../common/experimentGroups'; +import { DeprecatePythonPath } from '../../../common/experiments/groups'; import { IExperimentsManager, IPathUtils, Resource } from '../../../common/types'; import { IInterpreterSecurityService } from '../../autoSelection/types'; import { IInterpreterService, PythonInterpreter } from '../../contracts'; diff --git a/src/client/interpreter/configuration/pythonPathUpdaterServiceFactory.ts b/src/client/interpreter/configuration/pythonPathUpdaterServiceFactory.ts index b1e70235c4b8..aecf7beeb5b6 100644 --- a/src/client/interpreter/configuration/pythonPathUpdaterServiceFactory.ts +++ b/src/client/interpreter/configuration/pythonPathUpdaterServiceFactory.ts @@ -1,7 +1,7 @@ import { inject, injectable } from 'inversify'; import { Uri } from 'vscode'; import { IWorkspaceService } from '../../common/application/types'; -import { DeprecatePythonPath } from '../../common/experimentGroups'; +import { DeprecatePythonPath } from '../../common/experiments/groups'; import { IExperimentsManager, IInterpreterPathService } from '../../common/types'; import { IServiceContainer } from '../../ioc/types'; import { GlobalPythonPathUpdaterService } from './services/globalUpdaterService'; diff --git a/src/client/interpreter/interpreterService.ts b/src/client/interpreter/interpreterService.ts index 6eef287fe8e4..0bbc5b48ac2e 100644 --- a/src/client/interpreter/interpreterService.ts +++ b/src/client/interpreter/interpreterService.ts @@ -4,7 +4,7 @@ import * as path from 'path'; import { Disposable, Event, EventEmitter, Uri } from 'vscode'; import '../../client/common/extensions'; import { IDocumentManager, IWorkspaceService } from '../common/application/types'; -import { DeprecatePythonPath } from '../common/experimentGroups'; +import { DeprecatePythonPath } from '../common/experiments/groups'; import { traceError } from '../common/logger'; import { getArchitectureDisplayName } from '../common/platform/registry'; import { IFileSystem } from '../common/platform/types'; diff --git a/src/client/startupTelemetry.ts b/src/client/startupTelemetry.ts index c84e78c10837..d48ebeedd397 100644 --- a/src/client/startupTelemetry.ts +++ b/src/client/startupTelemetry.ts @@ -3,7 +3,7 @@ import { IWorkspaceService } from './common/application/types'; import { isTestExecution } from './common/constants'; -import { DeprecatePythonPath } from './common/experimentGroups'; +import { DeprecatePythonPath } from './common/experiments/groups'; import { traceError } from './common/logger'; import { ITerminalHelper } from './common/terminal/types'; import { diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index 89a918976ca9..dc8f5655f423 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -1,15 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// tslint:disable:no-reference no-any import-name no-any function-name -/// + import type { JSONObject } from '@phosphor/coreutils'; -import { basename as pathBasename, sep as pathSep } from 'path'; import * as stackTrace from 'stack-trace'; -import TelemetryReporter from 'vscode-extension-telemetry'; +// tslint:disable-next-line: import-name +import TelemetryReporter from 'vscode-extension-telemetry/lib/telemetryReporter'; import { DiagnosticCodes } from '../application/diagnostics/constants'; import { IWorkspaceService } from '../common/application/types'; -import { AppinsightsKey, EXTENSION_ROOT_DIR, isTestExecution, PVSC_EXTENSION_ID } from '../common/constants'; +import { AppinsightsKey, isTestExecution, PVSC_EXTENSION_ID } from '../common/constants'; import { traceError, traceInfo } from '../common/logger'; import { TerminalShellType } from '../common/terminal/types'; import { StopWatch } from '../common/utils/stopWatch'; @@ -28,10 +27,12 @@ import { TestProvider } from '../testing/common/types'; import { EventName, PlatformErrors } from './constants'; import { LinterTrigger, TestTool } from './types'; +// tslint:disable: no-any + /** * Checks whether telemetry is supported. * Its possible this function gets called within Debug Adapter, vscode isn't available in there. - * Withiin DA, there's a completely different way to send telemetry. + * Within DA, there's a completely different way to send telemetry. * @returns {boolean} */ function isTelemetrySupported(): boolean { @@ -55,6 +56,24 @@ export function isTelemetryDisabled(workspaceService: IWorkspaceService): boolea return settings.globalValue === false ? true : false; } +// Shared properties set by the IExperimentationTelemetry implementation. +const sharedProperties: Record = {}; +/** + * Set shared properties for all telemetry events. + */ +export function setSharedProperty(name: string, value: string): void { + sharedProperties[name] = value; +} + +/** + * Reset shared properties for testing purposes. + */ +export function _resetSharedProperties(): void { + for (const key of Object.keys(sharedProperties)) { + delete sharedProperties[key]; + } +} + let telemetryReporter: TelemetryReporter | undefined; function getTelemetryReporter() { if (!isTestExecution() && telemetryReporter) { @@ -63,14 +82,12 @@ function getTelemetryReporter() { const extensionId = PVSC_EXTENSION_ID; // tslint:disable-next-line:no-require-imports const extensions = (require('vscode') as typeof import('vscode')).extensions; - // tslint:disable-next-line:no-non-null-assertion const extension = extensions.getExtension(extensionId)!; - // tslint:disable-next-line:no-unsafe-any const extensionVersion = extension.packageJSON.version; // tslint:disable-next-line:no-require-imports const reporter = require('vscode-extension-telemetry').default as typeof TelemetryReporter; - return (telemetryReporter = new reporter(extensionId, extensionVersion, AppinsightsKey)); + return (telemetryReporter = new reporter(extensionId, extensionVersion, AppinsightsKey, true)); } export function clearTelemetryReporter() { @@ -88,45 +105,49 @@ export function sendTelemetryEvent

= {}; + let eventNameSent = eventName as string; - if (ex && (eventName as any) !== 'ERROR') { - // When sending `ERROR` telemetry event no need to send custom properties. + if (ex) { + // When sending telemetry events for exceptions no need to send custom properties. // Else we have to review all properties every time as part of GDPR. // Assume we have 10 events all with their own properties. // As we have errors for each event, those properties are treated as new data items. // Hence they need to be classified as part of the GDPR process, and thats unnecessary and onerous. - const props: Record = {}; - props.stackTrace = getStackTrace(ex); - props.originalEventName = (eventName as any) as string; - reporter.sendTelemetryEvent('ERROR', props, measures); - } - const customProperties: Record = {}; - if (properties) { - // tslint:disable-next-line:prefer-type-cast no-any - const data = properties as any; - Object.getOwnPropertyNames(data).forEach((prop) => { - if (data[prop] === undefined || data[prop] === null) { - return; - } - try { - // If there are any errors in serializing one property, ignore that and move on. - // Else nothign will be sent. - // tslint:disable-next-line:prefer-type-cast no-any no-unsafe-any - (customProperties as any)[prop] = - typeof data[prop] === 'string' - ? data[prop] - : typeof data[prop] === 'object' - ? 'object' - : data[prop].toString(); - } catch (ex) { - traceError(`Failed to serialize ${prop} for ${eventName}`, ex); - } - }); + eventNameSent = 'ERROR'; + customProperties = { originalEventName: eventName as string, stackTrace: serializeStackTrace(ex) }; + reporter.sendTelemetryErrorEvent(eventNameSent, customProperties, measures, []); + } else { + if (properties) { + const data = properties as any; + Object.getOwnPropertyNames(data).forEach((prop) => { + if (data[prop] === undefined || data[prop] === null) { + return; + } + try { + // If there are any errors in serializing one property, ignore that and move on. + // Else nothing will be sent. + customProperties[prop] = + typeof data[prop] === 'string' + ? data[prop] + : typeof data[prop] === 'object' + ? 'object' + : data[prop].toString(); + } catch (ex) { + traceError(`Failed to serialize ${prop} for ${eventName}`, ex); + } + }); + } + + // Add shared properties to telemetry props (we may overwrite existing ones). + Object.assign(customProperties, sharedProperties); + + reporter.sendTelemetryEvent(eventNameSent, customProperties, measures); } - reporter.sendTelemetryEvent((eventName as any) as string, customProperties, measures); + if (process.env && process.env.VSC_PYTHON_LOG_TELEMETRY) { traceInfo( - `Telemetry Event : ${eventName} Measures: ${JSON.stringify(measures)} Props: ${JSON.stringify( + `Telemetry Event : ${eventNameSent} Measures: ${JSON.stringify(measures)} Props: ${JSON.stringify( customProperties )} ` ); @@ -246,32 +267,12 @@ export function sendTelemetryWhenDone

${filename.substring(EXTENSION_ROOT_DIR.length)}`; - } else { - // We don't really care about files outside our extension. - filename = `${pathSep}${pathBasename(filename)}`; - } - return filename; -} - -function sanitizeName(name: string): string { - if (name.indexOf('/') === -1 && name.indexOf('\\') === -1) { - return name; - } else { - return ''; - } -} - -function getStackTrace(ex: Error): string { - // We aren't showing the error message (ex.message) since it might - // contain PII. +function serializeStackTrace(ex: Error): string { + // We aren't showing the error message (ex.message) since it might contain PII. let trace = ''; for (const frame of stackTrace.parse(ex)) { - let filename = frame.getFileName(); + const filename = frame.getFileName(); if (filename) { - filename = sanitizeFilename(filename); const lineno = frame.getLineNumber(); const colno = frame.getColumnNumber(); trace += `\n\tat ${getCallsite(frame)} ${filename}:${lineno}:${colno}`; @@ -279,8 +280,8 @@ function getStackTrace(ex: Error): string { trace += '\n\tat '; } } - // Ensure we always use `/` as path seperators. - // This way stack traces (with relative paths) comming from different OS will always look the same. + // Ensure we always use `/` as path separators. + // This way stack traces (with relative paths) coming from different OS will always look the same. return trace.trim().replace(/\\/g, '/'); } @@ -297,7 +298,7 @@ function getCallsite(frame: stackTrace.StackFrame) { parts.push(frame.getFunctionName()); } } - return parts.map(sanitizeName).join('.'); + return parts.join('.'); } // Map all events to their properties diff --git a/src/client/telemetry/vscode-extension-telemetry.d.ts b/src/client/telemetry/vscode-extension-telemetry.d.ts deleted file mode 100644 index a5d3d6294739..000000000000 --- a/src/client/telemetry/vscode-extension-telemetry.d.ts +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -declare module 'vscode-extension-telemetry' { - export default class TelemetryReporter { - /** - * Constructs a new telemetry reporter - * @param {string} extensionId All events will be prefixed with this event name - * @param {string} extensionVersion Extension version to be reported with each event - * @param {string} key The application insights key - */ - // tslint:disable-next-line:no-empty - constructor(extensionId: string, extensionVersion: string, key: string); - - /** - * Sends a telemetry event - * @param {string} eventName The event name - * @param {object} properties An associative array of strings - * @param {object} measures An associative array of numbers - */ - // tslint:disable-next-line:member-access - public sendTelemetryEvent( - eventName: string, - properties?: { - [key: string]: string; - }, - measures?: { - [key: string]: number; - // tslint:disable-next-line:no-empty - } - ): void; - } -} diff --git a/src/client/testing/main.ts b/src/client/testing/main.ts index f7e84b4b1ff0..f4763bd74b23 100644 --- a/src/client/testing/main.ts +++ b/src/client/testing/main.ts @@ -15,7 +15,7 @@ import { } from 'vscode'; import { IApplicationShell, ICommandManager, IDocumentManager, IWorkspaceService } from '../common/application/types'; import * as constants from '../common/constants'; -import { AlwaysDisplayTestExplorerGroups } from '../common/experimentGroups'; +import { AlwaysDisplayTestExplorerGroups } from '../common/experiments/groups'; import '../common/extensions'; import { traceError } from '../common/logger'; import { diff --git a/src/test/activation/aaTesting.unit.test.ts b/src/test/activation/aaTesting.unit.test.ts index 78e1af84a2c9..97cba28f3e03 100644 --- a/src/test/activation/aaTesting.unit.test.ts +++ b/src/test/activation/aaTesting.unit.test.ts @@ -5,7 +5,7 @@ import * as TypeMoq from 'typemoq'; import { AATesting } from '../../client/activation/aaTesting'; -import { ValidateABTesting } from '../../client/common/experimentGroups'; +import { ValidateABTesting } from '../../client/common/experiments/groups'; import { IExperimentsManager } from '../../client/common/types'; suite('A/A Testing', () => { diff --git a/src/test/activation/activationManager.unit.test.ts b/src/test/activation/activationManager.unit.test.ts index 43a14e4978c0..c4fc941ed17b 100644 --- a/src/test/activation/activationManager.unit.test.ts +++ b/src/test/activation/activationManager.unit.test.ts @@ -16,8 +16,8 @@ import { ActiveResourceService } from '../../client/common/application/activeRes import { IActiveResourceService, IDocumentManager, IWorkspaceService } from '../../client/common/application/types'; import { WorkspaceService } from '../../client/common/application/workspace'; import { PYTHON_LANGUAGE } from '../../client/common/constants'; -import { DeprecatePythonPath } from '../../client/common/experimentGroups'; -import { ExperimentsManager } from '../../client/common/experiments'; +import { DeprecatePythonPath } from '../../client/common/experiments/groups'; +import { ExperimentsManager } from '../../client/common/experiments/manager'; import { InterpreterPathService } from '../../client/common/interpreterPathService'; import { FileSystem } from '../../client/common/platform/fileSystem'; import { IFileSystem } from '../../client/common/platform/types'; diff --git a/src/test/activation/activationService.unit.test.ts b/src/test/activation/activationService.unit.test.ts index 4dccd02b03f3..8c8d884d1c93 100644 --- a/src/test/activation/activationService.unit.test.ts +++ b/src/test/activation/activationService.unit.test.ts @@ -16,7 +16,7 @@ import { import { LSNotSupportedDiagnosticServiceId } from '../../client/application/diagnostics/checks/lsNotSupported'; import { IDiagnostic, IDiagnosticsService } from '../../client/application/diagnostics/types'; import { IApplicationShell, ICommandManager, IWorkspaceService } from '../../client/common/application/types'; -import { LSControl, LSEnabled } from '../../client/common/experimentGroups'; +import { LSControl, LSEnabled } from '../../client/common/experiments/groups'; import { IPlatformService } from '../../client/common/platform/types'; import { IConfigurationService, diff --git a/src/test/activation/extensionSurvey.unit.test.ts b/src/test/activation/extensionSurvey.unit.test.ts index ddee96f38b58..e6ce42310631 100644 --- a/src/test/activation/extensionSurvey.unit.test.ts +++ b/src/test/activation/extensionSurvey.unit.test.ts @@ -9,7 +9,7 @@ import { anything, instance, mock, verify, when } from 'ts-mockito'; import * as TypeMoq from 'typemoq'; import { ExtensionSurveyPrompt, extensionSurveyStateKeys } from '../../client/activation/extensionSurvey'; import { IApplicationEnvironment, IApplicationShell } from '../../client/common/application/types'; -import { ShowExtensionSurveyPrompt } from '../../client/common/experimentGroups'; +import { ShowExtensionSurveyPrompt } from '../../client/common/experiments/groups'; import { PersistentStateFactory } from '../../client/common/persistentState'; import { IPlatformService } from '../../client/common/platform/types'; import { diff --git a/src/test/activation/languageServer/manager.unit.test.ts b/src/test/activation/languageServer/manager.unit.test.ts index 143a754a2dab..4e0b359c686c 100644 --- a/src/test/activation/languageServer/manager.unit.test.ts +++ b/src/test/activation/languageServer/manager.unit.test.ts @@ -17,7 +17,7 @@ import { ILanguageServerProxy } from '../../../client/activation/types'; import { ConfigurationService } from '../../../client/common/configuration/service'; -import { ExperimentsManager } from '../../../client/common/experiments'; +import { ExperimentsManager } from '../../../client/common/experiments/manager'; import { IConfigurationService, IExperimentsManager, IPythonExtensionBanner } from '../../../client/common/types'; import { ServiceContainer } from '../../../client/ioc/container'; import { IServiceContainer } from '../../../client/ioc/types'; diff --git a/src/test/api.functional.test.ts b/src/test/api.functional.test.ts index 3ddf70b837c3..72ca62d17580 100644 --- a/src/test/api.functional.test.ts +++ b/src/test/api.functional.test.ts @@ -12,7 +12,7 @@ import { Uri } from 'vscode'; import { buildApi } from '../client/api'; import { ConfigurationService } from '../client/common/configuration/service'; import { EXTENSION_ROOT_DIR } from '../client/common/constants'; -import { ExperimentsManager } from '../client/common/experiments'; +import { ExperimentsManager } from '../client/common/experiments/manager'; import { IConfigurationService, IExperimentsManager } from '../client/common/types'; import { ServiceContainer } from '../client/ioc/container'; import { ServiceManager } from '../client/ioc/serviceManager'; diff --git a/src/test/application/diagnostics/checks/macPythonInterpreter.unit.test.ts b/src/test/application/diagnostics/checks/macPythonInterpreter.unit.test.ts index eaa9315341d3..6389e7f81b4d 100644 --- a/src/test/application/diagnostics/checks/macPythonInterpreter.unit.test.ts +++ b/src/test/application/diagnostics/checks/macPythonInterpreter.unit.test.ts @@ -29,7 +29,7 @@ import { } from '../../../../client/application/diagnostics/types'; import { CommandsWithoutArgs } from '../../../../client/common/application/commands'; import { IWorkspaceService } from '../../../../client/common/application/types'; -import { DeprecatePythonPath } from '../../../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../../../client/common/experiments/groups'; import { IPlatformService } from '../../../../client/common/platform/types'; import { IConfigurationService, diff --git a/src/test/application/diagnostics/checks/pythonPathDeprecated.unit.test.ts b/src/test/application/diagnostics/checks/pythonPathDeprecated.unit.test.ts index 014bf92eff1e..32cdb98b2032 100644 --- a/src/test/application/diagnostics/checks/pythonPathDeprecated.unit.test.ts +++ b/src/test/application/diagnostics/checks/pythonPathDeprecated.unit.test.ts @@ -28,7 +28,7 @@ import { IDiagnosticHandlerService } from '../../../../client/application/diagnostics/types'; import { IWorkspaceService } from '../../../../client/common/application/types'; -import { DeprecatePythonPath } from '../../../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../../../client/common/experiments/groups'; import { IDisposableRegistry, IExperimentsManager, Resource } from '../../../../client/common/types'; import { Common, Diagnostics } from '../../../../client/common/utils/localize'; import { IServiceContainer } from '../../../../client/ioc/types'; diff --git a/src/test/application/diagnostics/checks/upgradeCodeRunner.unit.test.ts b/src/test/application/diagnostics/checks/upgradeCodeRunner.unit.test.ts index 64b4f9f35ee5..7664f16d937a 100644 --- a/src/test/application/diagnostics/checks/upgradeCodeRunner.unit.test.ts +++ b/src/test/application/diagnostics/checks/upgradeCodeRunner.unit.test.ts @@ -28,7 +28,7 @@ import { } from '../../../../client/application/diagnostics/types'; import { IWorkspaceService } from '../../../../client/common/application/types'; import { CODE_RUNNER_EXTENSION_ID } from '../../../../client/common/constants'; -import { DeprecatePythonPath } from '../../../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../../../client/common/experiments/groups'; import { IDisposableRegistry, IExperimentsManager, IExtensions, Resource } from '../../../../client/common/types'; import { Common, Diagnostics } from '../../../../client/common/utils/localize'; import { IServiceContainer } from '../../../../client/ioc/types'; diff --git a/src/test/common/configSettings/configSettings.pythonPath.unit.test.ts b/src/test/common/configSettings/configSettings.pythonPath.unit.test.ts index b15ebfff705e..43099b8d54cd 100644 --- a/src/test/common/configSettings/configSettings.pythonPath.unit.test.ts +++ b/src/test/common/configSettings/configSettings.pythonPath.unit.test.ts @@ -13,7 +13,7 @@ import * as typemoq from 'typemoq'; import { Uri, WorkspaceConfiguration } from 'vscode'; import { IWorkspaceService } from '../../../client/common/application/types'; import { PythonSettings } from '../../../client/common/configSettings'; -import { DeprecatePythonPath } from '../../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../../client/common/experiments/groups'; import { IExperimentsManager, IInterpreterPathService } from '../../../client/common/types'; import { noop } from '../../../client/common/utils/misc'; import { IInterpreterSecurityService } from '../../../client/interpreter/autoSelection/types'; diff --git a/src/test/common/configuration/service.unit.test.ts b/src/test/common/configuration/service.unit.test.ts index 87fceaad4a7c..887a0779acf7 100644 --- a/src/test/common/configuration/service.unit.test.ts +++ b/src/test/common/configuration/service.unit.test.ts @@ -9,7 +9,7 @@ import { ConfigurationTarget, Uri, WorkspaceConfiguration } from 'vscode'; import { IWorkspaceService } from '../../../client/common/application/types'; import { PythonSettings } from '../../../client/common/configSettings'; import { ConfigurationService } from '../../../client/common/configuration/service'; -import { DeprecatePythonPath } from '../../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../../client/common/experiments/groups'; import { IExperimentsManager, IInterpreterPathService } from '../../../client/common/types'; import { IInterpreterAutoSeletionProxyService, diff --git a/src/test/common/experiments.unit.test.ts b/src/test/common/experiments/manager.unit.test.ts similarity index 96% rename from src/test/common/experiments.unit.test.ts rename to src/test/common/experiments/manager.unit.test.ts index b36425f366f6..fc7fec8f4e1e 100644 --- a/src/test/common/experiments.unit.test.ts +++ b/src/test/common/experiments/manager.unit.test.ts @@ -9,11 +9,11 @@ import { assert, expect } from 'chai'; import * as sinon from 'sinon'; import { anything, instance, mock, verify, when } from 'ts-mockito'; import * as TypeMoq from 'typemoq'; -import { ApplicationEnvironment } from '../../client/common/application/applicationEnvironment'; -import { IApplicationEnvironment } from '../../client/common/application/types'; -import { PythonSettings } from '../../client/common/configSettings'; -import { ConfigurationService } from '../../client/common/configuration/service'; -import { CryptoUtils } from '../../client/common/crypto'; +import { ApplicationEnvironment } from '../../../client/common/application/applicationEnvironment'; +import { IApplicationEnvironment } from '../../../client/common/application/types'; +import { PythonSettings } from '../../../client/common/configSettings'; +import { ConfigurationService } from '../../../client/common/configuration/service'; +import { CryptoUtils } from '../../../client/common/crypto'; import { configUri, downloadedExperimentStorageKey, @@ -21,11 +21,11 @@ import { experimentStorageKey, isDownloadedStorageValidKey, oldExperimentSalts -} from '../../client/common/experiments'; -import { HttpClient } from '../../client/common/net/httpClient'; -import { PersistentStateFactory } from '../../client/common/persistentState'; -import { FileSystem } from '../../client/common/platform/fileSystem'; -import { IFileSystem } from '../../client/common/platform/types'; +} from '../../../client/common/experiments/manager'; +import { HttpClient } from '../../../client/common/net/httpClient'; +import { PersistentStateFactory } from '../../../client/common/persistentState'; +import { FileSystem } from '../../../client/common/platform/fileSystem'; +import { IFileSystem } from '../../../client/common/platform/types'; import { ICryptoUtils, IExperiments, @@ -33,10 +33,10 @@ import { IOutputChannel, IPersistentState, IPersistentStateFactory -} from '../../client/common/types'; -import { createDeferred, createDeferredFromPromise } from '../../client/common/utils/async'; -import { sleep } from '../common'; -import { noop } from '../core'; +} from '../../../client/common/types'; +import { createDeferred, createDeferredFromPromise } from '../../../client/common/utils/async'; +import { sleep } from '../../common'; +import { noop } from '../../core'; // tslint:disable: max-func-body-length @@ -840,6 +840,17 @@ suite('A/B experiments', () => { experimentsOptedInto: ['experiment1'], expectedResult: [{ name: 'experiment1', salt: 'salt', min: 79, max: 94 }] }, + { + testName: + 'User experiments list contains the experiment user has opened into and not the control experiment even if user is in the control experiment range', + experimentStorageValue: [ + { name: 'control', salt: 'salt', min: 0, max: 100 }, + { name: 'experiment', salt: 'salt', min: 0, max: 0 } + ], + hash: 8187, + experimentsOptedInto: ['experiment'], + expectedResult: [{ name: 'experiment', salt: 'salt', min: 0, max: 0 }] + }, { testName: 'User experiments list does not contain the experiment if user has both opted in and out of an experiment', diff --git a/src/test/common/experiments/service.unit.test.ts b/src/test/common/experiments/service.unit.test.ts new file mode 100644 index 000000000000..7cc26dcd0fc7 --- /dev/null +++ b/src/test/common/experiments/service.unit.test.ts @@ -0,0 +1,256 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { assert } from 'chai'; +import * as sinon from 'sinon'; +import { instance, mock, when } from 'ts-mockito'; +import * as tasClient from 'vscode-tas-client'; +import { ApplicationEnvironment } from '../../../client/common/application/applicationEnvironment'; +import { Channel, IApplicationEnvironment } from '../../../client/common/application/types'; +import { ConfigurationService } from '../../../client/common/configuration/service'; +import { ExperimentService } from '../../../client/common/experiments/service'; +import { IConfigurationService } from '../../../client/common/types'; +import * as Telemetry from '../../../client/telemetry'; +import { EventName } from '../../../client/telemetry/constants'; +import { PVSC_EXTENSION_ID_FOR_TESTS } from '../../constants'; +import { MockMemento } from '../../mocks/mementos'; + +suite('Experimentation service', () => { + const extensionVersion = '1.2.3'; + + let configurationService: IConfigurationService; + let appEnvironment: IApplicationEnvironment; + let globalMemento: MockMemento; + + setup(() => { + configurationService = mock(ConfigurationService); + appEnvironment = mock(ApplicationEnvironment); + globalMemento = new MockMemento(); + }); + + teardown(() => { + sinon.restore(); + }); + + function configureSettings(enabled: boolean, optInto: string[], optOutFrom: string[]) { + when(configurationService.getSettings(undefined)).thenReturn({ + experiments: { + enabled, + optInto, + optOutFrom + } + // tslint:disable-next-line: no-any + } as any); + } + + function configureApplicationEnvironment(channel: Channel, version: string) { + when(appEnvironment.channel).thenReturn(channel); + when(appEnvironment.extensionName).thenReturn(PVSC_EXTENSION_ID_FOR_TESTS); + when(appEnvironment.packageJson).thenReturn({ version }); + } + + suite('Initialization', () => { + test('Users with a release version of the extension should be in the Public target population', () => { + const getExperimentationServiceStub = sinon.stub(tasClient, 'getExperimentationService'); + + configureSettings(true, [], []); + configureApplicationEnvironment('stable', extensionVersion); + + // tslint:disable-next-line: no-unused-expression + new ExperimentService(instance(configurationService), instance(appEnvironment), globalMemento); + + sinon.assert.calledWithExactly( + getExperimentationServiceStub, + PVSC_EXTENSION_ID_FOR_TESTS, + extensionVersion, + tasClient.TargetPopulation.Public, + sinon.match.any, + globalMemento + ); + }); + + test('Users with an Insiders version of the extension should be the Insiders target population', () => { + const getExperimentationServiceStub = sinon.stub(tasClient, 'getExperimentationService'); + + configureSettings(true, [], []); + configureApplicationEnvironment('insiders', extensionVersion); + + // tslint:disable-next-line: no-unused-expression + new ExperimentService(instance(configurationService), instance(appEnvironment), globalMemento); + + sinon.assert.calledWithExactly( + getExperimentationServiceStub, + PVSC_EXTENSION_ID_FOR_TESTS, + extensionVersion, + tasClient.TargetPopulation.Insiders, + sinon.match.any, + globalMemento + ); + }); + + test('Users can only opt into experiment groups', () => { + sinon.stub(tasClient, 'getExperimentationService'); + + configureSettings(true, ['Foo - experiment', 'Bar - control'], []); + configureApplicationEnvironment('stable', extensionVersion); + + const experimentService = new ExperimentService( + instance(configurationService), + instance(appEnvironment), + globalMemento + ); + + assert.deepEqual(experimentService._optInto, ['Foo - experiment']); + }); + + test('Users can only opt out of experiment groups', () => { + sinon.stub(tasClient, 'getExperimentationService'); + configureSettings(true, [], ['Foo - experiment', 'Bar - control']); + configureApplicationEnvironment('stable', extensionVersion); + + const experimentService = new ExperimentService( + instance(configurationService), + instance(appEnvironment), + globalMemento + ); + + assert.deepEqual(experimentService._optOutFrom, ['Foo - experiment']); + }); + }); + + suite('In-experiment check', () => { + const experiment = 'Test Experiment - experiment'; + let telemetryEvents: { eventName: string; properties: object }[] = []; + let isCachedFlightEnabledStub: sinon.SinonStub; + let sendTelemetryEventStub: sinon.SinonStub; + + setup(() => { + sendTelemetryEventStub = sinon + .stub(Telemetry, 'sendTelemetryEvent') + .callsFake((eventName: string, _, properties: object) => { + const telemetry = { eventName, properties }; + telemetryEvents.push(telemetry); + }); + + isCachedFlightEnabledStub = sinon.stub().returns(Promise.resolve(true)); + sinon.stub(tasClient, 'getExperimentationService').returns({ + isCachedFlightEnabled: isCachedFlightEnabledStub + // tslint:disable-next-line: no-any + } as any); + + configureApplicationEnvironment('stable', extensionVersion); + }); + + teardown(() => { + telemetryEvents = []; + }); + + test('If the opt-in and opt-out arrays are empty, return the value from the experimentation framework for a given experiment', async () => { + configureSettings(true, [], []); + + const experimentService = new ExperimentService( + instance(configurationService), + instance(appEnvironment), + globalMemento + ); + const result = await experimentService.inExperiment(experiment); + + assert.isTrue(result); + sinon.assert.notCalled(sendTelemetryEventStub); + sinon.assert.calledOnce(isCachedFlightEnabledStub); + }); + + test('If the experiment setting is disabled, inExperiment should return false', async () => { + configureSettings(false, [], []); + + const experimentService = new ExperimentService( + instance(configurationService), + instance(appEnvironment), + globalMemento + ); + const result = await experimentService.inExperiment(experiment); + + assert.isFalse(result); + sinon.assert.notCalled(sendTelemetryEventStub); + sinon.assert.notCalled(isCachedFlightEnabledStub); + }); + + test('If the opt-in setting contains "All", inExperiment should return true', async () => { + configureSettings(true, ['All'], []); + + const experimentService = new ExperimentService( + instance(configurationService), + instance(appEnvironment), + globalMemento + ); + const result = await experimentService.inExperiment(experiment); + + assert.isTrue(result); + assert.equal(telemetryEvents.length, 1); + assert.deepEqual(telemetryEvents[0], { + eventName: EventName.PYTHON_EXPERIMENTS_OPT_IN_OUT, + properties: { expNameOptedInto: experiment } + }); + sinon.assert.notCalled(isCachedFlightEnabledStub); + }); + + test('If the opt-in setting contains the experiment name, inExperiment should return true', async () => { + configureSettings(true, [experiment], []); + + const experimentService = new ExperimentService( + instance(configurationService), + instance(appEnvironment), + globalMemento + ); + const result = await experimentService.inExperiment(experiment); + + assert.isTrue(result); + assert.equal(telemetryEvents.length, 1); + assert.deepEqual(telemetryEvents[0], { + eventName: EventName.PYTHON_EXPERIMENTS_OPT_IN_OUT, + properties: { expNameOptedInto: experiment } + }); + sinon.assert.notCalled(isCachedFlightEnabledStub); + }); + + test('If the opt-out setting contains "All", inExperiment should return false', async () => { + configureSettings(true, [], ['All']); + + const experimentService = new ExperimentService( + instance(configurationService), + instance(appEnvironment), + globalMemento + ); + const result = await experimentService.inExperiment(experiment); + + assert.isFalse(result); + assert.equal(telemetryEvents.length, 1); + assert.deepEqual(telemetryEvents[0], { + eventName: EventName.PYTHON_EXPERIMENTS_OPT_IN_OUT, + properties: { expNameOptedOutOf: experiment } + }); + sinon.assert.notCalled(isCachedFlightEnabledStub); + }); + + test('If the opt-out setting contains the experiment name, inExperiment should return false', async () => { + configureSettings(true, [], [experiment]); + + const experimentService = new ExperimentService( + instance(configurationService), + instance(appEnvironment), + globalMemento + ); + const result = await experimentService.inExperiment(experiment); + + assert.isFalse(result); + assert.equal(telemetryEvents.length, 1); + assert.deepEqual(telemetryEvents[0], { + eventName: EventName.PYTHON_EXPERIMENTS_OPT_IN_OUT, + properties: { expNameOptedOutOf: experiment } + }); + sinon.assert.notCalled(isCachedFlightEnabledStub); + }); + }); +}); diff --git a/src/test/common/experiments/telemetry.unit.test.ts b/src/test/common/experiments/telemetry.unit.test.ts new file mode 100644 index 000000000000..d49ba5599c71 --- /dev/null +++ b/src/test/common/experiments/telemetry.unit.test.ts @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { assert } from 'chai'; +import * as sinon from 'sinon'; +import { ExperimentationTelemetry } from '../../../client/common/experiments/telemetry'; +import * as Telemetry from '../../../client/telemetry'; + +suite('Experimentation telemetry', () => { + const event = 'SomeEventName'; + + let telemetryEvents: { eventName: string; properties: object }[] = []; + let sendTelemetryEventStub: sinon.SinonStub; + let setSharedPropertyStub: sinon.SinonStub; + let experimentTelemetry: ExperimentationTelemetry; + let eventProperties: Map; + + setup(() => { + sendTelemetryEventStub = sinon + .stub(Telemetry, 'sendTelemetryEvent') + .callsFake((eventName: string, _, properties: object) => { + const telemetry = { eventName, properties }; + telemetryEvents.push(telemetry); + }); + setSharedPropertyStub = sinon.stub(Telemetry, 'setSharedProperty'); + + eventProperties = new Map(); + eventProperties.set('foo', 'one'); + eventProperties.set('bar', 'two'); + + experimentTelemetry = new ExperimentationTelemetry(); + }); + + teardown(() => { + telemetryEvents = []; + sinon.restore(); + }); + + test('Calling postEvent should send a telemetry event', () => { + experimentTelemetry.postEvent(event, eventProperties); + + sinon.assert.calledOnce(sendTelemetryEventStub); + assert.equal(telemetryEvents.length, 1); + assert.deepEqual(telemetryEvents[0], { + eventName: event, + properties: { + foo: 'one', + bar: 'two' + } + }); + }); + + test('Shared properties should be set for all telemetry events', () => { + const shared = { key: 'shared', value: 'three' }; + + experimentTelemetry.setSharedProperty(shared.key, shared.value); + + sinon.assert.calledOnce(setSharedPropertyStub); + }); +}); diff --git a/src/test/common/installer.test.ts b/src/test/common/installer.test.ts index f3b6118ee4c1..4fe19d92e2c9 100644 --- a/src/test/common/installer.test.ts +++ b/src/test/common/installer.test.ts @@ -29,7 +29,8 @@ import { AsyncDisposableRegistry } from '../../client/common/asyncDisposableRegi import { ConfigurationService } from '../../client/common/configuration/service'; import { CryptoUtils } from '../../client/common/crypto'; import { EditorUtils } from '../../client/common/editor'; -import { ExperimentsManager } from '../../client/common/experiments'; +import { ExperimentsManager } from '../../client/common/experiments/manager'; +import { ExperimentService } from '../../client/common/experiments/service'; import { FeatureDeprecationManager } from '../../client/common/featureDeprecationManager'; import { ExtensionInsidersDailyChannelRule, @@ -102,6 +103,7 @@ import { ICryptoUtils, ICurrentProcess, IEditorUtils, + IExperimentService, IExperimentsManager, IExtensions, IFeatureDeprecationManager, @@ -253,6 +255,7 @@ suite('Installer', () => { ioc.serviceManager.addSingleton(ILiveShareApi, LiveShareApi); ioc.serviceManager.addSingleton(ICryptoUtils, CryptoUtils); ioc.serviceManager.addSingleton(IExperimentsManager, ExperimentsManager); + ioc.serviceManager.addSingleton(IExperimentService, ExperimentService); ioc.serviceManager.addSingleton(ITerminalHelper, TerminalHelper); ioc.serviceManager.addSingleton( diff --git a/src/test/common/moduleInstaller.test.ts b/src/test/common/moduleInstaller.test.ts index 92109eeca0e1..cf08e46e0074 100644 --- a/src/test/common/moduleInstaller.test.ts +++ b/src/test/common/moduleInstaller.test.ts @@ -36,7 +36,8 @@ import { AsyncDisposableRegistry } from '../../client/common/asyncDisposableRegi import { ConfigurationService } from '../../client/common/configuration/service'; import { CryptoUtils } from '../../client/common/crypto'; import { EditorUtils } from '../../client/common/editor'; -import { ExperimentsManager } from '../../client/common/experiments'; +import { ExperimentsManager } from '../../client/common/experiments/manager'; +import { ExperimentService } from '../../client/common/experiments/service'; import '../../client/common/extensions'; import { FeatureDeprecationManager } from '../../client/common/featureDeprecationManager'; import { @@ -101,6 +102,7 @@ import { ICryptoUtils, ICurrentProcess, IEditorUtils, + IExperimentService, IExperimentsManager, IExtensions, IFeatureDeprecationManager, @@ -266,6 +268,7 @@ suite('Module Installer', () => { ioc.serviceManager.addSingleton(ILiveShareApi, LiveShareApi); ioc.serviceManager.addSingleton(ICryptoUtils, CryptoUtils); ioc.serviceManager.addSingleton(IExperimentsManager, ExperimentsManager); + ioc.serviceManager.addSingleton(IExperimentService, ExperimentService); ioc.serviceManager.addSingleton( ITerminalActivationCommandProvider, diff --git a/src/test/common/serviceRegistry.unit.test.ts b/src/test/common/serviceRegistry.unit.test.ts index 62bab1a4a092..d40edb5e5378 100644 --- a/src/test/common/serviceRegistry.unit.test.ts +++ b/src/test/common/serviceRegistry.unit.test.ts @@ -34,7 +34,7 @@ import { AsyncDisposableRegistry } from '../../client/common/asyncDisposableRegi import { ConfigurationService } from '../../client/common/configuration/service'; import { CryptoUtils } from '../../client/common/crypto'; import { EditorUtils } from '../../client/common/editor'; -import { ExperimentsManager } from '../../client/common/experiments'; +import { ExperimentsManager } from '../../client/common/experiments/manager'; import { FeatureDeprecationManager } from '../../client/common/featureDeprecationManager'; import { ExtensionInsidersDailyChannelRule, diff --git a/src/test/common/terminals/environmentActivationProviders/terminalActivation.testvirtualenvs.ts b/src/test/common/terminals/environmentActivationProviders/terminalActivation.testvirtualenvs.ts index 180b790eb774..63d6958d9883 100644 --- a/src/test/common/terminals/environmentActivationProviders/terminalActivation.testvirtualenvs.ts +++ b/src/test/common/terminals/environmentActivationProviders/terminalActivation.testvirtualenvs.ts @@ -7,7 +7,7 @@ import { expect } from 'chai'; import * as fs from 'fs-extra'; import * as path from 'path'; import * as vscode from 'vscode'; -import { DeprecatePythonPath } from '../../../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../../../client/common/experiments/groups'; import { FileSystem } from '../../../../client/common/platform/fileSystem'; import { IExperimentsManager } from '../../../../client/common/types'; import { PYTHON_VIRTUAL_ENVS_LOCATION } from '../../../ciConstants'; diff --git a/src/test/configuration/interpreterSelector/interpreterSelector.unit.test.ts b/src/test/configuration/interpreterSelector/interpreterSelector.unit.test.ts index 0410a29dac78..5842768e3547 100644 --- a/src/test/configuration/interpreterSelector/interpreterSelector.unit.test.ts +++ b/src/test/configuration/interpreterSelector/interpreterSelector.unit.test.ts @@ -5,7 +5,7 @@ import * as assert from 'assert'; import { SemVer } from 'semver'; import * as TypeMoq from 'typemoq'; import { Uri } from 'vscode'; -import { DeprecatePythonPath } from '../../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../../client/common/experiments/groups'; import { PathUtils } from '../../../client/common/platform/pathUtils'; import { IFileSystem } from '../../../client/common/platform/types'; import { IExperimentsManager } from '../../../client/common/types'; diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 5ba7cc96afc7..47f813fe17d5 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -94,8 +94,8 @@ import { EXTENSION_ROOT_DIR, UseCustomEditorApi } from '../../client/common/cons import { CryptoUtils } from '../../client/common/crypto'; import { DotNetCompatibilityService } from '../../client/common/dotnet/compatibilityService'; import { IDotNetCompatibilityService } from '../../client/common/dotnet/types'; -import { LocalZMQKernel } from '../../client/common/experimentGroups'; -import { ExperimentsManager } from '../../client/common/experiments'; +import { LocalZMQKernel } from '../../client/common/experiments/groups'; +import { ExperimentsManager } from '../../client/common/experiments/manager'; import { InstallationChannelManager } from '../../client/common/installer/channelManager'; import { ProductInstaller } from '../../client/common/installer/productInstaller'; import { diff --git a/src/test/datascience/debugger.functional.test.tsx b/src/test/datascience/debugger.functional.test.tsx index 39ad034bf522..3cb83fa4ff9d 100644 --- a/src/test/datascience/debugger.functional.test.tsx +++ b/src/test/datascience/debugger.functional.test.tsx @@ -10,7 +10,7 @@ import { CancellationToken } from 'vscode-jsonrpc'; import * as vsls from 'vsls/vscode'; import { IApplicationShell, IDebugService, IDocumentManager } from '../../client/common/application/types'; -import { RunByLine } from '../../client/common/experimentGroups'; +import { RunByLine } from '../../client/common/experiments/groups'; import { IProcessServiceFactory, Output } from '../../client/common/process/types'; import { createDeferred, waitForPromise } from '../../client/common/utils/async'; import { noop } from '../../client/common/utils/misc'; diff --git a/src/test/datascience/notebook.functional.test.ts b/src/test/datascience/notebook.functional.test.ts index 3cc3f5dddb15..4dfa2a407902 100644 --- a/src/test/datascience/notebook.functional.test.ts +++ b/src/test/datascience/notebook.functional.test.ts @@ -19,7 +19,7 @@ import { ApplicationShell } from '../../client/common/application/applicationShe import { IApplicationShell } from '../../client/common/application/types'; import { Cancellation, CancellationError } from '../../client/common/cancellation'; import { EXTENSION_ROOT_DIR } from '../../client/common/constants'; -import { LocalZMQKernel } from '../../client/common/experimentGroups'; +import { LocalZMQKernel } from '../../client/common/experiments/groups'; import { traceError, traceInfo } from '../../client/common/logger'; import { IFileSystem } from '../../client/common/platform/types'; import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../client/common/process/types'; diff --git a/src/test/datascience/uiTests/ipywidget.ui.functional.test.ts b/src/test/datascience/uiTests/ipywidget.ui.functional.test.ts index 0c7bee367f83..f098220d3e03 100644 --- a/src/test/datascience/uiTests/ipywidget.ui.functional.test.ts +++ b/src/test/datascience/uiTests/ipywidget.ui.functional.test.ts @@ -13,7 +13,7 @@ import * as os from 'os'; import * as path from 'path'; import * as sinon from 'sinon'; import { Disposable } from 'vscode'; -import { LocalZMQKernel } from '../../../client/common/experimentGroups'; +import { LocalZMQKernel } from '../../../client/common/experiments/groups'; import { EXTENSION_ROOT_DIR } from '../../../client/constants'; import { retryIfFail as retryIfFailOriginal } from '../../common'; import { mockedVSCodeNamespaces } from '../../vscode-mock'; diff --git a/src/test/datascience/variableexplorer.functional.test.tsx b/src/test/datascience/variableexplorer.functional.test.tsx index 5e911332a6d0..1dd73dc3e382 100644 --- a/src/test/datascience/variableexplorer.functional.test.tsx +++ b/src/test/datascience/variableexplorer.functional.test.tsx @@ -8,7 +8,7 @@ import * as React from 'react'; import * as AdazzleReactDataGrid from 'react-data-grid'; import { Disposable } from 'vscode'; -import { RunByLine } from '../../client/common/experimentGroups'; +import { RunByLine } from '../../client/common/experiments/groups'; import { InteractiveWindowMessages } from '../../client/datascience/interactive-common/interactiveWindowTypes'; import { IJupyterVariable } from '../../client/datascience/types'; import { DataScienceIocContainer } from './dataScienceIocContainer'; diff --git a/src/test/debugger/attach.ptvsd.test.ts b/src/test/debugger/attach.ptvsd.test.ts index 0be2a3801c87..d846ad7c62c2 100644 --- a/src/test/debugger/attach.ptvsd.test.ts +++ b/src/test/debugger/attach.ptvsd.test.ts @@ -13,7 +13,7 @@ import { DebugClient } from 'vscode-debugadapter-testsupport'; import { IDocumentManager, IWorkspaceService } from '../../client/common/application/types'; import { EXTENSION_ROOT_DIR } from '../../client/common/constants'; -import { DebugAdapterNewPtvsd } from '../../client/common/experimentGroups'; +import { DebugAdapterNewPtvsd } from '../../client/common/experiments/groups'; import { IS_WINDOWS } from '../../client/common/platform/constants'; import { IPlatformService } from '../../client/common/platform/types'; import { IConfigurationService, IExperimentsManager } from '../../client/common/types'; diff --git a/src/test/debugger/extension/adapter/factory.unit.test.ts b/src/test/debugger/extension/adapter/factory.unit.test.ts index bd9e3d90d2d7..a96b32910957 100644 --- a/src/test/debugger/extension/adapter/factory.unit.test.ts +++ b/src/test/debugger/extension/adapter/factory.unit.test.ts @@ -17,8 +17,8 @@ import { ApplicationShell } from '../../../../client/common/application/applicat import { IApplicationShell } from '../../../../client/common/application/types'; import { ConfigurationService } from '../../../../client/common/configuration/service'; import { CryptoUtils } from '../../../../client/common/crypto'; -import { DebugAdapterNewPtvsd } from '../../../../client/common/experimentGroups'; -import { ExperimentsManager } from '../../../../client/common/experiments'; +import { DebugAdapterNewPtvsd } from '../../../../client/common/experiments/groups'; +import { ExperimentsManager } from '../../../../client/common/experiments/manager'; import { HttpClient } from '../../../../client/common/net/httpClient'; import { PersistentStateFactory } from '../../../../client/common/persistentState'; import { FileSystem } from '../../../../client/common/platform/fileSystem'; diff --git a/src/test/debugger/extension/adapter/outdatedDebuggerPrompt.unit.test.ts b/src/test/debugger/extension/adapter/outdatedDebuggerPrompt.unit.test.ts index a924be25a2db..7c262b56de20 100644 --- a/src/test/debugger/extension/adapter/outdatedDebuggerPrompt.unit.test.ts +++ b/src/test/debugger/extension/adapter/outdatedDebuggerPrompt.unit.test.ts @@ -12,8 +12,8 @@ import { ApplicationShell } from '../../../../client/common/application/applicat import { IApplicationShell } from '../../../../client/common/application/types'; import { ConfigurationService } from '../../../../client/common/configuration/service'; import { CryptoUtils } from '../../../../client/common/crypto'; -import { DebugAdapterNewPtvsd } from '../../../../client/common/experimentGroups'; -import { ExperimentsManager } from '../../../../client/common/experiments'; +import { DebugAdapterNewPtvsd } from '../../../../client/common/experiments/groups'; +import { ExperimentsManager } from '../../../../client/common/experiments/manager'; import { BrowserService } from '../../../../client/common/net/browser'; import { HttpClient } from '../../../../client/common/net/httpClient'; import { PersistentStateFactory } from '../../../../client/common/persistentState'; diff --git a/src/test/debugger/extension/configuration/resolvers/attach.unit.test.ts b/src/test/debugger/extension/configuration/resolvers/attach.unit.test.ts index a36aaf374a25..2e7b29dee168 100644 --- a/src/test/debugger/extension/configuration/resolvers/attach.unit.test.ts +++ b/src/test/debugger/extension/configuration/resolvers/attach.unit.test.ts @@ -10,7 +10,7 @@ import * as TypeMoq from 'typemoq'; import { DebugConfiguration, DebugConfigurationProvider, TextDocument, TextEditor, Uri, WorkspaceFolder } from 'vscode'; import { IDocumentManager, IWorkspaceService } from '../../../../../client/common/application/types'; import { PYTHON_LANGUAGE } from '../../../../../client/common/constants'; -import { DebugAdapterNewPtvsd } from '../../../../../client/common/experimentGroups'; +import { DebugAdapterNewPtvsd } from '../../../../../client/common/experiments/groups'; import { IFileSystem, IPlatformService } from '../../../../../client/common/platform/types'; import { IConfigurationService, IExperimentsManager } from '../../../../../client/common/types'; import { Diagnostics } from '../../../../../client/common/utils/localize'; diff --git a/src/test/debugger/extension/configuration/resolvers/launchConfigExperiments.unit.test.ts b/src/test/debugger/extension/configuration/resolvers/launchConfigExperiments.unit.test.ts index 339f3bd68ba1..62b034ac8771 100644 --- a/src/test/debugger/extension/configuration/resolvers/launchConfigExperiments.unit.test.ts +++ b/src/test/debugger/extension/configuration/resolvers/launchConfigExperiments.unit.test.ts @@ -10,8 +10,8 @@ import { instance, mock, spy, when } from 'ts-mockito'; import { ApplicationEnvironment } from '../../../../../client/common/application/applicationEnvironment'; import { ConfigurationService } from '../../../../../client/common/configuration/service'; import { CryptoUtils } from '../../../../../client/common/crypto'; -import { DebugAdapterNewPtvsd, WebAppReload } from '../../../../../client/common/experimentGroups'; -import { ExperimentsManager } from '../../../../../client/common/experiments'; +import { DebugAdapterNewPtvsd, WebAppReload } from '../../../../../client/common/experiments/groups'; +import { ExperimentsManager } from '../../../../../client/common/experiments/manager'; import { HttpClient } from '../../../../../client/common/net/httpClient'; import { PersistentStateFactory } from '../../../../../client/common/persistentState'; import { FileSystem } from '../../../../../client/common/platform/fileSystem'; diff --git a/src/test/interpreters/activation/wrapperEnvironmentActivationService.unit.test.ts b/src/test/interpreters/activation/wrapperEnvironmentActivationService.unit.test.ts index d09858a4412b..1d9677dbd4f0 100644 --- a/src/test/interpreters/activation/wrapperEnvironmentActivationService.unit.test.ts +++ b/src/test/interpreters/activation/wrapperEnvironmentActivationService.unit.test.ts @@ -8,7 +8,7 @@ import { EventEmitter, Uri } from 'vscode'; import { IWorkspaceService } from '../../../client/common/application/types'; import { WorkspaceService } from '../../../client/common/application/workspace'; import { CryptoUtils } from '../../../client/common/crypto'; -import { ExperimentsManager } from '../../../client/common/experiments'; +import { ExperimentsManager } from '../../../client/common/experiments/manager'; import { FileSystem } from '../../../client/common/platform/fileSystem'; import { IFileSystem } from '../../../client/common/platform/types'; import { ICryptoUtils, IExperimentsManager, IExtensionContext, Resource } from '../../../client/common/types'; diff --git a/src/test/interpreters/autoSelection/rules/settings.unit.test.ts b/src/test/interpreters/autoSelection/rules/settings.unit.test.ts index 73b9d647c5d7..eb561f754e92 100644 --- a/src/test/interpreters/autoSelection/rules/settings.unit.test.ts +++ b/src/test/interpreters/autoSelection/rules/settings.unit.test.ts @@ -9,8 +9,8 @@ import { expect } from 'chai'; import { anything, instance, mock, when } from 'ts-mockito'; import { IWorkspaceService } from '../../../../client/common/application/types'; import { WorkspaceService } from '../../../../client/common/application/workspace'; -import { DeprecatePythonPath } from '../../../../client/common/experimentGroups'; -import { ExperimentsManager } from '../../../../client/common/experiments'; +import { DeprecatePythonPath } from '../../../../client/common/experiments/groups'; +import { ExperimentsManager } from '../../../../client/common/experiments/manager'; import { InterpreterPathService } from '../../../../client/common/interpreterPathService'; import { PersistentState, PersistentStateFactory } from '../../../../client/common/persistentState'; import { FileSystem } from '../../../../client/common/platform/fileSystem'; diff --git a/src/test/interpreters/autoSelection/rules/workspaceEnv.unit.test.ts b/src/test/interpreters/autoSelection/rules/workspaceEnv.unit.test.ts index 91add8e2b24e..c347853d2dea 100644 --- a/src/test/interpreters/autoSelection/rules/workspaceEnv.unit.test.ts +++ b/src/test/interpreters/autoSelection/rules/workspaceEnv.unit.test.ts @@ -13,8 +13,8 @@ import * as typemoq from 'typemoq'; import { Uri, WorkspaceFolder } from 'vscode'; import { IWorkspaceService } from '../../../../client/common/application/types'; import { WorkspaceService } from '../../../../client/common/application/workspace'; -import { DeprecatePythonPath } from '../../../../client/common/experimentGroups'; -import { ExperimentsManager } from '../../../../client/common/experiments'; +import { DeprecatePythonPath } from '../../../../client/common/experiments/groups'; +import { ExperimentsManager } from '../../../../client/common/experiments/manager'; import { InterpreterPathService } from '../../../../client/common/interpreterPathService'; import { PersistentState, PersistentStateFactory } from '../../../../client/common/persistentState'; import { FileSystem } from '../../../../client/common/platform/fileSystem'; diff --git a/src/test/interpreters/interpreterService.unit.test.ts b/src/test/interpreters/interpreterService.unit.test.ts index 48b224666585..2b870efbdccc 100644 --- a/src/test/interpreters/interpreterService.unit.test.ts +++ b/src/test/interpreters/interpreterService.unit.test.ts @@ -14,7 +14,7 @@ import { SemVer } from 'semver'; import * as TypeMoq from 'typemoq'; import { ConfigurationTarget, Disposable, TextDocument, TextEditor, Uri, WorkspaceConfiguration } from 'vscode'; import { IDocumentManager, IWorkspaceService } from '../../client/common/application/types'; -import { DeprecatePythonPath } from '../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../client/common/experiments/groups'; import { getArchitectureDisplayName } from '../../client/common/platform/registry'; import { IFileSystem } from '../../client/common/platform/types'; import { IPythonExecutionFactory, IPythonExecutionService } from '../../client/common/process/types'; diff --git a/src/test/interpreters/pythonPathUpdaterFactory.unit.test.ts b/src/test/interpreters/pythonPathUpdaterFactory.unit.test.ts index e59e379cbc34..5ba72dc2c1bf 100644 --- a/src/test/interpreters/pythonPathUpdaterFactory.unit.test.ts +++ b/src/test/interpreters/pythonPathUpdaterFactory.unit.test.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as TypeMoq from 'typemoq'; import { ConfigurationTarget, Uri, WorkspaceConfiguration } from 'vscode'; import { IWorkspaceService } from '../../client/common/application/types'; -import { DeprecatePythonPath } from '../../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../../client/common/experiments/groups'; import { IExperimentsManager, IInterpreterPathService } from '../../client/common/types'; import { PythonPathUpdaterServiceFactory } from '../../client/interpreter/configuration/pythonPathUpdaterServiceFactory'; import { IPythonPathUpdaterServiceFactory } from '../../client/interpreter/configuration/types'; diff --git a/src/test/startupTelemetry.unit.test.ts b/src/test/startupTelemetry.unit.test.ts index 828a85a000ff..0099f4d4ff5e 100644 --- a/src/test/startupTelemetry.unit.test.ts +++ b/src/test/startupTelemetry.unit.test.ts @@ -7,7 +7,7 @@ import { expect } from 'chai'; import * as TypeMoq from 'typemoq'; import { Uri, WorkspaceConfiguration } from 'vscode'; import { IWorkspaceService } from '../client/common/application/types'; -import { DeprecatePythonPath } from '../client/common/experimentGroups'; +import { DeprecatePythonPath } from '../client/common/experiments/groups'; import { IExperimentsManager, IInterpreterPathService } from '../client/common/types'; import { IServiceContainer } from '../client/ioc/types'; import { hasUserDefinedPythonPath } from '../client/startupTelemetry'; diff --git a/src/test/telemetry/index.unit.test.ts b/src/test/telemetry/index.unit.test.ts index aad20386af82..3034f75f2446 100644 --- a/src/test/telemetry/index.unit.test.ts +++ b/src/test/telemetry/index.unit.test.ts @@ -12,7 +12,13 @@ import { WorkspaceConfiguration } from 'vscode'; import { IWorkspaceService } from '../../client/common/application/types'; import { WorkspaceService } from '../../client/common/application/workspace'; import { EXTENSION_ROOT_DIR } from '../../client/constants'; -import { clearTelemetryReporter, isTelemetryDisabled, sendTelemetryEvent } from '../../client/telemetry'; +import { + _resetSharedProperties, + clearTelemetryReporter, + isTelemetryDisabled, + sendTelemetryEvent, + setSharedProperty +} from '../../client/telemetry'; suite('Telemetry', () => { let workspaceService: IWorkspaceService; @@ -23,16 +29,22 @@ suite('Telemetry', () => { public static eventName: string[] = []; public static properties: Record[] = []; public static measures: {}[] = []; + public static errorProps: string[] | undefined; public static clear() { Reporter.eventName = []; Reporter.properties = []; Reporter.measures = []; + Reporter.errorProps = undefined; } public sendTelemetryEvent(eventName: string, properties?: {}, measures?: {}) { Reporter.eventName.push(eventName); Reporter.properties.push(properties!); Reporter.measures.push(measures!); } + public sendTelemetryErrorEvent(eventName: string, properties?: {}, measures?: {}, errorProps?: string[]) { + this.sendTelemetryEvent(eventName, properties, measures); + Reporter.errorProps = errorProps; + } } setup(() => { @@ -46,6 +58,7 @@ suite('Telemetry', () => { process.env.VSC_PYTHON_UNIT_TEST = oldValueOfVSC_PYTHON_UNIT_TEST; process.env.VSC_PYTHON_CI_TEST = oldValueOfVSC_PYTHON_CI_TEST; rewiremock.disable(); + _resetSharedProperties(); }); const testsForisTelemetryDisabled = [ @@ -94,7 +107,7 @@ suite('Telemetry', () => { expect(Reporter.measures).to.deep.equal([measures]); expect(Reporter.properties).to.deep.equal([properties]); }); - test('Send Telemetry', () => { + test('Send Telemetry with no properties', () => { rewiremock.enable(); rewiremock('vscode-extension-telemetry').with({ default: Reporter }); @@ -106,104 +119,45 @@ suite('Telemetry', () => { expect(Reporter.measures).to.deep.equal([undefined], 'Measures should be empty'); expect(Reporter.properties).to.deep.equal([{}], 'Properties should be empty'); }); - test('Send Error Telemetry', () => { + test('Send Telemetry with shared properties', () => { rewiremock.enable(); - const error = new Error('Boo'); rewiremock('vscode-extension-telemetry').with({ default: Reporter }); const eventName = 'Testing'; const properties = { hello: 'world', foo: 'bar' }; const measures = { start: 123, end: 987 }; + const expectedProperties = { ...properties, one: 'two' }; - // tslint:disable-next-line:no-any - sendTelemetryEvent(eventName as any, measures, properties as any, error); + setSharedProperty('one', 'two'); - const expectedErrorProperties = { - originalEventName: eventName - }; + // tslint:disable-next-line:no-any + sendTelemetryEvent(eventName as any, measures, properties as any); - expect(Reporter.eventName).to.deep.equal(['ERROR', eventName]); - expect(Reporter.measures).to.deep.equal([measures, measures]); - expect(Reporter.properties[0].stackTrace).to.be.length.greaterThan(1); - delete Reporter.properties[0].stackTrace; - expect(Reporter.properties).to.deep.equal([expectedErrorProperties, properties]); + expect(Reporter.eventName).to.deep.equal([eventName]); + expect(Reporter.measures).to.deep.equal([measures]); + expect(Reporter.properties).to.deep.equal([expectedProperties]); }); - test('Send Error Telemetry', () => { + test('Shared properties will replace existing ones', () => { rewiremock.enable(); - const error = new Error('Boo'); - error.stack = [ - 'Error: Boo', - `at Context.test (${EXTENSION_ROOT_DIR}/src/test/telemetry/index.unit.test.ts:50:23)`, - `at callFn (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runnable.js:372:21)`, - `at Test.Runnable.run (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runnable.js:364:7)`, - `at Runner.runTest (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:455:10)`, - `at ${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:573:12`, - `at next (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:369:14)`, - `at ${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:379:7`, - `at next (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:303:14)`, - `at ${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:342:7`, - `at done (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runnable.js:319:5)`, - `at callFn (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runnable.js:395:7)`, - `at Hook.Runnable.run (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runnable.js:364:7)`, - `at next (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:317:10)`, - `at Immediate. (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:347:5)`, - 'at runCallback (timers.js:789:20)', - 'at tryOnImmediate (timers.js:751:5)', - 'at processImmediate [as _immediateCallback] (timers.js:722:5)' - ].join('\n\t'); rewiremock('vscode-extension-telemetry').with({ default: Reporter }); const eventName = 'Testing'; const properties = { hello: 'world', foo: 'bar' }; const measures = { start: 123, end: 987 }; + const expectedProperties = { ...properties, foo: 'baz' }; - // tslint:disable-next-line:no-any - sendTelemetryEvent(eventName as any, measures, properties as any, error); + setSharedProperty('foo', 'baz'); - const expectedErrorProperties = { - originalEventName: eventName - }; - - const stackTrace = Reporter.properties[0].stackTrace; - delete Reporter.properties[0].stackTrace; - - expect(Reporter.eventName).to.deep.equal(['ERROR', eventName]); - expect(Reporter.measures).to.deep.equal([measures, measures]); - expect(Reporter.properties).to.deep.equal([expectedErrorProperties, properties]); - expect(stackTrace).to.be.length.greaterThan(1); - - const expectedStack = [ - 'at Context.test /src/test/telemetry/index.unit.test.ts:50:23\n\tat callFn /node_modules/mocha/lib/runnable.js:372:21', - 'at Test.Runnable.run /node_modules/mocha/lib/runnable.js:364:7', - 'at Runner.runTest /node_modules/mocha/lib/runner.js:455:10', - 'at /node_modules/mocha/lib/runner.js:573:12', - 'at next /node_modules/mocha/lib/runner.js:369:14', - 'at /node_modules/mocha/lib/runner.js:379:7', - 'at next /node_modules/mocha/lib/runner.js:303:14', - 'at /node_modules/mocha/lib/runner.js:342:7', - 'at done /node_modules/mocha/lib/runnable.js:319:5', - 'at callFn /node_modules/mocha/lib/runnable.js:395:7', - 'at Hook.Runnable.run /node_modules/mocha/lib/runnable.js:364:7', - 'at next /node_modules/mocha/lib/runner.js:317:10', - 'at Immediate /node_modules/mocha/lib/runner.js:347:5', - 'at runCallback /timers.js:789:20', - 'at tryOnImmediate /timers.js:751:5', - 'at processImmediate [as _immediateCallback] /timers.js:722:5' - ].join('\n\t'); + // tslint:disable-next-line:no-any + sendTelemetryEvent(eventName as any, measures, properties as any); - expect(stackTrace).to.be.equal(expectedStack); + expect(Reporter.eventName).to.deep.equal([eventName]); + expect(Reporter.measures).to.deep.equal([measures]); + expect(Reporter.properties).to.deep.equal([expectedProperties]); }); - test('Ensure non extension file paths are stripped from stack trace', () => { + test('Send Error Telemetry', () => { rewiremock.enable(); const error = new Error('Boo'); - error.stack = [ - 'Error: Boo', - `at Context.test (${EXTENSION_ROOT_DIR}/src/test/telemetry/index.unit.test.ts:50:23)`, - 'at callFn (c:/one/two/user/node_modules/mocha/lib/runnable.js:372:21)', - 'at Test.Runnable.run (/usr/Paul/Homer/desktop/node_modules/mocha/lib/runnable.js:364:7)', - 'at Runner.runTest (\\wowwee/node_modules/mocha/lib/runner.js:455:10)', - `at Immediate. (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:347:5)` - ].join('\n\t'); rewiremock('vscode-extension-telemetry').with({ default: Reporter }); const eventName = 'Testing'; @@ -217,34 +171,36 @@ suite('Telemetry', () => { originalEventName: eventName }; - const stackTrace = Reporter.properties[0].stackTrace; + expect(Reporter.eventName).to.deep.equal(['ERROR']); + expect(Reporter.measures).to.deep.equal([measures]); + expect(Reporter.properties[0].stackTrace).to.be.length.greaterThan(1); delete Reporter.properties[0].stackTrace; - - expect(Reporter.eventName).to.deep.equal(['ERROR', eventName]); - expect(Reporter.measures).to.deep.equal([measures, measures]); - expect(Reporter.properties).to.deep.equal([expectedErrorProperties, properties]); - expect(stackTrace).to.be.length.greaterThan(1); - - const expectedStack = [ - 'at Context.test /src/test/telemetry/index.unit.test.ts:50:23', - 'at callFn /runnable.js:372:21', - 'at Test.Runnable.run /runnable.js:364:7', - 'at Runner.runTest /runner.js:455:10', - 'at Immediate /node_modules/mocha/lib/runner.js:347:5' - ].join('\n\t'); - - expect(stackTrace).to.be.equal(expectedStack); + expect(Reporter.properties).to.deep.equal([expectedErrorProperties]); + expect(Reporter.errorProps).to.deep.equal([]); }); - test('Ensure non function names containing file names (unlikely, but for sake of completeness) are stripped from stack trace', () => { + test('Send Error Telemetry with stack trace', () => { rewiremock.enable(); const error = new Error('Boo'); + const root = EXTENSION_ROOT_DIR.replace(/\\/g, '/'); error.stack = [ 'Error: Boo', - `at Context.test (${EXTENSION_ROOT_DIR}/src/test/telemetry/index.unit.test.ts:50:23)`, - 'at callFn (c:/one/two/user/node_modules/mocha/lib/runnable.js:372:21)', - 'at Test./usr/Paul/Homer/desktop/node_modules/mocha/lib/runnable.run (/usr/Paul/Homer/desktop/node_modules/mocha/lib/runnable.js:364:7)', - 'at Runner.runTest (\\wowwee/node_modules/mocha/lib/runner.js:455:10)', - `at Immediate. (${EXTENSION_ROOT_DIR}/node_modules/mocha/lib/runner.js:347:5)` + `at Context.test (${root}/src/test/telemetry/index.unit.test.ts:50:23)`, + `at callFn (${root}/node_modules/mocha/lib/runnable.js:372:21)`, + `at Test.Runnable.run (${root}/node_modules/mocha/lib/runnable.js:364:7)`, + `at Runner.runTest (${root}/node_modules/mocha/lib/runner.js:455:10)`, + `at ${root}/node_modules/mocha/lib/runner.js:573:12`, + `at next (${root}/node_modules/mocha/lib/runner.js:369:14)`, + `at ${root}/node_modules/mocha/lib/runner.js:379:7`, + `at next (${root}/node_modules/mocha/lib/runner.js:303:14)`, + `at ${root}/node_modules/mocha/lib/runner.js:342:7`, + `at done (${root}/node_modules/mocha/lib/runnable.js:319:5)`, + `at callFn (${root}/node_modules/mocha/lib/runnable.js:395:7)`, + `at Hook.Runnable.run (${root}/node_modules/mocha/lib/runnable.js:364:7)`, + `at next (${root}/node_modules/mocha/lib/runner.js:317:10)`, + `at Immediate. (${root}/node_modules/mocha/lib/runner.js:347:5)`, + 'at runCallback (timers.js:789:20)', + 'at tryOnImmediate (timers.js:751:5)', + 'at processImmediate [as _immediateCallback] (timers.js:722:5)' ].join('\n\t'); rewiremock('vscode-extension-telemetry').with({ default: Reporter }); @@ -262,17 +218,30 @@ suite('Telemetry', () => { const stackTrace = Reporter.properties[0].stackTrace; delete Reporter.properties[0].stackTrace; - expect(Reporter.eventName).to.deep.equal(['ERROR', eventName]); - expect(Reporter.measures).to.deep.equal([measures, measures]); - expect(Reporter.properties).to.deep.equal([expectedErrorProperties, properties]); + expect(Reporter.eventName).to.deep.equal(['ERROR']); + expect(Reporter.measures).to.deep.equal([measures]); + expect(Reporter.properties).to.deep.equal([expectedErrorProperties]); expect(stackTrace).to.be.length.greaterThan(1); + expect(Reporter.errorProps).to.deep.equal([]); const expectedStack = [ - 'at Context.test /src/test/telemetry/index.unit.test.ts:50:23', - 'at callFn /runnable.js:372:21', - 'at .run /runnable.js:364:7', - 'at Runner.runTest /runner.js:455:10', - 'at Immediate /node_modules/mocha/lib/runner.js:347:5' + `at Context.test ${root}/src/test/telemetry/index.unit.test.ts:50:23`, + `at callFn ${root}/node_modules/mocha/lib/runnable.js:372:21`, + `at Test.Runnable.run ${root}/node_modules/mocha/lib/runnable.js:364:7`, + `at Runner.runTest ${root}/node_modules/mocha/lib/runner.js:455:10`, + `at ${root}/node_modules/mocha/lib/runner.js:573:12`, + `at next ${root}/node_modules/mocha/lib/runner.js:369:14`, + `at ${root}/node_modules/mocha/lib/runner.js:379:7`, + `at next ${root}/node_modules/mocha/lib/runner.js:303:14`, + `at ${root}/node_modules/mocha/lib/runner.js:342:7`, + `at done ${root}/node_modules/mocha/lib/runnable.js:319:5`, + `at callFn ${root}/node_modules/mocha/lib/runnable.js:395:7`, + `at Hook.Runnable.run ${root}/node_modules/mocha/lib/runnable.js:364:7`, + `at next ${root}/node_modules/mocha/lib/runner.js:317:10`, + `at Immediate ${root}/node_modules/mocha/lib/runner.js:347:5`, + 'at runCallback timers.js:789:20', + 'at tryOnImmediate timers.js:751:5', + 'at processImmediate [as _immediateCallback] timers.js:722:5' ].join('\n\t'); expect(stackTrace).to.be.equal(expectedStack); diff --git a/src/test/testing/helper.ts b/src/test/testing/helper.ts index d973f28d07d6..3f9ffbf44c89 100644 --- a/src/test/testing/helper.ts +++ b/src/test/testing/helper.ts @@ -32,7 +32,7 @@ export function lookForTestFile(tests: Tests, testFile: string) { // Only "/" (forward slash) in the given filename is affected. // // This helps with readability in test code. It allows us to use -// literals for filenames and dirnames instead of oath.join(). +// literals for filenames and dirnames instead of path.join(). export function fixPath(filename: string): string { return filename.replace(/\//, sep); } diff --git a/src/test/testing/main.unit.test.ts b/src/test/testing/main.unit.test.ts index 7f38e6ad6b72..147fbee07733 100644 --- a/src/test/testing/main.unit.test.ts +++ b/src/test/testing/main.unit.test.ts @@ -8,8 +8,8 @@ import { anything, instance, mock, verify, when } from 'ts-mockito'; import { Disposable } from 'vscode'; import { CommandManager } from '../../client/common/application/commandManager'; import { ICommandManager } from '../../client/common/application/types'; -import { AlwaysDisplayTestExplorerGroups } from '../../client/common/experimentGroups'; -import { ExperimentsManager } from '../../client/common/experiments'; +import { AlwaysDisplayTestExplorerGroups } from '../../client/common/experiments/groups'; +import { ExperimentsManager } from '../../client/common/experiments/manager'; import { IDisposableRegistry, IExperimentsManager } from '../../client/common/types'; import { ServiceContainer } from '../../client/ioc/container'; import { IServiceContainer } from '../../client/ioc/types';