Schedule performance tests

.github/workflows/build-test.yml

@@ -281,16 +281,23 @@ jobs:
             '@iw',
             '@webview|@export|@lsp|@variableViewer',
             '@debugger',
-            # '@notebookPerformance', # Disabled for now, separate PR this will be enabled via cron (want to make PR smaller)
+            '@notebookPerformance', # Disabled for now, separate PR this will be enabled via cron (want to make PR smaller)
             '@executionPerformance'
           ]
         # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
         # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case.
         os: [ubuntu-latest]
         ipywidgetsVersion: ['']
+        isScheduled:
+          - ${{ github.event_name == 'schedule' && 'true' || '' }} # We need an empty value to preserve the matrix name (as we enforce mandatory tests based on names).
         # More details on includes/excludes can be found here https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-including-additional-values-into-combinations
         # Basically with exclude, you can exclude any of the combinations from the result matrix.
         # & with include, you can include additional items to the result matrix.
+        exclude:
+          - tags: '@notebookPerformance' # Run only as part of scheduled CI runs
+            isScheduled: ''
+          - tags: '@executionPerformance' # Run only as part of scheduled CI runs
+            isScheduled: ''
         include:
           # Mandatory tests
           - jupyterConnection: remote

src/test/performance/executionPerf.vscode.test.ts

@@ -14,14 +14,14 @@ import {
 import { JupyterNotebookView, Telemetry } from '../../platform/common/constants';
 import { StopWatch } from '../../platform/common/utils/stopWatch';
 import { sendTelemetryEvent } from '../../telemetry';
-import { JVSC_EXTENSION_ID_FOR_TESTS } from '../constants.node';
+import { IS_PERF_TEST, JVSC_EXTENSION_ID_FOR_TESTS } from '../constants.node';
 import { PVSC_EXTENSION_ID, PythonExtension, type Environment } from '@vscode/python-extension';
 import { PYTHON_PATH, sleep } from '../common.node';
 import type { IExtensionApi } from '../../standalone/api';
 import { DisposableStore } from '../../platform/common/utils/lifecycle';
 import { countCells, startKernelUsingApiByRunningFirstAvailableCodeCell } from '../utils/notebook';
 
-suite('Extension Performance (@executionPerformance)', function () {
+(IS_PERF_TEST() ? suite : suite.skip)('Extension Performance (@executionPerformance)', function () {
     let notebook: NotebookDocument;
     let activeEnv: Environment;
     this.timeout(120_000);

src/test/performance/notebookPerf.vscode.common.test.ts

@@ -16,12 +16,12 @@ import {
 } from 'vscode';
 import { JupyterNotebookView, Telemetry } from '../../platform/common/constants';
 import type { API } from '../vscode-notebook-perf/src/api';
-import { PerformanceExtensionId } from '../constants';
+import { IS_PERF_TEST, PerformanceExtensionId } from '../constants';
 import { StopWatch } from '../../platform/common/utils/stopWatch';
 import { sendTelemetryEvent } from '../../telemetry';
 import { countCells } from '../utils/notebook';
 
-suite('Notebook Performance (@notebookPerformance)', function () {
+(IS_PERF_TEST() ? suite : suite.skip)('Notebook Performance (@notebookPerformance)', function () {
     let api: API;
     let notebook: NotebookDocument;
     this.timeout(120_000);