Merge master into ds/custom_editor

build/ci/templates/globals.yml

@@ -8,4 +8,4 @@ variables:
   VSC_PYTHON_LOG_FILE: '$(Build.ArtifactStagingDirectory)/pvsc.log'
   CI_BRANCH_NAME: ${Build.SourceBranchName}
   npm_config_cache: $(Pipeline.Workspace)/.npm
-
+  vmImageMacOS: 'macOS-10.15'

build/ci/vscode-python-ci-manual.yaml

@@ -168,7 +168,7 @@ stages:
           NeedsIPythonReqs: true
       #maxParallel: 3
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 
@@ -205,7 +205,7 @@ stages:
           NeedsPythonTestReqs: true
       #maxParallel: 3
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 

build/ci/vscode-python-ci.yaml

@@ -140,7 +140,7 @@ stages:
           NeedsIPythonReqs: true
       maxParallel: 2
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 

build/ci/vscode-python-nightly-ci.yaml

@@ -224,7 +224,7 @@ stages:
           NeedsIPythonReqs: true
       maxParallel: 1
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 
@@ -264,7 +264,7 @@ stages:
         # Note: We only run the smoke tests with the latest Python release.
       maxParallel: 1
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 
@@ -304,7 +304,7 @@ stages:
         # Note: We only run the smoke tests with the latest Python release.
       maxParallel: 1
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 
@@ -340,7 +340,7 @@ stages:
         # Note: We only run the smoke tests with the latest Python release.
       maxParallel: 1
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 

build/ci/vscode-python-nightly-flake-ci.yaml

@@ -81,7 +81,7 @@ stages:
           NeedsPythonFunctionalReqs: true
           VSCODE_PYTHON_ROLLING: true
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 
@@ -97,7 +97,7 @@ stages:
           NeedsPythonFunctionalReqs: true
           VSCODE_PYTHON_ROLLING: true
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 
@@ -134,4 +134,4 @@ stages:
     pool:
       vmImage: 'vs2017-win2016'
     steps:
-      - template: templates/test_phases.yml
+      - template: templates/test_phases.yml

build/ci/vscode-python-pr-validation.yaml

@@ -86,7 +86,7 @@ stages:
           NeedsPythonTestReqs: true
           NeedsPythonFunctionalReqs: true
     pool:
-      vmImage: 'macos-10.13'
+      vmImage: '$(vmImageMacOS)'
     steps:
       - template: templates/test_phases.yml
 

news/2 Fixes/10137.md

@@ -0,0 +1 @@
+Fix scrolling for output to consistently scroll even during execution.

news/2 Fixes/10204.md

@@ -0,0 +1 @@
+Fix empty variables to show an empty string in the Notebook/Interactive Window variable explorer.

news/2 Fixes/9935.md

@@ -0,0 +1 @@
+Switching kernels should disable the run/interrupt/restart buttons.

news/2 Fixes/9959.md

@@ -0,0 +1 @@
+Support opening spark dataframes in the data viewer.

news/3 Code Health/10288.md

@@ -0,0 +1 @@
+Use new MacOS VM in Pipelines.

pythonFiles/datascience/getJupyterVariableDataFrameInfo.py

@@ -10,6 +10,22 @@
 # Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable
 _VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""")
 
+# Function to compute row count for a value
+def _VSCODE_getRowCount(var):
+    if hasattr(var, "shape"):
+        try:
+            # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it
+            if isinstance(var.shape, tuple):
+                return var.shape[0]
+        except TypeError:
+            return 0
+    elif hasattr(var, "__len__"):
+        try:
+            return len(var)
+        except TypeError:
+            return 0
+
+
 # First check to see if we are a supported type, this prevents us from adding types that are not supported
 # and also keeps our types in sync with what the variable explorer says that we support
 if _VSCODE_targetVariable["type"] not in _VSCode_supportsDataExplorer:
@@ -21,18 +37,7 @@
     _VSCODE_evalResult = eval(_VSCODE_targetVariable["name"])
 
     # Figure out shape if not already there. Use the shape to compute the row count
-    if hasattr(_VSCODE_evalResult, "shape"):
-        try:
-            # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it
-            if isinstance(_VSCODE_evalResult.shape, tuple):
-                _VSCODE_targetVariable["rowCount"] = _VSCODE_evalResult.shape[0]
-        except TypeError:
-            _VSCODE_targetVariable["rowCount"] = 0
-    elif hasattr(_VSCODE_evalResult, "__len__"):
-        try:
-            _VSCODE_targetVariable["rowCount"] = len(_VSCODE_evalResult)
-        except TypeError:
-            _VSCODE_targetVariable["rowCount"] = 0
+    _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_evalResult)
 
     # Turn the eval result into a df
     _VSCODE_df = _VSCODE_evalResult
@@ -45,6 +50,9 @@
         _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult)
     elif _VSCODE_targetVariable["type"] == "ndarray":
         _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult)
+    elif hasattr(_VSCODE_df, "toPandas"):
+        _VSCODE_df = _VSCODE_df.toPandas()
+        _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_df)
 
     # If any rows, use pandas json to convert a single row to json. Extract
     # the column names and types from the json so we match what we'll fetch when

pythonFiles/datascience/getJupyterVariableDataFrameRows.py

@@ -24,6 +24,8 @@
     _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult)
 elif _VSCODE_targetVariable["type"] == "ndarray":
     _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult)
+elif hasattr(_VSCODE_df, "toPandas"):
+    _VSCODE_df = _VSCODE_df.toPandas()
 # If not a known type, then just let pandas handle it.
 elif not (hasattr(_VSCODE_df, "iloc")):
     _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult)

src/client/datascience/interactive-common/interactiveBase.ts

@@ -1418,7 +1418,12 @@ export abstract class InteractiveBase extends WebViewHost<IInteractiveWindowMapp
         if (!this._notebook) {
             return;
         }
-        await this.commandManager.executeCommand(Commands.SwitchJupyterKernel, this._notebook);
+        try {
+            this.startProgress();
+            await this.commandManager.executeCommand(Commands.SwitchJupyterKernel, this._notebook);
+        } finally {
+            this.stopProgress();
+        }
     }
     private async kernelChangeHandler(kernel: IJupyterKernelSpec | LiveKernelModel) {
         // Check if we are changing to LiveKernelModel

src/client/datascience/jupyter/jupyterVariables.ts

@@ -32,7 +32,7 @@ import { JupyterDataRateLimitError } from './jupyterDataRateLimitError';
 // kernels will add the ansi encoding.
 const TypeRegex = /.*?\[.*?;31mType:.*?\[0m\s+(\w+)/;
 const ValueRegex = /.*?\[.*?;31mValue:.*?\[0m\s+(.*)/;
-const StringFormRegex = /.*?\[.*?;31mString form:.*?\[0m\s+([\s\S]+?)\n.*?\[.*?/;
+const StringFormRegex = /.*?\[.*?;31mString form:.*?\[0m\s*?([\s\S]+?)\n(.*\[.*;31m?)/;
 const DocStringRegex = /.*?\[.*?;31mDocstring:.*?\[0m\s+(.*)/;
 const CountRegex = /.*?\[.*?;31mLength:.*?\[0m\s+(.*)/;
 const ShapeRegex = /^\s+\[(\d+) rows x (\d+) columns\]/m;

src/client/datascience/types.ts

@@ -561,6 +561,7 @@ export interface IDataScienceExtraSettings extends IDataScienceSettings {
             autoSurround: string;
             autoIndent: boolean;
             scrollBeyondLastLine: boolean;
+            horizontalScrollbarSize: number;
             verticalScrollbarSize: number;
             fontSize: number;
             fontFamily: string;

src/client/datascience/webViewHost.ts

@@ -188,8 +188,9 @@ export abstract class WebViewHost<IMapping> implements IDisposable {
                     autoIndent: this.getValue(editor, 'autoIndent', false),
                     fontLigatures: this.getValue(editor, 'fontLigatures', false),
                     scrollBeyondLastLine: this.getValue(editor, 'scrollBeyondLastLine', true),
-                    // VS Code puts a value for this, but it's 10 (the explorer bar size) not 14 the editor size
+                    // VS Code puts a value for this, but it's 10 (the explorer bar size) not 14 the editor size for vert
                     verticalScrollbarSize: this.getValue(editor, 'scrollbar.verticalScrollbarSize', 14),
+                    horizontalScrollbarSize: this.getValue(editor, 'scrollbar.horizontalScrollbarSize', 10),
                     fontSize: this.getValue(editor, 'fontSize', 14),
                     fontFamily: this.getValue(editor, 'fontFamily', "Consolas, 'Courier New', monospace")
                 },
@@ -346,6 +347,7 @@ export abstract class WebViewHost<IMapping> implements IDisposable {
             event.affectsConfiguration('editor.scrollBeyondLastLine') ||
             event.affectsConfiguration('editor.fontLigatures') ||
             event.affectsConfiguration('editor.scrollbar.verticalScrollbarSize') ||
+            event.affectsConfiguration('editor.scrollbar.horizontalScrollbarSize') ||
             event.affectsConfiguration('files.autoSave') ||
             event.affectsConfiguration('files.autoSaveDelay') ||
             event.affectsConfiguration('python.dataScience.enableGather')

src/datascience-ui/history-react/interactivePanel.less

@@ -1,11 +1,6 @@
 /* Import common styles and then override them below */
 @import "../interactive-common/common.css";
 
-:root {
-    --messages-offset-px: 20px;
-    --output-offset-px: 45px;
-}
-
 .toolbar-menu-bar-child {
     background: var(--override-background, var(--vscode-editor-background));
     z-index: 10;
@@ -19,7 +14,7 @@
 }
 
 .messages-result-container {
-    width: calc(100vw - var(--messages-offset-px));
+    width: 100%;
 }
 
 .messages-result-container pre {
@@ -38,19 +33,20 @@
 .cell-result-container {
     margin: 0px;
     display: grid;
+    grid-auto-columns: minmax(0, 1fr);
 }
 
 .cell-outer {
     display:grid;
-    grid-template-columns: auto 1fr 8px;
+    grid-template-columns: auto minmax(0, 1fr) 8px;
     grid-column-gap: 3px;
     width: 100%;
 }
 
 
 .cell-output {
     margin: 0px;
-    width: calc(100vw - var(--output-offset-px));
+    width: 100%;
     overflow-x: scroll;
     background: transparent;
 }
@@ -68,7 +64,7 @@ xmp {
 }
 
 .markdown-cell-output {
-    width: calc(100vw - var(--output-offset-px));
+    width: 100%;
     overflow-x: scroll;
 }
 

src/datascience-ui/history-react/interactivePanel.tsx

@@ -284,6 +284,7 @@ ${buildSettingsCss(this.props.settings)}`}</style>
             testMode: this.props.testMode,
             codeTheme: this.props.codeTheme,
             submittedText: this.props.submittedText,
+            settings: this.props.settings,
             skipNextScroll: this.props.skipNextScroll ? true : false,
             editable: false,
             renderCell: this.renderCell,
@@ -354,11 +355,8 @@ ${buildSettingsCss(this.props.settings)}`}</style>
             this.internalScrollCount += 1;
             // Force auto here as smooth scrolling can be canceled by updates to the window
             // from elsewhere (and keeping track of these would make this hard to maintain)
-            // tslint:disable: no-any
-            if ((div as any).scrollIntoViewIfNeeded) {
-                (div as any).scrollIntoViewIfNeeded(false);
-            } else if (div && div.scrollIntoView) {
-                div.scrollIntoView(false);
+            if (div && div.scrollIntoView) {
+                div.scrollIntoView({ behavior: 'auto', block: 'nearest', inline: 'nearest' });
             }
         }
     };

src/datascience-ui/history-react/redux/reducers/effects.ts

@@ -113,8 +113,7 @@ export namespace Effects {
             newVMs[index] = { ...newVMs[index], scrollCount: newVMs[index].scrollCount + 1 };
             return {
                 ...arg.prevState,
-                cellVMs: newVMs,
-                isAtBottom: false
+                cellVMs: newVMs
             };
         }
 

src/datascience-ui/interactive-common/buildSettingsCss.ts

@@ -11,6 +11,14 @@ export function buildSettingsCss(settings: IDataScienceExtraSettings | undefined
     return settings
         ? `#main-panel-content::-webkit-scrollbar {
     width: ${settings.extraSettings.editor.verticalScrollbarSize}px;
+}
+
+.cell-output::-webkit-scrollbar {
+    height: ${settings.extraSettings.editor.horizontalScrollbarSize}px;
+}
+
+.cell-output > *::-webkit-scrollbar {
+    width: ${settings.extraSettings.editor.verticalScrollbarSize}px;
 }`
         : '';
 }

src/datascience-ui/interactive-common/common.css

@@ -79,7 +79,7 @@ body, html {
 /* Cell */
 .cell-wrapper {
     margin: 0px;
-    padding: 2px;
+    padding: 2px 5px 2px 2px;
     display: block;
 }
 
@@ -310,14 +310,10 @@ body, html {
 }
 
 #cell-table {
-    display: table;
+    display: block;
     width: 100%;
 }
 
-#cell-table-body {
-    display: table-row-group;
-}
-
 .flash {
     animation-name: flash-animation;
     animation-duration: 1.0s;

src/datascience-ui/interactive-common/contentPanel.tsx

@@ -3,6 +3,7 @@
 'use strict';
 import * as React from 'react';
 
+import { IDataScienceExtraSettings } from '../../client/datascience/types';
 import { InputHistory } from './inputHistory';
 import { ICellViewModel } from './mainState';
 
@@ -16,6 +17,7 @@ export interface IContentPanelProps {
     cellVMs: ICellViewModel[];
     history?: InputHistory;
     testMode?: boolean;
+    settings?: IDataScienceExtraSettings;
     codeTheme: string;
     submittedText: boolean;
     skipNextScroll: boolean;
@@ -32,12 +34,10 @@ export class ContentPanel extends React.Component<IContentPanelProps> {
     constructor(prop: IContentPanelProps) {
         super(prop);
     }
-
     public componentDidMount() {
         this.scrollToBottom();
     }
-
-    public componentDidUpdate() {
+    public componentWillReceiveProps() {
         this.scrollToBottom();
     }
 
@@ -53,12 +53,10 @@ export class ContentPanel extends React.Component<IContentPanelProps> {
         const className = `${this.props.scrollBeyondLastLine ? 'content-panel-scrollBeyondLastLine' : ''}`;
         return (
             <div id="content-panel-div" ref={this.containerRef} className={className}>
-                <div id="cell-table">
-                    <div id="cell-table-body" role="list">
-                        {this.renderCells()}
-                    </div>
+                <div id="cell-table" role="list">
+                    {this.renderCells()}
                 </div>
-                <div ref={this.bottomRef} />
+                <div id="bottomDiv" ref={this.bottomRef} />
             </div>
         );
     }