diff --git a/pythonFiles/tests/ipython/scripts.py b/pythonFiles/tests/ipython/scripts.py index b36ee07ff430..cf015de053e0 100644 --- a/pythonFiles/tests/ipython/scripts.py +++ b/pythonFiles/tests/ipython/scripts.py @@ -75,7 +75,9 @@ def get_data_frame_info(variables, name, capsys): varJson = find_variable_json(variables, name) path = os.path.dirname(os.path.abspath(__file__)) file = os.path.abspath( - os.path.join(path, "../../datascience/getJupyterVariableDataFrameInfo.py") + os.path.join( + path, "../../vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py" + ) ) keys = dict([("_VSCode_JupyterTestValue", json.dumps(varJson))]) if execute_script(file, keys): @@ -88,7 +90,9 @@ def get_data_frame_info(variables, name, capsys): def get_data_frame_rows(varJson, start, end, capsys): path = os.path.dirname(os.path.abspath(__file__)) file = os.path.abspath( - os.path.join(path, "../../datascience/getJupyterVariableDataFrameRows.py") + os.path.join( + path, "../../vscode_datascience_helpers/getJupyterVariableDataFrameRows.py" + ) ) keys = dict( [ diff --git a/pythonFiles/datascience/__init__.py b/pythonFiles/vscode_datascience_helpers/__init__.py similarity index 100% rename from pythonFiles/datascience/__init__.py rename to pythonFiles/vscode_datascience_helpers/__init__.py diff --git a/pythonFiles/datascience/daemon/README.md b/pythonFiles/vscode_datascience_helpers/daemon/README.md similarity index 92% rename from pythonFiles/datascience/daemon/README.md rename to pythonFiles/vscode_datascience_helpers/daemon/README.md index 45b4ce6a97c0..8fdbc5eabdde 100644 --- a/pythonFiles/datascience/daemon/README.md +++ b/pythonFiles/vscode_datascience_helpers/daemon/README.md @@ -7,7 +7,7 @@ const env = { PYTHONUNBUFFERED: '1', PYTHONPATH: '/pythonFiles:/pythonFiles/lib/python' } -const childProcess = cp.spawn('', ['-m', 'datascience.daemon', '-v', '--log-file=log.log'], {env}); +const childProcess = cp.spawn('', ['-m', 'vscode_datascience_helpers.daemon', '-v', '--log-file=log.log'], {env}); const connection = rpc.createMessageConnection(new rpc.StreamMessageReader(childProcess.stdout),new rpc.StreamMessageWriter(childProcess.stdin)); connection.onClose(() => console.error('Closed')); diff --git a/pythonFiles/datascience/daemon/__init__.py b/pythonFiles/vscode_datascience_helpers/daemon/__init__.py similarity index 100% rename from pythonFiles/datascience/daemon/__init__.py rename to pythonFiles/vscode_datascience_helpers/daemon/__init__.py diff --git a/pythonFiles/datascience/daemon/__main__.py b/pythonFiles/vscode_datascience_helpers/daemon/__main__.py similarity index 98% rename from pythonFiles/datascience/daemon/__main__.py rename to pythonFiles/vscode_datascience_helpers/daemon/__main__.py index c1c047f233b9..2d03e03e613a 100644 --- a/pythonFiles/datascience/daemon/__main__.py +++ b/pythonFiles/vscode_datascience_helpers/daemon/__main__.py @@ -20,7 +20,7 @@ def add_arguments(parser): parser.add_argument( "--daemon-module", - default="datascience.daemon.daemon_python", + default="vscode_datascience_helpers.daemon.daemon_python", help="Daemon Module", ) diff --git a/pythonFiles/datascience/daemon/daemon_output.py b/pythonFiles/vscode_datascience_helpers/daemon/daemon_output.py similarity index 100% rename from pythonFiles/datascience/daemon/daemon_output.py rename to pythonFiles/vscode_datascience_helpers/daemon/daemon_output.py diff --git a/pythonFiles/datascience/daemon/daemon_python.py b/pythonFiles/vscode_datascience_helpers/daemon/daemon_python.py similarity index 98% rename from pythonFiles/datascience/daemon/daemon_python.py rename to pythonFiles/vscode_datascience_helpers/daemon/daemon_python.py index d9d85c587123..694934fefdb8 100644 --- a/pythonFiles/datascience/daemon/daemon_python.py +++ b/pythonFiles/vscode_datascience_helpers/daemon/daemon_python.py @@ -8,7 +8,7 @@ import traceback import runpy import importlib -from datascience.daemon.daemon_output import ( +from vscode_datascience_helpers.daemon.daemon_output import ( CustomWriter, IORedirector, get_io_buffers, @@ -63,7 +63,7 @@ def _decorator(self, *args, **kwargs): class PythonDaemon(MethodDispatcher): """ Base Python Daemon with simple methods to check if a module exists, get version info and the like. - To add additional methods, please create a separate class based off this and pass in the arg `--daemon-module` to `datascience.daemon`. + To add additional methods, please create a separate class based off this and pass in the arg `--daemon-module` to `vscode_datascience_helpers.daemon`. """ def __init__(self, rx, tx): diff --git a/pythonFiles/datascience/dummyJupyter.py b/pythonFiles/vscode_datascience_helpers/dummyJupyter.py similarity index 96% rename from pythonFiles/datascience/dummyJupyter.py rename to pythonFiles/vscode_datascience_helpers/dummyJupyter.py index 211af8cec79d..1a1a6cc07cb9 100644 --- a/pythonFiles/datascience/dummyJupyter.py +++ b/pythonFiles/vscode_datascience_helpers/dummyJupyter.py @@ -1,28 +1,28 @@ -# This file can mimic juypter running. Useful for testing jupyter crash handling - -import sys -import argparse -import time - - -def main(): - print("hello from dummy jupyter") - parser = argparse.ArgumentParser() - parser.add_argument("--version", type=bool, default=False, const=True, nargs="?") - parser.add_argument("notebook", type=bool, default=False, const=True, nargs="?") - parser.add_argument("--no-browser", type=bool, default=False, const=True, nargs="?") - parser.add_argument("--notebook-dir", default="") - parser.add_argument("--config", default="") - results = parser.parse_args() - if results.version: - print("1.1.dummy") - else: - print( - "http://localhost:8888/?token=012f08663a68e279fe0a5335e0b5dfe44759ddcccf0b3a56" - ) - time.sleep(5) - raise Exception("Dummy is dead") - - -if __name__ == "__main__": - main() +# This file can mimic juypter running. Useful for testing jupyter crash handling + +import sys +import argparse +import time + + +def main(): + print("hello from dummy jupyter") + parser = argparse.ArgumentParser() + parser.add_argument("--version", type=bool, default=False, const=True, nargs="?") + parser.add_argument("notebook", type=bool, default=False, const=True, nargs="?") + parser.add_argument("--no-browser", type=bool, default=False, const=True, nargs="?") + parser.add_argument("--notebook-dir", default="") + parser.add_argument("--config", default="") + results = parser.parse_args() + if results.version: + print("1.1.dummy") + else: + print( + "http://localhost:8888/?token=012f08663a68e279fe0a5335e0b5dfe44759ddcccf0b3a56" + ) + time.sleep(5) + raise Exception("Dummy is dead") + + +if __name__ == "__main__": + main() diff --git a/pythonFiles/datascience/getJupyterKernels.py b/pythonFiles/vscode_datascience_helpers/getJupyterKernels.py similarity index 100% rename from pythonFiles/datascience/getJupyterKernels.py rename to pythonFiles/vscode_datascience_helpers/getJupyterKernels.py diff --git a/pythonFiles/datascience/getJupyterKernelspecVersion.py b/pythonFiles/vscode_datascience_helpers/getJupyterKernelspecVersion.py similarity index 100% rename from pythonFiles/datascience/getJupyterKernelspecVersion.py rename to pythonFiles/vscode_datascience_helpers/getJupyterKernelspecVersion.py diff --git a/pythonFiles/datascience/getJupyterVariableDataFrameInfo.py b/pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py similarity index 97% rename from pythonFiles/datascience/getJupyterVariableDataFrameInfo.py rename to pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py index cf99979c47ba..856f5e282a18 100644 --- a/pythonFiles/datascience/getJupyterVariableDataFrameInfo.py +++ b/pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameInfo.py @@ -1,120 +1,120 @@ -# Query Jupyter server for the info about a dataframe -import json as _VSCODE_json -import pandas as _VSCODE_pd -import pandas.io.json as _VSCODE_pd_json -import builtins as _VSCODE_builtins - -# _VSCode_sub_supportsDataExplorer will contain our list of data explorer supported types -_VSCode_supportsDataExplorer = "['list', 'Series', 'dict', 'ndarray', 'DataFrame']" - -# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable -# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable -_VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""") - -# Function to compute row count for a value -def _VSCODE_getRowCount(var): - if hasattr(var, "shape"): - try: - # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it - if isinstance(var.shape, tuple): - return var.shape[0] - except TypeError: - return 0 - elif hasattr(var, "__len__"): - try: - return _VSCODE_builtins.len(var) - except TypeError: - return 0 - - -# First check to see if we are a supported type, this prevents us from adding types that are not supported -# and also keeps our types in sync with what the variable explorer says that we support -if _VSCODE_targetVariable["type"] not in _VSCode_supportsDataExplorer: - del _VSCode_supportsDataExplorer - print(_VSCODE_json.dumps(_VSCODE_targetVariable)) - del _VSCODE_targetVariable -else: - del _VSCode_supportsDataExplorer - _VSCODE_evalResult = _VSCODE_builtins.eval(_VSCODE_targetVariable["name"]) - - # Figure out shape if not already there. Use the shape to compute the row count - _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_evalResult) - - # Turn the eval result into a df - _VSCODE_df = _VSCODE_evalResult - if isinstance(_VSCODE_evalResult, list): - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) - elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series): - _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) - elif isinstance(_VSCODE_evalResult, dict): - _VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult) - _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) - elif _VSCODE_targetVariable["type"] == "ndarray": - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) - elif hasattr(_VSCODE_df, "toPandas"): - _VSCODE_df = _VSCODE_df.toPandas() - _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_df) - - # If any rows, use pandas json to convert a single row to json. Extract - # the column names and types from the json so we match what we'll fetch when - # we ask for all of the rows - if ( - hasattr(_VSCODE_targetVariable, "rowCount") - and _VSCODE_targetVariable["rowCount"] - ): - try: - _VSCODE_row = _VSCODE_df.iloc[0:1] - _VSCODE_json_row = _VSCODE_pd_json.to_json( - None, _VSCODE_row, date_format="iso" - ) - _VSCODE_columnNames = list(_VSCODE_json.loads(_VSCODE_json_row)) - del _VSCODE_row - del _VSCODE_json_row - except: - _VSCODE_columnNames = list(_VSCODE_df) - else: - _VSCODE_columnNames = list(_VSCODE_df) - - # Compute the index column. It may have been renamed - _VSCODE_indexColumn = _VSCODE_df.index.name if _VSCODE_df.index.name else "index" - _VSCODE_columnTypes = _VSCODE_builtins.list(_VSCODE_df.dtypes) - del _VSCODE_df - - # Make sure the index column exists - if _VSCODE_indexColumn not in _VSCODE_columnNames: - _VSCODE_columnNames.insert(0, _VSCODE_indexColumn) - _VSCODE_columnTypes.insert(0, "int64") - - # Then loop and generate our output json - _VSCODE_columns = [] - for _VSCODE_n in _VSCODE_builtins.range( - 0, _VSCODE_builtins.len(_VSCODE_columnNames) - ): - _VSCODE_column_type = _VSCODE_columnTypes[_VSCODE_n] - _VSCODE_column_name = str(_VSCODE_columnNames[_VSCODE_n]) - _VSCODE_colobj = {} - _VSCODE_colobj["key"] = _VSCODE_column_name - _VSCODE_colobj["name"] = _VSCODE_column_name - _VSCODE_colobj["type"] = str(_VSCODE_column_type) - _VSCODE_columns.append(_VSCODE_colobj) - del _VSCODE_column_name - del _VSCODE_column_type - - del _VSCODE_columnNames - del _VSCODE_columnTypes - - # Save this in our target - _VSCODE_targetVariable["columns"] = _VSCODE_columns - _VSCODE_targetVariable["indexColumn"] = _VSCODE_indexColumn - del _VSCODE_columns - del _VSCODE_indexColumn - - # Transform this back into a string - print(_VSCODE_json.dumps(_VSCODE_targetVariable)) - del _VSCODE_targetVariable - - # Cleanup imports - del _VSCODE_json - del _VSCODE_pd - del _VSCODE_pd_json - del _VSCODE_builtins +# Query Jupyter server for the info about a dataframe +import json as _VSCODE_json +import pandas as _VSCODE_pd +import pandas.io.json as _VSCODE_pd_json +import builtins as _VSCODE_builtins + +# _VSCode_sub_supportsDataExplorer will contain our list of data explorer supported types +_VSCode_supportsDataExplorer = "['list', 'Series', 'dict', 'ndarray', 'DataFrame']" + +# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable +# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable +_VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""") + +# Function to compute row count for a value +def _VSCODE_getRowCount(var): + if hasattr(var, "shape"): + try: + # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it + if isinstance(var.shape, tuple): + return var.shape[0] + except TypeError: + return 0 + elif hasattr(var, "__len__"): + try: + return _VSCODE_builtins.len(var) + except TypeError: + return 0 + + +# First check to see if we are a supported type, this prevents us from adding types that are not supported +# and also keeps our types in sync with what the variable explorer says that we support +if _VSCODE_targetVariable["type"] not in _VSCode_supportsDataExplorer: + del _VSCode_supportsDataExplorer + print(_VSCODE_json.dumps(_VSCODE_targetVariable)) + del _VSCODE_targetVariable +else: + del _VSCode_supportsDataExplorer + _VSCODE_evalResult = _VSCODE_builtins.eval(_VSCODE_targetVariable["name"]) + + # Figure out shape if not already there. Use the shape to compute the row count + _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_evalResult) + + # Turn the eval result into a df + _VSCODE_df = _VSCODE_evalResult + if isinstance(_VSCODE_evalResult, list): + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) + elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series): + _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) + elif isinstance(_VSCODE_evalResult, dict): + _VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult) + _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) + elif _VSCODE_targetVariable["type"] == "ndarray": + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) + elif hasattr(_VSCODE_df, "toPandas"): + _VSCODE_df = _VSCODE_df.toPandas() + _VSCODE_targetVariable["rowCount"] = _VSCODE_getRowCount(_VSCODE_df) + + # If any rows, use pandas json to convert a single row to json. Extract + # the column names and types from the json so we match what we'll fetch when + # we ask for all of the rows + if ( + hasattr(_VSCODE_targetVariable, "rowCount") + and _VSCODE_targetVariable["rowCount"] + ): + try: + _VSCODE_row = _VSCODE_df.iloc[0:1] + _VSCODE_json_row = _VSCODE_pd_json.to_json( + None, _VSCODE_row, date_format="iso" + ) + _VSCODE_columnNames = list(_VSCODE_json.loads(_VSCODE_json_row)) + del _VSCODE_row + del _VSCODE_json_row + except: + _VSCODE_columnNames = list(_VSCODE_df) + else: + _VSCODE_columnNames = list(_VSCODE_df) + + # Compute the index column. It may have been renamed + _VSCODE_indexColumn = _VSCODE_df.index.name if _VSCODE_df.index.name else "index" + _VSCODE_columnTypes = _VSCODE_builtins.list(_VSCODE_df.dtypes) + del _VSCODE_df + + # Make sure the index column exists + if _VSCODE_indexColumn not in _VSCODE_columnNames: + _VSCODE_columnNames.insert(0, _VSCODE_indexColumn) + _VSCODE_columnTypes.insert(0, "int64") + + # Then loop and generate our output json + _VSCODE_columns = [] + for _VSCODE_n in _VSCODE_builtins.range( + 0, _VSCODE_builtins.len(_VSCODE_columnNames) + ): + _VSCODE_column_type = _VSCODE_columnTypes[_VSCODE_n] + _VSCODE_column_name = str(_VSCODE_columnNames[_VSCODE_n]) + _VSCODE_colobj = {} + _VSCODE_colobj["key"] = _VSCODE_column_name + _VSCODE_colobj["name"] = _VSCODE_column_name + _VSCODE_colobj["type"] = str(_VSCODE_column_type) + _VSCODE_columns.append(_VSCODE_colobj) + del _VSCODE_column_name + del _VSCODE_column_type + + del _VSCODE_columnNames + del _VSCODE_columnTypes + + # Save this in our target + _VSCODE_targetVariable["columns"] = _VSCODE_columns + _VSCODE_targetVariable["indexColumn"] = _VSCODE_indexColumn + del _VSCODE_columns + del _VSCODE_indexColumn + + # Transform this back into a string + print(_VSCODE_json.dumps(_VSCODE_targetVariable)) + del _VSCODE_targetVariable + + # Cleanup imports + del _VSCODE_json + del _VSCODE_pd + del _VSCODE_pd_json + del _VSCODE_builtins diff --git a/pythonFiles/datascience/getJupyterVariableDataFrameRows.py b/pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameRows.py similarity index 97% rename from pythonFiles/datascience/getJupyterVariableDataFrameRows.py rename to pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameRows.py index b6f3ebe42726..e7d73ae656b2 100644 --- a/pythonFiles/datascience/getJupyterVariableDataFrameRows.py +++ b/pythonFiles/vscode_datascience_helpers/getJupyterVariableDataFrameRows.py @@ -1,52 +1,52 @@ -# Query Jupyter server for the rows of a data frame -import json as _VSCODE_json -import pandas as _VSCODE_pd -import pandas.io.json as _VSCODE_pd_json -import builtins as _VSCODE_builtins - -# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable -# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable -_VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""") -_VSCODE_evalResult = _VSCODE_builtins.eval(_VSCODE_targetVariable["name"]) - -# _VSCode_JupyterStartRow and _VSCode_JupyterEndRow should be replaced dynamically with the literals -# for our start and end rows -_VSCODE_startRow = _VSCODE_builtins.max(_VSCode_JupyterStartRow, 0) -_VSCODE_endRow = _VSCODE_builtins.min( - _VSCode_JupyterEndRow, _VSCODE_targetVariable["rowCount"] -) - -# Assume we have a dataframe. If not, turn our eval result into a dataframe -_VSCODE_df = _VSCODE_evalResult -if isinstance(_VSCODE_evalResult, list): - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) -elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series): - _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) -elif isinstance(_VSCODE_evalResult, dict): - _VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult) - _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) -elif _VSCODE_targetVariable["type"] == "ndarray": - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) -elif hasattr(_VSCODE_df, "toPandas"): - _VSCODE_df = _VSCODE_df.toPandas() -# If not a known type, then just let pandas handle it. -elif not (hasattr(_VSCODE_df, "iloc")): - _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) - -# Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON -_VSCODE_rows = _VSCODE_df.iloc[_VSCODE_startRow:_VSCODE_endRow] -_VSCODE_result = _VSCODE_pd_json.to_json( - None, _VSCODE_rows, orient="table", date_format="iso" -) -print(_VSCODE_result) - -# Cleanup our variables -del _VSCODE_df -del _VSCODE_endRow -del _VSCODE_startRow -del _VSCODE_rows -del _VSCODE_result -del _VSCODE_json -del _VSCODE_pd -del _VSCODE_pd_json -del _VSCODE_builtins +# Query Jupyter server for the rows of a data frame +import json as _VSCODE_json +import pandas as _VSCODE_pd +import pandas.io.json as _VSCODE_pd_json +import builtins as _VSCODE_builtins + +# In IJupyterVariables.getValue this '_VSCode_JupyterTestValue' will be replaced with the json stringified value of the target variable +# Indexes off of _VSCODE_targetVariable need to index types that are part of IJupyterVariable +_VSCODE_targetVariable = _VSCODE_json.loads("""_VSCode_JupyterTestValue""") +_VSCODE_evalResult = _VSCODE_builtins.eval(_VSCODE_targetVariable["name"]) + +# _VSCode_JupyterStartRow and _VSCode_JupyterEndRow should be replaced dynamically with the literals +# for our start and end rows +_VSCODE_startRow = _VSCODE_builtins.max(_VSCode_JupyterStartRow, 0) +_VSCODE_endRow = _VSCODE_builtins.min( + _VSCode_JupyterEndRow, _VSCODE_targetVariable["rowCount"] +) + +# Assume we have a dataframe. If not, turn our eval result into a dataframe +_VSCODE_df = _VSCODE_evalResult +if isinstance(_VSCODE_evalResult, list): + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) +elif isinstance(_VSCODE_evalResult, _VSCODE_pd.Series): + _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) +elif isinstance(_VSCODE_evalResult, dict): + _VSCODE_evalResult = _VSCODE_pd.Series(_VSCODE_evalResult) + _VSCODE_df = _VSCODE_pd.Series.to_frame(_VSCODE_evalResult) +elif _VSCODE_targetVariable["type"] == "ndarray": + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) +elif hasattr(_VSCODE_df, "toPandas"): + _VSCODE_df = _VSCODE_df.toPandas() +# If not a known type, then just let pandas handle it. +elif not (hasattr(_VSCODE_df, "iloc")): + _VSCODE_df = _VSCODE_pd.DataFrame(_VSCODE_evalResult) + +# Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON +_VSCODE_rows = _VSCODE_df.iloc[_VSCODE_startRow:_VSCODE_endRow] +_VSCODE_result = _VSCODE_pd_json.to_json( + None, _VSCODE_rows, orient="table", date_format="iso" +) +print(_VSCODE_result) + +# Cleanup our variables +del _VSCODE_df +del _VSCODE_endRow +del _VSCODE_startRow +del _VSCODE_rows +del _VSCODE_result +del _VSCODE_json +del _VSCODE_pd +del _VSCODE_pd_json +del _VSCODE_builtins diff --git a/pythonFiles/datascience/getServerInfo.py b/pythonFiles/vscode_datascience_helpers/getServerInfo.py similarity index 100% rename from pythonFiles/datascience/getServerInfo.py rename to pythonFiles/vscode_datascience_helpers/getServerInfo.py diff --git a/pythonFiles/datascience/jupyter_daemon.py b/pythonFiles/vscode_datascience_helpers/jupyter_daemon.py similarity index 99% rename from pythonFiles/datascience/jupyter_daemon.py rename to pythonFiles/vscode_datascience_helpers/jupyter_daemon.py index be51abab85b8..f83224a48d70 100644 --- a/pythonFiles/datascience/jupyter_daemon.py +++ b/pythonFiles/vscode_datascience_helpers/jupyter_daemon.py @@ -6,7 +6,7 @@ import os import subprocess import sys -from datascience.daemon.daemon_python import ( +from vscode_datascience_helpers.daemon.daemon_python import ( error_decorator, PythonDaemon as BasePythonDaemon, change_exec_context, diff --git a/pythonFiles/datascience/jupyter_nbInstalled.py b/pythonFiles/vscode_datascience_helpers/jupyter_nbInstalled.py similarity index 100% rename from pythonFiles/datascience/jupyter_nbInstalled.py rename to pythonFiles/vscode_datascience_helpers/jupyter_nbInstalled.py diff --git a/src/client/common/process/pythonDaemonPool.ts b/src/client/common/process/pythonDaemonPool.ts index 1248a0d8756a..7189f7c58d96 100644 --- a/src/client/common/process/pythonDaemonPool.ts +++ b/src/client/common/process/pythonDaemonPool.ts @@ -144,7 +144,11 @@ export class PythonDaemonExecutionServicePool implements IPythonDaemonExecutionS loggingArgs ); const env = this.envVariables; - const daemonProc = this.pythonExecutionService!.execModuleObservable('datascience.daemon', args, { env }); + const daemonProc = this.pythonExecutionService!.execModuleObservable( + 'vscode_datascience_helpers.daemon', + args, + { env } + ); if (!daemonProc.proc) { throw new Error('Failed to create Daemon Proc'); } diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 7e780dc5d4c4..a4a2b91f64d0 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -12,7 +12,7 @@ export const DefaultTheme = 'Default Light+'; export const JUPYTER_OUTPUT_CHANNEL = 'JUPYTER_OUTPUT_CHANNEL'; // Python Module to be used when instantiating the Python Daemon. -export const PythonDaemonModule = 'datascience.jupyter_daemon'; +export const PythonDaemonModule = 'vscode_datascience_helpers.jupyter_daemon'; // List of 'language' names that we know about. All should be lower case as that's how we compare. export const KnownNotebookLanguages: string[] = [ diff --git a/src/client/datascience/jupyter/interpreter/jupyterCommand.ts b/src/client/datascience/jupyter/interpreter/jupyterCommand.ts index 49e5a970bd82..342da28034a4 100644 --- a/src/client/datascience/jupyter/interpreter/jupyterCommand.ts +++ b/src/client/datascience/jupyter/interpreter/jupyterCommand.ts @@ -107,7 +107,14 @@ class InterpreterJupyterCommand implements IJupyterCommand { ) { try { const output = await svc.exec( - [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'jupyter_nbInstalled.py')], + [ + path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'jupyter_nbInstalled.py' + ) + ], {} ); if (output.stdout.toLowerCase().includes('available')) { @@ -264,7 +271,7 @@ export class InterpreterJupyterKernelSpecCommand extends InterpreterJupyterComma bypassCondaExecution: true }); return activatedEnv.exec( - [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterKernels.py')], + [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'getJupyterKernels.py')], { ...options, throwOnStdErr: true } ); } @@ -275,7 +282,14 @@ export class InterpreterJupyterKernelSpecCommand extends InterpreterJupyterComma bypassCondaExecution: true }); return activatedEnv.exec( - [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterKernelspecVersion.py')], + [ + path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getJupyterKernelspecVersion.py' + ) + ], { ...options, throwOnStdErr: true } ); } diff --git a/src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterExecutionService.ts b/src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterExecutionService.ts index 4888ab9c508a..cf9e70ec7a54 100644 --- a/src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterExecutionService.ts +++ b/src/client/datascience/jupyter/interpreter/jupyterCommandInterpreterExecutionService.ts @@ -102,7 +102,7 @@ export class JupyterCommandFinderInterpreterExecutionService implements IJupyter // We have a small python file here that we will execute to get the server info from all running Jupyter instances const newOptions: SpawnOptions = { mergeStdOutErr: true, token: token }; - const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'getServerInfo.py'); const serverInfoString = await daemon.exec([file], newOptions); let serverInfos: JupyterServerInfo[]; diff --git a/src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts b/src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts index e91889297535..350ddab38a5c 100644 --- a/src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts +++ b/src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts @@ -126,7 +126,7 @@ export class JupyterInterpreterSubCommandExecutionService // We have a small python file here that we will execute to get the server info from all running Jupyter instances const newOptions: SpawnOptions = { mergeStdOutErr: true, token: token }; - const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'getServerInfo.py'); const serverInfoString = await daemon.exec([file], newOptions); let serverInfos: JupyterServerInfo[]; @@ -199,7 +199,14 @@ export class JupyterInterpreterSubCommandExecutionService // Possible we cannot import ipykernel for some reason. (use as backup option). const stdoutFromFileExecPromise = daemon .exec( - [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterKernels.py')], + [ + path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getJupyterKernels.py' + ) + ], spawnOptions ) .then(output => output.stdout) diff --git a/src/client/datascience/jupyter/jupyterVariables.ts b/src/client/datascience/jupyter/jupyterVariables.ts index 35620498a1a9..0cba8b381613 100644 --- a/src/client/datascience/jupyter/jupyterVariables.ts +++ b/src/client/datascience/jupyter/jupyterVariables.ts @@ -94,10 +94,20 @@ export class JupyterVariables implements IJupyterVariables { // Private methods // Load our python files for fetching variables private async loadVariableFiles(): Promise { - let file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterVariableDataFrameInfo.py'); + let file = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getJupyterVariableDataFrameInfo.py' + ); this.fetchDataFrameInfoScript = await this.fileSystem.readFile(file); - file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getJupyterVariableDataFrameRows.py'); + file = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getJupyterVariableDataFrameRows.py' + ); this.fetchDataFrameRowsScript = await this.fileSystem.readFile(file); this.filesLoaded = true; diff --git a/src/test/common/process/pythonDaemon.functional.test.ts b/src/test/common/process/pythonDaemon.functional.test.ts index aec0af00fad4..8f81f83d7452 100644 --- a/src/test/common/process/pythonDaemon.functional.test.ts +++ b/src/test/common/process/pythonDaemon.functional.test.ts @@ -59,8 +59,8 @@ suite('Daemon', () => { return this.skip(); } // Enable the following to log everything going on at pyton end. - // pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'datascience.daemon', '-v', `--log-file=${path.join(EXTENSION_ROOT_DIR, 'test.log')}`], { env }); - pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'datascience.daemon'], { env }); + // pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'vscode_datascience_helpers.daemon', '-v', `--log-file=${path.join(EXTENSION_ROOT_DIR, 'test.log')}`], { env }); + pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'vscode_datascience_helpers.daemon'], { env }); connection = createMessageConnection( new StreamMessageReader(pythonProc.stdout), new StreamMessageWriter(pythonProc.stdin) diff --git a/src/test/common/process/pythonDaemonPool.functional.test.ts b/src/test/common/process/pythonDaemonPool.functional.test.ts index 3796e78c83c4..e8ad27f42a30 100644 --- a/src/test/common/process/pythonDaemonPool.functional.test.ts +++ b/src/test/common/process/pythonDaemonPool.functional.test.ts @@ -75,8 +75,10 @@ suite('Daemon - Python Daemon Pool', () => { logger = mock(ProcessLogger); createDaemonServicesSpy = sinon.spy(DaemonPool.prototype, 'createDaemonServices'); pythonExecutionService = mock(PythonExecutionService); - when(pythonExecutionService.execModuleObservable('datascience.daemon', anything(), anything())).thenCall(() => { - const pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'datascience.daemon'], { env }); + when( + pythonExecutionService.execModuleObservable('vscode_datascience_helpers.daemon', anything(), anything()) + ).thenCall(() => { + const pythonProc = spawn(fullyQualifiedPythonPath, ['-m', 'vscode_datascience_helpers.daemon'], { env }); const connection = createMessageConnection( new StreamMessageReader(pythonProc.stdout), new StreamMessageWriter(pythonProc.stdin) diff --git a/src/test/common/process/pythonDaemonPool.unit.test.ts b/src/test/common/process/pythonDaemonPool.unit.test.ts index 7b8defe65924..4264220ed352 100644 --- a/src/test/common/process/pythonDaemonPool.unit.test.ts +++ b/src/test/common/process/pythonDaemonPool.unit.test.ts @@ -69,7 +69,9 @@ suite('Daemon - Python Daemon Pool', () => { daemonProc.stdout = new EventEmitter() as any; daemonProc.stderr = new EventEmitter() as any; - when(pythonExecService.execModuleObservable('datascience.daemon', anything(), anything())).thenReturn({ + when( + pythonExecService.execModuleObservable('vscode_datascience_helpers.daemon', anything(), anything()) + ).thenReturn({ proc: daemonProc, dispose: noop, out: undefined as any diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index 4ecbe743fb4e..e9f8fd4f2977 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -463,7 +463,12 @@ suite('Jupyter Execution', async () => { return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); } ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); setupPythonService( service, undefined, @@ -502,7 +507,12 @@ suite('Jupyter Execution', async () => { ['kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }) ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); setupPythonService( service, undefined, @@ -599,7 +609,12 @@ suite('Jupyter Execution', async () => { return Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }); } ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); setupProcessServiceExec( service, workingPython.path, @@ -638,7 +653,12 @@ suite('Jupyter Execution', async () => { ['-m', 'jupyter', 'kernelspec', 'list', '--json'], Promise.resolve({ stdout: JSON.stringify(kernelSpecs) }) ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); setupProcessServiceExec( service, missingKernelPython.path, diff --git a/src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts b/src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts index 45ee5b4d54da..2cf83a1f143b 100644 --- a/src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts +++ b/src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts @@ -318,7 +318,7 @@ suite('Data Science - Jupyter InterpreterSubCommandExecutionService', () => { assert.equal(output, convertOutput); }); test('Return list of running jupyter servers', async () => { - const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const file = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'getServerInfo.py'); const expectedServers: JupyterServerInfo[] = [ { base_url: '1', diff --git a/src/test/datascience/mockJupyterManager.ts b/src/test/datascience/mockJupyterManager.ts index 76b1622667c9..e8ea67321d98 100644 --- a/src/test/datascience/mockJupyterManager.ts +++ b/src/test/datascience/mockJupyterManager.ts @@ -762,7 +762,12 @@ export class MockJupyterManager implements IJupyterSessionManager { }); } ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); this.setupProcessServiceExec(this.processService, workingPython.path, [getServerInfoPath], () => Promise.resolve({ stdout: 'failure to get server infos' }) ); @@ -812,7 +817,12 @@ export class MockJupyterManager implements IJupyterSessionManager { return Promise.resolve({ stdout: JSON.stringify(createKernelSpecs(kernels)) }); } ); - const getServerInfoPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'datascience', 'getServerInfo.py'); + const getServerInfoPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getServerInfo.py' + ); this.setupProcessServiceExec(this.processService, workingPython.path, [getServerInfoPath], () => Promise.resolve({ stdout: 'failure to get server infos' }) );