vdk-jupyter: convert job operation (#2406)

What: 
This change includes: 

- implementing the actual transform operation
- adding an archive of the old job to the parent directory of the job
- adding a handler that captures the request from the front-end for the
transform operation
- adding the necessary UI components

More info for the operation can be found:
https://github.com/vmware/versatile-data-kit/wiki/VDK-Jupyter-Integration-Convert-Job-Operation

Testing: python unit tests, TBD: jest tests

Example of how transformed job looks like: 
![Screenshot 2023-07-14 at 14 27
21](https://github.com/vmware/versatile-data-kit/assets/87015481/880c250f-f518-461c-92a3-e176f1b5b4f2)
![Screenshot 2023-07-14 at 14 27
30](https://github.com/vmware/versatile-data-kit/assets/87015481/659140d6-317c-4ed8-9ff2-9add227b0233)
![Screenshot 2023-07-14 at 14 28
07](https://github.com/vmware/versatile-data-kit/assets/87015481/1652dab9-c625-4e99-9d99-d2f5604415a1)


This is how a transformed job would look like, no UI changes are
introduced in this PR

Signed-off-by: Duygu Hasan [hduygu@vmware.com](mailto:hduygu@vmware.com)

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

projects/vdk-plugins/vdk-jupyter/vdk-jupyterlab-extension/jest.config.js

@@ -6,7 +6,8 @@
 const jestJupyterLab = require('@jupyterlab/testutils/lib/jest-config');
 
 const esModules = [
-  '@jupyterlab/',
+  '.*@jupyterlab/',
+  '@jupyter/ydoc',
   'lib0',
   'y\\-protocols',
   'y\\-websocket',
@@ -20,7 +21,6 @@ const {
   moduleNameMapper,
   preset,
   setupFilesAfterEnv,
-  setupFiles,
   testPathIgnorePatterns,
   transform
 } = jlabConfig;
@@ -30,7 +30,7 @@ module.exports = {
   moduleNameMapper,
   preset,
   setupFilesAfterEnv,
-  setupFiles,
+  setupFiles: ['<rootDir>/testutils/jest-setup-files.js'],
   testPathIgnorePatterns,
   transform,
   automock: false,

projects/vdk-plugins/vdk-jupyter/vdk-jupyterlab-extension/package.json

@@ -114,7 +114,8 @@
     "stylelint-config-recommended": "6.0.0",
     "stylelint-config-standard": "24.0.0",
     "stylelint-prettier": "2.0.0",
-    "ts-jest": "27.1.4"
+    "ts-jest": "27.1.4",
+    "jest-fetch-mock": "^3.0.0"
   },
   "sideEffects": [
     "style/*.css",

projects/vdk-plugins/vdk-jupyter/vdk-jupyterlab-extension/src/__tests__/convert-job-to-notebook-component.spec.ts

@@ -2,7 +2,9 @@
  * Copyright 2023-2023 VMware, Inc.
  * SPDX-License-Identifier: Apache-2.0
  */
-import ConvertJobToNotebookDialog from '../components/ConvertJobToNotebook';
+import ConvertJobToNotebookDialog, {
+  initializeNotebookContent
+} from '../components/ConvertJobToNotebook';
 import { render, fireEvent } from '@testing-library/react';
 import { jobData } from '../jobData';
 import { VdkOption } from '../vdkOptions/vdk_options';
@@ -36,3 +38,65 @@ describe('#onPathChange', () => {
     expect(jobData.get(VdkOption.PATH)).toEqual('other/path');
   });
 });
+
+describe('initializeNotebookContent', () => {
+  it('should create a notebook content correctly', () => {
+    const codeStructure = [
+      'def run(job_input: IJobInput)',
+      'print("Hello, world!")'
+    ];
+    const fileNames = ['file1', 'file2'];
+
+    const expectedNotebookContent = [
+      { source: '#### file1', type: 'markdown' },
+      { source: codeStructure[0], type: 'code' },
+      { source: 'run(job_input)', type: 'code' },
+      { source: '#### file2', type: 'markdown' },
+      { source: codeStructure[1], type: 'code' }
+    ];
+
+    const result = initializeNotebookContent(codeStructure, fileNames);
+    expect(result).toEqual(expectedNotebookContent);
+  });
+
+  it('should not add a run cell when there is no run function', () => {
+    const codeStructure = ['print("Hello, world!")'];
+    const fileNames = ['file1'];
+
+    const expectedNotebookContent = [
+      { source: '#### file1', type: 'markdown' },
+      { source: codeStructure[0], type: 'code' }
+    ];
+
+    const result = initializeNotebookContent(codeStructure, fileNames);
+    expect(result).toEqual(expectedNotebookContent);
+  });
+
+  it('should return an empty array when codeStructure and fileNames are empty', () => {
+    const codeStructure: string[] = [];
+    const fileNames: string[] = [];
+
+    const result = initializeNotebookContent(codeStructure, fileNames);
+    expect(result).toEqual([]);
+  });
+
+  it('should add a run cell for each run function in codeStructure', () => {
+    const codeStructure = [
+      'def run(job_input: IJobInput)',
+      'def run(job_input: IJobInput)'
+    ];
+    const fileNames = ['file1', 'file2'];
+
+    const expectedNotebookContent = [
+      { source: '#### file1', type: 'markdown' },
+      { source: codeStructure[0], type: 'code' },
+      { source: 'run(job_input)', type: 'code' },
+      { source: '#### file2', type: 'markdown' },
+      { source: codeStructure[1], type: 'code' },
+      { source: 'run(job_input)', type: 'code' }
+    ];
+
+    const result = initializeNotebookContent(codeStructure, fileNames);
+    expect(result).toEqual(expectedNotebookContent);
+  });
+});

projects/vdk-plugins/vdk-jupyter/vdk-jupyterlab-extension/src/commandsAndMenu.tsx

@@ -10,24 +10,27 @@ import { jobdDataRequest } from './serverRequests';
 import { VdkOption } from './vdkOptions/vdk_options';
 import { workingDirectory } from '.';
 import { IDocumentManager } from '@jupyterlab/docmanager';
+import { FileBrowser } from '@jupyterlab/filebrowser';
+import { INotebookTracker } from '@jupyterlab/notebook';
 
-var runningVdkOperation = false;
+export var runningVdkOperation = '';
 
-export function updateVDKMenu(commands: CommandRegistry, docManager: IDocumentManager) {
+export function updateVDKMenu(commands: CommandRegistry, docManager: IDocumentManager, fileBrowser: FileBrowser, notebookTracker: INotebookTracker) {
   // Add Run job command
-  add_command(commands, 'jp-vdk:menu-run','Run','Execute VDK Run Command', showRunJobDialog, docManager);
+  add_command(commands, 'jp-vdk:menu-run', 'Run', 'Execute VDK Run Command', showRunJobDialog, docManager);
 
   // Add Create job command
-  add_command(commands, 'jp-vdk:menu-create','Create','Execute VDK Create Command', showCreateJobDialog);
+  add_command(commands, 'jp-vdk:menu-create', 'Create', 'Execute VDK Create Command', showCreateJobDialog);
 
   // Add Download job command
-  add_command(commands, 'jp-vdk:menu-download','Download','Execute VDK Download Command', showDownloadJobDialog);
+  add_command(commands, 'jp-vdk:menu-download', 'Download', 'Execute VDK Download Command', showDownloadJobDialog);
 
   // Add Convert Job To Notebook command
-  add_command(commands, 'jp-vdk:menu-convert-job-to-notebook', 'Convert Job To Notebook', 'Convert Data Job To Jupyter Notebook', showConvertJobToNotebookDialog);
+  add_command(commands, 'jp-vdk:menu-convert-job-to-notebook', 'Convert Job To Notebook', 'Convert Data Job To Jupyter Notebook', showConvertJobToNotebookDialog, undefined,
+    fileBrowser, notebookTracker);
 
   // Add Create Deployment command
-  add_command(commands, 'jp-vdk:menu-create-deployment','Deploy','Create deployment of a VDK job', showCreateDeploymentDialog);
+  add_command(commands, 'jp-vdk:menu-create-deployment', 'Deploy', 'Create deployment of a VDK job', showCreateDeploymentDialog);
 }
 
 
@@ -38,20 +41,23 @@ export function updateVDKMenu(commands: CommandRegistry, docManager: IDocumentMa
  *@param getOperationDialog - function that will load the dialog for the command
  */
 function add_command(commands: CommandRegistry, schemaNaming: string, label: string, caption: string, getOperationDialog: Function,
-  docManager?: IDocumentManager){
+  docManager?: IDocumentManager, fileBrowser?: FileBrowser, notebookTracker?: INotebookTracker) {
   commands.addCommand(schemaNaming, {
     label: label,
     caption: caption,
     execute: async () => {
       try {
         if (!runningVdkOperation) {
-          runningVdkOperation = true;
+          runningVdkOperation = schemaNaming;
           jobData.set(VdkOption.PATH, workingDirectory);
           await jobdDataRequest();
-          if(docManager) await getOperationDialog(docManager);
-          else  await getOperationDialog();
+          if (label == 'Convert Job To Notebook') await getOperationDialog(commands, fileBrowser, notebookTracker);
+          else if (docManager) {
+            await getOperationDialog(docManager);
+          }
+          else await getOperationDialog();
           setJobDataToDefault();
-          runningVdkOperation = false;
+          runningVdkOperation = '';
         } else {
           showErrorMessage(
             'Another VDK operation is currently running!',