Merge branch 'master' into yiyi/update_SKU_API

microsoft · Oct 22, 2020 · ced90dc · ced90dc
2 parents 0e18971 + 166a0b6
commit ced90dc
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 95 deletions.
diff --git a/src/api/v2/clients/jobClient.ts b/src/api/v2/clients/jobClient.ts
@@ -37,6 +37,37 @@ export class JobClient extends OpenPAIBaseClient {
         });
     }
 
+    /**
+     * Submit a job in the system.
+     * @param jobConfig The job config.
+     */
+    public async createRunningJob(jobConfig: IJobConfig): Promise<IPAIResponse>  {
+        const url: string = Util.fixUrl(
+            `${this.cluster.rest_server_uri}/api/v2/jobs`,
+            this.cluster.https
+        );
+        const text: string = yaml.safeDump(jobConfig);
+
+        await this.httpClient.post(url, text, undefined, {
+            headers: {
+                'content-type': 'text/yaml'
+            }
+        });
+
+        let jobStatus : string;
+        while (true) {
+            const result: any = await this.getJob(this.cluster.username ? this.cluster.username : '', jobConfig.name);
+            jobStatus = result.jobStatus.state;
+            if (jobStatus === 'WAITING') {
+                setTimeout(() => { console.log('waiting for the job to run...'); }, 5000);
+                continue;
+            } else {
+                break;
+            }
+        }
+        return { code: '202', message: `The job ${jobConfig.name} is ${jobStatus}`};
+    }
+
     /**
      * Get the list of jobs.
      * @param query filter jobs by username, vc, state and keyword. Set offset, limit, order and withTotalCount.

diff --git a/src/api/v2/models/job.ts b/src/api/v2/models/job.ts
@@ -83,9 +83,12 @@ export interface IJobStatusDetails {
     retryDelayTime?: any;
     createdTime: number;
     completedTime: number | null;
+    attemptId: number | null;
+    attemptState: string | null;
     appId: string;
     appProgress: number;
     appTrackingUrl: string;
+    appCreatedTime: number | null;
     appLaunchedTime: number | null;
     appCompletedTime: number | null;
     appExitCode: number | null;
@@ -105,7 +108,14 @@ export interface IJobStatusDetails {
 
 export interface ITaskStatus {
     taskIndex: number;
+    taskUid: string;
     taskState: string;
+    retries: number;
+    accountableRetries: number;
+    createdTime: number | null;
+    completedTime: number | null;
+    attemptId: number | null;
+    attemptState: string | null;
     containerId: string;
     containerIp: string;
     containerPorts: { [index: string]: number | number[] | string; };
@@ -138,6 +148,7 @@ export interface IJobStatus {
 interface ITaskAttempt {
     attemptId: number;
     attemptState: string;
+    currentAttemptCreatedTime: number | null;
     currentAttemptLaunchedTime: number | null;
     currentAttemptCompletedTime: number | null;
     containerId: string | null;

diff --git a/src/api/v2/swagger.yaml b/src/api/v2/swagger.yaml
@@ -1212,6 +1212,7 @@ paths:
                   attemptId: 0
                   attemptState: SUCCEEDED,
                   appId: id
+                  appCreatedTime: 0
                   appLaunchedTime: 0
                   appCompletedTime: 0
                   appExitCode: 0
@@ -1265,6 +1266,7 @@ paths:
                   attemptId: 0
                   attemptState: SUCCEEDED,
                   appId: id
+                  appCreatedTime: 0
                   appLaunchedTime: 0
                   appCompletedTime: 0
                   appExitCode: 0
@@ -1548,7 +1550,6 @@ paths:
                   userName: userName
                   state: SUCCEEDED
                   originState: Completed
-                  maxAttemptCount: 1
                   attemptIndex: 0
                   jobStartedTime: 0
                   attemptStartedTime: 0
@@ -1625,7 +1626,6 @@ paths:
                 userName: userName
                 state: SUCCEEDED
                 originState: Completed
-                maxAttemptCount: 1
                 attemptIndex: 0
                 jobStartedTime: 0
                 attemptStartedTime: 0
@@ -1711,6 +1711,7 @@ paths:
                 attempts:
                   - attemptId: 0
                     attemptState: FAILED
+                    currentAttemptCreatedTime: 1602480558000
                     currentAttemptLaunchedTime: 1602480558000
                     currentAttemptCompletedTime: 1602480570000
                     containerId: a9f9f1f2-5e43-4423-88c3-022433b8cd7c
@@ -1737,6 +1738,7 @@ paths:
                     containerExitDiagnostics: Stop to complete current FrameworkAttempt
                   - attemptId: 1
                     attemptState: FAILED
+                    currentAttemptCreatedTime: 1602480591000
                     currentAttemptLaunchedTime: 1602480591000
                     currentAttemptCompletedTime: 1602480603000
                     containerId: a9f9f1f2-5e43-4423-88c3-022433b8cd7c
@@ -2216,6 +2218,12 @@ components:
             appId:
               type: string
               description: unique id for the job
+            appCreatedTime:
+              type: integer
+              nullable: true
+              description: >-
+                job created time for last attempt, in number of milliseconds
+                since the Unix Epoch.
             appLaunchedTime:
               type: integer
               nullable: true
@@ -2227,11 +2235,12 @@ components:
               nullable: true
               description: >
                 job completed time for last attempt, in number of milliseconds
-                since the Unix Epoch. to get durations for last attempt:
+                since the Unix Epoch. 
+                To get durations for last attempt:
                   wholeDuration = completedTime - createdTime;
                   runningDuration = appCompletedTime - appLaunchedTime;
                   waitingDuration = wholeDuration - runningDuration.
-                to get durations for all attempts:
+                To get durations for all attempts:
                   runningDuration = sum(appCompletedTime - appLaunchedTime) for all attempts.
             appExitCode:
               type: integer
@@ -2313,6 +2322,7 @@ components:
             - retries
             - createdTime
             - completedTime
+            - appCreatedTime
             - appLaunchedTime
             - appCompletedTime
             - appExitCode
@@ -2433,6 +2443,12 @@ components:
                       type: integer
                     attemptState:
                       type: string
+                    currentAttemptCreatedTime:
+                      type: integer
+                      nullable: true
+                      description: >-
+                        the current attempt created time, in number of milliseconds since the Unix
+                        Epoch.
                     currentAttemptLaunchedTime:
                       type: integer
                       nullable: true
@@ -2662,8 +2678,6 @@ components:
         originState:
           type: string
           nullable: true
-        maxAttemptCount:
-          type: integer
         attemptIndex:
           type: integer
           nullable: true
@@ -2891,6 +2905,12 @@ components:
                 type: integer
               attemptState:
                 type: string
+              currentAttemptCreatedTime:
+                type: integer
+                nullable: true
+                description: >-
+                  the current attempt created time, in number of milliseconds since the Unix
+                  Epoch.
               currentAttemptLaunchedTime:
                 type: integer
                 nullable: true
@@ -2969,7 +2989,7 @@ components:
                 nullable: true
                 properties:
                   affinityGroupName:
-                    type: object
+                    type: string
                     nullable: true
                   lazyPreempted:
                     type: object

diff --git a/tests/common/apiTestCases.ts b/tests/common/apiTestCases.ts
@@ -133,6 +133,20 @@ function createTestJob(): IApiOperation {
     };
 }
 
+function createTestRunningJob(): IApiOperation {
+    return {
+        tag: 'job',
+        operationId: 'createRunningJob',
+        parameters: [{
+            type: 'raw',
+            value: {
+                ...testJobConfig,
+                ...{ name: 'sdk_test_job' + randomString.new() }
+            }
+        }]
+    };
+}
+
 function updateTestJobExecutionType(executionType: 'START' | 'STOP'): IApiOperation {
     return {
         tag: 'job',
@@ -932,97 +946,13 @@ export const ApiDefaultTestCases: {[key: string]: IApiTestCase} = {
         after: [ updateTestJobExecutionType('STOP') ]
     },
     'get /api/v2/jobs/{user}~{job}/attempts/{jobAttemptId}/taskRoles/{taskRoleName}/taskIndex/{taskIndex}/attempts': {
-        before: [ createTestJob() ],
         tests: [
             {
-                description: 'Get task status',
-                operation: {
-                    parameters: [
-                        {
-                            type: 'raw',
-                            value: clustersJson[0].username
-                        },
-                        {
-                            type: 'raw',
-                            value: 'sdk_test_job' + randomString.get()
-                        },
-                        {
-                            type: 'raw',
-                            value: 0
-                        },
-                        {
-                            type: 'raw',
-                            value: 'worker'
-                        },
-                        {
-                            type: 'raw',
-                            value: 0
-                        }
-                    ]
-                }
-            },
-            {
-                description: 'Get task of a nonexist job',
-                operation: {
-                    parameters: [
-                        {
-                            type: 'raw',
-                            value: clustersJson[0].username
-                        },
-                        {
-                            type: 'raw',
-                            value: 'sdk_test_nonexist_job'
-                        },
-                        {
-                            type: 'raw',
-                            value: 0
-                        },
-                        {
-                            type: 'raw',
-                            value: 'worker'
-                        },
-                        {
-                            type: 'raw',
-                            value: 0
-                        }
-                    ],
-                    response: {
-                        statusCode: 404
-                    }
-                }
-            },
-            {
-                description: 'Get nonexist task attempt',
-                operation: {
-                    parameters: [
-                        {
-                            type: 'raw',
-                            value: clustersJson[0].username
-                        },
-                        {
-                            type: 'raw',
-                            value: 'sdk_test_job'
-                        },
-                        {
-                            type: 'raw',
-                            value: 0
-                        },
-                        {
-                            type: 'raw',
-                            value: 'worker'
-                        },
-                        {
-                            type: 'raw',
-                            value: 5
-                        }
-                    ],
-                    response: {
-                        statusCode: 404
-                    }
-                }
+                // Skip the test temporarily to avoid highly increasing CI cost.
+                description: 'Skip',
+                customizedTest: 'skipTest'
             }
-        ],
-        after: [ updateTestJobExecutionType('STOP') ]
+        ]
     },
     'get /api/v2/jobs/{user}~{job}/config': {
         before: [ createTestJob() ],

diff --git a/tests/common/test_data/testJobStatus.ts b/tests/common/test_data/testJobStatus.ts
@@ -21,9 +21,12 @@ export const testJobStatus: IJobStatus = {
         },
         createdTime: 1565329763372,
         completedTime: null,
+        attemptId: 0,
+        attemptState: 'RUNNING',
         appId: 'application_1565337391589_0002',
         appProgress: 0,
         appTrackingUrl: 'http://0.0.0.34/yarn/0.0.0.34:8088/proxy/application_1565337391589_0002/',
+        appCreatedTime: 1565337476313,
         appLaunchedTime: 1565337476313,
         appCompletedTime: null,
         appExitCode: null,
@@ -48,7 +51,14 @@ export const testJobStatus: IJobStatus = {
             taskStatuses: [
                 {
                     taskIndex: 0,
+                    taskUid: '65b2e476-1101-11eb-b396-96ba8c8049b4',
                     taskState: 'RUNNING',
+                    retries: 0,
+                    accountableRetries: 0,
+                    createdTime: 1602998326000,
+                    completedTime: null,
+                    attemptId: 0,
+                    attemptState: 'RUNNING',
                     containerId: 'container_e34_1565337391589_0002_01_000002',
                     containerIp: '0.0.0.38',
                     containerPorts: {

diff --git a/tests/common/test_data/testTaskDetail.ts b/tests/common/test_data/testTaskDetail.ts
@@ -22,6 +22,7 @@ export const testTaskDetail: ITaskDetail = {
        {
            attemptId: 1,
            attemptState: 'STOPPED',
+           currentAttemptCreatedTime: 1602665952000,
            currentAttemptLaunchedTime: 1602665952000,
            currentAttemptCompletedTime: 1602665969000,
            containerId: 'b62b9972-ee99-405d-a1bc-9f4467b7289a',
@@ -58,6 +59,7 @@ export const testTaskDetail: ITaskDetail = {
        {
            attemptId: 0,
            attemptState: 'STOPPED',
+           currentAttemptCreatedTime: 1602665934000,
            currentAttemptLaunchedTime: 1602665934000,
            currentAttemptCompletedTime: 1602665952000,
            containerId: '142b5212-1f2b-4ad6-83da-17d7d1e99e4f',